feat(tradein): EKB geoportal building ingest + geocoder tier (complete city buildings)

tradein-mvp/backend/app/services/geocoder.py

@@ -691,6 +691,62 @@ def _cadastral_house_match(db: Session, street: str, house: str) -> GeocodeSugge
     )
 
 
+_RE_HOUSE_INTERNAL_SPACES = re.compile(r"\s+")
+
+
+def _normalize_geoportal_house(house: str) -> str:
+    """house_norm как в ekb_geoportal_buildings: lower + удаление внутренних пробелов.
+
+    «7 б» → «7б». ДОЛЖНО совпадать с app.tasks.ekb_geoportal_ingest.normalize_house.
+    """
+    return _RE_HOUSE_INTERNAL_SPACES.sub("", house.strip().lower())
+
+
+def _geoportal_house_match(db: Session, street: str, house: str) -> GeocodeSuggestion | None:
+    """Точный матч по реестру зданий ЕКБ (городской геопортал, ekb_geoportal_buildings).
+
+    Полнее чем NSPD cad_buildings (~70% зданий ЕКБ отсутствуют в NSPD) — поэтому это
+    ПЕРВЫЙ локальный tier геокодера, до cad_buildings.
+
+    Нормализация: street_norm = lower(trim), house_norm = lower без внутренних пробелов
+    — ровно как наполняет лоадер. Параметры идут только bound-param'ами (без инъекций).
+    """
+    street_norm = street.strip().lower()
+    house_norm = _normalize_geoportal_house(house)
+    if not street_norm or not house_norm:
+        return None
+    try:
+        row = db.execute(
+            text("""
+                SELECT street, house, lat, lon
+                FROM ekb_geoportal_buildings
+                WHERE street_norm = lower(trim(CAST(:street AS text)))
+                  AND house_norm = CAST(:house AS text)
+                LIMIT 1
+            """),
+            {"street": street_norm, "house": house_norm},
+        ).first()
+    except Exception:
+        logger.warning(
+            "geoportal house match failed for street=%r house=%r",
+            street,
+            house,
+            exc_info=True,
+        )
+        return None
+
+    if row is None:
+        return None
+    label = f"{row.street}, {row.house}"
+    return GeocodeSuggestion(
+        label=label,
+        full_address=label,
+        lat=float(row.lat),
+        lon=float(row.lon),
+        kind="house",
+    )
+
+
 def _cadastral_reverse_sync(db: Session, lat: float, lon: float, radius_m: int = 200) -> str | None:
     """Reverse lookup via gendesign_cad_buildings FDW.
 
@@ -810,11 +866,38 @@ async def geocode(address: str, db: Session) -> GeocodeResult | None:
         logger.info("geocode cache hit: %s", addr_norm)
         return cached
 
-    # 2. Cadastral FDW (прямой запрос к gendesign_cad_buildings — без внешнего API)
-    # 2a. Anchored house-match: парсим street+house → точный матч по дом-маркеру.
-    # Это primary cadastral путь — raw-ILIKE по полному readable_address давал 0 hits
-    # для «Серова 27» / DaData-форм (литеральная подстрока не совпадает).
+    # 2. Локальные источники по street+house (без внешнего API).
     parsed = _parse_street_house(address.strip())
+
+    # 2a. Геопортал ЕКБ — ПЕРВЫЙ локальный tier (полнее cad_buildings ~на 70%).
+    if parsed is not None:
+        street, house = parsed
+        try:
+            hit = await asyncio.to_thread(_geoportal_house_match, db, street, house)
+        except Exception:
+            logger.warning("geoportal house-match raised — fall through", exc_info=True)
+            hit = None
+        if hit is not None:
+            result = GeocodeResult(
+                lat=hit.lat,
+                lon=hit.lon,
+                full_address=hit.full_address,
+                provider="cache",
+                confidence="exact",
+            )
+            await asyncio.to_thread(_cache_put, db, addr_norm, result)
+            logger.info(
+                "geocode geoportal house-match: %s → (%.5f, %.5f)",
+                addr_norm,
+                result.lat,
+                result.lon,
+            )
+            return result
+
+    # 2c. Cadastral FDW (прямой запрос к gendesign_cad_buildings — без внешнего API)
+    # Anchored house-match: парсим street+house → точный матч по дом-маркеру.
+    # raw-ILIKE по полному readable_address давал 0 hits для «Серова 27» / DaData-форм
+    # (литеральная подстрока не совпадает).
     if parsed is not None:
         street, house = parsed
         hit = await asyncio.to_thread(_cadastral_house_match, db, street, house)
@@ -835,7 +918,7 @@ async def geocode(address: str, db: Session) -> GeocodeResult | None:
             )
             return result
 
-    # 2b. Fallback: legacy raw-ILIKE forward search (для нераспарсенных форм)
+    # 2d. Fallback: legacy raw-ILIKE forward search (для нераспарсенных форм)
     cad_suggestions = await asyncio.to_thread(_cadastral_forward_sync, db, address.strip(), limit=1)
     if cad_suggestions:
         s = cad_suggestions[0]

tradein-mvp/backend/app/services/scrapers/ekb_geoportal_client.py

@@ -0,0 +1,233 @@
+"""Клиент городского геопортала Екатеринбурга (геопортал.екатеринбург.рф).
+
+Открытый GeoServer WFS, без авторизации. Слой `portal:portal_geo_topo_build` —
+футпринты зданий ЕКБ (намного полнее чем NSPD cad_buildings, который пропускает
+~70% зданий города).
+
+ВАЖНО: сервер отдаёт 403 на дефолтный User-Agent curl/httpx. Нужны браузерные
+заголовки (UA + Referer + Accept) — тогда 200.
+
+Геометрия `geoloc` в EPSG:4326 (MultiPolygon footprint). BBOX axis order =
+lon_min,lat_min,lon_max,lat_max. Сервер режет каждый ответ на 20000 features —
+лоадер тайлит мелко, чтобы не упереться.
+
+Центроид считается чистым Python'ом (area-weighted, без shapely — shapely не входит
+в зависимости tradein-mvp backend).
+
+Используется лоадером app.tasks.ekb_geoportal_ingest.
+"""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass
+
+import httpx
+
+logger = logging.getLogger(__name__)
+
+# Punycode-хост геопортал.екатеринбург.рф
+GEOPORTAL_HOST = "https://xn--80afgznagjs.xn--80acgfbsl1azdqr.xn--p1ai"
+WFS_PATH = "/api/gis/ows/3/portal/wfs"
+BUILD_LAYER = "portal:portal_geo_topo_build"
+
+# Браузерные заголовки — без них хост отдаёт 403.
+_BROWSER_HEADERS = {
+    "User-Agent": (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+        "AppleWebKit/537.36 (KHTML, like Gecko) "
+        "Chrome/148.0.0.0 Safari/537.36"
+    ),
+    "Referer": f"{GEOPORTAL_HOST}/",
+    "Accept": "application/json",
+}
+
+
+@dataclass(frozen=True, slots=True)
+class TopoBuilding:
+    """Одна часть-фича здания из слоя portal_geo_topo_build.
+
+    Здание может быть разбито на несколько part-features (одинаковые street/house,
+    разные key) — лоадер группирует их по (street_norm, house_norm).
+    """
+
+    street: str
+    house: str
+    floors: int | None
+    purpose: str | None
+    material: str | None
+    key: int | None
+    lat: float
+    lon: float
+
+
+def _parse_int(value: object) -> int | None:
+    if value is None:
+        return None
+    try:
+        return int(value)
+    except (ValueError, TypeError):
+        return None
+
+
+def _parse_str(value: object) -> str | None:
+    if value is None:
+        return None
+    s = str(value).strip()
+    return s or None
+
+
+def _ring_centroid(ring: list) -> tuple[float, float, float] | None:
+    """Area-weighted centroid of one linear ring → (cx, cy, signed_area2).
+
+    signed_area2 = 2× signed area (используется как вес при усреднении колец/полигонов).
+    Возвращает None если кольцо вырождено (нулевая площадь).
+    """
+    n = len(ring)
+    if n < 3:
+        return None
+    a2 = 0.0  # 2× signed area
+    cx = 0.0
+    cy = 0.0
+    for i in range(n):
+        x0, y0 = ring[i][0], ring[i][1]
+        x1, y1 = ring[(i + 1) % n][0], ring[(i + 1) % n][1]
+        cross = x0 * y1 - x1 * y0
+        a2 += cross
+        cx += (x0 + x1) * cross
+        cy += (y0 + y1) * cross
+    if a2 == 0.0:
+        return None
+    cx /= 3.0 * a2
+    cy /= 3.0 * a2
+    return (cx, cy, a2)
+
+
+def _multipolygon_centroid(coords: list) -> tuple[float, float] | None:
+    """Центроид GeoJSON MultiPolygon/Polygon-частей: усреднение по |площади| внешних колец.
+
+    coords — список полигонов, каждый = [outer_ring, hole1, ...]. Берём только
+    внешнее кольцо каждого полигона (дыр у footprint'ов зданий практически нет,
+    их вклад в положение центра пренебрежим). Возвращает (lon, lat) или None.
+    """
+    sum_w = 0.0
+    sum_x = 0.0
+    sum_y = 0.0
+    for polygon in coords:
+        if not polygon:
+            continue
+        outer = polygon[0]
+        c = _ring_centroid(outer)
+        if c is None:
+            continue
+        cx, cy, a2 = c
+        w = abs(a2)
+        sum_w += w
+        sum_x += cx * w
+        sum_y += cy * w
+    if sum_w == 0.0:
+        return None
+    return (sum_x / sum_w, sum_y / sum_w)
+
+
+def _geometry_centroid(geom: dict) -> tuple[float, float] | None:
+    """Центроид GeoJSON geometry (MultiPolygon или Polygon) → (lon, lat) или None."""
+    gtype = geom.get("type")
+    coords = geom.get("coordinates")
+    if not coords:
+        return None
+    if gtype == "MultiPolygon":
+        return _multipolygon_centroid(coords)
+    if gtype == "Polygon":
+        # Polygon coordinates = [outer_ring, hole, ...] → обернём в один полигон.
+        return _multipolygon_centroid([coords])
+    return None
+
+
+def _feature_to_building(feature: dict) -> TopoBuilding | None:
+    """GeoJSON Feature → TopoBuilding. None если нет улицы/дома или геометрии."""
+    props = feature.get("properties") or {}
+    street = _parse_str(props.get("topo_street"))
+    house = _parse_str(props.get("topo_num_house"))
+    if not street or not house:
+        return None
+
+    geom = feature.get("geometry")
+    if not isinstance(geom, dict):
+        return None
+    try:
+        centroid = _geometry_centroid(geom)
+    except Exception:
+        logger.warning("geoportal: bad geometry for street=%r house=%r", street, house)
+        return None
+    if centroid is None:
+        return None
+    lon, lat = centroid
+
+    return TopoBuilding(
+        street=street,
+        house=house,
+        floors=_parse_int(props.get("topo_floor")),
+        purpose=_parse_str(props.get("topo_purpose")),
+        material=_parse_str(props.get("topo_material")),
+        key=_parse_int(props.get("key")),
+        lat=lat,
+        lon=lon,
+    )
+
+
+def parse_feature_collection(payload: dict) -> list[TopoBuilding]:
+    """GeoJSON FeatureCollection → list[TopoBuilding]. Пропускает невалидные."""
+    features = payload.get("features")
+    if not isinstance(features, list):
+        return []
+    out: list[TopoBuilding] = []
+    for feature in features:
+        if not isinstance(feature, dict):
+            continue
+        building = _feature_to_building(feature)
+        if building is not None:
+            out.append(building)
+    return out
+
+
+class EkbGeoportalClient:
+    """Async WFS-клиент геопортала ЕКБ. Один клиент переиспользуется лоадером."""
+
+    def __init__(self, *, timeout: float = 30.0) -> None:
+        self._timeout = timeout
+
+    async def fetch_topo_buildings(
+        self,
+        bbox: tuple[float, float, float, float],
+        *,
+        count: int = 20000,
+    ) -> list[TopoBuilding]:
+        """Запрашивает здания в bbox (lon_min, lat_min, lon_max, lat_max).
+
+        Graceful: на non-200/parse-error логирует warning и возвращает [].
+        """
+        lon_min, lat_min, lon_max, lat_max = bbox
+        cql = f"BBOX(geoloc,{lon_min},{lat_min},{lon_max},{lat_max})"
+        params = {
+            "service": "WFS",
+            "version": "2.0.0",
+            "request": "GetFeature",
+            "typeNames": BUILD_LAYER,
+            "outputFormat": "application/json",
+            "CQL_FILTER": cql,
+            "count": str(count),
+        }
+        url = f"{GEOPORTAL_HOST}{WFS_PATH}"
+        try:
+            async with httpx.AsyncClient(timeout=self._timeout, headers=_BROWSER_HEADERS) as client:
+                resp = await client.get(url, params=params)
+            if resp.status_code != 200:
+                logger.warning("geoportal WFS → HTTP %d for bbox=%s", resp.status_code, bbox)
+                return []
+            payload = resp.json()
+        except Exception as exc:
+            logger.warning("geoportal WFS fetch/parse failed for bbox=%s: %s", bbox, exc)
+            return []
+
+        return parse_feature_collection(payload)

tradein-mvp/backend/app/tasks/ekb_geoportal_ingest.py

@@ -0,0 +1,273 @@
+"""Лоадер реестра зданий ЕКБ из городского геопортала в ekb_geoportal_buildings.
+
+Запуск: `python -m app.tasks.ekb_geoportal_ingest`
+
+Тайлит bbox ЕКБ (lon 60.45–60.85, lat 56.70–56.95) на мелкую сетку (~250-300 м),
+чтобы каждый тайл укладывался в server-cap 20000 features. Для каждого тайла
+запрашивает слой portal_geo_topo_build, группирует part-features по
+(street_norm, house_norm), считает центроид и UPSERT'ит в ekb_geoportal_buildings.
+
+Нормализация ОБЯЗАНА совпадать с geocoder._geoportal_house_match:
+  street_norm = lower(trim(street))
+  house_norm  = lower(house) с удалёнными внутренними пробелами («7 б» → «7б»)
+
+Configurable через argv/env (для частичного/тестового прогона):
+  --lon-min/--lon-max/--lat-min/--lat-max  или env BBOX="lon_min,lat_min,lon_max,lat_max"
+По умолчанию — полный ЕКБ.
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import logging
+import os
+import re
+
+from sqlalchemy import text
+from sqlalchemy.orm import Session
+
+from app.core.db import SessionLocal
+from app.services.scrapers.ekb_geoportal_client import (
+    EkbGeoportalClient,
+    TopoBuilding,
+)
+
+logger = logging.getLogger(__name__)
+
+# Полный bbox ЕКБ.
+EKB_LON_MIN = 60.45
+EKB_LON_MAX = 60.85
+EKB_LAT_MIN = 56.70
+EKB_LAT_MAX = 56.95
+
+# Шаг сетки: ~0.004° lon × ~0.003° lat (≈250-300 м). Центральный тайл 0.005×0.004
+# давал 6825 зданий → 0.004×0.003 заведомо < server-cap 20000.
+TILE_LON_STEP = 0.004
+TILE_LAT_STEP = 0.003
+
+# Пауза между запросами — гос-портал, вежливый rate-limit.
+REQUEST_SLEEP_SEC = 0.1
+
+# Каждые N тайлов — лог прогресса + commit.
+PROGRESS_EVERY = 50
+
+_INTERNAL_SPACES = re.compile(r"\s+")
+
+
+def normalize_street(street: str) -> str:
+    """street_norm = lower(trim(street))."""
+    return street.strip().lower()
+
+
+def normalize_house(house: str) -> str:
+    """house_norm = lower(house) с удалёнными внутренними пробелами («7 б» → «7б»)."""
+    return _INTERNAL_SPACES.sub("", house.strip().lower())
+
+
+def _tile_grid(
+    lon_min: float, lat_min: float, lon_max: float, lat_max: float
+) -> list[tuple[float, float, float, float]]:
+    """Сетка тайлов (lon_min, lat_min, lon_max, lat_max), шаг TILE_LON/LAT_STEP."""
+    tiles: list[tuple[float, float, float, float]] = []
+    lon = lon_min
+    while lon < lon_max:
+        lon_hi = min(lon + TILE_LON_STEP, lon_max)
+        lat = lat_min
+        while lat < lat_max:
+            lat_hi = min(lat + TILE_LAT_STEP, lat_max)
+            tiles.append((lon, lat, lon_hi, lat_hi))
+            lat = lat_hi
+        lon = lon_hi
+    return tiles
+
+
+def _group_buildings(
+    buildings: list[TopoBuilding],
+) -> dict[tuple[str, str], dict]:
+    """Группирует part-features по (street_norm, house_norm).
+
+    centroid = среднее центроидов частей; floors = max; purpose/material = первый
+    непустой; source_keys = собранные key.
+    """
+    grouped: dict[tuple[str, str], dict] = {}
+    for b in buildings:
+        key = (normalize_street(b.street), normalize_house(b.house))
+        if key not in grouped:
+            grouped[key] = {
+                "street": b.street.strip(),
+                "house": b.house.strip(),
+                "lat_sum": 0.0,
+                "lon_sum": 0.0,
+                "n": 0,
+                "floors": None,
+                "purpose": None,
+                "material": None,
+                "source_keys": [],
+            }
+        g = grouped[key]
+        g["lat_sum"] += b.lat
+        g["lon_sum"] += b.lon
+        g["n"] += 1
+        if b.floors is not None:
+            g["floors"] = b.floors if g["floors"] is None else max(g["floors"], b.floors)
+        if g["purpose"] is None and b.purpose is not None:
+            g["purpose"] = b.purpose
+        if g["material"] is None and b.material is not None:
+            g["material"] = b.material
+        if b.key is not None and b.key not in g["source_keys"]:
+            g["source_keys"].append(b.key)
+    return grouped
+
+
+def _upsert_building(
+    db: Session,
+    *,
+    street_norm: str,
+    house_norm: str,
+    agg: dict,
+) -> None:
+    """UPSERT одного здания в ekb_geoportal_buildings (ON CONFLICT DO UPDATE)."""
+    lat = agg["lat_sum"] / agg["n"]
+    lon = agg["lon_sum"] / agg["n"]
+    db.execute(
+        text("""
+            INSERT INTO ekb_geoportal_buildings (
+                street, house, street_norm, house_norm,
+                lat, lon, floors, purpose, material, source_keys, updated_at
+            ) VALUES (
+                :street, :house, :street_norm, :house_norm,
+                CAST(:lat AS double precision), CAST(:lon AS double precision),
+                CAST(:floors AS integer), :purpose, :material,
+                CAST(:source_keys AS bigint[]), now()
+            )
+            ON CONFLICT (street_norm, house_norm) DO UPDATE SET
+                street      = EXCLUDED.street,
+                house       = EXCLUDED.house,
+                lat         = EXCLUDED.lat,
+                lon         = EXCLUDED.lon,
+                floors      = EXCLUDED.floors,
+                purpose     = EXCLUDED.purpose,
+                material    = EXCLUDED.material,
+                source_keys = EXCLUDED.source_keys,
+                updated_at  = now()
+        """),
+        {
+            "street": agg["street"],
+            "house": agg["house"],
+            "street_norm": street_norm,
+            "house_norm": house_norm,
+            "lat": lat,
+            "lon": lon,
+            "floors": agg["floors"],
+            "purpose": agg["purpose"],
+            "material": agg["material"],
+            "source_keys": agg["source_keys"],
+        },
+    )
+
+
+async def ingest(
+    db: Session,
+    *,
+    lon_min: float,
+    lat_min: float,
+    lon_max: float,
+    lat_max: float,
+) -> int:
+    """Тайлит bbox, тянет здания, UPSERT'ит. Возвращает число upsert'нутых зданий."""
+    client = EkbGeoportalClient()
+    tiles = _tile_grid(lon_min, lat_min, lon_max, lat_max)
+    total_tiles = len(tiles)
+    logger.info(
+        "geoportal ingest: bbox=(%.4f,%.4f,%.4f,%.4f) → %d тайлов",
+        lon_min,
+        lat_min,
+        lon_max,
+        lat_max,
+        total_tiles,
+    )
+
+    upserted = 0
+    for i, tile in enumerate(tiles, start=1):
+        buildings = await client.fetch_topo_buildings(tile)
+        if buildings:
+            grouped = _group_buildings(buildings)
+            try:
+                with db.begin_nested():
+                    for (street_norm, house_norm), agg in grouped.items():
+                        _upsert_building(
+                            db,
+                            street_norm=street_norm,
+                            house_norm=house_norm,
+                            agg=agg,
+                        )
+                upserted += len(grouped)
+            except Exception:
+                logger.warning("geoportal ingest: batch failed for tile=%s", tile, exc_info=True)
+
+        await asyncio.sleep(REQUEST_SLEEP_SEC)
+
+        if i % PROGRESS_EVERY == 0 or i == total_tiles:
+            db.commit()
+            logger.info(
+                "geoportal ingest: %d/%d тайлов, upsert'нуто %d зданий, tile=%s",
+                i,
+                total_tiles,
+                upserted,
+                tile,
+            )
+
+    db.commit()
+    logger.info("geoportal ingest: готово — upsert'нуто %d зданий", upserted)
+    return upserted
+
+
+def _resolve_bbox(args: argparse.Namespace) -> tuple[float, float, float, float]:
+    """Разрешает bbox: argv → env BBOX → дефолтный ЕКБ."""
+    env_bbox = os.environ.get("BBOX")
+    if env_bbox:
+        parts = [float(p) for p in env_bbox.split(",")]
+        if len(parts) == 4:
+            return (parts[0], parts[1], parts[2], parts[3])
+        logger.warning("BBOX env malformed (%r) — игнорирую", env_bbox)
+
+    return (
+        args.lon_min if args.lon_min is not None else EKB_LON_MIN,
+        args.lat_min if args.lat_min is not None else EKB_LAT_MIN,
+        args.lon_max if args.lon_max is not None else EKB_LON_MAX,
+        args.lat_max if args.lat_max is not None else EKB_LAT_MAX,
+    )
+
+
+def main() -> None:
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(asctime)s %(levelname)s %(name)s: %(message)s",
+    )
+    parser = argparse.ArgumentParser(description="EKB geoportal building ingest")
+    parser.add_argument("--lon-min", type=float, default=None)
+    parser.add_argument("--lon-max", type=float, default=None)
+    parser.add_argument("--lat-min", type=float, default=None)
+    parser.add_argument("--lat-max", type=float, default=None)
+    args = parser.parse_args()
+
+    lon_min, lat_min, lon_max, lat_max = _resolve_bbox(args)
+
+    db = SessionLocal()
+    try:
+        asyncio.run(
+            ingest(
+                db,
+                lon_min=lon_min,
+                lat_min=lat_min,
+                lon_max=lon_max,
+                lat_max=lat_max,
+            )
+        )
+    finally:
+        db.close()
+
+
+if __name__ == "__main__":
+    main()

tradein-mvp/backend/data/sql/130_ekb_geoportal_buildings.sql

@@ -0,0 +1,38 @@
+-- 130_ekb_geoportal_buildings.sql
+-- Полный реестр зданий ЕКБ из городского геопортала (геопортал.екатеринбург.рф,
+-- слой WFS portal:portal_geo_topo_build). Намного полнее чем NSPD cad_buildings
+-- (который пропускает ~70% зданий ЕКБ) — используется как ПЕРВЫЙ локальный tier
+-- геокодера для разрешения «улица + дом» → координаты.
+--
+-- Наполняется лоадером app.tasks.ekb_geoportal_ingest (python -m). UPSERT по
+-- (street_norm, house_norm); координаты = центроид футпринта здания.
+--
+-- Нормализация (ДОЛЖНА совпадать с matcher'ом в geocoder._geoportal_house_match):
+--   street_norm = lower(trim(street))
+--   house_norm  = lower(house) с удалёнными внутренними пробелами («7 б» → «7б»)
+--
+-- Idempotent: CREATE TABLE IF NOT EXISTS + CREATE INDEX IF NOT EXISTS,
+-- безопасно при повторном применении.
+
+BEGIN;
+
+CREATE TABLE IF NOT EXISTS ekb_geoportal_buildings (
+    id           bigserial PRIMARY KEY,
+    street       text NOT NULL,
+    house        text NOT NULL,
+    street_norm  text NOT NULL,
+    house_norm   text NOT NULL,
+    lat          double precision NOT NULL,
+    lon          double precision NOT NULL,
+    floors       integer,
+    purpose      text,
+    material     text,
+    source_keys  bigint[],
+    updated_at   timestamptz NOT NULL DEFAULT now(),
+    UNIQUE (street_norm, house_norm)
+);
+
+CREATE INDEX IF NOT EXISTS ix_ekb_geoportal_buildings_street_house
+    ON ekb_geoportal_buildings (street_norm, house_norm);
+
+COMMIT;

tradein-mvp/backend/tests/test_ekb_geoportal_ingest.py

@@ -0,0 +1,329 @@
+"""Unit tests for the EKB geoportal building ingest pipeline.
+
+Covers:
+- ekb_geoportal_client: GeoJSON FeatureCollection → TopoBuilding parsing + centroid,
+  skip of features without street/house, graceful empty on bad input.
+- ekb_geoportal_ingest: house normalization ("7 б" → "7б"), part-feature grouping,
+  tile grid bounds.
+- geocoder._geoportal_house_match + geocode() wiring: mock db row → GeocodeResult
+  (provider="cache", confidence="exact"); parse-fail / no-row → falls through.
+
+All network + DB mocked.
+"""
+
+from __future__ import annotations
+
+import os
+import sys
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+
+os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
+
+_wp_mock = MagicMock()
+sys.modules.setdefault("weasyprint", _wp_mock)
+
+
+from app.services.scrapers.ekb_geoportal_client import (  # noqa: E402
+    TopoBuilding,
+    parse_feature_collection,
+)
+from app.tasks.ekb_geoportal_ingest import (  # noqa: E402
+    _group_buildings,
+    _tile_grid,
+    normalize_house,
+    normalize_street,
+)
+
+# ── client: GeoJSON parsing + centroid ───────────────────────────────────────
+
+
+def _multipolygon(coords: list) -> dict:
+    return {"type": "MultiPolygon", "coordinates": coords}
+
+
+# A unit square [0,0]-[2,2] → centroid (1, 1) in (lon, lat).
+_SQUARE = [[[[0.0, 0.0], [2.0, 0.0], [2.0, 2.0], [0.0, 2.0], [0.0, 0.0]]]]
+
+
+def test_parse_feature_collection_basic() -> None:
+    fc = {
+        "type": "FeatureCollection",
+        "features": [
+            {
+                "type": "Feature",
+                "geometry": _multipolygon(_SQUARE),
+                "properties": {
+                    "topo_street": "Космонавтов",
+                    "topo_num_house": "7б",
+                    "topo_floor": 9,
+                    "topo_purpose": "жилое",
+                    "topo_material": "3",
+                    "key": 12345,
+                },
+            }
+        ],
+    }
+    out = parse_feature_collection(fc)
+    assert len(out) == 1
+    b = out[0]
+    assert isinstance(b, TopoBuilding)
+    assert b.street == "Космонавтов"
+    assert b.house == "7б"
+    assert b.floors == 9
+    assert b.purpose == "жилое"
+    assert b.material == "3"
+    assert b.key == 12345
+    # centroid of the [0,0]-[2,2] square = (1, 1): lon=x, lat=y
+    assert b.lon == pytest.approx(1.0)
+    assert b.lat == pytest.approx(1.0)
+
+
+def test_parse_skips_features_without_street_or_house() -> None:
+    fc = {
+        "features": [
+            {
+                "geometry": _multipolygon(_SQUARE),
+                "properties": {"topo_street": "", "topo_num_house": "7б"},
+            },
+            {
+                "geometry": _multipolygon(_SQUARE),
+                "properties": {"topo_street": "Ленина", "topo_num_house": "  "},
+            },
+            {
+                "geometry": _multipolygon(_SQUARE),
+                "properties": {"topo_num_house": "5"},  # no street key
+            },
+        ]
+    }
+    assert parse_feature_collection(fc) == []
+
+
+def test_parse_skips_feature_without_geometry() -> None:
+    fc = {
+        "features": [
+            {
+                "geometry": None,
+                "properties": {"topo_street": "Ленина", "topo_num_house": "5"},
+            }
+        ]
+    }
+    assert parse_feature_collection(fc) == []
+
+
+def test_parse_empty_or_malformed_collection() -> None:
+    assert parse_feature_collection({}) == []
+    assert parse_feature_collection({"features": "nope"}) == []
+    assert parse_feature_collection({"features": [None, 42]}) == []
+
+
+# ── normalization ────────────────────────────────────────────────────────────
+
+
+@pytest.mark.parametrize(
+    ("raw", "expected"),
+    [
+        ("7 б", "7б"),
+        ("7Б", "7б"),
+        ("  7б  ", "7б"),
+        ("1в/1", "1в/1"),
+        ("12", "12"),
+        ("3 А", "3а"),
+    ],
+)
+def test_normalize_house(raw: str, expected: str) -> None:
+    assert normalize_house(raw) == expected
+
+
+def test_normalize_street() -> None:
+    assert normalize_street("  Космонавтов ") == "космонавтов"
+    assert normalize_street("8 Марта") == "8 марта"
+
+
+# ── grouping ─────────────────────────────────────────────────────────────────
+
+
+def test_group_buildings_merges_parts() -> None:
+    parts = [
+        TopoBuilding("Космонавтов", "7 б", 9, "жилое", "3", 1, lat=2.0, lon=4.0),
+        TopoBuilding("Космонавтов", "7б", None, None, None, 2, lat=4.0, lon=6.0),
+        TopoBuilding("Космонавтов", "7а", 5, None, None, 3, lat=1.0, lon=1.0),
+    ]
+    grouped = _group_buildings(parts)
+    # "7 б" and "7б" normalize to the same house → one merged entry
+    assert ("космонавтов", "7б") in grouped
+    assert ("космонавтов", "7а") in grouped
+    assert len(grouped) == 2
+
+    merged = grouped[("космонавтов", "7б")]
+    # centroid = mean of (2,4) and (4,6)
+    assert merged["lat_sum"] / merged["n"] == pytest.approx(3.0)
+    assert merged["lon_sum"] / merged["n"] == pytest.approx(5.0)
+    assert merged["floors"] == 9  # max across parts (other is None)
+    assert merged["purpose"] == "жилое"
+    assert merged["material"] == "3"
+    assert sorted(merged["source_keys"]) == [1, 2]
+
+
+# ── tile grid ────────────────────────────────────────────────────────────────
+
+
+def test_tile_grid_covers_bbox() -> None:
+    tiles = _tile_grid(60.45, 56.70, 60.85, 56.95)
+    assert len(tiles) > 0
+    # every tile within bbox bounds
+    for lon_lo, lat_lo, lon_hi, lat_hi in tiles:
+        assert 60.45 <= lon_lo < lon_hi <= 60.85 + 1e-9
+        assert 56.70 <= lat_lo < lat_hi <= 56.95 + 1e-9
+    # first tile anchored at SW corner
+    assert tiles[0][0] == pytest.approx(60.45)
+    assert tiles[0][1] == pytest.approx(56.70)
+
+
+# ── geocoder._geoportal_house_match + geocode() wiring ───────────────────────
+
+
+def test_geoportal_house_match_returns_suggestion() -> None:
+    from app.services.geocoder import GeocodeSuggestion, _geoportal_house_match
+
+    row = MagicMock()
+    row.street = "Космонавтов"
+    row.house = "7б"
+    row.lat = 56.864
+    row.lon = 60.611
+
+    db = MagicMock()
+    result = MagicMock()
+    result.first.return_value = row
+    db.execute.return_value = result
+
+    hit = _geoportal_house_match(db, "Космонавтов", "7 б")
+    assert isinstance(hit, GeocodeSuggestion)
+    assert hit.kind == "house"
+    assert hit.lat == 56.864
+    assert hit.lon == 60.611
+    # house param normalized: "7 б" → "7б"
+    params = db.execute.call_args.args[1]
+    assert params["house"] == "7б"
+    assert params["street"] == "космонавтов"
+
+
+def test_geoportal_house_match_none_when_no_row() -> None:
+    from app.services.geocoder import _geoportal_house_match
+
+    db = MagicMock()
+    result = MagicMock()
+    result.first.return_value = None
+    db.execute.return_value = result
+    assert _geoportal_house_match(db, "Космонавтов", "7б") is None
+
+
+def test_geoportal_house_match_none_on_db_error() -> None:
+    from app.services.geocoder import _geoportal_house_match
+
+    db = MagicMock()
+    db.execute.side_effect = RuntimeError("table missing")
+    assert _geoportal_house_match(db, "Космонавтов", "7б") is None
+
+
+def test_geoportal_house_match_none_for_empty_house() -> None:
+    from app.services.geocoder import _geoportal_house_match
+
+    db = MagicMock()
+    assert _geoportal_house_match(db, "Космонавтов", "   ") is None
+    db.execute.assert_not_called()
+
+
+async def test_geocode_uses_geoportal_first() -> None:
+    """Parse + geoportal hit → GeocodeResult(provider=cache, exact); cad NOT called."""
+    from app.services.geocoder import GeocodeSuggestion, geocode
+
+    db = MagicMock()
+    hit = GeocodeSuggestion(
+        label="Космонавтов, 7б",
+        full_address="Космонавтов, 7б",
+        lat=56.864,
+        lon=60.611,
+        kind="house",
+    )
+
+    with (
+        patch("app.services.geocoder._cache_get", return_value=None),
+        patch("app.services.geocoder._geoportal_house_match", return_value=hit) as mock_geo,
+        patch("app.services.geocoder._cadastral_house_match") as mock_cad,
+        patch("app.services.geocoder._cadastral_forward_sync") as mock_forward,
+        patch("app.services.geocoder._cache_put"),
+        patch("app.services.geocoder._yandex_lookup", new_callable=AsyncMock) as mock_yandex,
+    ):
+        result = await geocode("Космонавтов 7б", db)
+
+    assert result is not None
+    assert result.provider == "cache"
+    assert result.confidence == "exact"
+    assert result.lat == 56.864
+    assert result.lon == 60.611
+    mock_geo.assert_called_once()
+    # geoportal hit short-circuits everything downstream
+    mock_cad.assert_not_called()
+    mock_forward.assert_not_called()
+    mock_yandex.assert_not_called()
+
+
+async def test_geocode_falls_through_to_cadastral_when_geoportal_misses() -> None:
+    """Geoportal None → cadastral house-match IS called next."""
+    from app.services.geocoder import GeocodeSuggestion, geocode
+
+    db = MagicMock()
+    cad_hit = GeocodeSuggestion(
+        label="ул. Серова, д. 27",
+        full_address="ул. Серова, д. 27, Екатеринбург",
+        lat=56.81188,
+        lon=60.59739,
+        kind="house",
+    )
+
+    with (
+        patch("app.services.geocoder._cache_get", return_value=None),
+        patch("app.services.geocoder._geoportal_house_match", return_value=None) as mock_geo,
+        patch("app.services.geocoder._cadastral_house_match", return_value=cad_hit) as mock_cad,
+        patch("app.services.geocoder._cadastral_forward_sync") as mock_forward,
+        patch("app.services.geocoder._cache_put"),
+        patch("app.services.geocoder._yandex_lookup", new_callable=AsyncMock) as mock_yandex,
+    ):
+        result = await geocode("Серова 27", db)
+
+    assert result is not None
+    assert result.lat == 56.81188
+    mock_geo.assert_called_once()
+    mock_cad.assert_called_once()
+    mock_forward.assert_not_called()
+    mock_yandex.assert_not_called()
+
+
+async def test_geocode_skips_geoportal_when_parse_fails() -> None:
+    """Unparseable address → geoportal NOT called."""
+    from app.services.geocoder import geocode
+
+    db = MagicMock()
+
+    with (
+        patch("app.services.geocoder._cache_get", return_value=None),
+        patch("app.services.geocoder._geoportal_house_match") as mock_geo,
+        patch("app.services.geocoder._cadastral_house_match") as mock_cad,
+        patch("app.services.geocoder._cadastral_forward_sync", return_value=[]) as mock_forward,
+        patch("app.services.geocoder._cache_put"),
+        patch("app.services.geocoder.settings") as mock_settings,
+        patch(
+            "app.services.geocoder._nominatim_lookup",
+            new_callable=AsyncMock,
+            return_value=None,
+        ),
+    ):
+        mock_settings.yandex_geocoder_api_key = None
+        result = await geocode("полный мусор без дома", db)
+
+    assert result is None
+    mock_geo.assert_not_called()
+    mock_cad.assert_not_called()
+    mock_forward.assert_called_once()

tradein-mvp/backend/tests/test_geocoder_cadastral_matcher.py

@@ -178,6 +178,7 @@ async def test_geocode_uses_house_match_before_legacy_forward() -> None:
 
     with (
         patch("app.services.geocoder._cache_get", return_value=None),
+        patch("app.services.geocoder._geoportal_house_match", return_value=None),
         patch(
             "app.services.geocoder._cadastral_house_match",
             return_value=hit,
@@ -213,6 +214,7 @@ async def test_geocode_falls_back_to_legacy_forward_when_house_match_misses() ->
 
     with (
         patch("app.services.geocoder._cache_get", return_value=None),
+        patch("app.services.geocoder._geoportal_house_match", return_value=None),
         patch(
             "app.services.geocoder._cadastral_house_match",
             return_value=None,