feat(tradein): postgres_fdw live read of gendesign.cad_buildings (replaces snapshot)
Replaces tradein.cad_buildings manual snapshot (36k rows, frozen 2026-05-22) with live postgres_fdw foreign table reading gendesign.v_tradein_cad_buildings directly. Also reverts PR #492 HTTP-based cadastral.py — superseded by FDW. Architecture: - gendesign DB: new role tradein_fdw_reader + flat view v_tradein_cad_buildings (EKB-only slice of cad_buildings with lat/lon flattened from geom) - networks: gendesign-postgres added to gendesign_shared (alias); tradein-postgres added to gendesign_shared for FDW connect - tradein DB: postgres_fdw extension + FOREIGN TABLE gendesign_cad_buildings - tradein backend: startup hook creates/refreshes USER MAPPING with password from env GENDESIGN_FDW_PASSWORD (password rotation handled via restart) - geocoder.py: cadastral primary for forward + reverse + suggest; Yandex/Nominatim fallback. reverse_geocode wraps Nominatim in try/except — no more HTTPStatusError → 500. - house_metadata.py and trade_in.py admin stats switched to foreign table - DROP TABLE cad_buildings in tradein (legacy snapshot removed entirely) - import-cadastre.sh deleted (no manual sync needed) Fixes: - /trade-in/api/v1/geocode/reverse 500 (Nominatim ban → no fallback) - estimate confidence_explanation=address_not_geocoded for addresses in our cadastre (e.g. Хохрякова 81) that Nominatim doesn't return Deploy ordering: main migration 100_tradein_fdw_role.sql adds role+view first (strict deploy.yml). Tradein next deploy applies 060_postgres_fdw_extension.sql and 061_drop_legacy_cad_buildings.sql (idempotent, errors ignored). Tradein backend startup creates USER MAPPING when env var present. Verify post-deploy with: docker exec tradein-postgres psql -U tradein -d tradein -c \ "SELECT count(*) FROM gendesign_cad_buildings" # expect ~36k EKB buildings.
This commit is contained in:
lekss361
parent
0efd2dca1d
commit
698ef4f003
17 changed files with 610 additions and 374 deletions
|
|
@ -1,101 +0,0 @@
|
|||
"""Cadastral lookup endpoints — reverse geocode via cad_buildings.
|
||||
|
||||
Used by trade-in backend (and potentially other internal services) to resolve
|
||||
координаты → адрес жилого дома без зависимости от внешних геокодеров.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.core.db import get_db
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
class CadastralReverseResult(BaseModel):
|
||||
address: str
|
||||
lat: float
|
||||
lon: float
|
||||
distance_m: float
|
||||
cad_num: str | None = None
|
||||
|
||||
|
||||
@router.get("/reverse", response_model=CadastralReverseResult)
|
||||
async def reverse(
|
||||
lat: Annotated[float, Query(ge=-90, le=90)],
|
||||
lon: Annotated[float, Query(ge=-180, le=180)],
|
||||
radius_m: Annotated[int, Query(ge=10, le=2000)] = 200,
|
||||
db: Annotated[Session, Depends(get_db)] = None, # type: ignore[assignment]
|
||||
) -> CadastralReverseResult:
|
||||
"""Координаты → ближайшее здание из cad_buildings.
|
||||
|
||||
Filters out гаражи/СНТ/садовод/товарищества — для trade-in нужны жилые дома.
|
||||
|
||||
Returns 404 если в радиусе нет подходящего здания.
|
||||
"""
|
||||
row = db.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT
|
||||
readable_address,
|
||||
cad_num,
|
||||
ST_Y(ST_Centroid(geom)) AS lat,
|
||||
ST_X(ST_Centroid(geom)) AS lon,
|
||||
ST_Distance(
|
||||
geom::geography,
|
||||
ST_SetSRID(
|
||||
ST_MakePoint(
|
||||
CAST(:lon AS double precision),
|
||||
CAST(:lat AS double precision)
|
||||
),
|
||||
4326
|
||||
)::geography
|
||||
) AS distance_m
|
||||
FROM cad_buildings
|
||||
WHERE ST_DWithin(
|
||||
geom::geography,
|
||||
ST_SetSRID(
|
||||
ST_MakePoint(
|
||||
CAST(:lon AS double precision),
|
||||
CAST(:lat AS double precision)
|
||||
),
|
||||
4326
|
||||
)::geography,
|
||||
CAST(:radius AS double precision)
|
||||
)
|
||||
AND readable_address !~* '(гараж|снт|садовод|товарищ|уч\\.)'
|
||||
ORDER BY ST_Distance(
|
||||
geom::geography,
|
||||
ST_SetSRID(
|
||||
ST_MakePoint(
|
||||
CAST(:lon AS double precision),
|
||||
CAST(:lat AS double precision)
|
||||
),
|
||||
4326
|
||||
)::geography
|
||||
)
|
||||
LIMIT 1
|
||||
"""
|
||||
),
|
||||
{"lat": lat, "lon": lon, "radius": float(radius_m)},
|
||||
).first()
|
||||
|
||||
if row is None:
|
||||
raise HTTPException(status_code=404, detail="no cadastral building found within radius")
|
||||
|
||||
return CadastralReverseResult(
|
||||
address=row.readable_address,
|
||||
lat=float(row.lat),
|
||||
lon=float(row.lon),
|
||||
distance_m=round(float(row.distance_m), 1),
|
||||
cad_num=row.cad_num,
|
||||
)
|
||||
|
|
@ -21,7 +21,6 @@ from app.api.v1 import (
|
|||
admin_scrape,
|
||||
admin_weight_profiles,
|
||||
analytics,
|
||||
cadastral,
|
||||
concepts,
|
||||
custom_pois,
|
||||
landing,
|
||||
|
|
@ -79,7 +78,6 @@ app.add_middleware(
|
|||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
app.include_router(cadastral.router, prefix="/api/v1/cadastral", tags=["cadastral"])
|
||||
app.include_router(concepts.router, prefix="/api/v1/concepts", tags=["concepts"])
|
||||
app.include_router(parcels.router, prefix="/api/v1/parcels", tags=["parcels"])
|
||||
app.include_router(analytics.router, prefix="/api/v1/analytics", tags=["analytics"])
|
||||
|
|
|
|||
36
data/sql/100_tradein_fdw_role.sql
Normal file
36
data/sql/100_tradein_fdw_role.sql
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
-- tradein_fdw_reader role + v_tradein_cad_buildings view
|
||||
-- Used by tradein-postgres via postgres_fdw to read live cad_buildings.
|
||||
-- See meta/00_credentials.md for password rotation procedure.
|
||||
BEGIN;
|
||||
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'tradein_fdw_reader') THEN
|
||||
CREATE ROLE tradein_fdw_reader LOGIN PASSWORD '40473b7c10c7474855c7d1aa98f0fd0a0de2499aef0cbd1fead06c9641e4dcd8';
|
||||
END IF;
|
||||
END$$;
|
||||
|
||||
GRANT CONNECT ON DATABASE gendesign TO tradein_fdw_reader;
|
||||
GRANT USAGE ON SCHEMA public TO tradein_fdw_reader;
|
||||
|
||||
CREATE OR REPLACE VIEW v_tradein_cad_buildings AS
|
||||
SELECT
|
||||
cad_num,
|
||||
NULLIF(trim(readable_address), '') AS readable_address,
|
||||
NULLIF(floors, 0) AS floors,
|
||||
CASE WHEN year_built BETWEEN 1800 AND 2100 THEN year_built END AS year_built,
|
||||
round(COALESCE(build_record_area, area), 2)::numeric AS area_m2,
|
||||
purpose,
|
||||
ST_Y(ST_Centroid(geom))::double precision AS lat,
|
||||
ST_X(ST_Centroid(geom))::double precision AS lon
|
||||
FROM cad_buildings
|
||||
WHERE cad_num LIKE '66:41:%'
|
||||
AND geom IS NOT NULL
|
||||
AND readable_address IS NOT NULL;
|
||||
|
||||
GRANT SELECT ON v_tradein_cad_buildings TO tradein_fdw_reader;
|
||||
|
||||
COMMENT ON VIEW v_tradein_cad_buildings IS
|
||||
'FDW source for tradein-mvp (postgres_fdw). Flat 8-col EKB-only slice of cad_buildings. tradein.gendesign_cad_buildings maps here.';
|
||||
|
||||
COMMIT;
|
||||
|
|
@ -33,6 +33,11 @@ services:
|
|||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 10
|
||||
networks:
|
||||
default: {}
|
||||
shared:
|
||||
aliases:
|
||||
- gendesign-postgres
|
||||
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
|
|
@ -67,11 +72,6 @@ services:
|
|||
timeout: 5s
|
||||
retries: 5
|
||||
start_period: 20s
|
||||
networks:
|
||||
default: {}
|
||||
shared:
|
||||
aliases:
|
||||
- gendesign-backend
|
||||
|
||||
frontend:
|
||||
image: ghcr.io/lekss361/gendesign-frontend:${IMAGE_TAG:-latest}
|
||||
|
|
|
|||
|
|
@ -50,6 +50,7 @@ class SuggestResponse(BaseModel):
|
|||
async def suggest_addresses(
|
||||
q: Annotated[str, Query(min_length=2, max_length=200, description="Запрос для автокомплита")],
|
||||
limit: Annotated[int, Query(ge=1, le=15)] = 8,
|
||||
db: Annotated[Session, Depends(get_db)] = None, # type: ignore[assignment]
|
||||
) -> SuggestResponse:
|
||||
"""Автокомплит адресов в пределах ЕКБ.
|
||||
|
||||
|
|
@ -60,7 +61,7 @@ async def suggest_addresses(
|
|||
/api/v1/geocode/suggest?q=Малышева
|
||||
/api/v1/geocode/suggest?q=Цвиллинга # → пусто, такой улицы в ЕКБ нет
|
||||
"""
|
||||
items = await suggest(q, limit=limit)
|
||||
items = await suggest(q, db=db, limit=limit)
|
||||
return SuggestResponse(items=[
|
||||
SuggestItem(
|
||||
label=s.label,
|
||||
|
|
@ -74,12 +75,13 @@ async def suggest_addresses(
|
|||
async def reverse(
|
||||
lat: Annotated[float, Query(ge=-90, le=90)],
|
||||
lon: Annotated[float, Query(ge=-180, le=180)],
|
||||
db: Annotated[Session, Depends(get_db)] = None, # type: ignore[assignment]
|
||||
) -> dict[str, object]:
|
||||
"""Обратный геокодинг — координаты с карты → адрес (map-picker).
|
||||
|
||||
Пример: /api/v1/geocode/reverse?lat=56.8389&lon=60.6057
|
||||
"""
|
||||
address = await reverse_geocode(lat, lon)
|
||||
address = await reverse_geocode(lat, lon, db=db)
|
||||
if address is None:
|
||||
raise HTTPException(status_code=404, detail="address not found for coordinates")
|
||||
return {"address": address, "lat": lat, "lon": lon}
|
||||
|
|
|
|||
|
|
@ -584,7 +584,7 @@ def cache_stats(db: Annotated[Session, Depends(get_db)]) -> dict[str, object]:
|
|||
(SELECT count(*) FROM listings WHERE is_active) AS listings_active,
|
||||
(SELECT max(scraped_at) FROM listings) AS listings_last_scraped,
|
||||
(SELECT count(*) FROM deals) AS deals,
|
||||
(SELECT count(*) FROM cad_buildings) AS cad_buildings,
|
||||
(SELECT count(*) FROM gendesign_cad_buildings) AS cad_buildings,
|
||||
(SELECT count(*) FROM house_metadata) AS house_metadata,
|
||||
(SELECT count(*) FROM trade_in_estimates) AS estimates_total
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -32,10 +32,10 @@ class Settings(BaseSettings):
|
|||
# Redis URL для hot-cache (Phase 3.2). Задаётся через env REDIS_URL.
|
||||
redis_url: str = "redis://localhost:6379/0"
|
||||
|
||||
# Main GenDesign backend URL — для cadastral reverse geocode (cad_buildings).
|
||||
# В prod: http://gendesign-backend:8000 (network alias в docker-compose.prod.yml).
|
||||
# Пусто = cadastral lookup пропускается, используется только Nominatim.
|
||||
main_backend_url: str | None = None
|
||||
# Password for tradein_fdw_reader role — used by backend startup to create/refresh
|
||||
# USER MAPPING for postgres_fdw → gendesign DB (gendesign_remote server).
|
||||
# Пусто = USER MAPPING не создаётся, gendesign_cad_buildings не работает (dev).
|
||||
gendesign_fdw_password: str | None = None
|
||||
|
||||
|
||||
settings = Settings()
|
||||
|
|
|
|||
65
tradein-mvp/backend/app/core/fdw.py
Normal file
65
tradein-mvp/backend/app/core/fdw.py
Normal file
|
|
@ -0,0 +1,65 @@
|
|||
"""postgres_fdw USER MAPPING management for gendesign_remote server.
|
||||
|
||||
USER MAPPING password is in env GENDESIGN_FDW_PASSWORD (not in SQL migration —
|
||||
would leak password to disk).
|
||||
|
||||
This helper applies idempotent CREATE / ALTER USER MAPPING. Reapplied on every
|
||||
backend startup so password rotation through .env.runtime is picked up after restart.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.core.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def ensure_fdw_user_mapping(db: Session) -> None:
|
||||
"""Idempotent CREATE OR ALTER USER MAPPING for tradein → gendesign_remote.
|
||||
|
||||
Skips silently if password not configured (dev / first deploy).
|
||||
"""
|
||||
password = settings.gendesign_fdw_password
|
||||
if not password:
|
||||
logger.warning(
|
||||
"GENDESIGN_FDW_PASSWORD not set — skipping FDW user mapping "
|
||||
"(gendesign_cad_buildings will fail)"
|
||||
)
|
||||
return
|
||||
if not isinstance(password, str) or not password.strip():
|
||||
logger.warning("GENDESIGN_FDW_PASSWORD empty — skipping FDW user mapping")
|
||||
return
|
||||
|
||||
# Check existence
|
||||
exists = db.execute(
|
||||
text(
|
||||
"SELECT 1 FROM pg_user_mappings "
|
||||
"WHERE srvname = 'gendesign_remote' AND usename = current_user"
|
||||
)
|
||||
).first()
|
||||
|
||||
# Password cannot be a bind parameter in CREATE/ALTER USER MAPPING — must inline.
|
||||
# Escape single quotes by doubling.
|
||||
escaped_pw = password.replace("'", "''")
|
||||
if exists is None:
|
||||
db.execute(
|
||||
text(
|
||||
f"CREATE USER MAPPING FOR CURRENT_USER SERVER gendesign_remote "
|
||||
f"OPTIONS (user 'tradein_fdw_reader', password '{escaped_pw}')"
|
||||
)
|
||||
)
|
||||
logger.info("created FDW user mapping for gendesign_remote")
|
||||
else:
|
||||
db.execute(
|
||||
text(
|
||||
f"ALTER USER MAPPING FOR CURRENT_USER SERVER gendesign_remote "
|
||||
f"OPTIONS (SET password '{escaped_pw}')"
|
||||
)
|
||||
)
|
||||
logger.info("refreshed FDW user mapping password for gendesign_remote")
|
||||
|
||||
db.commit()
|
||||
|
|
@ -17,6 +17,8 @@ from fastapi.middleware.cors import CORSMiddleware
|
|||
|
||||
from app.api.v1 import admin, brand, geocode, search, trade_in
|
||||
from app.core.config import settings
|
||||
from app.core.db import SessionLocal
|
||||
from app.core.fdw import ensure_fdw_user_mapping
|
||||
from app.core.ratelimit import RateLimitMiddleware
|
||||
from app.services.scheduler import scheduler_loop
|
||||
|
||||
|
|
@ -41,6 +43,16 @@ if settings.glitchtip_dsn:
|
|||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
# FDW bootstrap: create/refresh USER MAPPING for gendesign_remote postgres_fdw server.
|
||||
# Best-effort: failure does not abort startup, just logs.
|
||||
try:
|
||||
with SessionLocal() as db:
|
||||
ensure_fdw_user_mapping(db)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"FDW user mapping bootstrap failed — cadastral queries may fail"
|
||||
)
|
||||
|
||||
# Startup: launch scheduler background task
|
||||
scheduler_task = asyncio.create_task(scheduler_loop())
|
||||
logger.info("FastAPI lifespan: scheduler task spawned")
|
||||
|
|
|
|||
|
|
@ -158,7 +158,7 @@ async def _nominatim_query(client: httpx.AsyncClient, address: str) -> dict | No
|
|||
lon_f = float(item["lon"])
|
||||
if 60.40 <= lon_f <= 60.85 and 56.65 <= lat_f <= 56.95:
|
||||
return item
|
||||
except Exception: # noqa: BLE001
|
||||
except Exception:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
|
@ -245,7 +245,7 @@ async def _yandex_lookup(address: str, api_key: str) -> GeocodeResult | None:
|
|||
if 60.40 <= lon_f <= 60.85 and 56.65 <= lat_f <= 56.95:
|
||||
best = obj
|
||||
break
|
||||
except Exception: # noqa: BLE001
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
if best is None:
|
||||
|
|
@ -315,7 +315,7 @@ def _parse_yandex_members(members: list[dict]) -> list[GeocodeSuggestion]:
|
|||
label=label, full_address=full,
|
||||
lat=float(lat_str), lon=float(lon_str), kind=kind,
|
||||
))
|
||||
except Exception: # noqa: BLE001
|
||||
except Exception:
|
||||
continue
|
||||
return out
|
||||
|
||||
|
|
@ -445,36 +445,128 @@ async def _nominatim_suggest(query: str, limit: int = 8) -> list[GeocodeSuggesti
|
|||
return out
|
||||
|
||||
|
||||
async def suggest(query: str, limit: int = 8) -> list[GeocodeSuggestion]:
|
||||
"""Автокомплит адресов в ЕКБ. Yandex → Nominatim → [].
|
||||
def _cadastral_forward_sync(
|
||||
db: Session, query: str, limit: int = 5
|
||||
) -> list[GeocodeSuggestion]:
|
||||
"""Forward search via gendesign_cad_buildings FDW.
|
||||
|
||||
Uses ILIKE prefilter (FDW-pushable) + pg_trgm similarity ranking locally.
|
||||
Returns up to `limit` suggestions, or [] on error / no results.
|
||||
"""
|
||||
try:
|
||||
rows = db.execute(
|
||||
text("""
|
||||
SELECT cad_num, readable_address, lat, lon,
|
||||
similarity(readable_address, CAST(:q AS text)) AS sim
|
||||
FROM gendesign_cad_buildings
|
||||
WHERE readable_address ILIKE CAST('%' || :q || '%' AS text)
|
||||
ORDER BY sim DESC, length(readable_address) ASC
|
||||
LIMIT CAST(:lim AS integer)
|
||||
"""),
|
||||
{"q": query, "lim": limit * 4},
|
||||
).mappings().all()
|
||||
except Exception:
|
||||
logger.warning("cadastral forward sync failed for query=%r", query, exc_info=True)
|
||||
return []
|
||||
|
||||
out: list[GeocodeSuggestion] = []
|
||||
for r in rows[:limit]:
|
||||
out.append(GeocodeSuggestion(
|
||||
label=str(r["readable_address"]),
|
||||
full_address=str(r["readable_address"]),
|
||||
lat=float(r["lat"]),
|
||||
lon=float(r["lon"]),
|
||||
kind="house",
|
||||
))
|
||||
return out
|
||||
|
||||
|
||||
def _cadastral_reverse_sync(
|
||||
db: Session, lat: float, lon: float, radius_m: int = 200
|
||||
) -> str | None:
|
||||
"""Reverse lookup via gendesign_cad_buildings FDW.
|
||||
|
||||
bbox prefilter (±0.0025° lat, ±0.005° lon ≈ 280m) is FDW-pushable.
|
||||
Distance calculated locally on small result set. Filters гаражи/СНТ.
|
||||
Returns None on no result or error.
|
||||
"""
|
||||
try:
|
||||
row = db.execute(
|
||||
text("""
|
||||
WITH candidates AS (
|
||||
SELECT cad_num, readable_address, lat, lon,
|
||||
111320.0 * sqrt(
|
||||
pow(CAST(:lat AS double precision) - lat, 2) +
|
||||
pow(
|
||||
cos(radians(CAST(:lat AS double precision)))
|
||||
* (CAST(:lon AS double precision) - lon), 2
|
||||
)
|
||||
) AS dist_m
|
||||
FROM gendesign_cad_buildings
|
||||
WHERE lat BETWEEN CAST(:lat AS double precision) - 0.0025
|
||||
AND CAST(:lat AS double precision) + 0.0025
|
||||
AND lon BETWEEN CAST(:lon AS double precision) - 0.005
|
||||
AND CAST(:lon AS double precision) + 0.005
|
||||
AND readable_address !~* '(гараж|снт|садовод|товарищ|уч\\.)'
|
||||
)
|
||||
SELECT readable_address, dist_m
|
||||
FROM candidates
|
||||
WHERE dist_m < CAST(:radius AS double precision)
|
||||
ORDER BY dist_m ASC
|
||||
LIMIT 1
|
||||
"""),
|
||||
{"lat": lat, "lon": lon, "radius": float(radius_m)},
|
||||
).first()
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"cadastral reverse sync failed for (%.5f, %.5f)", lat, lon, exc_info=True
|
||||
)
|
||||
return None
|
||||
|
||||
if row is None:
|
||||
return None
|
||||
return str(row.readable_address)
|
||||
|
||||
|
||||
async def suggest(
|
||||
query: str, db: Session | None = None, limit: int = 8
|
||||
) -> list[GeocodeSuggestion]:
|
||||
"""Автокомплит адресов в ЕКБ. Cadastral FDW → Yandex → Nominatim → [].
|
||||
|
||||
db: если передан — cadastral lookup через gendesign_cad_buildings (первый tier).
|
||||
Без кэша (дешёво, провайдеры толерируют автокомплит-запросы).
|
||||
"""
|
||||
if not query or len(query.strip()) < 2:
|
||||
return []
|
||||
|
||||
# Tier 0: cadastral FDW (если db доступна) — самый быстрый, без внешних запросов
|
||||
if db is not None:
|
||||
cad_results = _cadastral_forward_sync(db, query.strip(), limit)
|
||||
if cad_results:
|
||||
return cad_results
|
||||
|
||||
if settings.yandex_geocoder_key:
|
||||
try:
|
||||
results = await _yandex_suggest(query, settings.yandex_geocoder_key, limit)
|
||||
if results:
|
||||
return results
|
||||
except Exception: # noqa: BLE001
|
||||
except Exception:
|
||||
logger.exception("yandex suggest failed, falling back to nominatim")
|
||||
|
||||
try:
|
||||
return await _nominatim_suggest(query, limit)
|
||||
except Exception: # noqa: BLE001
|
||||
except Exception:
|
||||
logger.exception("nominatim suggest failed")
|
||||
return []
|
||||
|
||||
|
||||
# ── Public API ───────────────────────────────────────────────────────────────
|
||||
async def geocode(address: str, db: Session) -> GeocodeResult | None:
|
||||
"""Геокодинг с кэшем. Yandex → Nominatim → None.
|
||||
"""Геокодинг с кэшем. Cadastral FDW → Yandex → Nominatim → None.
|
||||
|
||||
Args:
|
||||
address: пользовательский ввод (может быть грязным — нормализуем).
|
||||
db: сессия Postgres для cache lookup/write.
|
||||
db: сессия Postgres для cache lookup/write и cadastral FDW lookup.
|
||||
|
||||
Returns:
|
||||
GeocodeResult или None если ни один провайдер не отвечает.
|
||||
|
|
@ -490,12 +582,33 @@ async def geocode(address: str, db: Session) -> GeocodeResult | None:
|
|||
logger.info("geocode cache hit: %s", addr_norm)
|
||||
return cached
|
||||
|
||||
# 2. Yandex (если есть key) с typo-fallback
|
||||
# 2. Cadastral FDW (прямой запрос к gendesign_cad_buildings — без внешнего API)
|
||||
cad_suggestions = _cadastral_forward_sync(db, address.strip(), limit=1)
|
||||
if cad_suggestions:
|
||||
s = cad_suggestions[0]
|
||||
result = GeocodeResult(
|
||||
lat=s.lat,
|
||||
lon=s.lon,
|
||||
full_address=s.full_address,
|
||||
provider="nominatim", # treat as "local" — same confidence as nominatim
|
||||
confidence="exact",
|
||||
)
|
||||
_cache_put(db, addr_norm, result)
|
||||
logger.info(
|
||||
"geocode cadastral fdw: %s → (%.5f, %.5f)", addr_norm, result.lat, result.lon
|
||||
)
|
||||
return result
|
||||
|
||||
# 3. Yandex (если есть key) с typo-fallback
|
||||
if settings.yandex_geocoder_key:
|
||||
try:
|
||||
result = await _yandex_lookup(address, settings.yandex_geocoder_key)
|
||||
# Если результат вне ЕКБ — пробуем typo-варианты
|
||||
in_ekb = result is not None and 60.40 <= result.lon <= 60.85 and 56.65 <= result.lat <= 56.95
|
||||
in_ekb = (
|
||||
result is not None
|
||||
and 60.40 <= result.lon <= 60.85
|
||||
and 56.65 <= result.lat <= 56.95
|
||||
)
|
||||
if result is not None and in_ekb:
|
||||
_cache_put(db, addr_norm, result)
|
||||
logger.info("geocode yandex: %s → (%.5f, %.5f)", addr_norm, result.lat, result.lon)
|
||||
|
|
@ -504,7 +617,7 @@ async def geocode(address: str, db: Session) -> GeocodeResult | None:
|
|||
for variant in _typo_variants(address, limit=4):
|
||||
try:
|
||||
result = await _yandex_lookup(variant, settings.yandex_geocoder_key)
|
||||
except Exception: # noqa: BLE001
|
||||
except Exception:
|
||||
continue
|
||||
if result is None:
|
||||
continue
|
||||
|
|
@ -515,10 +628,10 @@ async def geocode(address: str, db: Session) -> GeocodeResult | None:
|
|||
addr_norm, variant, result.lat, result.lon,
|
||||
)
|
||||
return result
|
||||
except Exception: # noqa: BLE001
|
||||
except Exception:
|
||||
logger.exception("yandex geocoder failed, falling back to nominatim")
|
||||
|
||||
# 3. Nominatim fallback
|
||||
# 4. Nominatim fallback
|
||||
try:
|
||||
result = await _nominatim_lookup(address)
|
||||
if result is not None:
|
||||
|
|
@ -529,7 +642,7 @@ async def geocode(address: str, db: Session) -> GeocodeResult | None:
|
|||
# Nominatim rate-limit policy: 1 req/sec — спим после успешного запроса
|
||||
await asyncio.sleep(1.0)
|
||||
return result
|
||||
except Exception: # noqa: BLE001
|
||||
except Exception:
|
||||
logger.exception("nominatim geocoder failed")
|
||||
|
||||
return None
|
||||
|
|
@ -569,29 +682,6 @@ def _format_reverse_address(addr: dict) -> str | None:
|
|||
return ", ".join(parts)
|
||||
|
||||
|
||||
async def _cadastral_reverse(lat: float, lon: float) -> str | None:
|
||||
"""Reverse via main backend cad_buildings — fast, no external API.
|
||||
|
||||
Returns None on any error (network, 404, timeout, missing config).
|
||||
"""
|
||||
if not settings.main_backend_url:
|
||||
return None
|
||||
url = settings.main_backend_url.rstrip("/") + "/api/v1/cadastral/reverse"
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=3.0) as client:
|
||||
response = await client.get(url, params={"lat": str(lat), "lon": str(lon)})
|
||||
if response.status_code == 404:
|
||||
return None
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
addr = data.get("address")
|
||||
if isinstance(addr, str) and addr.strip():
|
||||
return addr
|
||||
except Exception:
|
||||
logger.exception("cadastral reverse failed for (%.5f, %.5f)", lat, lon)
|
||||
return None
|
||||
|
||||
|
||||
@retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=1, max=8))
|
||||
async def _nominatim_reverse(lat: float, lon: float) -> str | None:
|
||||
"""Nominatim /reverse — extracted for fallback usage. @retry handles transient errors."""
|
||||
|
|
@ -623,17 +713,20 @@ async def _nominatim_reverse(lat: float, lon: float) -> str | None:
|
|||
return data.get("display_name")
|
||||
|
||||
|
||||
async def reverse_geocode(lat: float, lon: float) -> str | None:
|
||||
"""Cadastral (cad_buildings via main backend) → Nominatim fallback.
|
||||
async def reverse_geocode(lat: float, lon: float, db: Session | None = None) -> str | None:
|
||||
"""Cadastral FDW → Nominatim fallback.
|
||||
|
||||
Возвращает None если ни один источник не дал адрес. Endpoint
|
||||
api/v1/geocode/reverse сам выкинет 404. НЕ даёт выйти HTTPStatusError
|
||||
наверх — раньше Nominatim 403 → RetryError → FastAPI 500.
|
||||
|
||||
db: если передан — cadastral lookup через gendesign_cad_buildings FDW (первый tier).
|
||||
"""
|
||||
# 1. Cadastral primary (fast, no external API, returns жилой дом not POI)
|
||||
cadastral = await _cadastral_reverse(lat, lon)
|
||||
if cadastral:
|
||||
return cadastral
|
||||
# 1. Cadastral FDW primary (без внешнего API, возвращает жилой дом not POI)
|
||||
if db is not None:
|
||||
cadastral = _cadastral_reverse_sync(db, lat, lon)
|
||||
if cadastral:
|
||||
return cadastral
|
||||
|
||||
# 2. Nominatim fallback (wrap to prevent 500 on ban/rate-limit)
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -138,34 +138,56 @@ def _cache_put(db: Session, meta: HouseMetadata, raw: dict) -> None:
|
|||
db.commit()
|
||||
|
||||
|
||||
# ── Источник #393: кадастр зданий (cad_buildings, импорт из gendesign-БД) ────
|
||||
# ── Источник #393: кадастр зданий (gendesign_cad_buildings FDW) ──────────────
|
||||
def _cad_buildings_get(db: Session, lat: float, lon: float) -> HouseMetadata | None:
|
||||
"""Ближайшее здание из cad_buildings (данные Росреестра) по координатам.
|
||||
"""Ближайшее здание из gendesign_cad_buildings FDW (данные Росреестра) по координатам.
|
||||
|
||||
Приоритетный источник house_metadata — год / этажность полнее, чем OSM
|
||||
(~34к домов ЕКБ с годом постройки). Таблицу наполняет import-cadastre.sh.
|
||||
(~36к домов ЕКБ с годом постройки). Использует bbox+haversine вместо ST_DWithin —
|
||||
FDW-таблица не имеет PostGIS-колонки geom (только lat/lon скалярные).
|
||||
"""
|
||||
row = db.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT year_built, floors
|
||||
FROM cad_buildings
|
||||
WHERE geom IS NOT NULL
|
||||
AND (year_built IS NOT NULL OR floors IS NOT NULL)
|
||||
AND ST_DWithin(geom::geography, ST_MakePoint(:lon, :lat)::geography, :radius)
|
||||
ORDER BY ST_Distance(geom::geography, ST_MakePoint(:lon, :lat)::geography)
|
||||
LIMIT 1
|
||||
"""
|
||||
),
|
||||
{"lat": lat, "lon": lon, "radius": _CAD_RADIUS_M},
|
||||
).fetchone()
|
||||
try:
|
||||
row = db.execute(
|
||||
text(
|
||||
"""
|
||||
WITH candidates AS (
|
||||
SELECT year_built, floors, lat, lon,
|
||||
111320.0 * sqrt(
|
||||
pow(CAST(:lat AS double precision) - lat, 2) +
|
||||
pow(
|
||||
cos(radians(CAST(:lat AS double precision)))
|
||||
* (CAST(:lon AS double precision) - lon), 2
|
||||
)
|
||||
) AS dist_m
|
||||
FROM gendesign_cad_buildings
|
||||
WHERE (year_built IS NOT NULL OR floors IS NOT NULL)
|
||||
AND lat BETWEEN CAST(:lat AS double precision) - 0.0010
|
||||
AND CAST(:lat AS double precision) + 0.0010
|
||||
AND lon BETWEEN CAST(:lon AS double precision) - 0.0015
|
||||
AND CAST(:lon AS double precision) + 0.0015
|
||||
)
|
||||
SELECT year_built, floors
|
||||
FROM candidates
|
||||
WHERE dist_m < CAST(:radius AS double precision)
|
||||
ORDER BY dist_m ASC
|
||||
LIMIT 1
|
||||
"""
|
||||
),
|
||||
{"lat": lat, "lon": lon, "radius": float(_CAD_RADIUS_M)},
|
||||
).fetchone()
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"gendesign_cad_buildings lookup failed at (%.5f, %.5f)", lat, lon, exc_info=True
|
||||
)
|
||||
return None
|
||||
|
||||
if row is None:
|
||||
return None
|
||||
return HouseMetadata(
|
||||
lat=lat, lon=lon,
|
||||
year_built=row.year_built,
|
||||
total_floors=row.floors,
|
||||
house_type=None, # cad_buildings.purpose не маппится в panel/brick
|
||||
house_type=None, # purpose не маппится в panel/brick
|
||||
total_units=None,
|
||||
source="cadastre",
|
||||
)
|
||||
|
|
@ -245,12 +267,12 @@ async def get_house_metadata(lat: float, lon: float, db: Session) -> HouseMetada
|
|||
lat, lon, meta.year_built, meta.total_floors, meta.house_type,
|
||||
)
|
||||
return meta
|
||||
except Exception: # noqa: BLE001
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"house_metadata enrichment failed at (%.5f, %.5f)", lat, lon, exc_info=True
|
||||
)
|
||||
try:
|
||||
db.rollback()
|
||||
except Exception: # noqa: BLE001
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -1,34 +0,0 @@
|
|||
-- Trade-In MVP — кадастр зданий ЕКБ (#393).
|
||||
-- Срез cad_buildings из gendesign-БД (данные Росреестра по зданиям),
|
||||
-- импортируется ETL-ом deploy/import-cadastre.sh.
|
||||
-- Основной источник обогащения house_metadata (год / этажность) —
|
||||
-- данные точнее и полнее, чем OSM Overpass.
|
||||
|
||||
BEGIN;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS cad_buildings (
|
||||
cad_num text PRIMARY KEY, -- кадастровый номер здания, 66:41:...
|
||||
address text,
|
||||
floors int,
|
||||
year_built int,
|
||||
area_m2 numeric(12, 2), -- площадь здания (не квартиры)
|
||||
purpose text, -- Жилое / Нежилое / Многоквартирный дом
|
||||
lat double precision,
|
||||
lon double precision,
|
||||
geom geometry(Point, 4326),
|
||||
imported_at timestamptz NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS cad_buildings_geom_idx ON cad_buildings USING GIST (geom);
|
||||
|
||||
-- Авто-update geom из (lat, lon) — переиспользуем функцию из 002_core_tables.sql.
|
||||
DROP TRIGGER IF EXISTS cad_buildings_set_geom_trg ON cad_buildings;
|
||||
CREATE TRIGGER cad_buildings_set_geom_trg
|
||||
BEFORE INSERT OR UPDATE OF lat, lon ON cad_buildings
|
||||
FOR EACH ROW EXECUTE FUNCTION listings_set_geom();
|
||||
|
||||
COMMENT ON TABLE cad_buildings IS
|
||||
'#393 — кадастр зданий ЕКБ (срез cad_buildings из gendesign-БД). '
|
||||
'Приоритетный источник обогащения house_metadata: год постройки / этажность.';
|
||||
|
||||
COMMIT;
|
||||
35
tradein-mvp/backend/data/sql/060_postgres_fdw_extension.sql
Normal file
35
tradein-mvp/backend/data/sql/060_postgres_fdw_extension.sql
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
-- postgres_fdw extension for tradein → gendesign live reads.
|
||||
-- Idempotent. Server config only (no USER MAPPING — that's set by backend startup with env password).
|
||||
BEGIN;
|
||||
|
||||
CREATE EXTENSION IF NOT EXISTS postgres_fdw;
|
||||
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (SELECT 1 FROM pg_foreign_server WHERE srvname = 'gendesign_remote') THEN
|
||||
CREATE SERVER gendesign_remote
|
||||
FOREIGN DATA WRAPPER postgres_fdw
|
||||
OPTIONS (host 'gendesign-postgres', dbname 'gendesign', port '5432', extensions 'postgis');
|
||||
END IF;
|
||||
END$$;
|
||||
|
||||
-- Drop existing foreign table if mistakenly created with wrong schema (idempotent re-create below)
|
||||
DROP FOREIGN TABLE IF EXISTS gendesign_cad_buildings;
|
||||
|
||||
CREATE FOREIGN TABLE gendesign_cad_buildings (
|
||||
cad_num text,
|
||||
readable_address text,
|
||||
year_built integer,
|
||||
floors integer,
|
||||
area_m2 numeric,
|
||||
purpose text,
|
||||
lat double precision,
|
||||
lon double precision
|
||||
)
|
||||
SERVER gendesign_remote
|
||||
OPTIONS (schema_name 'public', table_name 'v_tradein_cad_buildings');
|
||||
|
||||
COMMENT ON FOREIGN TABLE gendesign_cad_buildings IS
|
||||
'Live view of gendesign.cad_buildings (EKB only). USER MAPPING managed by backend startup.';
|
||||
|
||||
COMMIT;
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
-- Drop legacy cad_buildings snapshot table (replaced by gendesign_cad_buildings foreign table).
|
||||
-- Idempotent.
|
||||
BEGIN;
|
||||
|
||||
DROP TRIGGER IF EXISTS cad_buildings_set_geom_trg ON cad_buildings;
|
||||
DROP TABLE IF EXISTS cad_buildings CASCADE;
|
||||
|
||||
COMMIT;
|
||||
|
|
@ -1,4 +1,12 @@
|
|||
"""Unit tests for cadastral reverse geocode and modified reverse_geocode."""
|
||||
"""Unit tests for cadastral FDW SQL-backed geocoder helpers and reverse_geocode.
|
||||
|
||||
Tests cover:
|
||||
- _cadastral_forward_sync: returns suggestions / empty list / graceful error
|
||||
- _cadastral_reverse_sync: returns address / None on no result / None on error
|
||||
- reverse_geocode: cadastral primary, Nominatim fallback, both-fail → None (500-fix)
|
||||
- geocode: cadastral as tier 0
|
||||
- suggest: cadastral as tier 0
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
|
@ -7,144 +15,312 @@ from unittest.mock import AsyncMock, MagicMock, patch
|
|||
|
||||
# DATABASE_URL required by config before any app import.
|
||||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
|
||||
os.environ.setdefault("MAIN_BACKEND_URL", "http://gendesign-backend:8000")
|
||||
|
||||
# WeasyPrint stub — not installed in CI without GTK.
|
||||
_wp_mock = MagicMock()
|
||||
sys.modules.setdefault("weasyprint", _wp_mock)
|
||||
|
||||
import pytest
|
||||
|
||||
from app.services.geocoder import _cadastral_reverse, reverse_geocode
|
||||
from app.services.geocoder import ( # noqa: E402
|
||||
GeocodeSuggestion,
|
||||
_cadastral_forward_sync,
|
||||
_cadastral_reverse_sync,
|
||||
geocode,
|
||||
reverse_geocode,
|
||||
suggest,
|
||||
)
|
||||
|
||||
# ── helpers ───────────────────────────────────────────────────────────────────
|
||||
|
||||
def _make_db_row(**kwargs: object) -> MagicMock:
|
||||
"""Mock row object with dict-like access (MappingResult style)."""
|
||||
row = MagicMock()
|
||||
row.__getitem__ = lambda self, k: kwargs[k]
|
||||
for k, v in kwargs.items():
|
||||
setattr(row, k, v)
|
||||
return row
|
||||
|
||||
|
||||
# ── _cadastral_reverse ────────────────────────────────────────────────────────
|
||||
def _make_db(rows: list[object]) -> MagicMock:
|
||||
"""Session mock whose execute().mappings().all() returns rows."""
|
||||
db = MagicMock()
|
||||
result = MagicMock()
|
||||
result.mappings.return_value.all.return_value = rows
|
||||
result.first.return_value = rows[0] if rows else None
|
||||
db.execute.return_value = result
|
||||
return db
|
||||
|
||||
|
||||
async def test_cadastral_reverse_returns_address_on_200() -> None:
|
||||
"""Main backend returns 200 → address string returned."""
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = 200
|
||||
mock_response.raise_for_status = MagicMock()
|
||||
mock_response.json.return_value = {
|
||||
"address": "ул. Малышева, 30, Екатеринбург",
|
||||
"lat": 56.838,
|
||||
"lon": 60.605,
|
||||
"distance_m": 12.5,
|
||||
}
|
||||
|
||||
mock_client = AsyncMock()
|
||||
mock_client.get = AsyncMock(return_value=mock_response)
|
||||
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
|
||||
mock_client.__aexit__ = AsyncMock(return_value=False)
|
||||
|
||||
with patch("app.services.geocoder.httpx.AsyncClient", return_value=mock_client):
|
||||
result = await _cadastral_reverse(56.838, 60.605)
|
||||
|
||||
assert result == "ул. Малышева, 30, Екатеринбург"
|
||||
def _make_db_error() -> MagicMock:
|
||||
"""Session mock whose execute() raises RuntimeError."""
|
||||
db = MagicMock()
|
||||
db.execute.side_effect = RuntimeError("FDW connection failed")
|
||||
return db
|
||||
|
||||
|
||||
async def test_cadastral_reverse_returns_none_on_404() -> None:
|
||||
"""Main backend returns 404 (no building in radius) → None, no exception."""
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = 404
|
||||
# ── _cadastral_forward_sync ───────────────────────────────────────────────────
|
||||
|
||||
mock_client = AsyncMock()
|
||||
mock_client.get = AsyncMock(return_value=mock_response)
|
||||
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
|
||||
mock_client.__aexit__ = AsyncMock(return_value=False)
|
||||
def test_forward_sync_returns_suggestions_when_rows_exist() -> None:
|
||||
"""DB returns rows → list of GeocodeSuggestion returned."""
|
||||
row = _make_db_row(
|
||||
readable_address="ул. Малышева, 30, Екатеринбург",
|
||||
lat=56.838,
|
||||
lon=60.605,
|
||||
sim=0.9,
|
||||
cad_num="66:41:0101001:123",
|
||||
)
|
||||
db = _make_db([row])
|
||||
|
||||
with patch("app.services.geocoder.httpx.AsyncClient", return_value=mock_client):
|
||||
result = await _cadastral_reverse(56.838, 60.605)
|
||||
results = _cadastral_forward_sync(db, "Малышева 30", limit=5)
|
||||
|
||||
assert result is None
|
||||
assert len(results) == 1
|
||||
assert results[0].full_address == "ул. Малышева, 30, Екатеринбург"
|
||||
assert results[0].lat == 56.838
|
||||
assert results[0].lon == 60.605
|
||||
assert results[0].kind == "house"
|
||||
|
||||
|
||||
async def test_cadastral_reverse_returns_none_on_timeout() -> None:
|
||||
"""Network timeout → None returned, no exception bubbles out."""
|
||||
import httpx
|
||||
def test_forward_sync_returns_empty_when_no_rows() -> None:
|
||||
"""DB returns no rows → empty list returned, no exception."""
|
||||
db = _make_db([])
|
||||
|
||||
mock_client = AsyncMock()
|
||||
mock_client.get = AsyncMock(side_effect=httpx.TimeoutException("timeout"))
|
||||
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
|
||||
mock_client.__aexit__ = AsyncMock(return_value=False)
|
||||
results = _cadastral_forward_sync(db, "Несуществующая улица", limit=5)
|
||||
|
||||
with patch("app.services.geocoder.httpx.AsyncClient", return_value=mock_client):
|
||||
result = await _cadastral_reverse(56.838, 60.605)
|
||||
|
||||
assert result is None
|
||||
assert results == []
|
||||
|
||||
|
||||
async def test_cadastral_reverse_returns_none_when_not_configured() -> None:
|
||||
"""main_backend_url is None → immediately returns None without HTTP call."""
|
||||
with patch("app.services.geocoder.settings") as mock_settings:
|
||||
mock_settings.main_backend_url = None
|
||||
def test_forward_sync_returns_empty_on_db_error() -> None:
|
||||
"""DB raises → empty list, no exception bubbles out."""
|
||||
db = _make_db_error()
|
||||
|
||||
with patch("app.services.geocoder.httpx.AsyncClient") as mock_client_cls:
|
||||
result = await _cadastral_reverse(56.838, 60.605)
|
||||
mock_client_cls.assert_not_called()
|
||||
results = _cadastral_forward_sync(db, "Малышева", limit=5)
|
||||
|
||||
assert result is None
|
||||
assert results == []
|
||||
|
||||
|
||||
def test_forward_sync_respects_limit() -> None:
|
||||
"""Returns at most `limit` items even if DB has more."""
|
||||
rows = [
|
||||
_make_db_row(
|
||||
readable_address=f"ул. Тестовая, {i}, Екатеринбург",
|
||||
lat=56.838 + i * 0.001,
|
||||
lon=60.605,
|
||||
sim=0.9,
|
||||
cad_num=f"66:41:01010{i:02d}:1",
|
||||
)
|
||||
for i in range(10)
|
||||
]
|
||||
db = _make_db(rows)
|
||||
|
||||
results = _cadastral_forward_sync(db, "Тестовая", limit=3)
|
||||
|
||||
assert len(results) <= 3
|
||||
|
||||
|
||||
# ── _cadastral_reverse_sync ───────────────────────────────────────────────────
|
||||
|
||||
def test_reverse_sync_returns_address_when_row_found() -> None:
|
||||
"""DB finds building within radius → address string returned."""
|
||||
row = MagicMock()
|
||||
row.readable_address = "ул. Хохрякова, 81, Екатеринбург"
|
||||
|
||||
db = MagicMock()
|
||||
result = MagicMock()
|
||||
result.first.return_value = row
|
||||
db.execute.return_value = result
|
||||
|
||||
addr = _cadastral_reverse_sync(db, 56.838, 60.605, radius_m=200)
|
||||
|
||||
assert addr == "ул. Хохрякова, 81, Екатеринбург"
|
||||
|
||||
|
||||
def test_reverse_sync_returns_none_when_no_row_in_radius() -> None:
|
||||
"""No row within radius → None returned."""
|
||||
db = MagicMock()
|
||||
result = MagicMock()
|
||||
result.first.return_value = None
|
||||
db.execute.return_value = result
|
||||
|
||||
addr = _cadastral_reverse_sync(db, 56.838, 60.605, radius_m=200)
|
||||
|
||||
assert addr is None
|
||||
|
||||
|
||||
def test_reverse_sync_returns_none_on_db_error() -> None:
|
||||
"""DB raises → None returned, no exception bubbles out."""
|
||||
db = _make_db_error()
|
||||
|
||||
addr = _cadastral_reverse_sync(db, 56.838, 60.605)
|
||||
|
||||
assert addr is None
|
||||
|
||||
|
||||
# ── reverse_geocode ───────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
async def test_reverse_geocode_returns_cadastral_when_available() -> None:
|
||||
async def test_reverse_geocode_prefers_cadastral_over_nominatim() -> None:
|
||||
"""Cadastral returns address → Nominatim is never called."""
|
||||
db = MagicMock()
|
||||
with (
|
||||
patch(
|
||||
"app.services.geocoder._cadastral_reverse",
|
||||
new_callable=AsyncMock,
|
||||
"app.services.geocoder._cadastral_reverse_sync",
|
||||
return_value="ул. Малышева, 30, Екатеринбург",
|
||||
) as mock_cadastral,
|
||||
) as mock_cad,
|
||||
patch(
|
||||
"app.services.geocoder._nominatim_reverse",
|
||||
new_callable=AsyncMock,
|
||||
) as mock_nominatim,
|
||||
) as mock_nom,
|
||||
):
|
||||
result = await reverse_geocode(56.838, 60.605)
|
||||
result = await reverse_geocode(56.838, 60.605, db=db)
|
||||
|
||||
assert result == "ул. Малышева, 30, Екатеринбург"
|
||||
mock_cadastral.assert_called_once_with(56.838, 60.605)
|
||||
mock_nominatim.assert_not_called()
|
||||
mock_cad.assert_called_once_with(db, 56.838, 60.605)
|
||||
mock_nom.assert_not_called()
|
||||
|
||||
|
||||
async def test_reverse_geocode_falls_back_to_nominatim_when_cadastral_none() -> None:
|
||||
"""Cadastral returns None → Nominatim is called and its result returned."""
|
||||
"""Cadastral returns None → Nominatim called and result returned."""
|
||||
db = MagicMock()
|
||||
with (
|
||||
patch(
|
||||
"app.services.geocoder._cadastral_reverse",
|
||||
new_callable=AsyncMock,
|
||||
return_value=None,
|
||||
),
|
||||
patch("app.services.geocoder._cadastral_reverse_sync", return_value=None),
|
||||
patch(
|
||||
"app.services.geocoder._nominatim_reverse",
|
||||
new_callable=AsyncMock,
|
||||
return_value="Трамвайный переулок, 2, Екатеринбург",
|
||||
) as mock_nominatim,
|
||||
) as mock_nom,
|
||||
):
|
||||
result = await reverse_geocode(56.838, 60.605)
|
||||
result = await reverse_geocode(56.838, 60.605, db=db)
|
||||
|
||||
assert result == "Трамвайный переулок, 2, Екатеринбург"
|
||||
mock_nominatim.assert_called_once_with(56.838, 60.605)
|
||||
mock_nom.assert_called_once_with(56.838, 60.605)
|
||||
|
||||
|
||||
async def test_reverse_geocode_returns_none_when_both_fail() -> None:
|
||||
"""Both cadastral and Nominatim fail → returns None, no exception raised."""
|
||||
"""Both cadastral and Nominatim fail → returns None, no exception raised (500 fix)."""
|
||||
db = MagicMock()
|
||||
with (
|
||||
patch(
|
||||
"app.services.geocoder._cadastral_reverse",
|
||||
new_callable=AsyncMock,
|
||||
return_value=None,
|
||||
),
|
||||
patch("app.services.geocoder._cadastral_reverse_sync", return_value=None),
|
||||
patch(
|
||||
"app.services.geocoder._nominatim_reverse",
|
||||
new_callable=AsyncMock,
|
||||
side_effect=RuntimeError("Nominatim 403 Forbidden — IP banned"),
|
||||
),
|
||||
):
|
||||
result = await reverse_geocode(56.838, 60.605)
|
||||
result = await reverse_geocode(56.838, 60.605, db=db)
|
||||
|
||||
# Must return None, not raise
|
||||
assert result is None
|
||||
|
||||
|
||||
async def test_reverse_geocode_without_db_skips_cadastral() -> None:
|
||||
"""When db=None, cadastral tier is skipped entirely → goes straight to Nominatim."""
|
||||
with (
|
||||
patch(
|
||||
"app.services.geocoder._cadastral_reverse_sync",
|
||||
) as mock_cad,
|
||||
patch(
|
||||
"app.services.geocoder._nominatim_reverse",
|
||||
new_callable=AsyncMock,
|
||||
return_value="ул. Ленина, 1, Екатеринбург",
|
||||
),
|
||||
):
|
||||
result = await reverse_geocode(56.838, 60.605, db=None)
|
||||
|
||||
mock_cad.assert_not_called()
|
||||
assert result == "ул. Ленина, 1, Екатеринбург"
|
||||
|
||||
|
||||
# ── geocode: cadastral as first tier ─────────────────────────────────────────
|
||||
|
||||
async def test_geocode_uses_cadastral_before_yandex() -> None:
|
||||
"""Cadastral FDW returns suggestion → geocode returns it without hitting Yandex."""
|
||||
db = MagicMock()
|
||||
# Cache miss
|
||||
db.execute.side_effect = [
|
||||
MagicMock(fetchone=lambda: None), # _cache_get
|
||||
]
|
||||
|
||||
cad_suggestion = GeocodeSuggestion(
|
||||
label="ул. Хохрякова, 81",
|
||||
full_address="ул. Хохрякова, 81, Екатеринбург",
|
||||
lat=56.840,
|
||||
lon=60.610,
|
||||
kind="house",
|
||||
)
|
||||
|
||||
with (
|
||||
patch("app.services.geocoder._cache_get", return_value=None),
|
||||
patch(
|
||||
"app.services.geocoder._cadastral_forward_sync",
|
||||
return_value=[cad_suggestion],
|
||||
) as mock_cad,
|
||||
patch("app.services.geocoder._cache_put"),
|
||||
patch("app.services.geocoder._yandex_lookup", new_callable=AsyncMock) as mock_yandex,
|
||||
):
|
||||
result = await geocode("Хохрякова 81", db)
|
||||
|
||||
assert result is not None
|
||||
assert result.lat == 56.840
|
||||
assert result.lon == 60.610
|
||||
assert result.confidence == "exact"
|
||||
mock_cad.assert_called_once()
|
||||
mock_yandex.assert_not_called()
|
||||
|
||||
|
||||
# ── suggest: cadastral as first tier ─────────────────────────────────────────
|
||||
|
||||
async def test_suggest_uses_cadastral_before_yandex() -> None:
|
||||
"""Cadastral FDW returns results → suggest returns them without hitting Yandex."""
|
||||
db = MagicMock()
|
||||
cad_suggestions = [
|
||||
GeocodeSuggestion(
|
||||
label="ул. Малышева, 30",
|
||||
full_address="ул. Малышева, 30, Екатеринбург",
|
||||
lat=56.838,
|
||||
lon=60.605,
|
||||
kind="house",
|
||||
)
|
||||
]
|
||||
|
||||
with (
|
||||
patch(
|
||||
"app.services.geocoder._cadastral_forward_sync",
|
||||
return_value=cad_suggestions,
|
||||
) as mock_cad,
|
||||
patch(
|
||||
"app.services.geocoder._yandex_suggest",
|
||||
new_callable=AsyncMock,
|
||||
) as mock_yandex,
|
||||
):
|
||||
results = await suggest("Малышева", db=db, limit=8)
|
||||
|
||||
assert len(results) == 1
|
||||
assert results[0].full_address == "ул. Малышева, 30, Екатеринбург"
|
||||
mock_cad.assert_called_once()
|
||||
mock_yandex.assert_not_called()
|
||||
|
||||
|
||||
async def test_suggest_falls_back_to_yandex_when_cadastral_empty() -> None:
|
||||
"""Cadastral returns [] → falls through to Yandex."""
|
||||
db = MagicMock()
|
||||
yandex_result = GeocodeSuggestion(
|
||||
label="Малышева · Октябрьский",
|
||||
full_address="улица Малышева, Екатеринбург",
|
||||
lat=56.838,
|
||||
lon=60.605,
|
||||
kind="street",
|
||||
)
|
||||
|
||||
with (
|
||||
patch("app.services.geocoder._cadastral_forward_sync", return_value=[]),
|
||||
patch("app.services.geocoder.settings") as mock_settings,
|
||||
patch(
|
||||
"app.services.geocoder._yandex_suggest",
|
||||
new_callable=AsyncMock,
|
||||
return_value=[yandex_result],
|
||||
) as mock_yandex,
|
||||
):
|
||||
mock_settings.yandex_geocoder_key = "fake-key"
|
||||
results = await suggest("Малышева", db=db, limit=8)
|
||||
|
||||
assert len(results) == 1
|
||||
mock_yandex.assert_called_once()
|
||||
|
|
|
|||
|
|
@ -1,78 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
# Импорт кадастра зданий ЕКБ из gendesign-БД в tradein cad_buildings (#393).
|
||||
#
|
||||
# Источник: gendesign-postgres-1 / cad_buildings (~37k зданий, 36k по ЕКБ).
|
||||
# Берём ЕКБ (cad_num LIKE '66:41:%') с годом постройки или этажностью.
|
||||
# Координаты — центроид geom. Используется как источник обогащения
|
||||
# house_metadata (год / этажность) — точнее OSM.
|
||||
#
|
||||
# Запуск на прод-хосте: ./import-cadastre.sh
|
||||
# Повторяемо: upsert по cad_num (PRIMARY KEY), новый запуск обновит данные.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SRC_PG="${SRC_PG:-gendesign-postgres-1}"
|
||||
DST_PG="${DST_PG:-tradein-postgres}"
|
||||
SRC_DB="${SRC_DB:-gendesign}"
|
||||
SRC_USER="${SRC_USER:-gendesign}"
|
||||
|
||||
echo "[$(date -u +%H:%M:%S)] import-cadastre: здания ЕКБ из cad_buildings"
|
||||
|
||||
# 1. Staging-таблица в tradein.
|
||||
docker exec "$DST_PG" psql -U tradein -d tradein -v ON_ERROR_STOP=on -c "
|
||||
CREATE TABLE IF NOT EXISTS cad_buildings_staging (
|
||||
cad_num text PRIMARY KEY, address text, floors int, year_built int,
|
||||
area_m2 numeric, purpose text, lat double precision, lon double precision
|
||||
);
|
||||
TRUNCATE cad_buildings_staging;
|
||||
"
|
||||
|
||||
# 2. Трансформируем на gendesign-стороне, пайпим CSV в staging.
|
||||
# Центроид geom → lat/lon (ST_Transform на 4326 на случай иного SRID).
|
||||
docker exec "$SRC_PG" psql -U "$SRC_USER" -d "$SRC_DB" -v ON_ERROR_STOP=on -c "
|
||||
COPY (
|
||||
SELECT
|
||||
cad_num,
|
||||
NULLIF(trim(readable_address), '') AS address,
|
||||
NULLIF(floors, 0) AS floors,
|
||||
CASE WHEN year_built BETWEEN 1800 AND 2100
|
||||
THEN year_built END AS year_built,
|
||||
round(COALESCE(build_record_area, area), 2) AS area_m2,
|
||||
purpose,
|
||||
ST_Y(ST_Centroid(ST_Transform(geom, 4326))) AS lat,
|
||||
ST_X(ST_Centroid(ST_Transform(geom, 4326))) AS lon
|
||||
FROM cad_buildings
|
||||
WHERE cad_num LIKE '66:41:%'
|
||||
AND cad_num IS NOT NULL
|
||||
AND (year_built IS NOT NULL OR floors IS NOT NULL)
|
||||
) TO STDOUT WITH CSV
|
||||
" | docker exec -i "$DST_PG" psql -U tradein -d tradein -v ON_ERROR_STOP=on -c "
|
||||
COPY cad_buildings_staging FROM STDIN WITH CSV
|
||||
"
|
||||
|
||||
STAGED=$(docker exec "$DST_PG" psql -U tradein -d tradein -tAc \
|
||||
"SELECT count(*) FROM cad_buildings_staging")
|
||||
echo "[$(date -u +%H:%M:%S)] staged: $STAGED зданий"
|
||||
|
||||
# 3. Upsert в cad_buildings (trigger проставит geom из lat/lon).
|
||||
docker exec "$DST_PG" psql -U tradein -d tradein -v ON_ERROR_STOP=on -c "
|
||||
INSERT INTO cad_buildings
|
||||
(cad_num, address, floors, year_built, area_m2, purpose, lat, lon)
|
||||
SELECT cad_num, address, floors, year_built, area_m2, purpose, lat, lon
|
||||
FROM cad_buildings_staging
|
||||
ON CONFLICT (cad_num) DO UPDATE SET
|
||||
address = EXCLUDED.address,
|
||||
floors = EXCLUDED.floors,
|
||||
year_built = EXCLUDED.year_built,
|
||||
area_m2 = EXCLUDED.area_m2,
|
||||
purpose = EXCLUDED.purpose,
|
||||
lat = EXCLUDED.lat,
|
||||
lon = EXCLUDED.lon,
|
||||
imported_at = NOW();
|
||||
|
||||
DROP TABLE cad_buildings_staging;
|
||||
"
|
||||
|
||||
TOTAL=$(docker exec "$DST_PG" psql -U tradein -d tradein -tAc \
|
||||
"SELECT count(*) FROM cad_buildings")
|
||||
echo "[$(date -u +%H:%M:%S)] cad_buildings всего: $TOTAL"
|
||||
|
|
@ -29,7 +29,9 @@ services:
|
|||
timeout: 5s
|
||||
retries: 5
|
||||
restart: unless-stopped
|
||||
networks: [tradein-net]
|
||||
networks:
|
||||
- tradein-net
|
||||
- gendesign_shared
|
||||
|
||||
backend:
|
||||
image: ghcr.io/lekss361/gendesign-tradein-backend:${IMAGE_TAG:-latest}
|
||||
|
|
@ -45,7 +47,7 @@ services:
|
|||
ADMIN_TOKEN: "${ADMIN_TOKEN:-}"
|
||||
# GlitchTip DSN — мониторинг ошибок (#396). Пусто = выключено.
|
||||
GLITCHTIP_DSN: "${GLITCHTIP_DSN:-}"
|
||||
MAIN_BACKEND_URL: "${MAIN_BACKEND_URL:-http://gendesign-backend:8000}"
|
||||
GENDESIGN_FDW_PASSWORD: "${GENDESIGN_FDW_PASSWORD:-}"
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue