feat(tradein): postgres_fdw live read of gendesign.cad_buildings (replaces snapshot) (#493)
Some checks failed
Deploy Trade-In / changes (push) Successful in 5s
Deploy / changes (push) Successful in 5s
Deploy Trade-In / build-frontend (push) Successful in 26s
Deploy Trade-In / build-backend (push) Successful in 47s
Deploy / build-frontend (push) Successful in 29s
Deploy / build-backend (push) Successful in 1m24s
Deploy Trade-In / deploy (push) Successful in 40s
Deploy / build-worker (push) Successful in 2m57s
Deploy / deploy (push) Failing after 37s

Replaces tradein.cad_buildings snapshot with live postgres_fdw foreign table reading gendesign.v_tradein_cad_buildings. Fixes /trade-in/api/v1/geocode/reverse 500 (Nominatim ban) and address_not_geocoded for cadastre addresses (e.g. Хохрякова 81).

Security (deep-review fixes):
- 100_tradein_fdw_role.sql: passwordless CREATE ROLE; password set by deploy.yml ALTER ROLE bootstrap reading GENDESIGN_FDW_PASSWORD from backend/.env.runtime (via psql :'pw' var → format %L — injection-safe).
- core/fdw.py: regex whitelist [A-Za-z0-9_-]{32,256} on password, ValueError without echoing value, try/rollback on commit.
- 060_postgres_fdw_extension.sql: connect_timeout='3' on FOREIGN SERVER + ALTER ADD/SET fallback.
- geocoder.py: _cadastral_forward_sync / _cadastral_reverse_sync wrapped in asyncio.to_thread.
- 100_*.sql: REVOKE ALL ON ALL TABLES/SEQUENCES/FUNCTIONS IN SCHEMA public; only GRANT SELECT on v_tradein_cad_buildings.
- pg_user_mappings query handles PUBLIC mapping (usename IS NULL).

Tests: 3 SQL-injection guards on ensure_fdw_user_mapping + rewritten cadastral suite.
This commit is contained in:
lekss361 2026-05-24 08:57:30 +00:00
parent 73bdff41a7
commit 94cf199276
18 changed files with 715 additions and 374 deletions

View file

@ -251,6 +251,27 @@ jobs:
done done
echo "All migrations applied." echo "All migrations applied."
# Set tradein_fdw_reader password from env (post-migration bootstrap).
# SQL migration creates role passwordless; password lives only in
# /opt/gendesign/backend/.env.runtime (sourced below via set -a / source).
# Idempotent: ALTER is no-op if password unchanged on subsequent deploys.
set -a; source backend/.env.runtime; set +a
if [ -n "${GENDESIGN_FDW_PASSWORD:-}" ]; then
echo "→ Applying tradein_fdw_reader password from env"
docker compose -p gendesign -f docker-compose.prod.yml exec -T postgres \
psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -v ON_ERROR_STOP=on \
-v "pw=$GENDESIGN_FDW_PASSWORD" \
-c "DO \$\$ BEGIN \
IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'tradein_fdw_reader') THEN \
EXECUTE format('ALTER ROLE tradein_fdw_reader WITH PASSWORD %L', :'pw'); \
ELSE \
RAISE NOTICE 'tradein_fdw_reader role missing; migration not yet applied'; \
END IF; \
END \$\$;"
else
echo "⚠️ GENDESIGN_FDW_PASSWORD not set in backend/.env.runtime — skipping ALTER ROLE for tradein_fdw_reader"
fi
# Build local-only sidecar images (glitchtip-auth-forwarder). # Build local-only sidecar images (glitchtip-auth-forwarder).
# Эти services не в GHCR — сборка происходит на VPS на каждом deploy. # Эти services не в GHCR — сборка происходит на VPS на каждом deploy.
# Cache-friendly: первый build ~30s, последующие 1-3s если файлы не менялись. # Cache-friendly: первый build ~30s, последующие 1-3s если файлы не менялись.

View file

@ -1,101 +0,0 @@
"""Cadastral lookup endpoints — reverse geocode via cad_buildings.
Used by trade-in backend (and potentially other internal services) to resolve
координаты адрес жилого дома без зависимости от внешних геокодеров.
"""
from __future__ import annotations
import logging
from typing import Annotated
from fastapi import APIRouter, Depends, HTTPException, Query
from pydantic import BaseModel
from sqlalchemy import text
from sqlalchemy.orm import Session
from app.core.db import get_db
logger = logging.getLogger(__name__)
router = APIRouter()
class CadastralReverseResult(BaseModel):
address: str
lat: float
lon: float
distance_m: float
cad_num: str | None = None
@router.get("/reverse", response_model=CadastralReverseResult)
async def reverse(
lat: Annotated[float, Query(ge=-90, le=90)],
lon: Annotated[float, Query(ge=-180, le=180)],
radius_m: Annotated[int, Query(ge=10, le=2000)] = 200,
db: Annotated[Session, Depends(get_db)] = None, # type: ignore[assignment]
) -> CadastralReverseResult:
"""Координаты → ближайшее здание из cad_buildings.
Filters out гаражи/СНТ/садовод/товарищества для trade-in нужны жилые дома.
Returns 404 если в радиусе нет подходящего здания.
"""
row = db.execute(
text(
"""
SELECT
readable_address,
cad_num,
ST_Y(ST_Centroid(geom)) AS lat,
ST_X(ST_Centroid(geom)) AS lon,
ST_Distance(
geom::geography,
ST_SetSRID(
ST_MakePoint(
CAST(:lon AS double precision),
CAST(:lat AS double precision)
),
4326
)::geography
) AS distance_m
FROM cad_buildings
WHERE ST_DWithin(
geom::geography,
ST_SetSRID(
ST_MakePoint(
CAST(:lon AS double precision),
CAST(:lat AS double precision)
),
4326
)::geography,
CAST(:radius AS double precision)
)
AND readable_address !~* '(гараж|снт|садовод|товарищ|уч\\.)'
ORDER BY ST_Distance(
geom::geography,
ST_SetSRID(
ST_MakePoint(
CAST(:lon AS double precision),
CAST(:lat AS double precision)
),
4326
)::geography
)
LIMIT 1
"""
),
{"lat": lat, "lon": lon, "radius": float(radius_m)},
).first()
if row is None:
raise HTTPException(status_code=404, detail="no cadastral building found within radius")
return CadastralReverseResult(
address=row.readable_address,
lat=float(row.lat),
lon=float(row.lon),
distance_m=round(float(row.distance_m), 1),
cad_num=row.cad_num,
)

View file

@ -21,7 +21,6 @@ from app.api.v1 import (
admin_scrape, admin_scrape,
admin_weight_profiles, admin_weight_profiles,
analytics, analytics,
cadastral,
concepts, concepts,
custom_pois, custom_pois,
landing, landing,
@ -79,7 +78,6 @@ app.add_middleware(
allow_headers=["*"], allow_headers=["*"],
) )
app.include_router(cadastral.router, prefix="/api/v1/cadastral", tags=["cadastral"])
app.include_router(concepts.router, prefix="/api/v1/concepts", tags=["concepts"]) app.include_router(concepts.router, prefix="/api/v1/concepts", tags=["concepts"])
app.include_router(parcels.router, prefix="/api/v1/parcels", tags=["parcels"]) app.include_router(parcels.router, prefix="/api/v1/parcels", tags=["parcels"])
app.include_router(analytics.router, prefix="/api/v1/analytics", tags=["analytics"]) app.include_router(analytics.router, prefix="/api/v1/analytics", tags=["analytics"])

View file

@ -0,0 +1,45 @@
-- tradein_fdw_reader role + v_tradein_cad_buildings view
-- Used by tradein-postgres via postgres_fdw to read live cad_buildings.
-- Password is set by .forgejo/workflows/deploy.yml ALTER ROLE bootstrap step
-- reading GENDESIGN_FDW_PASSWORD from backend/.env.runtime — never embed password in SQL.
BEGIN;
DO $$
BEGIN
IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'tradein_fdw_reader') THEN
CREATE ROLE tradein_fdw_reader LOGIN;
END IF;
END$$;
COMMENT ON ROLE tradein_fdw_reader IS
'FDW reader from tradein-postgres. Password set by .forgejo/workflows/deploy.yml from env GENDESIGN_FDW_PASSWORD post-migration step. Never embed password in SQL migrations.';
-- Defense-in-depth: explicit REVOKE perimeter (any existing PUBLIC grants ignored).
REVOKE ALL ON ALL TABLES IN SCHEMA public FROM tradein_fdw_reader;
REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM tradein_fdw_reader;
REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM tradein_fdw_reader;
GRANT CONNECT ON DATABASE gendesign TO tradein_fdw_reader;
GRANT USAGE ON SCHEMA public TO tradein_fdw_reader;
CREATE OR REPLACE VIEW v_tradein_cad_buildings AS
SELECT
cad_num,
NULLIF(trim(readable_address), '') AS readable_address,
NULLIF(floors, 0) AS floors,
CASE WHEN year_built BETWEEN 1800 AND 2100 THEN year_built END AS year_built,
round(COALESCE(build_record_area, area), 2)::numeric AS area_m2,
purpose,
ST_Y(ST_Centroid(geom))::double precision AS lat,
ST_X(ST_Centroid(geom))::double precision AS lon
FROM cad_buildings
WHERE cad_num LIKE '66:41:%'
AND geom IS NOT NULL
AND readable_address IS NOT NULL;
GRANT SELECT ON v_tradein_cad_buildings TO tradein_fdw_reader;
COMMENT ON VIEW v_tradein_cad_buildings IS
'FDW source for tradein-mvp (postgres_fdw). Flat 8-col EKB-only slice of cad_buildings. tradein.gendesign_cad_buildings maps here.';
COMMIT;

View file

@ -33,6 +33,11 @@ services:
interval: 5s interval: 5s
timeout: 5s timeout: 5s
retries: 10 retries: 10
networks:
default: {}
shared:
aliases:
- gendesign-postgres
redis: redis:
image: redis:7-alpine image: redis:7-alpine
@ -67,11 +72,6 @@ services:
timeout: 5s timeout: 5s
retries: 5 retries: 5
start_period: 20s start_period: 20s
networks:
default: {}
shared:
aliases:
- gendesign-backend
frontend: frontend:
image: ghcr.io/lekss361/gendesign-frontend:${IMAGE_TAG:-latest} image: ghcr.io/lekss361/gendesign-frontend:${IMAGE_TAG:-latest}

View file

@ -50,6 +50,7 @@ class SuggestResponse(BaseModel):
async def suggest_addresses( async def suggest_addresses(
q: Annotated[str, Query(min_length=2, max_length=200, description="Запрос для автокомплита")], q: Annotated[str, Query(min_length=2, max_length=200, description="Запрос для автокомплита")],
limit: Annotated[int, Query(ge=1, le=15)] = 8, limit: Annotated[int, Query(ge=1, le=15)] = 8,
db: Annotated[Session, Depends(get_db)] = None, # type: ignore[assignment]
) -> SuggestResponse: ) -> SuggestResponse:
"""Автокомплит адресов в пределах ЕКБ. """Автокомплит адресов в пределах ЕКБ.
@ -60,7 +61,7 @@ async def suggest_addresses(
/api/v1/geocode/suggest?q=Малышева /api/v1/geocode/suggest?q=Малышева
/api/v1/geocode/suggest?q=Цвиллинга # → пусто, такой улицы в ЕКБ нет /api/v1/geocode/suggest?q=Цвиллинга # → пусто, такой улицы в ЕКБ нет
""" """
items = await suggest(q, limit=limit) items = await suggest(q, db=db, limit=limit)
return SuggestResponse(items=[ return SuggestResponse(items=[
SuggestItem( SuggestItem(
label=s.label, label=s.label,
@ -74,12 +75,13 @@ async def suggest_addresses(
async def reverse( async def reverse(
lat: Annotated[float, Query(ge=-90, le=90)], lat: Annotated[float, Query(ge=-90, le=90)],
lon: Annotated[float, Query(ge=-180, le=180)], lon: Annotated[float, Query(ge=-180, le=180)],
db: Annotated[Session, Depends(get_db)] = None, # type: ignore[assignment]
) -> dict[str, object]: ) -> dict[str, object]:
"""Обратный геокодинг — координаты с карты → адрес (map-picker). """Обратный геокодинг — координаты с карты → адрес (map-picker).
Пример: /api/v1/geocode/reverse?lat=56.8389&lon=60.6057 Пример: /api/v1/geocode/reverse?lat=56.8389&lon=60.6057
""" """
address = await reverse_geocode(lat, lon) address = await reverse_geocode(lat, lon, db=db)
if address is None: if address is None:
raise HTTPException(status_code=404, detail="address not found for coordinates") raise HTTPException(status_code=404, detail="address not found for coordinates")
return {"address": address, "lat": lat, "lon": lon} return {"address": address, "lat": lat, "lon": lon}

View file

@ -584,7 +584,7 @@ def cache_stats(db: Annotated[Session, Depends(get_db)]) -> dict[str, object]:
(SELECT count(*) FROM listings WHERE is_active) AS listings_active, (SELECT count(*) FROM listings WHERE is_active) AS listings_active,
(SELECT max(scraped_at) FROM listings) AS listings_last_scraped, (SELECT max(scraped_at) FROM listings) AS listings_last_scraped,
(SELECT count(*) FROM deals) AS deals, (SELECT count(*) FROM deals) AS deals,
(SELECT count(*) FROM cad_buildings) AS cad_buildings, (SELECT count(*) FROM gendesign_cad_buildings) AS cad_buildings,
(SELECT count(*) FROM house_metadata) AS house_metadata, (SELECT count(*) FROM house_metadata) AS house_metadata,
(SELECT count(*) FROM trade_in_estimates) AS estimates_total (SELECT count(*) FROM trade_in_estimates) AS estimates_total
""" """

View file

@ -32,10 +32,10 @@ class Settings(BaseSettings):
# Redis URL для hot-cache (Phase 3.2). Задаётся через env REDIS_URL. # Redis URL для hot-cache (Phase 3.2). Задаётся через env REDIS_URL.
redis_url: str = "redis://localhost:6379/0" redis_url: str = "redis://localhost:6379/0"
# Main GenDesign backend URL — для cadastral reverse geocode (cad_buildings). # Password for tradein_fdw_reader role — used by backend startup to create/refresh
# В prod: http://gendesign-backend:8000 (network alias в docker-compose.prod.yml). # USER MAPPING for postgres_fdw → gendesign DB (gendesign_remote server).
# Пусто = cadastral lookup пропускается, используется только Nominatim. # Пусто = USER MAPPING не создаётся, gendesign_cad_buildings не работает (dev).
main_backend_url: str | None = None gendesign_fdw_password: str | None = None
settings = Settings() settings = Settings()

View file

@ -0,0 +1,81 @@
"""postgres_fdw USER MAPPING management for gendesign_remote server.
Password lives in env GENDESIGN_FDW_PASSWORD never in SQL migration or git.
This helper:
- validates password format (alphanumeric/hex, length >= 32) to prevent SQL
injection via embedded quotes (the postgres CREATE USER MAPPING syntax does
not accept bind parameters password must be inlined into the DDL string),
- applies idempotent CREATE or ALTER mapping on every backend startup so
password rotation through .env.runtime is picked up after restart.
"""
from __future__ import annotations
import logging
import re
from sqlalchemy import text
from sqlalchemy.orm import Session
from app.core.config import settings
logger = logging.getLogger(__name__)
# Strict whitelist — only hex/alphanumeric and a few separators. Refuses ANY
# quote, backslash, unicode whitespace, control char. Min length 32 forces
# rotation away from weak defaults.
_PASSWORD_RE = re.compile(r"^[A-Za-z0-9_\-]{32,256}$")
def ensure_fdw_user_mapping(db: Session) -> None:
"""Idempotent CREATE OR ALTER USER MAPPING for tradein → gendesign_remote.
Skips silently if password not configured (dev / first deploy / forgotten env).
Raises ValueError if password fails format whitelist fail-fast on misconfig.
"""
password = settings.gendesign_fdw_password
if not password or not str(password).strip():
logger.warning(
"GENDESIGN_FDW_PASSWORD not set — skipping FDW user mapping "
"(gendesign_cad_buildings queries will fail; cadastral lookups will "
"fall back to Yandex/Nominatim)"
)
return
if not _PASSWORD_RE.fullmatch(password):
# Do NOT echo the password into logs / exception even partially.
raise ValueError(
"GENDESIGN_FDW_PASSWORD failed format whitelist "
"(allowed: 32-256 chars, [A-Za-z0-9_-]). Refusing to apply USER MAPPING "
"— password rotation must use the same format."
)
# NB: even after whitelist validation we still cannot use SQLAlchemy bind
# parameters here — postgres CREATE/ALTER USER MAPPING is DDL and treats
# OPTIONS values as literal text. Whitelist guarantees the value is
# injection-safe; we still wrap in single quotes for the DDL syntax.
exists = db.execute(
text(
"SELECT 1 FROM pg_user_mappings "
"WHERE srvname = 'gendesign_remote' "
"AND (usename = current_user OR usename IS NULL)"
)
).first()
if exists is None:
db.execute(text(
f"CREATE USER MAPPING FOR CURRENT_USER SERVER gendesign_remote "
f"OPTIONS (user 'tradein_fdw_reader', password '{password}')"
))
logger.info("created FDW user mapping for gendesign_remote")
else:
db.execute(text(
f"ALTER USER MAPPING FOR CURRENT_USER SERVER gendesign_remote "
f"OPTIONS (SET password '{password}')"
))
logger.info("refreshed FDW user mapping password for gendesign_remote")
try:
db.commit()
except Exception:
db.rollback()
raise

View file

@ -17,6 +17,8 @@ from fastapi.middleware.cors import CORSMiddleware
from app.api.v1 import admin, brand, geocode, search, trade_in from app.api.v1 import admin, brand, geocode, search, trade_in
from app.core.config import settings from app.core.config import settings
from app.core.db import SessionLocal
from app.core.fdw import ensure_fdw_user_mapping
from app.core.ratelimit import RateLimitMiddleware from app.core.ratelimit import RateLimitMiddleware
from app.services.scheduler import scheduler_loop from app.services.scheduler import scheduler_loop
@ -41,6 +43,16 @@ if settings.glitchtip_dsn:
@asynccontextmanager @asynccontextmanager
async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]: async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
# FDW bootstrap: create/refresh USER MAPPING for gendesign_remote postgres_fdw server.
# Best-effort: failure does not abort startup, just logs.
try:
with SessionLocal() as db:
ensure_fdw_user_mapping(db)
except Exception:
logger.exception(
"FDW user mapping bootstrap failed — cadastral queries may fail"
)
# Startup: launch scheduler background task # Startup: launch scheduler background task
scheduler_task = asyncio.create_task(scheduler_loop()) scheduler_task = asyncio.create_task(scheduler_loop())
logger.info("FastAPI lifespan: scheduler task spawned") logger.info("FastAPI lifespan: scheduler task spawned")

View file

@ -158,7 +158,7 @@ async def _nominatim_query(client: httpx.AsyncClient, address: str) -> dict | No
lon_f = float(item["lon"]) lon_f = float(item["lon"])
if 60.40 <= lon_f <= 60.85 and 56.65 <= lat_f <= 56.95: if 60.40 <= lon_f <= 60.85 and 56.65 <= lat_f <= 56.95:
return item return item
except Exception: # noqa: BLE001 except Exception:
continue continue
return None return None
@ -245,7 +245,7 @@ async def _yandex_lookup(address: str, api_key: str) -> GeocodeResult | None:
if 60.40 <= lon_f <= 60.85 and 56.65 <= lat_f <= 56.95: if 60.40 <= lon_f <= 60.85 and 56.65 <= lat_f <= 56.95:
best = obj best = obj
break break
except Exception: # noqa: BLE001 except Exception:
continue continue
if best is None: if best is None:
@ -315,7 +315,7 @@ def _parse_yandex_members(members: list[dict]) -> list[GeocodeSuggestion]:
label=label, full_address=full, label=label, full_address=full,
lat=float(lat_str), lon=float(lon_str), kind=kind, lat=float(lat_str), lon=float(lon_str), kind=kind,
)) ))
except Exception: # noqa: BLE001 except Exception:
continue continue
return out return out
@ -445,36 +445,128 @@ async def _nominatim_suggest(query: str, limit: int = 8) -> list[GeocodeSuggesti
return out return out
async def suggest(query: str, limit: int = 8) -> list[GeocodeSuggestion]: def _cadastral_forward_sync(
"""Автокомплит адресов в ЕКБ. Yandex → Nominatim → []. db: Session, query: str, limit: int = 5
) -> list[GeocodeSuggestion]:
"""Forward search via gendesign_cad_buildings FDW.
Uses ILIKE prefilter (FDW-pushable) + pg_trgm similarity ranking locally.
Returns up to `limit` suggestions, or [] on error / no results.
"""
try:
rows = db.execute(
text("""
SELECT cad_num, readable_address, lat, lon,
similarity(readable_address, CAST(:q AS text)) AS sim
FROM gendesign_cad_buildings
WHERE readable_address ILIKE CAST('%' || :q || '%' AS text)
ORDER BY sim DESC, length(readable_address) ASC
LIMIT CAST(:lim AS integer)
"""),
{"q": query, "lim": limit * 4},
).mappings().all()
except Exception:
logger.warning("cadastral forward sync failed for query=%r", query, exc_info=True)
return []
out: list[GeocodeSuggestion] = []
for r in rows[:limit]:
out.append(GeocodeSuggestion(
label=str(r["readable_address"]),
full_address=str(r["readable_address"]),
lat=float(r["lat"]),
lon=float(r["lon"]),
kind="house",
))
return out
def _cadastral_reverse_sync(
db: Session, lat: float, lon: float, radius_m: int = 200
) -> str | None:
"""Reverse lookup via gendesign_cad_buildings FDW.
bbox prefilter (±0.0025° lat, ±0.005° lon 280m) is FDW-pushable.
Distance calculated locally on small result set. Filters гаражи/СНТ.
Returns None on no result or error.
"""
try:
row = db.execute(
text("""
WITH candidates AS (
SELECT cad_num, readable_address, lat, lon,
111320.0 * sqrt(
pow(CAST(:lat AS double precision) - lat, 2) +
pow(
cos(radians(CAST(:lat AS double precision)))
* (CAST(:lon AS double precision) - lon), 2
)
) AS dist_m
FROM gendesign_cad_buildings
WHERE lat BETWEEN CAST(:lat AS double precision) - 0.0025
AND CAST(:lat AS double precision) + 0.0025
AND lon BETWEEN CAST(:lon AS double precision) - 0.005
AND CAST(:lon AS double precision) + 0.005
AND readable_address !~* '(гараж|снт|садовод|товарищ|уч\\.)'
)
SELECT readable_address, dist_m
FROM candidates
WHERE dist_m < CAST(:radius AS double precision)
ORDER BY dist_m ASC
LIMIT 1
"""),
{"lat": lat, "lon": lon, "radius": float(radius_m)},
).first()
except Exception:
logger.warning(
"cadastral reverse sync failed for (%.5f, %.5f)", lat, lon, exc_info=True
)
return None
if row is None:
return None
return str(row.readable_address)
async def suggest(
query: str, db: Session | None = None, limit: int = 8
) -> list[GeocodeSuggestion]:
"""Автокомплит адресов в ЕКБ. Cadastral FDW → Yandex → Nominatim → [].
db: если передан cadastral lookup через gendesign_cad_buildings (первый tier).
Без кэша (дешёво, провайдеры толерируют автокомплит-запросы). Без кэша (дешёво, провайдеры толерируют автокомплит-запросы).
""" """
if not query or len(query.strip()) < 2: if not query or len(query.strip()) < 2:
return [] return []
# Tier 0: cadastral FDW (если db доступна) — самый быстрый, без внешних запросов
if db is not None:
cad_results = await asyncio.to_thread(_cadastral_forward_sync, db, query.strip(), limit)
if cad_results:
return cad_results
if settings.yandex_geocoder_key: if settings.yandex_geocoder_key:
try: try:
results = await _yandex_suggest(query, settings.yandex_geocoder_key, limit) results = await _yandex_suggest(query, settings.yandex_geocoder_key, limit)
if results: if results:
return results return results
except Exception: # noqa: BLE001 except Exception:
logger.exception("yandex suggest failed, falling back to nominatim") logger.exception("yandex suggest failed, falling back to nominatim")
try: try:
return await _nominatim_suggest(query, limit) return await _nominatim_suggest(query, limit)
except Exception: # noqa: BLE001 except Exception:
logger.exception("nominatim suggest failed") logger.exception("nominatim suggest failed")
return [] return []
# ── Public API ─────────────────────────────────────────────────────────────── # ── Public API ───────────────────────────────────────────────────────────────
async def geocode(address: str, db: Session) -> GeocodeResult | None: async def geocode(address: str, db: Session) -> GeocodeResult | None:
"""Геокодинг с кэшем. Yandex → Nominatim → None. """Геокодинг с кэшем. Cadastral FDW → Yandex → Nominatim → None.
Args: Args:
address: пользовательский ввод (может быть грязным нормализуем). address: пользовательский ввод (может быть грязным нормализуем).
db: сессия Postgres для cache lookup/write. db: сессия Postgres для cache lookup/write и cadastral FDW lookup.
Returns: Returns:
GeocodeResult или None если ни один провайдер не отвечает. GeocodeResult или None если ни один провайдер не отвечает.
@ -490,12 +582,33 @@ async def geocode(address: str, db: Session) -> GeocodeResult | None:
logger.info("geocode cache hit: %s", addr_norm) logger.info("geocode cache hit: %s", addr_norm)
return cached return cached
# 2. Yandex (если есть key) с typo-fallback # 2. Cadastral FDW (прямой запрос к gendesign_cad_buildings — без внешнего API)
cad_suggestions = await asyncio.to_thread(_cadastral_forward_sync, db, address.strip(), limit=1)
if cad_suggestions:
s = cad_suggestions[0]
result = GeocodeResult(
lat=s.lat,
lon=s.lon,
full_address=s.full_address,
provider="nominatim", # treat as "local" — same confidence as nominatim
confidence="exact",
)
_cache_put(db, addr_norm, result)
logger.info(
"geocode cadastral fdw: %s → (%.5f, %.5f)", addr_norm, result.lat, result.lon
)
return result
# 3. Yandex (если есть key) с typo-fallback
if settings.yandex_geocoder_key: if settings.yandex_geocoder_key:
try: try:
result = await _yandex_lookup(address, settings.yandex_geocoder_key) result = await _yandex_lookup(address, settings.yandex_geocoder_key)
# Если результат вне ЕКБ — пробуем typo-варианты # Если результат вне ЕКБ — пробуем typo-варианты
in_ekb = result is not None and 60.40 <= result.lon <= 60.85 and 56.65 <= result.lat <= 56.95 in_ekb = (
result is not None
and 60.40 <= result.lon <= 60.85
and 56.65 <= result.lat <= 56.95
)
if result is not None and in_ekb: if result is not None and in_ekb:
_cache_put(db, addr_norm, result) _cache_put(db, addr_norm, result)
logger.info("geocode yandex: %s → (%.5f, %.5f)", addr_norm, result.lat, result.lon) logger.info("geocode yandex: %s → (%.5f, %.5f)", addr_norm, result.lat, result.lon)
@ -504,7 +617,7 @@ async def geocode(address: str, db: Session) -> GeocodeResult | None:
for variant in _typo_variants(address, limit=4): for variant in _typo_variants(address, limit=4):
try: try:
result = await _yandex_lookup(variant, settings.yandex_geocoder_key) result = await _yandex_lookup(variant, settings.yandex_geocoder_key)
except Exception: # noqa: BLE001 except Exception:
continue continue
if result is None: if result is None:
continue continue
@ -515,10 +628,10 @@ async def geocode(address: str, db: Session) -> GeocodeResult | None:
addr_norm, variant, result.lat, result.lon, addr_norm, variant, result.lat, result.lon,
) )
return result return result
except Exception: # noqa: BLE001 except Exception:
logger.exception("yandex geocoder failed, falling back to nominatim") logger.exception("yandex geocoder failed, falling back to nominatim")
# 3. Nominatim fallback # 4. Nominatim fallback
try: try:
result = await _nominatim_lookup(address) result = await _nominatim_lookup(address)
if result is not None: if result is not None:
@ -529,7 +642,7 @@ async def geocode(address: str, db: Session) -> GeocodeResult | None:
# Nominatim rate-limit policy: 1 req/sec — спим после успешного запроса # Nominatim rate-limit policy: 1 req/sec — спим после успешного запроса
await asyncio.sleep(1.0) await asyncio.sleep(1.0)
return result return result
except Exception: # noqa: BLE001 except Exception:
logger.exception("nominatim geocoder failed") logger.exception("nominatim geocoder failed")
return None return None
@ -569,29 +682,6 @@ def _format_reverse_address(addr: dict) -> str | None:
return ", ".join(parts) return ", ".join(parts)
async def _cadastral_reverse(lat: float, lon: float) -> str | None:
"""Reverse via main backend cad_buildings — fast, no external API.
Returns None on any error (network, 404, timeout, missing config).
"""
if not settings.main_backend_url:
return None
url = settings.main_backend_url.rstrip("/") + "/api/v1/cadastral/reverse"
try:
async with httpx.AsyncClient(timeout=3.0) as client:
response = await client.get(url, params={"lat": str(lat), "lon": str(lon)})
if response.status_code == 404:
return None
response.raise_for_status()
data = response.json()
addr = data.get("address")
if isinstance(addr, str) and addr.strip():
return addr
except Exception:
logger.exception("cadastral reverse failed for (%.5f, %.5f)", lat, lon)
return None
@retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=1, max=8)) @retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=1, max=8))
async def _nominatim_reverse(lat: float, lon: float) -> str | None: async def _nominatim_reverse(lat: float, lon: float) -> str | None:
"""Nominatim /reverse — extracted for fallback usage. @retry handles transient errors.""" """Nominatim /reverse — extracted for fallback usage. @retry handles transient errors."""
@ -623,17 +713,20 @@ async def _nominatim_reverse(lat: float, lon: float) -> str | None:
return data.get("display_name") return data.get("display_name")
async def reverse_geocode(lat: float, lon: float) -> str | None: async def reverse_geocode(lat: float, lon: float, db: Session | None = None) -> str | None:
"""Cadastral (cad_buildings via main backend) → Nominatim fallback. """Cadastral FDW → Nominatim fallback.
Возвращает None если ни один источник не дал адрес. Endpoint Возвращает None если ни один источник не дал адрес. Endpoint
api/v1/geocode/reverse сам выкинет 404. НЕ даёт выйти HTTPStatusError api/v1/geocode/reverse сам выкинет 404. НЕ даёт выйти HTTPStatusError
наверх раньше Nominatim 403 RetryError FastAPI 500. наверх раньше Nominatim 403 RetryError FastAPI 500.
db: если передан cadastral lookup через gendesign_cad_buildings FDW (первый tier).
""" """
# 1. Cadastral primary (fast, no external API, returns жилой дом not POI) # 1. Cadastral FDW primary (без внешнего API, возвращает жилой дом not POI)
cadastral = await _cadastral_reverse(lat, lon) if db is not None:
if cadastral: cadastral = await asyncio.to_thread(_cadastral_reverse_sync, db, lat, lon)
return cadastral if cadastral:
return cadastral
# 2. Nominatim fallback (wrap to prevent 500 on ban/rate-limit) # 2. Nominatim fallback (wrap to prevent 500 on ban/rate-limit)
try: try:

View file

@ -138,34 +138,56 @@ def _cache_put(db: Session, meta: HouseMetadata, raw: dict) -> None:
db.commit() db.commit()
# ── Источник #393: кадастр зданий (cad_buildings, импорт из gendesign-БД) ──── # ── Источник #393: кадастр зданий (gendesign_cad_buildings FDW) ──────────────
def _cad_buildings_get(db: Session, lat: float, lon: float) -> HouseMetadata | None: def _cad_buildings_get(db: Session, lat: float, lon: float) -> HouseMetadata | None:
"""Ближайшее здание из cad_buildings (данные Росреестра) по координатам. """Ближайшее здание из gendesign_cad_buildings FDW (данные Росреестра) по координатам.
Приоритетный источник house_metadata год / этажность полнее, чем OSM Приоритетный источник house_metadata год / этажность полнее, чем OSM
(~34к домов ЕКБ с годом постройки). Таблицу наполняет import-cadastre.sh. (~36к домов ЕКБ с годом постройки). Использует bbox+haversine вместо ST_DWithin
FDW-таблица не имеет PostGIS-колонки geom (только lat/lon скалярные).
""" """
row = db.execute( try:
text( row = db.execute(
""" text(
SELECT year_built, floors """
FROM cad_buildings WITH candidates AS (
WHERE geom IS NOT NULL SELECT year_built, floors, lat, lon,
AND (year_built IS NOT NULL OR floors IS NOT NULL) 111320.0 * sqrt(
AND ST_DWithin(geom::geography, ST_MakePoint(:lon, :lat)::geography, :radius) pow(CAST(:lat AS double precision) - lat, 2) +
ORDER BY ST_Distance(geom::geography, ST_MakePoint(:lon, :lat)::geography) pow(
LIMIT 1 cos(radians(CAST(:lat AS double precision)))
""" * (CAST(:lon AS double precision) - lon), 2
), )
{"lat": lat, "lon": lon, "radius": _CAD_RADIUS_M}, ) AS dist_m
).fetchone() FROM gendesign_cad_buildings
WHERE (year_built IS NOT NULL OR floors IS NOT NULL)
AND lat BETWEEN CAST(:lat AS double precision) - 0.0010
AND CAST(:lat AS double precision) + 0.0010
AND lon BETWEEN CAST(:lon AS double precision) - 0.0015
AND CAST(:lon AS double precision) + 0.0015
)
SELECT year_built, floors
FROM candidates
WHERE dist_m < CAST(:radius AS double precision)
ORDER BY dist_m ASC
LIMIT 1
"""
),
{"lat": lat, "lon": lon, "radius": float(_CAD_RADIUS_M)},
).fetchone()
except Exception:
logger.warning(
"gendesign_cad_buildings lookup failed at (%.5f, %.5f)", lat, lon, exc_info=True
)
return None
if row is None: if row is None:
return None return None
return HouseMetadata( return HouseMetadata(
lat=lat, lon=lon, lat=lat, lon=lon,
year_built=row.year_built, year_built=row.year_built,
total_floors=row.floors, total_floors=row.floors,
house_type=None, # cad_buildings.purpose не маппится в panel/brick house_type=None, # purpose не маппится в panel/brick
total_units=None, total_units=None,
source="cadastre", source="cadastre",
) )
@ -245,12 +267,12 @@ async def get_house_metadata(lat: float, lon: float, db: Session) -> HouseMetada
lat, lon, meta.year_built, meta.total_floors, meta.house_type, lat, lon, meta.year_built, meta.total_floors, meta.house_type,
) )
return meta return meta
except Exception: # noqa: BLE001 except Exception:
logger.warning( logger.warning(
"house_metadata enrichment failed at (%.5f, %.5f)", lat, lon, exc_info=True "house_metadata enrichment failed at (%.5f, %.5f)", lat, lon, exc_info=True
) )
try: try:
db.rollback() db.rollback()
except Exception: # noqa: BLE001 except Exception:
pass pass
return None return None

View file

@ -1,34 +0,0 @@
-- Trade-In MVP — кадастр зданий ЕКБ (#393).
-- Срез cad_buildings из gendesign-БД (данные Росреестра по зданиям),
-- импортируется ETL-ом deploy/import-cadastre.sh.
-- Основной источник обогащения house_metadata (год / этажность) —
-- данные точнее и полнее, чем OSM Overpass.
BEGIN;
CREATE TABLE IF NOT EXISTS cad_buildings (
cad_num text PRIMARY KEY, -- кадастровый номер здания, 66:41:...
address text,
floors int,
year_built int,
area_m2 numeric(12, 2), -- площадь здания (не квартиры)
purpose text, -- Жилое / Нежилое / Многоквартирный дом
lat double precision,
lon double precision,
geom geometry(Point, 4326),
imported_at timestamptz NOT NULL DEFAULT NOW()
);
CREATE INDEX IF NOT EXISTS cad_buildings_geom_idx ON cad_buildings USING GIST (geom);
-- Авто-update geom из (lat, lon) — переиспользуем функцию из 002_core_tables.sql.
DROP TRIGGER IF EXISTS cad_buildings_set_geom_trg ON cad_buildings;
CREATE TRIGGER cad_buildings_set_geom_trg
BEFORE INSERT OR UPDATE OF lat, lon ON cad_buildings
FOR EACH ROW EXECUTE FUNCTION listings_set_geom();
COMMENT ON TABLE cad_buildings IS
'#393 — кадастр зданий ЕКБ (срез cad_buildings из gendesign-БД). '
'Приоритетный источник обогащения house_metadata: год постройки / этажность.';
COMMIT;

View file

@ -0,0 +1,43 @@
-- postgres_fdw extension for tradein → gendesign live reads.
-- Idempotent. Server config only (no USER MAPPING — that's set by backend startup with env password).
BEGIN;
CREATE EXTENSION IF NOT EXISTS postgres_fdw;
DO $$
BEGIN
IF NOT EXISTS (SELECT 1 FROM pg_foreign_server WHERE srvname = 'gendesign_remote') THEN
CREATE SERVER gendesign_remote
FOREIGN DATA WRAPPER postgres_fdw
OPTIONS (host 'gendesign-postgres', dbname 'gendesign', port '5432',
extensions 'postgis', connect_timeout '3');
ELSE
-- Update existing server: add connect_timeout if missing, or set if already present.
BEGIN
ALTER SERVER gendesign_remote OPTIONS (ADD connect_timeout '3');
EXCEPTION WHEN OTHERS THEN
ALTER SERVER gendesign_remote OPTIONS (SET connect_timeout '3');
END;
END IF;
END$$;
-- Drop existing foreign table if mistakenly created with wrong schema (idempotent re-create below)
DROP FOREIGN TABLE IF EXISTS gendesign_cad_buildings;
CREATE FOREIGN TABLE gendesign_cad_buildings (
cad_num text,
readable_address text,
year_built integer,
floors integer,
area_m2 numeric,
purpose text,
lat double precision,
lon double precision
)
SERVER gendesign_remote
OPTIONS (schema_name 'public', table_name 'v_tradein_cad_buildings');
COMMENT ON FOREIGN TABLE gendesign_cad_buildings IS
'Live view of gendesign.cad_buildings (EKB only). USER MAPPING managed by backend startup.';
COMMIT;

View file

@ -0,0 +1,8 @@
-- Drop legacy cad_buildings snapshot table (replaced by gendesign_cad_buildings foreign table).
-- Idempotent.
BEGIN;
DROP TRIGGER IF EXISTS cad_buildings_set_geom_trg ON cad_buildings;
DROP TABLE IF EXISTS cad_buildings CASCADE;
COMMIT;

View file

@ -1,4 +1,12 @@
"""Unit tests for cadastral reverse geocode and modified reverse_geocode.""" """Unit tests for cadastral FDW SQL-backed geocoder helpers and reverse_geocode.
Tests cover:
- _cadastral_forward_sync: returns suggestions / empty list / graceful error
- _cadastral_reverse_sync: returns address / None on no result / None on error
- reverse_geocode: cadastral primary, Nominatim fallback, both-fail None (500-fix)
- geocode: cadastral as tier 0
- suggest: cadastral as tier 0
"""
from __future__ import annotations from __future__ import annotations
import os import os
@ -7,144 +15,363 @@ from unittest.mock import AsyncMock, MagicMock, patch
# DATABASE_URL required by config before any app import. # DATABASE_URL required by config before any app import.
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test") os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
os.environ.setdefault("MAIN_BACKEND_URL", "http://gendesign-backend:8000")
# WeasyPrint stub — not installed in CI without GTK. # WeasyPrint stub — not installed in CI without GTK.
_wp_mock = MagicMock() _wp_mock = MagicMock()
sys.modules.setdefault("weasyprint", _wp_mock) sys.modules.setdefault("weasyprint", _wp_mock)
import pytest
from app.services.geocoder import _cadastral_reverse, reverse_geocode from app.services.geocoder import ( # noqa: E402
GeocodeSuggestion,
_cadastral_forward_sync,
_cadastral_reverse_sync,
geocode,
reverse_geocode,
suggest,
)
# ── helpers ───────────────────────────────────────────────────────────────────
def _make_db_row(**kwargs: object) -> MagicMock:
"""Mock row object with dict-like access (MappingResult style)."""
row = MagicMock()
row.__getitem__ = lambda self, k: kwargs[k]
for k, v in kwargs.items():
setattr(row, k, v)
return row
# ── _cadastral_reverse ──────────────────────────────────────────────────────── def _make_db(rows: list[object]) -> MagicMock:
"""Session mock whose execute().mappings().all() returns rows."""
db = MagicMock()
result = MagicMock()
result.mappings.return_value.all.return_value = rows
result.first.return_value = rows[0] if rows else None
db.execute.return_value = result
return db
async def test_cadastral_reverse_returns_address_on_200() -> None: def _make_db_error() -> MagicMock:
"""Main backend returns 200 → address string returned.""" """Session mock whose execute() raises RuntimeError."""
mock_response = MagicMock() db = MagicMock()
mock_response.status_code = 200 db.execute.side_effect = RuntimeError("FDW connection failed")
mock_response.raise_for_status = MagicMock() return db
mock_response.json.return_value = {
"address": "ул. Малышева, 30, Екатеринбург",
"lat": 56.838,
"lon": 60.605,
"distance_m": 12.5,
}
mock_client = AsyncMock()
mock_client.get = AsyncMock(return_value=mock_response)
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
mock_client.__aexit__ = AsyncMock(return_value=False)
with patch("app.services.geocoder.httpx.AsyncClient", return_value=mock_client):
result = await _cadastral_reverse(56.838, 60.605)
assert result == "ул. Малышева, 30, Екатеринбург"
async def test_cadastral_reverse_returns_none_on_404() -> None: # ── _cadastral_forward_sync ───────────────────────────────────────────────────
"""Main backend returns 404 (no building in radius) → None, no exception."""
mock_response = MagicMock()
mock_response.status_code = 404
mock_client = AsyncMock() def test_forward_sync_returns_suggestions_when_rows_exist() -> None:
mock_client.get = AsyncMock(return_value=mock_response) """DB returns rows → list of GeocodeSuggestion returned."""
mock_client.__aenter__ = AsyncMock(return_value=mock_client) row = _make_db_row(
mock_client.__aexit__ = AsyncMock(return_value=False) readable_address="ул. Малышева, 30, Екатеринбург",
lat=56.838,
lon=60.605,
sim=0.9,
cad_num="66:41:0101001:123",
)
db = _make_db([row])
with patch("app.services.geocoder.httpx.AsyncClient", return_value=mock_client): results = _cadastral_forward_sync(db, "Малышева 30", limit=5)
result = await _cadastral_reverse(56.838, 60.605)
assert result is None assert len(results) == 1
assert results[0].full_address == "ул. Малышева, 30, Екатеринбург"
assert results[0].lat == 56.838
assert results[0].lon == 60.605
assert results[0].kind == "house"
async def test_cadastral_reverse_returns_none_on_timeout() -> None: def test_forward_sync_returns_empty_when_no_rows() -> None:
"""Network timeout → None returned, no exception bubbles out.""" """DB returns no rows → empty list returned, no exception."""
import httpx db = _make_db([])
mock_client = AsyncMock() results = _cadastral_forward_sync(db, "Несуществующая улица", limit=5)
mock_client.get = AsyncMock(side_effect=httpx.TimeoutException("timeout"))
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
mock_client.__aexit__ = AsyncMock(return_value=False)
with patch("app.services.geocoder.httpx.AsyncClient", return_value=mock_client): assert results == []
result = await _cadastral_reverse(56.838, 60.605)
assert result is None
async def test_cadastral_reverse_returns_none_when_not_configured() -> None: def test_forward_sync_returns_empty_on_db_error() -> None:
"""main_backend_url is None → immediately returns None without HTTP call.""" """DB raises → empty list, no exception bubbles out."""
with patch("app.services.geocoder.settings") as mock_settings: db = _make_db_error()
mock_settings.main_backend_url = None
with patch("app.services.geocoder.httpx.AsyncClient") as mock_client_cls: results = _cadastral_forward_sync(db, "Малышева", limit=5)
result = await _cadastral_reverse(56.838, 60.605)
mock_client_cls.assert_not_called()
assert result is None assert results == []
def test_forward_sync_respects_limit() -> None:
"""Returns at most `limit` items even if DB has more."""
rows = [
_make_db_row(
readable_address=f"ул. Тестовая, {i}, Екатеринбург",
lat=56.838 + i * 0.001,
lon=60.605,
sim=0.9,
cad_num=f"66:41:01010{i:02d}:1",
)
for i in range(10)
]
db = _make_db(rows)
results = _cadastral_forward_sync(db, "Тестовая", limit=3)
assert len(results) <= 3
# ── _cadastral_reverse_sync ───────────────────────────────────────────────────
def test_reverse_sync_returns_address_when_row_found() -> None:
"""DB finds building within radius → address string returned."""
row = MagicMock()
row.readable_address = "ул. Хохрякова, 81, Екатеринбург"
db = MagicMock()
result = MagicMock()
result.first.return_value = row
db.execute.return_value = result
addr = _cadastral_reverse_sync(db, 56.838, 60.605, radius_m=200)
assert addr == "ул. Хохрякова, 81, Екатеринбург"
def test_reverse_sync_returns_none_when_no_row_in_radius() -> None:
"""No row within radius → None returned."""
db = MagicMock()
result = MagicMock()
result.first.return_value = None
db.execute.return_value = result
addr = _cadastral_reverse_sync(db, 56.838, 60.605, radius_m=200)
assert addr is None
def test_reverse_sync_returns_none_on_db_error() -> None:
"""DB raises → None returned, no exception bubbles out."""
db = _make_db_error()
addr = _cadastral_reverse_sync(db, 56.838, 60.605)
assert addr is None
# ── reverse_geocode ─────────────────────────────────────────────────────────── # ── reverse_geocode ───────────────────────────────────────────────────────────
async def test_reverse_geocode_prefers_cadastral_over_nominatim() -> None:
async def test_reverse_geocode_returns_cadastral_when_available() -> None:
"""Cadastral returns address → Nominatim is never called.""" """Cadastral returns address → Nominatim is never called."""
db = MagicMock()
with ( with (
patch( patch(
"app.services.geocoder._cadastral_reverse", "app.services.geocoder._cadastral_reverse_sync",
new_callable=AsyncMock,
return_value="ул. Малышева, 30, Екатеринбург", return_value="ул. Малышева, 30, Екатеринбург",
) as mock_cadastral, ) as mock_cad,
patch( patch(
"app.services.geocoder._nominatim_reverse", "app.services.geocoder._nominatim_reverse",
new_callable=AsyncMock, new_callable=AsyncMock,
) as mock_nominatim, ) as mock_nom,
): ):
result = await reverse_geocode(56.838, 60.605) result = await reverse_geocode(56.838, 60.605, db=db)
assert result == "ул. Малышева, 30, Екатеринбург" assert result == "ул. Малышева, 30, Екатеринбург"
mock_cadastral.assert_called_once_with(56.838, 60.605) mock_cad.assert_called_once_with(db, 56.838, 60.605)
mock_nominatim.assert_not_called() mock_nom.assert_not_called()
async def test_reverse_geocode_falls_back_to_nominatim_when_cadastral_none() -> None: async def test_reverse_geocode_falls_back_to_nominatim_when_cadastral_none() -> None:
"""Cadastral returns None → Nominatim is called and its result returned.""" """Cadastral returns None → Nominatim called and result returned."""
db = MagicMock()
with ( with (
patch( patch("app.services.geocoder._cadastral_reverse_sync", return_value=None),
"app.services.geocoder._cadastral_reverse",
new_callable=AsyncMock,
return_value=None,
),
patch( patch(
"app.services.geocoder._nominatim_reverse", "app.services.geocoder._nominatim_reverse",
new_callable=AsyncMock, new_callable=AsyncMock,
return_value="Трамвайный переулок, 2, Екатеринбург", return_value="Трамвайный переулок, 2, Екатеринбург",
) as mock_nominatim, ) as mock_nom,
): ):
result = await reverse_geocode(56.838, 60.605) result = await reverse_geocode(56.838, 60.605, db=db)
assert result == "Трамвайный переулок, 2, Екатеринбург" assert result == "Трамвайный переулок, 2, Екатеринбург"
mock_nominatim.assert_called_once_with(56.838, 60.605) mock_nom.assert_called_once_with(56.838, 60.605)
async def test_reverse_geocode_returns_none_when_both_fail() -> None: async def test_reverse_geocode_returns_none_when_both_fail() -> None:
"""Both cadastral and Nominatim fail → returns None, no exception raised.""" """Both cadastral and Nominatim fail → returns None, no exception raised (500 fix)."""
db = MagicMock()
with ( with (
patch( patch("app.services.geocoder._cadastral_reverse_sync", return_value=None),
"app.services.geocoder._cadastral_reverse",
new_callable=AsyncMock,
return_value=None,
),
patch( patch(
"app.services.geocoder._nominatim_reverse", "app.services.geocoder._nominatim_reverse",
new_callable=AsyncMock, new_callable=AsyncMock,
side_effect=RuntimeError("Nominatim 403 Forbidden — IP banned"), side_effect=RuntimeError("Nominatim 403 Forbidden — IP banned"),
), ),
): ):
result = await reverse_geocode(56.838, 60.605) result = await reverse_geocode(56.838, 60.605, db=db)
# Must return None, not raise # Must return None, not raise
assert result is None assert result is None
async def test_reverse_geocode_without_db_skips_cadastral() -> None:
"""When db=None, cadastral tier is skipped entirely → goes straight to Nominatim."""
with (
patch(
"app.services.geocoder._cadastral_reverse_sync",
) as mock_cad,
patch(
"app.services.geocoder._nominatim_reverse",
new_callable=AsyncMock,
return_value="ул. Ленина, 1, Екатеринбург",
),
):
result = await reverse_geocode(56.838, 60.605, db=None)
mock_cad.assert_not_called()
assert result == "ул. Ленина, 1, Екатеринбург"
# ── geocode: cadastral as first tier ─────────────────────────────────────────
async def test_geocode_uses_cadastral_before_yandex() -> None:
"""Cadastral FDW returns suggestion → geocode returns it without hitting Yandex."""
db = MagicMock()
# Cache miss
db.execute.side_effect = [
MagicMock(fetchone=lambda: None), # _cache_get
]
cad_suggestion = GeocodeSuggestion(
label="ул. Хохрякова, 81",
full_address="ул. Хохрякова, 81, Екатеринбург",
lat=56.840,
lon=60.610,
kind="house",
)
with (
patch("app.services.geocoder._cache_get", return_value=None),
patch(
"app.services.geocoder._cadastral_forward_sync",
return_value=[cad_suggestion],
) as mock_cad,
patch("app.services.geocoder._cache_put"),
patch("app.services.geocoder._yandex_lookup", new_callable=AsyncMock) as mock_yandex,
):
result = await geocode("Хохрякова 81", db)
assert result is not None
assert result.lat == 56.840
assert result.lon == 60.610
assert result.confidence == "exact"
mock_cad.assert_called_once()
mock_yandex.assert_not_called()
# ── suggest: cadastral as first tier ─────────────────────────────────────────
async def test_suggest_uses_cadastral_before_yandex() -> None:
"""Cadastral FDW returns results → suggest returns them without hitting Yandex."""
db = MagicMock()
cad_suggestions = [
GeocodeSuggestion(
label="ул. Малышева, 30",
full_address="ул. Малышева, 30, Екатеринбург",
lat=56.838,
lon=60.605,
kind="house",
)
]
with (
patch(
"app.services.geocoder._cadastral_forward_sync",
return_value=cad_suggestions,
) as mock_cad,
patch(
"app.services.geocoder._yandex_suggest",
new_callable=AsyncMock,
) as mock_yandex,
):
results = await suggest("Малышева", db=db, limit=8)
assert len(results) == 1
assert results[0].full_address == "ул. Малышева, 30, Екатеринбург"
mock_cad.assert_called_once()
mock_yandex.assert_not_called()
async def test_suggest_falls_back_to_yandex_when_cadastral_empty() -> None:
"""Cadastral returns [] → falls through to Yandex."""
db = MagicMock()
yandex_result = GeocodeSuggestion(
label="Малышева · Октябрьский",
full_address="улица Малышева, Екатеринбург",
lat=56.838,
lon=60.605,
kind="street",
)
with (
patch("app.services.geocoder._cadastral_forward_sync", return_value=[]),
patch("app.services.geocoder.settings") as mock_settings,
patch(
"app.services.geocoder._yandex_suggest",
new_callable=AsyncMock,
return_value=[yandex_result],
) as mock_yandex,
):
mock_settings.yandex_geocoder_key = "fake-key"
results = await suggest("Малышева", db=db, limit=8)
assert len(results) == 1
mock_yandex.assert_called_once()
# ── ensure_fdw_user_mapping: SQL injection / whitelist guards ─────────────────
def test_ensure_fdw_user_mapping_rejects_quote_in_password() -> None:
"""Password with embedded quote must be rejected (SQL injection guard)."""
from app.core.fdw import ensure_fdw_user_mapping
fake_settings = MagicMock()
fake_settings.gendesign_fdw_password = "x' OR '1'='1"
with patch("app.core.fdw.settings", fake_settings):
db = MagicMock()
try:
ensure_fdw_user_mapping(db)
assert False, "should have raised ValueError"
except ValueError as e:
assert "format whitelist" in str(e)
# No DB call should have happened
db.execute.assert_not_called()
def test_ensure_fdw_user_mapping_rejects_short_password() -> None:
"""Password shorter than 32 chars rejected — forces rotation away from weak defaults."""
from app.core.fdw import ensure_fdw_user_mapping
fake_settings = MagicMock()
fake_settings.gendesign_fdw_password = "short"
with patch("app.core.fdw.settings", fake_settings):
db = MagicMock()
try:
ensure_fdw_user_mapping(db)
assert False, "should have raised ValueError"
except ValueError:
pass
db.execute.assert_not_called()
def test_ensure_fdw_user_mapping_accepts_valid_password() -> None:
"""Hex password 32+ chars passes whitelist and issues CREATE USER MAPPING."""
from app.core.fdw import ensure_fdw_user_mapping
fake_settings = MagicMock()
fake_settings.gendesign_fdw_password = "a" * 64 # valid hex-like
with patch("app.core.fdw.settings", fake_settings):
db = MagicMock()
db.execute.return_value.first.return_value = None # mapping doesn't exist
ensure_fdw_user_mapping(db)
# CREATE USER MAPPING called — inspect TextClause text via str(args[0])
sql_texts = [str(c.args[0]) for c in db.execute.call_args_list]
assert any("CREATE USER MAPPING" in s for s in sql_texts)
db.commit.assert_called_once()

View file

@ -1,78 +0,0 @@
#!/usr/bin/env bash
# Импорт кадастра зданий ЕКБ из gendesign-БД в tradein cad_buildings (#393).
#
# Источник: gendesign-postgres-1 / cad_buildings (~37k зданий, 36k по ЕКБ).
# Берём ЕКБ (cad_num LIKE '66:41:%') с годом постройки или этажностью.
# Координаты — центроид geom. Используется как источник обогащения
# house_metadata (год / этажность) — точнее OSM.
#
# Запуск на прод-хосте: ./import-cadastre.sh
# Повторяемо: upsert по cad_num (PRIMARY KEY), новый запуск обновит данные.
set -euo pipefail
SRC_PG="${SRC_PG:-gendesign-postgres-1}"
DST_PG="${DST_PG:-tradein-postgres}"
SRC_DB="${SRC_DB:-gendesign}"
SRC_USER="${SRC_USER:-gendesign}"
echo "[$(date -u +%H:%M:%S)] import-cadastre: здания ЕКБ из cad_buildings"
# 1. Staging-таблица в tradein.
docker exec "$DST_PG" psql -U tradein -d tradein -v ON_ERROR_STOP=on -c "
CREATE TABLE IF NOT EXISTS cad_buildings_staging (
cad_num text PRIMARY KEY, address text, floors int, year_built int,
area_m2 numeric, purpose text, lat double precision, lon double precision
);
TRUNCATE cad_buildings_staging;
"
# 2. Трансформируем на gendesign-стороне, пайпим CSV в staging.
# Центроид geom → lat/lon (ST_Transform на 4326 на случай иного SRID).
docker exec "$SRC_PG" psql -U "$SRC_USER" -d "$SRC_DB" -v ON_ERROR_STOP=on -c "
COPY (
SELECT
cad_num,
NULLIF(trim(readable_address), '') AS address,
NULLIF(floors, 0) AS floors,
CASE WHEN year_built BETWEEN 1800 AND 2100
THEN year_built END AS year_built,
round(COALESCE(build_record_area, area), 2) AS area_m2,
purpose,
ST_Y(ST_Centroid(ST_Transform(geom, 4326))) AS lat,
ST_X(ST_Centroid(ST_Transform(geom, 4326))) AS lon
FROM cad_buildings
WHERE cad_num LIKE '66:41:%'
AND cad_num IS NOT NULL
AND (year_built IS NOT NULL OR floors IS NOT NULL)
) TO STDOUT WITH CSV
" | docker exec -i "$DST_PG" psql -U tradein -d tradein -v ON_ERROR_STOP=on -c "
COPY cad_buildings_staging FROM STDIN WITH CSV
"
STAGED=$(docker exec "$DST_PG" psql -U tradein -d tradein -tAc \
"SELECT count(*) FROM cad_buildings_staging")
echo "[$(date -u +%H:%M:%S)] staged: $STAGED зданий"
# 3. Upsert в cad_buildings (trigger проставит geom из lat/lon).
docker exec "$DST_PG" psql -U tradein -d tradein -v ON_ERROR_STOP=on -c "
INSERT INTO cad_buildings
(cad_num, address, floors, year_built, area_m2, purpose, lat, lon)
SELECT cad_num, address, floors, year_built, area_m2, purpose, lat, lon
FROM cad_buildings_staging
ON CONFLICT (cad_num) DO UPDATE SET
address = EXCLUDED.address,
floors = EXCLUDED.floors,
year_built = EXCLUDED.year_built,
area_m2 = EXCLUDED.area_m2,
purpose = EXCLUDED.purpose,
lat = EXCLUDED.lat,
lon = EXCLUDED.lon,
imported_at = NOW();
DROP TABLE cad_buildings_staging;
"
TOTAL=$(docker exec "$DST_PG" psql -U tradein -d tradein -tAc \
"SELECT count(*) FROM cad_buildings")
echo "[$(date -u +%H:%M:%S)] cad_buildings всего: $TOTAL"

View file

@ -29,7 +29,9 @@ services:
timeout: 5s timeout: 5s
retries: 5 retries: 5
restart: unless-stopped restart: unless-stopped
networks: [tradein-net] networks:
- tradein-net
- gendesign_shared
backend: backend:
image: ghcr.io/lekss361/gendesign-tradein-backend:${IMAGE_TAG:-latest} image: ghcr.io/lekss361/gendesign-tradein-backend:${IMAGE_TAG:-latest}
@ -45,7 +47,7 @@ services:
ADMIN_TOKEN: "${ADMIN_TOKEN:-}" ADMIN_TOKEN: "${ADMIN_TOKEN:-}"
# GlitchTip DSN — мониторинг ошибок (#396). Пусто = выключено. # GlitchTip DSN — мониторинг ошибок (#396). Пусто = выключено.
GLITCHTIP_DSN: "${GLITCHTIP_DSN:-}" GLITCHTIP_DSN: "${GLITCHTIP_DSN:-}"
MAIN_BACKEND_URL: "${MAIN_BACKEND_URL:-http://gendesign-backend:8000}" GENDESIGN_FDW_PASSWORD: "${GENDESIGN_FDW_PASSWORD:-}"
depends_on: depends_on:
postgres: postgres:
condition: service_healthy condition: service_healthy