feat(tradein): postgres_fdw live read of gendesign.cad_buildings (replaces snapshot)
Replaces tradein.cad_buildings manual snapshot (36k rows, frozen 2026-05-22) with live postgres_fdw foreign table reading gendesign.v_tradein_cad_buildings directly. Also reverts PR #492 HTTP-based cadastral.py — superseded by FDW. Architecture: - gendesign DB: new role tradein_fdw_reader + flat view v_tradein_cad_buildings (EKB-only slice of cad_buildings with lat/lon flattened from geom) - networks: gendesign-postgres added to gendesign_shared (alias); tradein-postgres added to gendesign_shared for FDW connect - tradein DB: postgres_fdw extension + FOREIGN TABLE gendesign_cad_buildings - tradein backend: startup hook creates/refreshes USER MAPPING with password from env GENDESIGN_FDW_PASSWORD (password rotation handled via restart) - geocoder.py: cadastral primary for forward + reverse + suggest; Yandex/Nominatim fallback. reverse_geocode wraps Nominatim in try/except — no more HTTPStatusError → 500. - house_metadata.py and trade_in.py admin stats switched to foreign table - DROP TABLE cad_buildings in tradein (legacy snapshot removed entirely) - import-cadastre.sh deleted (no manual sync needed) Fixes: - /trade-in/api/v1/geocode/reverse 500 (Nominatim ban → no fallback) - estimate confidence_explanation=address_not_geocoded for addresses in our cadastre (e.g. Хохрякова 81) that Nominatim doesn't return Deploy ordering: main migration 100_tradein_fdw_role.sql adds role+view first (strict deploy.yml). Tradein next deploy applies 060_postgres_fdw_extension.sql and 061_drop_legacy_cad_buildings.sql (idempotent, errors ignored). Tradein backend startup creates USER MAPPING when env var present. Verify post-deploy with: docker exec tradein-postgres psql -U tradein -d tradein -c \ "SELECT count(*) FROM gendesign_cad_buildings" # expect ~36k EKB buildings.
This commit is contained in:
parent
0efd2dca1d
commit
698ef4f003
17 changed files with 610 additions and 374 deletions
|
|
@ -1,101 +0,0 @@
|
||||||
"""Cadastral lookup endpoints — reverse geocode via cad_buildings.
|
|
||||||
|
|
||||||
Used by trade-in backend (and potentially other internal services) to resolve
|
|
||||||
координаты → адрес жилого дома без зависимости от внешних геокодеров.
|
|
||||||
"""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import logging
|
|
||||||
from typing import Annotated
|
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
|
||||||
from pydantic import BaseModel
|
|
||||||
from sqlalchemy import text
|
|
||||||
from sqlalchemy.orm import Session
|
|
||||||
|
|
||||||
from app.core.db import get_db
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
router = APIRouter()
|
|
||||||
|
|
||||||
|
|
||||||
class CadastralReverseResult(BaseModel):
|
|
||||||
address: str
|
|
||||||
lat: float
|
|
||||||
lon: float
|
|
||||||
distance_m: float
|
|
||||||
cad_num: str | None = None
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/reverse", response_model=CadastralReverseResult)
|
|
||||||
async def reverse(
|
|
||||||
lat: Annotated[float, Query(ge=-90, le=90)],
|
|
||||||
lon: Annotated[float, Query(ge=-180, le=180)],
|
|
||||||
radius_m: Annotated[int, Query(ge=10, le=2000)] = 200,
|
|
||||||
db: Annotated[Session, Depends(get_db)] = None, # type: ignore[assignment]
|
|
||||||
) -> CadastralReverseResult:
|
|
||||||
"""Координаты → ближайшее здание из cad_buildings.
|
|
||||||
|
|
||||||
Filters out гаражи/СНТ/садовод/товарищества — для trade-in нужны жилые дома.
|
|
||||||
|
|
||||||
Returns 404 если в радиусе нет подходящего здания.
|
|
||||||
"""
|
|
||||||
row = db.execute(
|
|
||||||
text(
|
|
||||||
"""
|
|
||||||
SELECT
|
|
||||||
readable_address,
|
|
||||||
cad_num,
|
|
||||||
ST_Y(ST_Centroid(geom)) AS lat,
|
|
||||||
ST_X(ST_Centroid(geom)) AS lon,
|
|
||||||
ST_Distance(
|
|
||||||
geom::geography,
|
|
||||||
ST_SetSRID(
|
|
||||||
ST_MakePoint(
|
|
||||||
CAST(:lon AS double precision),
|
|
||||||
CAST(:lat AS double precision)
|
|
||||||
),
|
|
||||||
4326
|
|
||||||
)::geography
|
|
||||||
) AS distance_m
|
|
||||||
FROM cad_buildings
|
|
||||||
WHERE ST_DWithin(
|
|
||||||
geom::geography,
|
|
||||||
ST_SetSRID(
|
|
||||||
ST_MakePoint(
|
|
||||||
CAST(:lon AS double precision),
|
|
||||||
CAST(:lat AS double precision)
|
|
||||||
),
|
|
||||||
4326
|
|
||||||
)::geography,
|
|
||||||
CAST(:radius AS double precision)
|
|
||||||
)
|
|
||||||
AND readable_address !~* '(гараж|снт|садовод|товарищ|уч\\.)'
|
|
||||||
ORDER BY ST_Distance(
|
|
||||||
geom::geography,
|
|
||||||
ST_SetSRID(
|
|
||||||
ST_MakePoint(
|
|
||||||
CAST(:lon AS double precision),
|
|
||||||
CAST(:lat AS double precision)
|
|
||||||
),
|
|
||||||
4326
|
|
||||||
)::geography
|
|
||||||
)
|
|
||||||
LIMIT 1
|
|
||||||
"""
|
|
||||||
),
|
|
||||||
{"lat": lat, "lon": lon, "radius": float(radius_m)},
|
|
||||||
).first()
|
|
||||||
|
|
||||||
if row is None:
|
|
||||||
raise HTTPException(status_code=404, detail="no cadastral building found within radius")
|
|
||||||
|
|
||||||
return CadastralReverseResult(
|
|
||||||
address=row.readable_address,
|
|
||||||
lat=float(row.lat),
|
|
||||||
lon=float(row.lon),
|
|
||||||
distance_m=round(float(row.distance_m), 1),
|
|
||||||
cad_num=row.cad_num,
|
|
||||||
)
|
|
||||||
|
|
@ -21,7 +21,6 @@ from app.api.v1 import (
|
||||||
admin_scrape,
|
admin_scrape,
|
||||||
admin_weight_profiles,
|
admin_weight_profiles,
|
||||||
analytics,
|
analytics,
|
||||||
cadastral,
|
|
||||||
concepts,
|
concepts,
|
||||||
custom_pois,
|
custom_pois,
|
||||||
landing,
|
landing,
|
||||||
|
|
@ -79,7 +78,6 @@ app.add_middleware(
|
||||||
allow_headers=["*"],
|
allow_headers=["*"],
|
||||||
)
|
)
|
||||||
|
|
||||||
app.include_router(cadastral.router, prefix="/api/v1/cadastral", tags=["cadastral"])
|
|
||||||
app.include_router(concepts.router, prefix="/api/v1/concepts", tags=["concepts"])
|
app.include_router(concepts.router, prefix="/api/v1/concepts", tags=["concepts"])
|
||||||
app.include_router(parcels.router, prefix="/api/v1/parcels", tags=["parcels"])
|
app.include_router(parcels.router, prefix="/api/v1/parcels", tags=["parcels"])
|
||||||
app.include_router(analytics.router, prefix="/api/v1/analytics", tags=["analytics"])
|
app.include_router(analytics.router, prefix="/api/v1/analytics", tags=["analytics"])
|
||||||
|
|
|
||||||
36
data/sql/100_tradein_fdw_role.sql
Normal file
36
data/sql/100_tradein_fdw_role.sql
Normal file
|
|
@ -0,0 +1,36 @@
|
||||||
|
-- tradein_fdw_reader role + v_tradein_cad_buildings view
|
||||||
|
-- Used by tradein-postgres via postgres_fdw to read live cad_buildings.
|
||||||
|
-- See meta/00_credentials.md for password rotation procedure.
|
||||||
|
BEGIN;
|
||||||
|
|
||||||
|
DO $$
|
||||||
|
BEGIN
|
||||||
|
IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'tradein_fdw_reader') THEN
|
||||||
|
CREATE ROLE tradein_fdw_reader LOGIN PASSWORD '40473b7c10c7474855c7d1aa98f0fd0a0de2499aef0cbd1fead06c9641e4dcd8';
|
||||||
|
END IF;
|
||||||
|
END$$;
|
||||||
|
|
||||||
|
GRANT CONNECT ON DATABASE gendesign TO tradein_fdw_reader;
|
||||||
|
GRANT USAGE ON SCHEMA public TO tradein_fdw_reader;
|
||||||
|
|
||||||
|
CREATE OR REPLACE VIEW v_tradein_cad_buildings AS
|
||||||
|
SELECT
|
||||||
|
cad_num,
|
||||||
|
NULLIF(trim(readable_address), '') AS readable_address,
|
||||||
|
NULLIF(floors, 0) AS floors,
|
||||||
|
CASE WHEN year_built BETWEEN 1800 AND 2100 THEN year_built END AS year_built,
|
||||||
|
round(COALESCE(build_record_area, area), 2)::numeric AS area_m2,
|
||||||
|
purpose,
|
||||||
|
ST_Y(ST_Centroid(geom))::double precision AS lat,
|
||||||
|
ST_X(ST_Centroid(geom))::double precision AS lon
|
||||||
|
FROM cad_buildings
|
||||||
|
WHERE cad_num LIKE '66:41:%'
|
||||||
|
AND geom IS NOT NULL
|
||||||
|
AND readable_address IS NOT NULL;
|
||||||
|
|
||||||
|
GRANT SELECT ON v_tradein_cad_buildings TO tradein_fdw_reader;
|
||||||
|
|
||||||
|
COMMENT ON VIEW v_tradein_cad_buildings IS
|
||||||
|
'FDW source for tradein-mvp (postgres_fdw). Flat 8-col EKB-only slice of cad_buildings. tradein.gendesign_cad_buildings maps here.';
|
||||||
|
|
||||||
|
COMMIT;
|
||||||
|
|
@ -33,6 +33,11 @@ services:
|
||||||
interval: 5s
|
interval: 5s
|
||||||
timeout: 5s
|
timeout: 5s
|
||||||
retries: 10
|
retries: 10
|
||||||
|
networks:
|
||||||
|
default: {}
|
||||||
|
shared:
|
||||||
|
aliases:
|
||||||
|
- gendesign-postgres
|
||||||
|
|
||||||
redis:
|
redis:
|
||||||
image: redis:7-alpine
|
image: redis:7-alpine
|
||||||
|
|
@ -67,11 +72,6 @@ services:
|
||||||
timeout: 5s
|
timeout: 5s
|
||||||
retries: 5
|
retries: 5
|
||||||
start_period: 20s
|
start_period: 20s
|
||||||
networks:
|
|
||||||
default: {}
|
|
||||||
shared:
|
|
||||||
aliases:
|
|
||||||
- gendesign-backend
|
|
||||||
|
|
||||||
frontend:
|
frontend:
|
||||||
image: ghcr.io/lekss361/gendesign-frontend:${IMAGE_TAG:-latest}
|
image: ghcr.io/lekss361/gendesign-frontend:${IMAGE_TAG:-latest}
|
||||||
|
|
|
||||||
|
|
@ -50,6 +50,7 @@ class SuggestResponse(BaseModel):
|
||||||
async def suggest_addresses(
|
async def suggest_addresses(
|
||||||
q: Annotated[str, Query(min_length=2, max_length=200, description="Запрос для автокомплита")],
|
q: Annotated[str, Query(min_length=2, max_length=200, description="Запрос для автокомплита")],
|
||||||
limit: Annotated[int, Query(ge=1, le=15)] = 8,
|
limit: Annotated[int, Query(ge=1, le=15)] = 8,
|
||||||
|
db: Annotated[Session, Depends(get_db)] = None, # type: ignore[assignment]
|
||||||
) -> SuggestResponse:
|
) -> SuggestResponse:
|
||||||
"""Автокомплит адресов в пределах ЕКБ.
|
"""Автокомплит адресов в пределах ЕКБ.
|
||||||
|
|
||||||
|
|
@ -60,7 +61,7 @@ async def suggest_addresses(
|
||||||
/api/v1/geocode/suggest?q=Малышева
|
/api/v1/geocode/suggest?q=Малышева
|
||||||
/api/v1/geocode/suggest?q=Цвиллинга # → пусто, такой улицы в ЕКБ нет
|
/api/v1/geocode/suggest?q=Цвиллинга # → пусто, такой улицы в ЕКБ нет
|
||||||
"""
|
"""
|
||||||
items = await suggest(q, limit=limit)
|
items = await suggest(q, db=db, limit=limit)
|
||||||
return SuggestResponse(items=[
|
return SuggestResponse(items=[
|
||||||
SuggestItem(
|
SuggestItem(
|
||||||
label=s.label,
|
label=s.label,
|
||||||
|
|
@ -74,12 +75,13 @@ async def suggest_addresses(
|
||||||
async def reverse(
|
async def reverse(
|
||||||
lat: Annotated[float, Query(ge=-90, le=90)],
|
lat: Annotated[float, Query(ge=-90, le=90)],
|
||||||
lon: Annotated[float, Query(ge=-180, le=180)],
|
lon: Annotated[float, Query(ge=-180, le=180)],
|
||||||
|
db: Annotated[Session, Depends(get_db)] = None, # type: ignore[assignment]
|
||||||
) -> dict[str, object]:
|
) -> dict[str, object]:
|
||||||
"""Обратный геокодинг — координаты с карты → адрес (map-picker).
|
"""Обратный геокодинг — координаты с карты → адрес (map-picker).
|
||||||
|
|
||||||
Пример: /api/v1/geocode/reverse?lat=56.8389&lon=60.6057
|
Пример: /api/v1/geocode/reverse?lat=56.8389&lon=60.6057
|
||||||
"""
|
"""
|
||||||
address = await reverse_geocode(lat, lon)
|
address = await reverse_geocode(lat, lon, db=db)
|
||||||
if address is None:
|
if address is None:
|
||||||
raise HTTPException(status_code=404, detail="address not found for coordinates")
|
raise HTTPException(status_code=404, detail="address not found for coordinates")
|
||||||
return {"address": address, "lat": lat, "lon": lon}
|
return {"address": address, "lat": lat, "lon": lon}
|
||||||
|
|
|
||||||
|
|
@ -584,7 +584,7 @@ def cache_stats(db: Annotated[Session, Depends(get_db)]) -> dict[str, object]:
|
||||||
(SELECT count(*) FROM listings WHERE is_active) AS listings_active,
|
(SELECT count(*) FROM listings WHERE is_active) AS listings_active,
|
||||||
(SELECT max(scraped_at) FROM listings) AS listings_last_scraped,
|
(SELECT max(scraped_at) FROM listings) AS listings_last_scraped,
|
||||||
(SELECT count(*) FROM deals) AS deals,
|
(SELECT count(*) FROM deals) AS deals,
|
||||||
(SELECT count(*) FROM cad_buildings) AS cad_buildings,
|
(SELECT count(*) FROM gendesign_cad_buildings) AS cad_buildings,
|
||||||
(SELECT count(*) FROM house_metadata) AS house_metadata,
|
(SELECT count(*) FROM house_metadata) AS house_metadata,
|
||||||
(SELECT count(*) FROM trade_in_estimates) AS estimates_total
|
(SELECT count(*) FROM trade_in_estimates) AS estimates_total
|
||||||
"""
|
"""
|
||||||
|
|
|
||||||
|
|
@ -32,10 +32,10 @@ class Settings(BaseSettings):
|
||||||
# Redis URL для hot-cache (Phase 3.2). Задаётся через env REDIS_URL.
|
# Redis URL для hot-cache (Phase 3.2). Задаётся через env REDIS_URL.
|
||||||
redis_url: str = "redis://localhost:6379/0"
|
redis_url: str = "redis://localhost:6379/0"
|
||||||
|
|
||||||
# Main GenDesign backend URL — для cadastral reverse geocode (cad_buildings).
|
# Password for tradein_fdw_reader role — used by backend startup to create/refresh
|
||||||
# В prod: http://gendesign-backend:8000 (network alias в docker-compose.prod.yml).
|
# USER MAPPING for postgres_fdw → gendesign DB (gendesign_remote server).
|
||||||
# Пусто = cadastral lookup пропускается, используется только Nominatim.
|
# Пусто = USER MAPPING не создаётся, gendesign_cad_buildings не работает (dev).
|
||||||
main_backend_url: str | None = None
|
gendesign_fdw_password: str | None = None
|
||||||
|
|
||||||
|
|
||||||
settings = Settings()
|
settings = Settings()
|
||||||
|
|
|
||||||
65
tradein-mvp/backend/app/core/fdw.py
Normal file
65
tradein-mvp/backend/app/core/fdw.py
Normal file
|
|
@ -0,0 +1,65 @@
|
||||||
|
"""postgres_fdw USER MAPPING management for gendesign_remote server.
|
||||||
|
|
||||||
|
USER MAPPING password is in env GENDESIGN_FDW_PASSWORD (not in SQL migration —
|
||||||
|
would leak password to disk).
|
||||||
|
|
||||||
|
This helper applies idempotent CREATE / ALTER USER MAPPING. Reapplied on every
|
||||||
|
backend startup so password rotation through .env.runtime is picked up after restart.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
|
||||||
|
from sqlalchemy import text
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app.core.config import settings
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def ensure_fdw_user_mapping(db: Session) -> None:
|
||||||
|
"""Idempotent CREATE OR ALTER USER MAPPING for tradein → gendesign_remote.
|
||||||
|
|
||||||
|
Skips silently if password not configured (dev / first deploy).
|
||||||
|
"""
|
||||||
|
password = settings.gendesign_fdw_password
|
||||||
|
if not password:
|
||||||
|
logger.warning(
|
||||||
|
"GENDESIGN_FDW_PASSWORD not set — skipping FDW user mapping "
|
||||||
|
"(gendesign_cad_buildings will fail)"
|
||||||
|
)
|
||||||
|
return
|
||||||
|
if not isinstance(password, str) or not password.strip():
|
||||||
|
logger.warning("GENDESIGN_FDW_PASSWORD empty — skipping FDW user mapping")
|
||||||
|
return
|
||||||
|
|
||||||
|
# Check existence
|
||||||
|
exists = db.execute(
|
||||||
|
text(
|
||||||
|
"SELECT 1 FROM pg_user_mappings "
|
||||||
|
"WHERE srvname = 'gendesign_remote' AND usename = current_user"
|
||||||
|
)
|
||||||
|
).first()
|
||||||
|
|
||||||
|
# Password cannot be a bind parameter in CREATE/ALTER USER MAPPING — must inline.
|
||||||
|
# Escape single quotes by doubling.
|
||||||
|
escaped_pw = password.replace("'", "''")
|
||||||
|
if exists is None:
|
||||||
|
db.execute(
|
||||||
|
text(
|
||||||
|
f"CREATE USER MAPPING FOR CURRENT_USER SERVER gendesign_remote "
|
||||||
|
f"OPTIONS (user 'tradein_fdw_reader', password '{escaped_pw}')"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
logger.info("created FDW user mapping for gendesign_remote")
|
||||||
|
else:
|
||||||
|
db.execute(
|
||||||
|
text(
|
||||||
|
f"ALTER USER MAPPING FOR CURRENT_USER SERVER gendesign_remote "
|
||||||
|
f"OPTIONS (SET password '{escaped_pw}')"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
logger.info("refreshed FDW user mapping password for gendesign_remote")
|
||||||
|
|
||||||
|
db.commit()
|
||||||
|
|
@ -17,6 +17,8 @@ from fastapi.middleware.cors import CORSMiddleware
|
||||||
|
|
||||||
from app.api.v1 import admin, brand, geocode, search, trade_in
|
from app.api.v1 import admin, brand, geocode, search, trade_in
|
||||||
from app.core.config import settings
|
from app.core.config import settings
|
||||||
|
from app.core.db import SessionLocal
|
||||||
|
from app.core.fdw import ensure_fdw_user_mapping
|
||||||
from app.core.ratelimit import RateLimitMiddleware
|
from app.core.ratelimit import RateLimitMiddleware
|
||||||
from app.services.scheduler import scheduler_loop
|
from app.services.scheduler import scheduler_loop
|
||||||
|
|
||||||
|
|
@ -41,6 +43,16 @@ if settings.glitchtip_dsn:
|
||||||
|
|
||||||
@asynccontextmanager
|
@asynccontextmanager
|
||||||
async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||||
|
# FDW bootstrap: create/refresh USER MAPPING for gendesign_remote postgres_fdw server.
|
||||||
|
# Best-effort: failure does not abort startup, just logs.
|
||||||
|
try:
|
||||||
|
with SessionLocal() as db:
|
||||||
|
ensure_fdw_user_mapping(db)
|
||||||
|
except Exception:
|
||||||
|
logger.exception(
|
||||||
|
"FDW user mapping bootstrap failed — cadastral queries may fail"
|
||||||
|
)
|
||||||
|
|
||||||
# Startup: launch scheduler background task
|
# Startup: launch scheduler background task
|
||||||
scheduler_task = asyncio.create_task(scheduler_loop())
|
scheduler_task = asyncio.create_task(scheduler_loop())
|
||||||
logger.info("FastAPI lifespan: scheduler task spawned")
|
logger.info("FastAPI lifespan: scheduler task spawned")
|
||||||
|
|
|
||||||
|
|
@ -158,7 +158,7 @@ async def _nominatim_query(client: httpx.AsyncClient, address: str) -> dict | No
|
||||||
lon_f = float(item["lon"])
|
lon_f = float(item["lon"])
|
||||||
if 60.40 <= lon_f <= 60.85 and 56.65 <= lat_f <= 56.95:
|
if 60.40 <= lon_f <= 60.85 and 56.65 <= lat_f <= 56.95:
|
||||||
return item
|
return item
|
||||||
except Exception: # noqa: BLE001
|
except Exception:
|
||||||
continue
|
continue
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
@ -245,7 +245,7 @@ async def _yandex_lookup(address: str, api_key: str) -> GeocodeResult | None:
|
||||||
if 60.40 <= lon_f <= 60.85 and 56.65 <= lat_f <= 56.95:
|
if 60.40 <= lon_f <= 60.85 and 56.65 <= lat_f <= 56.95:
|
||||||
best = obj
|
best = obj
|
||||||
break
|
break
|
||||||
except Exception: # noqa: BLE001
|
except Exception:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
if best is None:
|
if best is None:
|
||||||
|
|
@ -315,7 +315,7 @@ def _parse_yandex_members(members: list[dict]) -> list[GeocodeSuggestion]:
|
||||||
label=label, full_address=full,
|
label=label, full_address=full,
|
||||||
lat=float(lat_str), lon=float(lon_str), kind=kind,
|
lat=float(lat_str), lon=float(lon_str), kind=kind,
|
||||||
))
|
))
|
||||||
except Exception: # noqa: BLE001
|
except Exception:
|
||||||
continue
|
continue
|
||||||
return out
|
return out
|
||||||
|
|
||||||
|
|
@ -445,36 +445,128 @@ async def _nominatim_suggest(query: str, limit: int = 8) -> list[GeocodeSuggesti
|
||||||
return out
|
return out
|
||||||
|
|
||||||
|
|
||||||
async def suggest(query: str, limit: int = 8) -> list[GeocodeSuggestion]:
|
def _cadastral_forward_sync(
|
||||||
"""Автокомплит адресов в ЕКБ. Yandex → Nominatim → [].
|
db: Session, query: str, limit: int = 5
|
||||||
|
) -> list[GeocodeSuggestion]:
|
||||||
|
"""Forward search via gendesign_cad_buildings FDW.
|
||||||
|
|
||||||
|
Uses ILIKE prefilter (FDW-pushable) + pg_trgm similarity ranking locally.
|
||||||
|
Returns up to `limit` suggestions, or [] on error / no results.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
rows = db.execute(
|
||||||
|
text("""
|
||||||
|
SELECT cad_num, readable_address, lat, lon,
|
||||||
|
similarity(readable_address, CAST(:q AS text)) AS sim
|
||||||
|
FROM gendesign_cad_buildings
|
||||||
|
WHERE readable_address ILIKE CAST('%' || :q || '%' AS text)
|
||||||
|
ORDER BY sim DESC, length(readable_address) ASC
|
||||||
|
LIMIT CAST(:lim AS integer)
|
||||||
|
"""),
|
||||||
|
{"q": query, "lim": limit * 4},
|
||||||
|
).mappings().all()
|
||||||
|
except Exception:
|
||||||
|
logger.warning("cadastral forward sync failed for query=%r", query, exc_info=True)
|
||||||
|
return []
|
||||||
|
|
||||||
|
out: list[GeocodeSuggestion] = []
|
||||||
|
for r in rows[:limit]:
|
||||||
|
out.append(GeocodeSuggestion(
|
||||||
|
label=str(r["readable_address"]),
|
||||||
|
full_address=str(r["readable_address"]),
|
||||||
|
lat=float(r["lat"]),
|
||||||
|
lon=float(r["lon"]),
|
||||||
|
kind="house",
|
||||||
|
))
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def _cadastral_reverse_sync(
|
||||||
|
db: Session, lat: float, lon: float, radius_m: int = 200
|
||||||
|
) -> str | None:
|
||||||
|
"""Reverse lookup via gendesign_cad_buildings FDW.
|
||||||
|
|
||||||
|
bbox prefilter (±0.0025° lat, ±0.005° lon ≈ 280m) is FDW-pushable.
|
||||||
|
Distance calculated locally on small result set. Filters гаражи/СНТ.
|
||||||
|
Returns None on no result or error.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
row = db.execute(
|
||||||
|
text("""
|
||||||
|
WITH candidates AS (
|
||||||
|
SELECT cad_num, readable_address, lat, lon,
|
||||||
|
111320.0 * sqrt(
|
||||||
|
pow(CAST(:lat AS double precision) - lat, 2) +
|
||||||
|
pow(
|
||||||
|
cos(radians(CAST(:lat AS double precision)))
|
||||||
|
* (CAST(:lon AS double precision) - lon), 2
|
||||||
|
)
|
||||||
|
) AS dist_m
|
||||||
|
FROM gendesign_cad_buildings
|
||||||
|
WHERE lat BETWEEN CAST(:lat AS double precision) - 0.0025
|
||||||
|
AND CAST(:lat AS double precision) + 0.0025
|
||||||
|
AND lon BETWEEN CAST(:lon AS double precision) - 0.005
|
||||||
|
AND CAST(:lon AS double precision) + 0.005
|
||||||
|
AND readable_address !~* '(гараж|снт|садовод|товарищ|уч\\.)'
|
||||||
|
)
|
||||||
|
SELECT readable_address, dist_m
|
||||||
|
FROM candidates
|
||||||
|
WHERE dist_m < CAST(:radius AS double precision)
|
||||||
|
ORDER BY dist_m ASC
|
||||||
|
LIMIT 1
|
||||||
|
"""),
|
||||||
|
{"lat": lat, "lon": lon, "radius": float(radius_m)},
|
||||||
|
).first()
|
||||||
|
except Exception:
|
||||||
|
logger.warning(
|
||||||
|
"cadastral reverse sync failed for (%.5f, %.5f)", lat, lon, exc_info=True
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
|
||||||
|
if row is None:
|
||||||
|
return None
|
||||||
|
return str(row.readable_address)
|
||||||
|
|
||||||
|
|
||||||
|
async def suggest(
|
||||||
|
query: str, db: Session | None = None, limit: int = 8
|
||||||
|
) -> list[GeocodeSuggestion]:
|
||||||
|
"""Автокомплит адресов в ЕКБ. Cadastral FDW → Yandex → Nominatim → [].
|
||||||
|
|
||||||
|
db: если передан — cadastral lookup через gendesign_cad_buildings (первый tier).
|
||||||
Без кэша (дешёво, провайдеры толерируют автокомплит-запросы).
|
Без кэша (дешёво, провайдеры толерируют автокомплит-запросы).
|
||||||
"""
|
"""
|
||||||
if not query or len(query.strip()) < 2:
|
if not query or len(query.strip()) < 2:
|
||||||
return []
|
return []
|
||||||
|
|
||||||
|
# Tier 0: cadastral FDW (если db доступна) — самый быстрый, без внешних запросов
|
||||||
|
if db is not None:
|
||||||
|
cad_results = _cadastral_forward_sync(db, query.strip(), limit)
|
||||||
|
if cad_results:
|
||||||
|
return cad_results
|
||||||
|
|
||||||
if settings.yandex_geocoder_key:
|
if settings.yandex_geocoder_key:
|
||||||
try:
|
try:
|
||||||
results = await _yandex_suggest(query, settings.yandex_geocoder_key, limit)
|
results = await _yandex_suggest(query, settings.yandex_geocoder_key, limit)
|
||||||
if results:
|
if results:
|
||||||
return results
|
return results
|
||||||
except Exception: # noqa: BLE001
|
except Exception:
|
||||||
logger.exception("yandex suggest failed, falling back to nominatim")
|
logger.exception("yandex suggest failed, falling back to nominatim")
|
||||||
|
|
||||||
try:
|
try:
|
||||||
return await _nominatim_suggest(query, limit)
|
return await _nominatim_suggest(query, limit)
|
||||||
except Exception: # noqa: BLE001
|
except Exception:
|
||||||
logger.exception("nominatim suggest failed")
|
logger.exception("nominatim suggest failed")
|
||||||
return []
|
return []
|
||||||
|
|
||||||
|
|
||||||
# ── Public API ───────────────────────────────────────────────────────────────
|
# ── Public API ───────────────────────────────────────────────────────────────
|
||||||
async def geocode(address: str, db: Session) -> GeocodeResult | None:
|
async def geocode(address: str, db: Session) -> GeocodeResult | None:
|
||||||
"""Геокодинг с кэшем. Yandex → Nominatim → None.
|
"""Геокодинг с кэшем. Cadastral FDW → Yandex → Nominatim → None.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
address: пользовательский ввод (может быть грязным — нормализуем).
|
address: пользовательский ввод (может быть грязным — нормализуем).
|
||||||
db: сессия Postgres для cache lookup/write.
|
db: сессия Postgres для cache lookup/write и cadastral FDW lookup.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
GeocodeResult или None если ни один провайдер не отвечает.
|
GeocodeResult или None если ни один провайдер не отвечает.
|
||||||
|
|
@ -490,12 +582,33 @@ async def geocode(address: str, db: Session) -> GeocodeResult | None:
|
||||||
logger.info("geocode cache hit: %s", addr_norm)
|
logger.info("geocode cache hit: %s", addr_norm)
|
||||||
return cached
|
return cached
|
||||||
|
|
||||||
# 2. Yandex (если есть key) с typo-fallback
|
# 2. Cadastral FDW (прямой запрос к gendesign_cad_buildings — без внешнего API)
|
||||||
|
cad_suggestions = _cadastral_forward_sync(db, address.strip(), limit=1)
|
||||||
|
if cad_suggestions:
|
||||||
|
s = cad_suggestions[0]
|
||||||
|
result = GeocodeResult(
|
||||||
|
lat=s.lat,
|
||||||
|
lon=s.lon,
|
||||||
|
full_address=s.full_address,
|
||||||
|
provider="nominatim", # treat as "local" — same confidence as nominatim
|
||||||
|
confidence="exact",
|
||||||
|
)
|
||||||
|
_cache_put(db, addr_norm, result)
|
||||||
|
logger.info(
|
||||||
|
"geocode cadastral fdw: %s → (%.5f, %.5f)", addr_norm, result.lat, result.lon
|
||||||
|
)
|
||||||
|
return result
|
||||||
|
|
||||||
|
# 3. Yandex (если есть key) с typo-fallback
|
||||||
if settings.yandex_geocoder_key:
|
if settings.yandex_geocoder_key:
|
||||||
try:
|
try:
|
||||||
result = await _yandex_lookup(address, settings.yandex_geocoder_key)
|
result = await _yandex_lookup(address, settings.yandex_geocoder_key)
|
||||||
# Если результат вне ЕКБ — пробуем typo-варианты
|
# Если результат вне ЕКБ — пробуем typo-варианты
|
||||||
in_ekb = result is not None and 60.40 <= result.lon <= 60.85 and 56.65 <= result.lat <= 56.95
|
in_ekb = (
|
||||||
|
result is not None
|
||||||
|
and 60.40 <= result.lon <= 60.85
|
||||||
|
and 56.65 <= result.lat <= 56.95
|
||||||
|
)
|
||||||
if result is not None and in_ekb:
|
if result is not None and in_ekb:
|
||||||
_cache_put(db, addr_norm, result)
|
_cache_put(db, addr_norm, result)
|
||||||
logger.info("geocode yandex: %s → (%.5f, %.5f)", addr_norm, result.lat, result.lon)
|
logger.info("geocode yandex: %s → (%.5f, %.5f)", addr_norm, result.lat, result.lon)
|
||||||
|
|
@ -504,7 +617,7 @@ async def geocode(address: str, db: Session) -> GeocodeResult | None:
|
||||||
for variant in _typo_variants(address, limit=4):
|
for variant in _typo_variants(address, limit=4):
|
||||||
try:
|
try:
|
||||||
result = await _yandex_lookup(variant, settings.yandex_geocoder_key)
|
result = await _yandex_lookup(variant, settings.yandex_geocoder_key)
|
||||||
except Exception: # noqa: BLE001
|
except Exception:
|
||||||
continue
|
continue
|
||||||
if result is None:
|
if result is None:
|
||||||
continue
|
continue
|
||||||
|
|
@ -515,10 +628,10 @@ async def geocode(address: str, db: Session) -> GeocodeResult | None:
|
||||||
addr_norm, variant, result.lat, result.lon,
|
addr_norm, variant, result.lat, result.lon,
|
||||||
)
|
)
|
||||||
return result
|
return result
|
||||||
except Exception: # noqa: BLE001
|
except Exception:
|
||||||
logger.exception("yandex geocoder failed, falling back to nominatim")
|
logger.exception("yandex geocoder failed, falling back to nominatim")
|
||||||
|
|
||||||
# 3. Nominatim fallback
|
# 4. Nominatim fallback
|
||||||
try:
|
try:
|
||||||
result = await _nominatim_lookup(address)
|
result = await _nominatim_lookup(address)
|
||||||
if result is not None:
|
if result is not None:
|
||||||
|
|
@ -529,7 +642,7 @@ async def geocode(address: str, db: Session) -> GeocodeResult | None:
|
||||||
# Nominatim rate-limit policy: 1 req/sec — спим после успешного запроса
|
# Nominatim rate-limit policy: 1 req/sec — спим после успешного запроса
|
||||||
await asyncio.sleep(1.0)
|
await asyncio.sleep(1.0)
|
||||||
return result
|
return result
|
||||||
except Exception: # noqa: BLE001
|
except Exception:
|
||||||
logger.exception("nominatim geocoder failed")
|
logger.exception("nominatim geocoder failed")
|
||||||
|
|
||||||
return None
|
return None
|
||||||
|
|
@ -569,29 +682,6 @@ def _format_reverse_address(addr: dict) -> str | None:
|
||||||
return ", ".join(parts)
|
return ", ".join(parts)
|
||||||
|
|
||||||
|
|
||||||
async def _cadastral_reverse(lat: float, lon: float) -> str | None:
|
|
||||||
"""Reverse via main backend cad_buildings — fast, no external API.
|
|
||||||
|
|
||||||
Returns None on any error (network, 404, timeout, missing config).
|
|
||||||
"""
|
|
||||||
if not settings.main_backend_url:
|
|
||||||
return None
|
|
||||||
url = settings.main_backend_url.rstrip("/") + "/api/v1/cadastral/reverse"
|
|
||||||
try:
|
|
||||||
async with httpx.AsyncClient(timeout=3.0) as client:
|
|
||||||
response = await client.get(url, params={"lat": str(lat), "lon": str(lon)})
|
|
||||||
if response.status_code == 404:
|
|
||||||
return None
|
|
||||||
response.raise_for_status()
|
|
||||||
data = response.json()
|
|
||||||
addr = data.get("address")
|
|
||||||
if isinstance(addr, str) and addr.strip():
|
|
||||||
return addr
|
|
||||||
except Exception:
|
|
||||||
logger.exception("cadastral reverse failed for (%.5f, %.5f)", lat, lon)
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
@retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=1, max=8))
|
@retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=1, max=8))
|
||||||
async def _nominatim_reverse(lat: float, lon: float) -> str | None:
|
async def _nominatim_reverse(lat: float, lon: float) -> str | None:
|
||||||
"""Nominatim /reverse — extracted for fallback usage. @retry handles transient errors."""
|
"""Nominatim /reverse — extracted for fallback usage. @retry handles transient errors."""
|
||||||
|
|
@ -623,17 +713,20 @@ async def _nominatim_reverse(lat: float, lon: float) -> str | None:
|
||||||
return data.get("display_name")
|
return data.get("display_name")
|
||||||
|
|
||||||
|
|
||||||
async def reverse_geocode(lat: float, lon: float) -> str | None:
|
async def reverse_geocode(lat: float, lon: float, db: Session | None = None) -> str | None:
|
||||||
"""Cadastral (cad_buildings via main backend) → Nominatim fallback.
|
"""Cadastral FDW → Nominatim fallback.
|
||||||
|
|
||||||
Возвращает None если ни один источник не дал адрес. Endpoint
|
Возвращает None если ни один источник не дал адрес. Endpoint
|
||||||
api/v1/geocode/reverse сам выкинет 404. НЕ даёт выйти HTTPStatusError
|
api/v1/geocode/reverse сам выкинет 404. НЕ даёт выйти HTTPStatusError
|
||||||
наверх — раньше Nominatim 403 → RetryError → FastAPI 500.
|
наверх — раньше Nominatim 403 → RetryError → FastAPI 500.
|
||||||
|
|
||||||
|
db: если передан — cadastral lookup через gendesign_cad_buildings FDW (первый tier).
|
||||||
"""
|
"""
|
||||||
# 1. Cadastral primary (fast, no external API, returns жилой дом not POI)
|
# 1. Cadastral FDW primary (без внешнего API, возвращает жилой дом not POI)
|
||||||
cadastral = await _cadastral_reverse(lat, lon)
|
if db is not None:
|
||||||
if cadastral:
|
cadastral = _cadastral_reverse_sync(db, lat, lon)
|
||||||
return cadastral
|
if cadastral:
|
||||||
|
return cadastral
|
||||||
|
|
||||||
# 2. Nominatim fallback (wrap to prevent 500 on ban/rate-limit)
|
# 2. Nominatim fallback (wrap to prevent 500 on ban/rate-limit)
|
||||||
try:
|
try:
|
||||||
|
|
|
||||||
|
|
@ -138,34 +138,56 @@ def _cache_put(db: Session, meta: HouseMetadata, raw: dict) -> None:
|
||||||
db.commit()
|
db.commit()
|
||||||
|
|
||||||
|
|
||||||
# ── Источник #393: кадастр зданий (cad_buildings, импорт из gendesign-БД) ────
|
# ── Источник #393: кадастр зданий (gendesign_cad_buildings FDW) ──────────────
|
||||||
def _cad_buildings_get(db: Session, lat: float, lon: float) -> HouseMetadata | None:
|
def _cad_buildings_get(db: Session, lat: float, lon: float) -> HouseMetadata | None:
|
||||||
"""Ближайшее здание из cad_buildings (данные Росреестра) по координатам.
|
"""Ближайшее здание из gendesign_cad_buildings FDW (данные Росреестра) по координатам.
|
||||||
|
|
||||||
Приоритетный источник house_metadata — год / этажность полнее, чем OSM
|
Приоритетный источник house_metadata — год / этажность полнее, чем OSM
|
||||||
(~34к домов ЕКБ с годом постройки). Таблицу наполняет import-cadastre.sh.
|
(~36к домов ЕКБ с годом постройки). Использует bbox+haversine вместо ST_DWithin —
|
||||||
|
FDW-таблица не имеет PostGIS-колонки geom (только lat/lon скалярные).
|
||||||
"""
|
"""
|
||||||
row = db.execute(
|
try:
|
||||||
text(
|
row = db.execute(
|
||||||
"""
|
text(
|
||||||
SELECT year_built, floors
|
"""
|
||||||
FROM cad_buildings
|
WITH candidates AS (
|
||||||
WHERE geom IS NOT NULL
|
SELECT year_built, floors, lat, lon,
|
||||||
AND (year_built IS NOT NULL OR floors IS NOT NULL)
|
111320.0 * sqrt(
|
||||||
AND ST_DWithin(geom::geography, ST_MakePoint(:lon, :lat)::geography, :radius)
|
pow(CAST(:lat AS double precision) - lat, 2) +
|
||||||
ORDER BY ST_Distance(geom::geography, ST_MakePoint(:lon, :lat)::geography)
|
pow(
|
||||||
LIMIT 1
|
cos(radians(CAST(:lat AS double precision)))
|
||||||
"""
|
* (CAST(:lon AS double precision) - lon), 2
|
||||||
),
|
)
|
||||||
{"lat": lat, "lon": lon, "radius": _CAD_RADIUS_M},
|
) AS dist_m
|
||||||
).fetchone()
|
FROM gendesign_cad_buildings
|
||||||
|
WHERE (year_built IS NOT NULL OR floors IS NOT NULL)
|
||||||
|
AND lat BETWEEN CAST(:lat AS double precision) - 0.0010
|
||||||
|
AND CAST(:lat AS double precision) + 0.0010
|
||||||
|
AND lon BETWEEN CAST(:lon AS double precision) - 0.0015
|
||||||
|
AND CAST(:lon AS double precision) + 0.0015
|
||||||
|
)
|
||||||
|
SELECT year_built, floors
|
||||||
|
FROM candidates
|
||||||
|
WHERE dist_m < CAST(:radius AS double precision)
|
||||||
|
ORDER BY dist_m ASC
|
||||||
|
LIMIT 1
|
||||||
|
"""
|
||||||
|
),
|
||||||
|
{"lat": lat, "lon": lon, "radius": float(_CAD_RADIUS_M)},
|
||||||
|
).fetchone()
|
||||||
|
except Exception:
|
||||||
|
logger.warning(
|
||||||
|
"gendesign_cad_buildings lookup failed at (%.5f, %.5f)", lat, lon, exc_info=True
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
|
||||||
if row is None:
|
if row is None:
|
||||||
return None
|
return None
|
||||||
return HouseMetadata(
|
return HouseMetadata(
|
||||||
lat=lat, lon=lon,
|
lat=lat, lon=lon,
|
||||||
year_built=row.year_built,
|
year_built=row.year_built,
|
||||||
total_floors=row.floors,
|
total_floors=row.floors,
|
||||||
house_type=None, # cad_buildings.purpose не маппится в panel/brick
|
house_type=None, # purpose не маппится в panel/brick
|
||||||
total_units=None,
|
total_units=None,
|
||||||
source="cadastre",
|
source="cadastre",
|
||||||
)
|
)
|
||||||
|
|
@ -245,12 +267,12 @@ async def get_house_metadata(lat: float, lon: float, db: Session) -> HouseMetada
|
||||||
lat, lon, meta.year_built, meta.total_floors, meta.house_type,
|
lat, lon, meta.year_built, meta.total_floors, meta.house_type,
|
||||||
)
|
)
|
||||||
return meta
|
return meta
|
||||||
except Exception: # noqa: BLE001
|
except Exception:
|
||||||
logger.warning(
|
logger.warning(
|
||||||
"house_metadata enrichment failed at (%.5f, %.5f)", lat, lon, exc_info=True
|
"house_metadata enrichment failed at (%.5f, %.5f)", lat, lon, exc_info=True
|
||||||
)
|
)
|
||||||
try:
|
try:
|
||||||
db.rollback()
|
db.rollback()
|
||||||
except Exception: # noqa: BLE001
|
except Exception:
|
||||||
pass
|
pass
|
||||||
return None
|
return None
|
||||||
|
|
|
||||||
|
|
@ -1,34 +0,0 @@
|
||||||
-- Trade-In MVP — кадастр зданий ЕКБ (#393).
|
|
||||||
-- Срез cad_buildings из gendesign-БД (данные Росреестра по зданиям),
|
|
||||||
-- импортируется ETL-ом deploy/import-cadastre.sh.
|
|
||||||
-- Основной источник обогащения house_metadata (год / этажность) —
|
|
||||||
-- данные точнее и полнее, чем OSM Overpass.
|
|
||||||
|
|
||||||
BEGIN;
|
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS cad_buildings (
|
|
||||||
cad_num text PRIMARY KEY, -- кадастровый номер здания, 66:41:...
|
|
||||||
address text,
|
|
||||||
floors int,
|
|
||||||
year_built int,
|
|
||||||
area_m2 numeric(12, 2), -- площадь здания (не квартиры)
|
|
||||||
purpose text, -- Жилое / Нежилое / Многоквартирный дом
|
|
||||||
lat double precision,
|
|
||||||
lon double precision,
|
|
||||||
geom geometry(Point, 4326),
|
|
||||||
imported_at timestamptz NOT NULL DEFAULT NOW()
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE INDEX IF NOT EXISTS cad_buildings_geom_idx ON cad_buildings USING GIST (geom);
|
|
||||||
|
|
||||||
-- Авто-update geom из (lat, lon) — переиспользуем функцию из 002_core_tables.sql.
|
|
||||||
DROP TRIGGER IF EXISTS cad_buildings_set_geom_trg ON cad_buildings;
|
|
||||||
CREATE TRIGGER cad_buildings_set_geom_trg
|
|
||||||
BEFORE INSERT OR UPDATE OF lat, lon ON cad_buildings
|
|
||||||
FOR EACH ROW EXECUTE FUNCTION listings_set_geom();
|
|
||||||
|
|
||||||
COMMENT ON TABLE cad_buildings IS
|
|
||||||
'#393 — кадастр зданий ЕКБ (срез cad_buildings из gendesign-БД). '
|
|
||||||
'Приоритетный источник обогащения house_metadata: год постройки / этажность.';
|
|
||||||
|
|
||||||
COMMIT;
|
|
||||||
35
tradein-mvp/backend/data/sql/060_postgres_fdw_extension.sql
Normal file
35
tradein-mvp/backend/data/sql/060_postgres_fdw_extension.sql
Normal file
|
|
@ -0,0 +1,35 @@
|
||||||
|
-- postgres_fdw extension for tradein → gendesign live reads.
|
||||||
|
-- Idempotent. Server config only (no USER MAPPING — that's set by backend startup with env password).
|
||||||
|
BEGIN;
|
||||||
|
|
||||||
|
CREATE EXTENSION IF NOT EXISTS postgres_fdw;
|
||||||
|
|
||||||
|
DO $$
|
||||||
|
BEGIN
|
||||||
|
IF NOT EXISTS (SELECT 1 FROM pg_foreign_server WHERE srvname = 'gendesign_remote') THEN
|
||||||
|
CREATE SERVER gendesign_remote
|
||||||
|
FOREIGN DATA WRAPPER postgres_fdw
|
||||||
|
OPTIONS (host 'gendesign-postgres', dbname 'gendesign', port '5432', extensions 'postgis');
|
||||||
|
END IF;
|
||||||
|
END$$;
|
||||||
|
|
||||||
|
-- Drop existing foreign table if mistakenly created with wrong schema (idempotent re-create below)
|
||||||
|
DROP FOREIGN TABLE IF EXISTS gendesign_cad_buildings;
|
||||||
|
|
||||||
|
CREATE FOREIGN TABLE gendesign_cad_buildings (
|
||||||
|
cad_num text,
|
||||||
|
readable_address text,
|
||||||
|
year_built integer,
|
||||||
|
floors integer,
|
||||||
|
area_m2 numeric,
|
||||||
|
purpose text,
|
||||||
|
lat double precision,
|
||||||
|
lon double precision
|
||||||
|
)
|
||||||
|
SERVER gendesign_remote
|
||||||
|
OPTIONS (schema_name 'public', table_name 'v_tradein_cad_buildings');
|
||||||
|
|
||||||
|
COMMENT ON FOREIGN TABLE gendesign_cad_buildings IS
|
||||||
|
'Live view of gendesign.cad_buildings (EKB only). USER MAPPING managed by backend startup.';
|
||||||
|
|
||||||
|
COMMIT;
|
||||||
|
|
@ -0,0 +1,8 @@
|
||||||
|
-- Drop legacy cad_buildings snapshot table (replaced by gendesign_cad_buildings foreign table).
|
||||||
|
-- Idempotent.
|
||||||
|
BEGIN;
|
||||||
|
|
||||||
|
DROP TRIGGER IF EXISTS cad_buildings_set_geom_trg ON cad_buildings;
|
||||||
|
DROP TABLE IF EXISTS cad_buildings CASCADE;
|
||||||
|
|
||||||
|
COMMIT;
|
||||||
|
|
@ -1,4 +1,12 @@
|
||||||
"""Unit tests for cadastral reverse geocode and modified reverse_geocode."""
|
"""Unit tests for cadastral FDW SQL-backed geocoder helpers and reverse_geocode.
|
||||||
|
|
||||||
|
Tests cover:
|
||||||
|
- _cadastral_forward_sync: returns suggestions / empty list / graceful error
|
||||||
|
- _cadastral_reverse_sync: returns address / None on no result / None on error
|
||||||
|
- reverse_geocode: cadastral primary, Nominatim fallback, both-fail → None (500-fix)
|
||||||
|
- geocode: cadastral as tier 0
|
||||||
|
- suggest: cadastral as tier 0
|
||||||
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import os
|
import os
|
||||||
|
|
@ -7,144 +15,312 @@ from unittest.mock import AsyncMock, MagicMock, patch
|
||||||
|
|
||||||
# DATABASE_URL required by config before any app import.
|
# DATABASE_URL required by config before any app import.
|
||||||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
|
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
|
||||||
os.environ.setdefault("MAIN_BACKEND_URL", "http://gendesign-backend:8000")
|
|
||||||
|
|
||||||
# WeasyPrint stub — not installed in CI without GTK.
|
# WeasyPrint stub — not installed in CI without GTK.
|
||||||
_wp_mock = MagicMock()
|
_wp_mock = MagicMock()
|
||||||
sys.modules.setdefault("weasyprint", _wp_mock)
|
sys.modules.setdefault("weasyprint", _wp_mock)
|
||||||
|
|
||||||
import pytest
|
|
||||||
|
|
||||||
from app.services.geocoder import _cadastral_reverse, reverse_geocode
|
from app.services.geocoder import ( # noqa: E402
|
||||||
|
GeocodeSuggestion,
|
||||||
|
_cadastral_forward_sync,
|
||||||
|
_cadastral_reverse_sync,
|
||||||
|
geocode,
|
||||||
|
reverse_geocode,
|
||||||
|
suggest,
|
||||||
|
)
|
||||||
|
|
||||||
|
# ── helpers ───────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def _make_db_row(**kwargs: object) -> MagicMock:
|
||||||
|
"""Mock row object with dict-like access (MappingResult style)."""
|
||||||
|
row = MagicMock()
|
||||||
|
row.__getitem__ = lambda self, k: kwargs[k]
|
||||||
|
for k, v in kwargs.items():
|
||||||
|
setattr(row, k, v)
|
||||||
|
return row
|
||||||
|
|
||||||
|
|
||||||
# ── _cadastral_reverse ────────────────────────────────────────────────────────
|
def _make_db(rows: list[object]) -> MagicMock:
|
||||||
|
"""Session mock whose execute().mappings().all() returns rows."""
|
||||||
|
db = MagicMock()
|
||||||
|
result = MagicMock()
|
||||||
|
result.mappings.return_value.all.return_value = rows
|
||||||
|
result.first.return_value = rows[0] if rows else None
|
||||||
|
db.execute.return_value = result
|
||||||
|
return db
|
||||||
|
|
||||||
|
|
||||||
async def test_cadastral_reverse_returns_address_on_200() -> None:
|
def _make_db_error() -> MagicMock:
|
||||||
"""Main backend returns 200 → address string returned."""
|
"""Session mock whose execute() raises RuntimeError."""
|
||||||
mock_response = MagicMock()
|
db = MagicMock()
|
||||||
mock_response.status_code = 200
|
db.execute.side_effect = RuntimeError("FDW connection failed")
|
||||||
mock_response.raise_for_status = MagicMock()
|
return db
|
||||||
mock_response.json.return_value = {
|
|
||||||
"address": "ул. Малышева, 30, Екатеринбург",
|
|
||||||
"lat": 56.838,
|
|
||||||
"lon": 60.605,
|
|
||||||
"distance_m": 12.5,
|
|
||||||
}
|
|
||||||
|
|
||||||
mock_client = AsyncMock()
|
|
||||||
mock_client.get = AsyncMock(return_value=mock_response)
|
|
||||||
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
|
|
||||||
mock_client.__aexit__ = AsyncMock(return_value=False)
|
|
||||||
|
|
||||||
with patch("app.services.geocoder.httpx.AsyncClient", return_value=mock_client):
|
|
||||||
result = await _cadastral_reverse(56.838, 60.605)
|
|
||||||
|
|
||||||
assert result == "ул. Малышева, 30, Екатеринбург"
|
|
||||||
|
|
||||||
|
|
||||||
async def test_cadastral_reverse_returns_none_on_404() -> None:
|
# ── _cadastral_forward_sync ───────────────────────────────────────────────────
|
||||||
"""Main backend returns 404 (no building in radius) → None, no exception."""
|
|
||||||
mock_response = MagicMock()
|
|
||||||
mock_response.status_code = 404
|
|
||||||
|
|
||||||
mock_client = AsyncMock()
|
def test_forward_sync_returns_suggestions_when_rows_exist() -> None:
|
||||||
mock_client.get = AsyncMock(return_value=mock_response)
|
"""DB returns rows → list of GeocodeSuggestion returned."""
|
||||||
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
|
row = _make_db_row(
|
||||||
mock_client.__aexit__ = AsyncMock(return_value=False)
|
readable_address="ул. Малышева, 30, Екатеринбург",
|
||||||
|
lat=56.838,
|
||||||
|
lon=60.605,
|
||||||
|
sim=0.9,
|
||||||
|
cad_num="66:41:0101001:123",
|
||||||
|
)
|
||||||
|
db = _make_db([row])
|
||||||
|
|
||||||
with patch("app.services.geocoder.httpx.AsyncClient", return_value=mock_client):
|
results = _cadastral_forward_sync(db, "Малышева 30", limit=5)
|
||||||
result = await _cadastral_reverse(56.838, 60.605)
|
|
||||||
|
|
||||||
assert result is None
|
assert len(results) == 1
|
||||||
|
assert results[0].full_address == "ул. Малышева, 30, Екатеринбург"
|
||||||
|
assert results[0].lat == 56.838
|
||||||
|
assert results[0].lon == 60.605
|
||||||
|
assert results[0].kind == "house"
|
||||||
|
|
||||||
|
|
||||||
async def test_cadastral_reverse_returns_none_on_timeout() -> None:
|
def test_forward_sync_returns_empty_when_no_rows() -> None:
|
||||||
"""Network timeout → None returned, no exception bubbles out."""
|
"""DB returns no rows → empty list returned, no exception."""
|
||||||
import httpx
|
db = _make_db([])
|
||||||
|
|
||||||
mock_client = AsyncMock()
|
results = _cadastral_forward_sync(db, "Несуществующая улица", limit=5)
|
||||||
mock_client.get = AsyncMock(side_effect=httpx.TimeoutException("timeout"))
|
|
||||||
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
|
|
||||||
mock_client.__aexit__ = AsyncMock(return_value=False)
|
|
||||||
|
|
||||||
with patch("app.services.geocoder.httpx.AsyncClient", return_value=mock_client):
|
assert results == []
|
||||||
result = await _cadastral_reverse(56.838, 60.605)
|
|
||||||
|
|
||||||
assert result is None
|
|
||||||
|
|
||||||
|
|
||||||
async def test_cadastral_reverse_returns_none_when_not_configured() -> None:
|
def test_forward_sync_returns_empty_on_db_error() -> None:
|
||||||
"""main_backend_url is None → immediately returns None without HTTP call."""
|
"""DB raises → empty list, no exception bubbles out."""
|
||||||
with patch("app.services.geocoder.settings") as mock_settings:
|
db = _make_db_error()
|
||||||
mock_settings.main_backend_url = None
|
|
||||||
|
|
||||||
with patch("app.services.geocoder.httpx.AsyncClient") as mock_client_cls:
|
results = _cadastral_forward_sync(db, "Малышева", limit=5)
|
||||||
result = await _cadastral_reverse(56.838, 60.605)
|
|
||||||
mock_client_cls.assert_not_called()
|
|
||||||
|
|
||||||
assert result is None
|
assert results == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_forward_sync_respects_limit() -> None:
|
||||||
|
"""Returns at most `limit` items even if DB has more."""
|
||||||
|
rows = [
|
||||||
|
_make_db_row(
|
||||||
|
readable_address=f"ул. Тестовая, {i}, Екатеринбург",
|
||||||
|
lat=56.838 + i * 0.001,
|
||||||
|
lon=60.605,
|
||||||
|
sim=0.9,
|
||||||
|
cad_num=f"66:41:01010{i:02d}:1",
|
||||||
|
)
|
||||||
|
for i in range(10)
|
||||||
|
]
|
||||||
|
db = _make_db(rows)
|
||||||
|
|
||||||
|
results = _cadastral_forward_sync(db, "Тестовая", limit=3)
|
||||||
|
|
||||||
|
assert len(results) <= 3
|
||||||
|
|
||||||
|
|
||||||
|
# ── _cadastral_reverse_sync ───────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def test_reverse_sync_returns_address_when_row_found() -> None:
|
||||||
|
"""DB finds building within radius → address string returned."""
|
||||||
|
row = MagicMock()
|
||||||
|
row.readable_address = "ул. Хохрякова, 81, Екатеринбург"
|
||||||
|
|
||||||
|
db = MagicMock()
|
||||||
|
result = MagicMock()
|
||||||
|
result.first.return_value = row
|
||||||
|
db.execute.return_value = result
|
||||||
|
|
||||||
|
addr = _cadastral_reverse_sync(db, 56.838, 60.605, radius_m=200)
|
||||||
|
|
||||||
|
assert addr == "ул. Хохрякова, 81, Екатеринбург"
|
||||||
|
|
||||||
|
|
||||||
|
def test_reverse_sync_returns_none_when_no_row_in_radius() -> None:
|
||||||
|
"""No row within radius → None returned."""
|
||||||
|
db = MagicMock()
|
||||||
|
result = MagicMock()
|
||||||
|
result.first.return_value = None
|
||||||
|
db.execute.return_value = result
|
||||||
|
|
||||||
|
addr = _cadastral_reverse_sync(db, 56.838, 60.605, radius_m=200)
|
||||||
|
|
||||||
|
assert addr is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_reverse_sync_returns_none_on_db_error() -> None:
|
||||||
|
"""DB raises → None returned, no exception bubbles out."""
|
||||||
|
db = _make_db_error()
|
||||||
|
|
||||||
|
addr = _cadastral_reverse_sync(db, 56.838, 60.605)
|
||||||
|
|
||||||
|
assert addr is None
|
||||||
|
|
||||||
|
|
||||||
# ── reverse_geocode ───────────────────────────────────────────────────────────
|
# ── reverse_geocode ───────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
async def test_reverse_geocode_prefers_cadastral_over_nominatim() -> None:
|
||||||
async def test_reverse_geocode_returns_cadastral_when_available() -> None:
|
|
||||||
"""Cadastral returns address → Nominatim is never called."""
|
"""Cadastral returns address → Nominatim is never called."""
|
||||||
|
db = MagicMock()
|
||||||
with (
|
with (
|
||||||
patch(
|
patch(
|
||||||
"app.services.geocoder._cadastral_reverse",
|
"app.services.geocoder._cadastral_reverse_sync",
|
||||||
new_callable=AsyncMock,
|
|
||||||
return_value="ул. Малышева, 30, Екатеринбург",
|
return_value="ул. Малышева, 30, Екатеринбург",
|
||||||
) as mock_cadastral,
|
) as mock_cad,
|
||||||
patch(
|
patch(
|
||||||
"app.services.geocoder._nominatim_reverse",
|
"app.services.geocoder._nominatim_reverse",
|
||||||
new_callable=AsyncMock,
|
new_callable=AsyncMock,
|
||||||
) as mock_nominatim,
|
) as mock_nom,
|
||||||
):
|
):
|
||||||
result = await reverse_geocode(56.838, 60.605)
|
result = await reverse_geocode(56.838, 60.605, db=db)
|
||||||
|
|
||||||
assert result == "ул. Малышева, 30, Екатеринбург"
|
assert result == "ул. Малышева, 30, Екатеринбург"
|
||||||
mock_cadastral.assert_called_once_with(56.838, 60.605)
|
mock_cad.assert_called_once_with(db, 56.838, 60.605)
|
||||||
mock_nominatim.assert_not_called()
|
mock_nom.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
async def test_reverse_geocode_falls_back_to_nominatim_when_cadastral_none() -> None:
|
async def test_reverse_geocode_falls_back_to_nominatim_when_cadastral_none() -> None:
|
||||||
"""Cadastral returns None → Nominatim is called and its result returned."""
|
"""Cadastral returns None → Nominatim called and result returned."""
|
||||||
|
db = MagicMock()
|
||||||
with (
|
with (
|
||||||
patch(
|
patch("app.services.geocoder._cadastral_reverse_sync", return_value=None),
|
||||||
"app.services.geocoder._cadastral_reverse",
|
|
||||||
new_callable=AsyncMock,
|
|
||||||
return_value=None,
|
|
||||||
),
|
|
||||||
patch(
|
patch(
|
||||||
"app.services.geocoder._nominatim_reverse",
|
"app.services.geocoder._nominatim_reverse",
|
||||||
new_callable=AsyncMock,
|
new_callable=AsyncMock,
|
||||||
return_value="Трамвайный переулок, 2, Екатеринбург",
|
return_value="Трамвайный переулок, 2, Екатеринбург",
|
||||||
) as mock_nominatim,
|
) as mock_nom,
|
||||||
):
|
):
|
||||||
result = await reverse_geocode(56.838, 60.605)
|
result = await reverse_geocode(56.838, 60.605, db=db)
|
||||||
|
|
||||||
assert result == "Трамвайный переулок, 2, Екатеринбург"
|
assert result == "Трамвайный переулок, 2, Екатеринбург"
|
||||||
mock_nominatim.assert_called_once_with(56.838, 60.605)
|
mock_nom.assert_called_once_with(56.838, 60.605)
|
||||||
|
|
||||||
|
|
||||||
async def test_reverse_geocode_returns_none_when_both_fail() -> None:
|
async def test_reverse_geocode_returns_none_when_both_fail() -> None:
|
||||||
"""Both cadastral and Nominatim fail → returns None, no exception raised."""
|
"""Both cadastral and Nominatim fail → returns None, no exception raised (500 fix)."""
|
||||||
|
db = MagicMock()
|
||||||
with (
|
with (
|
||||||
patch(
|
patch("app.services.geocoder._cadastral_reverse_sync", return_value=None),
|
||||||
"app.services.geocoder._cadastral_reverse",
|
|
||||||
new_callable=AsyncMock,
|
|
||||||
return_value=None,
|
|
||||||
),
|
|
||||||
patch(
|
patch(
|
||||||
"app.services.geocoder._nominatim_reverse",
|
"app.services.geocoder._nominatim_reverse",
|
||||||
new_callable=AsyncMock,
|
new_callable=AsyncMock,
|
||||||
side_effect=RuntimeError("Nominatim 403 Forbidden — IP banned"),
|
side_effect=RuntimeError("Nominatim 403 Forbidden — IP banned"),
|
||||||
),
|
),
|
||||||
):
|
):
|
||||||
result = await reverse_geocode(56.838, 60.605)
|
result = await reverse_geocode(56.838, 60.605, db=db)
|
||||||
|
|
||||||
# Must return None, not raise
|
# Must return None, not raise
|
||||||
assert result is None
|
assert result is None
|
||||||
|
|
||||||
|
|
||||||
|
async def test_reverse_geocode_without_db_skips_cadastral() -> None:
|
||||||
|
"""When db=None, cadastral tier is skipped entirely → goes straight to Nominatim."""
|
||||||
|
with (
|
||||||
|
patch(
|
||||||
|
"app.services.geocoder._cadastral_reverse_sync",
|
||||||
|
) as mock_cad,
|
||||||
|
patch(
|
||||||
|
"app.services.geocoder._nominatim_reverse",
|
||||||
|
new_callable=AsyncMock,
|
||||||
|
return_value="ул. Ленина, 1, Екатеринбург",
|
||||||
|
),
|
||||||
|
):
|
||||||
|
result = await reverse_geocode(56.838, 60.605, db=None)
|
||||||
|
|
||||||
|
mock_cad.assert_not_called()
|
||||||
|
assert result == "ул. Ленина, 1, Екатеринбург"
|
||||||
|
|
||||||
|
|
||||||
|
# ── geocode: cadastral as first tier ─────────────────────────────────────────
|
||||||
|
|
||||||
|
async def test_geocode_uses_cadastral_before_yandex() -> None:
|
||||||
|
"""Cadastral FDW returns suggestion → geocode returns it without hitting Yandex."""
|
||||||
|
db = MagicMock()
|
||||||
|
# Cache miss
|
||||||
|
db.execute.side_effect = [
|
||||||
|
MagicMock(fetchone=lambda: None), # _cache_get
|
||||||
|
]
|
||||||
|
|
||||||
|
cad_suggestion = GeocodeSuggestion(
|
||||||
|
label="ул. Хохрякова, 81",
|
||||||
|
full_address="ул. Хохрякова, 81, Екатеринбург",
|
||||||
|
lat=56.840,
|
||||||
|
lon=60.610,
|
||||||
|
kind="house",
|
||||||
|
)
|
||||||
|
|
||||||
|
with (
|
||||||
|
patch("app.services.geocoder._cache_get", return_value=None),
|
||||||
|
patch(
|
||||||
|
"app.services.geocoder._cadastral_forward_sync",
|
||||||
|
return_value=[cad_suggestion],
|
||||||
|
) as mock_cad,
|
||||||
|
patch("app.services.geocoder._cache_put"),
|
||||||
|
patch("app.services.geocoder._yandex_lookup", new_callable=AsyncMock) as mock_yandex,
|
||||||
|
):
|
||||||
|
result = await geocode("Хохрякова 81", db)
|
||||||
|
|
||||||
|
assert result is not None
|
||||||
|
assert result.lat == 56.840
|
||||||
|
assert result.lon == 60.610
|
||||||
|
assert result.confidence == "exact"
|
||||||
|
mock_cad.assert_called_once()
|
||||||
|
mock_yandex.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
|
# ── suggest: cadastral as first tier ─────────────────────────────────────────
|
||||||
|
|
||||||
|
async def test_suggest_uses_cadastral_before_yandex() -> None:
|
||||||
|
"""Cadastral FDW returns results → suggest returns them without hitting Yandex."""
|
||||||
|
db = MagicMock()
|
||||||
|
cad_suggestions = [
|
||||||
|
GeocodeSuggestion(
|
||||||
|
label="ул. Малышева, 30",
|
||||||
|
full_address="ул. Малышева, 30, Екатеринбург",
|
||||||
|
lat=56.838,
|
||||||
|
lon=60.605,
|
||||||
|
kind="house",
|
||||||
|
)
|
||||||
|
]
|
||||||
|
|
||||||
|
with (
|
||||||
|
patch(
|
||||||
|
"app.services.geocoder._cadastral_forward_sync",
|
||||||
|
return_value=cad_suggestions,
|
||||||
|
) as mock_cad,
|
||||||
|
patch(
|
||||||
|
"app.services.geocoder._yandex_suggest",
|
||||||
|
new_callable=AsyncMock,
|
||||||
|
) as mock_yandex,
|
||||||
|
):
|
||||||
|
results = await suggest("Малышева", db=db, limit=8)
|
||||||
|
|
||||||
|
assert len(results) == 1
|
||||||
|
assert results[0].full_address == "ул. Малышева, 30, Екатеринбург"
|
||||||
|
mock_cad.assert_called_once()
|
||||||
|
mock_yandex.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
|
async def test_suggest_falls_back_to_yandex_when_cadastral_empty() -> None:
|
||||||
|
"""Cadastral returns [] → falls through to Yandex."""
|
||||||
|
db = MagicMock()
|
||||||
|
yandex_result = GeocodeSuggestion(
|
||||||
|
label="Малышева · Октябрьский",
|
||||||
|
full_address="улица Малышева, Екатеринбург",
|
||||||
|
lat=56.838,
|
||||||
|
lon=60.605,
|
||||||
|
kind="street",
|
||||||
|
)
|
||||||
|
|
||||||
|
with (
|
||||||
|
patch("app.services.geocoder._cadastral_forward_sync", return_value=[]),
|
||||||
|
patch("app.services.geocoder.settings") as mock_settings,
|
||||||
|
patch(
|
||||||
|
"app.services.geocoder._yandex_suggest",
|
||||||
|
new_callable=AsyncMock,
|
||||||
|
return_value=[yandex_result],
|
||||||
|
) as mock_yandex,
|
||||||
|
):
|
||||||
|
mock_settings.yandex_geocoder_key = "fake-key"
|
||||||
|
results = await suggest("Малышева", db=db, limit=8)
|
||||||
|
|
||||||
|
assert len(results) == 1
|
||||||
|
mock_yandex.assert_called_once()
|
||||||
|
|
|
||||||
|
|
@ -1,78 +0,0 @@
|
||||||
#!/usr/bin/env bash
|
|
||||||
# Импорт кадастра зданий ЕКБ из gendesign-БД в tradein cad_buildings (#393).
|
|
||||||
#
|
|
||||||
# Источник: gendesign-postgres-1 / cad_buildings (~37k зданий, 36k по ЕКБ).
|
|
||||||
# Берём ЕКБ (cad_num LIKE '66:41:%') с годом постройки или этажностью.
|
|
||||||
# Координаты — центроид geom. Используется как источник обогащения
|
|
||||||
# house_metadata (год / этажность) — точнее OSM.
|
|
||||||
#
|
|
||||||
# Запуск на прод-хосте: ./import-cadastre.sh
|
|
||||||
# Повторяемо: upsert по cad_num (PRIMARY KEY), новый запуск обновит данные.
|
|
||||||
|
|
||||||
set -euo pipefail
|
|
||||||
|
|
||||||
SRC_PG="${SRC_PG:-gendesign-postgres-1}"
|
|
||||||
DST_PG="${DST_PG:-tradein-postgres}"
|
|
||||||
SRC_DB="${SRC_DB:-gendesign}"
|
|
||||||
SRC_USER="${SRC_USER:-gendesign}"
|
|
||||||
|
|
||||||
echo "[$(date -u +%H:%M:%S)] import-cadastre: здания ЕКБ из cad_buildings"
|
|
||||||
|
|
||||||
# 1. Staging-таблица в tradein.
|
|
||||||
docker exec "$DST_PG" psql -U tradein -d tradein -v ON_ERROR_STOP=on -c "
|
|
||||||
CREATE TABLE IF NOT EXISTS cad_buildings_staging (
|
|
||||||
cad_num text PRIMARY KEY, address text, floors int, year_built int,
|
|
||||||
area_m2 numeric, purpose text, lat double precision, lon double precision
|
|
||||||
);
|
|
||||||
TRUNCATE cad_buildings_staging;
|
|
||||||
"
|
|
||||||
|
|
||||||
# 2. Трансформируем на gendesign-стороне, пайпим CSV в staging.
|
|
||||||
# Центроид geom → lat/lon (ST_Transform на 4326 на случай иного SRID).
|
|
||||||
docker exec "$SRC_PG" psql -U "$SRC_USER" -d "$SRC_DB" -v ON_ERROR_STOP=on -c "
|
|
||||||
COPY (
|
|
||||||
SELECT
|
|
||||||
cad_num,
|
|
||||||
NULLIF(trim(readable_address), '') AS address,
|
|
||||||
NULLIF(floors, 0) AS floors,
|
|
||||||
CASE WHEN year_built BETWEEN 1800 AND 2100
|
|
||||||
THEN year_built END AS year_built,
|
|
||||||
round(COALESCE(build_record_area, area), 2) AS area_m2,
|
|
||||||
purpose,
|
|
||||||
ST_Y(ST_Centroid(ST_Transform(geom, 4326))) AS lat,
|
|
||||||
ST_X(ST_Centroid(ST_Transform(geom, 4326))) AS lon
|
|
||||||
FROM cad_buildings
|
|
||||||
WHERE cad_num LIKE '66:41:%'
|
|
||||||
AND cad_num IS NOT NULL
|
|
||||||
AND (year_built IS NOT NULL OR floors IS NOT NULL)
|
|
||||||
) TO STDOUT WITH CSV
|
|
||||||
" | docker exec -i "$DST_PG" psql -U tradein -d tradein -v ON_ERROR_STOP=on -c "
|
|
||||||
COPY cad_buildings_staging FROM STDIN WITH CSV
|
|
||||||
"
|
|
||||||
|
|
||||||
STAGED=$(docker exec "$DST_PG" psql -U tradein -d tradein -tAc \
|
|
||||||
"SELECT count(*) FROM cad_buildings_staging")
|
|
||||||
echo "[$(date -u +%H:%M:%S)] staged: $STAGED зданий"
|
|
||||||
|
|
||||||
# 3. Upsert в cad_buildings (trigger проставит geom из lat/lon).
|
|
||||||
docker exec "$DST_PG" psql -U tradein -d tradein -v ON_ERROR_STOP=on -c "
|
|
||||||
INSERT INTO cad_buildings
|
|
||||||
(cad_num, address, floors, year_built, area_m2, purpose, lat, lon)
|
|
||||||
SELECT cad_num, address, floors, year_built, area_m2, purpose, lat, lon
|
|
||||||
FROM cad_buildings_staging
|
|
||||||
ON CONFLICT (cad_num) DO UPDATE SET
|
|
||||||
address = EXCLUDED.address,
|
|
||||||
floors = EXCLUDED.floors,
|
|
||||||
year_built = EXCLUDED.year_built,
|
|
||||||
area_m2 = EXCLUDED.area_m2,
|
|
||||||
purpose = EXCLUDED.purpose,
|
|
||||||
lat = EXCLUDED.lat,
|
|
||||||
lon = EXCLUDED.lon,
|
|
||||||
imported_at = NOW();
|
|
||||||
|
|
||||||
DROP TABLE cad_buildings_staging;
|
|
||||||
"
|
|
||||||
|
|
||||||
TOTAL=$(docker exec "$DST_PG" psql -U tradein -d tradein -tAc \
|
|
||||||
"SELECT count(*) FROM cad_buildings")
|
|
||||||
echo "[$(date -u +%H:%M:%S)] cad_buildings всего: $TOTAL"
|
|
||||||
|
|
@ -29,7 +29,9 @@ services:
|
||||||
timeout: 5s
|
timeout: 5s
|
||||||
retries: 5
|
retries: 5
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
networks: [tradein-net]
|
networks:
|
||||||
|
- tradein-net
|
||||||
|
- gendesign_shared
|
||||||
|
|
||||||
backend:
|
backend:
|
||||||
image: ghcr.io/lekss361/gendesign-tradein-backend:${IMAGE_TAG:-latest}
|
image: ghcr.io/lekss361/gendesign-tradein-backend:${IMAGE_TAG:-latest}
|
||||||
|
|
@ -45,7 +47,7 @@ services:
|
||||||
ADMIN_TOKEN: "${ADMIN_TOKEN:-}"
|
ADMIN_TOKEN: "${ADMIN_TOKEN:-}"
|
||||||
# GlitchTip DSN — мониторинг ошибок (#396). Пусто = выключено.
|
# GlitchTip DSN — мониторинг ошибок (#396). Пусто = выключено.
|
||||||
GLITCHTIP_DSN: "${GLITCHTIP_DSN:-}"
|
GLITCHTIP_DSN: "${GLITCHTIP_DSN:-}"
|
||||||
MAIN_BACKEND_URL: "${MAIN_BACKEND_URL:-http://gendesign-backend:8000}"
|
GENDESIGN_FDW_PASSWORD: "${GENDESIGN_FDW_PASSWORD:-}"
|
||||||
depends_on:
|
depends_on:
|
||||||
postgres:
|
postgres:
|
||||||
condition: service_healthy
|
condition: service_healthy
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue