From 698ef4f00382947b68a735c37c328f39a3640ceb Mon Sep 17 00:00:00 2001 From: lekss361 Date: Sun, 24 May 2026 11:13:44 +0300 Subject: [PATCH] feat(tradein): postgres_fdw live read of gendesign.cad_buildings (replaces snapshot) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces tradein.cad_buildings manual snapshot (36k rows, frozen 2026-05-22) with live postgres_fdw foreign table reading gendesign.v_tradein_cad_buildings directly. Also reverts PR #492 HTTP-based cadastral.py — superseded by FDW. Architecture: - gendesign DB: new role tradein_fdw_reader + flat view v_tradein_cad_buildings (EKB-only slice of cad_buildings with lat/lon flattened from geom) - networks: gendesign-postgres added to gendesign_shared (alias); tradein-postgres added to gendesign_shared for FDW connect - tradein DB: postgres_fdw extension + FOREIGN TABLE gendesign_cad_buildings - tradein backend: startup hook creates/refreshes USER MAPPING with password from env GENDESIGN_FDW_PASSWORD (password rotation handled via restart) - geocoder.py: cadastral primary for forward + reverse + suggest; Yandex/Nominatim fallback. reverse_geocode wraps Nominatim in try/except — no more HTTPStatusError → 500. - house_metadata.py and trade_in.py admin stats switched to foreign table - DROP TABLE cad_buildings in tradein (legacy snapshot removed entirely) - import-cadastre.sh deleted (no manual sync needed) Fixes: - /trade-in/api/v1/geocode/reverse 500 (Nominatim ban → no fallback) - estimate confidence_explanation=address_not_geocoded for addresses in our cadastre (e.g. Хохрякова 81) that Nominatim doesn't return Deploy ordering: main migration 100_tradein_fdw_role.sql adds role+view first (strict deploy.yml). Tradein next deploy applies 060_postgres_fdw_extension.sql and 061_drop_legacy_cad_buildings.sql (idempotent, errors ignored). Tradein backend startup creates USER MAPPING when env var present. Verify post-deploy with: docker exec tradein-postgres psql -U tradein -d tradein -c \ "SELECT count(*) FROM gendesign_cad_buildings" # expect ~36k EKB buildings. --- backend/app/api/v1/cadastral.py | 101 ------ backend/app/main.py | 2 - data/sql/100_tradein_fdw_role.sql | 36 ++ docker-compose.prod.yml | 10 +- tradein-mvp/backend/app/api/v1/geocode.py | 6 +- tradein-mvp/backend/app/api/v1/trade_in.py | 2 +- tradein-mvp/backend/app/core/config.py | 8 +- tradein-mvp/backend/app/core/fdw.py | 65 ++++ tradein-mvp/backend/app/main.py | 12 + tradein-mvp/backend/app/services/geocoder.py | 181 +++++++--- .../backend/app/services/house_metadata.py | 62 ++-- .../backend/data/sql/006_cad_buildings.sql | 34 -- .../data/sql/060_postgres_fdw_extension.sql | 35 ++ .../sql/061_drop_legacy_cad_buildings.sql | 8 + .../tests/services/test_cadastral_reverse.py | 338 +++++++++++++----- tradein-mvp/deploy/import-cadastre.sh | 78 ---- tradein-mvp/docker-compose.prod.yml | 6 +- 17 files changed, 610 insertions(+), 374 deletions(-) delete mode 100644 backend/app/api/v1/cadastral.py create mode 100644 data/sql/100_tradein_fdw_role.sql create mode 100644 tradein-mvp/backend/app/core/fdw.py delete mode 100644 tradein-mvp/backend/data/sql/006_cad_buildings.sql create mode 100644 tradein-mvp/backend/data/sql/060_postgres_fdw_extension.sql create mode 100644 tradein-mvp/backend/data/sql/061_drop_legacy_cad_buildings.sql delete mode 100755 tradein-mvp/deploy/import-cadastre.sh diff --git a/backend/app/api/v1/cadastral.py b/backend/app/api/v1/cadastral.py deleted file mode 100644 index a1b17b0f..00000000 --- a/backend/app/api/v1/cadastral.py +++ /dev/null @@ -1,101 +0,0 @@ -"""Cadastral lookup endpoints — reverse geocode via cad_buildings. - -Used by trade-in backend (and potentially other internal services) to resolve -координаты → адрес жилого дома без зависимости от внешних геокодеров. -""" - -from __future__ import annotations - -import logging -from typing import Annotated - -from fastapi import APIRouter, Depends, HTTPException, Query -from pydantic import BaseModel -from sqlalchemy import text -from sqlalchemy.orm import Session - -from app.core.db import get_db - -logger = logging.getLogger(__name__) - -router = APIRouter() - - -class CadastralReverseResult(BaseModel): - address: str - lat: float - lon: float - distance_m: float - cad_num: str | None = None - - -@router.get("/reverse", response_model=CadastralReverseResult) -async def reverse( - lat: Annotated[float, Query(ge=-90, le=90)], - lon: Annotated[float, Query(ge=-180, le=180)], - radius_m: Annotated[int, Query(ge=10, le=2000)] = 200, - db: Annotated[Session, Depends(get_db)] = None, # type: ignore[assignment] -) -> CadastralReverseResult: - """Координаты → ближайшее здание из cad_buildings. - - Filters out гаражи/СНТ/садовод/товарищества — для trade-in нужны жилые дома. - - Returns 404 если в радиусе нет подходящего здания. - """ - row = db.execute( - text( - """ - SELECT - readable_address, - cad_num, - ST_Y(ST_Centroid(geom)) AS lat, - ST_X(ST_Centroid(geom)) AS lon, - ST_Distance( - geom::geography, - ST_SetSRID( - ST_MakePoint( - CAST(:lon AS double precision), - CAST(:lat AS double precision) - ), - 4326 - )::geography - ) AS distance_m - FROM cad_buildings - WHERE ST_DWithin( - geom::geography, - ST_SetSRID( - ST_MakePoint( - CAST(:lon AS double precision), - CAST(:lat AS double precision) - ), - 4326 - )::geography, - CAST(:radius AS double precision) - ) - AND readable_address !~* '(гараж|снт|садовод|товарищ|уч\\.)' - ORDER BY ST_Distance( - geom::geography, - ST_SetSRID( - ST_MakePoint( - CAST(:lon AS double precision), - CAST(:lat AS double precision) - ), - 4326 - )::geography - ) - LIMIT 1 - """ - ), - {"lat": lat, "lon": lon, "radius": float(radius_m)}, - ).first() - - if row is None: - raise HTTPException(status_code=404, detail="no cadastral building found within radius") - - return CadastralReverseResult( - address=row.readable_address, - lat=float(row.lat), - lon=float(row.lon), - distance_m=round(float(row.distance_m), 1), - cad_num=row.cad_num, - ) diff --git a/backend/app/main.py b/backend/app/main.py index bd7afc5b..07071a70 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -21,7 +21,6 @@ from app.api.v1 import ( admin_scrape, admin_weight_profiles, analytics, - cadastral, concepts, custom_pois, landing, @@ -79,7 +78,6 @@ app.add_middleware( allow_headers=["*"], ) -app.include_router(cadastral.router, prefix="/api/v1/cadastral", tags=["cadastral"]) app.include_router(concepts.router, prefix="/api/v1/concepts", tags=["concepts"]) app.include_router(parcels.router, prefix="/api/v1/parcels", tags=["parcels"]) app.include_router(analytics.router, prefix="/api/v1/analytics", tags=["analytics"]) diff --git a/data/sql/100_tradein_fdw_role.sql b/data/sql/100_tradein_fdw_role.sql new file mode 100644 index 00000000..cfe605f7 --- /dev/null +++ b/data/sql/100_tradein_fdw_role.sql @@ -0,0 +1,36 @@ +-- tradein_fdw_reader role + v_tradein_cad_buildings view +-- Used by tradein-postgres via postgres_fdw to read live cad_buildings. +-- See meta/00_credentials.md for password rotation procedure. +BEGIN; + +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'tradein_fdw_reader') THEN + CREATE ROLE tradein_fdw_reader LOGIN PASSWORD '40473b7c10c7474855c7d1aa98f0fd0a0de2499aef0cbd1fead06c9641e4dcd8'; + END IF; +END$$; + +GRANT CONNECT ON DATABASE gendesign TO tradein_fdw_reader; +GRANT USAGE ON SCHEMA public TO tradein_fdw_reader; + +CREATE OR REPLACE VIEW v_tradein_cad_buildings AS +SELECT + cad_num, + NULLIF(trim(readable_address), '') AS readable_address, + NULLIF(floors, 0) AS floors, + CASE WHEN year_built BETWEEN 1800 AND 2100 THEN year_built END AS year_built, + round(COALESCE(build_record_area, area), 2)::numeric AS area_m2, + purpose, + ST_Y(ST_Centroid(geom))::double precision AS lat, + ST_X(ST_Centroid(geom))::double precision AS lon +FROM cad_buildings +WHERE cad_num LIKE '66:41:%' + AND geom IS NOT NULL + AND readable_address IS NOT NULL; + +GRANT SELECT ON v_tradein_cad_buildings TO tradein_fdw_reader; + +COMMENT ON VIEW v_tradein_cad_buildings IS + 'FDW source for tradein-mvp (postgres_fdw). Flat 8-col EKB-only slice of cad_buildings. tradein.gendesign_cad_buildings maps here.'; + +COMMIT; diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index c121a217..c4d1c455 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -33,6 +33,11 @@ services: interval: 5s timeout: 5s retries: 10 + networks: + default: {} + shared: + aliases: + - gendesign-postgres redis: image: redis:7-alpine @@ -67,11 +72,6 @@ services: timeout: 5s retries: 5 start_period: 20s - networks: - default: {} - shared: - aliases: - - gendesign-backend frontend: image: ghcr.io/lekss361/gendesign-frontend:${IMAGE_TAG:-latest} diff --git a/tradein-mvp/backend/app/api/v1/geocode.py b/tradein-mvp/backend/app/api/v1/geocode.py index 0324b0b1..a9e9a5f8 100644 --- a/tradein-mvp/backend/app/api/v1/geocode.py +++ b/tradein-mvp/backend/app/api/v1/geocode.py @@ -50,6 +50,7 @@ class SuggestResponse(BaseModel): async def suggest_addresses( q: Annotated[str, Query(min_length=2, max_length=200, description="Запрос для автокомплита")], limit: Annotated[int, Query(ge=1, le=15)] = 8, + db: Annotated[Session, Depends(get_db)] = None, # type: ignore[assignment] ) -> SuggestResponse: """Автокомплит адресов в пределах ЕКБ. @@ -60,7 +61,7 @@ async def suggest_addresses( /api/v1/geocode/suggest?q=Малышева /api/v1/geocode/suggest?q=Цвиллинга # → пусто, такой улицы в ЕКБ нет """ - items = await suggest(q, limit=limit) + items = await suggest(q, db=db, limit=limit) return SuggestResponse(items=[ SuggestItem( label=s.label, @@ -74,12 +75,13 @@ async def suggest_addresses( async def reverse( lat: Annotated[float, Query(ge=-90, le=90)], lon: Annotated[float, Query(ge=-180, le=180)], + db: Annotated[Session, Depends(get_db)] = None, # type: ignore[assignment] ) -> dict[str, object]: """Обратный геокодинг — координаты с карты → адрес (map-picker). Пример: /api/v1/geocode/reverse?lat=56.8389&lon=60.6057 """ - address = await reverse_geocode(lat, lon) + address = await reverse_geocode(lat, lon, db=db) if address is None: raise HTTPException(status_code=404, detail="address not found for coordinates") return {"address": address, "lat": lat, "lon": lon} diff --git a/tradein-mvp/backend/app/api/v1/trade_in.py b/tradein-mvp/backend/app/api/v1/trade_in.py index e0001804..f7a33939 100644 --- a/tradein-mvp/backend/app/api/v1/trade_in.py +++ b/tradein-mvp/backend/app/api/v1/trade_in.py @@ -584,7 +584,7 @@ def cache_stats(db: Annotated[Session, Depends(get_db)]) -> dict[str, object]: (SELECT count(*) FROM listings WHERE is_active) AS listings_active, (SELECT max(scraped_at) FROM listings) AS listings_last_scraped, (SELECT count(*) FROM deals) AS deals, - (SELECT count(*) FROM cad_buildings) AS cad_buildings, + (SELECT count(*) FROM gendesign_cad_buildings) AS cad_buildings, (SELECT count(*) FROM house_metadata) AS house_metadata, (SELECT count(*) FROM trade_in_estimates) AS estimates_total """ diff --git a/tradein-mvp/backend/app/core/config.py b/tradein-mvp/backend/app/core/config.py index f3d7bf6f..6d6b422d 100644 --- a/tradein-mvp/backend/app/core/config.py +++ b/tradein-mvp/backend/app/core/config.py @@ -32,10 +32,10 @@ class Settings(BaseSettings): # Redis URL для hot-cache (Phase 3.2). Задаётся через env REDIS_URL. redis_url: str = "redis://localhost:6379/0" - # Main GenDesign backend URL — для cadastral reverse geocode (cad_buildings). - # В prod: http://gendesign-backend:8000 (network alias в docker-compose.prod.yml). - # Пусто = cadastral lookup пропускается, используется только Nominatim. - main_backend_url: str | None = None + # Password for tradein_fdw_reader role — used by backend startup to create/refresh + # USER MAPPING for postgres_fdw → gendesign DB (gendesign_remote server). + # Пусто = USER MAPPING не создаётся, gendesign_cad_buildings не работает (dev). + gendesign_fdw_password: str | None = None settings = Settings() diff --git a/tradein-mvp/backend/app/core/fdw.py b/tradein-mvp/backend/app/core/fdw.py new file mode 100644 index 00000000..49d1e78c --- /dev/null +++ b/tradein-mvp/backend/app/core/fdw.py @@ -0,0 +1,65 @@ +"""postgres_fdw USER MAPPING management for gendesign_remote server. + +USER MAPPING password is in env GENDESIGN_FDW_PASSWORD (not in SQL migration — +would leak password to disk). + +This helper applies idempotent CREATE / ALTER USER MAPPING. Reapplied on every +backend startup so password rotation through .env.runtime is picked up after restart. +""" +from __future__ import annotations + +import logging + +from sqlalchemy import text +from sqlalchemy.orm import Session + +from app.core.config import settings + +logger = logging.getLogger(__name__) + + +def ensure_fdw_user_mapping(db: Session) -> None: + """Idempotent CREATE OR ALTER USER MAPPING for tradein → gendesign_remote. + + Skips silently if password not configured (dev / first deploy). + """ + password = settings.gendesign_fdw_password + if not password: + logger.warning( + "GENDESIGN_FDW_PASSWORD not set — skipping FDW user mapping " + "(gendesign_cad_buildings will fail)" + ) + return + if not isinstance(password, str) or not password.strip(): + logger.warning("GENDESIGN_FDW_PASSWORD empty — skipping FDW user mapping") + return + + # Check existence + exists = db.execute( + text( + "SELECT 1 FROM pg_user_mappings " + "WHERE srvname = 'gendesign_remote' AND usename = current_user" + ) + ).first() + + # Password cannot be a bind parameter in CREATE/ALTER USER MAPPING — must inline. + # Escape single quotes by doubling. + escaped_pw = password.replace("'", "''") + if exists is None: + db.execute( + text( + f"CREATE USER MAPPING FOR CURRENT_USER SERVER gendesign_remote " + f"OPTIONS (user 'tradein_fdw_reader', password '{escaped_pw}')" + ) + ) + logger.info("created FDW user mapping for gendesign_remote") + else: + db.execute( + text( + f"ALTER USER MAPPING FOR CURRENT_USER SERVER gendesign_remote " + f"OPTIONS (SET password '{escaped_pw}')" + ) + ) + logger.info("refreshed FDW user mapping password for gendesign_remote") + + db.commit() diff --git a/tradein-mvp/backend/app/main.py b/tradein-mvp/backend/app/main.py index a32cb039..293f7877 100644 --- a/tradein-mvp/backend/app/main.py +++ b/tradein-mvp/backend/app/main.py @@ -17,6 +17,8 @@ from fastapi.middleware.cors import CORSMiddleware from app.api.v1 import admin, brand, geocode, search, trade_in from app.core.config import settings +from app.core.db import SessionLocal +from app.core.fdw import ensure_fdw_user_mapping from app.core.ratelimit import RateLimitMiddleware from app.services.scheduler import scheduler_loop @@ -41,6 +43,16 @@ if settings.glitchtip_dsn: @asynccontextmanager async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]: + # FDW bootstrap: create/refresh USER MAPPING for gendesign_remote postgres_fdw server. + # Best-effort: failure does not abort startup, just logs. + try: + with SessionLocal() as db: + ensure_fdw_user_mapping(db) + except Exception: + logger.exception( + "FDW user mapping bootstrap failed — cadastral queries may fail" + ) + # Startup: launch scheduler background task scheduler_task = asyncio.create_task(scheduler_loop()) logger.info("FastAPI lifespan: scheduler task spawned") diff --git a/tradein-mvp/backend/app/services/geocoder.py b/tradein-mvp/backend/app/services/geocoder.py index bc8c7982..0b07a47f 100644 --- a/tradein-mvp/backend/app/services/geocoder.py +++ b/tradein-mvp/backend/app/services/geocoder.py @@ -158,7 +158,7 @@ async def _nominatim_query(client: httpx.AsyncClient, address: str) -> dict | No lon_f = float(item["lon"]) if 60.40 <= lon_f <= 60.85 and 56.65 <= lat_f <= 56.95: return item - except Exception: # noqa: BLE001 + except Exception: continue return None @@ -245,7 +245,7 @@ async def _yandex_lookup(address: str, api_key: str) -> GeocodeResult | None: if 60.40 <= lon_f <= 60.85 and 56.65 <= lat_f <= 56.95: best = obj break - except Exception: # noqa: BLE001 + except Exception: continue if best is None: @@ -315,7 +315,7 @@ def _parse_yandex_members(members: list[dict]) -> list[GeocodeSuggestion]: label=label, full_address=full, lat=float(lat_str), lon=float(lon_str), kind=kind, )) - except Exception: # noqa: BLE001 + except Exception: continue return out @@ -445,36 +445,128 @@ async def _nominatim_suggest(query: str, limit: int = 8) -> list[GeocodeSuggesti return out -async def suggest(query: str, limit: int = 8) -> list[GeocodeSuggestion]: - """Автокомплит адресов в ЕКБ. Yandex → Nominatim → []. +def _cadastral_forward_sync( + db: Session, query: str, limit: int = 5 +) -> list[GeocodeSuggestion]: + """Forward search via gendesign_cad_buildings FDW. + Uses ILIKE prefilter (FDW-pushable) + pg_trgm similarity ranking locally. + Returns up to `limit` suggestions, or [] on error / no results. + """ + try: + rows = db.execute( + text(""" + SELECT cad_num, readable_address, lat, lon, + similarity(readable_address, CAST(:q AS text)) AS sim + FROM gendesign_cad_buildings + WHERE readable_address ILIKE CAST('%' || :q || '%' AS text) + ORDER BY sim DESC, length(readable_address) ASC + LIMIT CAST(:lim AS integer) + """), + {"q": query, "lim": limit * 4}, + ).mappings().all() + except Exception: + logger.warning("cadastral forward sync failed for query=%r", query, exc_info=True) + return [] + + out: list[GeocodeSuggestion] = [] + for r in rows[:limit]: + out.append(GeocodeSuggestion( + label=str(r["readable_address"]), + full_address=str(r["readable_address"]), + lat=float(r["lat"]), + lon=float(r["lon"]), + kind="house", + )) + return out + + +def _cadastral_reverse_sync( + db: Session, lat: float, lon: float, radius_m: int = 200 +) -> str | None: + """Reverse lookup via gendesign_cad_buildings FDW. + + bbox prefilter (±0.0025° lat, ±0.005° lon ≈ 280m) is FDW-pushable. + Distance calculated locally on small result set. Filters гаражи/СНТ. + Returns None on no result or error. + """ + try: + row = db.execute( + text(""" + WITH candidates AS ( + SELECT cad_num, readable_address, lat, lon, + 111320.0 * sqrt( + pow(CAST(:lat AS double precision) - lat, 2) + + pow( + cos(radians(CAST(:lat AS double precision))) + * (CAST(:lon AS double precision) - lon), 2 + ) + ) AS dist_m + FROM gendesign_cad_buildings + WHERE lat BETWEEN CAST(:lat AS double precision) - 0.0025 + AND CAST(:lat AS double precision) + 0.0025 + AND lon BETWEEN CAST(:lon AS double precision) - 0.005 + AND CAST(:lon AS double precision) + 0.005 + AND readable_address !~* '(гараж|снт|садовод|товарищ|уч\\.)' + ) + SELECT readable_address, dist_m + FROM candidates + WHERE dist_m < CAST(:radius AS double precision) + ORDER BY dist_m ASC + LIMIT 1 + """), + {"lat": lat, "lon": lon, "radius": float(radius_m)}, + ).first() + except Exception: + logger.warning( + "cadastral reverse sync failed for (%.5f, %.5f)", lat, lon, exc_info=True + ) + return None + + if row is None: + return None + return str(row.readable_address) + + +async def suggest( + query: str, db: Session | None = None, limit: int = 8 +) -> list[GeocodeSuggestion]: + """Автокомплит адресов в ЕКБ. Cadastral FDW → Yandex → Nominatim → []. + + db: если передан — cadastral lookup через gendesign_cad_buildings (первый tier). Без кэша (дешёво, провайдеры толерируют автокомплит-запросы). """ if not query or len(query.strip()) < 2: return [] + # Tier 0: cadastral FDW (если db доступна) — самый быстрый, без внешних запросов + if db is not None: + cad_results = _cadastral_forward_sync(db, query.strip(), limit) + if cad_results: + return cad_results + if settings.yandex_geocoder_key: try: results = await _yandex_suggest(query, settings.yandex_geocoder_key, limit) if results: return results - except Exception: # noqa: BLE001 + except Exception: logger.exception("yandex suggest failed, falling back to nominatim") try: return await _nominatim_suggest(query, limit) - except Exception: # noqa: BLE001 + except Exception: logger.exception("nominatim suggest failed") return [] # ── Public API ─────────────────────────────────────────────────────────────── async def geocode(address: str, db: Session) -> GeocodeResult | None: - """Геокодинг с кэшем. Yandex → Nominatim → None. + """Геокодинг с кэшем. Cadastral FDW → Yandex → Nominatim → None. Args: address: пользовательский ввод (может быть грязным — нормализуем). - db: сессия Postgres для cache lookup/write. + db: сессия Postgres для cache lookup/write и cadastral FDW lookup. Returns: GeocodeResult или None если ни один провайдер не отвечает. @@ -490,12 +582,33 @@ async def geocode(address: str, db: Session) -> GeocodeResult | None: logger.info("geocode cache hit: %s", addr_norm) return cached - # 2. Yandex (если есть key) с typo-fallback + # 2. Cadastral FDW (прямой запрос к gendesign_cad_buildings — без внешнего API) + cad_suggestions = _cadastral_forward_sync(db, address.strip(), limit=1) + if cad_suggestions: + s = cad_suggestions[0] + result = GeocodeResult( + lat=s.lat, + lon=s.lon, + full_address=s.full_address, + provider="nominatim", # treat as "local" — same confidence as nominatim + confidence="exact", + ) + _cache_put(db, addr_norm, result) + logger.info( + "geocode cadastral fdw: %s → (%.5f, %.5f)", addr_norm, result.lat, result.lon + ) + return result + + # 3. Yandex (если есть key) с typo-fallback if settings.yandex_geocoder_key: try: result = await _yandex_lookup(address, settings.yandex_geocoder_key) # Если результат вне ЕКБ — пробуем typo-варианты - in_ekb = result is not None and 60.40 <= result.lon <= 60.85 and 56.65 <= result.lat <= 56.95 + in_ekb = ( + result is not None + and 60.40 <= result.lon <= 60.85 + and 56.65 <= result.lat <= 56.95 + ) if result is not None and in_ekb: _cache_put(db, addr_norm, result) logger.info("geocode yandex: %s → (%.5f, %.5f)", addr_norm, result.lat, result.lon) @@ -504,7 +617,7 @@ async def geocode(address: str, db: Session) -> GeocodeResult | None: for variant in _typo_variants(address, limit=4): try: result = await _yandex_lookup(variant, settings.yandex_geocoder_key) - except Exception: # noqa: BLE001 + except Exception: continue if result is None: continue @@ -515,10 +628,10 @@ async def geocode(address: str, db: Session) -> GeocodeResult | None: addr_norm, variant, result.lat, result.lon, ) return result - except Exception: # noqa: BLE001 + except Exception: logger.exception("yandex geocoder failed, falling back to nominatim") - # 3. Nominatim fallback + # 4. Nominatim fallback try: result = await _nominatim_lookup(address) if result is not None: @@ -529,7 +642,7 @@ async def geocode(address: str, db: Session) -> GeocodeResult | None: # Nominatim rate-limit policy: 1 req/sec — спим после успешного запроса await asyncio.sleep(1.0) return result - except Exception: # noqa: BLE001 + except Exception: logger.exception("nominatim geocoder failed") return None @@ -569,29 +682,6 @@ def _format_reverse_address(addr: dict) -> str | None: return ", ".join(parts) -async def _cadastral_reverse(lat: float, lon: float) -> str | None: - """Reverse via main backend cad_buildings — fast, no external API. - - Returns None on any error (network, 404, timeout, missing config). - """ - if not settings.main_backend_url: - return None - url = settings.main_backend_url.rstrip("/") + "/api/v1/cadastral/reverse" - try: - async with httpx.AsyncClient(timeout=3.0) as client: - response = await client.get(url, params={"lat": str(lat), "lon": str(lon)}) - if response.status_code == 404: - return None - response.raise_for_status() - data = response.json() - addr = data.get("address") - if isinstance(addr, str) and addr.strip(): - return addr - except Exception: - logger.exception("cadastral reverse failed for (%.5f, %.5f)", lat, lon) - return None - - @retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=1, max=8)) async def _nominatim_reverse(lat: float, lon: float) -> str | None: """Nominatim /reverse — extracted for fallback usage. @retry handles transient errors.""" @@ -623,17 +713,20 @@ async def _nominatim_reverse(lat: float, lon: float) -> str | None: return data.get("display_name") -async def reverse_geocode(lat: float, lon: float) -> str | None: - """Cadastral (cad_buildings via main backend) → Nominatim fallback. +async def reverse_geocode(lat: float, lon: float, db: Session | None = None) -> str | None: + """Cadastral FDW → Nominatim fallback. Возвращает None если ни один источник не дал адрес. Endpoint api/v1/geocode/reverse сам выкинет 404. НЕ даёт выйти HTTPStatusError наверх — раньше Nominatim 403 → RetryError → FastAPI 500. + + db: если передан — cadastral lookup через gendesign_cad_buildings FDW (первый tier). """ - # 1. Cadastral primary (fast, no external API, returns жилой дом not POI) - cadastral = await _cadastral_reverse(lat, lon) - if cadastral: - return cadastral + # 1. Cadastral FDW primary (без внешнего API, возвращает жилой дом not POI) + if db is not None: + cadastral = _cadastral_reverse_sync(db, lat, lon) + if cadastral: + return cadastral # 2. Nominatim fallback (wrap to prevent 500 on ban/rate-limit) try: diff --git a/tradein-mvp/backend/app/services/house_metadata.py b/tradein-mvp/backend/app/services/house_metadata.py index e552693c..ff2d1f87 100644 --- a/tradein-mvp/backend/app/services/house_metadata.py +++ b/tradein-mvp/backend/app/services/house_metadata.py @@ -138,34 +138,56 @@ def _cache_put(db: Session, meta: HouseMetadata, raw: dict) -> None: db.commit() -# ── Источник #393: кадастр зданий (cad_buildings, импорт из gendesign-БД) ──── +# ── Источник #393: кадастр зданий (gendesign_cad_buildings FDW) ────────────── def _cad_buildings_get(db: Session, lat: float, lon: float) -> HouseMetadata | None: - """Ближайшее здание из cad_buildings (данные Росреестра) по координатам. + """Ближайшее здание из gendesign_cad_buildings FDW (данные Росреестра) по координатам. Приоритетный источник house_metadata — год / этажность полнее, чем OSM - (~34к домов ЕКБ с годом постройки). Таблицу наполняет import-cadastre.sh. + (~36к домов ЕКБ с годом постройки). Использует bbox+haversine вместо ST_DWithin — + FDW-таблица не имеет PostGIS-колонки geom (только lat/lon скалярные). """ - row = db.execute( - text( - """ - SELECT year_built, floors - FROM cad_buildings - WHERE geom IS NOT NULL - AND (year_built IS NOT NULL OR floors IS NOT NULL) - AND ST_DWithin(geom::geography, ST_MakePoint(:lon, :lat)::geography, :radius) - ORDER BY ST_Distance(geom::geography, ST_MakePoint(:lon, :lat)::geography) - LIMIT 1 - """ - ), - {"lat": lat, "lon": lon, "radius": _CAD_RADIUS_M}, - ).fetchone() + try: + row = db.execute( + text( + """ + WITH candidates AS ( + SELECT year_built, floors, lat, lon, + 111320.0 * sqrt( + pow(CAST(:lat AS double precision) - lat, 2) + + pow( + cos(radians(CAST(:lat AS double precision))) + * (CAST(:lon AS double precision) - lon), 2 + ) + ) AS dist_m + FROM gendesign_cad_buildings + WHERE (year_built IS NOT NULL OR floors IS NOT NULL) + AND lat BETWEEN CAST(:lat AS double precision) - 0.0010 + AND CAST(:lat AS double precision) + 0.0010 + AND lon BETWEEN CAST(:lon AS double precision) - 0.0015 + AND CAST(:lon AS double precision) + 0.0015 + ) + SELECT year_built, floors + FROM candidates + WHERE dist_m < CAST(:radius AS double precision) + ORDER BY dist_m ASC + LIMIT 1 + """ + ), + {"lat": lat, "lon": lon, "radius": float(_CAD_RADIUS_M)}, + ).fetchone() + except Exception: + logger.warning( + "gendesign_cad_buildings lookup failed at (%.5f, %.5f)", lat, lon, exc_info=True + ) + return None + if row is None: return None return HouseMetadata( lat=lat, lon=lon, year_built=row.year_built, total_floors=row.floors, - house_type=None, # cad_buildings.purpose не маппится в panel/brick + house_type=None, # purpose не маппится в panel/brick total_units=None, source="cadastre", ) @@ -245,12 +267,12 @@ async def get_house_metadata(lat: float, lon: float, db: Session) -> HouseMetada lat, lon, meta.year_built, meta.total_floors, meta.house_type, ) return meta - except Exception: # noqa: BLE001 + except Exception: logger.warning( "house_metadata enrichment failed at (%.5f, %.5f)", lat, lon, exc_info=True ) try: db.rollback() - except Exception: # noqa: BLE001 + except Exception: pass return None diff --git a/tradein-mvp/backend/data/sql/006_cad_buildings.sql b/tradein-mvp/backend/data/sql/006_cad_buildings.sql deleted file mode 100644 index 98589ec4..00000000 --- a/tradein-mvp/backend/data/sql/006_cad_buildings.sql +++ /dev/null @@ -1,34 +0,0 @@ --- Trade-In MVP — кадастр зданий ЕКБ (#393). --- Срез cad_buildings из gendesign-БД (данные Росреестра по зданиям), --- импортируется ETL-ом deploy/import-cadastre.sh. --- Основной источник обогащения house_metadata (год / этажность) — --- данные точнее и полнее, чем OSM Overpass. - -BEGIN; - -CREATE TABLE IF NOT EXISTS cad_buildings ( - cad_num text PRIMARY KEY, -- кадастровый номер здания, 66:41:... - address text, - floors int, - year_built int, - area_m2 numeric(12, 2), -- площадь здания (не квартиры) - purpose text, -- Жилое / Нежилое / Многоквартирный дом - lat double precision, - lon double precision, - geom geometry(Point, 4326), - imported_at timestamptz NOT NULL DEFAULT NOW() -); - -CREATE INDEX IF NOT EXISTS cad_buildings_geom_idx ON cad_buildings USING GIST (geom); - --- Авто-update geom из (lat, lon) — переиспользуем функцию из 002_core_tables.sql. -DROP TRIGGER IF EXISTS cad_buildings_set_geom_trg ON cad_buildings; -CREATE TRIGGER cad_buildings_set_geom_trg - BEFORE INSERT OR UPDATE OF lat, lon ON cad_buildings - FOR EACH ROW EXECUTE FUNCTION listings_set_geom(); - -COMMENT ON TABLE cad_buildings IS - '#393 — кадастр зданий ЕКБ (срез cad_buildings из gendesign-БД). ' - 'Приоритетный источник обогащения house_metadata: год постройки / этажность.'; - -COMMIT; diff --git a/tradein-mvp/backend/data/sql/060_postgres_fdw_extension.sql b/tradein-mvp/backend/data/sql/060_postgres_fdw_extension.sql new file mode 100644 index 00000000..a8c88316 --- /dev/null +++ b/tradein-mvp/backend/data/sql/060_postgres_fdw_extension.sql @@ -0,0 +1,35 @@ +-- postgres_fdw extension for tradein → gendesign live reads. +-- Idempotent. Server config only (no USER MAPPING — that's set by backend startup with env password). +BEGIN; + +CREATE EXTENSION IF NOT EXISTS postgres_fdw; + +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_foreign_server WHERE srvname = 'gendesign_remote') THEN + CREATE SERVER gendesign_remote + FOREIGN DATA WRAPPER postgres_fdw + OPTIONS (host 'gendesign-postgres', dbname 'gendesign', port '5432', extensions 'postgis'); + END IF; +END$$; + +-- Drop existing foreign table if mistakenly created with wrong schema (idempotent re-create below) +DROP FOREIGN TABLE IF EXISTS gendesign_cad_buildings; + +CREATE FOREIGN TABLE gendesign_cad_buildings ( + cad_num text, + readable_address text, + year_built integer, + floors integer, + area_m2 numeric, + purpose text, + lat double precision, + lon double precision +) +SERVER gendesign_remote +OPTIONS (schema_name 'public', table_name 'v_tradein_cad_buildings'); + +COMMENT ON FOREIGN TABLE gendesign_cad_buildings IS + 'Live view of gendesign.cad_buildings (EKB only). USER MAPPING managed by backend startup.'; + +COMMIT; diff --git a/tradein-mvp/backend/data/sql/061_drop_legacy_cad_buildings.sql b/tradein-mvp/backend/data/sql/061_drop_legacy_cad_buildings.sql new file mode 100644 index 00000000..64a8640a --- /dev/null +++ b/tradein-mvp/backend/data/sql/061_drop_legacy_cad_buildings.sql @@ -0,0 +1,8 @@ +-- Drop legacy cad_buildings snapshot table (replaced by gendesign_cad_buildings foreign table). +-- Idempotent. +BEGIN; + +DROP TRIGGER IF EXISTS cad_buildings_set_geom_trg ON cad_buildings; +DROP TABLE IF EXISTS cad_buildings CASCADE; + +COMMIT; diff --git a/tradein-mvp/backend/tests/services/test_cadastral_reverse.py b/tradein-mvp/backend/tests/services/test_cadastral_reverse.py index d5585432..6b7c3ee5 100644 --- a/tradein-mvp/backend/tests/services/test_cadastral_reverse.py +++ b/tradein-mvp/backend/tests/services/test_cadastral_reverse.py @@ -1,4 +1,12 @@ -"""Unit tests for cadastral reverse geocode and modified reverse_geocode.""" +"""Unit tests for cadastral FDW SQL-backed geocoder helpers and reverse_geocode. + +Tests cover: +- _cadastral_forward_sync: returns suggestions / empty list / graceful error +- _cadastral_reverse_sync: returns address / None on no result / None on error +- reverse_geocode: cadastral primary, Nominatim fallback, both-fail → None (500-fix) +- geocode: cadastral as tier 0 +- suggest: cadastral as tier 0 +""" from __future__ import annotations import os @@ -7,144 +15,312 @@ from unittest.mock import AsyncMock, MagicMock, patch # DATABASE_URL required by config before any app import. os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test") -os.environ.setdefault("MAIN_BACKEND_URL", "http://gendesign-backend:8000") # WeasyPrint stub — not installed in CI without GTK. _wp_mock = MagicMock() sys.modules.setdefault("weasyprint", _wp_mock) -import pytest -from app.services.geocoder import _cadastral_reverse, reverse_geocode +from app.services.geocoder import ( # noqa: E402 + GeocodeSuggestion, + _cadastral_forward_sync, + _cadastral_reverse_sync, + geocode, + reverse_geocode, + suggest, +) + +# ── helpers ─────────────────────────────────────────────────────────────────── + +def _make_db_row(**kwargs: object) -> MagicMock: + """Mock row object with dict-like access (MappingResult style).""" + row = MagicMock() + row.__getitem__ = lambda self, k: kwargs[k] + for k, v in kwargs.items(): + setattr(row, k, v) + return row -# ── _cadastral_reverse ──────────────────────────────────────────────────────── +def _make_db(rows: list[object]) -> MagicMock: + """Session mock whose execute().mappings().all() returns rows.""" + db = MagicMock() + result = MagicMock() + result.mappings.return_value.all.return_value = rows + result.first.return_value = rows[0] if rows else None + db.execute.return_value = result + return db -async def test_cadastral_reverse_returns_address_on_200() -> None: - """Main backend returns 200 → address string returned.""" - mock_response = MagicMock() - mock_response.status_code = 200 - mock_response.raise_for_status = MagicMock() - mock_response.json.return_value = { - "address": "ул. Малышева, 30, Екатеринбург", - "lat": 56.838, - "lon": 60.605, - "distance_m": 12.5, - } - - mock_client = AsyncMock() - mock_client.get = AsyncMock(return_value=mock_response) - mock_client.__aenter__ = AsyncMock(return_value=mock_client) - mock_client.__aexit__ = AsyncMock(return_value=False) - - with patch("app.services.geocoder.httpx.AsyncClient", return_value=mock_client): - result = await _cadastral_reverse(56.838, 60.605) - - assert result == "ул. Малышева, 30, Екатеринбург" +def _make_db_error() -> MagicMock: + """Session mock whose execute() raises RuntimeError.""" + db = MagicMock() + db.execute.side_effect = RuntimeError("FDW connection failed") + return db -async def test_cadastral_reverse_returns_none_on_404() -> None: - """Main backend returns 404 (no building in radius) → None, no exception.""" - mock_response = MagicMock() - mock_response.status_code = 404 +# ── _cadastral_forward_sync ─────────────────────────────────────────────────── - mock_client = AsyncMock() - mock_client.get = AsyncMock(return_value=mock_response) - mock_client.__aenter__ = AsyncMock(return_value=mock_client) - mock_client.__aexit__ = AsyncMock(return_value=False) +def test_forward_sync_returns_suggestions_when_rows_exist() -> None: + """DB returns rows → list of GeocodeSuggestion returned.""" + row = _make_db_row( + readable_address="ул. Малышева, 30, Екатеринбург", + lat=56.838, + lon=60.605, + sim=0.9, + cad_num="66:41:0101001:123", + ) + db = _make_db([row]) - with patch("app.services.geocoder.httpx.AsyncClient", return_value=mock_client): - result = await _cadastral_reverse(56.838, 60.605) + results = _cadastral_forward_sync(db, "Малышева 30", limit=5) - assert result is None + assert len(results) == 1 + assert results[0].full_address == "ул. Малышева, 30, Екатеринбург" + assert results[0].lat == 56.838 + assert results[0].lon == 60.605 + assert results[0].kind == "house" -async def test_cadastral_reverse_returns_none_on_timeout() -> None: - """Network timeout → None returned, no exception bubbles out.""" - import httpx +def test_forward_sync_returns_empty_when_no_rows() -> None: + """DB returns no rows → empty list returned, no exception.""" + db = _make_db([]) - mock_client = AsyncMock() - mock_client.get = AsyncMock(side_effect=httpx.TimeoutException("timeout")) - mock_client.__aenter__ = AsyncMock(return_value=mock_client) - mock_client.__aexit__ = AsyncMock(return_value=False) + results = _cadastral_forward_sync(db, "Несуществующая улица", limit=5) - with patch("app.services.geocoder.httpx.AsyncClient", return_value=mock_client): - result = await _cadastral_reverse(56.838, 60.605) - - assert result is None + assert results == [] -async def test_cadastral_reverse_returns_none_when_not_configured() -> None: - """main_backend_url is None → immediately returns None without HTTP call.""" - with patch("app.services.geocoder.settings") as mock_settings: - mock_settings.main_backend_url = None +def test_forward_sync_returns_empty_on_db_error() -> None: + """DB raises → empty list, no exception bubbles out.""" + db = _make_db_error() - with patch("app.services.geocoder.httpx.AsyncClient") as mock_client_cls: - result = await _cadastral_reverse(56.838, 60.605) - mock_client_cls.assert_not_called() + results = _cadastral_forward_sync(db, "Малышева", limit=5) - assert result is None + assert results == [] + + +def test_forward_sync_respects_limit() -> None: + """Returns at most `limit` items even if DB has more.""" + rows = [ + _make_db_row( + readable_address=f"ул. Тестовая, {i}, Екатеринбург", + lat=56.838 + i * 0.001, + lon=60.605, + sim=0.9, + cad_num=f"66:41:01010{i:02d}:1", + ) + for i in range(10) + ] + db = _make_db(rows) + + results = _cadastral_forward_sync(db, "Тестовая", limit=3) + + assert len(results) <= 3 + + +# ── _cadastral_reverse_sync ─────────────────────────────────────────────────── + +def test_reverse_sync_returns_address_when_row_found() -> None: + """DB finds building within radius → address string returned.""" + row = MagicMock() + row.readable_address = "ул. Хохрякова, 81, Екатеринбург" + + db = MagicMock() + result = MagicMock() + result.first.return_value = row + db.execute.return_value = result + + addr = _cadastral_reverse_sync(db, 56.838, 60.605, radius_m=200) + + assert addr == "ул. Хохрякова, 81, Екатеринбург" + + +def test_reverse_sync_returns_none_when_no_row_in_radius() -> None: + """No row within radius → None returned.""" + db = MagicMock() + result = MagicMock() + result.first.return_value = None + db.execute.return_value = result + + addr = _cadastral_reverse_sync(db, 56.838, 60.605, radius_m=200) + + assert addr is None + + +def test_reverse_sync_returns_none_on_db_error() -> None: + """DB raises → None returned, no exception bubbles out.""" + db = _make_db_error() + + addr = _cadastral_reverse_sync(db, 56.838, 60.605) + + assert addr is None # ── reverse_geocode ─────────────────────────────────────────────────────────── - -async def test_reverse_geocode_returns_cadastral_when_available() -> None: +async def test_reverse_geocode_prefers_cadastral_over_nominatim() -> None: """Cadastral returns address → Nominatim is never called.""" + db = MagicMock() with ( patch( - "app.services.geocoder._cadastral_reverse", - new_callable=AsyncMock, + "app.services.geocoder._cadastral_reverse_sync", return_value="ул. Малышева, 30, Екатеринбург", - ) as mock_cadastral, + ) as mock_cad, patch( "app.services.geocoder._nominatim_reverse", new_callable=AsyncMock, - ) as mock_nominatim, + ) as mock_nom, ): - result = await reverse_geocode(56.838, 60.605) + result = await reverse_geocode(56.838, 60.605, db=db) assert result == "ул. Малышева, 30, Екатеринбург" - mock_cadastral.assert_called_once_with(56.838, 60.605) - mock_nominatim.assert_not_called() + mock_cad.assert_called_once_with(db, 56.838, 60.605) + mock_nom.assert_not_called() async def test_reverse_geocode_falls_back_to_nominatim_when_cadastral_none() -> None: - """Cadastral returns None → Nominatim is called and its result returned.""" + """Cadastral returns None → Nominatim called and result returned.""" + db = MagicMock() with ( - patch( - "app.services.geocoder._cadastral_reverse", - new_callable=AsyncMock, - return_value=None, - ), + patch("app.services.geocoder._cadastral_reverse_sync", return_value=None), patch( "app.services.geocoder._nominatim_reverse", new_callable=AsyncMock, return_value="Трамвайный переулок, 2, Екатеринбург", - ) as mock_nominatim, + ) as mock_nom, ): - result = await reverse_geocode(56.838, 60.605) + result = await reverse_geocode(56.838, 60.605, db=db) assert result == "Трамвайный переулок, 2, Екатеринбург" - mock_nominatim.assert_called_once_with(56.838, 60.605) + mock_nom.assert_called_once_with(56.838, 60.605) async def test_reverse_geocode_returns_none_when_both_fail() -> None: - """Both cadastral and Nominatim fail → returns None, no exception raised.""" + """Both cadastral and Nominatim fail → returns None, no exception raised (500 fix).""" + db = MagicMock() with ( - patch( - "app.services.geocoder._cadastral_reverse", - new_callable=AsyncMock, - return_value=None, - ), + patch("app.services.geocoder._cadastral_reverse_sync", return_value=None), patch( "app.services.geocoder._nominatim_reverse", new_callable=AsyncMock, side_effect=RuntimeError("Nominatim 403 Forbidden — IP banned"), ), ): - result = await reverse_geocode(56.838, 60.605) + result = await reverse_geocode(56.838, 60.605, db=db) # Must return None, not raise assert result is None + + +async def test_reverse_geocode_without_db_skips_cadastral() -> None: + """When db=None, cadastral tier is skipped entirely → goes straight to Nominatim.""" + with ( + patch( + "app.services.geocoder._cadastral_reverse_sync", + ) as mock_cad, + patch( + "app.services.geocoder._nominatim_reverse", + new_callable=AsyncMock, + return_value="ул. Ленина, 1, Екатеринбург", + ), + ): + result = await reverse_geocode(56.838, 60.605, db=None) + + mock_cad.assert_not_called() + assert result == "ул. Ленина, 1, Екатеринбург" + + +# ── geocode: cadastral as first tier ───────────────────────────────────────── + +async def test_geocode_uses_cadastral_before_yandex() -> None: + """Cadastral FDW returns suggestion → geocode returns it without hitting Yandex.""" + db = MagicMock() + # Cache miss + db.execute.side_effect = [ + MagicMock(fetchone=lambda: None), # _cache_get + ] + + cad_suggestion = GeocodeSuggestion( + label="ул. Хохрякова, 81", + full_address="ул. Хохрякова, 81, Екатеринбург", + lat=56.840, + lon=60.610, + kind="house", + ) + + with ( + patch("app.services.geocoder._cache_get", return_value=None), + patch( + "app.services.geocoder._cadastral_forward_sync", + return_value=[cad_suggestion], + ) as mock_cad, + patch("app.services.geocoder._cache_put"), + patch("app.services.geocoder._yandex_lookup", new_callable=AsyncMock) as mock_yandex, + ): + result = await geocode("Хохрякова 81", db) + + assert result is not None + assert result.lat == 56.840 + assert result.lon == 60.610 + assert result.confidence == "exact" + mock_cad.assert_called_once() + mock_yandex.assert_not_called() + + +# ── suggest: cadastral as first tier ───────────────────────────────────────── + +async def test_suggest_uses_cadastral_before_yandex() -> None: + """Cadastral FDW returns results → suggest returns them without hitting Yandex.""" + db = MagicMock() + cad_suggestions = [ + GeocodeSuggestion( + label="ул. Малышева, 30", + full_address="ул. Малышева, 30, Екатеринбург", + lat=56.838, + lon=60.605, + kind="house", + ) + ] + + with ( + patch( + "app.services.geocoder._cadastral_forward_sync", + return_value=cad_suggestions, + ) as mock_cad, + patch( + "app.services.geocoder._yandex_suggest", + new_callable=AsyncMock, + ) as mock_yandex, + ): + results = await suggest("Малышева", db=db, limit=8) + + assert len(results) == 1 + assert results[0].full_address == "ул. Малышева, 30, Екатеринбург" + mock_cad.assert_called_once() + mock_yandex.assert_not_called() + + +async def test_suggest_falls_back_to_yandex_when_cadastral_empty() -> None: + """Cadastral returns [] → falls through to Yandex.""" + db = MagicMock() + yandex_result = GeocodeSuggestion( + label="Малышева · Октябрьский", + full_address="улица Малышева, Екатеринбург", + lat=56.838, + lon=60.605, + kind="street", + ) + + with ( + patch("app.services.geocoder._cadastral_forward_sync", return_value=[]), + patch("app.services.geocoder.settings") as mock_settings, + patch( + "app.services.geocoder._yandex_suggest", + new_callable=AsyncMock, + return_value=[yandex_result], + ) as mock_yandex, + ): + mock_settings.yandex_geocoder_key = "fake-key" + results = await suggest("Малышева", db=db, limit=8) + + assert len(results) == 1 + mock_yandex.assert_called_once() diff --git a/tradein-mvp/deploy/import-cadastre.sh b/tradein-mvp/deploy/import-cadastre.sh deleted file mode 100755 index 9764fab3..00000000 --- a/tradein-mvp/deploy/import-cadastre.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env bash -# Импорт кадастра зданий ЕКБ из gendesign-БД в tradein cad_buildings (#393). -# -# Источник: gendesign-postgres-1 / cad_buildings (~37k зданий, 36k по ЕКБ). -# Берём ЕКБ (cad_num LIKE '66:41:%') с годом постройки или этажностью. -# Координаты — центроид geom. Используется как источник обогащения -# house_metadata (год / этажность) — точнее OSM. -# -# Запуск на прод-хосте: ./import-cadastre.sh -# Повторяемо: upsert по cad_num (PRIMARY KEY), новый запуск обновит данные. - -set -euo pipefail - -SRC_PG="${SRC_PG:-gendesign-postgres-1}" -DST_PG="${DST_PG:-tradein-postgres}" -SRC_DB="${SRC_DB:-gendesign}" -SRC_USER="${SRC_USER:-gendesign}" - -echo "[$(date -u +%H:%M:%S)] import-cadastre: здания ЕКБ из cad_buildings" - -# 1. Staging-таблица в tradein. -docker exec "$DST_PG" psql -U tradein -d tradein -v ON_ERROR_STOP=on -c " - CREATE TABLE IF NOT EXISTS cad_buildings_staging ( - cad_num text PRIMARY KEY, address text, floors int, year_built int, - area_m2 numeric, purpose text, lat double precision, lon double precision - ); - TRUNCATE cad_buildings_staging; -" - -# 2. Трансформируем на gendesign-стороне, пайпим CSV в staging. -# Центроид geom → lat/lon (ST_Transform на 4326 на случай иного SRID). -docker exec "$SRC_PG" psql -U "$SRC_USER" -d "$SRC_DB" -v ON_ERROR_STOP=on -c " - COPY ( - SELECT - cad_num, - NULLIF(trim(readable_address), '') AS address, - NULLIF(floors, 0) AS floors, - CASE WHEN year_built BETWEEN 1800 AND 2100 - THEN year_built END AS year_built, - round(COALESCE(build_record_area, area), 2) AS area_m2, - purpose, - ST_Y(ST_Centroid(ST_Transform(geom, 4326))) AS lat, - ST_X(ST_Centroid(ST_Transform(geom, 4326))) AS lon - FROM cad_buildings - WHERE cad_num LIKE '66:41:%' - AND cad_num IS NOT NULL - AND (year_built IS NOT NULL OR floors IS NOT NULL) - ) TO STDOUT WITH CSV -" | docker exec -i "$DST_PG" psql -U tradein -d tradein -v ON_ERROR_STOP=on -c " - COPY cad_buildings_staging FROM STDIN WITH CSV -" - -STAGED=$(docker exec "$DST_PG" psql -U tradein -d tradein -tAc \ - "SELECT count(*) FROM cad_buildings_staging") -echo "[$(date -u +%H:%M:%S)] staged: $STAGED зданий" - -# 3. Upsert в cad_buildings (trigger проставит geom из lat/lon). -docker exec "$DST_PG" psql -U tradein -d tradein -v ON_ERROR_STOP=on -c " - INSERT INTO cad_buildings - (cad_num, address, floors, year_built, area_m2, purpose, lat, lon) - SELECT cad_num, address, floors, year_built, area_m2, purpose, lat, lon - FROM cad_buildings_staging - ON CONFLICT (cad_num) DO UPDATE SET - address = EXCLUDED.address, - floors = EXCLUDED.floors, - year_built = EXCLUDED.year_built, - area_m2 = EXCLUDED.area_m2, - purpose = EXCLUDED.purpose, - lat = EXCLUDED.lat, - lon = EXCLUDED.lon, - imported_at = NOW(); - - DROP TABLE cad_buildings_staging; -" - -TOTAL=$(docker exec "$DST_PG" psql -U tradein -d tradein -tAc \ - "SELECT count(*) FROM cad_buildings") -echo "[$(date -u +%H:%M:%S)] cad_buildings всего: $TOTAL" diff --git a/tradein-mvp/docker-compose.prod.yml b/tradein-mvp/docker-compose.prod.yml index f1b55f33..8f894833 100644 --- a/tradein-mvp/docker-compose.prod.yml +++ b/tradein-mvp/docker-compose.prod.yml @@ -29,7 +29,9 @@ services: timeout: 5s retries: 5 restart: unless-stopped - networks: [tradein-net] + networks: + - tradein-net + - gendesign_shared backend: image: ghcr.io/lekss361/gendesign-tradein-backend:${IMAGE_TAG:-latest} @@ -45,7 +47,7 @@ services: ADMIN_TOKEN: "${ADMIN_TOKEN:-}" # GlitchTip DSN — мониторинг ошибок (#396). Пусто = выключено. GLITCHTIP_DSN: "${GLITCHTIP_DSN:-}" - MAIN_BACKEND_URL: "${MAIN_BACKEND_URL:-http://gendesign-backend:8000}" + GENDESIGN_FDW_PASSWORD: "${GENDESIGN_FDW_PASSWORD:-}" depends_on: postgres: condition: service_healthy