feat(tradein): DaData /suggest tier в /suggest endpoint (PR Q2) #605

Merged
Light1YT merged 1 commit from feat/tradein-dadata-suggest into main 2026-05-27 14:09:31 +00:00
4 changed files with 581 additions and 17 deletions

View file

@ -1,21 +1,23 @@
"""DaData /clean/address client — enrich address с canonical form,
kadastr_num, ФИАС, координаты, метро.
"""DaData clients — /clean/address (enrichment) + /suggest/address (autocomplete).
Используется в estimator flow для on-demand обогащения **целевого адреса**
(payload.address) заполняет недостающий kadastr_num, который open dataset
Росреестра не даёт (см. PR Q1 / vault `meta/00_credentials.md`).
/clean/address обогащение **целевого адреса** канонической формой,
kadastr_num, ФИАС, координаты, метро. Используется в estimator flow для
on-demand обогащения payload.address (PR Q1). Требует token + secret,
demo tier 100/день.
/suggest/address автокомплит для /api/v1/geocode/suggest (PR Q2).
Заменяет заблокированный Yandex Suggest как второй tier после Cadastral FDW.
Token-only (X-Secret не нужен), 10k/день free. Возвращает список candidate'ов
с координатами + ФИАС + структурными полями.
ENV: DADATA_API_TOKEN, DADATA_API_SECRET.
Если не задан хотя бы один service возвращает None gracefully (не break flow).
- clean_address: требует оба.
- suggest_addresses: требует только DADATA_API_TOKEN.
Если требуемые credentials не заданы graceful возврат None / [] (не break flow).
Rate limit: 100/день demo tier. Для production надо upgrade'нуть.
Endpoint: POST https://cleaner.dadata.ru/api/v1/clean/address
Headers: Authorization: Token <T>, X-Secret: <S>, Content-Type: application/json
Body: ["<address text>"]
Response: list[dict] обычно один элемент с полями result/house_cadnum/geo_lat/...
Docs: https://dadata.ru/api/clean/address/
Docs:
- https://dadata.ru/api/clean/address/
- https://dadata.ru/api/suggest/address/
"""
from __future__ import annotations
@ -31,7 +33,9 @@ from app.core.config import settings
logger = logging.getLogger(__name__)
DADATA_CLEAN_URL = "https://cleaner.dadata.ru/api/v1/clean/address"
DADATA_SUGGEST_URL = "https://suggestions.dadata.ru/suggestions/api/4_1/rs/suggest/address"
_DADATA_TIMEOUT_S = 8.0
_DADATA_SUGGEST_TIMEOUT_S = 5.0
@dataclass(frozen=True, slots=True)
@ -212,3 +216,168 @@ async def clean_address(address: str) -> DadataAddressResult | None:
result.qc_house,
)
return result
# ── Suggest (autocomplete) ───────────────────────────────────────────────────
@dataclass(frozen=True, slots=True)
class DadataSuggestion:
"""Один candidate из ответа DaData /suggest/address.
value короткая строка для отображения («ул Малышева, д 30»).
unrestricted_value полная строка с регионом/страной («г Екатеринбург, ул »).
lat / lon координаты (если DaData их вернул; для some сегментов None).
fias_id UUID ФИАС матчнутого объекта (дома/улицы/города).
house номер дома если distinguished (строкой: «30» / «30к2»).
street название улицы.
city locality.
kind `house` (fias_level=8) / `street` (=7) / `city` (<=6).
Маппится на GeocodeSuggestion.kind в geocoder.py.
"""
value: str
unrestricted_value: str
lat: float | None
lon: float | None
fias_id: str | None
house: str | None
street: str | None
city: str | None
kind: str
def _classify_kind(fias_level: Any) -> str:
"""Маппит DaData `fias_level` → kind для GeocodeSuggestion.
DaData fias_level (https://dadata.ru/api/suggest/address/#fias_level):
0=country, 1=region, 3=area, 4=city, 5=settlement, 6=street_alt,
7=street, 8=house, 9=flat, 65=planning, 75=plot.
Для autocomplete нам важно: house / street / city.
"""
level = _coerce_int(fias_level)
if level is None:
return "city" # safe default
if level >= 8:
return "house"
if level == 7:
return "street"
return "city"
def _parse_suggestion(item: dict[str, Any]) -> DadataSuggestion | None:
"""Парсит один элемент `suggestions[]` → DadataSuggestion.
Возвращает None если shape невалидный (нет value / нет data).
"""
if not isinstance(item, dict):
return None
value = item.get("value")
if not value or not isinstance(value, str):
return None
data = item.get("data") or {}
if not isinstance(data, dict):
data = {}
return DadataSuggestion(
value=value,
unrestricted_value=str(item.get("unrestricted_value") or value),
lat=_coerce_float(data.get("geo_lat")),
lon=_coerce_float(data.get("geo_lon")),
fias_id=data.get("fias_id"),
house=data.get("house"),
street=data.get("street"),
city=data.get("city"),
kind=_classify_kind(data.get("fias_level")),
)
async def suggest_addresses(
query: str, limit: int = 8, city: str = "Екатеринбург"
) -> list[DadataSuggestion]:
"""Автокомплит адресов через DaData /suggest/address.
Args:
query: текст для подсказки (минимум 2 символа).
limit: сколько вариантов вернуть (DaData параметр `count`, max 20).
city: locality для constraint (передаётся в `locations`). По умолчанию ЕКБ.
Returns:
list[DadataSuggestion] пустой список если:
- DADATA_API_TOKEN не задан (graceful disable)
- query короче 2 символов
- сетевая / HTTP ошибка / 429 quota / malformed response
Token-only auth (X-Secret для suggest не нужен отличается от clean_address).
Quota: 10000/день на free tier (в 100x больше чем у clean_address).
"""
if not query or len(query.strip()) < 2:
return []
token = (settings.dadata_api_token or "").strip()
if not token:
logger.debug("dadata suggest: DADATA_API_TOKEN не задан — skip")
return []
headers = {
"Authorization": f"Token {token}",
"Content-Type": "application/json",
"Accept": "application/json",
}
body: dict[str, Any] = {
"query": query.strip(),
"count": max(1, min(int(limit), 20)),
}
if city:
body["locations"] = [{"city": city}]
try:
async with httpx.AsyncClient(timeout=_DADATA_SUGGEST_TIMEOUT_S) as client:
response = await client.post(DADATA_SUGGEST_URL, headers=headers, json=body)
except (httpx.TimeoutException, httpx.NetworkError) as exc:
logger.warning("dadata suggest: network error для %r: %s", query[:60], exc)
return []
except Exception as exc: # pragma: no cover — defensive
logger.warning("dadata suggest: unexpected client error для %r: %s", query[:60], exc)
return []
status = response.status_code
if status == 429:
logger.warning("dadata suggest: HTTP 429 — quota exceeded (10k/день free?)")
return []
if status in (401, 403):
logger.error(
"dadata suggest: HTTP %d — auth rejected. Проверь DADATA_API_TOKEN.", status
)
return []
if status >= 500:
logger.warning("dadata suggest: HTTP %d — transient server error", status)
return []
if status >= 400:
body_preview = (response.text or "")[:200]
logger.warning("dadata suggest: HTTP %d — bad request: %r", status, body_preview)
return []
try:
payload = response.json()
except ValueError as exc:
logger.warning("dadata suggest: невалидный JSON в ответе: %s", exc)
return []
if not isinstance(payload, dict):
logger.warning("dadata suggest: response не dict (%s)", type(payload).__name__)
return []
raw_suggestions = payload.get("suggestions")
if not isinstance(raw_suggestions, list):
logger.info("dadata suggest: нет ключа `suggestions` в ответе для %r", query[:60])
return []
out: list[DadataSuggestion] = []
for item in raw_suggestions:
parsed = _parse_suggestion(item)
if parsed is not None:
out.append(parsed)
logger.info("dadata suggest: %r%d вариантов", query[:60], len(out))
return out

View file

@ -24,6 +24,7 @@ from sqlalchemy.orm import Session
from tenacity import retry, stop_after_attempt, wait_exponential
from app.core.config import settings
from app.services import dadata
logger = logging.getLogger(__name__)
@ -320,6 +321,36 @@ def _parse_yandex_members(members: list[dict]) -> list[GeocodeSuggestion]:
return out
# ── DaData suggest wrapper ──────────────────────────────────────────────────
# Маппинг DaData kind → GeocodeSuggestion.kind (внутри geocoder используется
# 'locality' вместо 'city' — consistent с Yandex/Nominatim ветками).
_DADATA_KIND_MAP = {"house": "house", "street": "street", "city": "locality"}
async def _dadata_suggest(query: str, limit: int = 8) -> list[GeocodeSuggestion]:
"""Обёртка над `dadata.suggest_addresses` — конвертит в GeocodeSuggestion.
Дроп candidate'ов без координат (DaData возвращает их для широких categories
типа город/район, для autocomplete с привязкой к карте они бесполезны).
Label собирается из DaData `value` (короткая форма «ул Малышева, д 30»).
Без city suffix query уже ограничен `locations=[{city: 'Екатеринбург'}]`.
"""
raw = await dadata.suggest_addresses(query, limit=limit)
out: list[GeocodeSuggestion] = []
for s in raw:
if s.lat is None or s.lon is None:
continue
out.append(GeocodeSuggestion(
label=s.value,
full_address=s.unrestricted_value,
lat=s.lat,
lon=s.lon,
kind=_DADATA_KIND_MAP.get(s.kind, "locality"),
))
return out
async def _yandex_geocode_request(
client: httpx.AsyncClient, api_key: str, query: str, limit: int, bounded: bool
) -> list[dict]:
@ -531,20 +562,34 @@ def _cadastral_reverse_sync(
async def suggest(
query: str, db: Session | None = None, limit: int = 8
) -> list[GeocodeSuggestion]:
"""Автокомплит адресов в ЕКБ. Cadastral FDW → Yandex → Nominatim → [].
"""Автокомплит адресов в ЕКБ. Cadastral FDW → DaData → Yandex → Nominatim → [].
db: если передан cadastral lookup через gendesign_cad_buildings (первый tier).
DaData /suggest (PR Q2) token-only, 10k/день, заменяет Yandex который
заблокирован (1k/день demo limit исчерпан).
Без кэша (дешёво, провайдеры толерируют автокомплит-запросы).
"""
if not query or len(query.strip()) < 2:
return []
# Tier 0: cadastral FDW (если db доступна) — самый быстрый, без внешних запросов
# Tier 1: cadastral FDW (если db доступна) — самый быстрый, без внешних запросов
if db is not None:
cad_results = await asyncio.to_thread(_cadastral_forward_sync, db, query.strip(), limit)
if cad_results:
return cad_results
# Tier 2: DaData /suggest — token-only (X-Secret не нужен), 10k/день free,
# лучший fit для РФ адресов. Заменил Yandex как primary external provider
# после того как demo-key Yandex (1k/день) был исчерпан.
if settings.dadata_api_token:
try:
dadata_results = await _dadata_suggest(query, limit)
if dadata_results:
return dadata_results
except Exception:
logger.exception("dadata suggest failed, falling back to yandex")
# Tier 3: Yandex (legacy — оставляем как fallback, если key есть)
if settings.yandex_geocoder_api_key:
try:
results = await _yandex_suggest(query, settings.yandex_geocoder_api_key, limit)
@ -553,6 +598,7 @@ async def suggest(
except Exception:
logger.exception("yandex suggest failed, falling back to nominatim")
# Tier 4: Nominatim (последний fallback — OSM, без ключа)
try:
return await _nominatim_suggest(query, limit)
except Exception:

View file

@ -396,7 +396,7 @@ async def test_suggest_uses_cadastral_before_yandex() -> None:
async def test_suggest_falls_back_to_yandex_when_cadastral_empty() -> None:
"""Cadastral returns [] → falls through to Yandex."""
"""Cadastral returns [] → DaData not configured → falls through to Yandex."""
db = MagicMock()
yandex_result = GeocodeSuggestion(
label="Малышева · Октябрьский",
@ -409,12 +409,18 @@ async def test_suggest_falls_back_to_yandex_when_cadastral_empty() -> None:
with (
patch("app.services.geocoder._cadastral_forward_sync", return_value=[]),
patch("app.services.geocoder.settings") as mock_settings,
patch(
"app.services.geocoder._dadata_suggest",
new_callable=AsyncMock,
return_value=[], # DaData empty → falls through
),
patch(
"app.services.geocoder._yandex_suggest",
new_callable=AsyncMock,
return_value=[yandex_result],
) as mock_yandex,
):
mock_settings.dadata_api_token = "fake-dadata-token" # gate to call _dadata_suggest
mock_settings.yandex_geocoder_api_key = "fake-key"
results = await suggest("Малышева", db=db, limit=8)
@ -422,6 +428,40 @@ async def test_suggest_falls_back_to_yandex_when_cadastral_empty() -> None:
mock_yandex.assert_called_once()
async def test_suggest_uses_dadata_before_yandex_when_cadastral_empty() -> None:
"""Cadastral returns [] → DaData tier returns results → Yandex never called (PR Q2)."""
db = MagicMock()
dadata_result = GeocodeSuggestion(
label="г Екатеринбург, ул Малышева, д 30",
full_address="620075, Свердловская обл, г Екатеринбург, ул Малышева, д 30",
lat=56.838011,
lon=60.601023,
kind="house",
)
with (
patch("app.services.geocoder._cadastral_forward_sync", return_value=[]),
patch("app.services.geocoder.settings") as mock_settings,
patch(
"app.services.geocoder._dadata_suggest",
new_callable=AsyncMock,
return_value=[dadata_result],
) as mock_dadata,
patch(
"app.services.geocoder._yandex_suggest",
new_callable=AsyncMock,
) as mock_yandex,
):
mock_settings.dadata_api_token = "fake-dadata-token"
mock_settings.yandex_geocoder_api_key = "fake-yandex-key"
results = await suggest("Малышева", db=db, limit=8)
assert len(results) == 1
assert results[0].lat == 56.838011
mock_dadata.assert_called_once()
mock_yandex.assert_not_called()
# ── ensure_fdw_user_mapping: SQL injection / whitelist guards ─────────────────
def test_ensure_fdw_user_mapping_rejects_quote_in_password() -> None:

View file

@ -362,3 +362,312 @@ async def test_clean_address_sends_proper_request_body() -> None:
assert captured["body"] == ["Екатеринбург, Малышева 125"] # trimmed
assert captured["auth"] == "Token my-token"
assert captured["secret"] == "my-secret"
# ─────────────────────────────────────────────────────────────────────────────
# DaData /suggest/address tests (PR Q2)
# ─────────────────────────────────────────────────────────────────────────────
SAMPLE_SUGGEST_PAYLOAD: dict = {
"suggestions": [
{
"value": "г Екатеринбург, ул Малышева, д 30",
"unrestricted_value": "620075, Свердловская обл, г Екатеринбург, ул Малышева, д 30",
"data": {
"geo_lat": "56.838011",
"geo_lon": "60.601023",
"fias_id": "00000000-aaaa-bbbb-cccc-000000000001",
"fias_level": "8",
"house": "30",
"street": "Малышева",
"city": "Екатеринбург",
},
},
{
"value": "г Екатеринбург, ул Малышева",
"unrestricted_value": "Свердловская обл, г Екатеринбург, ул Малышева",
"data": {
"geo_lat": "56.838",
"geo_lon": "60.605",
"fias_id": "00000000-aaaa-bbbb-cccc-000000000002",
"fias_level": "7",
"house": None,
"street": "Малышева",
"city": "Екатеринбург",
},
},
{
"value": "г Екатеринбург",
"unrestricted_value": "Свердловская обл, г Екатеринбург",
"data": {
"geo_lat": "56.838",
"geo_lon": "60.605",
"fias_id": "00000000-aaaa-bbbb-cccc-000000000003",
"fias_level": "4",
"house": None,
"street": None,
"city": "Екатеринбург",
},
},
]
}
def _mock_suggest_transport(
status_code: int = 200, payload: object = SAMPLE_SUGGEST_PAYLOAD
) -> httpx.MockTransport:
"""MockTransport для /suggest endpoint — отличается URL и headers от /clean."""
def handler(request: httpx.Request) -> httpx.Response:
assert "suggestions.dadata.ru" in str(request.url)
assert request.headers.get("Authorization", "").startswith("Token ")
# Suggest endpoint НЕ требует X-Secret (отличие от /clean)
assert request.headers.get("X-Secret") is None
return httpx.Response(status_code, json=payload)
return httpx.MockTransport(handler)
async def test_suggest_addresses_happy_path() -> None:
"""200 → list из 3 DadataSuggestion с правильным kind для каждого fias_level."""
from app.services import dadata
transport = _mock_suggest_transport(200, SAMPLE_SUGGEST_PAYLOAD)
with _patch_settings(), _patch_async_client(transport):
results = await dadata.suggest_addresses("Малышева")
assert len(results) == 3
# First — house (fias_level=8)
assert results[0].value == "г Екатеринбург, ул Малышева, д 30"
assert results[0].lat is not None
assert abs(results[0].lat - 56.838011) < 1e-6
assert results[0].lon is not None
assert abs(results[0].lon - 60.601023) < 1e-6
assert results[0].fias_id == "00000000-aaaa-bbbb-cccc-000000000001"
assert results[0].house == "30"
assert results[0].street == "Малышева"
assert results[0].city == "Екатеринбург"
assert results[0].kind == "house"
# Second — street (fias_level=7)
assert results[1].kind == "street"
# Third — city (fias_level=4)
assert results[2].kind == "city"
async def test_suggest_addresses_returns_empty_on_no_token() -> None:
"""DADATA_API_TOKEN не задан → [], никакого httpx-вызова."""
from app.services import dadata
with _patch_settings(token=None, secret="anything"):
with patch("app.services.dadata.httpx.AsyncClient") as mock_client:
results = await dadata.suggest_addresses("Малышева")
assert results == []
mock_client.assert_not_called()
async def test_suggest_addresses_returns_empty_on_short_query() -> None:
"""Query < 2 символов → [], никакого вызова DaData."""
from app.services import dadata
with _patch_settings():
with patch("app.services.dadata.httpx.AsyncClient") as mock_client:
assert await dadata.suggest_addresses("") == []
assert await dadata.suggest_addresses("a") == []
assert await dadata.suggest_addresses(" ") == []
mock_client.assert_not_called()
async def test_suggest_addresses_returns_empty_on_network_error() -> None:
"""httpx TimeoutException → [] gracefully."""
from app.services import dadata
def handler(request: httpx.Request) -> httpx.Response:
raise httpx.TimeoutException("read timed out")
transport = httpx.MockTransport(handler)
with _patch_settings(), _patch_async_client(transport):
results = await dadata.suggest_addresses("Малышева")
assert results == []
@pytest.mark.parametrize("status", [429, 401, 403, 500, 502, 503])
async def test_suggest_addresses_returns_empty_on_http_error(status: int) -> None:
"""4xx/5xx → []."""
from app.services import dadata
transport = _mock_suggest_transport(status, {"error": "rejected"})
with _patch_settings(), _patch_async_client(transport):
results = await dadata.suggest_addresses("Малышева")
assert results == []
async def test_suggest_addresses_returns_empty_on_malformed_response() -> None:
"""Нет ключа `suggestions` / не dict / список вместо dict → []."""
from app.services import dadata
# Case 1: dict без ключа `suggestions`
transport = _mock_suggest_transport(200, {"other": "key"})
with _patch_settings(), _patch_async_client(transport):
results = await dadata.suggest_addresses("Малышева")
assert results == []
# Case 2: пустой suggestions list
transport2 = _mock_suggest_transport(200, {"suggestions": []})
with _patch_settings(), _patch_async_client(transport2):
results = await dadata.suggest_addresses("Малышева")
assert results == []
# Case 3: response — list вместо dict
transport3 = _mock_suggest_transport(200, ["wrong shape"])
with _patch_settings(), _patch_async_client(transport3):
results = await dadata.suggest_addresses("Малышева")
assert results == []
async def test_suggest_addresses_filters_null_coords() -> None:
"""Suggest должен парсить запись даже без координат — фильтр координат
отвечает _dadata_suggest в geocoder.py. Сам suggest_addresses возвращает
DadataSuggestion с lat=None/lon=None (не отбрасывает на этом уровне)."""
from app.services import dadata
payload = {
"suggestions": [
{
"value": "г Екатеринбург, район Кировский",
"unrestricted_value": "Свердловская обл, г Екатеринбург, р-н Кировский",
"data": {
"geo_lat": None,
"geo_lon": None,
"fias_level": "6",
"city": "Екатеринбург",
},
},
{
"value": "г Екатеринбург, ул Малышева, д 30",
"unrestricted_value": "620075, Свердловская обл, г Екатеринбург, ул Малышева, д 30",
"data": {
"geo_lat": "56.838",
"geo_lon": "60.601",
"fias_level": "8",
"house": "30",
"street": "Малышева",
"city": "Екатеринбург",
},
},
]
}
transport = _mock_suggest_transport(200, payload)
with _patch_settings(), _patch_async_client(transport):
results = await dadata.suggest_addresses("Кировский")
# suggest_addresses вернёт оба — фильтр координат в _dadata_suggest (geocoder)
assert len(results) == 2
assert results[0].lat is None
assert results[0].lon is None
assert results[1].lat is not None
async def test_suggest_addresses_classifies_kind() -> None:
"""fias_level → kind. 8→house, 7→street, ≤6→city. Также coerces str→int."""
from app.services import dadata
payload = {
"suggestions": [
{
"value": "Дом",
"unrestricted_value": "Дом",
"data": {"geo_lat": "56.84", "geo_lon": "60.6", "fias_level": "8"},
},
{
"value": "Улица",
"unrestricted_value": "Улица",
"data": {"geo_lat": "56.84", "geo_lon": "60.6", "fias_level": "7"},
},
{
"value": "Город",
"unrestricted_value": "Город",
"data": {"geo_lat": "56.84", "geo_lon": "60.6", "fias_level": "4"},
},
{
"value": "Регион",
"unrestricted_value": "Регион",
"data": {"geo_lat": "56.84", "geo_lon": "60.6", "fias_level": "1"},
},
{
"value": "Квартира", # fias_level=9 (>= 8) → house bucket
"unrestricted_value": "Квартира",
"data": {"geo_lat": "56.84", "geo_lon": "60.6", "fias_level": "9"},
},
{
"value": "Без уровня", # fias_level отсутствует → city safe default
"unrestricted_value": "Без уровня",
"data": {"geo_lat": "56.84", "geo_lon": "60.6"},
},
]
}
transport = _mock_suggest_transport(200, payload)
with _patch_settings(), _patch_async_client(transport):
results = await dadata.suggest_addresses("test")
assert len(results) == 6
assert results[0].kind == "house" # 8
assert results[1].kind == "street" # 7
assert results[2].kind == "city" # 4
assert results[3].kind == "city" # 1
assert results[4].kind == "house" # 9 (>=8)
assert results[5].kind == "city" # missing → default
async def test_suggest_addresses_sends_proper_request() -> None:
"""Body содержит query/count/locations, header только Authorization (без X-Secret)."""
from app.services import dadata
captured: dict[str, object] = {}
def handler(request: httpx.Request) -> httpx.Response:
import json as _json
assert "suggestions.dadata.ru" in str(request.url)
captured["body"] = _json.loads(request.content.decode("utf-8"))
captured["auth"] = request.headers.get("Authorization")
captured["x_secret"] = request.headers.get("X-Secret")
return httpx.Response(200, json=SAMPLE_SUGGEST_PAYLOAD)
transport = httpx.MockTransport(handler)
with _patch_settings(token="my-token"), _patch_async_client(transport):
results = await dadata.suggest_addresses(" Малышева ", limit=5, city="Екатеринбург")
assert len(results) > 0
body = captured["body"]
assert isinstance(body, dict)
assert body["query"] == "Малышева" # trimmed
assert body["count"] == 5
assert body["locations"] == [{"city": "Екатеринбург"}]
assert captured["auth"] == "Token my-token"
# Critical: suggest endpoint should NOT require X-Secret
assert captured["x_secret"] is None
async def test_suggest_addresses_clamps_limit() -> None:
"""count должен быть clamped в [1, 20] — DaData ограничение."""
from app.services import dadata
captured: dict[str, object] = {}
def handler(request: httpx.Request) -> httpx.Response:
import json as _json
captured["body"] = _json.loads(request.content.decode("utf-8"))
return httpx.Response(200, json={"suggestions": []})
transport = httpx.MockTransport(handler)
with _patch_settings(), _patch_async_client(transport):
await dadata.suggest_addresses("Малышева", limit=100)
body = captured["body"]
assert isinstance(body, dict)
assert body["count"] == 20 # clamped to max