380 lines
15 KiB
Python
380 lines
15 KiB
Python
"""Yandex Geocoder helpers for the address-mismatch audit + backfill (issue #582).
|
|
|
|
Three geocoding paths exposed:
|
|
|
|
- `reverse_via_api()` — Yandex Geocoder HTTP API, lon/lat → address. Fast,
|
|
structured response, needs a valid API key (env `YANDEX_GEOCODER_API_KEY`).
|
|
Free tier is 25k req/day, fine for ~8.5k houses + audit (~17k total).
|
|
|
|
- `reverse_via_playwright()` — fallback when no API key is available. Drives
|
|
a real browser session at https://yandex.ru/maps/?…&mode=whatshere. Slower
|
|
and CAPTCHA-prone, so the driver inserts 4-7s sleeps between calls and we
|
|
raise a dedicated exception on CAPTCHA so the batch can pause-and-resume.
|
|
|
|
- `forward_via_api()` — address → lon/lat + canonical address (Phase 2 of
|
|
issue #582). Used by `backfill_house_coords.py` to fill `houses.lat/lon`
|
|
for the 4141 houses scraped from sources that didn't include coords (esp.
|
|
yandex_valuation, which only returns an address string).
|
|
|
|
All three return a `YandexReverseResult` dataclass — same shape regardless
|
|
of direction so the driver code stays implementation-agnostic. The `raw`
|
|
field always carries the full source payload for post-hoc diagnostics, and
|
|
`precision` / `kind` are filled in by the API paths so the caller can skip
|
|
imprecise matches (e.g. only-street-level results during backfill).
|
|
|
|
Why three paths:
|
|
The user (issue #582 discussion) wants the audit to run on dev machines
|
|
that may not have an API key, but on prod we already provision the key for
|
|
estimator.py. Forward geocode is API-only — Playwright forward geocoding
|
|
through Yandex Maps search is too fragile (relevance ranking, suggest
|
|
dropdown). For dev without a key, backfill simply doesn't run.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import logging
|
|
import random
|
|
from dataclasses import dataclass, field
|
|
from typing import Any
|
|
|
|
import httpx
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# Yandex Maps "what's here" URL — wraps a reverse-geocode in browser-driven UI.
|
|
# `whatshere[point]` accepts "<lon>,<lat>" (note: lon first, Yandex convention).
|
|
_YANDEX_MAPS_WHATSHERE = (
|
|
"https://yandex.ru/maps/?ll={lon:.6f}%2C{lat:.6f}&z=18&mode=whatshere"
|
|
"&whatshere%5Bpoint%5D={lon:.6f}%2C{lat:.6f}&whatshere%5Bzoom%5D=18"
|
|
)
|
|
|
|
# Geocoder HTTP API. `kind=house` narrows the result to a building if possible,
|
|
# which is what we want for cadastr-style addresses (улица + дом).
|
|
_YANDEX_GEOCODE_API = "https://geocode-maps.yandex.ru/1.x/"
|
|
|
|
# Reasonable timeouts: API call should be sub-second; we give it generous
|
|
# headroom for slow networks but not so much that a hang stalls the batch.
|
|
_API_TIMEOUT = httpx.Timeout(connect=5.0, read=10.0, write=5.0, pool=5.0)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Dataclasses + exceptions
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@dataclass
|
|
class YandexReverseResult:
|
|
"""Normalized result of a geocode call (forward, reverse-API, or browser).
|
|
|
|
Attributes:
|
|
address: Human-readable canonical address Yandex returned. For
|
|
reverse, this is the snapped address at the queried point. For
|
|
forward, this is the canonical form of the input address. None
|
|
if Yandex returned no match.
|
|
snapped_lat: Latitude of the matched object's geometric centre.
|
|
snapped_lon: Longitude of the matched object's geometric centre.
|
|
precision: For forward calls — Yandex match precision tag (`exact`,
|
|
`number`, `near`, `range`, `street`, `other`). For reverse —
|
|
same field is filled when present (usually `house` / `street`).
|
|
None for the playwright path. Used by the backfill driver to
|
|
skip imprecise matches.
|
|
kind: Object kind from Yandex (`house`, `street`, `locality`, ...).
|
|
Same source as `precision` — see metaDataProperty.GeocoderMetaData.
|
|
raw: Raw response payload retained for forensics (JSON dict from API,
|
|
or snapshot dict from playwright). Used to populate
|
|
`address_mismatch_audit.raw_payload` and
|
|
`houses.raw_payload.yandex_geocode`.
|
|
"""
|
|
|
|
address: str | None
|
|
snapped_lat: float | None
|
|
snapped_lon: float | None
|
|
raw: dict[str, Any] = field(default_factory=dict)
|
|
precision: str | None = None
|
|
kind: str | None = None
|
|
|
|
|
|
class YandexBlockedError(RuntimeError):
|
|
"""Raised when Yandex returns a CAPTCHA / anti-bot challenge.
|
|
|
|
The driver catches this, marks the row `audit_status='blocked'`, logs the
|
|
current batch position, then exits cleanly so a human can intervene.
|
|
"""
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Path A — HTTP Geocoder API
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
async def reverse_via_api(
|
|
lat: float,
|
|
lon: float,
|
|
api_key: str,
|
|
*,
|
|
client: httpx.AsyncClient | None = None,
|
|
) -> YandexReverseResult:
|
|
"""Reverse-geocode (lat, lon) via the Yandex Geocoder HTTP API.
|
|
|
|
Why a separate `client` parameter: lets the driver reuse one
|
|
`AsyncClient` across all 200 calls (TCP keep-alive + connection pool),
|
|
and lets the tests inject a `MockTransport` to assert request shape.
|
|
|
|
Args:
|
|
lat: latitude in WGS84.
|
|
lon: longitude in WGS84.
|
|
api_key: Yandex Geocoder API key.
|
|
client: optional pre-built async client. If None, a one-shot client
|
|
is created.
|
|
|
|
Returns:
|
|
`YandexReverseResult` with the first `featureMember[0].GeoObject`
|
|
result, or all-None if Yandex returned no match (still includes
|
|
`raw` payload so we can later inspect why).
|
|
"""
|
|
params = {
|
|
"apikey": api_key,
|
|
# Yandex expects "lon,lat" (longitude first) per docs — same
|
|
# convention as the "whatshere" map URL above.
|
|
"geocode": f"{lon},{lat}",
|
|
"format": "json",
|
|
"kind": "house",
|
|
"results": "1",
|
|
}
|
|
|
|
own_client = client is None
|
|
if client is None:
|
|
client = httpx.AsyncClient(timeout=_API_TIMEOUT)
|
|
|
|
try:
|
|
resp = await client.get(_YANDEX_GEOCODE_API, params=params)
|
|
resp.raise_for_status()
|
|
data = resp.json()
|
|
finally:
|
|
if own_client:
|
|
await client.aclose()
|
|
|
|
return _parse_api_payload(data)
|
|
|
|
|
|
def _parse_api_payload(data: dict[str, Any]) -> YandexReverseResult:
|
|
"""Extract address + snapped point from a Yandex Geocoder API JSON response.
|
|
|
|
Split out so unit tests can feed a fixture file directly without spinning
|
|
up an HTTP mock. Same payload shape for forward and reverse calls —
|
|
Yandex's response envelope is symmetric.
|
|
"""
|
|
try:
|
|
members = data.get("response", {}).get("GeoObjectCollection", {}).get("featureMember", [])
|
|
if not members:
|
|
return YandexReverseResult(address=None, snapped_lat=None, snapped_lon=None, raw=data)
|
|
|
|
geo_obj = members[0].get("GeoObject", {})
|
|
|
|
# Address: prefer the long `metaDataProperty.GeocoderMetaData.text`
|
|
# (full canonical) and fall back to `name` (street + house number).
|
|
meta = geo_obj.get("metaDataProperty", {}).get("GeocoderMetaData", {})
|
|
address = meta.get("text") or geo_obj.get("name")
|
|
precision = meta.get("precision")
|
|
kind = meta.get("kind")
|
|
|
|
# Point format: "<lon> <lat>" — space-separated string.
|
|
point_str = geo_obj.get("Point", {}).get("pos", "")
|
|
snapped_lon: float | None
|
|
snapped_lat: float | None
|
|
if point_str:
|
|
try:
|
|
lon_s, lat_s = point_str.split()
|
|
snapped_lon = float(lon_s)
|
|
snapped_lat = float(lat_s)
|
|
except (ValueError, TypeError):
|
|
snapped_lon = None
|
|
snapped_lat = None
|
|
else:
|
|
snapped_lon = None
|
|
snapped_lat = None
|
|
|
|
return YandexReverseResult(
|
|
address=address,
|
|
snapped_lat=snapped_lat,
|
|
snapped_lon=snapped_lon,
|
|
raw=data,
|
|
precision=precision,
|
|
kind=kind,
|
|
)
|
|
except Exception as e: # pragma: no cover — defensive; tests cover happy paths
|
|
logger.warning("yandex API payload parse failed: %s", e)
|
|
return YandexReverseResult(address=None, snapped_lat=None, snapped_lon=None, raw=data)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Path A.2 — Forward geocode (address → lon/lat) via HTTP API
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
async def forward_via_api(
|
|
address: str,
|
|
api_key: str,
|
|
*,
|
|
client: httpx.AsyncClient | None = None,
|
|
) -> YandexReverseResult:
|
|
"""Forward-geocode an address string via the Yandex Geocoder HTTP API.
|
|
|
|
Phase 2 of issue #582 — used by `backfill_house_coords.py` to populate
|
|
`houses.lat/lon` for houses that were scraped without coords (esp.
|
|
yandex_valuation rows, which only carry an address).
|
|
|
|
Args:
|
|
address: free-form address ("ул Малышева 51", "Екатеринбург, Ленина 5",
|
|
etc.). Yandex's NLU is forgiving — no need to pre-normalize.
|
|
api_key: Yandex Geocoder API key.
|
|
client: optional pre-built async client. If None, a one-shot client
|
|
is created (matches `reverse_via_api` ergonomics).
|
|
|
|
Returns:
|
|
`YandexReverseResult` with the canonical address + snapped point of
|
|
the first matching feature. `precision` and `kind` are populated so
|
|
the backfill driver can skip imprecise hits (e.g. precision='street'
|
|
means we landed on the road, not the building — too vague for
|
|
comparable-listings spatial queries).
|
|
|
|
Same envelope as `reverse_via_api` — `_parse_api_payload` handles both.
|
|
"""
|
|
params = {
|
|
"apikey": api_key,
|
|
"geocode": address,
|
|
"format": "json",
|
|
# `kind=house` filters out street-only / locality-only matches at
|
|
# the API level when possible. Yandex still returns lower-precision
|
|
# results when no building matches, so the caller must double-check
|
|
# `precision` before writing to houses.
|
|
"kind": "house",
|
|
"results": "1",
|
|
# Locality bias for EKB — improves recall when the input address
|
|
# omits the city. The audit population is 99% EKB houses, so this
|
|
# is safe; non-EKB inputs (rare) still resolve, just with the bias.
|
|
"ll": "60.6122,56.8389",
|
|
"spn": "0.6,0.4",
|
|
}
|
|
|
|
own_client = client is None
|
|
if client is None:
|
|
client = httpx.AsyncClient(timeout=_API_TIMEOUT)
|
|
|
|
try:
|
|
resp = await client.get(_YANDEX_GEOCODE_API, params=params)
|
|
resp.raise_for_status()
|
|
data = resp.json()
|
|
finally:
|
|
if own_client:
|
|
await client.aclose()
|
|
|
|
return _parse_api_payload(data)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Path B — Playwright fallback
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
async def reverse_via_playwright(
|
|
lat: float,
|
|
lon: float,
|
|
page: Any,
|
|
) -> YandexReverseResult:
|
|
"""Reverse-geocode (lat, lon) by driving yandex.ru/maps with Playwright.
|
|
|
|
Why this exists:
|
|
The Yandex Geocoder API requires a key with paid quota for >25k/day. The
|
|
audit only needs 200 rows but a dev without a key still needs a way to
|
|
run the script, so we ship a browser-driven fallback.
|
|
|
|
Implementation:
|
|
1. Navigate to the `whatshere` URL — Yandex Maps responds by opening a
|
|
toponym card at the requested coordinates and rendering the resolved
|
|
address in the side panel.
|
|
2. Wait for client hydration (`networkidle`).
|
|
3. First try to read `window.__INITIAL_STATE__` — Yandex stores the
|
|
toponym address inside the hydrated Redux tree, which is more
|
|
stable across UI redesigns than DOM selectors.
|
|
4. Fall back to DOM selectors (`.toponym-card-title-view__title` +
|
|
`__subtitle`) if the state walk doesn't find an address.
|
|
5. Detect CAPTCHA (`.CheckboxCaptcha`) early and raise `YandexBlockedError`
|
|
so the batch can pause-and-resume without spamming Yandex.
|
|
|
|
`page` is typed as `Any` to keep playwright a dev-only dep — runtime
|
|
importers don't need playwright installed if they only use the API path.
|
|
"""
|
|
url = _YANDEX_MAPS_WHATSHERE.format(lat=lat, lon=lon)
|
|
await page.goto(url, wait_until="domcontentloaded")
|
|
|
|
# Light wait for client-side hydration. Yandex Maps fires lots of
|
|
# background XHRs so `networkidle` is too aggressive; this small wait is
|
|
# enough for the toponym card to render.
|
|
try:
|
|
await page.wait_for_load_state("networkidle", timeout=8000)
|
|
except Exception as e:
|
|
# Slow networks: continue — selectors will retry with their own waits.
|
|
logger.debug("networkidle wait timed out, continuing: %s", e)
|
|
await asyncio.sleep(random.uniform(0.5, 1.2))
|
|
|
|
# CAPTCHA gate — Yandex shows a `.CheckboxCaptcha` form when it suspects
|
|
# automation. Once we see it, every subsequent reverse call will also be
|
|
# blocked, so we raise immediately and let the driver stop the batch.
|
|
captcha = await page.query_selector(".CheckboxCaptcha")
|
|
if captcha is not None:
|
|
raise YandexBlockedError("Yandex CAPTCHA detected on maps page")
|
|
|
|
# Attempt 1 — initial state walk.
|
|
state_addr: str | None = None
|
|
state_pos: tuple[float, float] | None = None
|
|
try:
|
|
state_addr, state_pos = await page.evaluate(
|
|
"() => {\n"
|
|
" const s = window.__INITIAL_STATE__ || {};\n"
|
|
" const card = (s.cards && s.cards.toponym) || (s.card && s.card.toponym) || null;\n"
|
|
" if (!card) return [null, null];\n"
|
|
" const addr = card.title || card.address || null;\n"
|
|
" const pos = card.coords || card.point || null;\n"
|
|
" if (pos && pos.length === 2) return [addr, [pos[0], pos[1]]];\n"
|
|
" return [addr, null];\n"
|
|
"}"
|
|
)
|
|
except Exception as e:
|
|
logger.debug("playwright state walk failed (will fall back to DOM): %s", e)
|
|
|
|
address = state_addr
|
|
|
|
# Attempt 2 — DOM fallback.
|
|
if not address:
|
|
title_el = await page.query_selector(".toponym-card-title-view__title")
|
|
subtitle_el = await page.query_selector(".toponym-card-title-view__subtitle")
|
|
title = (await title_el.inner_text()).strip() if title_el else ""
|
|
subtitle = (await subtitle_el.inner_text()).strip() if subtitle_el else ""
|
|
# subtitle often holds "Екатеринбург, район", title the street + house
|
|
address = ", ".join([p for p in (subtitle, title) if p]) or None
|
|
|
|
snapped_lat: float | None
|
|
snapped_lon: float | None
|
|
if state_pos:
|
|
# State stored as [lon, lat] in Yandex's coordinate convention.
|
|
snapped_lon = float(state_pos[0])
|
|
snapped_lat = float(state_pos[1])
|
|
else:
|
|
snapped_lon = None
|
|
snapped_lat = None
|
|
|
|
raw = {
|
|
"url": url,
|
|
"state_addr": state_addr,
|
|
"state_pos": list(state_pos) if state_pos else None,
|
|
"dom_address": address if not state_addr else None,
|
|
}
|
|
|
|
return YandexReverseResult(
|
|
address=address,
|
|
snapped_lat=snapped_lat,
|
|
snapped_lon=snapped_lon,
|
|
raw=raw,
|
|
)
|