Light1YT
02ad9a1f58
Stratified 200-house Yandex reverse-geocode audit comparing our DB's (address, lat/lon) pair against what Yandex.Карты returns for those coordinates. Лежит фундамент данных для Phases 2-5 (canonical = Yandex per user decision). What ships: - Migration 066: address_mismatch_audit table + FDW foreign table gendesign_ekb_districts_geom для stratified sampling. - Sampling SQL: 8 EKB districts × 25 houses = 200. - Driver scripts/audit_address_mismatch.py с двумя режимами: API (если YANDEX_GEOCODER_API_KEY есть) и Playwright (CAPTCHA-aware, персистентный контекст). Resumable по --batch. - Report SQL: p50/p75/p95 distance, top-20 outliers, per-district breakdown. - 19 unit-тестов: normalize, distance bind, API parser, resume idempotency, captcha → status=blocked. Devops note: после деплоя миграции — выдать tradein_fdw_reader SELECT на gendesign.public.ekb_districts_geom перед первым запуском audit. Refs #582 (Phase 1 of 5). Phase 3 backfill заблокирован до Yandex API key (#402).
595 lines
21 KiB
Python
595 lines
21 KiB
Python
"""Audit driver — compares houses.address vs Yandex reverse geocode.
|
||
|
||
Phase 1 of Forgejo issue #582. Pulls a stratified sample of EKB houses (25
|
||
per admin district = 200 total), reverse-geocodes each via Yandex, computes
|
||
the distance between the stored coordinates and the snapped Yandex point,
|
||
and writes the result into `address_mismatch_audit`.
|
||
|
||
Design choices:
|
||
- **Resumable**: the audit table has UNIQUE (house_id, audit_batch). Re-run
|
||
with the same `--batch` skips rows already inserted, so a partial run can
|
||
be picked up after CAPTCHA / network blip.
|
||
- **Mode auto**: prefer API when `YANDEX_GEOCODER_API_KEY` is set, fall back
|
||
to Playwright otherwise. Explicit override via `--mode {api,playwright}`.
|
||
- **No prod side effects**: the script only writes to one new audit table;
|
||
it never touches `houses`, `house_sources`, or any matching/listing row.
|
||
- **Per-row SAVEPOINT**: a single Yandex error must not nuke the entire
|
||
batch — wrap each INSERT in `db.begin_nested()` per backend.md.
|
||
|
||
How to run:
|
||
DATABASE_URL=postgresql+psycopg://... \
|
||
YANDEX_GEOCODER_API_KEY=... \
|
||
python -m scripts.audit_address_mismatch --batch 2026-05-25_run1
|
||
|
||
Outputs (post-run):
|
||
- New rows in `address_mismatch_audit` with batch label.
|
||
- `scripts/address_audit_report.sql :batch=<id>` for summary.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import asyncio
|
||
import json
|
||
import logging
|
||
import os
|
||
import random
|
||
from dataclasses import dataclass
|
||
from datetime import date
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
import httpx
|
||
from sqlalchemy import text
|
||
from sqlalchemy.orm import Session
|
||
|
||
# Allow running both as `python -m scripts.audit_address_mismatch` (preferred)
|
||
# and as a stand-alone file (`python scripts/audit_address_mismatch.py`)
|
||
# without requiring package install.
|
||
try:
|
||
from app.core.db import SessionLocal # type: ignore[import-not-found]
|
||
from app.services.matching.normalize import normalize_address # type: ignore[import-not-found]
|
||
except ImportError: # pragma: no cover — fallback for adhoc invocation
|
||
import sys
|
||
|
||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||
from app.core.db import SessionLocal
|
||
from app.services.matching.normalize import normalize_address
|
||
|
||
# `from .` works when run via -m; the absolute import works under pytest.
|
||
try:
|
||
from scripts._yandex_reverse import ( # type: ignore[import-not-found]
|
||
YandexBlockedError,
|
||
YandexReverseResult,
|
||
reverse_via_api,
|
||
reverse_via_playwright,
|
||
)
|
||
except ImportError:
|
||
from _yandex_reverse import ( # type: ignore[no-redef]
|
||
YandexBlockedError,
|
||
YandexReverseResult,
|
||
reverse_via_api,
|
||
reverse_via_playwright,
|
||
)
|
||
|
||
logging.basicConfig(
|
||
level=logging.INFO,
|
||
format="%(asctime)s %(levelname)s %(name)s %(message)s",
|
||
)
|
||
logger = logging.getLogger("audit_address_mismatch")
|
||
|
||
# Playwright persistent context location — keeps cookies/local storage between
|
||
# runs so we look like a returning user, reducing CAPTCHA frequency.
|
||
_PLAYWRIGHT_USER_DATA = Path.home() / ".cache" / "tradein-audit-playwright"
|
||
|
||
_PLAYWRIGHT_UA = (
|
||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
|
||
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
|
||
)
|
||
|
||
_SAMPLE_SQL_PATH = Path(__file__).parent / "audit_address_sample.sql"
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Domain helpers
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@dataclass
|
||
class SampleRow:
|
||
"""One house from the stratified sampling query."""
|
||
|
||
id: int
|
||
address: str
|
||
lat: float
|
||
lon: float
|
||
district: str | None
|
||
|
||
|
||
# Words that introduce a street rather than identify it. We skip these so the
|
||
# comparison lands on the actual street name ('малышева' / 'ленина'). Mirrors
|
||
# the canonical forms produced by `normalize_address` (which expands all known
|
||
# abbreviations to these full words).
|
||
_STREET_TYPE_WORDS = frozenset(
|
||
{
|
||
"улица",
|
||
"проспект",
|
||
"переулок",
|
||
"бульвар",
|
||
"проезд",
|
||
"шоссе",
|
||
"площадь",
|
||
"набережная",
|
||
"тупик",
|
||
"строение",
|
||
"корпус",
|
||
"дом",
|
||
}
|
||
)
|
||
|
||
# Geographic prefix words that addresses sometimes carry before the street
|
||
# (e.g. 'россия екатеринбург улица малышева 51'). We skip them too so we
|
||
# converge on the same identifying token regardless of how verbose the
|
||
# source representation is.
|
||
_GEO_PREFIX_WORDS = frozenset(
|
||
{
|
||
"россия",
|
||
"свердловская",
|
||
"область",
|
||
"екатеринбург",
|
||
"город",
|
||
"г",
|
||
}
|
||
)
|
||
|
||
|
||
def _first_street_token(address: str | None) -> str | None:
|
||
"""Extract the first street-name token of a normalized address.
|
||
|
||
Phase-1 heuristic for "do the streets agree": skip numeric tokens (house
|
||
numbers), street-type words ('улица', 'проспект', …), and geographic
|
||
prefixes ('россия', 'екатеринбург', …) — the next token is the street
|
||
name itself, which is the identifying part we want to compare.
|
||
|
||
Returns None for an empty address or when no candidate token remains.
|
||
"""
|
||
norm = normalize_address(address or "")
|
||
if not norm:
|
||
return None
|
||
for tok in norm.split():
|
||
if not tok:
|
||
continue
|
||
# Skip purely numeric tokens (e.g. '5', '17а' if it starts with digit).
|
||
if tok[0].isdigit():
|
||
continue
|
||
# Skip street type words and geographic prefixes.
|
||
if tok in _STREET_TYPE_WORDS or tok in _GEO_PREFIX_WORDS:
|
||
continue
|
||
return tok
|
||
return None
|
||
|
||
|
||
def _street_differs(original: str | None, snapped: str | None) -> bool | None:
|
||
"""True iff first non-numeric token differs between the two addresses.
|
||
|
||
Returns None when either side is empty — we cannot compute a meaningful
|
||
diff (caller writes NULL into the audit row).
|
||
"""
|
||
a = _first_street_token(original)
|
||
b = _first_street_token(snapped)
|
||
if a is None or b is None:
|
||
return None
|
||
return a != b
|
||
|
||
|
||
def _distance_meters(
|
||
db: Session,
|
||
olat: float,
|
||
olon: float,
|
||
slat: float,
|
||
slon: float,
|
||
) -> float | None:
|
||
"""Compute great-circle distance via PostGIS geography type.
|
||
|
||
We could do this in Python with a haversine formula, but the audit table
|
||
uses ST_Distance results elsewhere so we use the same authority to avoid
|
||
drift. ST_MakePoint(lon, lat) — PostGIS convention is lon first.
|
||
"""
|
||
row = db.execute(
|
||
text(
|
||
"SELECT ST_Distance("
|
||
" ST_SetSRID(ST_MakePoint(CAST(:olon AS double precision), "
|
||
" CAST(:olat AS double precision)), 4326)::geography, "
|
||
" ST_SetSRID(ST_MakePoint(CAST(:slon AS double precision), "
|
||
" CAST(:slat AS double precision)), 4326)::geography"
|
||
") AS m"
|
||
),
|
||
{"olat": olat, "olon": olon, "slat": slat, "slon": slon},
|
||
).first()
|
||
if row is None or row[0] is None:
|
||
return None
|
||
return float(row[0])
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Sampling + resumption queries
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def _load_sample(db: Session, limit_per_district: int) -> list[SampleRow]:
|
||
"""Run the stratified sampling SQL → list of SampleRow."""
|
||
sql = _SAMPLE_SQL_PATH.read_text(encoding="utf-8")
|
||
rows = db.execute(text(sql), {"limit_per_district": limit_per_district}).mappings().all()
|
||
return [
|
||
SampleRow(
|
||
id=r["id"],
|
||
address=r["address"],
|
||
lat=float(r["lat"]),
|
||
lon=float(r["lon"]),
|
||
district=r["district"],
|
||
)
|
||
for r in rows
|
||
]
|
||
|
||
|
||
def _already_processed_ids(db: Session, batch: str) -> set[int]:
|
||
"""Return the set of house_id already in the audit table for this batch.
|
||
|
||
Drives resumability: drop these from the sample before geocoding.
|
||
"""
|
||
rows = db.execute(
|
||
text("SELECT house_id FROM address_mismatch_audit WHERE audit_batch = CAST(:b AS text)"),
|
||
{"b": batch},
|
||
).all()
|
||
return {r[0] for r in rows}
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Insert helper
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def _insert_audit_row(
|
||
db: Session,
|
||
*,
|
||
house_id: int,
|
||
batch: str,
|
||
district: str | None,
|
||
original_address: str | None,
|
||
original_lat: float | None,
|
||
original_lon: float | None,
|
||
snapped_address: str | None,
|
||
snapped_lat: float | None,
|
||
snapped_lon: float | None,
|
||
distance_m: float | None,
|
||
street_differs: bool | None,
|
||
audit_status: str,
|
||
error_message: str | None,
|
||
raw_payload: dict[str, Any] | None,
|
||
) -> None:
|
||
"""INSERT … ON CONFLICT DO NOTHING into address_mismatch_audit.
|
||
|
||
Wrapped in begin_nested by the caller per backend.md SAVEPOINT pattern.
|
||
"""
|
||
db.execute(
|
||
text(
|
||
"INSERT INTO address_mismatch_audit ("
|
||
" house_id, audit_batch, district,"
|
||
" original_address, original_lat, original_lon,"
|
||
" snapped_address, snapped_lat, snapped_lon,"
|
||
" distance_m, street_differs,"
|
||
" audit_status, error_message, raw_payload"
|
||
") VALUES ("
|
||
" CAST(:house_id AS bigint), CAST(:batch AS text), :district,"
|
||
" :original_address, :original_lat, :original_lon,"
|
||
" :snapped_address, :snapped_lat, :snapped_lon,"
|
||
" :distance_m, :street_differs,"
|
||
" CAST(:audit_status AS text), :error_message,"
|
||
" CAST(:raw_payload AS jsonb)"
|
||
") ON CONFLICT (house_id, audit_batch) DO NOTHING"
|
||
),
|
||
{
|
||
"house_id": house_id,
|
||
"batch": batch,
|
||
"district": district,
|
||
"original_address": original_address,
|
||
"original_lat": original_lat,
|
||
"original_lon": original_lon,
|
||
"snapped_address": snapped_address,
|
||
"snapped_lat": snapped_lat,
|
||
"snapped_lon": snapped_lon,
|
||
"distance_m": distance_m,
|
||
"street_differs": street_differs,
|
||
"audit_status": audit_status,
|
||
"error_message": error_message,
|
||
"raw_payload": json.dumps(raw_payload) if raw_payload is not None else None,
|
||
},
|
||
)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Mode dispatcher
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def _resolve_mode(mode: str, api_key: str | None) -> str:
|
||
"""Translate `--mode auto` → concrete 'api' / 'playwright' choice.
|
||
|
||
Explicit modes are passed through unchanged; auto chooses api iff a key
|
||
is configured (fail-fast: we don't want a "should have used the API but
|
||
silently fell back to slow scraping" surprise).
|
||
"""
|
||
if mode == "auto":
|
||
return "api" if api_key else "playwright"
|
||
return mode
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Main loop
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
async def _run_api_mode(
|
||
db: Session,
|
||
sample: list[SampleRow],
|
||
batch: str,
|
||
api_key: str,
|
||
) -> int:
|
||
"""Geocode the sample using the HTTP Geocoder API."""
|
||
processed = 0
|
||
last_distance: float | None = None
|
||
async with httpx.AsyncClient(timeout=httpx.Timeout(10.0)) as client:
|
||
for i, row in enumerate(sample, start=1):
|
||
status = "ok"
|
||
err: str | None = None
|
||
res: YandexReverseResult | None = None
|
||
try:
|
||
res = await reverse_via_api(row.lat, row.lon, api_key, client=client)
|
||
except httpx.HTTPError as e:
|
||
status = "error"
|
||
err = f"http_error: {e!s}"
|
||
except Exception as e: # pragma: no cover — defensive
|
||
status = "error"
|
||
err = f"unhandled: {e!s}"
|
||
|
||
distance = None
|
||
street_diff: bool | None = None
|
||
if res is not None and status == "ok":
|
||
if res.address is None:
|
||
status = "no_match"
|
||
else:
|
||
if res.snapped_lat is not None and res.snapped_lon is not None:
|
||
distance = _distance_meters(
|
||
db, row.lat, row.lon, res.snapped_lat, res.snapped_lon
|
||
)
|
||
last_distance = distance
|
||
street_diff = _street_differs(row.address, res.address)
|
||
|
||
try:
|
||
with db.begin_nested():
|
||
_insert_audit_row(
|
||
db,
|
||
house_id=row.id,
|
||
batch=batch,
|
||
district=row.district,
|
||
original_address=row.address,
|
||
original_lat=row.lat,
|
||
original_lon=row.lon,
|
||
snapped_address=res.address if res else None,
|
||
snapped_lat=res.snapped_lat if res else None,
|
||
snapped_lon=res.snapped_lon if res else None,
|
||
distance_m=distance,
|
||
street_differs=street_diff,
|
||
audit_status=status,
|
||
error_message=err,
|
||
raw_payload=res.raw if res else None,
|
||
)
|
||
# Per-row commit: each row is durable on disk before the next
|
||
# Yandex call; --batch resume picks up exactly where we crashed.
|
||
db.commit()
|
||
processed += 1
|
||
except Exception as e:
|
||
db.rollback()
|
||
logger.warning("insert failed for house_id=%s: %s", row.id, e)
|
||
|
||
if i % 10 == 0:
|
||
logger.info(
|
||
"progress %d/%d, mode=api, last_distance=%s",
|
||
i,
|
||
len(sample),
|
||
f"{last_distance:.1f}m" if last_distance is not None else "n/a",
|
||
)
|
||
|
||
return processed
|
||
|
||
|
||
async def _run_playwright_mode(
|
||
db: Session,
|
||
sample: list[SampleRow],
|
||
batch: str,
|
||
) -> int:
|
||
"""Geocode via a persistent Playwright context (CAPTCHA-aware)."""
|
||
try:
|
||
from playwright.async_api import async_playwright # type: ignore[import-not-found]
|
||
except ImportError as e:
|
||
raise RuntimeError(
|
||
"Playwright is required for --mode playwright. "
|
||
"Install with `uv sync --group dev` and `playwright install chromium`."
|
||
) from e
|
||
|
||
_PLAYWRIGHT_USER_DATA.mkdir(parents=True, exist_ok=True)
|
||
processed = 0
|
||
last_distance: float | None = None
|
||
|
||
async with async_playwright() as p:
|
||
context = await p.chromium.launch_persistent_context(
|
||
user_data_dir=str(_PLAYWRIGHT_USER_DATA),
|
||
headless=False,
|
||
user_agent=_PLAYWRIGHT_UA,
|
||
locale="ru-RU",
|
||
timezone_id="Asia/Yekaterinburg",
|
||
)
|
||
page = await context.new_page()
|
||
|
||
try:
|
||
for i, row in enumerate(sample, start=1):
|
||
status = "ok"
|
||
err: str | None = None
|
||
res: YandexReverseResult | None = None
|
||
stop_batch = False
|
||
try:
|
||
res = await reverse_via_playwright(row.lat, row.lon, page)
|
||
except YandexBlockedError as e:
|
||
status = "blocked"
|
||
err = str(e)
|
||
stop_batch = True
|
||
except Exception as e:
|
||
status = "error"
|
||
err = f"playwright: {e!s}"
|
||
|
||
distance = None
|
||
street_diff: bool | None = None
|
||
if res is not None and status == "ok":
|
||
if res.address is None:
|
||
status = "no_match"
|
||
else:
|
||
if res.snapped_lat is not None and res.snapped_lon is not None:
|
||
distance = _distance_meters(
|
||
db, row.lat, row.lon, res.snapped_lat, res.snapped_lon
|
||
)
|
||
last_distance = distance
|
||
street_diff = _street_differs(row.address, res.address)
|
||
|
||
try:
|
||
with db.begin_nested():
|
||
_insert_audit_row(
|
||
db,
|
||
house_id=row.id,
|
||
batch=batch,
|
||
district=row.district,
|
||
original_address=row.address,
|
||
original_lat=row.lat,
|
||
original_lon=row.lon,
|
||
snapped_address=res.address if res else None,
|
||
snapped_lat=res.snapped_lat if res else None,
|
||
snapped_lon=res.snapped_lon if res else None,
|
||
distance_m=distance,
|
||
street_differs=street_diff,
|
||
audit_status=status,
|
||
error_message=err,
|
||
raw_payload=res.raw if res else None,
|
||
)
|
||
db.commit()
|
||
processed += 1
|
||
except Exception as e:
|
||
db.rollback()
|
||
logger.warning("insert failed for house_id=%s: %s", row.id, e)
|
||
|
||
if stop_batch:
|
||
logger.error(
|
||
"Yandex CAPTCHA detected at position %d/%d (house_id=%s). "
|
||
"Stopping batch — re-run with same --batch to resume.",
|
||
i,
|
||
len(sample),
|
||
row.id,
|
||
)
|
||
break
|
||
|
||
if i % 10 == 0:
|
||
logger.info(
|
||
"progress %d/%d, mode=playwright, last_distance=%s",
|
||
i,
|
||
len(sample),
|
||
f"{last_distance:.1f}m" if last_distance is not None else "n/a",
|
||
)
|
||
|
||
# Random delay 4-7s between requests — keeps us under Yandex's
|
||
# heuristic rate limit while still finishing 200 rows in <30min.
|
||
# Skip the wait on the last iteration (no next request to space).
|
||
if i < len(sample):
|
||
await asyncio.sleep(random.uniform(4.0, 7.0))
|
||
finally:
|
||
await context.close()
|
||
|
||
return processed
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Entry point
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def _parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
||
"""argparse setup, factored out for testability."""
|
||
p = argparse.ArgumentParser(
|
||
description="Phase 1 audit — houses.address vs Yandex reverse geocode.",
|
||
)
|
||
p.add_argument(
|
||
"--batch",
|
||
default=f"{date.today().isoformat()}_run1",
|
||
help="Audit batch label. Same batch re-run skips already-processed houses.",
|
||
)
|
||
p.add_argument(
|
||
"--limit-per-district",
|
||
type=int,
|
||
default=25,
|
||
help="Houses to sample per district (default 25 → ~200 total for EKB).",
|
||
)
|
||
p.add_argument(
|
||
"--mode",
|
||
choices=("auto", "api", "playwright"),
|
||
default="auto",
|
||
help="auto = API if YANDEX_GEOCODER_API_KEY set, else playwright.",
|
||
)
|
||
return p.parse_args(argv)
|
||
|
||
|
||
async def main(argv: list[str] | None = None) -> int:
|
||
"""CLI entry point. Returns the number of rows processed this run."""
|
||
args = _parse_args(argv)
|
||
api_key = os.environ.get("YANDEX_GEOCODER_API_KEY")
|
||
mode = _resolve_mode(args.mode, api_key)
|
||
|
||
if mode == "api" and not api_key:
|
||
raise SystemExit("mode=api requested but YANDEX_GEOCODER_API_KEY is not set")
|
||
|
||
logger.info(
|
||
"starting audit batch=%s mode=%s limit_per_district=%d",
|
||
args.batch,
|
||
mode,
|
||
args.limit_per_district,
|
||
)
|
||
|
||
db = SessionLocal()
|
||
try:
|
||
sample = _load_sample(db, args.limit_per_district)
|
||
logger.info("loaded sample: %d houses", len(sample))
|
||
|
||
# Resume support — drop already-processed house_ids.
|
||
done = _already_processed_ids(db, args.batch)
|
||
if done:
|
||
logger.info(
|
||
"resuming batch %s: %d rows already processed, %d remaining",
|
||
args.batch,
|
||
len(done),
|
||
len(sample) - sum(1 for s in sample if s.id in done),
|
||
)
|
||
remaining = [s for s in sample if s.id not in done]
|
||
|
||
if not remaining:
|
||
logger.info("nothing to do — batch %s is complete", args.batch)
|
||
return 0
|
||
|
||
if mode == "api":
|
||
n = await _run_api_mode(db, remaining, args.batch, api_key or "")
|
||
else:
|
||
n = await _run_playwright_mode(db, remaining, args.batch)
|
||
|
||
logger.info("done: processed=%d batch=%s mode=%s", n, args.batch, mode)
|
||
return n
|
||
finally:
|
||
db.close()
|
||
|
||
|
||
if __name__ == "__main__": # pragma: no cover
|
||
asyncio.run(main())
|