"""Batch geocoding для listings с NULL lat/lon. Запускается: - Manual через POST /admin/scrape/geocode-missing-listings - Future: cron job (Celery beat если bootstrap'нут / OS cron) Pattern: dedup по address (1 unique address → 1 geocode call → UPDATE all listings). Rate limit: Nominatim 1 req/sec. Yandex 25K/day если YANDEX_GEOCODER_API_KEY set. Отличие от /admin/geocode-missing (per-ID): - Этот модуль группирует по address → меньше API calls (dedup). - Поддерживает all sources включая Avito (после PR #487 убрали jitter). - Возвращает GeocodeBackfillResult с детальными counters. """ from __future__ import annotations import logging import time from dataclasses import dataclass, field from sqlalchemy import text from sqlalchemy.orm import Session from app.services.estimator import _geocode_is_coarse from app.services.geocoder import geocode logger = logging.getLogger(__name__) @dataclass class GeocodeBackfillResult: addresses_total: int = 0 # unique addresses pending geocode в этом batch addresses_processed: int = 0 # фактически обработано addresses_geocoded: int = 0 # успешно получили coords addresses_failed: int = 0 # geocoder вернул None listings_updated: int = 0 # total listings затронуто (1 address → N listings) cache_hits: int = 0 # из geocode_cache (instant) cache_misses: int = 0 # реальные geocoder calls duration_sec: float = field(default=0.0) async def geocode_missing_listings( db: Session, *, batch_size: int = 200, dry_run: bool = False, ) -> GeocodeBackfillResult: """Geocode listings с NULL coords (любой source). Steps: 1. SELECT DISTINCT address FROM listings WHERE lat IS NULL AND address IS NOT NULL GROUP BY address ORDER BY COUNT(*) DESC LIMIT batch_size (приоритет адресам с большим числом listings — больший ROI per geocode call) 2. Для каждого address: - geocode(address, db) — auto-cache (hit или miss) - Если есть результат: UPDATE listings SET lat, lon WHERE address = :addr AND lat IS NULL - PostGIS trigger (listings_set_geom_trg) автоматически обновит geom 3. Log progress каждые 50 addresses. Args: batch_size: max addresses to process per call (default 200 ≈ 3.5 min Nominatim) dry_run: только показать что бы сделалось, без UPDATE Returns: GeocodeBackfillResult с counters. """ start = time.monotonic() result = GeocodeBackfillResult() # 1. Найти top-N адресов с NULL coords (DESC by occurrence count) rows = ( db.execute( text( """ SELECT address, COUNT(*) AS listings_count FROM listings WHERE lat IS NULL AND address IS NOT NULL AND length(trim(address)) >= 5 GROUP BY address ORDER BY listings_count DESC, address ASC LIMIT :limit """ ), {"limit": batch_size}, ) .mappings() .all() ) result.addresses_total = len(rows) if not rows: logger.info("geocode_missing: 0 pending addresses — nothing to do") result.duration_sec = time.monotonic() - start return result logger.info( "geocode_missing: starting batch=%d total_pending_addresses=%d (top by listings count)", batch_size, result.addresses_total, ) for idx, row in enumerate(rows): address: str = row["address"] listings_count: int = row["listings_count"] result.addresses_processed += 1 try: geo = await geocode(address, db) except Exception as exc: logger.warning("geocode_missing: geocode raised for '%s': %s", address[:60], exc) result.addresses_failed += 1 continue if geo is None: result.addresses_failed += 1 logger.info( "geocode_missing: NOT FOUND '%s' (used in %d listings)", address[:60], listings_count, ) continue if geo.provider == "cache": result.cache_hits += 1 else: result.cache_misses += 1 result.addresses_geocoded += 1 if dry_run: logger.info( "geocode_missing[dry]: '%s' → (%.5f, %.5f) provider=%s would update %d listings", address[:60], geo.lat, geo.lon, geo.provider, listings_count, ) continue # Определяем точность геокода: city-centroid (нет номера дома) → 'city'. # _geocode_is_coarse() проверяет confidence='locality' ИЛИ отсутствие # house-number токена (1-3 цифры) в full_address — оба случая означают # что геокодер не дошёл до дома и вернул центр НП/города (#769 Part E). precision: str | None = "city" if _geocode_is_coarse(geo) else None # UPDATE listings — PostGIS trigger (listings_set_geom_trg) обновит geom автоматически. # geo_precision проставляется одновременно с координатами. update_result = db.execute( text( """ UPDATE listings SET lat = :lat, lon = :lon, geo_precision = :precision WHERE address = :addr AND lat IS NULL """ ), {"lat": geo.lat, "lon": geo.lon, "precision": precision, "addr": address}, ) db.commit() result.listings_updated += update_result.rowcount if (idx + 1) % 50 == 0: elapsed = time.monotonic() - start rate = (idx + 1) / elapsed if elapsed > 0 else 0 logger.info( "geocode_missing: progress %d/%d " "(geocoded=%d failed=%d cache_hits=%d listings_updated=%d) rate=%.1f addr/s", idx + 1, len(rows), result.addresses_geocoded, result.addresses_failed, result.cache_hits, result.listings_updated, rate, ) result.duration_sec = time.monotonic() - start logger.info( "geocode_missing: DONE batch=%d processed=%d geocoded=%d failed=%d " "cache=(hit=%d miss=%d) listings_updated=%d duration=%.1fs", batch_size, result.addresses_processed, result.addresses_geocoded, result.addresses_failed, result.cache_hits, result.cache_misses, result.listings_updated, result.duration_sec, ) return result