gendesign/tradein-mvp/backend/app/tasks/geocode_missing.py

"""Batch geocoding для listings с NULL lat/lon.

Запускается:
  - Manual через POST /admin/scrape/geocode-missing-listings
  - Future: cron job (Celery beat если bootstrap'нут / OS cron)

Pattern: dedup по address (1 unique address → 1 geocode call → UPDATE all listings).
Rate limit: Nominatim 1 req/sec. Yandex 25K/day если YANDEX_GEOCODER_API_KEY set.

Отличие от /admin/geocode-missing (per-ID):
  - Этот модуль группирует по address → меньше API calls (dedup).
  - Поддерживает all sources включая Avito (после PR #487 убрали jitter).
  - Возвращает GeocodeBackfillResult с детальными counters.
"""
from __future__ import annotations

import logging
import time
from dataclasses import dataclass, field

from sqlalchemy import text
from sqlalchemy.orm import Session

from app.services.geocoder import geocode

logger = logging.getLogger(__name__)


@dataclass
class GeocodeBackfillResult:
    addresses_total: int = 0          # unique addresses pending geocode в этом batch
    addresses_processed: int = 0      # фактически обработано
    addresses_geocoded: int = 0       # успешно получили coords
    addresses_failed: int = 0         # geocoder вернул None
    listings_updated: int = 0         # total listings затронуто (1 address → N listings)
    cache_hits: int = 0               # из geocode_cache (instant)
    cache_misses: int = 0             # реальные geocoder calls
    duration_sec: float = field(default=0.0)


async def geocode_missing_listings(
    db: Session,
    *,
    batch_size: int = 200,
    dry_run: bool = False,
) -> GeocodeBackfillResult:
    """Geocode listings с NULL coords (любой source).

    Steps:
    1. SELECT DISTINCT address FROM listings WHERE lat IS NULL AND address IS NOT NULL
       GROUP BY address ORDER BY COUNT(*) DESC LIMIT batch_size
       (приоритет адресам с большим числом listings — больший ROI per geocode call)

    2. Для каждого address:
       - geocode(address, db) — auto-cache (hit или miss)
       - Если есть результат: UPDATE listings SET lat, lon WHERE address = :addr AND lat IS NULL
       - PostGIS trigger (listings_set_geom_trg) автоматически обновит geom

    3. Log progress каждые 50 addresses.

    Args:
        batch_size: max addresses to process per call (default 200 ≈ 3.5 min Nominatim)
        dry_run: только показать что бы сделалось, без UPDATE

    Returns:
        GeocodeBackfillResult с counters.
    """
    start = time.monotonic()
    result = GeocodeBackfillResult()

    # 1. Найти top-N адресов с NULL coords (DESC by occurrence count)
    rows = db.execute(
        text(
            """
            SELECT address, COUNT(*) AS listings_count
            FROM listings
            WHERE lat IS NULL
              AND address IS NOT NULL
              AND length(trim(address)) >= 5
            GROUP BY address
            ORDER BY listings_count DESC, address ASC
            LIMIT :limit
            """
        ),
        {"limit": batch_size},
    ).mappings().all()

    result.addresses_total = len(rows)

    if not rows:
        logger.info("geocode_missing: 0 pending addresses — nothing to do")
        result.duration_sec = time.monotonic() - start
        return result

    logger.info(
        "geocode_missing: starting batch=%d total_pending_addresses=%d (top by listings count)",
        batch_size,
        result.addresses_total,
    )

    for idx, row in enumerate(rows):
        address: str = row["address"]
        listings_count: int = row["listings_count"]
        result.addresses_processed += 1

        try:
            geo = await geocode(address, db)
        except Exception as exc:
            logger.warning(
                "geocode_missing: geocode raised for '%s': %s", address[:60], exc
            )
            result.addresses_failed += 1
            continue

        if geo is None:
            result.addresses_failed += 1
            logger.info(
                "geocode_missing: NOT FOUND '%s' (used in %d listings)",
                address[:60],
                listings_count,
            )
            continue

        if geo.provider == "cache":
            result.cache_hits += 1
        else:
            result.cache_misses += 1

        result.addresses_geocoded += 1

        if dry_run:
            logger.info(
                "geocode_missing[dry]: '%s' → (%.5f, %.5f) provider=%s would update %d listings",
                address[:60],
                geo.lat,
                geo.lon,
                geo.provider,
                listings_count,
            )
            continue

        # UPDATE listings — PostGIS trigger (listings_set_geom_trg) обновит geom автоматически
        update_result = db.execute(
            text(
                """
                UPDATE listings
                SET lat = :lat, lon = :lon
                WHERE address = :addr AND lat IS NULL
                """
            ),
            {"lat": geo.lat, "lon": geo.lon, "addr": address},
        )
        db.commit()
        result.listings_updated += update_result.rowcount

        if (idx + 1) % 50 == 0:
            elapsed = time.monotonic() - start
            rate = (idx + 1) / elapsed if elapsed > 0 else 0
            logger.info(
                "geocode_missing: progress %d/%d "
                "(geocoded=%d failed=%d cache_hits=%d listings_updated=%d) rate=%.1f addr/s",
                idx + 1,
                len(rows),
                result.addresses_geocoded,
                result.addresses_failed,
                result.cache_hits,
                result.listings_updated,
                rate,
            )

    result.duration_sec = time.monotonic() - start

    logger.info(
        "geocode_missing: DONE batch=%d processed=%d geocoded=%d failed=%d "
        "cache=(hit=%d miss=%d) listings_updated=%d duration=%.1fs",
        batch_size,
        result.addresses_processed,
        result.addresses_geocoded,
        result.addresses_failed,
        result.cache_hits,
        result.cache_misses,
        result.listings_updated,
        result.duration_sec,
    )

    return result