"""Scheduled task: backfill listings.address for EKB yandex listings with street-only addresses. Pilot scope: Екатеринбург only (region_code=66, city hardcoded). Yandex SERP cards include only the street name (e.g. «улица Горького»); the offer detail page contains the full address including house number: «Продажа квартиры — Екатеринбург, улица Горького, 36 — id 7654321 на Яндекс.Недвижимости» This task selects active listings that match the predicate (source='yandex', region_code=66, address IS NOT NULL, address has no house number, source_url IS NOT NULL), fetches each detail page via curl_cffi(chrome120), extracts the full address from the <title>, and UPDATE listings.address ONLY when the extracted value differs from the current one. Wired into the in-app scheduler as source='yandex_address_backfill'. Triggered nightly via the scrape_schedules seed (migration 091). Manual trigger also available via POST /api/v1/admin/scrape/yandex-address-backfill. Delegates actual HTTP + DB work to app.services.yandex_address_backfill.backfill_yandex_addresses (the proven service module that already implements RE_TITLE_ADDRESS, _extract_address_from_title, and _eligible_listings). """ from __future__ import annotations import logging from dataclasses import dataclass, field from sqlalchemy.orm import Session from app.services import scrape_runs as runs_mod from app.services.yandex_address_backfill import ( YandexAddressBackfillResult, backfill_yandex_addresses, ) logger = logging.getLogger(__name__) __all__ = [ "YandexAddressBackfillTaskResult", "run_yandex_address_backfill", ] @dataclass class YandexAddressBackfillTaskResult: """Result counters from a scheduled backfill run.""" checked: int = 0 saved: int = 0 skipped: int = 0 errors: int = 0 duration_sec: float = field(default=0.0) async def run_yandex_address_backfill( db: Session, *, run_id: int, params: dict, ) -> YandexAddressBackfillTaskResult: """Execute Yandex address backfill with run lifecycle management. Wraps backfill_yandex_addresses(), emitting heartbeat before the batch call and marking the scrape_run as done/failed on completion. Params (from default_params jsonb in scrape_schedules): batch_size: int — max listings to process (default 200). request_delay_sec: float — seconds between requests (default 3.0). """ batch_size = int(params.get("batch_size", 200)) request_delay_sec = float(params.get("request_delay_sec", 3.0)) counters: dict[str, int] = { "checked": 0, "saved": 0, "skipped": 0, "errors": 0, } try: runs_mod.update_heartbeat(db, run_id, counters) result: YandexAddressBackfillResult = await backfill_yandex_addresses( db, limit=batch_size, request_delay_sec=request_delay_sec, ) counters = { "checked": result.checked, "saved": result.saved, "skipped": result.skipped, "errors": result.errors, "duration_sec": int(result.duration_sec), } runs_mod.mark_done(db, run_id, counters) logger.info( "scheduler: yandex_address_backfill run_id=%d done — " "checked=%d saved=%d skipped=%d errors=%d %.1fs", run_id, result.checked, result.saved, result.skipped, result.errors, result.duration_sec, ) return YandexAddressBackfillTaskResult( checked=result.checked, saved=result.saved, skipped=result.skipped, errors=result.errors, duration_sec=result.duration_sec, ) except Exception as exc: logger.exception("scheduler: yandex_address_backfill run_id=%d failed", run_id) runs_mod.mark_failed(db, run_id, str(exc)[:1000], counters) raise