bot-backend
5a9fb806bb
The backfill service (app/services/yandex_address_backfill.py) + its manual admin endpoint already exist, but it was never wired into the in-app scheduler — so residual street-only yandex listings (no house number) were only fixed on manual trigger. Add a recurring scheduler task: trigger_yandex_address_backfill_run mirrors trigger_cian_backfill_run (claim run + mark done/failed), new task wrapper app/tasks/yandex_address_backfill.py delegates to the existing backfill_yandex_addresses(), dispatch branch + seed migration 091 (window 02:00-03:00 UTC, ON CONFLICT(source) idempotent). EKB-only (region_code=66). Refs #855, #726
116 lines
4 KiB
Python
116 lines
4 KiB
Python
"""Scheduled task: backfill listings.address for EKB yandex listings with street-only addresses.
|
|
|
|
Pilot scope: Екатеринбург only (region_code=66, city hardcoded).
|
|
Yandex SERP cards include only the street name (e.g. «улица Горького»); the offer detail
|
|
page <title> contains the full address including house number:
|
|
«Продажа квартиры — Екатеринбург, улица Горького, 36 — id 7654321 на Яндекс.Недвижимости»
|
|
|
|
This task selects active listings that match the predicate (source='yandex', region_code=66,
|
|
address IS NOT NULL, address has no house number, source_url IS NOT NULL), fetches each
|
|
detail page via curl_cffi(chrome120), extracts the full address from the <title>, and
|
|
UPDATE listings.address ONLY when the extracted value differs from the current one.
|
|
|
|
Wired into the in-app scheduler as source='yandex_address_backfill'. Triggered nightly via
|
|
the scrape_schedules seed (migration 091). Manual trigger also available via POST
|
|
/api/v1/admin/scrape/yandex-address-backfill.
|
|
|
|
Delegates actual HTTP + DB work to app.services.yandex_address_backfill.backfill_yandex_addresses
|
|
(the proven service module that already implements RE_TITLE_ADDRESS, _extract_address_from_title,
|
|
and _eligible_listings).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
from dataclasses import dataclass, field
|
|
|
|
from sqlalchemy.orm import Session
|
|
|
|
from app.services import scrape_runs as runs_mod
|
|
from app.services.yandex_address_backfill import (
|
|
YandexAddressBackfillResult,
|
|
backfill_yandex_addresses,
|
|
)
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
__all__ = [
|
|
"YandexAddressBackfillTaskResult",
|
|
"run_yandex_address_backfill",
|
|
]
|
|
|
|
|
|
@dataclass
|
|
class YandexAddressBackfillTaskResult:
|
|
"""Result counters from a scheduled backfill run."""
|
|
|
|
checked: int = 0
|
|
saved: int = 0
|
|
skipped: int = 0
|
|
errors: int = 0
|
|
duration_sec: float = field(default=0.0)
|
|
|
|
|
|
async def run_yandex_address_backfill(
|
|
db: Session,
|
|
*,
|
|
run_id: int,
|
|
params: dict,
|
|
) -> YandexAddressBackfillTaskResult:
|
|
"""Execute Yandex address backfill with run lifecycle management.
|
|
|
|
Wraps backfill_yandex_addresses(), emitting heartbeat before the batch call
|
|
and marking the scrape_run as done/failed on completion.
|
|
|
|
Params (from default_params jsonb in scrape_schedules):
|
|
batch_size: int — max listings to process (default 200).
|
|
request_delay_sec: float — seconds between requests (default 3.0).
|
|
"""
|
|
batch_size = int(params.get("batch_size", 200))
|
|
request_delay_sec = float(params.get("request_delay_sec", 3.0))
|
|
|
|
counters: dict[str, int] = {
|
|
"checked": 0,
|
|
"saved": 0,
|
|
"skipped": 0,
|
|
"errors": 0,
|
|
}
|
|
|
|
try:
|
|
runs_mod.update_heartbeat(db, run_id, counters)
|
|
|
|
result: YandexAddressBackfillResult = await backfill_yandex_addresses(
|
|
db,
|
|
limit=batch_size,
|
|
request_delay_sec=request_delay_sec,
|
|
)
|
|
|
|
counters = {
|
|
"checked": result.checked,
|
|
"saved": result.saved,
|
|
"skipped": result.skipped,
|
|
"errors": result.errors,
|
|
"duration_sec": int(result.duration_sec),
|
|
}
|
|
runs_mod.mark_done(db, run_id, counters)
|
|
logger.info(
|
|
"scheduler: yandex_address_backfill run_id=%d done — "
|
|
"checked=%d saved=%d skipped=%d errors=%d %.1fs",
|
|
run_id,
|
|
result.checked,
|
|
result.saved,
|
|
result.skipped,
|
|
result.errors,
|
|
result.duration_sec,
|
|
)
|
|
return YandexAddressBackfillTaskResult(
|
|
checked=result.checked,
|
|
saved=result.saved,
|
|
skipped=result.skipped,
|
|
errors=result.errors,
|
|
duration_sec=result.duration_sec,
|
|
)
|
|
except Exception as exc:
|
|
logger.exception("scheduler: yandex_address_backfill run_id=%d failed", run_id)
|
|
runs_mod.mark_failed(db, run_id, str(exc)[:1000], counters)
|
|
raise
|