eb564cfc01
Migrates legacy app.services.scrapers.* imports to scraper_kit equivalents for house_imv_backfill.py, avito_detail_backfill.py, cian_history_backfill.py, ekb_geoportal_ingest.py, and yandex_detail_backfill.py, proving parity via tests/support/parity.assert_parity per the epic's gate (#2304). newbuilding_enrich_backfill.py and yandex_newbuilding_sweep.py are left fully on legacy imports: both call into scraper_kit.providers.{cian,yandex}.newbuilding, which construct BrowserFetcher(source=...) without the now-mandatory endpoint= kwarg (issue #2322, verified still open against the actual provider source, not just issue status) -- no caller-side fix is possible, matching Group A's (#2305) precedent for the same bug in admin.py. Config-gated kit function footguns found and fixed (Group B #2306 pattern): - avito fetch_detail's backconnect-on-403 retry silently drops when config= is omitted -- now passes config=RealScraperConfig() explicitly, with a regression test proving the gate. - kit AvitoScraper's constructor now requires ScraperConfig positionally -- wired via RealScraperConfig(), which _rotate_ip() reads for avito_proxy_rotate_url. - BrowserFetcher(source=...) call sites (house_imv_backfill, avito/cian detail backfills) now pass the mandatory endpoint=settings.browser_http_endpoint. New footgun discovered (NOT #2322, flagged for follow-up): kit's build_warmed_session() builds its curl_cffi session via _build_detail_session() with no config parameter at all, unlike fetch_detail -- migrating it would silently drop the sticky MGTS-proxy egress on avito_detail_backfill's warm-batch path (the prod default). Left build_warmed_session/ _AVITO_WARM_SEARCH_URL on legacy imports, documented inline. Refs #2310
301 lines
12 KiB
Python
301 lines
12 KiB
Python
"""Scheduled backfill: detail-enrichment for legacy yandex listings (#1553).
|
|
|
|
Nightly window 12:00-15:00 UTC (migration 113, source=yandex_detail_backfill).
|
|
Offset from avito_detail_backfill (09-12 UTC) to avoid parallel egress on shared IP.
|
|
|
|
Problem: ~3952 yandex listings have detail_enriched_at IS NULL.
|
|
Yandex city sweep does not call YandexDetailScraper — SERP data only.
|
|
area_m2 coverage 23%, living/kitchen 0%, repair_state 1% on prod.
|
|
|
|
Solution: single snapshot SELECT at start (guarantees termination), fetch each
|
|
offer detail page via curl_cffi AsyncSession (chrome120 + proxy) — mirrors
|
|
yandex_address_backfill.py which already gets full HTML from Yandex on prod.
|
|
Parse HTML via YandexDetailScraper.parse (pure, no network). Persist via
|
|
save_detail_enrichment. Track consecutive parse→None results; abort after
|
|
max_consecutive_blocks (mark_done, not mark_failed — retry next night via
|
|
NULL detail_enriched_at).
|
|
|
|
Why curl_cffi and not YandexDetailScraper.fetch_detail:
|
|
fetch_detail uses BaseScraper._http_get (plain httpx, no proxy, no TLS
|
|
fingerprinting). On datacenter IPs Yandex returns captcha / shell-HTML
|
|
→ parse always returns None → backfill would be 0% effective. The
|
|
curl_cffi path (chrome120 impersonation + mobile proxy) is already proven
|
|
by yandex_address_backfill, which fetches identical offer detail pages.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import logging
|
|
import time
|
|
from dataclasses import dataclass, field
|
|
|
|
from curl_cffi.requests import AsyncSession
|
|
from scraper_kit.providers.yandex.detail import YandexDetailScraper, save_detail_enrichment
|
|
from sqlalchemy import text
|
|
from sqlalchemy.orm import Session
|
|
|
|
from app.core.config import settings
|
|
from app.services import scrape_runs as runs_mod
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
__all__ = [
|
|
"YandexDetailBackfillResult",
|
|
"run_yandex_detail_backfill",
|
|
]
|
|
|
|
|
|
@dataclass
|
|
class YandexDetailBackfillResult:
|
|
"""Counters for one yandex detail backfill run."""
|
|
|
|
attempted: int = 0
|
|
enriched: int = 0
|
|
failed: int = 0
|
|
duration_sec: float = field(default=0.0)
|
|
|
|
def to_dict(self) -> dict[str, int]:
|
|
return {
|
|
"attempted": self.attempted,
|
|
"enriched": self.enriched,
|
|
"failed": self.failed,
|
|
"duration_sec": int(self.duration_sec),
|
|
}
|
|
|
|
|
|
async def run_yandex_detail_backfill(
|
|
db: Session,
|
|
*,
|
|
run_id: int,
|
|
params: dict,
|
|
) -> YandexDetailBackfillResult:
|
|
"""Backfill detail_enriched_at for legacy yandex listings via curl_cffi + parse.
|
|
|
|
Params (from default_params jsonb in scrape_schedules):
|
|
batch_size: int -- snapshot size (SELECT LIMIT), default 800.
|
|
budget_sec: float -- wall-clock budget per run, default 3600s.
|
|
request_delay_sec: float -- delay between listings, default 5s.
|
|
max_consecutive_blocks: int -- consecutive parse→None before abort, default 5.
|
|
|
|
Fetch mechanism:
|
|
curl_cffi AsyncSession(impersonate="chrome120") + scraper_proxy_url — mirrors
|
|
yandex_address_backfill. On HTTP 200: pass resp.text to
|
|
YandexDetailScraper().parse(html, offer_url). parse→None counts as a fail
|
|
(possible captcha wall); consecutive None → abort after max_consecutive_blocks.
|
|
|
|
Lifecycle: update_heartbeat -> snapshot -> loop with budget guard ->
|
|
mark_done (incl. partial / consecutive-None abort) / mark_failed (exception only).
|
|
"""
|
|
batch_size = int(params.get("batch_size", 800))
|
|
budget_sec = float(params.get("budget_sec", 3600))
|
|
request_delay_sec = float(params.get("request_delay_sec", 5.0))
|
|
max_consecutive_blocks = int(params.get("max_consecutive_blocks", 5))
|
|
|
|
counters = YandexDetailBackfillResult()
|
|
current_counters: dict[str, int] = counters.to_dict()
|
|
|
|
start = time.monotonic()
|
|
|
|
try:
|
|
runs_mod.update_heartbeat(db, run_id, current_counters)
|
|
|
|
# SNAPSHOT: single SELECT at start -- NOT re-selected in loop.
|
|
# Priority: is_active DESC (active first), scraped_at DESC (newest first).
|
|
snapshot = (
|
|
db.execute(
|
|
text(
|
|
"""
|
|
SELECT id, source_url
|
|
FROM listings
|
|
WHERE source = 'yandex'
|
|
AND detail_enriched_at IS NULL
|
|
AND source_url IS NOT NULL
|
|
ORDER BY is_active DESC NULLS LAST, scraped_at DESC NULLS LAST
|
|
LIMIT CAST(:batch_size AS int)
|
|
"""
|
|
),
|
|
{"batch_size": batch_size},
|
|
)
|
|
.mappings()
|
|
.all()
|
|
)
|
|
|
|
if not snapshot:
|
|
logger.info(
|
|
"yandex_detail_backfill: run_id=%d -- no pending listings "
|
|
"(detail_enriched_at IS NULL = 0), done",
|
|
run_id,
|
|
)
|
|
runs_mod.mark_done(db, run_id, current_counters)
|
|
return counters
|
|
|
|
logger.info(
|
|
"yandex_detail_backfill: run_id=%d snapshot=%d (budget=%.0fs "
|
|
"delay=%.1fs max_consecutive_none=%d)",
|
|
run_id,
|
|
len(snapshot),
|
|
budget_sec,
|
|
request_delay_sec,
|
|
max_consecutive_blocks,
|
|
)
|
|
|
|
# Build proxies dict once — mirrors yandex_address_backfill.py
|
|
_proxy = settings.scraper_proxy_url
|
|
_proxies = {"http": _proxy, "https": _proxy} if _proxy else None
|
|
|
|
consecutive_none = 0
|
|
do_sleep = False
|
|
scraper = YandexDetailScraper()
|
|
|
|
async with AsyncSession(
|
|
impersonate="chrome120",
|
|
timeout=30.0,
|
|
proxies=_proxies,
|
|
headers={
|
|
"Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8",
|
|
},
|
|
) as session:
|
|
for idx, row in enumerate(snapshot):
|
|
# Budget guard
|
|
elapsed = time.monotonic() - start
|
|
if elapsed > budget_sec:
|
|
logger.info(
|
|
"yandex_detail_backfill: run_id=%d -- budget %.0fs exhausted "
|
|
"(elapsed=%.1fs), stopping at #%d/%d",
|
|
run_id,
|
|
budget_sec,
|
|
elapsed,
|
|
idx,
|
|
len(snapshot),
|
|
)
|
|
break
|
|
|
|
# Delay before each request except the first
|
|
if do_sleep:
|
|
await asyncio.sleep(request_delay_sec)
|
|
do_sleep = True
|
|
|
|
source_url: str = row["source_url"]
|
|
listing_id: int = row["id"]
|
|
counters.attempted += 1
|
|
|
|
try:
|
|
try:
|
|
resp = await session.get(source_url, allow_redirects=True)
|
|
except Exception as fetch_exc:
|
|
consecutive_none += 1
|
|
counters.failed += 1
|
|
logger.warning(
|
|
"yandex_detail_backfill: run_id=%d listing_id=%d "
|
|
"fetch error (consecutive=%d): %s",
|
|
run_id,
|
|
listing_id,
|
|
consecutive_none,
|
|
fetch_exc,
|
|
)
|
|
if consecutive_none >= max_consecutive_blocks:
|
|
logger.error(
|
|
"yandex_detail_backfill: run_id=%d ABORT -- %d consecutive "
|
|
"errors. enriched=%d attempted=%d",
|
|
run_id,
|
|
consecutive_none,
|
|
counters.enriched,
|
|
counters.attempted,
|
|
)
|
|
break
|
|
continue
|
|
|
|
if resp.status_code != 200:
|
|
consecutive_none += 1
|
|
counters.failed += 1
|
|
logger.warning(
|
|
"yandex_detail_backfill: run_id=%d listing_id=%d "
|
|
"HTTP %d (consecutive=%d)",
|
|
run_id,
|
|
listing_id,
|
|
resp.status_code,
|
|
consecutive_none,
|
|
)
|
|
if consecutive_none >= max_consecutive_blocks:
|
|
logger.error(
|
|
"yandex_detail_backfill: run_id=%d ABORT -- %d consecutive "
|
|
"non-200 responses. enriched=%d attempted=%d",
|
|
run_id,
|
|
consecutive_none,
|
|
counters.enriched,
|
|
counters.attempted,
|
|
)
|
|
break
|
|
continue
|
|
|
|
enrichment = scraper.parse(resp.text, offer_url=source_url)
|
|
|
|
if enrichment is None:
|
|
# parse→None: captcha wall / shell-HTML / no JSON-LD.
|
|
# Do not mark listing as done — retry next night.
|
|
consecutive_none += 1
|
|
counters.failed += 1
|
|
logger.warning(
|
|
"yandex_detail_backfill: run_id=%d listing_id=%d source_url=%s "
|
|
"-> parse None (consecutive=%d)",
|
|
run_id,
|
|
listing_id,
|
|
source_url,
|
|
consecutive_none,
|
|
)
|
|
if consecutive_none >= max_consecutive_blocks:
|
|
logger.error(
|
|
"yandex_detail_backfill: run_id=%d ABORT -- %d consecutive "
|
|
"parse-None results (captcha wall?). enriched=%d attempted=%d",
|
|
run_id,
|
|
consecutive_none,
|
|
counters.enriched,
|
|
counters.attempted,
|
|
)
|
|
break
|
|
continue
|
|
|
|
consecutive_none = 0
|
|
if save_detail_enrichment(db, listing_id, enrichment):
|
|
counters.enriched += 1
|
|
|
|
except Exception as exc:
|
|
counters.failed += 1
|
|
logger.warning(
|
|
"yandex_detail_backfill: save/iteration error for listing_id=%d: %s",
|
|
listing_id,
|
|
exc,
|
|
)
|
|
try:
|
|
db.rollback()
|
|
except Exception:
|
|
pass
|
|
|
|
if counters.attempted % 25 == 0:
|
|
current_counters = counters.to_dict()
|
|
runs_mod.update_heartbeat(db, run_id, current_counters)
|
|
|
|
counters.duration_sec = time.monotonic() - start
|
|
current_counters = counters.to_dict()
|
|
runs_mod.mark_done(db, run_id, current_counters)
|
|
logger.info(
|
|
"yandex_detail_backfill: run_id=%d DONE -- attempted=%d enriched=%d "
|
|
"failed=%d duration=%.1fs",
|
|
run_id,
|
|
counters.attempted,
|
|
counters.enriched,
|
|
counters.failed,
|
|
counters.duration_sec,
|
|
)
|
|
return counters
|
|
|
|
except Exception as exc:
|
|
counters.duration_sec = time.monotonic() - start
|
|
logger.exception(
|
|
"yandex_detail_backfill: run_id=%d FAILED after %.1fs",
|
|
run_id,
|
|
counters.duration_sec,
|
|
)
|
|
runs_mod.mark_failed(db, run_id, str(exc)[:1000], counters.to_dict())
|
|
raise
|