gendesign/tradein-mvp/backend/app/tasks/avito_detail_backfill.py

"""Scheduled backfill: detail-enrichment for legacy avito listings (#1551).

Nightly window 09:00-12:00 UTC (migration 112, source=avito_detail_backfill).
Offset from avito_city_sweep (~03-04 UTC) to avoid sharing proxy IP simultaneously.

Problem: ~15243/15917 avito listings have detail_enriched_at IS NULL.
City sweep ENRICH_DETAIL only enriches top-N proximate listings per anchor.
Legacy listings (older than 2h or outside radius) are never enriched.

Solution: single snapshot SELECT at start (guarantees termination), same proxy
session path as scrape_pipeline.py step 5. Block handling mirrors step 5:
rotate IP on every block, abort after max_consecutive_blocks (mark_done not
mark_failed -- block is temporary, retry next night via NULL detail_enriched_at).
"""

from __future__ import annotations

import asyncio
import logging
import time
from dataclasses import dataclass, field
from urllib.parse import urlparse

from curl_cffi.requests import AsyncSession
from sqlalchemy import text
from sqlalchemy.orm import Session

from app.core.config import settings
from app.services import scrape_runs as runs_mod
from app.services.scrape_pipeline import _CHROME_HEADERS, _avito_proxies
from app.services.scrapers.avito import AvitoScraper
from app.services.scrapers.avito_detail import fetch_detail, save_detail_enrichment
from app.services.scrapers.avito_exceptions import (
    AvitoBlockedError,
    AvitoRateLimitedError,
)
from app.services.scrapers.browser_fetcher import BrowserFetcher

logger = logging.getLogger(__name__)

__all__ = [
    "AvitoDetailBackfillResult",
    "run_avito_detail_backfill",
]


@dataclass
class AvitoDetailBackfillResult:
    """Counters for one backfill run."""

    attempted: int = 0
    enriched: int = 0
    blocked: int = 0
    failed: int = 0
    duration_sec: float = field(default=0.0)

    def to_dict(self) -> dict[str, int]:
        return {
            "attempted": self.attempted,
            "enriched": self.enriched,
            "blocked": self.blocked,
            "failed": self.failed,
            "duration_sec": int(self.duration_sec),
        }


async def run_avito_detail_backfill(
    db: Session,
    *,
    run_id: int,
    params: dict,
) -> AvitoDetailBackfillResult:
    """Backfill detail_enriched_at for legacy avito listings via mobile proxy.

    Params (from default_params jsonb in scrape_schedules):
        batch_size: int -- snapshot size (SELECT LIMIT), default 800.
        budget_sec: float -- wall-clock budget per run, default 3600s.
        request_delay_sec: float -- delay between listings, default 6.0s.
        max_consecutive_blocks: int -- abort threshold, default 5.

    Lifecycle: update_heartbeat -> snapshot -> loop with budget guard ->
    mark_done (incl. partial/block-abort) / mark_failed (exception only).
    """
    batch_size = int(params.get("batch_size", 800))
    budget_sec = float(params.get("budget_sec", 3600))
    request_delay_sec = float(params.get("request_delay_sec", 6.0))
    max_consecutive_blocks = int(params.get("max_consecutive_blocks", 5))

    counters = AvitoDetailBackfillResult()
    current_counters: dict[str, int] = counters.to_dict()

    browser_mode = settings.scraper_fetch_mode == "browser"
    session: AsyncSession | None = None
    browser_fetcher: BrowserFetcher | None = None
    own_session = False
    own_browser = False

    scraper = AvitoScraper()
    start = time.monotonic()

    try:
        # Setup session (mirrors run_avito_pipeline lines 161-183)
        if browser_mode:
            browser_fetcher = BrowserFetcher()
            await browser_fetcher.__aenter__()
            own_browser = True
            scraper._browser = browser_fetcher
        else:
            own_session = True
            session = AsyncSession(
                impersonate="chrome120",
                timeout=25,
                headers=_CHROME_HEADERS,
                proxies=_avito_proxies(),
            )
            scraper._cffi = session

        runs_mod.update_heartbeat(db, run_id, current_counters)

        # SNAPSHOT: single SELECT at start -- NOT re-selected in loop.
        # Priority: is_active DESC (active first), scraped_at DESC (newest first).
        snapshot = (
            db.execute(
                text(
                    """
                    SELECT id, source_url
                    FROM listings
                    WHERE source = 'avito'
                      AND detail_enriched_at IS NULL
                      AND source_url IS NOT NULL
                    ORDER BY is_active DESC NULLS LAST, scraped_at DESC NULLS LAST
                    LIMIT CAST(:batch_size AS int)
                    """
                ),
                {"batch_size": batch_size},
            )
            .mappings()
            .all()
        )

        if not snapshot:
            logger.info(
                "avito_detail_backfill: run_id=%d -- no pending listings "
                "(detail_enriched_at IS NULL = 0), done",
                run_id,
            )
            runs_mod.mark_done(db, run_id, current_counters)
            return counters

        logger.info(
            "avito_detail_backfill: run_id=%d snapshot=%d (budget=%.0fs "
            "delay=%.1fs max_blocks=%d browser=%s)",
            run_id,
            len(snapshot),
            budget_sec,
            request_delay_sec,
            max_consecutive_blocks,
            browser_mode,
        )

        consecutive_blocks = 0
        do_sleep = False

        for idx, row in enumerate(snapshot):
            # Budget guard
            elapsed = time.monotonic() - start
            if elapsed > budget_sec:
                logger.info(
                    "avito_detail_backfill: run_id=%d -- budget %.0fs exhausted "
                    "(elapsed=%.1fs), stopping at #%d/%d",
                    run_id,
                    budget_sec,
                    elapsed,
                    idx,
                    len(snapshot),
                )
                break

            # Delay before each request except the first
            if do_sleep:
                await asyncio.sleep(request_delay_sec)
            do_sleep = True

            source_url: str = row["source_url"]
            counters.attempted += 1

            # Normalise URL -> path (mirrors scrape_pipeline.py line 311)
            item_url = urlparse(source_url).path if source_url.startswith("http") else source_url

            try:
                enrichment = await fetch_detail(
                    item_url,
                    cffi_session=session,
                    browser_fetcher=browser_fetcher,
                )
                if save_detail_enrichment(db, enrichment):
                    counters.enriched += 1
                consecutive_blocks = 0

            except (AvitoBlockedError, AvitoRateLimitedError) as e:
                consecutive_blocks += 1
                counters.blocked += 1
                do_sleep = False
                logger.warning(
                    "avito_detail_backfill: run_id=%d BLOCKED #%d/%d (consecutive=%d): %s",
                    run_id,
                    idx + 1,
                    len(snapshot),
                    consecutive_blocks,
                    e,
                )
                await scraper._rotate_ip()
                if consecutive_blocks >= max_consecutive_blocks:
                    logger.error(
                        "avito_detail_backfill: run_id=%d ABORT -- %d consecutive blocks, "
                        "IP rate-limited. enriched=%d attempted=%d",
                        run_id,
                        consecutive_blocks,
                        counters.enriched,
                        counters.attempted,
                    )
                    break

            except Exception as e:
                counters.failed += 1
                logger.warning(
                    "avito_detail_backfill: run_id=%d listing %s failed: %s",
                    run_id,
                    source_url,
                    e,
                )
                try:
                    db.rollback()
                except Exception:
                    pass

            if counters.attempted % 25 == 0:
                current_counters = counters.to_dict()
                runs_mod.update_heartbeat(db, run_id, current_counters)

        counters.duration_sec = time.monotonic() - start
        current_counters = counters.to_dict()
        runs_mod.mark_done(db, run_id, current_counters)
        logger.info(
            "avito_detail_backfill: run_id=%d DONE -- attempted=%d enriched=%d "
            "blocked=%d failed=%d duration=%.1fs",
            run_id,
            counters.attempted,
            counters.enriched,
            counters.blocked,
            counters.failed,
            counters.duration_sec,
        )
        return counters

    except Exception as exc:
        counters.duration_sec = time.monotonic() - start
        logger.exception(
            "avito_detail_backfill: run_id=%d FAILED after %.1fs",
            run_id,
            counters.duration_sec,
        )
        runs_mod.mark_failed(db, run_id, str(exc)[:1000], counters.to_dict())
        raise

    finally:
        if own_session and session is not None:
            await session.close()
        if own_browser and browser_fetcher is not None:
            await browser_fetcher.__aexit__(None, None, None)