gendesign/tradein-mvp/backend/app/services/cian_price_history.py

"""T8a: Cian offer price-history backfill service.

Fetches Cian detail pages (curl_cffi, без Playwright) for listings that have
no rows in offer_price_history, extracts priceChanges from
_cianConfig['frontend-offer-card'] defaultState, writes to offer_price_history.

Deliberately NOT wired into scheduler — rate-limit risk with datacenter IPs.
Triggered manually via POST /api/v1/admin/scrape/cian-price-history.
"""

from __future__ import annotations

import asyncio
import logging
import time
from dataclasses import dataclass, field

# #2306: fetch_detail/save_detail_enrichment migrated to scraper_kit (byte-identical
# golden-parity была доказана против legacy app.services.scrapers.cian_detail до его
# удаления, #2397 Part E2 — см. tests/test_scraper_kit_pricehistory_session_parity.py
# для оставшейся extract_state/ScrapedLot parity). RealScraperConfig — тот же
# read-only адаптер над settings, что и остальные kit-инжекции (#2131) — сохраняет
# proxy-поведение (config.cian_proxy_url) идентичным прежнему прямому импорту settings.
from scraper_kit.providers.cian.detail import fetch_detail, save_detail_enrichment
from sqlalchemy import text
from sqlalchemy.orm import Session

from app.services.scraper_adapters import RealScraperConfig
from app.services.scraper_settings import get_scraper_delay

logger = logging.getLogger(__name__)


@dataclass
class CianPriceHistoryResult:
    checked: int = 0
    saved: int = 0
    skipped: int = 0
    errors: int = 0
    duration_sec: float = field(default=0.0)


async def backfill_cian_price_history(
    db: Session,
    *,
    batch_size: int = 50,
    listing_id: int | None = None,
) -> CianPriceHistoryResult:
    """Fetch Cian detail pages and write missing price-history rows.

    Args:
        db: SQLAlchemy session (caller-owned; commits internally per listing).
        batch_size: max listings to process when listing_id is None.
        listing_id: process a single specific listing (ignores batch_size).

    Selection query picks cian listings with no existing offer_price_history rows.
    Idempotent: re-run safe via ON CONFLICT DO NOTHING in save_detail_enrichment.
    """
    result = CianPriceHistoryResult()
    t0 = time.time()
    delay = get_scraper_delay("cian")  # default 5.0s

    if listing_id is not None:
        rows = (
            db.execute(
                text("""
                    SELECT l.id, l.source_url
                    FROM listings l
                    WHERE l.id = CAST(:lid AS bigint)
                      AND l.source = 'cian'
                      AND l.source_url IS NOT NULL
                """),
                {"lid": listing_id},
            )
            .mappings()
            .all()
        )
    else:
        rows = (
            db.execute(
                text("""
                    SELECT l.id, l.source_url
                    FROM listings l
                    LEFT JOIN offer_price_history oph ON oph.listing_id = l.id
                    WHERE l.source = 'cian'
                      AND l.source_url IS NOT NULL
                      AND oph.listing_id IS NULL
                    ORDER BY l.id
                    LIMIT :lim
                """),
                {"lim": batch_size},
            )
            .mappings()
            .all()
        )

    result.checked = len(rows)
    logger.info(
        "cian_price_history backfill: checked=%d delay=%.1fs",
        result.checked,
        delay,
    )

    for i, row in enumerate(rows):
        lid: int = row["id"]
        url: str = row["source_url"]

        try:
            # config= обязателен — kit fetch_detail без него не читает cian_proxy_url
            # (direct connection), а без прокси datacenter-IP блокируется Cian (#806).
            enrichment = await fetch_detail(url, config=RealScraperConfig())
        except Exception as exc:
            logger.warning(
                "cian_price_history: fetch failed listing_id=%s url=%s: %s",
                lid,
                url,
                exc,
            )
            result.errors += 1
            await asyncio.sleep(delay)
            continue

        if enrichment is None:
            logger.warning(
                "cian_price_history: fetch returned None listing_id=%s url=%s",
                lid,
                url,
            )
            result.errors += 1
            await asyncio.sleep(delay)
            continue

        if not enrichment.price_changes:
            logger.debug("cian_price_history: no price_changes listing_id=%s", lid)
            result.skipped += 1
        else:
            try:
                # Count rows actually inserted: save_detail_enrichment skips
                # changes without change_time/price_rub and uses ON CONFLICT
                # DO NOTHING, so len(price_changes) overcounts on invalid
                # elements or idempotent re-runs. Diff the row count instead.
                before = db.execute(
                    text(
                        "SELECT COUNT(*) FROM offer_price_history "
                        "WHERE listing_id = CAST(:lid AS bigint)"
                    ),
                    {"lid": lid},
                ).scalar_one()
                save_detail_enrichment(db, lid, enrichment)
                after = db.execute(
                    text(
                        "SELECT COUNT(*) FROM offer_price_history "
                        "WHERE listing_id = CAST(:lid AS bigint)"
                    ),
                    {"lid": lid},
                ).scalar_one()
                result.saved += max(0, int(after) - int(before))
            except Exception as exc:
                logger.warning("cian_price_history: save failed listing_id=%s: %s", lid, exc)
                result.errors += 1
                try:
                    db.rollback()
                except Exception:
                    pass
                await asyncio.sleep(delay)
                continue

        if i < len(rows) - 1:
            await asyncio.sleep(delay)

    result.duration_sec = time.time() - t0
    logger.info(
        "cian_price_history done: checked=%d saved=%d skipped=%d errors=%d %.1fs",
        result.checked,
        result.saved,
        result.skipped,
        result.errors,
        result.duration_sec,
    )
    return result