"""T8a: Cian offer price-history backfill service. Fetches Cian detail pages (curl_cffi, без Playwright) for listings that have no rows in offer_price_history, extracts priceChanges from _cianConfig['frontend-offer-card'] defaultState, writes to offer_price_history. Deliberately NOT wired into scheduler — rate-limit risk with datacenter IPs. Triggered manually via POST /api/v1/admin/scrape/cian-price-history. """ from __future__ import annotations import asyncio import logging import time from dataclasses import dataclass, field from sqlalchemy import text from sqlalchemy.orm import Session from app.services.scraper_settings import get_scraper_delay from app.services.scrapers.cian_detail import fetch_detail, save_detail_enrichment logger = logging.getLogger(__name__) @dataclass class CianPriceHistoryResult: checked: int = 0 saved: int = 0 skipped: int = 0 errors: int = 0 duration_sec: float = field(default=0.0) async def backfill_cian_price_history( db: Session, *, batch_size: int = 50, listing_id: int | None = None, ) -> CianPriceHistoryResult: """Fetch Cian detail pages and write missing price-history rows. Args: db: SQLAlchemy session (caller-owned; commits internally per listing). batch_size: max listings to process when listing_id is None. listing_id: process a single specific listing (ignores batch_size). Selection query picks cian listings with no existing offer_price_history rows. Idempotent: re-run safe via ON CONFLICT DO NOTHING in save_detail_enrichment. """ result = CianPriceHistoryResult() t0 = time.time() delay = get_scraper_delay("cian") # default 5.0s if listing_id is not None: rows = ( db.execute( text(""" SELECT l.id, l.source_url FROM listings l WHERE l.id = CAST(:lid AS bigint) AND l.source = 'cian' AND l.source_url IS NOT NULL """), {"lid": listing_id}, ) .mappings() .all() ) else: rows = ( db.execute( text(""" SELECT l.id, l.source_url FROM listings l LEFT JOIN offer_price_history oph ON oph.listing_id = l.id WHERE l.source = 'cian' AND l.source_url IS NOT NULL AND oph.listing_id IS NULL ORDER BY l.id LIMIT :lim """), {"lim": batch_size}, ) .mappings() .all() ) result.checked = len(rows) logger.info( "cian_price_history backfill: checked=%d delay=%.1fs", result.checked, delay, ) for i, row in enumerate(rows): lid: int = row["id"] url: str = row["source_url"] try: enrichment = await fetch_detail(url) except Exception as exc: logger.warning( "cian_price_history: fetch failed listing_id=%s url=%s: %s", lid, url, exc, ) result.errors += 1 await asyncio.sleep(delay) continue if enrichment is None: logger.warning( "cian_price_history: fetch returned None listing_id=%s url=%s", lid, url, ) result.errors += 1 await asyncio.sleep(delay) continue if not enrichment.price_changes: logger.debug("cian_price_history: no price_changes listing_id=%s", lid) result.skipped += 1 else: try: save_detail_enrichment(db, lid, enrichment) result.saved += len(enrichment.price_changes) except Exception as exc: logger.warning("cian_price_history: save failed listing_id=%s: %s", lid, exc) result.errors += 1 try: db.rollback() except Exception: pass await asyncio.sleep(delay) continue if i < len(rows) - 1: await asyncio.sleep(delay) result.duration_sec = time.time() - t0 logger.info( "cian_price_history done: checked=%d saved=%d skipped=%d errors=%d %.1fs", result.checked, result.saved, result.skipped, result.errors, result.duration_sec, ) return result