feat(tradein/cian): valuation backfill + runner script для prod

tradein-mvp/backend/app/api/v1/admin.py

@@ -1035,6 +1035,10 @@ class CianBackfillResp(BaseModel):
     houses_succeeded: int
     houses_failed_fetch: int
     houses_failed_save: int
+    valuations_total: int
+    valuations_processed: int
+    valuations_succeeded: int
+    valuations_failed: int
     duration_sec: float
 
 
@@ -1043,6 +1047,7 @@ async def scrape_cian_backfill_history(
     batch_size: int = Query(50, ge=1, le=200),
     do_listings: bool = True,
     do_houses: bool = True,
+    do_valuations: bool = False,
     dry_run: bool = False,
     db: Session = Depends(get_db),  # noqa: B008
 ) -> CianBackfillResp:
@@ -1065,6 +1070,7 @@ async def scrape_cian_backfill_history(
         batch_size=batch_size,
         do_listings=do_listings,
         do_houses=do_houses,
+        do_valuations=do_valuations,
         dry_run=dry_run,
     )
     return CianBackfillResp(ok=True, **result.__dict__)

tradein-mvp/backend/app/tasks/cian_history_backfill.py

@@ -29,6 +29,7 @@ from sqlalchemy.orm import Session
 
 from app.services.scraper_settings import get_scraper_delay
 from app.services.scrapers.cian_detail import fetch_detail, save_detail_enrichment
+from app.services.scrapers.cian_valuation import estimate_via_cian_valuation
 
 logger = logging.getLogger(__name__)
 
@@ -46,6 +47,10 @@ class CianBackfillResult:
     houses_succeeded: int = 0
     houses_failed_fetch: int = 0
     houses_failed_save: int = 0
+    valuations_total: int = 0
+    valuations_processed: int = 0
+    valuations_succeeded: int = 0
+    valuations_failed: int = 0
     duration_sec: float = field(default=0.0)
 
 
@@ -55,6 +60,7 @@ async def backfill_cian_history(
     batch_size: int = 50,
     do_listings: bool = True,
     do_houses: bool = True,
+    do_valuations: bool = False,
     dry_run: bool = False,
 ) -> CianBackfillResult:
     """Iterate Cian listings + houses with missing history, fetch+save.
@@ -62,10 +68,13 @@ async def backfill_cian_history(
     Args:
         db: SQLAlchemy session (caller-owned; each entity commits internally via
             save_detail_enrichment / save_newbuilding_enrichment).
-        batch_size: max entities to process per call (listings + houses counted separately).
+        batch_size: max entities to process per call (listings + houses + valuations counted
+            separately).
         do_listings: process listings batch (offer_price_history backfill).
         do_houses: reserved for future houses_price_dynamics backfill (currently no-op —
             see module docstring TODO).
+        do_valuations: process Cian Valuation Calculator batch (external_valuations backfill).
+            Default False — opt-in because each call hits Cian auth-gated API.
         dry_run: skip all fetch+save; only count and log pending rows.
 
     Returns:
@@ -152,10 +161,66 @@ async def backfill_cian_history(
             "(see TODO in cian_history_backfill.py)"
         )
 
+    # ── 3. Cian listings без external_valuations (price prediction backfill) ──
+    if do_valuations:
+        rows = db.execute(
+            text("""
+                SELECT l.id, l.address, l.area_m2, l.rooms, l.floor, l.total_floors
+                FROM listings l
+                WHERE l.source = 'cian'
+                  AND l.address IS NOT NULL
+                  AND l.area_m2 IS NOT NULL
+                  AND l.rooms IS NOT NULL
+                  AND l.floor IS NOT NULL
+                  AND l.total_floors IS NOT NULL
+                  AND NOT EXISTS (
+                      SELECT 1 FROM external_valuations ev
+                      WHERE ev.source = 'cian_valuation'
+                        AND ev.listing_id = l.id
+                        AND ev.expires_at > NOW()
+                  )
+                LIMIT :lim
+            """),
+            {"lim": batch_size},
+        ).mappings().all()
+        result.valuations_total = len(rows)
+
+        if dry_run:
+            logger.info(
+                "dry_run: would process %d cian listings for valuation", result.valuations_total
+            )
+        else:
+            for row in rows:
+                result.valuations_processed += 1
+                try:
+                    cval = await estimate_via_cian_valuation(
+                        db,
+                        address=row["address"],
+                        total_area=float(row["area_m2"]),
+                        rooms_count=int(row["rooms"]),
+                        floor=int(row["floor"]),
+                        total_floors=int(row["total_floors"]),
+                        repair_type="cosmetic",
+                        deal_type="sale",
+                        use_cache=False,  # force fresh fetch — populate cache
+                        listing_id=int(row["id"]),  # canonical link via mig 044
+                    )
+                    if cval is not None and cval.sale_price_rub:
+                        result.valuations_succeeded += 1
+                    else:
+                        result.valuations_failed += 1
+                except Exception as exc:
+                    logger.warning(
+                        "cian_valuation backfill failed for listing_id=%s: %s", row["id"], exc
+                    )
+                    result.valuations_failed += 1
+                await asyncio.sleep(delay)
+
     result.duration_sec = time.time() - t0
     logger.info(
         "cian_backfill done: listings=%d/%d (ok=%d fetch_fail=%d save_fail=%d "
-        "price_rows=%d) houses=%d/%d (ok=%d fail=%d) %.1fs",
+        "price_rows=%d) houses=%d/%d (ok=%d fail=%d) "
+        "valuations=%d/%d (ok=%d fail=%d) %.1fs",
         result.listings_processed,
         result.listings_total,
         result.listings_succeeded,
@@ -166,6 +231,10 @@ async def backfill_cian_history(
         result.houses_total,
         result.houses_succeeded,
         result.houses_failed_fetch + result.houses_failed_save,
+        result.valuations_processed,
+        result.valuations_total,
+        result.valuations_succeeded,
+        result.valuations_failed,
         result.duration_sec,
     )
     return result

tradein-mvp/scripts/cian_backfill_all.sh

@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+API="${TRADEIN_API_URL:-http://localhost:8000}"
+BATCH="${BATCH_SIZE:-100}"
+MAX_ITERS="${MAX_ITERS:-1000}"
+DO_LISTINGS="${DO_LISTINGS:-true}"
+DO_VALUATIONS="${DO_VALUATIONS:-true}"
+DO_HOUSES="${DO_HOUSES:-false}"  # houses stubbed until cian_zhk_url migration
+
+usage() {
+  cat <<EOF
+Cian backfill runner — drains offer_price_history + external_valuations backlog.
+
+Usage:
+  $0 [--dry-run]
+
+Env vars:
+  TRADEIN_API_URL  default: http://localhost:8000
+  BATCH_SIZE       default: 100   (max 200 per endpoint cap)
+  MAX_ITERS        default: 1000  (safety stop)
+  DO_LISTINGS      default: true
+  DO_VALUATIONS    default: true
+  DO_HOUSES        default: false (no cian_zhk_url col yet)
+
+Examples:
+  $0 --dry-run                       # count what would be processed
+  BATCH_SIZE=50 $0                   # drain at half pace
+  DO_VALUATIONS=false $0             # only history
+  TRADEIN_API_URL=https://gendsgn.ru/api $0
+EOF
+}
+
+DRY=""
+if [[ "${1:-}" == "--dry-run" ]]; then DRY="&dry_run=true"; fi
+if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then usage; exit 0; fi
+
+echo "Cian backfill: API=$API BATCH=$BATCH listings=$DO_LISTINGS valuations=$DO_VALUATIONS houses=$DO_HOUSES dry=${DRY:-no}"
+
+cum_listings=0
+cum_valuations=0
+i=0
+while (( i < MAX_ITERS )); do
+  i=$((i+1))
+  url="$API/api/v1/admin/scrape/cian-backfill-history?batch_size=$BATCH&do_listings=$DO_LISTINGS&do_valuations=$DO_VALUATIONS&do_houses=$DO_HOUSES${DRY}"
+  resp=$(curl -fsS -X POST "$url")
+  lt=$(echo "$resp" | jq -r '.listings_total // 0')
+  vt=$(echo "$resp" | jq -r '.valuations_total // 0')
+  ls=$(echo "$resp" | jq -r '.listings_succeeded // 0')
+  vs=$(echo "$resp" | jq -r '.valuations_succeeded // 0')
+  dur=$(echo "$resp" | jq -r '.duration_sec // 0')
+  cum_listings=$((cum_listings + ls))
+  cum_valuations=$((cum_valuations + vs))
+  echo "[$i] listings=$lt (ok=$ls) valuations=$vt (ok=$vs) ${dur}s  cum: hist=$cum_listings val=$cum_valuations"
+  if (( lt == 0 && vt == 0 )); then
+    echo "Drained — stopping."
+    break
+  fi
+  if [[ -n "$DRY" ]]; then
+    echo "Dry-run — no DB writes; stopping after first probe."
+    break
+  fi
+done
+echo "Done. Total: history=$cum_listings valuations=$cum_valuations iterations=$i"