From da38cd0a5fd8eddcdf77715a7fd8936c9dc2ec60 Mon Sep 17 00:00:00 2001 From: lekss361 Date: Sun, 24 May 2026 18:07:17 +0300 Subject: [PATCH] =?UTF-8?q?feat(tradein/cian):=20valuation=20backfill=20+?= =?UTF-8?q?=20runner=20script=20=D0=B4=D0=BB=D1=8F=20prod?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tradein-mvp/backend/app/api/v1/admin.py | 6 ++ .../app/tasks/cian_history_backfill.py | 73 ++++++++++++++++++- tradein-mvp/scripts/cian_backfill_all.sh | 64 ++++++++++++++++ 3 files changed, 141 insertions(+), 2 deletions(-) create mode 100644 tradein-mvp/scripts/cian_backfill_all.sh diff --git a/tradein-mvp/backend/app/api/v1/admin.py b/tradein-mvp/backend/app/api/v1/admin.py index 695aabbf..18576cbd 100644 --- a/tradein-mvp/backend/app/api/v1/admin.py +++ b/tradein-mvp/backend/app/api/v1/admin.py @@ -1035,6 +1035,10 @@ class CianBackfillResp(BaseModel): houses_succeeded: int houses_failed_fetch: int houses_failed_save: int + valuations_total: int + valuations_processed: int + valuations_succeeded: int + valuations_failed: int duration_sec: float @@ -1043,6 +1047,7 @@ async def scrape_cian_backfill_history( batch_size: int = Query(50, ge=1, le=200), do_listings: bool = True, do_houses: bool = True, + do_valuations: bool = False, dry_run: bool = False, db: Session = Depends(get_db), # noqa: B008 ) -> CianBackfillResp: @@ -1065,6 +1070,7 @@ async def scrape_cian_backfill_history( batch_size=batch_size, do_listings=do_listings, do_houses=do_houses, + do_valuations=do_valuations, dry_run=dry_run, ) return CianBackfillResp(ok=True, **result.__dict__) diff --git a/tradein-mvp/backend/app/tasks/cian_history_backfill.py b/tradein-mvp/backend/app/tasks/cian_history_backfill.py index af81a736..b899b801 100644 --- a/tradein-mvp/backend/app/tasks/cian_history_backfill.py +++ b/tradein-mvp/backend/app/tasks/cian_history_backfill.py @@ -29,6 +29,7 @@ from sqlalchemy.orm import Session from app.services.scraper_settings import get_scraper_delay from app.services.scrapers.cian_detail import fetch_detail, save_detail_enrichment +from app.services.scrapers.cian_valuation import estimate_via_cian_valuation logger = logging.getLogger(__name__) @@ -46,6 +47,10 @@ class CianBackfillResult: houses_succeeded: int = 0 houses_failed_fetch: int = 0 houses_failed_save: int = 0 + valuations_total: int = 0 + valuations_processed: int = 0 + valuations_succeeded: int = 0 + valuations_failed: int = 0 duration_sec: float = field(default=0.0) @@ -55,6 +60,7 @@ async def backfill_cian_history( batch_size: int = 50, do_listings: bool = True, do_houses: bool = True, + do_valuations: bool = False, dry_run: bool = False, ) -> CianBackfillResult: """Iterate Cian listings + houses with missing history, fetch+save. @@ -62,10 +68,13 @@ async def backfill_cian_history( Args: db: SQLAlchemy session (caller-owned; each entity commits internally via save_detail_enrichment / save_newbuilding_enrichment). - batch_size: max entities to process per call (listings + houses counted separately). + batch_size: max entities to process per call (listings + houses + valuations counted + separately). do_listings: process listings batch (offer_price_history backfill). do_houses: reserved for future houses_price_dynamics backfill (currently no-op — see module docstring TODO). + do_valuations: process Cian Valuation Calculator batch (external_valuations backfill). + Default False — opt-in because each call hits Cian auth-gated API. dry_run: skip all fetch+save; only count and log pending rows. Returns: @@ -152,10 +161,66 @@ async def backfill_cian_history( "(see TODO in cian_history_backfill.py)" ) + # ── 3. Cian listings без external_valuations (price prediction backfill) ── + if do_valuations: + rows = db.execute( + text(""" + SELECT l.id, l.address, l.area_m2, l.rooms, l.floor, l.total_floors + FROM listings l + WHERE l.source = 'cian' + AND l.address IS NOT NULL + AND l.area_m2 IS NOT NULL + AND l.rooms IS NOT NULL + AND l.floor IS NOT NULL + AND l.total_floors IS NOT NULL + AND NOT EXISTS ( + SELECT 1 FROM external_valuations ev + WHERE ev.source = 'cian_valuation' + AND ev.listing_id = l.id + AND ev.expires_at > NOW() + ) + LIMIT :lim + """), + {"lim": batch_size}, + ).mappings().all() + result.valuations_total = len(rows) + + if dry_run: + logger.info( + "dry_run: would process %d cian listings for valuation", result.valuations_total + ) + else: + for row in rows: + result.valuations_processed += 1 + try: + cval = await estimate_via_cian_valuation( + db, + address=row["address"], + total_area=float(row["area_m2"]), + rooms_count=int(row["rooms"]), + floor=int(row["floor"]), + total_floors=int(row["total_floors"]), + repair_type="cosmetic", + deal_type="sale", + use_cache=False, # force fresh fetch — populate cache + listing_id=int(row["id"]), # canonical link via mig 044 + ) + if cval is not None and cval.sale_price_rub: + result.valuations_succeeded += 1 + else: + result.valuations_failed += 1 + except Exception as exc: + logger.warning( + "cian_valuation backfill failed for listing_id=%s: %s", row["id"], exc + ) + result.valuations_failed += 1 + await asyncio.sleep(delay) + result.duration_sec = time.time() - t0 logger.info( "cian_backfill done: listings=%d/%d (ok=%d fetch_fail=%d save_fail=%d " - "price_rows=%d) houses=%d/%d (ok=%d fail=%d) %.1fs", + "price_rows=%d) houses=%d/%d (ok=%d fail=%d) " + "valuations=%d/%d (ok=%d fail=%d) %.1fs", result.listings_processed, result.listings_total, result.listings_succeeded, @@ -166,6 +231,10 @@ async def backfill_cian_history( result.houses_total, result.houses_succeeded, result.houses_failed_fetch + result.houses_failed_save, + result.valuations_processed, + result.valuations_total, + result.valuations_succeeded, + result.valuations_failed, result.duration_sec, ) return result diff --git a/tradein-mvp/scripts/cian_backfill_all.sh b/tradein-mvp/scripts/cian_backfill_all.sh new file mode 100644 index 00000000..c9067bfd --- /dev/null +++ b/tradein-mvp/scripts/cian_backfill_all.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +set -euo pipefail + +API="${TRADEIN_API_URL:-http://localhost:8000}" +BATCH="${BATCH_SIZE:-100}" +MAX_ITERS="${MAX_ITERS:-1000}" +DO_LISTINGS="${DO_LISTINGS:-true}" +DO_VALUATIONS="${DO_VALUATIONS:-true}" +DO_HOUSES="${DO_HOUSES:-false}" # houses stubbed until cian_zhk_url migration + +usage() { + cat <