feat(tradein/cian): valuation backfill + runner script для prod
This commit is contained in:
lekss361
parent
499ca76aed
commit
da38cd0a5f
3 changed files with 141 additions and 2 deletions
|
|
@ -1035,6 +1035,10 @@ class CianBackfillResp(BaseModel):
|
|||
houses_succeeded: int
|
||||
houses_failed_fetch: int
|
||||
houses_failed_save: int
|
||||
valuations_total: int
|
||||
valuations_processed: int
|
||||
valuations_succeeded: int
|
||||
valuations_failed: int
|
||||
duration_sec: float
|
||||
|
||||
|
||||
|
|
@ -1043,6 +1047,7 @@ async def scrape_cian_backfill_history(
|
|||
batch_size: int = Query(50, ge=1, le=200),
|
||||
do_listings: bool = True,
|
||||
do_houses: bool = True,
|
||||
do_valuations: bool = False,
|
||||
dry_run: bool = False,
|
||||
db: Session = Depends(get_db), # noqa: B008
|
||||
) -> CianBackfillResp:
|
||||
|
|
@ -1065,6 +1070,7 @@ async def scrape_cian_backfill_history(
|
|||
batch_size=batch_size,
|
||||
do_listings=do_listings,
|
||||
do_houses=do_houses,
|
||||
do_valuations=do_valuations,
|
||||
dry_run=dry_run,
|
||||
)
|
||||
return CianBackfillResp(ok=True, **result.__dict__)
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ from sqlalchemy.orm import Session
|
|||
|
||||
from app.services.scraper_settings import get_scraper_delay
|
||||
from app.services.scrapers.cian_detail import fetch_detail, save_detail_enrichment
|
||||
from app.services.scrapers.cian_valuation import estimate_via_cian_valuation
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -46,6 +47,10 @@ class CianBackfillResult:
|
|||
houses_succeeded: int = 0
|
||||
houses_failed_fetch: int = 0
|
||||
houses_failed_save: int = 0
|
||||
valuations_total: int = 0
|
||||
valuations_processed: int = 0
|
||||
valuations_succeeded: int = 0
|
||||
valuations_failed: int = 0
|
||||
duration_sec: float = field(default=0.0)
|
||||
|
||||
|
||||
|
|
@ -55,6 +60,7 @@ async def backfill_cian_history(
|
|||
batch_size: int = 50,
|
||||
do_listings: bool = True,
|
||||
do_houses: bool = True,
|
||||
do_valuations: bool = False,
|
||||
dry_run: bool = False,
|
||||
) -> CianBackfillResult:
|
||||
"""Iterate Cian listings + houses with missing history, fetch+save.
|
||||
|
|
@ -62,10 +68,13 @@ async def backfill_cian_history(
|
|||
Args:
|
||||
db: SQLAlchemy session (caller-owned; each entity commits internally via
|
||||
save_detail_enrichment / save_newbuilding_enrichment).
|
||||
batch_size: max entities to process per call (listings + houses counted separately).
|
||||
batch_size: max entities to process per call (listings + houses + valuations counted
|
||||
separately).
|
||||
do_listings: process listings batch (offer_price_history backfill).
|
||||
do_houses: reserved for future houses_price_dynamics backfill (currently no-op —
|
||||
see module docstring TODO).
|
||||
do_valuations: process Cian Valuation Calculator batch (external_valuations backfill).
|
||||
Default False — opt-in because each call hits Cian auth-gated API.
|
||||
dry_run: skip all fetch+save; only count and log pending rows.
|
||||
|
||||
Returns:
|
||||
|
|
@ -152,10 +161,66 @@ async def backfill_cian_history(
|
|||
"(see TODO in cian_history_backfill.py)"
|
||||
)
|
||||
|
||||
# ── 3. Cian listings без external_valuations (price prediction backfill) ──
|
||||
if do_valuations:
|
||||
rows = db.execute(
|
||||
text("""
|
||||
SELECT l.id, l.address, l.area_m2, l.rooms, l.floor, l.total_floors
|
||||
FROM listings l
|
||||
WHERE l.source = 'cian'
|
||||
AND l.address IS NOT NULL
|
||||
AND l.area_m2 IS NOT NULL
|
||||
AND l.rooms IS NOT NULL
|
||||
AND l.floor IS NOT NULL
|
||||
AND l.total_floors IS NOT NULL
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM external_valuations ev
|
||||
WHERE ev.source = 'cian_valuation'
|
||||
AND ev.listing_id = l.id
|
||||
AND ev.expires_at > NOW()
|
||||
)
|
||||
LIMIT :lim
|
||||
"""),
|
||||
{"lim": batch_size},
|
||||
).mappings().all()
|
||||
result.valuations_total = len(rows)
|
||||
|
||||
if dry_run:
|
||||
logger.info(
|
||||
"dry_run: would process %d cian listings for valuation", result.valuations_total
|
||||
)
|
||||
else:
|
||||
for row in rows:
|
||||
result.valuations_processed += 1
|
||||
try:
|
||||
cval = await estimate_via_cian_valuation(
|
||||
db,
|
||||
address=row["address"],
|
||||
total_area=float(row["area_m2"]),
|
||||
rooms_count=int(row["rooms"]),
|
||||
floor=int(row["floor"]),
|
||||
total_floors=int(row["total_floors"]),
|
||||
repair_type="cosmetic",
|
||||
deal_type="sale",
|
||||
use_cache=False, # force fresh fetch — populate cache
|
||||
listing_id=int(row["id"]), # canonical link via mig 044
|
||||
)
|
||||
if cval is not None and cval.sale_price_rub:
|
||||
result.valuations_succeeded += 1
|
||||
else:
|
||||
result.valuations_failed += 1
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"cian_valuation backfill failed for listing_id=%s: %s", row["id"], exc
|
||||
)
|
||||
result.valuations_failed += 1
|
||||
await asyncio.sleep(delay)
|
||||
|
||||
result.duration_sec = time.time() - t0
|
||||
logger.info(
|
||||
"cian_backfill done: listings=%d/%d (ok=%d fetch_fail=%d save_fail=%d "
|
||||
"price_rows=%d) houses=%d/%d (ok=%d fail=%d) %.1fs",
|
||||
"price_rows=%d) houses=%d/%d (ok=%d fail=%d) "
|
||||
"valuations=%d/%d (ok=%d fail=%d) %.1fs",
|
||||
result.listings_processed,
|
||||
result.listings_total,
|
||||
result.listings_succeeded,
|
||||
|
|
@ -166,6 +231,10 @@ async def backfill_cian_history(
|
|||
result.houses_total,
|
||||
result.houses_succeeded,
|
||||
result.houses_failed_fetch + result.houses_failed_save,
|
||||
result.valuations_processed,
|
||||
result.valuations_total,
|
||||
result.valuations_succeeded,
|
||||
result.valuations_failed,
|
||||
result.duration_sec,
|
||||
)
|
||||
return result
|
||||
|
|
|
|||
64
tradein-mvp/scripts/cian_backfill_all.sh
Normal file
64
tradein-mvp/scripts/cian_backfill_all.sh
Normal file
|
|
@ -0,0 +1,64 @@
|
|||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
API="${TRADEIN_API_URL:-http://localhost:8000}"
|
||||
BATCH="${BATCH_SIZE:-100}"
|
||||
MAX_ITERS="${MAX_ITERS:-1000}"
|
||||
DO_LISTINGS="${DO_LISTINGS:-true}"
|
||||
DO_VALUATIONS="${DO_VALUATIONS:-true}"
|
||||
DO_HOUSES="${DO_HOUSES:-false}" # houses stubbed until cian_zhk_url migration
|
||||
|
||||
usage() {
|
||||
cat <<EOF
|
||||
Cian backfill runner — drains offer_price_history + external_valuations backlog.
|
||||
|
||||
Usage:
|
||||
$0 [--dry-run]
|
||||
|
||||
Env vars:
|
||||
TRADEIN_API_URL default: http://localhost:8000
|
||||
BATCH_SIZE default: 100 (max 200 per endpoint cap)
|
||||
MAX_ITERS default: 1000 (safety stop)
|
||||
DO_LISTINGS default: true
|
||||
DO_VALUATIONS default: true
|
||||
DO_HOUSES default: false (no cian_zhk_url col yet)
|
||||
|
||||
Examples:
|
||||
$0 --dry-run # count what would be processed
|
||||
BATCH_SIZE=50 $0 # drain at half pace
|
||||
DO_VALUATIONS=false $0 # only history
|
||||
TRADEIN_API_URL=https://gendsgn.ru/api $0
|
||||
EOF
|
||||
}
|
||||
|
||||
DRY=""
|
||||
if [[ "${1:-}" == "--dry-run" ]]; then DRY="&dry_run=true"; fi
|
||||
if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then usage; exit 0; fi
|
||||
|
||||
echo "Cian backfill: API=$API BATCH=$BATCH listings=$DO_LISTINGS valuations=$DO_VALUATIONS houses=$DO_HOUSES dry=${DRY:-no}"
|
||||
|
||||
cum_listings=0
|
||||
cum_valuations=0
|
||||
i=0
|
||||
while (( i < MAX_ITERS )); do
|
||||
i=$((i+1))
|
||||
url="$API/api/v1/admin/scrape/cian-backfill-history?batch_size=$BATCH&do_listings=$DO_LISTINGS&do_valuations=$DO_VALUATIONS&do_houses=$DO_HOUSES${DRY}"
|
||||
resp=$(curl -fsS -X POST "$url")
|
||||
lt=$(echo "$resp" | jq -r '.listings_total // 0')
|
||||
vt=$(echo "$resp" | jq -r '.valuations_total // 0')
|
||||
ls=$(echo "$resp" | jq -r '.listings_succeeded // 0')
|
||||
vs=$(echo "$resp" | jq -r '.valuations_succeeded // 0')
|
||||
dur=$(echo "$resp" | jq -r '.duration_sec // 0')
|
||||
cum_listings=$((cum_listings + ls))
|
||||
cum_valuations=$((cum_valuations + vs))
|
||||
echo "[$i] listings=$lt (ok=$ls) valuations=$vt (ok=$vs) ${dur}s cum: hist=$cum_listings val=$cum_valuations"
|
||||
if (( lt == 0 && vt == 0 )); then
|
||||
echo "Drained — stopping."
|
||||
break
|
||||
fi
|
||||
if [[ -n "$DRY" ]]; then
|
||||
echo "Dry-run — no DB writes; stopping after first probe."
|
||||
break
|
||||
fi
|
||||
done
|
||||
echo "Done. Total: history=$cum_listings valuations=$cum_valuations iterations=$i"
|
||||
Loading…
Add table
Reference in a new issue