eb564cfc01
Migrates legacy app.services.scrapers.* imports to scraper_kit equivalents for house_imv_backfill.py, avito_detail_backfill.py, cian_history_backfill.py, ekb_geoportal_ingest.py, and yandex_detail_backfill.py, proving parity via tests/support/parity.assert_parity per the epic's gate (#2304). newbuilding_enrich_backfill.py and yandex_newbuilding_sweep.py are left fully on legacy imports: both call into scraper_kit.providers.{cian,yandex}.newbuilding, which construct BrowserFetcher(source=...) without the now-mandatory endpoint= kwarg (issue #2322, verified still open against the actual provider source, not just issue status) -- no caller-side fix is possible, matching Group A's (#2305) precedent for the same bug in admin.py. Config-gated kit function footguns found and fixed (Group B #2306 pattern): - avito fetch_detail's backconnect-on-403 retry silently drops when config= is omitted -- now passes config=RealScraperConfig() explicitly, with a regression test proving the gate. - kit AvitoScraper's constructor now requires ScraperConfig positionally -- wired via RealScraperConfig(), which _rotate_ip() reads for avito_proxy_rotate_url. - BrowserFetcher(source=...) call sites (house_imv_backfill, avito/cian detail backfills) now pass the mandatory endpoint=settings.browser_http_endpoint. New footgun discovered (NOT #2322, flagged for follow-up): kit's build_warmed_session() builds its curl_cffi session via _build_detail_session() with no config parameter at all, unlike fetch_detail -- migrating it would silently drop the sticky MGTS-proxy egress on avito_detail_backfill's warm-batch path (the prod default). Left build_warmed_session/ _AVITO_WARM_SEARCH_URL on legacy imports, documented inline. Refs #2310
328 lines
13 KiB
Python
328 lines
13 KiB
Python
"""Batch backfill для Cian historical data.
|
||
|
||
Запускается:
|
||
- Manual через POST /admin/scrape/cian-backfill-history
|
||
- Future: in-app scheduler (после bootstrap'а Celery beat)
|
||
|
||
Что делает:
|
||
1. SELECT listings WHERE source='cian' AND source_url IS NOT NULL
|
||
AND NOT EXISTS (SELECT 1 FROM offer_price_history WHERE listing_id=...)
|
||
LIMIT batch_size
|
||
→ fetch_detail(browser_fetcher=bf) + save_detail_enrichment per listing
|
||
Использует BrowserFetcher (camoufox) для получения JS-rendered HTML, потому что
|
||
curl_cffi не рендерит priceChanges (они hidden в raw HTML).
|
||
2. Houses block: SELECT houses WHERE cian_zhk_url IS NOT NULL
|
||
AND NOT EXISTS (SELECT 1 FROM houses_price_dynamics WHERE house_id=...)
|
||
LIMIT batch_size
|
||
→ fetch_newbuilding(cian_zhk_url) + save_newbuilding_enrichment per house
|
||
Requires migration 071_houses_cian_zhk_url.sql (cian_zhk_url column).
|
||
|
||
Rate limit: scraper_settings.get_scraper_delay('cian') between requests.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import logging
|
||
import time
|
||
from dataclasses import dataclass, field
|
||
|
||
from scraper_kit.browser_fetcher import BrowserFetcher
|
||
from scraper_kit.providers.cian.detail import fetch_detail, save_detail_enrichment
|
||
from sqlalchemy import text
|
||
from sqlalchemy.orm import Session
|
||
|
||
from app.core.config import settings
|
||
from app.services.scraper_settings import get_scraper_delay
|
||
|
||
# cian_valuation остаётся legacy — валюационные зависимости (avito_imv/
|
||
# cian_valuation/yandex_valuation) выделены в отдельный, более рискованный
|
||
# issue #2308 (см. epic #2277, issue #2310 Group C), не в этой миграции.
|
||
from app.services.scrapers.cian_valuation import estimate_via_cian_valuation
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
|
||
@dataclass
|
||
class CianBackfillResult:
|
||
listings_total: int = 0
|
||
listings_processed: int = 0
|
||
listings_succeeded: int = 0
|
||
listings_failed_fetch: int = 0
|
||
listings_failed_save: int = 0
|
||
price_changes_attempted: int = 0
|
||
houses_total: int = 0
|
||
houses_processed: int = 0
|
||
houses_succeeded: int = 0
|
||
houses_failed_fetch: int = 0
|
||
houses_failed_save: int = 0
|
||
valuations_total: int = 0
|
||
valuations_processed: int = 0
|
||
valuations_succeeded: int = 0
|
||
valuations_failed: int = 0
|
||
duration_sec: float = field(default=0.0)
|
||
|
||
|
||
async def backfill_cian_history(
|
||
db: Session,
|
||
*,
|
||
batch_size: int = 50,
|
||
do_listings: bool = True,
|
||
do_houses: bool = True,
|
||
do_valuations: bool = False,
|
||
dry_run: bool = False,
|
||
) -> CianBackfillResult:
|
||
"""Iterate Cian listings + houses with missing history, fetch+save.
|
||
|
||
Args:
|
||
db: SQLAlchemy session (caller-owned; each entity commits internally via
|
||
save_detail_enrichment / save_newbuilding_enrichment).
|
||
batch_size: max entities to process per call (listings + houses + valuations counted
|
||
separately).
|
||
do_listings: process listings batch (offer_price_history backfill).
|
||
do_houses: process houses batch (houses_price_dynamics backfill via
|
||
fetch_newbuilding). Requires migration 071_houses_cian_zhk_url.sql applied.
|
||
do_valuations: process Cian Valuation Calculator batch (external_valuations backfill).
|
||
Default False — opt-in because each call hits Cian auth-gated API.
|
||
dry_run: skip all fetch+save; only count and log pending rows.
|
||
|
||
Returns:
|
||
CianBackfillResult with per-domain counters + total wall-clock duration.
|
||
"""
|
||
result = CianBackfillResult()
|
||
t0 = time.time()
|
||
delay = get_scraper_delay("cian") # seconds; default 5.0
|
||
|
||
# ── 1. Listings: missing offer_price_history ──────────────────────────────
|
||
if do_listings:
|
||
rows = (
|
||
db.execute(
|
||
text("""
|
||
SELECT l.id, l.source_url
|
||
FROM listings l
|
||
LEFT JOIN offer_price_history oph ON oph.listing_id = l.id
|
||
WHERE l.source = 'cian'
|
||
AND l.source_url IS NOT NULL
|
||
AND oph.listing_id IS NULL
|
||
LIMIT :lim
|
||
"""),
|
||
{"lim": batch_size},
|
||
)
|
||
.mappings()
|
||
.all()
|
||
)
|
||
result.listings_total = len(rows)
|
||
|
||
if dry_run:
|
||
logger.info("dry_run: would process %d cian listings", result.listings_total)
|
||
else:
|
||
# One BrowserFetcher instance shared across all listings in this batch.
|
||
# priceChanges requires JS rendering — curl_cffi returns empty list (#1574).
|
||
async with BrowserFetcher(source="cian", endpoint=settings.browser_http_endpoint) as bf:
|
||
for row in rows:
|
||
listing_id: int = row["id"]
|
||
source_url: str = row["source_url"]
|
||
result.listings_processed += 1
|
||
|
||
enrichment = None
|
||
try:
|
||
enrichment = await fetch_detail(source_url, browser_fetcher=bf)
|
||
except Exception as exc:
|
||
logger.warning(
|
||
"cian_detail fetch failed for listing_id=%s url=%s: %s",
|
||
listing_id,
|
||
source_url,
|
||
exc,
|
||
)
|
||
result.listings_failed_fetch += 1
|
||
await asyncio.sleep(delay)
|
||
continue
|
||
|
||
if enrichment is None:
|
||
logger.warning(
|
||
"cian_detail fetch returned None for listing_id=%s url=%s",
|
||
listing_id,
|
||
source_url,
|
||
)
|
||
result.listings_failed_fetch += 1
|
||
await asyncio.sleep(delay)
|
||
continue
|
||
|
||
try:
|
||
save_detail_enrichment(db, listing_id, enrichment)
|
||
result.listings_succeeded += 1
|
||
result.price_changes_attempted += len(enrichment.price_changes or [])
|
||
except Exception as exc:
|
||
logger.warning(
|
||
"cian_detail save failed for listing_id=%s: %s", listing_id, exc
|
||
)
|
||
result.listings_failed_save += 1
|
||
|
||
await asyncio.sleep(delay)
|
||
|
||
# ── 2. Houses: missing houses_price_dynamics ──────────────────────────────
|
||
if do_houses:
|
||
# cian_newbuilding остаётся legacy (issue #2310, NOT migrated): kit's
|
||
# scraper_kit.providers.cian.newbuilding.fetch_newbuilding() constructs
|
||
# BrowserFetcher(source="cian") WITHOUT the now-mandatory endpoint= kwarg
|
||
# → TypeError on every call, no caller-side fix possible (issue #2322,
|
||
# still open — verified against provider source, not just issue status).
|
||
from app.services.scrapers.cian_newbuilding import (
|
||
fetch_newbuilding,
|
||
save_newbuilding_enrichment,
|
||
)
|
||
|
||
house_rows = (
|
||
db.execute(
|
||
text("""
|
||
SELECT h.id, h.cian_zhk_url
|
||
FROM houses h
|
||
WHERE h.cian_zhk_url IS NOT NULL
|
||
AND NOT EXISTS (
|
||
SELECT 1 FROM houses_price_dynamics hpd
|
||
WHERE hpd.house_id = h.id
|
||
)
|
||
LIMIT :lim
|
||
"""),
|
||
{"lim": batch_size},
|
||
)
|
||
.mappings()
|
||
.all()
|
||
)
|
||
result.houses_total = len(house_rows)
|
||
|
||
if dry_run:
|
||
logger.info("dry_run: would process %d cian houses", result.houses_total)
|
||
else:
|
||
for hrow in house_rows:
|
||
house_id: int = hrow["id"]
|
||
zhk_url: str = hrow["cian_zhk_url"]
|
||
result.houses_processed += 1
|
||
|
||
enrichment = None
|
||
try:
|
||
enrichment = await fetch_newbuilding(zhk_url)
|
||
except Exception as exc:
|
||
logger.warning(
|
||
"cian_newbuilding fetch failed for house_id=%s url=%s: %s",
|
||
house_id,
|
||
zhk_url,
|
||
exc,
|
||
)
|
||
result.houses_failed_fetch += 1
|
||
await asyncio.sleep(delay)
|
||
continue
|
||
|
||
if enrichment is None:
|
||
logger.warning(
|
||
"cian_newbuilding fetch returned None for house_id=%s url=%s",
|
||
house_id,
|
||
zhk_url,
|
||
)
|
||
result.houses_failed_fetch += 1
|
||
await asyncio.sleep(delay)
|
||
continue
|
||
|
||
try:
|
||
save_newbuilding_enrichment(db, house_id, enrichment)
|
||
result.houses_succeeded += 1
|
||
except Exception as exc:
|
||
logger.warning(
|
||
"cian_newbuilding save failed for house_id=%s: %s", house_id, exc
|
||
)
|
||
result.houses_failed_save += 1
|
||
# Roll back to clean session state so next house can proceed.
|
||
# save_newbuilding_enrichment commits on success; on failure the
|
||
# transaction is left open/dirty — rollback to avoid session poison.
|
||
try:
|
||
db.rollback()
|
||
except Exception as rb_exc:
|
||
logger.warning(
|
||
"cian_newbuilding rollback failed for house_id=%s: %s",
|
||
house_id,
|
||
rb_exc,
|
||
)
|
||
|
||
await asyncio.sleep(delay)
|
||
|
||
# ── 3. Cian listings без external_valuations (price prediction backfill) ──
|
||
if do_valuations:
|
||
rows = (
|
||
db.execute(
|
||
text("""
|
||
SELECT l.id, l.address, l.area_m2, l.rooms, l.floor, l.total_floors
|
||
FROM listings l
|
||
WHERE l.source = 'cian'
|
||
AND l.address IS NOT NULL
|
||
AND l.area_m2 IS NOT NULL
|
||
AND l.rooms IS NOT NULL
|
||
AND l.floor IS NOT NULL
|
||
AND l.total_floors IS NOT NULL
|
||
AND NOT EXISTS (
|
||
SELECT 1 FROM external_valuations ev
|
||
WHERE ev.source = 'cian_valuation'
|
||
AND ev.listing_id = l.id
|
||
AND ev.expires_at > NOW()
|
||
)
|
||
LIMIT :lim
|
||
"""),
|
||
{"lim": batch_size},
|
||
)
|
||
.mappings()
|
||
.all()
|
||
)
|
||
result.valuations_total = len(rows)
|
||
|
||
if dry_run:
|
||
logger.info(
|
||
"dry_run: would process %d cian listings for valuation", result.valuations_total
|
||
)
|
||
else:
|
||
for row in rows:
|
||
result.valuations_processed += 1
|
||
try:
|
||
cval = await estimate_via_cian_valuation(
|
||
db,
|
||
address=row["address"],
|
||
total_area=float(row["area_m2"]),
|
||
rooms_count=int(row["rooms"]),
|
||
floor=int(row["floor"]),
|
||
total_floors=int(row["total_floors"]),
|
||
repair_type="cosmetic",
|
||
deal_type="sale",
|
||
use_cache=False, # force fresh fetch — populate cache
|
||
listing_id=int(row["id"]), # canonical link via mig 044
|
||
)
|
||
if cval is not None and cval.sale_price_rub:
|
||
result.valuations_succeeded += 1
|
||
else:
|
||
result.valuations_failed += 1
|
||
except Exception as exc:
|
||
logger.warning(
|
||
"cian_valuation backfill failed for listing_id=%s: %s", row["id"], exc
|
||
)
|
||
result.valuations_failed += 1
|
||
await asyncio.sleep(delay)
|
||
|
||
result.duration_sec = time.time() - t0
|
||
logger.info(
|
||
"cian_backfill done: listings=%d/%d (ok=%d fetch_fail=%d save_fail=%d "
|
||
"price_rows=%d) houses=%d/%d (ok=%d fail=%d) "
|
||
"valuations=%d/%d (ok=%d fail=%d) %.1fs",
|
||
result.listings_processed,
|
||
result.listings_total,
|
||
result.listings_succeeded,
|
||
result.listings_failed_fetch,
|
||
result.listings_failed_save,
|
||
result.price_changes_attempted,
|
||
result.houses_processed,
|
||
result.houses_total,
|
||
result.houses_succeeded,
|
||
result.houses_failed_fetch + result.houses_failed_save,
|
||
result.valuations_processed,
|
||
result.valuations_total,
|
||
result.valuations_succeeded,
|
||
result.valuations_failed,
|
||
result.duration_sec,
|
||
)
|
||
return result
|