lekss361
2914ff2ef2
Stage 4d: full city sweep ЕКБ (5 anchors × N pages + enrichment + cooperative cancel). - avito.py fetch_around(pages=N) — pagination backward compat default=1 - scrape_pipeline.run_avito_city_sweep — anchors loop + heartbeat + cancel check - scrape_runs.py utility (create/heartbeat/done/failed/cancelled/list_recent) - admin.py 3 endpoints: start (BackgroundTasks) / runs / cancel - migration 051 ADD COLUMN IF NOT EXISTS + DROP/ADD CHECK constraint (text+CHECK, не enum — safe в transaction) - deploy/avito-city-sweep.sh с random delay 0-3h (anti-pattern) - 16 offline tests Deep review approved: migration idempotent, cancel priority correct (mark_done guarded by status='running'), no SQL injection, psycopg v3 compliant.
364 lines
14 KiB
Python
364 lines
14 KiB
Python
"""scrape_pipeline.py — Avito full orchestrator (Stage 2e + city sweep).
|
||
|
||
Координирует 5 шагов одного pipeline run:
|
||
1. SEARCH — AvitoScraper().fetch_around(lat, lon, radius_m, pages=N)
|
||
2. SAVE — save_listings(db, lots) → (inserted, updated)
|
||
3. GROUP — dedupe house_url из lots → unique set
|
||
4. ENRICH_HOUSES — fetch_house_catalog + save_house_catalog_enrichment per unique house
|
||
5. ENRICH_DETAIL — top-N listings (detail_enriched_at IS NULL) → fetch_detail + save
|
||
|
||
City sweep: run_avito_city_sweep — итерирует EKB_ANCHORS × pages_per_anchor,
|
||
с cooperative cancel через scrape_runs.status и heartbeat update каждый anchor.
|
||
|
||
Graceful degradation на каждом step: exception в одной house/listing
|
||
не валит весь pipeline.
|
||
|
||
IMV ОТСУТСТВУЕТ — он on-demand (Stage 3 estimator integration), не cron.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import logging
|
||
from dataclasses import dataclass, field, fields
|
||
from urllib.parse import urlparse
|
||
|
||
from sqlalchemy import text
|
||
from sqlalchemy.orm import Session
|
||
|
||
from app.services.scrapers.avito import AvitoScraper
|
||
from app.services.scrapers.avito_detail import fetch_detail, save_detail_enrichment
|
||
from app.services.scrapers.avito_houses import fetch_house_catalog, save_house_catalog_enrichment
|
||
from app.services.scrapers.base import ScrapedLot, save_listings
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
# Default anchors ЕКБ — 5 точек покрытия города
|
||
EKB_ANCHORS: list[tuple[float, float, str]] = [
|
||
(56.8400, 60.6050, "Центр"),
|
||
(56.7950, 60.5300, "ЮЗ"),
|
||
(56.8970, 60.6100, "Уралмаш"),
|
||
(56.7700, 60.5500, "Академический"),
|
||
(56.8650, 60.6200, "Пионерский"),
|
||
]
|
||
|
||
|
||
# ── Result dataclasses ──────────────────────────────────────────────────────
|
||
|
||
|
||
@dataclass
|
||
class PipelineCounters:
|
||
"""Counters одного pipeline run для логов/админки."""
|
||
|
||
lots_fetched: int = 0
|
||
lots_inserted: int = 0
|
||
lots_updated: int = 0
|
||
unique_houses: int = 0
|
||
houses_enriched: int = 0
|
||
houses_failed: int = 0
|
||
detail_attempted: int = 0
|
||
detail_enriched: int = 0
|
||
detail_failed: int = 0
|
||
errors: list[str] = field(default_factory=list)
|
||
|
||
|
||
@dataclass
|
||
class PipelineResult:
|
||
"""Полный результат full Avito pipeline."""
|
||
|
||
anchor_lat: float
|
||
anchor_lon: float
|
||
radius_m: int
|
||
counters: PipelineCounters
|
||
enrich_houses: bool
|
||
enrich_detail_top_n: int
|
||
|
||
|
||
# ── Main orchestrator ───────────────────────────────────────────────────────
|
||
|
||
|
||
async def run_avito_pipeline(
|
||
db: Session,
|
||
lat: float,
|
||
lon: float,
|
||
radius_m: int = 1500,
|
||
*,
|
||
enrich_houses: bool = True,
|
||
enrich_detail_top_n: int = 10,
|
||
pages: int = 1,
|
||
request_delay_sec: float | None = None,
|
||
) -> PipelineResult:
|
||
"""Full Avito search → houses → detail enrichment pipeline.
|
||
|
||
Steps:
|
||
1. SEARCH: AvitoScraper().fetch_around(lat, lon, radius_m, pages=pages)
|
||
2. SAVE: save_listings(db, lots) → (inserted, updated)
|
||
3. GROUP: dedupe house_url из новых lots → unique house path set
|
||
4. ENRICH_HOUSES (если enrich_houses=True): для каждого unique house path
|
||
→ fetch_house_catalog + save_house_catalog_enrichment (try/except per house)
|
||
5. ENRICH_DETAIL (если enrich_detail_top_n > 0): select top-N priority listings
|
||
(приоритет — detail_enriched_at IS NULL + price > 0)
|
||
→ fetch_detail + save_detail_enrichment (try/except per listing)
|
||
6. Return PipelineResult с counters
|
||
|
||
pages=1 (default) — backward compat, одна страница.
|
||
pages>1 — pagination (city sweep). request_delay_sec — override задержки между страницами.
|
||
|
||
Graceful degradation на каждом step — exception на одной house/listing
|
||
не должна провалить весь pipeline.
|
||
"""
|
||
counters = PipelineCounters()
|
||
lots: list[ScrapedLot] = []
|
||
|
||
# ── Step 1: search ──────────────────────────────────────────────────────
|
||
async with AvitoScraper() as scraper:
|
||
try:
|
||
lots = await scraper.fetch_around(
|
||
lat, lon, radius_m, pages=pages, delay_override_sec=request_delay_sec
|
||
)
|
||
counters.lots_fetched = len(lots)
|
||
logger.info(
|
||
"pipeline:search anchor=(%.4f,%.4f) r=%dm fetched=%d",
|
||
lat,
|
||
lon,
|
||
radius_m,
|
||
len(lots),
|
||
)
|
||
except Exception as e:
|
||
logger.exception("pipeline:search failed")
|
||
counters.errors.append(f"search: {e}")
|
||
return PipelineResult(lat, lon, radius_m, counters, enrich_houses, enrich_detail_top_n)
|
||
|
||
# ── Step 2: save listings ───────────────────────────────────────────────
|
||
if lots:
|
||
try:
|
||
counters.lots_inserted, counters.lots_updated = save_listings(db, lots)
|
||
except Exception as e:
|
||
logger.exception("pipeline:save_listings failed")
|
||
counters.errors.append(f"save_listings: {e}")
|
||
|
||
# ── Step 3: group by house ──────────────────────────────────────────────
|
||
# Dedupe house path (без host) для consistency; fetch_house_catalog принимает path
|
||
unique_house_paths: set[str] = set()
|
||
for lot in lots:
|
||
if lot.house_url:
|
||
try:
|
||
parsed = urlparse(lot.house_url)
|
||
path = parsed.path if parsed.path else lot.house_url
|
||
unique_house_paths.add(path)
|
||
except Exception:
|
||
continue
|
||
|
||
counters.unique_houses = len(unique_house_paths)
|
||
logger.info("pipeline:group_by_house unique=%d", counters.unique_houses)
|
||
|
||
# ── Step 4: enrich houses ───────────────────────────────────────────────
|
||
if enrich_houses and unique_house_paths:
|
||
for house_path in unique_house_paths:
|
||
try:
|
||
enrichment = await fetch_house_catalog(house_path)
|
||
save_house_catalog_enrichment(db, enrichment)
|
||
counters.houses_enriched += 1
|
||
except Exception as e:
|
||
logger.warning(
|
||
"pipeline:house_enrich failed for %s: %s", house_path, e
|
||
)
|
||
counters.houses_failed += 1
|
||
counters.errors.append(f"house {house_path}: {e}")
|
||
|
||
# ── Step 5: enrich detail для top-N priority listings ──────────────────
|
||
if enrich_detail_top_n > 0:
|
||
# Priority: листинги ещё не enriched (detail_enriched_at IS NULL)
|
||
# в радиусе 2× от anchor ИЛИ только что scraped (последние 2 часа).
|
||
# OR обёрнут в скобки — приоритет AND выше OR.
|
||
priority_rows = db.execute(
|
||
text("""
|
||
SELECT source_url
|
||
FROM listings
|
||
WHERE source = 'avito'
|
||
AND source_url IS NOT NULL
|
||
AND (
|
||
(
|
||
detail_enriched_at IS NULL
|
||
AND price_rub > 0
|
||
AND ST_DWithin(
|
||
geom::geography,
|
||
ST_MakePoint(:lon, :lat)::geography,
|
||
:radius
|
||
)
|
||
)
|
||
OR (
|
||
detail_enriched_at IS NULL
|
||
AND scraped_at > NOW() - INTERVAL '2 hours'
|
||
)
|
||
)
|
||
ORDER BY scraped_at DESC NULLS LAST
|
||
LIMIT :limit
|
||
"""),
|
||
{
|
||
"lat": lat,
|
||
"lon": lon,
|
||
"radius": radius_m * 2,
|
||
"limit": enrich_detail_top_n,
|
||
},
|
||
).mappings().all()
|
||
|
||
for row in priority_rows:
|
||
source_url: str = row["source_url"]
|
||
counters.detail_attempted += 1
|
||
try:
|
||
# fetch_detail принимает absolute URL или relative path
|
||
item_url = (
|
||
urlparse(source_url).path
|
||
if source_url.startswith("http")
|
||
else source_url
|
||
)
|
||
enrichment_detail = await fetch_detail(item_url)
|
||
if save_detail_enrichment(db, enrichment_detail):
|
||
counters.detail_enriched += 1
|
||
except Exception as e:
|
||
logger.warning(
|
||
"pipeline:detail_enrich failed for %s: %s", source_url, e
|
||
)
|
||
counters.detail_failed += 1
|
||
counters.errors.append(f"detail {source_url}: {e}")
|
||
|
||
logger.info(
|
||
"pipeline:done anchor=(%.4f,%.4f) lots=%d (ins=%d/upd=%d) "
|
||
"houses=%d/%d detail=%d/%d errors=%d",
|
||
lat,
|
||
lon,
|
||
counters.lots_fetched,
|
||
counters.lots_inserted,
|
||
counters.lots_updated,
|
||
counters.houses_enriched,
|
||
counters.unique_houses,
|
||
counters.detail_enriched,
|
||
counters.detail_attempted,
|
||
len(counters.errors),
|
||
)
|
||
|
||
return PipelineResult(lat, lon, radius_m, counters, enrich_houses, enrich_detail_top_n)
|
||
|
||
|
||
@dataclass
|
||
class CitySweepCounters:
|
||
"""Aggregate counters для full city sweep run."""
|
||
|
||
anchors_total: int = 0
|
||
anchors_done: int = 0
|
||
lots_fetched: int = 0
|
||
lots_inserted: int = 0
|
||
lots_updated: int = 0
|
||
unique_houses: int = 0
|
||
houses_enriched: int = 0
|
||
houses_failed: int = 0
|
||
detail_attempted: int = 0
|
||
detail_enriched: int = 0
|
||
detail_failed: int = 0
|
||
errors_count: int = 0
|
||
|
||
def to_dict(self) -> dict[str, int]:
|
||
return {f.name: getattr(self, f.name) for f in fields(self)}
|
||
|
||
|
||
async def run_avito_city_sweep(
|
||
db: Session,
|
||
*,
|
||
run_id: int,
|
||
anchors: list[tuple[float, float, str]] | None = None,
|
||
radius_m: int = 1500,
|
||
pages_per_anchor: int = 3,
|
||
enrich_houses: bool = True,
|
||
detail_top_n: int = 20,
|
||
request_delay_sec: float = 7.0,
|
||
) -> CitySweepCounters:
|
||
"""Full city sweep: iterate anchors × pages → save → enrich houses + detail.
|
||
|
||
Алгоритм:
|
||
- Для каждого anchor: run_avito_pipeline(pages=pages_per_anchor)
|
||
- Перед каждым anchor проверяет scrape_runs.status — если 'cancelled' → stop
|
||
- Heartbeat update каждый anchor с текущими counters
|
||
- Errors per-anchor логируются, не валят весь sweep
|
||
- По завершению вызывает mark_done; при исключении — mark_failed
|
||
"""
|
||
from app.services import scrape_runs as runs
|
||
|
||
_anchors = anchors if anchors is not None else EKB_ANCHORS
|
||
counters = CitySweepCounters(anchors_total=len(_anchors))
|
||
|
||
try:
|
||
for idx, (lat, lon, name) in enumerate(_anchors, start=1):
|
||
# Cooperative cancel: проверяем статус перед каждым anchor
|
||
if runs.is_cancelled(db, run_id):
|
||
logger.info(
|
||
"city-sweep run_id=%d: cancelled by signal at anchor #%d/%d (%s)",
|
||
run_id, idx, len(_anchors), name,
|
||
)
|
||
runs.update_heartbeat(db, run_id, counters.to_dict())
|
||
return counters
|
||
|
||
logger.info(
|
||
"city-sweep run_id=%d anchor #%d/%d (%s, %.4f, %.4f)",
|
||
run_id, idx, len(_anchors), name, lat, lon,
|
||
)
|
||
try:
|
||
result = await run_avito_pipeline(
|
||
db,
|
||
lat=lat,
|
||
lon=lon,
|
||
radius_m=radius_m,
|
||
enrich_houses=enrich_houses,
|
||
enrich_detail_top_n=detail_top_n,
|
||
pages=pages_per_anchor,
|
||
request_delay_sec=request_delay_sec,
|
||
)
|
||
counters.lots_fetched += result.counters.lots_fetched
|
||
counters.lots_inserted += result.counters.lots_inserted
|
||
counters.lots_updated += result.counters.lots_updated
|
||
counters.unique_houses += result.counters.unique_houses
|
||
counters.houses_enriched += result.counters.houses_enriched
|
||
counters.houses_failed += result.counters.houses_failed
|
||
counters.detail_attempted += result.counters.detail_attempted
|
||
counters.detail_enriched += result.counters.detail_enriched
|
||
counters.detail_failed += result.counters.detail_failed
|
||
counters.errors_count += len(result.counters.errors)
|
||
except Exception:
|
||
logger.exception("city-sweep run_id=%d: anchor %s failed", run_id, name)
|
||
counters.errors_count += 1
|
||
|
||
counters.anchors_done = idx
|
||
runs.update_heartbeat(db, run_id, counters.to_dict())
|
||
|
||
runs.mark_done(db, run_id, counters.to_dict())
|
||
logger.info(
|
||
"city-sweep run_id=%d done: anchors=%d/%d lots=%d (ins=%d/upd=%d) "
|
||
"houses=%d/%d detail=%d/%d errors=%d",
|
||
run_id,
|
||
counters.anchors_done,
|
||
counters.anchors_total,
|
||
counters.lots_fetched,
|
||
counters.lots_inserted,
|
||
counters.lots_updated,
|
||
counters.houses_enriched,
|
||
counters.unique_houses,
|
||
counters.detail_enriched,
|
||
counters.detail_attempted,
|
||
counters.errors_count,
|
||
)
|
||
return counters
|
||
|
||
except Exception as exc:
|
||
logger.exception("city-sweep run_id=%d: fatal error", run_id)
|
||
runs.mark_failed(db, run_id, str(exc), counters.to_dict())
|
||
raise
|
||
|
||
|
||
# ── Public re-exports ───────────────────────────────────────────────────────
|
||
__all__ = [
|
||
"EKB_ANCHORS",
|
||
"CitySweepCounters",
|
||
"PipelineCounters",
|
||
"PipelineResult",
|
||
"run_avito_city_sweep",
|
||
"run_avito_pipeline",
|
||
]
|