feat(scraper-kit): copy sweep pipeline orchestration with protocol injection, strangler (#2135)
This commit is contained in:
bot-backend
parent
f7063d49b5
commit
773cbdc69b
6 changed files with 2294 additions and 0 deletions
|
|
@ -23,6 +23,9 @@ from app.core.config import settings as _settings
|
|||
from app.core.db import SessionLocal as _SessionLocal
|
||||
from app.services.house_dedup_merge import run_house_dedup_merge as _run_house_dedup_merge
|
||||
from app.services.house_imv_backfill import backfill_house_imv as _backfill_house_imv
|
||||
from app.services.house_imv_backfill import (
|
||||
process_houses_imv_batch as _process_houses_imv_batch,
|
||||
)
|
||||
from app.services.matching import match_or_create_house as _match_or_create_house
|
||||
from app.services.matching import upsert_listing_source as _upsert_listing_source
|
||||
from app.services.yandex_address_backfill import (
|
||||
|
|
@ -154,6 +157,43 @@ class RealScraperConfig:
|
|||
def glitchtip_dsn(self) -> str | None:
|
||||
return _settings.glitchtip_dsn
|
||||
|
||||
# ── Оркестрация sweep-pipeline (#2135) ───────────────────────────────
|
||||
@property
|
||||
def avito_serp_ok_not_banned(self) -> bool:
|
||||
return _settings.avito_serp_ok_not_banned
|
||||
|
||||
@property
|
||||
def avito_proxy_rotate_settle_s(self) -> float:
|
||||
return _settings.avito_proxy_rotate_settle_s
|
||||
|
||||
@property
|
||||
def proxy_rotate_attempts(self) -> int:
|
||||
return _settings.proxy_rotate_attempts
|
||||
|
||||
@property
|
||||
def proxy_rotate_attempt_timeout_s(self) -> float:
|
||||
return _settings.proxy_rotate_attempt_timeout_s
|
||||
|
||||
@property
|
||||
def cian_proxy_rotate_url(self) -> str | None:
|
||||
return _settings.cian_proxy_rotate_url
|
||||
|
||||
@property
|
||||
def cian_proxy_max_rotations(self) -> int:
|
||||
return _settings.cian_proxy_max_rotations
|
||||
|
||||
@property
|
||||
def yandex_proxy_max_rotations(self) -> int:
|
||||
return _settings.yandex_proxy_max_rotations
|
||||
|
||||
@property
|
||||
def scraper_skip_seen_today(self) -> bool:
|
||||
return _settings.scraper_skip_seen_today
|
||||
|
||||
@property
|
||||
def cian_full_load_per_fetch_timeout_s(self) -> float:
|
||||
return _settings.cian_full_load_per_fetch_timeout_s
|
||||
|
||||
|
||||
class RealSessionFactory:
|
||||
"""SessionFactory-адаптер над `app.core.db.SessionLocal`."""
|
||||
|
|
@ -206,6 +246,21 @@ class RealEnrichmentJobs:
|
|||
request_delay_sec=request_delay_sec,
|
||||
)
|
||||
|
||||
async def process_houses_imv_batch(
|
||||
self,
|
||||
db: Session,
|
||||
house_ids: set[int],
|
||||
*,
|
||||
request_delay_sec: float = 5.0,
|
||||
heartbeat: Callable[[], None] | None = None,
|
||||
) -> HouseIMVBackfillResult:
|
||||
return await _process_houses_imv_batch(
|
||||
db,
|
||||
house_ids,
|
||||
request_delay_sec=request_delay_sec,
|
||||
heartbeat=heartbeat,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"RealEnrichmentJobs",
|
||||
|
|
|
|||
383
tradein-mvp/backend/tests/test_scraper_kit_pipeline_parity.py
Normal file
383
tradein-mvp/backend/tests/test_scraper_kit_pipeline_parity.py
Normal file
|
|
@ -0,0 +1,383 @@
|
|||
"""Golden-parity: `scraper_kit.orchestration.pipeline` ≡ `app.services.scrape_pipeline`.
|
||||
|
||||
Strangler-инвариант (#2135): новая scraper_kit-копия оркестрации sweep-pipeline
|
||||
должна давать ТОТ ЖЕ наблюдаемый эффект, что и старый боевой orchestrator, на
|
||||
одинаковом мокнутом I/O. Развязка `app.*` (config/matcher/enrichment/runs/shutdown
|
||||
инжектируются) не должна менять ветвление.
|
||||
|
||||
Фокус — КРИТИЧНАЯ логика оркестрации (то, ради чего PR):
|
||||
- ban/rotation state-machine: SERP-блок на anchor'е → abort sweep;
|
||||
- partial-ban intake (#1950): SERP intake сохранён + detail заблокирован →
|
||||
mark_done (не mark_banned) под флагом avito_serp_ok_not_banned;
|
||||
- detail-фаза: N подряд блоков → propagate → anchor-handler;
|
||||
- counters aggregation по anchor'ам + IMV-фаза;
|
||||
- последовательность вызовов scrape_runs (heartbeat / mark_done / mark_banned).
|
||||
|
||||
Метод: гоняем ОБА orchestrator'а (`_drive_old` / `_drive_new`) на идентичном
|
||||
описании сценария, перехватывая scrape_runs recording-mock'ом. Сравниваем:
|
||||
- итоговый counters.to_dict();
|
||||
- последовательность (method, numeric-counters) вызовов scrape_runs.
|
||||
|
||||
Без сети, без БД.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
|
||||
|
||||
from scraper_kit.avito_exceptions import (
|
||||
AvitoBlockedError as NewAvitoBlockedError,
|
||||
)
|
||||
from scraper_kit.orchestration.pipeline import (
|
||||
run_avito_city_sweep as new_city_sweep,
|
||||
)
|
||||
|
||||
from app.services.scrape_pipeline import run_avito_city_sweep as old_city_sweep
|
||||
from app.services.scrapers.avito_exceptions import (
|
||||
AvitoBlockedError as OldAvitoBlockedError,
|
||||
)
|
||||
|
||||
# ── recording scrape_runs ────────────────────────────────────────────────────
|
||||
|
||||
_NON_NUMERIC_KEYS = {"enrichment_abort_note"}
|
||||
|
||||
|
||||
class _RunsRecorder:
|
||||
"""Записывает вызовы scrape_runs-финализаторов в общий список.
|
||||
|
||||
is_cancelled всегда False (кооп-cancel не тестируем здесь). Каждый терминальный
|
||||
вызов пишет (method, counters_dict) — сравниваем последовательности old/new.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.calls: list[tuple[str, dict[str, Any]]] = []
|
||||
|
||||
def is_cancelled(self, db: Any, run_id: int) -> bool:
|
||||
return False
|
||||
|
||||
def update_heartbeat(self, db: Any, run_id: int, counters: dict[str, Any]) -> None:
|
||||
self.calls.append(("update_heartbeat", dict(counters)))
|
||||
|
||||
def mark_done(self, db: Any, run_id: int, counters: dict[str, Any]) -> None:
|
||||
self.calls.append(("mark_done", dict(counters)))
|
||||
|
||||
def mark_banned(self, db: Any, run_id: int, error: str, counters: dict[str, Any]) -> None:
|
||||
self.calls.append(("mark_banned", dict(counters)))
|
||||
|
||||
def mark_failed(self, db: Any, run_id: int, error: str, counters: dict[str, Any]) -> None:
|
||||
self.calls.append(("mark_failed", dict(counters)))
|
||||
|
||||
|
||||
_DriveResult = tuple[dict[str, int], list[tuple[str, dict[str, int]]]]
|
||||
|
||||
|
||||
def _normalize(calls: list[tuple[str, dict[str, Any]]]) -> list[tuple[str, dict[str, int]]]:
|
||||
"""Оставить только имя метода + числовые counters (убрать note-строки)."""
|
||||
out: list[tuple[str, dict[str, int]]] = []
|
||||
for method, counters in calls:
|
||||
numeric = {k: v for k, v in counters.items() if k not in _NON_NUMERIC_KEYS}
|
||||
out.append((method, numeric))
|
||||
return out
|
||||
|
||||
|
||||
# ── scenario description ─────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class _Scenario:
|
||||
"""Декларативное описание одного прогона city-sweep для обоих orchestrator'ов."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
anchors: list[tuple[float, float, str]],
|
||||
# per-anchor: ("lots", n_lots, ins, upd) | ("block",)
|
||||
per_anchor: list[tuple[Any, ...]],
|
||||
enrich_houses: bool = False,
|
||||
detail_top_n: int = 0,
|
||||
detail_rows: int = 0,
|
||||
detail_behavior: str = "ok", # "ok" | "block_all"
|
||||
enrich_imv: bool = False,
|
||||
imv_result: tuple[int, int, int] | None = None,
|
||||
avito_serp_ok_not_banned: bool = True,
|
||||
avito_proxy_max_rotations: int = 0,
|
||||
lots_have_house_url: bool = False,
|
||||
) -> None:
|
||||
self.anchors = anchors
|
||||
self.per_anchor = per_anchor
|
||||
self.enrich_houses = enrich_houses
|
||||
self.detail_top_n = detail_top_n
|
||||
self.detail_rows = detail_rows
|
||||
self.detail_behavior = detail_behavior
|
||||
self.enrich_imv = enrich_imv
|
||||
self.imv_result = imv_result
|
||||
self.avito_serp_ok_not_banned = avito_serp_ok_not_banned
|
||||
self.avito_proxy_max_rotations = avito_proxy_max_rotations
|
||||
self.lots_have_house_url = lots_have_house_url
|
||||
|
||||
def _config(self) -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
scraper_fetch_mode="curl_cffi",
|
||||
browser_http_endpoint="http://browser.test/fetch",
|
||||
scraper_proxy_url=None,
|
||||
avito_proxy_rotate_url=None,
|
||||
avito_proxy_max_rotations=self.avito_proxy_max_rotations,
|
||||
avito_serp_ok_not_banned=self.avito_serp_ok_not_banned,
|
||||
avito_proxy_rotate_settle_s=0.0,
|
||||
proxy_rotate_attempts=1,
|
||||
proxy_rotate_attempt_timeout_s=1.0,
|
||||
cian_proxy_rotate_url=None,
|
||||
cian_proxy_max_rotations=0,
|
||||
yandex_proxy_rotate_url=None,
|
||||
yandex_proxy_max_rotations=0,
|
||||
scraper_skip_seen_today=False,
|
||||
)
|
||||
|
||||
def _fetch_around_side_effects(self, blocked_exc: type[Exception]) -> list[Any]:
|
||||
effects: list[Any] = []
|
||||
for spec in self.per_anchor:
|
||||
if spec[0] == "block":
|
||||
effects.append(blocked_exc("SERP blocked"))
|
||||
else:
|
||||
_, n_lots, _ins, _upd = spec
|
||||
house_url = "/catalog/houses/ekb/h1/100" if self.lots_have_house_url else None
|
||||
effects.append([MagicMock(house_url=house_url) for _ in range(n_lots)])
|
||||
return effects
|
||||
|
||||
def _save_side_effects(self) -> list[tuple[int, int]]:
|
||||
return [(spec[2], spec[3]) for spec in self.per_anchor if spec[0] == "lots" and spec[1] > 0]
|
||||
|
||||
def _detail_rows(self) -> list[dict[str, str]]:
|
||||
return [{"source_url": f"https://www.avito.ru/x/{i}"} for i in range(self.detail_rows)]
|
||||
|
||||
def _fetch_detail_side_effects(self, blocked_exc: type[Exception]) -> Any:
|
||||
if self.detail_behavior == "block_all":
|
||||
return blocked_exc("detail blocked")
|
||||
return MagicMock(house_catalog_url=None)
|
||||
|
||||
|
||||
def _make_db(scenario: _Scenario) -> MagicMock:
|
||||
db = MagicMock()
|
||||
# detail-фаза: db.execute(...).mappings().all() → priority rows
|
||||
db.execute.return_value.mappings.return_value.all.return_value = scenario._detail_rows()
|
||||
return db
|
||||
|
||||
|
||||
def _make_scraper(scenario: _Scenario, blocked_exc: type[Exception]) -> MagicMock:
|
||||
scraper = MagicMock()
|
||||
scraper._cffi = None
|
||||
scraper._browser = None
|
||||
scraper.fetch_around = AsyncMock(side_effect=scenario._fetch_around_side_effects(blocked_exc))
|
||||
return scraper
|
||||
|
||||
|
||||
def _async_session_cm() -> MagicMock:
|
||||
sess = MagicMock()
|
||||
sess.__aenter__ = AsyncMock(return_value=sess)
|
||||
sess.__aexit__ = AsyncMock(return_value=None)
|
||||
sess.close = AsyncMock()
|
||||
return sess
|
||||
|
||||
|
||||
async def _drive_old(scenario: _Scenario) -> _DriveResult:
|
||||
recorder = _RunsRecorder()
|
||||
db = _make_db(scenario)
|
||||
scraper = _make_scraper(scenario, OldAvitoBlockedError)
|
||||
save_mock = MagicMock(side_effect=scenario._save_side_effects())
|
||||
|
||||
imv_res = None
|
||||
if scenario.imv_result is not None:
|
||||
checked, saved, errors = scenario.imv_result
|
||||
imv_res = SimpleNamespace(checked=checked, saved=saved, errors=errors)
|
||||
|
||||
with (
|
||||
patch("app.services.scrape_pipeline.AvitoScraper", return_value=scraper),
|
||||
patch("app.services.scrape_pipeline.save_listings", save_mock),
|
||||
patch(
|
||||
"app.services.scrape_pipeline.fetch_house_catalog",
|
||||
AsyncMock(return_value=MagicMock()),
|
||||
),
|
||||
patch(
|
||||
"app.services.scrape_pipeline.save_house_catalog_enrichment",
|
||||
return_value={"house_id": 1},
|
||||
),
|
||||
patch(
|
||||
"app.services.scrape_pipeline.fetch_detail",
|
||||
AsyncMock(side_effect=scenario._fetch_detail_side_effects(OldAvitoBlockedError)),
|
||||
),
|
||||
patch("app.services.scrape_pipeline.save_detail_enrichment", return_value=True),
|
||||
patch("app.services.scrape_pipeline.scrape_runs", recorder),
|
||||
patch(
|
||||
"app.services.house_imv_backfill.process_houses_imv_batch",
|
||||
AsyncMock(return_value=imv_res),
|
||||
),
|
||||
patch("app.services.scrape_pipeline.settings", scenario._config()),
|
||||
patch("app.services.scrape_pipeline.shutdown_requested", return_value=False),
|
||||
patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
|
||||
patch("app.services.scrape_pipeline.AsyncSession", return_value=_async_session_cm()),
|
||||
):
|
||||
counters = await old_city_sweep(
|
||||
db,
|
||||
run_id=1,
|
||||
radius_m=1000,
|
||||
anchors=scenario.anchors,
|
||||
pages_per_anchor=1,
|
||||
enrich_houses=scenario.enrich_houses,
|
||||
detail_top_n=scenario.detail_top_n,
|
||||
request_delay_sec=0.0,
|
||||
enrich_imv=scenario.enrich_imv,
|
||||
)
|
||||
return counters.to_dict(), _normalize(recorder.calls)
|
||||
|
||||
|
||||
async def _drive_new(scenario: _Scenario) -> _DriveResult:
|
||||
recorder = _RunsRecorder()
|
||||
db = _make_db(scenario)
|
||||
scraper = _make_scraper(scenario, NewAvitoBlockedError)
|
||||
save_mock = MagicMock(side_effect=scenario._save_side_effects())
|
||||
|
||||
imv_res = None
|
||||
if scenario.imv_result is not None:
|
||||
checked, saved, errors = scenario.imv_result
|
||||
imv_res = SimpleNamespace(checked=checked, saved=saved, errors=errors)
|
||||
|
||||
enrichment = MagicMock()
|
||||
enrichment.process_houses_imv_batch = AsyncMock(return_value=imv_res)
|
||||
|
||||
pfx = "scraper_kit.orchestration.pipeline"
|
||||
with (
|
||||
patch(f"{pfx}.AvitoScraper", return_value=scraper),
|
||||
patch(f"{pfx}.save_listings", save_mock),
|
||||
patch(f"{pfx}.fetch_house_catalog", AsyncMock(return_value=MagicMock())),
|
||||
patch(f"{pfx}.save_house_catalog_enrichment", return_value={"house_id": 1}),
|
||||
patch(
|
||||
f"{pfx}.fetch_detail",
|
||||
AsyncMock(side_effect=scenario._fetch_detail_side_effects(NewAvitoBlockedError)),
|
||||
),
|
||||
patch(f"{pfx}.save_detail_enrichment", return_value=True),
|
||||
patch(f"{pfx}.runs", recorder),
|
||||
patch(f"{pfx}.asyncio.sleep", AsyncMock()),
|
||||
patch(f"{pfx}.AsyncSession", return_value=_async_session_cm()),
|
||||
):
|
||||
counters = await new_city_sweep(
|
||||
db,
|
||||
run_id=1,
|
||||
config=scenario._config(),
|
||||
matcher=MagicMock(),
|
||||
enrichment=enrichment,
|
||||
shutdown_requested=lambda: False,
|
||||
radius_m=1000,
|
||||
anchors=scenario.anchors,
|
||||
pages_per_anchor=1,
|
||||
enrich_houses=scenario.enrich_houses,
|
||||
detail_top_n=scenario.detail_top_n,
|
||||
request_delay_sec=0.0,
|
||||
enrich_imv=scenario.enrich_imv,
|
||||
)
|
||||
return counters.to_dict(), _normalize(recorder.calls)
|
||||
|
||||
|
||||
async def _assert_parity(scenario: _Scenario) -> dict[str, int]:
|
||||
old_counters, old_calls = await _drive_old(scenario)
|
||||
new_counters, new_calls = await _drive_new(scenario)
|
||||
assert (
|
||||
new_counters == old_counters
|
||||
), f"counters diverged:\n old={old_counters}\n new={new_counters}"
|
||||
assert (
|
||||
new_calls == old_calls
|
||||
), f"scrape_runs calls diverged:\n old={old_calls}\n new={new_calls}"
|
||||
return new_counters
|
||||
|
||||
|
||||
# ── scenarios ────────────────────────────────────────────────────────────────
|
||||
|
||||
_ANCHORS_2 = [(56.84, 60.60, "A1"), (56.79, 60.53, "A2")]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_parity_happy_path_counters_aggregation() -> None:
|
||||
"""2 anchor'а, SERP+save, без houses/detail/imv → mark_done + агрегированные counters."""
|
||||
scenario = _Scenario(
|
||||
anchors=_ANCHORS_2,
|
||||
per_anchor=[("lots", 10, 8, 2), ("lots", 6, 5, 1)],
|
||||
)
|
||||
counters = await _assert_parity(scenario)
|
||||
# Убеждаемся что сценарий действительно прогнал агрегацию (не пустой прогон).
|
||||
assert counters["lots_fetched"] == 16
|
||||
assert counters["lots_inserted"] == 13
|
||||
assert counters["lots_updated"] == 3
|
||||
assert counters["anchors_done"] == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_parity_full_ban_no_intake_marks_banned() -> None:
|
||||
"""Первый же anchor SERP-блок, лоты не сохранены → mark_banned (не done)."""
|
||||
scenario = _Scenario(
|
||||
anchors=_ANCHORS_2,
|
||||
per_anchor=[("block",), ("block",)],
|
||||
)
|
||||
await _assert_parity(scenario)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_parity_partial_ban_intake_marks_done() -> None:
|
||||
"""Anchor1 intake сохранён, anchor2 SERP-блок → partial-intake → mark_done не banned."""
|
||||
scenario = _Scenario(
|
||||
anchors=_ANCHORS_2,
|
||||
per_anchor=[("lots", 10, 9, 1), ("block",)],
|
||||
avito_serp_ok_not_banned=True,
|
||||
)
|
||||
await _assert_parity(scenario)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_parity_partial_ban_flag_off_marks_banned() -> None:
|
||||
"""avito_serp_ok_not_banned=False: даже при intake SERP-блок → mark_banned."""
|
||||
scenario = _Scenario(
|
||||
anchors=_ANCHORS_2,
|
||||
per_anchor=[("lots", 10, 9, 1), ("block",)],
|
||||
avito_serp_ok_not_banned=False,
|
||||
)
|
||||
await _assert_parity(scenario)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_parity_detail_consecutive_block_propagates() -> None:
|
||||
"""detail-фаза: 3 подряд блока (rotations=0) → propagate → anchor-handler.
|
||||
|
||||
Лоты сохранены на этом же anchor'е → partial-intake → mark_done.
|
||||
"""
|
||||
scenario = _Scenario(
|
||||
anchors=[(56.84, 60.60, "A1")],
|
||||
per_anchor=[("lots", 5, 5, 0)],
|
||||
detail_top_n=5,
|
||||
detail_rows=3,
|
||||
detail_behavior="block_all",
|
||||
avito_serp_ok_not_banned=True,
|
||||
avito_proxy_max_rotations=0,
|
||||
)
|
||||
await _assert_parity(scenario)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_parity_imv_phase_counters() -> None:
|
||||
"""IMV-фаза: touched houses → process_houses_imv_batch → imv-counters агрегированы."""
|
||||
scenario = _Scenario(
|
||||
anchors=[(56.84, 60.60, "A1")],
|
||||
per_anchor=[("lots", 4, 4, 0)],
|
||||
enrich_houses=True,
|
||||
lots_have_house_url=True,
|
||||
enrich_imv=True,
|
||||
imv_result=(3, 2, 1),
|
||||
)
|
||||
counters = await _assert_parity(scenario)
|
||||
# touched house → IMV-фаза отработала, imv-counters агрегированы одинаково.
|
||||
assert counters["imv_attempted"] == 3
|
||||
assert counters["imv_enriched"] == 2
|
||||
assert counters["imv_failed"] == 1
|
||||
|
|
@ -112,6 +112,28 @@ class ScraperConfig(Protocol):
|
|||
cookie_encryption_key: str
|
||||
# Sentry/GlitchTip DSN (опционально).
|
||||
glitchtip_dsn: str | None
|
||||
# ── Оркестрация sweep-pipeline (#2135) ───────────────────────────────────
|
||||
# Поля, читаемые слоем оркестрации (scraper_kit.orchestration.pipeline).
|
||||
# Собраны grep'ом `settings.<name>` по app/services/scrape_pipeline.py.
|
||||
# Проксируются той же продуктовой реализацией (RealScraperConfig).
|
||||
#
|
||||
# Partial-ban intake: SERP уже собрал лоты, а detail/houses заблокировало —
|
||||
# run помечается 'done' (не 'banned'), intake сохранён (#1950).
|
||||
avito_serp_ok_not_banned: bool
|
||||
# changeip settle-пауза (секунды) после ротации IP.
|
||||
avito_proxy_rotate_settle_s: float
|
||||
# Ретраи changeip-GET: N попыток по proxy_rotate_attempt_timeout_s каждая (#1950).
|
||||
proxy_rotate_attempts: int
|
||||
proxy_rotate_attempt_timeout_s: float
|
||||
# Cian rotate-прокси + лимит ротаций.
|
||||
cian_proxy_rotate_url: str | None
|
||||
cian_proxy_max_rotations: int
|
||||
# Yandex лимит ротаций (rotate_url уже выше).
|
||||
yandex_proxy_max_rotations: int
|
||||
# Пропускать листинги, уже обновлённые сегодня по МСК (full-load дедуп).
|
||||
scraper_skip_seen_today: bool
|
||||
# Per-fetch watchdog-таймаут Cian full-load (секунды).
|
||||
cian_full_load_per_fetch_timeout_s: float
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
|
|
@ -171,6 +193,21 @@ class EnrichmentJobs(Protocol):
|
|||
"""Догрузить точные адреса Yandex. Проксирует backfill_yandex_addresses."""
|
||||
...
|
||||
|
||||
async def process_houses_imv_batch(
|
||||
self,
|
||||
db: Session,
|
||||
house_ids: set[int],
|
||||
*,
|
||||
request_delay_sec: float = ...,
|
||||
heartbeat: Callable[[], None] | None = ...,
|
||||
) -> Any:
|
||||
"""Avito IMV-оценка явного набора house_id (финальная IMV-фаза city-sweep).
|
||||
|
||||
Проксирует process_houses_imv_batch. Возвращаемый результат имеет поля
|
||||
.checked/.saved/.errors (тип остаётся Any — не тянем app.* в контракты).
|
||||
"""
|
||||
...
|
||||
|
||||
|
||||
__all__ = [
|
||||
"EnrichmentJobs",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,15 @@
|
|||
"""Оркестрация sweep-pipeline scraper_kit — strangler-копия `app.services.scrape_pipeline`.
|
||||
|
||||
Координирует полный прогон скрапа (SERP → save → houses → detail → IMV) поверх
|
||||
провайдеров `scraper_kit.providers.*` и базового слоя `scraper_kit.base`. Развязана
|
||||
от `app.*` через Protocol-инжекцию (`scraper_kit.contracts`):
|
||||
|
||||
- `app.core.config.settings` → инжектируемый `ScraperConfig`
|
||||
- `app.services.matching` → инжектируемый `HouseMatcher`
|
||||
- `app.services.house_imv_backfill` → инжектируемый `EnrichmentJobs`
|
||||
- `app.core.shutdown.shutdown_requested` → инжектируемый `Callable[[], bool]`
|
||||
- `app.services.scrape_runs` → локальная копия `scraper_kit.orchestration.runs`
|
||||
|
||||
Боевой scheduler по-прежнему зовёт старый `app.services.scrape_pipeline` (strangler:
|
||||
COPY, не MOVE). Переключение рантайма на этот модуль — отдельный поздний шаг.
|
||||
"""
|
||||
File diff suppressed because it is too large
Load diff
|
|
@ -0,0 +1,392 @@
|
|||
"""scrape_runs helpers — tracking long-running pipeline runs (strangler-копия #2135).
|
||||
|
||||
Байт-эквивалент `app.services.scrape_runs` — чистые SQL-хелперы поверх таблицы
|
||||
`scrape_runs` (миграции 015 + 051). Развязка от `app.*`: единственное намеренное
|
||||
отличие — `sentry_sdk` импортируется опционально (kit standalone-импортируем, а
|
||||
`sentry-sdk` не входит в его зависимости). Если пакет не установлен — alert-хук
|
||||
best-effort no-op, поведение SQL-финализаторов идентично старому.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
try: # sentry опционален — kit не тянет его в зависимостях
|
||||
import sentry_sdk
|
||||
except ImportError: # pragma: no cover - в проде backend-env sentry_sdk присутствует
|
||||
sentry_sdk = None # type: ignore[assignment]
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Количество последовательных неудачных запусков (failed/banned), при достижении
|
||||
# которого отправляется Sentry alert. Алерт срабатывает ровно при N-й подряд ошибке
|
||||
# (anti-spam: не на каждой последующей).
|
||||
CONSECUTIVE_FAILURE_ALERT_THRESHOLD = 3
|
||||
|
||||
|
||||
def _column_counts(counters: dict[str, int]) -> tuple[int | None, int | None]:
|
||||
"""Извлечь значения для dedicated-колонок total_seen / new_count из jsonb-counters.
|
||||
|
||||
Все sweep-counters (YandexCitySweepCounters / CitySweepCounters / …) пишут число
|
||||
объявлений в выдаче как ``lots_fetched`` и число впервые вставленных как
|
||||
``lots_inserted``. Раньше эти значения попадали ТОЛЬКО в counters jsonb, а
|
||||
выделенные колонки total_seen/new_count оставались 0 → admin/observability
|
||||
показывала total_seen=0 при реально сохранённых строках (audit #1871/#1926).
|
||||
|
||||
Приоритет ключей:
|
||||
- total_seen ← 'total_seen' (если уже есть в counters) иначе 'lots_fetched'
|
||||
- new_count ← 'new_count' (если уже есть) иначе 'lots_inserted'
|
||||
|
||||
Возвращает (total_seen, new_count); None для ключа, которого нет в counters —
|
||||
тогда соответствующая колонка не перезаписывается (COALESCE-семантика в UPDATE).
|
||||
"""
|
||||
|
||||
def _pick(*keys: str) -> int | None:
|
||||
for key in keys:
|
||||
val = counters.get(key)
|
||||
if val is not None:
|
||||
try:
|
||||
return int(val)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return None
|
||||
|
||||
return _pick("total_seen", "lots_fetched"), _pick("new_count", "lots_inserted")
|
||||
|
||||
|
||||
def _alert_if_consecutive_failures(db: Session, source: str) -> None:
|
||||
"""Отправить Sentry alert если последние CONSECUTIVE_FAILURE_ALERT_THRESHOLD
|
||||
завершённых запусков для данного source имеют статус 'failed' или 'banned'.
|
||||
|
||||
Anti-spam: алерт срабатывает ТОЛЬКО когда стрик РОВНО равен порогу — т.е. запрос
|
||||
возвращает ровно N последних (failed|banned) и (N+1)-й, если существует, НЕ является
|
||||
failed/banned. Это предотвращает повторный алерт на каждой ошибке сверх порога.
|
||||
|
||||
Best-effort: весь блок обёрнут в try/except — сбой запроса или неинициализированный
|
||||
Sentry НЕ должен нарушать вызывающий mark_* путь.
|
||||
"""
|
||||
if sentry_sdk is None:
|
||||
return
|
||||
n = CONSECUTIVE_FAILURE_ALERT_THRESHOLD
|
||||
try:
|
||||
# Берём последние N+1 завершённых (non-running) запусков по source.
|
||||
# Сортируем по finished_at DESC чтобы самые свежие шли первыми.
|
||||
rows = db.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT status FROM scrape_runs
|
||||
WHERE source = :source
|
||||
AND status IN ('failed', 'banned', 'done', 'cancelled')
|
||||
ORDER BY finished_at DESC NULLS LAST
|
||||
LIMIT :limit
|
||||
"""
|
||||
),
|
||||
{"source": source, "limit": n + 1},
|
||||
).fetchall()
|
||||
|
||||
if len(rows) < n:
|
||||
# Ещё не набралось N завершённых запусков вообще — алерт не нужен.
|
||||
return
|
||||
|
||||
# Первые N должны быть все failed/banned.
|
||||
first_n = rows[:n]
|
||||
if not all(r.status in ("failed", "banned") for r in first_n):
|
||||
return
|
||||
|
||||
# (N+1)-й запуск, если есть, тоже должен НЕ быть failed/banned — иначе мы уже
|
||||
# должны были отправить алерт раньше и не стоит дублировать.
|
||||
if len(rows) > n and rows[n].status in ("failed", "banned"):
|
||||
return
|
||||
|
||||
# Стрик ровно достиг порога — отправляем алерт.
|
||||
sentry_sdk.capture_message(
|
||||
f"Scraper source '{source}' has {n} consecutive failed/banned runs — "
|
||||
"manual intervention may be required (expired cookies / ban / broken parser).",
|
||||
level="error",
|
||||
)
|
||||
logger.error(
|
||||
"sentry alert sent: source=%s has %d consecutive failed/banned runs", source, n
|
||||
)
|
||||
except Exception:
|
||||
pass # sentry_sdk not initialised in dev, or query failed — best-effort only
|
||||
|
||||
|
||||
def _alert_on_run_id(db: Session, run_id: int) -> None:
|
||||
"""Вспомогательная обёртка: извлекает source по run_id и вызывает
|
||||
_alert_if_consecutive_failures. Best-effort — не бросает исключений.
|
||||
"""
|
||||
try:
|
||||
row = db.execute(
|
||||
text("SELECT source FROM scrape_runs WHERE id = :run_id"),
|
||||
{"run_id": run_id},
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return
|
||||
_alert_if_consecutive_failures(db, str(row.source))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def create_run(db: Session, *, source: str, params: dict[str, Any]) -> int:
|
||||
"""INSERT scrape_runs(source, status='running', params, started_at=NOW()).
|
||||
|
||||
run_type DEFAULT 'city_sweep' (из 051 миграции).
|
||||
Returns run_id (bigint).
|
||||
"""
|
||||
row = db.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO scrape_runs (source, status, params, started_at, heartbeat_at)
|
||||
VALUES (:source, 'running', CAST(:params AS jsonb), NOW(), NOW())
|
||||
RETURNING id
|
||||
"""
|
||||
),
|
||||
{"source": source, "params": json.dumps(params, ensure_ascii=False)},
|
||||
).fetchone()
|
||||
db.commit()
|
||||
assert row is not None, "scrape_runs INSERT returned no id"
|
||||
return int(row.id)
|
||||
|
||||
|
||||
def update_heartbeat(db: Session, run_id: int, counters: dict[str, int]) -> None:
|
||||
"""UPDATE heartbeat_at=NOW(), counters=:counters + total_seen/new_count колонки.
|
||||
|
||||
total_seen/new_count извлекаются из counters (lots_fetched/lots_inserted) и
|
||||
пишутся в выделенные колонки, чтобы observability не показывала 0 (audit #1926).
|
||||
COALESCE: если ключа нет в counters — старое значение колонки сохраняется.
|
||||
"""
|
||||
total_seen, new_count = _column_counts(counters)
|
||||
db.execute(
|
||||
text(
|
||||
"""
|
||||
UPDATE scrape_runs
|
||||
SET heartbeat_at = NOW(),
|
||||
counters = CAST(:counters AS jsonb),
|
||||
total_seen = COALESCE(CAST(:total_seen AS int), total_seen),
|
||||
new_count = COALESCE(CAST(:new_count AS int), new_count)
|
||||
WHERE id = :run_id
|
||||
"""
|
||||
),
|
||||
{
|
||||
"run_id": run_id,
|
||||
"counters": json.dumps(counters),
|
||||
"total_seen": total_seen,
|
||||
"new_count": new_count,
|
||||
},
|
||||
)
|
||||
db.commit()
|
||||
|
||||
|
||||
def is_cancelled(db: Session, run_id: int) -> bool:
|
||||
"""Проверить status='cancelled' (cooperative cancel в long-running pipeline)."""
|
||||
row = db.execute(
|
||||
text("SELECT status FROM scrape_runs WHERE id = :id"),
|
||||
{"id": run_id},
|
||||
).fetchone()
|
||||
return row is not None and row.status == "cancelled"
|
||||
|
||||
|
||||
def mark_done(db: Session, run_id: int, counters: dict[str, int]) -> None:
|
||||
"""Финализация run: status='done', finished_at=NOW(), counters + total_seen/new_count.
|
||||
|
||||
total_seen/new_count извлекаются из counters (lots_fetched/lots_inserted) и пишутся
|
||||
в выделенные колонки — иначе admin/observability показывает 0 (audit #1926).
|
||||
"""
|
||||
total_seen, new_count = _column_counts(counters)
|
||||
row = db.execute(
|
||||
text(
|
||||
"""
|
||||
UPDATE scrape_runs
|
||||
SET status = 'done', finished_at = NOW(), heartbeat_at = NOW(),
|
||||
counters = CAST(:counters AS jsonb),
|
||||
total_seen = COALESCE(CAST(:total_seen AS int), total_seen),
|
||||
new_count = COALESCE(CAST(:new_count AS int), new_count)
|
||||
WHERE id = :run_id AND status = 'running'
|
||||
RETURNING id
|
||||
"""
|
||||
),
|
||||
{
|
||||
"run_id": run_id,
|
||||
"counters": json.dumps(counters),
|
||||
"total_seen": total_seen,
|
||||
"new_count": new_count,
|
||||
},
|
||||
).first()
|
||||
if row is None:
|
||||
logger.warning("mark_done no-op: run_id=%d not in 'running' state", run_id)
|
||||
db.commit()
|
||||
|
||||
|
||||
def mark_failed(db: Session, run_id: int, error: str, counters: dict[str, int]) -> None:
|
||||
"""Финализация run: status='failed', error (первые 1000 символов).
|
||||
|
||||
Defensive rollback: если до этого вызова в той же транзакции был ошибочный UPDATE,
|
||||
он мог оставить сессию в error state — rollback сбрасывает состояние.
|
||||
"""
|
||||
try:
|
||||
db.rollback() # defensive: cascade-safe if prior UPDATE left txn in error state
|
||||
except Exception:
|
||||
pass
|
||||
total_seen, new_count = _column_counts(counters)
|
||||
row = db.execute(
|
||||
text(
|
||||
"""
|
||||
UPDATE scrape_runs
|
||||
SET status = 'failed', finished_at = NOW(), heartbeat_at = NOW(),
|
||||
error = :error, counters = CAST(:counters AS jsonb),
|
||||
total_seen = COALESCE(CAST(:total_seen AS int), total_seen),
|
||||
new_count = COALESCE(CAST(:new_count AS int), new_count)
|
||||
WHERE id = :run_id AND status = 'running'
|
||||
RETURNING id
|
||||
"""
|
||||
),
|
||||
{
|
||||
"run_id": run_id,
|
||||
"error": error[:1000],
|
||||
"counters": json.dumps(counters),
|
||||
"total_seen": total_seen,
|
||||
"new_count": new_count,
|
||||
},
|
||||
).first()
|
||||
if row is None:
|
||||
logger.warning("mark_failed no-op: run_id=%d not in 'running' state", run_id)
|
||||
db.commit()
|
||||
# Алерт после коммита — статус уже персистирован в БД; best-effort (не бросает).
|
||||
_alert_on_run_id(db, run_id)
|
||||
|
||||
|
||||
def mark_banned(db: Session, run_id: int, error: str, counters: dict[str, int]) -> None:
|
||||
"""Финализация run: status='banned' (IP заблокирован Avito — 403/captcha).
|
||||
|
||||
Per migration 015 — 'banned' задокументирован как 'Avito вернул 403/captcha'.
|
||||
Отличается от 'failed': это external constraint, не наш bug. Cooldown 2-4 часа.
|
||||
|
||||
Defensive rollback: если до этого вызова в той же транзакции был ошибочный UPDATE,
|
||||
он мог оставить сессию в error state — rollback сбрасывает состояние.
|
||||
"""
|
||||
try:
|
||||
db.rollback() # defensive: cascade-safe if prior UPDATE left txn in error state
|
||||
except Exception:
|
||||
pass
|
||||
total_seen, new_count = _column_counts(counters)
|
||||
row = db.execute(
|
||||
text(
|
||||
"""
|
||||
UPDATE scrape_runs
|
||||
SET status = 'banned', finished_at = NOW(), heartbeat_at = NOW(),
|
||||
error = :error, counters = CAST(:counters AS jsonb),
|
||||
total_seen = COALESCE(CAST(:total_seen AS int), total_seen),
|
||||
new_count = COALESCE(CAST(:new_count AS int), new_count)
|
||||
WHERE id = :run_id AND status = 'running'
|
||||
RETURNING id
|
||||
"""
|
||||
),
|
||||
{
|
||||
"run_id": run_id,
|
||||
"error": error[:1000],
|
||||
"counters": json.dumps(counters),
|
||||
"total_seen": total_seen,
|
||||
"new_count": new_count,
|
||||
},
|
||||
).first()
|
||||
if row is None:
|
||||
logger.warning("mark_banned no-op: run_id=%d not in 'running' state", run_id)
|
||||
db.commit()
|
||||
# Алерт после коммита — статус уже персистирован в БД; best-effort (не бросает).
|
||||
_alert_on_run_id(db, run_id)
|
||||
|
||||
|
||||
def mark_cancelled(db: Session, run_id: int) -> bool:
|
||||
"""Set status='cancelled' если currently 'running'. Returns True если cancelled."""
|
||||
result = db.execute(
|
||||
text(
|
||||
"""
|
||||
UPDATE scrape_runs
|
||||
SET status = 'cancelled', finished_at = NOW()
|
||||
WHERE id = :run_id AND status = 'running'
|
||||
RETURNING id
|
||||
"""
|
||||
),
|
||||
{"run_id": run_id},
|
||||
).fetchone()
|
||||
db.commit()
|
||||
return result is not None
|
||||
|
||||
|
||||
def list_recent(db: Session, *, source: str, limit: int = 10) -> list[dict[str, Any]]:
|
||||
"""Список последних N runs по source, в порядке убывания started_at."""
|
||||
rows = (
|
||||
db.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT id AS run_id, source, status, params, counters, error,
|
||||
started_at, finished_at, heartbeat_at
|
||||
FROM scrape_runs
|
||||
WHERE source = :source
|
||||
ORDER BY started_at DESC NULLS LAST
|
||||
LIMIT :limit
|
||||
"""
|
||||
),
|
||||
{"source": source, "limit": limit},
|
||||
)
|
||||
.mappings()
|
||||
.all()
|
||||
)
|
||||
return [dict(r) for r in rows]
|
||||
|
||||
|
||||
def list_all(
|
||||
db: Session,
|
||||
*,
|
||||
source: str | None = None,
|
||||
status: str | None = None,
|
||||
limit: int = 50,
|
||||
offset: int = 0,
|
||||
) -> tuple[int, list[dict[str, Any]]]:
|
||||
"""Unified-список runs по всем source'ам с опц. фильтрами source/status.
|
||||
|
||||
Возвращает (total, rows): total — кол-во строк под фильтром (без limit/offset),
|
||||
rows — текущая страница (ORDER BY started_at DESC, LIMIT/OFFSET).
|
||||
|
||||
Используется единой scrapers-страницей вместо 3 per-source /runs.
|
||||
"""
|
||||
where = ["1=1"]
|
||||
params: dict[str, Any] = {"limit": limit, "offset": offset}
|
||||
if source is not None:
|
||||
where.append("source = :source")
|
||||
params["source"] = source
|
||||
if status is not None:
|
||||
where.append("status = :status")
|
||||
params["status"] = status
|
||||
where_sql = " AND ".join(where)
|
||||
|
||||
total_row = db.execute(
|
||||
text(f"SELECT COUNT(*) AS n FROM scrape_runs WHERE {where_sql}"),
|
||||
params,
|
||||
).fetchone()
|
||||
total = int(total_row.n) if total_row is not None else 0
|
||||
|
||||
rows = (
|
||||
db.execute(
|
||||
text(
|
||||
f"""
|
||||
SELECT id AS run_id, source, run_type, status, params, counters,
|
||||
total_seen, new_count, started_at, finished_at,
|
||||
heartbeat_at, error AS error_text
|
||||
FROM scrape_runs
|
||||
WHERE {where_sql}
|
||||
ORDER BY started_at DESC NULLS LAST
|
||||
LIMIT CAST(:limit AS int) OFFSET CAST(:offset AS int)
|
||||
"""
|
||||
),
|
||||
params,
|
||||
)
|
||||
.mappings()
|
||||
.all()
|
||||
)
|
||||
return total, [dict(r) for r in rows]
|
||||
Loading…
Add table
Reference in a new issue