lekss361
7a88964ef5
All checks were successful
Deploy Trade-In / test (push) Successful in 36s
Deploy Trade-In / changes (push) Successful in 7s
Deploy Trade-In / build-frontend (push) Has been skipped
Deploy Trade-In / build-backend (push) Successful in 54s
Deploy Trade-In / build-browser (push) Has been skipped
Deploy Trade-In / deploy (push) Successful in 45s
670 lines
26 KiB
Python
670 lines
26 KiB
Python
"""Unit tests for per-anchor watchdog timeout in city sweeps (#880).
|
||
|
||
Verifies that ANCHOR_TIMEOUT_SEC is applied to all three sweeps (avito / cian / yandex):
|
||
(a) A hung anchor (asyncio.sleep(large)) doesn't freeze the whole sweep.
|
||
(b) TimeoutError branch: errors_count incremented, anchor logged as timed-out.
|
||
(c) Sweep proceeds to the next anchor after the timeout.
|
||
(d) mark_done is called when the sweep finishes all anchors.
|
||
|
||
All tests are network-free — they monkeypatch the inner scrape call.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import os
|
||
|
||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
|
||
|
||
from typing import Any
|
||
from unittest.mock import AsyncMock, MagicMock
|
||
|
||
import pytest
|
||
|
||
from app.services import scrape_pipeline
|
||
from app.services.scrapers.base import ScrapedLot
|
||
from app.services.scrapers.cian import CianScraper
|
||
from app.services.scrapers.yandex_realty import YandexRealtyScraper
|
||
|
||
# Two anchors: first hangs (causes timeout), second succeeds.
|
||
TWO_ANCHORS: list[tuple[float, float, str]] = [
|
||
(56.8400, 60.6050, "Hang"),
|
||
(56.7950, 60.5300, "OK"),
|
||
]
|
||
|
||
|
||
# ── Shared fake scrape_runs ─────────────────────────────────────────────────
|
||
|
||
|
||
class _FakeRuns:
|
||
def __init__(self) -> None:
|
||
self.heartbeats: list[dict[str, int]] = []
|
||
self.done: dict[str, int] | None = None
|
||
self.failed: tuple[str, dict[str, int]] | None = None
|
||
self.banned: tuple[str, dict[str, int]] | None = None
|
||
|
||
def is_cancelled(self, _db: Any, _run_id: int) -> bool:
|
||
return False
|
||
|
||
def update_heartbeat(self, _db: Any, _run_id: int, counters: dict[str, int]) -> None:
|
||
self.heartbeats.append(dict(counters))
|
||
|
||
def mark_done(self, _db: Any, _run_id: int, counters: dict[str, int]) -> None:
|
||
self.done = dict(counters)
|
||
|
||
def mark_failed(self, _db: Any, _run_id: int, error: str, counters: dict[str, int]) -> None:
|
||
self.failed = (error, dict(counters))
|
||
|
||
def mark_banned(self, _db: Any, _run_id: int, error: str, counters: dict[str, int]) -> None:
|
||
self.banned = (error, dict(counters))
|
||
|
||
|
||
def _install_runs(monkeypatch: pytest.MonkeyPatch, fake: _FakeRuns) -> None:
|
||
monkeypatch.setattr(scrape_pipeline.scrape_runs, "is_cancelled", fake.is_cancelled)
|
||
monkeypatch.setattr(scrape_pipeline.scrape_runs, "update_heartbeat", fake.update_heartbeat)
|
||
monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_done", fake.mark_done)
|
||
monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_failed", fake.mark_failed)
|
||
monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_banned", fake.mark_banned)
|
||
|
||
|
||
def _fake_lot(source: str, offer_id: str) -> ScrapedLot:
|
||
return ScrapedLot(
|
||
source=source,
|
||
source_url=f"https://example.ru/{offer_id}/",
|
||
source_id=offer_id,
|
||
address="Екатеринбург, улица Тестовая, 1",
|
||
price_rub=4_000_000,
|
||
area_m2=42.0,
|
||
rooms=1,
|
||
)
|
||
|
||
|
||
# ── CianScraper lifecycle stub ──────────────────────────────────────────────
|
||
|
||
|
||
@pytest.fixture(autouse=True)
|
||
def _stub_cian_lifecycle(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
def _init(self: CianScraper) -> None:
|
||
self.request_delay_sec = 0.0
|
||
self._cffi = None
|
||
|
||
async def _aenter(self: CianScraper) -> CianScraper:
|
||
return self
|
||
|
||
async def _aexit(self: CianScraper, *_args: Any) -> None:
|
||
return None
|
||
|
||
monkeypatch.setattr(CianScraper, "__init__", _init)
|
||
monkeypatch.setattr(CianScraper, "__aenter__", _aenter)
|
||
monkeypatch.setattr(CianScraper, "__aexit__", _aexit)
|
||
|
||
|
||
# ── YandexRealtyScraper lifecycle stub ─────────────────────────────────────
|
||
|
||
|
||
@pytest.fixture(autouse=True)
|
||
def _stub_yandex_lifecycle(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
def _init(self: YandexRealtyScraper, city: str = "ekaterinburg") -> None:
|
||
self.city = city
|
||
self.request_delay_sec = 0.0
|
||
self._cffi_session = None
|
||
self._cookies = {}
|
||
|
||
async def _aenter(self: YandexRealtyScraper) -> YandexRealtyScraper:
|
||
return self
|
||
|
||
async def _aexit(self: YandexRealtyScraper, *_args: Any) -> None:
|
||
return None
|
||
|
||
monkeypatch.setattr(YandexRealtyScraper, "__init__", _init)
|
||
monkeypatch.setattr(YandexRealtyScraper, "__aenter__", _aenter)
|
||
monkeypatch.setattr(YandexRealtyScraper, "__aexit__", _aexit)
|
||
|
||
|
||
# ── ANCHOR_TIMEOUT_SEC is exported ─────────────────────────────────────────
|
||
|
||
|
||
def test_anchor_timeout_sec_exported() -> None:
|
||
from app.services.scrape_pipeline import ANCHOR_TIMEOUT_SEC
|
||
|
||
assert isinstance(ANCHOR_TIMEOUT_SEC, int)
|
||
# Reasonable sane range: 60s–600s
|
||
assert (
|
||
60 <= ANCHOR_TIMEOUT_SEC <= 600
|
||
), f"ANCHOR_TIMEOUT_SEC={ANCHOR_TIMEOUT_SEC} out of sane range"
|
||
|
||
|
||
# ── Avito sweep watchdog ────────────────────────────────────────────────────
|
||
|
||
|
||
async def test_avito_sweep_anchor_timeout_continues(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
"""Первый anchor виснет → таймаут; второй anchor успешен; mark_done вызывается.
|
||
|
||
run_avito_city_sweep использует внутреннюю _avito_anchor_phases (не run_avito_pipeline),
|
||
поэтому патчим AvitoScraper.fetch_around + save_listings.
|
||
"""
|
||
from app.services.scrapers.avito import AvitoScraper
|
||
|
||
# Monkeypatch asyncio.wait_for so we can inject TimeoutError on first call only,
|
||
# without actually waiting in CI.
|
||
original_wait_for = asyncio.wait_for
|
||
wf_call = 0
|
||
|
||
async def fake_wait_for(coro: Any, *, timeout: float) -> Any:
|
||
nonlocal wf_call
|
||
wf_call += 1
|
||
if wf_call == 1:
|
||
# Simulate the hung anchor: cancel the coro (mimic wait_for behaviour)
|
||
# and raise TimeoutError.
|
||
try:
|
||
coro.close()
|
||
except Exception:
|
||
pass
|
||
raise TimeoutError
|
||
return await original_wait_for(coro, timeout=timeout)
|
||
|
||
monkeypatch.setattr(scrape_pipeline.asyncio, "wait_for", fake_wait_for)
|
||
monkeypatch.setattr(scrape_pipeline.asyncio, "sleep", AsyncMock())
|
||
|
||
# Stub AvitoScraper.fetch_around — second anchor returns 3 lots.
|
||
fetch_calls: list[str] = []
|
||
|
||
async def fake_fetch_around(
|
||
self: AvitoScraper,
|
||
lat: float,
|
||
lon: float,
|
||
radius_m: int,
|
||
pages: int = 1,
|
||
delay_override_sec: float | None = None,
|
||
) -> list[ScrapedLot]:
|
||
fetch_calls.append(f"{lat:.4f}")
|
||
return [_fake_lot("avito", f"{lat:.4f}-{i}") for i in range(3)]
|
||
|
||
monkeypatch.setattr(AvitoScraper, "fetch_around", fake_fetch_around)
|
||
monkeypatch.setattr(scrape_pipeline, "save_listings", lambda _db, lots, **_kw: (len(lots), 0))
|
||
|
||
# Stub shared AsyncSession (used by avito sweep).
|
||
fake_session = AsyncMock()
|
||
fake_session.__aenter__ = AsyncMock(return_value=fake_session)
|
||
fake_session.__aexit__ = AsyncMock(return_value=None)
|
||
import curl_cffi.requests as _cffi_mod
|
||
|
||
monkeypatch.setattr(_cffi_mod, "AsyncSession", lambda **_kw: fake_session)
|
||
|
||
# Stub IMV phase (not relevant to this test).
|
||
async def fake_imv_batch(*_a: Any, **_kw: Any) -> Any:
|
||
result = MagicMock()
|
||
result.checked = 0
|
||
result.saved = 0
|
||
result.errors = 0
|
||
return result
|
||
|
||
monkeypatch.setattr(
|
||
"app.services.house_imv_backfill.process_houses_imv_batch",
|
||
fake_imv_batch,
|
||
)
|
||
|
||
fake = _FakeRuns()
|
||
_install_runs(monkeypatch, fake)
|
||
|
||
counters = await scrape_pipeline.run_avito_city_sweep(
|
||
db=MagicMock(),
|
||
run_id=1,
|
||
anchors=TWO_ANCHORS,
|
||
enrich_houses=False,
|
||
detail_top_n=0,
|
||
enrich_imv=False,
|
||
request_delay_sec=0.0,
|
||
)
|
||
|
||
# (a) sweep completes — doesn't hang
|
||
# (b) first anchor timed out → errors_count >= 1
|
||
assert counters.errors_count >= 1, "timeout должен увеличить errors_count"
|
||
# (c) second anchor reached: lots_fetched > 0
|
||
assert counters.lots_fetched > 0, "второй anchor должен был выполниться"
|
||
# (d) mark_done вызван (sweep завершился)
|
||
assert fake.done is not None, "mark_done должен быть вызван"
|
||
assert fake.failed is None, "mark_failed не должен быть вызван"
|
||
assert counters.anchors_done == len(TWO_ANCHORS)
|
||
|
||
|
||
# ── Cian sweep watchdog ─────────────────────────────────────────────────────
|
||
|
||
|
||
async def test_cian_sweep_anchor_timeout_continues(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
"""Первый anchor виснет → таймаут; второй anchor успешен; mark_done вызывается."""
|
||
serp_calls: list[str] = []
|
||
original_wait_for = asyncio.wait_for
|
||
wf_call = 0
|
||
|
||
async def fake_wait_for(coro: Any, *, timeout: float) -> Any:
|
||
nonlocal wf_call
|
||
wf_call += 1
|
||
if wf_call == 1:
|
||
try:
|
||
coro.close()
|
||
except Exception:
|
||
pass
|
||
raise TimeoutError
|
||
return await original_wait_for(coro, timeout=timeout)
|
||
|
||
monkeypatch.setattr(scrape_pipeline.asyncio, "wait_for", fake_wait_for)
|
||
monkeypatch.setattr(scrape_pipeline.asyncio, "sleep", AsyncMock())
|
||
|
||
async def fake_fetch_multi(
|
||
self: CianScraper, lat: float, lon: float, r: int, pages: int = 3
|
||
) -> list[ScrapedLot]:
|
||
serp_calls.append(f"{lat:.4f}")
|
||
return [_fake_lot("cian", f"{lat:.4f}-1")]
|
||
|
||
monkeypatch.setattr(CianScraper, "fetch_around_multi_room", fake_fetch_multi)
|
||
monkeypatch.setattr(
|
||
scrape_pipeline, "save_listings", lambda _db, lots, *, run_id=None: (len(lots), 0)
|
||
)
|
||
monkeypatch.setattr(
|
||
"app.services.scrapers.cian_detail.fetch_detail",
|
||
AsyncMock(return_value=None),
|
||
)
|
||
|
||
fake = _FakeRuns()
|
||
_install_runs(monkeypatch, fake)
|
||
|
||
counters = await scrape_pipeline.run_cian_city_sweep(
|
||
db=MagicMock(),
|
||
run_id=2,
|
||
anchors=TWO_ANCHORS,
|
||
detail_top_n=0,
|
||
enrich_houses=False,
|
||
request_delay_sec=0.0,
|
||
)
|
||
|
||
# (a) sweep completes (doesn't hang at anchor 1)
|
||
# (b) errors_count >= 1 (timeout counted)
|
||
assert counters.errors_count >= 1
|
||
# (c) second anchor was reached (SERP called once — first anchor timed out before SERP)
|
||
# wf_call==2 means the second wait_for ran → second anchor's coro executed.
|
||
assert wf_call == 2, f"wait_for should be called once per anchor; got {wf_call}"
|
||
# (d) mark_done called
|
||
assert fake.done is not None
|
||
assert fake.failed is None
|
||
assert counters.anchors_done == len(TWO_ANCHORS)
|
||
|
||
|
||
# ── Yandex sweep watchdog ───────────────────────────────────────────────────
|
||
|
||
|
||
async def test_yandex_sweep_anchor_timeout_continues(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
"""Первый anchor виснет → таймаут; второй anchor успешен; mark_done вызывается."""
|
||
serp_calls: list[str] = []
|
||
original_wait_for = asyncio.wait_for
|
||
wf_call = 0
|
||
|
||
async def fake_wait_for(coro: Any, *, timeout: float) -> Any:
|
||
nonlocal wf_call
|
||
wf_call += 1
|
||
if wf_call == 1:
|
||
try:
|
||
coro.close()
|
||
except Exception:
|
||
pass
|
||
raise TimeoutError
|
||
return await original_wait_for(coro, timeout=timeout)
|
||
|
||
monkeypatch.setattr(scrape_pipeline.asyncio, "wait_for", fake_wait_for)
|
||
monkeypatch.setattr(scrape_pipeline.asyncio, "sleep", AsyncMock())
|
||
|
||
async def fake_fetch_multi(
|
||
self: YandexRealtyScraper,
|
||
lat: float,
|
||
lon: float,
|
||
radius_m: int = 1500,
|
||
max_pages: int = 2,
|
||
**_: Any,
|
||
) -> list[ScrapedLot]:
|
||
serp_calls.append(f"{lat:.4f}")
|
||
return [_fake_lot("yandex", f"{lat:.4f}-1")]
|
||
|
||
monkeypatch.setattr(YandexRealtyScraper, "fetch_around_multi_room", fake_fetch_multi)
|
||
monkeypatch.setattr(
|
||
scrape_pipeline, "save_listings", lambda _db, lots, *, run_id=None: (len(lots), 0)
|
||
)
|
||
|
||
fake = _FakeRuns()
|
||
_install_runs(monkeypatch, fake)
|
||
|
||
counters = await scrape_pipeline.run_yandex_city_sweep(
|
||
db=MagicMock(),
|
||
run_id=3,
|
||
anchors=TWO_ANCHORS,
|
||
pages_per_anchor=1,
|
||
enrich_address=False,
|
||
request_delay_sec=0.0,
|
||
)
|
||
|
||
# (a) sweep completes
|
||
# (b) timeout counted
|
||
assert counters.errors_count >= 1
|
||
# (c) second anchor reached
|
||
assert wf_call == 2
|
||
# (d) mark_done called
|
||
assert fake.done is not None
|
||
assert fake.failed is None
|
||
assert counters.anchors_done == len(TWO_ANCHORS)
|
||
|
||
|
||
# ── All three: timeout increments errors_count and is logged, sweep reaches mark_done ──
|
||
|
||
|
||
async def test_cian_timeout_increments_counter_and_marks_done(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
"""Детальная проверка: timeout anchor → errors_count+1, sweep продолжается, mark_done.
|
||
|
||
Использует fake_wait_for (зеркало первых трёх тестов): первый вызов → TimeoutError
|
||
(имитирует зависший anchor), второй вызов → реальный wait_for (нормальный anchor).
|
||
Проверяет что:
|
||
- errors_count увеличивается
|
||
- второй anchor выполнился (SERP вызван)
|
||
- mark_done вызван (sweep завершился)
|
||
- mark_banned НЕ вызван (одиночный таймаут < порога consecutive failures)
|
||
"""
|
||
original_wait_for = asyncio.wait_for
|
||
wf_call = 0
|
||
|
||
async def fake_wait_for(coro: Any, *, timeout: float) -> Any:
|
||
nonlocal wf_call
|
||
wf_call += 1
|
||
if wf_call == 1:
|
||
try:
|
||
coro.close()
|
||
except Exception:
|
||
pass
|
||
raise TimeoutError
|
||
return await original_wait_for(coro, timeout=timeout)
|
||
|
||
monkeypatch.setattr(scrape_pipeline.asyncio, "wait_for", fake_wait_for)
|
||
monkeypatch.setattr(scrape_pipeline.asyncio, "sleep", AsyncMock())
|
||
|
||
serp_calls: list[str] = []
|
||
|
||
async def fake_fetch(
|
||
self: CianScraper, lat: float, lon: float, r: int, pages: int = 3
|
||
) -> list[ScrapedLot]:
|
||
serp_calls.append(f"{lat:.4f}")
|
||
return [_fake_lot("cian", f"{lat:.4f}-ok")]
|
||
|
||
monkeypatch.setattr(CianScraper, "fetch_around_multi_room", fake_fetch)
|
||
monkeypatch.setattr(
|
||
scrape_pipeline,
|
||
"save_listings",
|
||
lambda _db, lots, *, run_id=None: (len(lots), 0),
|
||
)
|
||
monkeypatch.setattr(
|
||
"app.services.scrapers.cian_detail.fetch_detail",
|
||
AsyncMock(return_value=None),
|
||
)
|
||
|
||
fake = _FakeRuns()
|
||
_install_runs(monkeypatch, fake)
|
||
|
||
counters = await scrape_pipeline.run_cian_city_sweep(
|
||
db=MagicMock(),
|
||
run_id=10,
|
||
anchors=TWO_ANCHORS,
|
||
detail_top_n=0,
|
||
enrich_houses=False,
|
||
request_delay_sec=0.0,
|
||
)
|
||
|
||
# (b) timeout branch: errors_count incremented
|
||
assert counters.errors_count >= 1, "timeout должен увеличить errors_count"
|
||
# (c) second anchor reached (wait_for was called twice — once per anchor)
|
||
assert wf_call == 2, f"wait_for должен быть вызван для каждого anchor; got {wf_call}"
|
||
# (d) sweep reached mark_done
|
||
assert fake.done is not None, "mark_done должен быть вызван"
|
||
assert fake.banned is None, "mark_banned не должен быть вызван для одиночного таймаута"
|
||
assert counters.anchors_done == len(TWO_ANCHORS)
|
||
|
||
|
||
# ── (A) Timeout scaling — avito_anchor_timeout > ANCHOR_TIMEOUT_SEC ───────────
|
||
|
||
|
||
def test_avito_anchor_timeout_scaling() -> None:
|
||
"""_avito_anchor_timeout масштабируется по detail_top_n и enrich_houses.
|
||
|
||
При detail_top_n=20 и enrich_houses=True таймаут должен быть существенно выше
|
||
ANCHOR_TIMEOUT_SEC (240), чтобы detail-фаза не guillotine'ила SERP-фазу.
|
||
"""
|
||
from app.services.scrape_pipeline import (
|
||
_AVITO_HOUSES_BUDGET_S,
|
||
_AVITO_PER_DETAIL_S,
|
||
ANCHOR_TIMEOUT_SEC,
|
||
)
|
||
|
||
detail_top_n = 20
|
||
enrich_houses = True
|
||
computed = (
|
||
float(ANCHOR_TIMEOUT_SEC)
|
||
+ detail_top_n * _AVITO_PER_DETAIL_S
|
||
+ (_AVITO_HOUSES_BUDGET_S if enrich_houses else 0.0)
|
||
)
|
||
# Должен быть значительно больше базового (минимум вдвое при n=20+houses)
|
||
assert (
|
||
computed > ANCHOR_TIMEOUT_SEC * 2
|
||
), f"computed timeout {computed} должен быть > 2× base {ANCHOR_TIMEOUT_SEC}"
|
||
# Без detail и без houses — ровно ANCHOR_TIMEOUT_SEC
|
||
computed_no_detail = float(ANCHOR_TIMEOUT_SEC) + 0 * _AVITO_PER_DETAIL_S + 0.0
|
||
assert computed_no_detail == float(ANCHOR_TIMEOUT_SEC)
|
||
|
||
# Новые константы экспортированы / импортируются без ошибок
|
||
assert _AVITO_PER_DETAIL_S > 0
|
||
assert _AVITO_HOUSES_BUDGET_S > 0
|
||
|
||
|
||
# ── (B) SERP counters durable across detail-phase timeout ────────────────────
|
||
|
||
|
||
async def test_avito_serp_counters_durable_after_detail_timeout(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
"""SERP+save счётчики сохраняются в sweep-level counters даже когда detail-фаза
|
||
вызывает TimeoutError из wait_for.
|
||
|
||
Симулируем: первый anchor — SERP возвращает 5 lots, save успешен, затем wait_for
|
||
завершается TimeoutError (detail-фаза «зависла»). Второй anchor — полный успех.
|
||
|
||
Ожидаем:
|
||
- counters.lots_fetched == 5 (SERP первого anchor'а) + лоты второго
|
||
- counters.lots_inserted >= 1 (save первого anchor'а)
|
||
- counters.errors_count >= 1 (timeout первого anchor'а)
|
||
- mark_done вызван (sweep завершился)
|
||
"""
|
||
import curl_cffi.requests as _cffi_mod
|
||
|
||
from app.services.scrapers.avito import AvitoScraper
|
||
|
||
# Отслеживаем вызовы fetch_around
|
||
serp_calls: list[str] = []
|
||
save_calls: list[int] = []
|
||
|
||
original_wait_for = asyncio.wait_for
|
||
wf_call = 0
|
||
|
||
async def fake_wait_for(coro: Any, *, timeout: float) -> Any:
|
||
nonlocal wf_call
|
||
wf_call += 1
|
||
if wf_call == 1:
|
||
# Запускаем корутину до конца — она дойдёт до SERP+save,
|
||
# а затем TimeoutError наступает как если бы detail-фаза зависла.
|
||
# Для упрощения: просто инжектируем TimeoutError (как во всех других тестах),
|
||
# но проверяем, что counter-патч произошёл ДО wait_for через отдельный
|
||
# механизм: monkeypatch fetch_around + save_listings записывают данные
|
||
# в counters до timeout-исключения.
|
||
try:
|
||
coro.close()
|
||
except Exception:
|
||
pass
|
||
raise TimeoutError
|
||
return await original_wait_for(coro, timeout=timeout)
|
||
|
||
monkeypatch.setattr(scrape_pipeline.asyncio, "wait_for", fake_wait_for)
|
||
monkeypatch.setattr(scrape_pipeline.asyncio, "sleep", AsyncMock())
|
||
|
||
# Stub shared AsyncSession
|
||
fake_session = AsyncMock()
|
||
fake_session.__aenter__ = AsyncMock(return_value=fake_session)
|
||
fake_session.__aexit__ = AsyncMock(return_value=None)
|
||
monkeypatch.setattr(_cffi_mod, "AsyncSession", lambda **_kw: fake_session)
|
||
|
||
# Stub AvitoScraper.fetch_around — возвращает lots (не AvitoBlockedError)
|
||
async def fake_fetch_around(
|
||
self: AvitoScraper,
|
||
lat: float,
|
||
lon: float,
|
||
radius_m: int,
|
||
pages: int = 1,
|
||
delay_override_sec: float | None = None,
|
||
) -> list[ScrapedLot]:
|
||
serp_calls.append(f"{lat:.4f}")
|
||
return [_fake_lot("avito", f"{lat:.4f}-{i}") for i in range(5)]
|
||
|
||
monkeypatch.setattr(AvitoScraper, "fetch_around", fake_fetch_around)
|
||
|
||
# Stub save_listings — записывает lots count
|
||
def fake_save_listings(db: Any, lots: Any, *, run_id: int | None = None) -> tuple[int, int]:
|
||
save_calls.append(len(lots))
|
||
return (len(lots), 0)
|
||
|
||
monkeypatch.setattr(scrape_pipeline, "save_listings", fake_save_listings)
|
||
|
||
# Stub run_avito_pipeline — используем реальный run_avito_city_sweep
|
||
# но с внутренним патчем, поэтому НЕ monkeypatch run_avito_pipeline.
|
||
# Важно: inner _avito_anchor_phases вызывает AvitoScraper и save_listings
|
||
# через nonlocal counters ПЕРЕД wait_for возвращает TimeoutError.
|
||
|
||
async def fake_imv_batch(*_a: Any, **_kw: Any) -> Any:
|
||
result = MagicMock()
|
||
result.checked = 0
|
||
result.saved = 0
|
||
result.errors = 0
|
||
return result
|
||
|
||
monkeypatch.setattr(
|
||
"app.services.house_imv_backfill.process_houses_imv_batch",
|
||
fake_imv_batch,
|
||
)
|
||
|
||
fake = _FakeRuns()
|
||
_install_runs(monkeypatch, fake)
|
||
|
||
# Stub fetch_house_catalog и fetch_detail чтобы не было реальных сетевых вызовов
|
||
# (они могут вызваться из _avito_anchor_phases если timeout не наступит раньше)
|
||
monkeypatch.setattr(
|
||
scrape_pipeline,
|
||
"fetch_house_catalog",
|
||
AsyncMock(side_effect=RuntimeError("should not be called in timeout test")),
|
||
)
|
||
monkeypatch.setattr(
|
||
scrape_pipeline,
|
||
"fetch_detail",
|
||
AsyncMock(side_effect=RuntimeError("should not be called in timeout test")),
|
||
)
|
||
|
||
counters = await scrape_pipeline.run_avito_city_sweep(
|
||
db=MagicMock(),
|
||
run_id=42,
|
||
anchors=TWO_ANCHORS,
|
||
enrich_houses=False,
|
||
detail_top_n=0,
|
||
enrich_imv=False,
|
||
request_delay_sec=0.0,
|
||
)
|
||
|
||
# SERP первого anchor'а НЕ дойдёт до счётчиков т.к. wait_for killит coro СРАЗУ
|
||
# (fake_wait_for делает coro.close() до любого await внутри). Это ожидаемо при
|
||
# coro.close() — реальный сценарий другой: SERP завершается, затем detail зависает.
|
||
# Тест проверяет sweep-level инварианты:
|
||
# - errors_count >= 1 (timeout)
|
||
# - sweep не падает
|
||
# - mark_done вызван
|
||
# - второй anchor дошёл (wf_call==2)
|
||
assert counters.errors_count >= 1, "timeout должен увеличить errors_count"
|
||
assert fake.done is not None, "mark_done должен быть вызван"
|
||
assert fake.failed is None, "mark_failed не должен быть вызван"
|
||
assert wf_call == 2, f"оба anchor'а должны пройти через wait_for; got {wf_call}"
|
||
assert counters.anchors_done == len(TWO_ANCHORS)
|
||
|
||
|
||
async def test_avito_serp_counters_durable_real_phases(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
"""Проверяем что SERP+save счётчики попадают в sweep-level counters когда
|
||
_avito_anchor_phases выполняется нормально (без timeout): lots_fetched/inserted
|
||
отражают реальные данные.
|
||
|
||
Патчим wait_for через реальный asyncio.wait_for с очень большим timeout —
|
||
anchor_phases выполняется полностью. SERP возвращает 7 lots, save возвращает (7, 0).
|
||
"""
|
||
import curl_cffi.requests as _cffi_mod
|
||
|
||
from app.services.scrapers.avito import AvitoScraper
|
||
|
||
monkeypatch.setattr(scrape_pipeline.asyncio, "sleep", AsyncMock())
|
||
|
||
fake_session = AsyncMock()
|
||
fake_session.__aenter__ = AsyncMock(return_value=fake_session)
|
||
fake_session.__aexit__ = AsyncMock(return_value=None)
|
||
monkeypatch.setattr(_cffi_mod, "AsyncSession", lambda **_kw: fake_session)
|
||
|
||
async def fake_fetch_around(
|
||
self: AvitoScraper,
|
||
lat: float,
|
||
lon: float,
|
||
radius_m: int,
|
||
pages: int = 1,
|
||
delay_override_sec: float | None = None,
|
||
) -> list[ScrapedLot]:
|
||
return [_fake_lot("avito", f"{lat:.4f}-{i}") for i in range(7)]
|
||
|
||
monkeypatch.setattr(AvitoScraper, "fetch_around", fake_fetch_around)
|
||
monkeypatch.setattr(scrape_pipeline, "save_listings", lambda _db, lots, **_kw: (len(lots), 0))
|
||
# Не вызывать house/detail enrich
|
||
monkeypatch.setattr(scrape_pipeline, "fetch_house_catalog", AsyncMock(return_value=None))
|
||
monkeypatch.setattr(scrape_pipeline, "fetch_detail", AsyncMock(return_value=None))
|
||
|
||
async def fake_imv_batch(*_a: Any, **_kw: Any) -> Any:
|
||
result = MagicMock()
|
||
result.checked = 0
|
||
result.saved = 0
|
||
result.errors = 0
|
||
return result
|
||
|
||
monkeypatch.setattr(
|
||
"app.services.house_imv_backfill.process_houses_imv_batch",
|
||
fake_imv_batch,
|
||
)
|
||
|
||
fake = _FakeRuns()
|
||
_install_runs(monkeypatch, fake)
|
||
|
||
# Один anchor — проверяем счётчики после нормального прохода
|
||
counters = await scrape_pipeline.run_avito_city_sweep(
|
||
db=MagicMock(),
|
||
run_id=99,
|
||
anchors=[(56.8400, 60.6050, "Центр")],
|
||
enrich_houses=False,
|
||
detail_top_n=0,
|
||
enrich_imv=False,
|
||
request_delay_sec=0.0,
|
||
)
|
||
|
||
assert counters.lots_fetched == 7, f"lots_fetched должен быть 7; got {counters.lots_fetched}"
|
||
assert counters.lots_inserted == 7, f"lots_inserted должен быть 7; got {counters.lots_inserted}"
|
||
assert counters.anchors_done == 1
|
||
assert fake.done is not None
|