bot-backend
40c044e2a6
All checks were successful
CI / changes (push) Successful in 7s
CI / backend-tests (push) Has been skipped
CI / frontend-tests (push) Has been skipped
CI / openapi-codegen-check (push) Has been skipped
CI / changes (pull_request) Successful in 6s
CI / backend-tests (pull_request) Has been skipped
CI / frontend-tests (pull_request) Has been skipped
CI / openapi-codegen-check (pull_request) Has been skipped
- Compute _avito_anchor_timeout = ANCHOR_TIMEOUT_SEC + detail_top_n * 50s + 180s (houses). Eliminates false TimeoutError on anchors with many detail requests. - Refactor run_avito_city_sweep to use inner _avito_anchor_phases coroutine (mirrors Yandex/Cian pattern): counters updated immediately after SERP+save (nonlocal), before detail phase. TimeoutError in detail no longer erases lots_fetched/inserted. - Add consecutive-detail-timeout abort after 5 consecutive TimeoutErrors. - Update test_anchor_watchdog: new tests for timeout scaling formula and counter durability. - Update test_avito_anti_bot + test_sweep_imv_phase: patch AvitoScraper.fetch_around directly (run_avito_city_sweep no longer calls run_avito_pipeline); use ExitStack instead of starred-expression unpacking (invalid syntax in Python 3.12/3.14).
489 lines
18 KiB
Python
489 lines
18 KiB
Python
"""Tests for IMV enrichment phase integrated into Avito city sweep.
|
||
|
||
Проверяет:
|
||
1. enrich_imv=True — sweep вызывает IMV-фазу для touched house_id.
|
||
2. enrich_imv=False — IMV не вызывается.
|
||
3. IMV counters (imv_attempted, imv_enriched, imv_failed) заполняются.
|
||
4. CitySweepCounters содержит новые imv_* поля.
|
||
5. process_houses_imv_batch пропускает пустой набор house_ids.
|
||
6. CitySweepStartRequest принимает enrich_imv (API-level param).
|
||
7. Cooperative cancel перед IMV-фазой — skip IMV, sweep marked done.
|
||
|
||
NB: run_avito_city_sweep использует внутренние _avito_anchor_phases (не run_avito_pipeline).
|
||
Тесты патчат AvitoScraper.fetch_around + save_listings + fetch_house_catalog +
|
||
save_house_catalog_enrichment для контроля touched_house_ids.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import os
|
||
from contextlib import ExitStack
|
||
from dataclasses import fields
|
||
from typing import Any
|
||
from unittest.mock import AsyncMock, MagicMock, patch
|
||
|
||
import pytest
|
||
|
||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
|
||
|
||
|
||
def _make_house_lot(offer_id: str, house_path: str) -> Any:
|
||
"""ScrapedLot с house_url для тестов house-enrich пути."""
|
||
from app.services.scrapers.base import ScrapedLot
|
||
|
||
return ScrapedLot(
|
||
source="avito",
|
||
source_url=f"https://www.avito.ru{house_path}/{offer_id}",
|
||
source_id=offer_id,
|
||
address="Тест",
|
||
price_rub=3_000_000,
|
||
area_m2=40.0,
|
||
rooms=1,
|
||
house_url=f"https://www.avito.ru{house_path}",
|
||
)
|
||
|
||
|
||
def _enter_common_patches(stack: ExitStack, house_ids: list[int]) -> None:
|
||
"""Входит в набор patch'ей для базового Avito sweep через ExitStack:
|
||
- AvitoScraper.fetch_around → lots с house_url (по одному на house_id)
|
||
- save_listings → (n, 0)
|
||
- fetch_house_catalog → MagicMock()
|
||
- save_house_catalog_enrichment → {"house_id": X} по очереди
|
||
- curl_cffi.requests.AsyncSession → fake
|
||
- asyncio.sleep → no-op
|
||
"""
|
||
from app.services.scrapers.avito import AvitoScraper
|
||
|
||
lots = [_make_house_lot(f"lot-{hid}", f"/ekaterinburg/houses/test/{hid}") for hid in house_ids]
|
||
|
||
async def fake_fetch_around(self: Any, lat: float, lon: float, *_a: Any, **_kw: Any) -> list:
|
||
return lots
|
||
|
||
# save_house_catalog_enrichment вызывается по одному разу на каждый дом
|
||
house_id_iter = iter(house_ids)
|
||
|
||
def fake_save_house(db: Any, enrichment: Any) -> dict:
|
||
try:
|
||
return {"house_id": next(house_id_iter)}
|
||
except StopIteration:
|
||
return {}
|
||
|
||
fake_session = AsyncMock()
|
||
fake_session.__aenter__ = AsyncMock(return_value=fake_session)
|
||
fake_session.__aexit__ = AsyncMock(return_value=None)
|
||
|
||
stack.enter_context(patch.object(AvitoScraper, "fetch_around", fake_fetch_around))
|
||
stack.enter_context(
|
||
patch(
|
||
"app.services.scrape_pipeline.save_listings",
|
||
lambda _db, _lots, **_kw: (len(_lots), 0),
|
||
)
|
||
)
|
||
stack.enter_context(
|
||
patch(
|
||
"app.services.scrape_pipeline.fetch_house_catalog",
|
||
AsyncMock(return_value=MagicMock()),
|
||
)
|
||
)
|
||
stack.enter_context(
|
||
patch(
|
||
"app.services.scrape_pipeline.save_house_catalog_enrichment",
|
||
fake_save_house,
|
||
)
|
||
)
|
||
stack.enter_context(patch("curl_cffi.requests.AsyncSession", return_value=fake_session))
|
||
stack.enter_context(patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()))
|
||
|
||
|
||
# ── CitySweepCounters has imv_* fields ────────────────────────────────────────
|
||
|
||
|
||
def test_city_sweep_counters_has_imv_fields() -> None:
|
||
from app.services.scrape_pipeline import CitySweepCounters
|
||
|
||
c = CitySweepCounters()
|
||
assert hasattr(c, "imv_attempted")
|
||
assert hasattr(c, "imv_enriched")
|
||
assert hasattr(c, "imv_failed")
|
||
assert c.imv_attempted == 0
|
||
assert c.imv_enriched == 0
|
||
assert c.imv_failed == 0
|
||
|
||
|
||
def test_city_sweep_counters_to_dict_has_imv_keys() -> None:
|
||
from app.services.scrape_pipeline import CitySweepCounters
|
||
|
||
c = CitySweepCounters(imv_attempted=5, imv_enriched=3, imv_failed=2)
|
||
d = c.to_dict()
|
||
assert d["imv_attempted"] == 5
|
||
assert d["imv_enriched"] == 3
|
||
assert d["imv_failed"] == 2
|
||
# Все dataclass-поля должны быть в to_dict
|
||
expected_keys = {f.name for f in fields(c)}
|
||
assert set(d.keys()) == expected_keys
|
||
|
||
|
||
# ── PipelineResult имеет touched_house_ids ────────────────────────────────────
|
||
|
||
|
||
def test_pipeline_result_has_touched_house_ids() -> None:
|
||
from app.services.scrape_pipeline import PipelineCounters, PipelineResult
|
||
|
||
r = PipelineResult(
|
||
anchor_lat=56.84,
|
||
anchor_lon=60.6,
|
||
radius_m=1500,
|
||
counters=PipelineCounters(),
|
||
enrich_houses=True,
|
||
enrich_detail_top_n=10,
|
||
)
|
||
assert hasattr(r, "touched_house_ids")
|
||
assert isinstance(r.touched_house_ids, set)
|
||
assert len(r.touched_house_ids) == 0
|
||
|
||
|
||
def test_pipeline_result_touched_house_ids_set() -> None:
|
||
from app.services.scrape_pipeline import PipelineCounters, PipelineResult
|
||
|
||
r = PipelineResult(
|
||
anchor_lat=56.84,
|
||
anchor_lon=60.6,
|
||
radius_m=1500,
|
||
counters=PipelineCounters(),
|
||
enrich_houses=True,
|
||
enrich_detail_top_n=10,
|
||
touched_house_ids={1, 2, 3},
|
||
)
|
||
assert r.touched_house_ids == {1, 2, 3}
|
||
|
||
|
||
# ── enrich_imv=True → IMV phase вызывается ───────────────────────────────────
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_sweep_enrich_imv_true_calls_imv_batch() -> None:
|
||
"""enrich_imv=True + есть touched house_ids → process_houses_imv_batch вызывается.
|
||
|
||
Патчим AvitoScraper.fetch_around → lots с house_url, save_house_catalog_enrichment
|
||
→ {house_id: X} чтобы all_touched_house_ids = {10, 20}.
|
||
"""
|
||
from app.services.house_imv_backfill import HouseIMVBackfillResult
|
||
from app.services.scrape_pipeline import run_avito_city_sweep
|
||
|
||
mock_db = MagicMock()
|
||
mock_imv_result = HouseIMVBackfillResult(checked=2, saved=2, skipped=0, errors=0)
|
||
|
||
with ExitStack() as stack:
|
||
_enter_common_patches(stack, [10, 20])
|
||
stack.enter_context(patch("app.services.scrape_runs.is_cancelled", return_value=False))
|
||
stack.enter_context(patch("app.services.scrape_runs.update_heartbeat"))
|
||
stack.enter_context(patch("app.services.scrape_runs.mark_done"))
|
||
mock_imv_batch = stack.enter_context(
|
||
patch(
|
||
"app.services.house_imv_backfill.process_houses_imv_batch",
|
||
new_callable=AsyncMock,
|
||
return_value=mock_imv_result,
|
||
)
|
||
)
|
||
counters = await run_avito_city_sweep(
|
||
mock_db,
|
||
run_id=1,
|
||
anchors=[(56.84, 60.6, "TestAnchor")],
|
||
enrich_imv=True,
|
||
enrich_houses=True,
|
||
detail_top_n=0,
|
||
)
|
||
|
||
mock_imv_batch.assert_awaited_once()
|
||
# Первый позиционный аргумент — db, второй — house_ids
|
||
call_args = mock_imv_batch.call_args
|
||
assert call_args.args[1] == {10, 20}
|
||
assert counters.imv_attempted == 2
|
||
assert counters.imv_enriched == 2
|
||
assert counters.imv_failed == 0
|
||
|
||
|
||
# ── enrich_imv=False → IMV не вызывается ────────────────────────────────────
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_sweep_enrich_imv_false_skips_imv() -> None:
|
||
"""enrich_imv=False → process_houses_imv_batch НЕ вызывается."""
|
||
from app.services.scrape_pipeline import run_avito_city_sweep
|
||
|
||
mock_db = MagicMock()
|
||
|
||
with ExitStack() as stack:
|
||
_enter_common_patches(stack, [99])
|
||
stack.enter_context(patch("app.services.scrape_runs.is_cancelled", return_value=False))
|
||
stack.enter_context(patch("app.services.scrape_runs.update_heartbeat"))
|
||
stack.enter_context(patch("app.services.scrape_runs.mark_done"))
|
||
mock_imv_batch = stack.enter_context(
|
||
patch(
|
||
"app.services.house_imv_backfill.process_houses_imv_batch",
|
||
new_callable=AsyncMock,
|
||
)
|
||
)
|
||
counters = await run_avito_city_sweep(
|
||
mock_db,
|
||
run_id=2,
|
||
anchors=[(56.84, 60.6, "TestAnchor")],
|
||
enrich_imv=False,
|
||
enrich_houses=True,
|
||
detail_top_n=0,
|
||
)
|
||
|
||
mock_imv_batch.assert_not_awaited()
|
||
assert counters.imv_attempted == 0
|
||
assert counters.imv_enriched == 0
|
||
assert counters.imv_failed == 0
|
||
|
||
|
||
# ── enrich_imv=True но нет touched house_ids → IMV не вызывается ────────────
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_sweep_enrich_imv_no_touched_ids_skips() -> None:
|
||
"""enrich_imv=True, но touched_house_ids пустой → IMV не вызывается.
|
||
|
||
Лоты без house_url → unique_house_paths пуст → touched_house_ids пуст.
|
||
"""
|
||
from app.services.scrape_pipeline import run_avito_city_sweep
|
||
from app.services.scrapers.avito import AvitoScraper
|
||
|
||
mock_db = MagicMock()
|
||
fake_session = AsyncMock()
|
||
fake_session.__aenter__ = AsyncMock(return_value=fake_session)
|
||
fake_session.__aexit__ = AsyncMock(return_value=None)
|
||
|
||
# Лоты БЕЗ house_url → нет домов → IMV не вызывается
|
||
async def fake_fetch_around(self: Any, lat: float, lon: float, *_a: Any, **_kw: Any) -> list:
|
||
return []
|
||
|
||
with (
|
||
patch.object(AvitoScraper, "fetch_around", fake_fetch_around),
|
||
patch("app.services.scrape_pipeline.save_listings", lambda _db, _lots, **_kw: (0, 0)),
|
||
patch("curl_cffi.requests.AsyncSession", return_value=fake_session),
|
||
patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
|
||
patch("app.services.scrape_runs.is_cancelled", return_value=False),
|
||
patch("app.services.scrape_runs.update_heartbeat"),
|
||
patch("app.services.scrape_runs.mark_done"),
|
||
patch(
|
||
"app.services.house_imv_backfill.process_houses_imv_batch",
|
||
new_callable=AsyncMock,
|
||
) as mock_imv_batch,
|
||
):
|
||
await run_avito_city_sweep(
|
||
mock_db,
|
||
run_id=3,
|
||
anchors=[(56.84, 60.6, "TestAnchor")],
|
||
enrich_imv=True,
|
||
enrich_houses=True,
|
||
detail_top_n=0,
|
||
)
|
||
|
||
mock_imv_batch.assert_not_awaited()
|
||
|
||
|
||
# ── IMV-ошибки не валят sweep ────────────────────────────────────────────────
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_sweep_imv_crash_does_not_abort_sweep() -> None:
|
||
"""Если process_houses_imv_batch raises — sweep всё равно завершается (mark_done)."""
|
||
from app.services.scrape_pipeline import run_avito_city_sweep
|
||
|
||
mock_db = MagicMock()
|
||
|
||
with ExitStack() as stack:
|
||
_enter_common_patches(stack, [55])
|
||
stack.enter_context(patch("app.services.scrape_runs.is_cancelled", return_value=False))
|
||
stack.enter_context(patch("app.services.scrape_runs.update_heartbeat"))
|
||
mock_mark_done = stack.enter_context(patch("app.services.scrape_runs.mark_done"))
|
||
stack.enter_context(
|
||
patch(
|
||
"app.services.house_imv_backfill.process_houses_imv_batch",
|
||
new_callable=AsyncMock,
|
||
side_effect=RuntimeError("IMV network error"),
|
||
)
|
||
)
|
||
counters = await run_avito_city_sweep(
|
||
mock_db,
|
||
run_id=4,
|
||
anchors=[(56.84, 60.6, "TestAnchor")],
|
||
enrich_imv=True,
|
||
enrich_houses=True,
|
||
detail_top_n=0,
|
||
)
|
||
|
||
mock_mark_done.assert_called_once()
|
||
# IMV crash → ошибка считается, но sweep завершается как done
|
||
assert counters.errors_count >= 1
|
||
|
||
|
||
# ── IMV counters накапливают failed ────────────────────────────────────────────
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_sweep_imv_failed_counter() -> None:
|
||
"""imv_failed отражает errors из HouseIMVBackfillResult."""
|
||
from app.services.house_imv_backfill import HouseIMVBackfillResult
|
||
from app.services.scrape_pipeline import run_avito_city_sweep
|
||
|
||
mock_db = MagicMock()
|
||
# 3 attempted, 1 enriched, 2 failed
|
||
mock_imv_result = HouseIMVBackfillResult(checked=3, saved=1, skipped=0, errors=2)
|
||
|
||
with ExitStack() as stack:
|
||
_enter_common_patches(stack, [1, 2, 3])
|
||
stack.enter_context(patch("app.services.scrape_runs.is_cancelled", return_value=False))
|
||
stack.enter_context(patch("app.services.scrape_runs.update_heartbeat"))
|
||
stack.enter_context(patch("app.services.scrape_runs.mark_done"))
|
||
stack.enter_context(
|
||
patch(
|
||
"app.services.house_imv_backfill.process_houses_imv_batch",
|
||
new_callable=AsyncMock,
|
||
return_value=mock_imv_result,
|
||
)
|
||
)
|
||
counters = await run_avito_city_sweep(
|
||
mock_db,
|
||
run_id=5,
|
||
anchors=[(56.84, 60.6, "TestAnchor")],
|
||
enrich_imv=True,
|
||
enrich_houses=True,
|
||
detail_top_n=0,
|
||
)
|
||
|
||
assert counters.imv_attempted == 3
|
||
assert counters.imv_enriched == 1
|
||
assert counters.imv_failed == 2
|
||
# errors_count должен включить imv_failed
|
||
assert counters.errors_count >= 2
|
||
|
||
|
||
# ── Cancel перед IMV-фазой ────────────────────────────────────────────────────
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_sweep_cancel_before_imv_skips_imv() -> None:
|
||
"""Если is_cancelled=True в IMV-фазе → IMV пропускается, sweep всё равно done."""
|
||
from app.services.scrape_pipeline import run_avito_city_sweep
|
||
|
||
mock_db = MagicMock()
|
||
|
||
# is_cancelled: первый вызов (anchor loop) = False, второй (IMV phase) = True
|
||
cancel_side_effects = [False, True]
|
||
call_idx = 0
|
||
|
||
def is_cancelled_side_effect(db: Any, run_id: int) -> bool:
|
||
nonlocal call_idx
|
||
val = cancel_side_effects[call_idx] if call_idx < len(cancel_side_effects) else True
|
||
call_idx += 1
|
||
return val
|
||
|
||
with ExitStack() as stack:
|
||
_enter_common_patches(stack, [77])
|
||
stack.enter_context(
|
||
patch(
|
||
"app.services.scrape_runs.is_cancelled",
|
||
side_effect=is_cancelled_side_effect,
|
||
)
|
||
)
|
||
stack.enter_context(patch("app.services.scrape_runs.update_heartbeat"))
|
||
mock_mark_done = stack.enter_context(patch("app.services.scrape_runs.mark_done"))
|
||
mock_imv_batch = stack.enter_context(
|
||
patch(
|
||
"app.services.house_imv_backfill.process_houses_imv_batch",
|
||
new_callable=AsyncMock,
|
||
)
|
||
)
|
||
await run_avito_city_sweep(
|
||
mock_db,
|
||
run_id=6,
|
||
anchors=[(56.84, 60.6, "TestAnchor")],
|
||
enrich_imv=True,
|
||
enrich_houses=True,
|
||
detail_top_n=0,
|
||
)
|
||
|
||
mock_imv_batch.assert_not_awaited()
|
||
mock_mark_done.assert_called_once()
|
||
|
||
|
||
# ── process_houses_imv_batch с пустым set ────────────────────────────────────
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_process_houses_imv_batch_empty_ids() -> None:
|
||
"""process_houses_imv_batch с пустым house_ids → result.checked=0, быстрый return."""
|
||
from app.services.house_imv_backfill import process_houses_imv_batch
|
||
|
||
mock_db = MagicMock()
|
||
|
||
result = await process_houses_imv_batch(mock_db, set(), request_delay_sec=1.0)
|
||
|
||
assert result.checked == 0
|
||
assert result.saved == 0
|
||
assert result.errors == 0
|
||
# DB не должна вызываться при пустом наборе
|
||
mock_db.execute.assert_not_called()
|
||
|
||
|
||
# ── CitySweepStartRequest принимает enrich_imv ───────────────────────────────
|
||
|
||
|
||
def test_city_sweep_start_request_enrich_imv_default_true() -> None:
|
||
"""enrich_imv по умолчанию True."""
|
||
from app.api.v1.admin import CitySweepStartRequest
|
||
|
||
req = CitySweepStartRequest()
|
||
assert req.enrich_imv is True
|
||
|
||
|
||
def test_city_sweep_start_request_enrich_imv_false() -> None:
|
||
"""enrich_imv=False принимается."""
|
||
from app.api.v1.admin import CitySweepStartRequest
|
||
|
||
req = CitySweepStartRequest(enrich_imv=False)
|
||
assert req.enrich_imv is False
|
||
|
||
|
||
# ── API endpoint передаёт enrich_imv в sweep ────────────────────────────────
|
||
|
||
|
||
def test_sweep_endpoint_passes_enrich_imv(app_with_admin) -> None:
|
||
"""POST /scrape/avito-city-sweep с enrich_imv=False → sweep вызывается без IMV."""
|
||
with (
|
||
patch("app.api.v1.admin.has_running_run", return_value=False),
|
||
patch("app.services.scrape_runs.create_run", return_value=42),
|
||
patch(
|
||
"app.services.scrape_pipeline.run_avito_city_sweep",
|
||
new_callable=AsyncMock,
|
||
),
|
||
):
|
||
r = app_with_admin.post(
|
||
"/api/v1/admin/scrape/avito-city-sweep",
|
||
json={"pages_per_anchor": 2, "enrich_imv": False},
|
||
)
|
||
assert r.status_code == 200
|
||
body = r.json()
|
||
assert body["run_id"] == 42
|
||
|
||
|
||
@pytest.fixture
|
||
def app_with_admin():
|
||
from fastapi import FastAPI
|
||
from fastapi.testclient import TestClient
|
||
|
||
from app.api.v1 import admin as admin_module
|
||
from app.core.db import get_db
|
||
|
||
app = FastAPI()
|
||
app.include_router(admin_module.router, prefix="/api/v1/admin")
|
||
|
||
def fake_db():
|
||
yield MagicMock()
|
||
|
||
app.dependency_overrides[get_db] = fake_db
|
||
return TestClient(app)
|