lekss361
f93ef5374d
All checks were successful
Deploy Trade-In / changes (push) Successful in 10s
Deploy Trade-In / build-frontend (push) Has been skipped
Deploy Trade-In / build-browser (push) Has been skipped
Deploy Trade-In / test (push) Successful in 47s
Deploy Trade-In / build-backend (push) Successful in 1m30s
Deploy Trade-In / deploy (push) Successful in 1m32s
390 lines
14 KiB
Python
390 lines
14 KiB
Python
"""Tests для DomClick detail-backfill orchestrator (issue #2000 Layer B).
|
||
|
||
Зеркалит конвенции test_avito_detail_backfill.py: module-level patch-target строки,
|
||
_mock_db(snapshot) helper, runs = MagicMock() assertions. DomClick-специфика:
|
||
единственный BrowserFetcher (нет curl-fallback), cookies из domclick_session_svc
|
||
(мокается целиком как модуль), эксепшн-триада DomClickBlockedError/DomClickParseError
|
||
вместо Avito's Blocked/RateLimited/ListingGone.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import os
|
||
import sys
|
||
from unittest.mock import AsyncMock, MagicMock, patch
|
||
|
||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
|
||
|
||
_wp_mock = MagicMock()
|
||
sys.modules.setdefault("weasyprint", _wp_mock)
|
||
|
||
import pytest # noqa: E402
|
||
from scraper_kit.domclick_exceptions import DomClickBlockedError, DomClickParseError # noqa: E402
|
||
|
||
from app.core import shutdown as _sd # noqa: E402
|
||
from app.tasks.domclick_detail_backfill import ( # noqa: E402
|
||
DomClickDetailBackfillResult,
|
||
run_domclick_detail_backfill,
|
||
)
|
||
|
||
|
||
@pytest.fixture(autouse=True)
|
||
def _reset_shutdown() -> None:
|
||
"""shutdown — module-global Event: чистим вокруг каждого теста (изоляция #1182)."""
|
||
_sd.reset_shutdown()
|
||
yield
|
||
_sd.reset_shutdown()
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Helpers
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def _make_snapshot(n: int) -> list[dict]:
|
||
return [
|
||
{
|
||
"id": i + 1,
|
||
"source_url": f"https://ekaterinburg.domclick.ru/card/sale__flat__{i + 1}",
|
||
}
|
||
for i in range(n)
|
||
]
|
||
|
||
|
||
def _mock_db(snapshot: list[dict]) -> MagicMock:
|
||
"""Fake Session: first execute() returns snapshot via .mappings().all()."""
|
||
db = MagicMock()
|
||
sel = MagicMock()
|
||
sel.mappings.return_value.all.return_value = snapshot
|
||
db.execute.return_value = sel
|
||
return db
|
||
|
||
|
||
def _mock_browser_fetcher_cls() -> MagicMock:
|
||
"""MagicMock class whose instance is a working async context manager."""
|
||
instance = AsyncMock()
|
||
instance.__aenter__ = AsyncMock(return_value=instance)
|
||
instance.__aexit__ = AsyncMock(return_value=False)
|
||
return MagicMock(return_value=instance)
|
||
|
||
|
||
_FETCH = "app.tasks.domclick_detail_backfill.fetch_detail"
|
||
_SAVE = "app.tasks.domclick_detail_backfill.save_detail_enrichment"
|
||
_RUNS = "app.tasks.domclick_detail_backfill.runs_mod"
|
||
_SLEEP = "app.tasks.domclick_detail_backfill.asyncio.sleep"
|
||
_SETTINGS = "app.tasks.domclick_detail_backfill.settings"
|
||
_SHUTDOWN = "app.tasks.domclick_detail_backfill.shutdown_requested"
|
||
_BROWSER_FETCHER = "app.tasks.domclick_detail_backfill.BrowserFetcher"
|
||
_SESSION_SVC = "app.tasks.domclick_detail_backfill.domclick_session_svc"
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Tests
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_backfill_empty_snapshot_marks_done() -> None:
|
||
"""Empty snapshot -> mark_done immediately, no fetch calls, no BrowserFetcher open."""
|
||
db = _mock_db([])
|
||
runs = MagicMock()
|
||
fake_settings = MagicMock(browser_http_endpoint="http://browser:9000")
|
||
mock_svc = MagicMock()
|
||
mock_svc.load_session.return_value = {"CAS_ID": "123"}
|
||
mock_bf_cls = _mock_browser_fetcher_cls()
|
||
with (
|
||
patch(_SETTINGS, fake_settings),
|
||
patch(_SESSION_SVC, mock_svc),
|
||
patch(_RUNS, runs),
|
||
patch(_BROWSER_FETCHER, mock_bf_cls),
|
||
patch(_FETCH) as mock_fetch,
|
||
):
|
||
result = await run_domclick_detail_backfill(
|
||
db, run_id=1, params={"batch_size": 10, "budget_sec": 60}
|
||
)
|
||
|
||
assert isinstance(result, DomClickDetailBackfillResult)
|
||
assert result.attempted == 0
|
||
assert result.enriched == 0
|
||
mock_fetch.assert_not_called()
|
||
mock_bf_cls.assert_not_called()
|
||
runs.mark_done.assert_called_once()
|
||
runs.mark_failed.assert_not_called()
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_backfill_processes_snapshot_with_cookies_threaded() -> None:
|
||
"""3 listings -> all fetched+enriched, cookies from load_session() threaded into
|
||
every fetch_detail() call kwargs, mark_done called."""
|
||
snapshot = _make_snapshot(3)
|
||
db = _mock_db(snapshot)
|
||
runs = MagicMock()
|
||
mock_enrichment = MagicMock()
|
||
mock_fetch = AsyncMock(return_value=mock_enrichment)
|
||
mock_save = MagicMock(return_value=True)
|
||
fake_settings = MagicMock(browser_http_endpoint="http://browser:9000")
|
||
fake_cookies = {"CAS_ID": "999", "qrator_jsid2": "abc"}
|
||
mock_svc = MagicMock()
|
||
mock_svc.load_session.return_value = fake_cookies
|
||
mock_bf_cls = _mock_browser_fetcher_cls()
|
||
with (
|
||
patch(_SETTINGS, fake_settings),
|
||
patch(_SESSION_SVC, mock_svc),
|
||
patch(_RUNS, runs),
|
||
patch(_BROWSER_FETCHER, mock_bf_cls),
|
||
patch(_FETCH, mock_fetch),
|
||
patch(_SAVE, mock_save),
|
||
patch(_SLEEP, new_callable=AsyncMock),
|
||
):
|
||
result = await run_domclick_detail_backfill(
|
||
db, run_id=2, params={"batch_size": 10, "budget_sec": 3600}
|
||
)
|
||
|
||
assert result.attempted == 3
|
||
assert result.enriched == 3
|
||
assert result.blocked == 0
|
||
assert result.failed == 0
|
||
assert mock_fetch.call_count == 3
|
||
mock_bf_cls.assert_called_once_with(
|
||
source="domclick", endpoint=fake_settings.browser_http_endpoint
|
||
)
|
||
for call in mock_fetch.call_args_list:
|
||
_, kwargs = call
|
||
assert kwargs.get("cookies") == fake_cookies
|
||
assert kwargs.get("browser_fetcher") is not None
|
||
runs.mark_done.assert_called_once()
|
||
runs.mark_failed.assert_not_called()
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_backfill_cookies_none_still_proceeds_with_warning(caplog) -> None:
|
||
"""No valid session (load_session()->None) -> run still proceeds (fetch_detail
|
||
called with cookies=None), but a warning is logged so operators refresh the session.
|
||
"""
|
||
snapshot = _make_snapshot(1)
|
||
db = _mock_db(snapshot)
|
||
runs = MagicMock()
|
||
mock_enrichment = MagicMock()
|
||
mock_fetch = AsyncMock(return_value=mock_enrichment)
|
||
mock_save = MagicMock(return_value=True)
|
||
fake_settings = MagicMock(browser_http_endpoint="http://browser:9000")
|
||
mock_svc = MagicMock()
|
||
mock_svc.load_session.return_value = None
|
||
mock_bf_cls = _mock_browser_fetcher_cls()
|
||
with (
|
||
caplog.at_level("WARNING"),
|
||
patch(_SETTINGS, fake_settings),
|
||
patch(_SESSION_SVC, mock_svc),
|
||
patch(_RUNS, runs),
|
||
patch(_BROWSER_FETCHER, mock_bf_cls),
|
||
patch(_FETCH, mock_fetch),
|
||
patch(_SAVE, mock_save),
|
||
patch(_SLEEP, new_callable=AsyncMock),
|
||
):
|
||
result = await run_domclick_detail_backfill(
|
||
db, run_id=3, params={"batch_size": 10, "budget_sec": 3600}
|
||
)
|
||
|
||
assert result.enriched == 1
|
||
_, kwargs = mock_fetch.call_args
|
||
assert kwargs.get("cookies") is None
|
||
assert "no valid DomClick session cookies" in caplog.text
|
||
runs.mark_done.assert_called_once()
|
||
runs.mark_failed.assert_not_called()
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_backfill_blocked_abort_after_max_consecutive() -> None:
|
||
"""3 consecutive DomClickBlockedError -> abort, mark_done (NOT mark_failed).
|
||
|
||
No IP-rotation recovery step exists for DomClick (single dedicated proxy) --
|
||
abort happens on the SAME iteration the threshold is hit, no extra recovery calls.
|
||
"""
|
||
snapshot = _make_snapshot(10)
|
||
db = _mock_db(snapshot)
|
||
runs = MagicMock()
|
||
blocked_exc = DomClickBlockedError("QRATOR challenge page detected")
|
||
mock_fetch = AsyncMock(side_effect=blocked_exc)
|
||
fake_settings = MagicMock(browser_http_endpoint="http://browser:9000")
|
||
mock_svc = MagicMock()
|
||
mock_svc.load_session.return_value = {"CAS_ID": "123"}
|
||
mock_bf_cls = _mock_browser_fetcher_cls()
|
||
with (
|
||
patch(_SETTINGS, fake_settings),
|
||
patch(_SESSION_SVC, mock_svc),
|
||
patch(_RUNS, runs),
|
||
patch(_BROWSER_FETCHER, mock_bf_cls),
|
||
patch(_FETCH, mock_fetch),
|
||
patch(_SLEEP, new_callable=AsyncMock),
|
||
):
|
||
result = await run_domclick_detail_backfill(
|
||
db,
|
||
run_id=4,
|
||
params={"batch_size": 10, "budget_sec": 3600, "max_consecutive_blocks": 3},
|
||
)
|
||
|
||
assert result.blocked == 3
|
||
assert result.attempted == 3
|
||
assert result.enriched == 0
|
||
runs.mark_done.assert_called_once()
|
||
runs.mark_failed.assert_not_called()
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_backfill_parse_error_counts_failed_no_abort() -> None:
|
||
"""DomClickParseError (schema drift, not a block) -> failed++, consecutive-block
|
||
breaker NOT touched, run does NOT abort, continues to next listing.
|
||
"""
|
||
snapshot = _make_snapshot(2)
|
||
db = _mock_db(snapshot)
|
||
runs = MagicMock()
|
||
mock_enrichment = MagicMock()
|
||
mock_fetch = AsyncMock(
|
||
side_effect=[DomClickParseError("__SSR_STATE__ not found"), mock_enrichment]
|
||
)
|
||
mock_save = MagicMock(return_value=True)
|
||
fake_settings = MagicMock(browser_http_endpoint="http://browser:9000")
|
||
mock_svc = MagicMock()
|
||
mock_svc.load_session.return_value = {"CAS_ID": "123"}
|
||
mock_bf_cls = _mock_browser_fetcher_cls()
|
||
with (
|
||
patch(_SETTINGS, fake_settings),
|
||
patch(_SESSION_SVC, mock_svc),
|
||
patch(_RUNS, runs),
|
||
patch(_BROWSER_FETCHER, mock_bf_cls),
|
||
patch(_FETCH, mock_fetch),
|
||
patch(_SAVE, mock_save),
|
||
patch(_SLEEP, new_callable=AsyncMock),
|
||
):
|
||
result = await run_domclick_detail_backfill(
|
||
db,
|
||
run_id=5,
|
||
params={"batch_size": 10, "budget_sec": 3600, "max_consecutive_blocks": 3},
|
||
)
|
||
|
||
assert result.failed == 1
|
||
assert result.enriched == 1
|
||
assert result.attempted == 2
|
||
assert result.blocked == 0
|
||
runs.mark_done.assert_called_once()
|
||
runs.mark_failed.assert_not_called()
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_backfill_sigterm_drain_breaks_and_marks_done_partial() -> None:
|
||
"""#1182 Phase 2: shutdown_requested() True -> loop выходит на границе карточки,
|
||
mark_done вызывается с ЧАСТИЧНЫМИ счётчиками (не mark_failed).
|
||
"""
|
||
snapshot = _make_snapshot(3)
|
||
db = _mock_db(snapshot)
|
||
runs = MagicMock()
|
||
mock_enrichment = MagicMock()
|
||
mock_fetch = AsyncMock(return_value=mock_enrichment)
|
||
mock_save = MagicMock(return_value=True)
|
||
fake_settings = MagicMock(browser_http_endpoint="http://browser:9000")
|
||
mock_svc = MagicMock()
|
||
mock_svc.load_session.return_value = {"CAS_ID": "123"}
|
||
mock_bf_cls = _mock_browser_fetcher_cls()
|
||
with (
|
||
patch(_SETTINGS, fake_settings),
|
||
patch(_SESSION_SVC, mock_svc),
|
||
patch(_RUNS, runs),
|
||
patch(_BROWSER_FETCHER, mock_bf_cls),
|
||
patch(_FETCH, mock_fetch),
|
||
patch(_SAVE, mock_save),
|
||
patch(_SLEEP, new_callable=AsyncMock),
|
||
patch(_SHUTDOWN, side_effect=[False, True]),
|
||
):
|
||
result = await run_domclick_detail_backfill(
|
||
db, run_id=6, params={"batch_size": 10, "budget_sec": 3600}
|
||
)
|
||
|
||
assert result.attempted == 1
|
||
assert result.enriched == 1
|
||
assert mock_fetch.call_count == 1
|
||
runs.mark_done.assert_called_once()
|
||
runs.mark_failed.assert_not_called()
|
||
runs.mark_cancelled.assert_not_called()
|
||
done_counters = runs.mark_done.call_args.args[2]
|
||
assert done_counters["attempted"] == 1
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_backfill_budget_guard_stops_loop() -> None:
|
||
"""Budget expired before first listing -> fetch_detail not called."""
|
||
snapshot = _make_snapshot(5)
|
||
db = _mock_db(snapshot)
|
||
runs = MagicMock()
|
||
mono_values = iter([0.0, 999.0, 999.0])
|
||
fake_settings = MagicMock(browser_http_endpoint="http://browser:9000")
|
||
mock_svc = MagicMock()
|
||
mock_svc.load_session.return_value = {"CAS_ID": "123"}
|
||
mock_bf_cls = _mock_browser_fetcher_cls()
|
||
with (
|
||
patch(_SETTINGS, fake_settings),
|
||
patch(_SESSION_SVC, mock_svc),
|
||
patch(_RUNS, runs),
|
||
patch(_BROWSER_FETCHER, mock_bf_cls),
|
||
patch("app.tasks.domclick_detail_backfill.time.monotonic", side_effect=mono_values),
|
||
patch(_FETCH) as mock_fetch,
|
||
):
|
||
await run_domclick_detail_backfill(db, run_id=7, params={"batch_size": 5, "budget_sec": 1})
|
||
|
||
mock_fetch.assert_not_called()
|
||
runs.mark_done.assert_called_once()
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_backfill_top_level_exception_marks_failed() -> None:
|
||
"""db.execute raises -> mark_failed called, exception re-raised."""
|
||
db = MagicMock()
|
||
db.execute.side_effect = RuntimeError("DB connection lost")
|
||
runs = MagicMock()
|
||
fake_settings = MagicMock(browser_http_endpoint="http://browser:9000")
|
||
mock_svc = MagicMock()
|
||
mock_svc.load_session.return_value = {"CAS_ID": "123"}
|
||
mock_bf_cls = _mock_browser_fetcher_cls()
|
||
with (
|
||
patch(_SETTINGS, fake_settings),
|
||
patch(_SESSION_SVC, mock_svc),
|
||
patch(_RUNS, runs),
|
||
patch(_BROWSER_FETCHER, mock_bf_cls),
|
||
):
|
||
with pytest.raises(RuntimeError, match="DB connection lost"):
|
||
await run_domclick_detail_backfill(
|
||
db, run_id=8, params={"batch_size": 5, "budget_sec": 60}
|
||
)
|
||
|
||
runs.mark_failed.assert_called_once()
|
||
runs.mark_done.assert_not_called()
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_backfill_generic_exception_continues_and_rolls_back() -> None:
|
||
"""Unexpected exception on one listing -> failed++, db.rollback(), loop continues."""
|
||
snapshot = _make_snapshot(2)
|
||
db = _mock_db(snapshot)
|
||
runs = MagicMock()
|
||
mock_enrichment = MagicMock()
|
||
mock_fetch = AsyncMock(side_effect=[RuntimeError("unexpected"), mock_enrichment])
|
||
fake_settings = MagicMock(browser_http_endpoint="http://browser:9000")
|
||
mock_svc = MagicMock()
|
||
mock_svc.load_session.return_value = {"CAS_ID": "123"}
|
||
mock_bf_cls = _mock_browser_fetcher_cls()
|
||
with (
|
||
patch(_SETTINGS, fake_settings),
|
||
patch(_SESSION_SVC, mock_svc),
|
||
patch(_RUNS, runs),
|
||
patch(_BROWSER_FETCHER, mock_bf_cls),
|
||
patch(_FETCH, mock_fetch),
|
||
patch(_SAVE, return_value=True),
|
||
patch(_SLEEP, new_callable=AsyncMock),
|
||
):
|
||
result = await run_domclick_detail_backfill(
|
||
db, run_id=9, params={"batch_size": 10, "budget_sec": 3600}
|
||
)
|
||
|
||
assert result.failed == 1
|
||
assert result.enriched == 1
|
||
assert result.attempted == 2
|
||
db.rollback.assert_called()
|
||
runs.mark_done.assert_called_once()
|