"""#1945 flats/extras isolation tests (scrapers/domrf_kn). PROVEN mechanism: extras /сервисы/api/object/{id}/* return 403 (volume-independent, dead since 2026-06-03) AND poison the session cookies → subsequent FLATS on the same session flip to 403. flats_count collapsed 3670→9. Fix: flats never share a session with extras; extras run as a separate best-effort pass that RECYCLES the session on any WAF-403 so poison never accumulates (and never touches flats). These tests assert: • _extras_coros builds ONLY extras endpoints (never flats); • _is_waf_poisoned flips on any WafBlockedError; • _run_extras_pass recycles the BrowserSession on a poisoned object; • _run_extras_pass aborts early when extras are wholesale-403 (no ~1500 relaunches). """ from __future__ import annotations import datetime from typing import Any, ClassVar import pytest import app.services.scrapers.domrf_kn as kn from app.services.scrapers.stealth import WafBlockedError class TestExtrasCorosPurity: """_extras_coros НИКОГДА не включает flats — иначе изоляция бессмысленна.""" def test_extras_coros_excludes_flats(self) -> None: coros = kn._extras_coros(sess=object(), obj_id=123) # type: ignore[arg-type] try: names = {c.__qualname__ for c in coros} assert not any("flats" in n for n in names) # 10 extras: 2 sale_graph + sales_agg + infra + photos + 5 docs. assert len(coros) == 10 finally: for c in coros: c.close() # avoid un-awaited coroutine warnings class TestWafPoisonDetection: def test_poisoned_on_any_waf(self) -> None: results = [ ("sales_agg", "u", {"ok": 1}), ("infrastructure", "u", WafBlockedError("403 waf")), ] assert kn._is_waf_poisoned(results) is True def test_not_poisoned_without_waf(self) -> None: results = [ ("sales_agg", "u", {"ok": 1}), ("infrastructure", "u", RuntimeError("http 500")), # transient, not WAF ] assert kn._is_waf_poisoned(results) is False def test_empty_not_poisoned(self) -> None: assert kn._is_waf_poisoned([]) is False # ── _run_extras_pass session-lifecycle tests ───────────────────────────────── class _FakeSession: """BrowserSession stand-in: tracks bootstrap/warm/exit, no real browser.""" instances: ClassVar[list[_FakeSession]] = [] def __init__(self, **_kw: Any) -> None: self.request_count = 7 self.warmed = 0 self.exited = False _FakeSession.instances.append(self) async def __aenter__(self) -> _FakeSession: return self async def __aexit__(self, *_exc: Any) -> None: self.exited = True async def warm_up(self, force: bool = False) -> None: self.warmed += 1 def _objs(n: int) -> list[dict[str, Any]]: return [{"objId": 1000 + i} for i in range(n)] @pytest.fixture(autouse=True) def _patch(monkeypatch: pytest.MonkeyPatch) -> None: _FakeSession.instances = [] monkeypatch.setattr(kn, "BrowserSession", _FakeSession) monkeypatch.setattr(kn, "log_progress", lambda *a, **k: None) monkeypatch.setattr(kn, "_checkpoint", lambda *a, **k: None) monkeypatch.setattr(kn, "upsert_documents", lambda *a, **k: (0, 0)) # _process_extras_result is real upsert plumbing — no-op it (we test lifecycle). async def _noop_process(*_a: Any, **_k: Any) -> None: return None monkeypatch.setattr(kn, "_process_extras_result", _noop_process) def _make_coros_factory(per_obj_results: Any) -> Any: """Return a _extras_coros replacement: per-object it yields coroutines that each resolve to one staged (kind, url, result) tuple, so real asyncio.gather is used.""" state = {"i": 0} def _coros(_sess: Any, _oid: int) -> list[Any]: results = per_obj_results(state["i"]) state["i"] += 1 async def _wrap(res: tuple[str, str, Any]) -> tuple[str, str, Any]: return res return [_wrap(r) for r in results] return _coros async def _drive(objs: list[dict[str, Any]]) -> int: extras_counts = dict.fromkeys( ( "sale_graph_rows", "sales_agg_rows", "infra_rows", "photos_rows", "photos_downloaded", "documents_rows", "checks_rows", ), 0, ) return await kn._run_extras_pass( db=object(), # type: ignore[arg-type] run_id=1, region_code=66, all_objects=objs, start_index=0, snapshot_date=datetime.date(2026, 6, 28), extras_counts=extras_counts, pdir=kn.PHOTOS_DIR_DEFAULT, download_photos_binary=False, load_state=None, headed=False, browser_concurrency=2, request_jitter_min_ms=1200, request_jitter_max_ms=3000, proxy_url=None, ) class TestRunExtrasPassLifecycle: @pytest.mark.asyncio async def test_clean_results_use_single_session(self, monkeypatch: pytest.MonkeyPatch) -> None: # Every object returns clean JSON → no poison → one session for the whole pass. def clean(_i: int) -> list[tuple[str, str, Any]]: return [("sales_agg", "u", ({"x": 1}, "u")), ("infrastructure", "u", ([], "u"))] monkeypatch.setattr(kn, "_extras_coros", _make_coros_factory(clean)) req = await _drive(_objs(4)) # Exactly 1 BrowserSession constructed for a fully-clean pass. assert len(_FakeSession.instances) == 1 assert _FakeSession.instances[0].exited is True assert req == 7 # single session.request_count @pytest.mark.asyncio async def test_recycle_on_poison(self, monkeypatch: pytest.MonkeyPatch) -> None: # Object 0 poisoned (1 WAF among results) but NOT all-WAF (so no early abort); # objects 1+ clean → pass recycles once after object 0, then continues clean. def mixed(i: int) -> list[tuple[str, str, Any]]: if i == 0: return [ ("sales_agg", "u", ({"x": 1}, "u")), # one ok → not all-WAF ("infrastructure", "u", WafBlockedError("403")), # poison ] return [("sales_agg", "u", ({"x": 1}, "u")), ("infrastructure", "u", ([], "u"))] monkeypatch.setattr(kn, "_extras_coros", _make_coros_factory(mixed)) await _drive(_objs(3)) # 1 initial + 1 recycle after the poisoned object 0 = 2 sessions. assert len(_FakeSession.instances) == 2 assert all(s.exited for s in _FakeSession.instances) @pytest.mark.asyncio async def test_early_abort_on_wholesale_waf(self, monkeypatch: pytest.MonkeyPatch) -> None: # Every object fully WAF-403 (extras dead) → abort after the threshold, # NOT grinding through all 1000 objects with a relaunch each time. def all_waf(_i: int) -> list[tuple[str, str, Any]]: return [ ("sales_agg", "u", WafBlockedError("403")), ("infrastructure", "u", WafBlockedError("403")), ] monkeypatch.setattr(kn, "_extras_coros", _make_coros_factory(all_waf)) await _drive(_objs(1000)) # Abort kicks in at _EXTRAS_ABORT_AFTER_CONSEC_WAF consecutive all-WAF objects. # Sessions: each poisoned object recycles UNLESS it's the abort object. # Bounded well under 1000 — proves we don't relaunch per object forever. assert len(_FakeSession.instances) <= kn._EXTRAS_ABORT_AFTER_CONSEC_WAF + 1