200 lines
7.5 KiB
Python
200 lines
7.5 KiB
Python
"""#1945 flats/extras isolation tests (scrapers/domrf_kn).
|
|
|
|
PROVEN mechanism: extras /сервисы/api/object/{id}/* return 403 (volume-independent,
|
|
dead since 2026-06-03) AND poison the session cookies → subsequent FLATS on the same
|
|
session flip to 403. flats_count collapsed 3670→9. Fix: flats never share a session
|
|
with extras; extras run as a separate best-effort pass that RECYCLES the session on
|
|
any WAF-403 so poison never accumulates (and never touches flats).
|
|
|
|
These tests assert:
|
|
• _extras_coros builds ONLY extras endpoints (never flats);
|
|
• _is_waf_poisoned flips on any WafBlockedError;
|
|
• _run_extras_pass recycles the BrowserSession on a poisoned object;
|
|
• _run_extras_pass aborts early when extras are wholesale-403 (no ~1500 relaunches).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import datetime
|
|
from typing import Any, ClassVar
|
|
|
|
import pytest
|
|
|
|
import app.services.scrapers.domrf_kn as kn
|
|
from app.services.scrapers.stealth import WafBlockedError
|
|
|
|
|
|
class TestExtrasCorosPurity:
|
|
"""_extras_coros НИКОГДА не включает flats — иначе изоляция бессмысленна."""
|
|
|
|
def test_extras_coros_excludes_flats(self) -> None:
|
|
coros = kn._extras_coros(sess=object(), obj_id=123) # type: ignore[arg-type]
|
|
try:
|
|
names = {c.__qualname__ for c in coros}
|
|
assert not any("flats" in n for n in names)
|
|
# 10 extras: 2 sale_graph + sales_agg + infra + photos + 5 docs.
|
|
assert len(coros) == 10
|
|
finally:
|
|
for c in coros:
|
|
c.close() # avoid un-awaited coroutine warnings
|
|
|
|
|
|
class TestWafPoisonDetection:
|
|
def test_poisoned_on_any_waf(self) -> None:
|
|
results = [
|
|
("sales_agg", "u", {"ok": 1}),
|
|
("infrastructure", "u", WafBlockedError("403 waf")),
|
|
]
|
|
assert kn._is_waf_poisoned(results) is True
|
|
|
|
def test_not_poisoned_without_waf(self) -> None:
|
|
results = [
|
|
("sales_agg", "u", {"ok": 1}),
|
|
("infrastructure", "u", RuntimeError("http 500")), # transient, not WAF
|
|
]
|
|
assert kn._is_waf_poisoned(results) is False
|
|
|
|
def test_empty_not_poisoned(self) -> None:
|
|
assert kn._is_waf_poisoned([]) is False
|
|
|
|
|
|
# ── _run_extras_pass session-lifecycle tests ─────────────────────────────────
|
|
|
|
|
|
class _FakeSession:
|
|
"""BrowserSession stand-in: tracks bootstrap/warm/exit, no real browser."""
|
|
|
|
instances: ClassVar[list[_FakeSession]] = []
|
|
|
|
def __init__(self, **_kw: Any) -> None:
|
|
self.request_count = 7
|
|
self.warmed = 0
|
|
self.exited = False
|
|
_FakeSession.instances.append(self)
|
|
|
|
async def __aenter__(self) -> _FakeSession:
|
|
return self
|
|
|
|
async def __aexit__(self, *_exc: Any) -> None:
|
|
self.exited = True
|
|
|
|
async def warm_up(self, force: bool = False) -> None:
|
|
self.warmed += 1
|
|
|
|
|
|
def _objs(n: int) -> list[dict[str, Any]]:
|
|
return [{"objId": 1000 + i} for i in range(n)]
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _patch(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
_FakeSession.instances = []
|
|
monkeypatch.setattr(kn, "BrowserSession", _FakeSession)
|
|
monkeypatch.setattr(kn, "log_progress", lambda *a, **k: None)
|
|
monkeypatch.setattr(kn, "_checkpoint", lambda *a, **k: None)
|
|
monkeypatch.setattr(kn, "upsert_documents", lambda *a, **k: (0, 0))
|
|
|
|
# _process_extras_result is real upsert plumbing — no-op it (we test lifecycle).
|
|
async def _noop_process(*_a: Any, **_k: Any) -> None:
|
|
return None
|
|
|
|
monkeypatch.setattr(kn, "_process_extras_result", _noop_process)
|
|
|
|
|
|
def _make_coros_factory(per_obj_results: Any) -> Any:
|
|
"""Return a _extras_coros replacement: per-object it yields coroutines that each
|
|
resolve to one staged (kind, url, result) tuple, so real asyncio.gather is used."""
|
|
state = {"i": 0}
|
|
|
|
def _coros(_sess: Any, _oid: int) -> list[Any]:
|
|
results = per_obj_results(state["i"])
|
|
state["i"] += 1
|
|
|
|
async def _wrap(res: tuple[str, str, Any]) -> tuple[str, str, Any]:
|
|
return res
|
|
|
|
return [_wrap(r) for r in results]
|
|
|
|
return _coros
|
|
|
|
|
|
async def _drive(objs: list[dict[str, Any]]) -> int:
|
|
extras_counts = dict.fromkeys(
|
|
(
|
|
"sale_graph_rows",
|
|
"sales_agg_rows",
|
|
"infra_rows",
|
|
"photos_rows",
|
|
"photos_downloaded",
|
|
"documents_rows",
|
|
"checks_rows",
|
|
),
|
|
0,
|
|
)
|
|
return await kn._run_extras_pass(
|
|
db=object(), # type: ignore[arg-type]
|
|
run_id=1,
|
|
region_code=66,
|
|
all_objects=objs,
|
|
start_index=0,
|
|
snapshot_date=datetime.date(2026, 6, 28),
|
|
extras_counts=extras_counts,
|
|
pdir=kn.PHOTOS_DIR_DEFAULT,
|
|
download_photos_binary=False,
|
|
load_state=None,
|
|
headed=False,
|
|
browser_concurrency=2,
|
|
request_jitter_min_ms=1200,
|
|
request_jitter_max_ms=3000,
|
|
proxy_url=None,
|
|
)
|
|
|
|
|
|
class TestRunExtrasPassLifecycle:
|
|
@pytest.mark.asyncio
|
|
async def test_clean_results_use_single_session(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
# Every object returns clean JSON → no poison → one session for the whole pass.
|
|
def clean(_i: int) -> list[tuple[str, str, Any]]:
|
|
return [("sales_agg", "u", ({"x": 1}, "u")), ("infrastructure", "u", ([], "u"))]
|
|
|
|
monkeypatch.setattr(kn, "_extras_coros", _make_coros_factory(clean))
|
|
req = await _drive(_objs(4))
|
|
# Exactly 1 BrowserSession constructed for a fully-clean pass.
|
|
assert len(_FakeSession.instances) == 1
|
|
assert _FakeSession.instances[0].exited is True
|
|
assert req == 7 # single session.request_count
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_recycle_on_poison(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
# Object 0 poisoned (1 WAF among results) but NOT all-WAF (so no early abort);
|
|
# objects 1+ clean → pass recycles once after object 0, then continues clean.
|
|
def mixed(i: int) -> list[tuple[str, str, Any]]:
|
|
if i == 0:
|
|
return [
|
|
("sales_agg", "u", ({"x": 1}, "u")), # one ok → not all-WAF
|
|
("infrastructure", "u", WafBlockedError("403")), # poison
|
|
]
|
|
return [("sales_agg", "u", ({"x": 1}, "u")), ("infrastructure", "u", ([], "u"))]
|
|
|
|
monkeypatch.setattr(kn, "_extras_coros", _make_coros_factory(mixed))
|
|
await _drive(_objs(3))
|
|
# 1 initial + 1 recycle after the poisoned object 0 = 2 sessions.
|
|
assert len(_FakeSession.instances) == 2
|
|
assert all(s.exited for s in _FakeSession.instances)
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_early_abort_on_wholesale_waf(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
# Every object fully WAF-403 (extras dead) → abort after the threshold,
|
|
# NOT grinding through all 1000 objects with a relaunch each time.
|
|
def all_waf(_i: int) -> list[tuple[str, str, Any]]:
|
|
return [
|
|
("sales_agg", "u", WafBlockedError("403")),
|
|
("infrastructure", "u", WafBlockedError("403")),
|
|
]
|
|
|
|
monkeypatch.setattr(kn, "_extras_coros", _make_coros_factory(all_waf))
|
|
await _drive(_objs(1000))
|
|
# Abort kicks in at _EXTRAS_ABORT_AFTER_CONSEC_WAF consecutive all-WAF objects.
|
|
# Sessions: each poisoned object recycles UNLESS it's the abort object.
|
|
# Bounded well under 1000 — proves we don't relaunch per object forever.
|
|
assert len(_FakeSession.instances) <= kn._EXTRAS_ABORT_AFTER_CONSEC_WAF + 1
|