bot-backend
4b527751b9
All checks were successful
Deploy Trade-In / changes (push) Successful in 11s
Deploy Trade-In / build-frontend (push) Has been skipped
Deploy Trade-In / build-browser (push) Has been skipped
Deploy Trade-In / test (push) Successful in 3m27s
Deploy Trade-In / build-backend (push) Successful in 1m37s
Deploy Trade-In / deploy (push) Successful in 2m49s
86 lines
3.5 KiB
Python
86 lines
3.5 KiB
Python
"""#2687: распознанный QRATOR-блок домкликового свипа — это 'platform', не 'unknown'.
|
||
|
||
Ветка `if counters.blocked:` в `run_domclick_city_sweep` входится ТОЛЬКО когда
|
||
скрейпер поднял `DomClickBlockedError` (площадка отдала QRATOR block-страницу). Это
|
||
ровно определение `BAN_KIND_PLATFORM`. До правки свип звал `mark_banned` без
|
||
`ban_kind`, и все его баны падали в дефолт 'unknown' (#2764) — диагноз был
|
||
УСТАНОВЛЕН (блок распознан), но не передавался, и мониторинг не мог отличить
|
||
«площадка отбила» (platform) от «тракт/прокси» (infra).
|
||
|
||
Прод 2026-08-12: domclick_city_sweep 3751/3675/3594 все `banned` с `blocked=1`,
|
||
`ban_kind='unknown'`. На старом коде тест ниже падает (ban_kind='unknown').
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import os
|
||
from types import SimpleNamespace
|
||
from typing import Any
|
||
from unittest.mock import AsyncMock, MagicMock, patch
|
||
|
||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
|
||
|
||
from scraper_kit.orchestration.pipeline import run_domclick_city_sweep
|
||
|
||
PFX = "scraper_kit.orchestration.pipeline"
|
||
|
||
|
||
class _BanKindRecorder:
|
||
"""Ловит ban_kind, переданный в mark_banned (остальное — no-op)."""
|
||
|
||
def __init__(self) -> None:
|
||
self.ban_kind: str | None = None
|
||
self.marked: list[str] = []
|
||
|
||
def is_cancelled(self, db: Any, run_id: int) -> bool:
|
||
return False
|
||
|
||
def update_heartbeat(self, db: Any, run_id: int, counters: dict[str, Any]) -> None:
|
||
return None
|
||
|
||
def mark_done(self, db: Any, run_id: int, counters: dict[str, Any]) -> None:
|
||
self.marked.append("done")
|
||
|
||
def mark_failed(self, db: Any, run_id: int, error: str, counters: dict[str, Any]) -> None:
|
||
self.marked.append("failed")
|
||
|
||
def mark_banned(
|
||
self, db: Any, run_id: int, error: str, counters: dict[str, Any], **kw: Any
|
||
) -> None:
|
||
self.marked.append("banned")
|
||
self.ban_kind = kw.get("ban_kind")
|
||
|
||
|
||
async def _drive_blocked() -> _BanKindRecorder:
|
||
recorder = _BanKindRecorder()
|
||
scraper = MagicMock()
|
||
scraper.__aenter__ = AsyncMock(return_value=scraper)
|
||
scraper.__aexit__ = AsyncMock(return_value=None)
|
||
scraper.fetch_city = AsyncMock(return_value=[MagicMock() for _ in range(39)])
|
||
scraper.blocked = True # распознанный QRATOR-блок
|
||
scraper.geo_filtered = 0
|
||
scraper.fetch_errors = 1
|
||
scraper.buckets_completed = 2
|
||
scraper.buckets_total = 6
|
||
with (
|
||
patch(f"{PFX}.DomClickScraper", return_value=scraper),
|
||
patch(f"{PFX}.save_listings", MagicMock(return_value=(39, 0))),
|
||
patch(f"{PFX}.runs", recorder),
|
||
):
|
||
await run_domclick_city_sweep(
|
||
MagicMock(),
|
||
config=SimpleNamespace(browser_http_endpoint="http://x:9000"),
|
||
matcher=MagicMock(),
|
||
run_id=1,
|
||
city_id=4,
|
||
pages=1,
|
||
request_delay_sec=0.0,
|
||
)
|
||
return recorder
|
||
|
||
|
||
async def test_qrator_block_marks_platform_ban_kind() -> None:
|
||
"""Блок распознан → ban_kind='platform'. На старом коде было 'unknown' (дефолт)."""
|
||
recorder = await _drive_blocked()
|
||
assert recorder.marked[-1] == "banned"
|
||
assert recorder.ban_kind == "platform"
|