bot-backend
a50af40ad6
avito_detail_backfill (mode=browser) aborted prod run 458 with
attempted=5 enriched=0 blocked=5: the lat-null pending queue is dominated
by DEAD listings (404 — that's why they are stale coord-holes). In browser
mode a dead listing's 404 page ("Ошибка 404. Страница не найдена") has no
item-view, so _is_detail_soft_block returned True → AvitoBlockedError →
backfill counted it as `blocked` and the consecutive-block breaker aborted
the run before reaching live listings.
- New AvitoListingGoneError (subclass of common AvitoError, NOT of
AvitoBlockedError/AvitoRateLimitedError) so the backfill block-trap
does not catch it.
- New _is_detail_not_found() detector + browser-mode branch in fetch_detail,
checked BEFORE firewall/soft-block (keyed on the 404 title so it does not
swallow the generic soft-block deflect). Curl path unchanged (404 → 302/
non-200 → ValueError → failed, as before).
- Backfill: new `gone` counter; dedicated except AvitoListingGoneError branch
that is neutral to the consecutive-block breaker and marks the listing
is_active=FALSE (SAVEPOINT) so it leaves the snapshot scope.
Tests: unit for _is_detail_not_found (404 True; soft-block/item-view/empty
False), browser-mode fetch_detail raising AvitoListingGoneError (not Blocked),
and backfill gone-row marks inactive without aborting the breaker.
82 lines
4.1 KiB
Python
82 lines
4.1 KiB
Python
"""404 / removed-listing detection на detail-странице (#2034).
|
||
|
||
Lat-null pending-очередь backfill в основном содержит МЁРТВЫЕ листинги (404 — потому
|
||
они и стали координатными дырами). В browser-mode 404-страница Avito («Ошибка 404.
|
||
Страница не найдена», тело тоже содержит generic «Объявления на сайте Авито») не имеет
|
||
item-view → раньше ловилась как soft-block → AvitoBlockedError → backfill крутил
|
||
consecutive-block breaker и абортил run ДО live-листингов (run 458: attempted=5
|
||
enriched=0 blocked=5 → abort).
|
||
|
||
_is_detail_not_found детектит её ОТДЕЛЬНО → fetch_detail кидает AvitoListingGoneError
|
||
(НЕ AvitoBlockedError), backfill метит листинг is_active=FALSE и продолжает без
|
||
ротации/ретрая.
|
||
|
||
Зеркало fixture-стиля test_avito_detail_soft_block.py / test_avito_detail_browser_only.py.
|
||
"""
|
||
|
||
from unittest.mock import AsyncMock, MagicMock, patch
|
||
|
||
import pytest
|
||
|
||
from app.services.scrapers import avito_detail as detail_mod
|
||
from app.services.scrapers.avito_detail import fetch_detail
|
||
from app.services.scrapers.avito_exceptions import AvitoBlockedError, AvitoListingGoneError
|
||
|
||
_SENTINEL = object()
|
||
|
||
# 404-страница Avito: специфичный <title>; тело тоже содержит generic-витринный текст,
|
||
# который сам по себе тригерит soft-block — поэтому not-found проверяется ПЕРВЫМ.
|
||
_NOT_FOUND_HTML = (
|
||
'<!DOCTYPE html><html lang="ru"><head>'
|
||
"<title>Ошибка 404. Страница не найдена</title>"
|
||
"</head><body><div>Объявления на сайте Авито</div></body></html>"
|
||
)
|
||
# Soft-block deflect: generic-витрина БЕЗ 404-текста — это блок, НЕ gone.
|
||
_SOFT_BLOCK_HTML = (
|
||
'<!DOCTYPE html><html lang="ru"><head>'
|
||
"<title>Авито — Объявления на сайте Авито</title>"
|
||
"</head><body><div>...</div></body></html>"
|
||
)
|
||
_ITEM_VIEW_HTML = "<html><body><span data-marker='item-view/item-id'>№ 123</span></body></html>"
|
||
|
||
_ASYNC_SESSION = "app.services.scrapers.avito_detail.AsyncSession"
|
||
_BUILD_SESSION = "app.services.scrapers.avito_detail._build_detail_session"
|
||
_PARSE = "app.services.scrapers.avito_detail.parse_detail_html"
|
||
|
||
|
||
def test_is_detail_not_found_unit() -> None:
|
||
"""Юнит _is_detail_not_found: 404 → True; soft-block deflect / item-view / пусто → False."""
|
||
assert detail_mod._is_detail_not_found(_NOT_FOUND_HTML) is True
|
||
# soft-block deflect (generic-витрина без 404-текста) НЕ должен ловиться как gone
|
||
assert detail_mod._is_detail_not_found(_SOFT_BLOCK_HTML) is False
|
||
# реальное объявление с item-view → False
|
||
assert detail_mod._is_detail_not_found(_ITEM_VIEW_HTML) is False
|
||
assert detail_mod._is_detail_not_found("") is False
|
||
|
||
|
||
def _browser_returning(html: str) -> MagicMock:
|
||
bf = MagicMock()
|
||
bf.fetch = AsyncMock(return_value=html)
|
||
return bf
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_browser_mode_404_raises_listing_gone_not_blocked() -> None:
|
||
"""browser_fetcher вернул 404-HTML → AvitoListingGoneError (НЕ AvitoBlockedError),
|
||
без curl-сессии."""
|
||
bf = _browser_returning(_NOT_FOUND_HTML)
|
||
with (
|
||
patch(_ASYNC_SESSION) as async_session,
|
||
patch(_BUILD_SESSION) as build_session,
|
||
patch(_PARSE, return_value=_SENTINEL) as parse,
|
||
):
|
||
with pytest.raises(AvitoListingGoneError) as exc_info:
|
||
await fetch_detail("/items/123", browser_fetcher=bf)
|
||
|
||
# Тип строго AvitoListingGoneError и НЕ подкласс AvitoBlockedError —
|
||
# иначе backfill-ловушка `except (AvitoBlockedError, ...)` перехватила бы его.
|
||
assert not isinstance(exc_info.value, AvitoBlockedError)
|
||
bf.fetch.assert_awaited_once()
|
||
parse.assert_not_called()
|
||
async_session.assert_not_called()
|
||
build_session.assert_not_called()
|