fix(tradein/scraper): detail-backfill с нулём обогащений перестаёт называться успехом (#2674) #2695
8 changed files with 232 additions and 50 deletions
|
|
@ -394,6 +394,68 @@ def mark_banned(db: Session, run_id: int, error: str, counters: dict[str, int])
|
|||
_alert_on_run_id(db, run_id)
|
||||
|
||||
|
||||
def mark_backfill_finished(
|
||||
db: Session,
|
||||
run_id: int,
|
||||
counters: dict[str, int],
|
||||
*,
|
||||
source: str,
|
||||
aborted_by_blocks: bool = False,
|
||||
) -> None:
|
||||
"""Честный финал detail-backfill'а (#2674): нулевой прогон ≠ 'done'.
|
||||
|
||||
Все три detail-backfill'а (avito/yandex/domclick) финализировались ОДНИМ
|
||||
mark_done: прогон, который сделал N попыток и не обогатил НИ ОДНОГО объявления,
|
||||
отчитывался успехом. На проде (2026-08-06) это 78 прогонов из 158 —
|
||||
avito 23/76 (в т.ч. 5 прогонов по 1500-1600 попыток с нулём обогащений),
|
||||
yandex 31/52 (все attempted=5 failed=5), domclick 24/30 (494 попытки → 0).
|
||||
|
||||
Существующие алерты этот класс не ловили: _alert_if_consecutive_failures
|
||||
считает только failed/banned, а _alert_if_consecutive_zero_results смотрит
|
||||
total_seen, которого в counters backfill'ов нет вовсе (всегда 0 → стрик не
|
||||
прерывается никогда → анти-спам молчит после первого раза).
|
||||
|
||||
Правила (порядок важен), по образцу #2657 для domclick_city_sweep:
|
||||
- попыток не было (attempted=0) → 'done', честная пустота: кандидатов нет;
|
||||
- есть блоки источника И (прогон оборван брейкером ИЛИ ноль результата)
|
||||
→ 'banned': external constraint, не наш баг (и триггер ротации IP #2611);
|
||||
- ноль результата без блоков → 'failed': это наша поломка (парсер/сеть/БД);
|
||||
- иначе (обогатили хоть что-то) → 'done', в т.ч. частичный прогон.
|
||||
|
||||
`gone` (404 у avito) считается результатом наравне с `enriched`: прогон,
|
||||
который подтвердил снятие объявлений, работу сделал.
|
||||
"""
|
||||
attempted = int(counters.get("attempted") or 0)
|
||||
enriched = int(counters.get("enriched") or 0)
|
||||
blocked = int(counters.get("blocked") or 0)
|
||||
produced = enriched + int(counters.get("gone") or 0)
|
||||
|
||||
if attempted == 0:
|
||||
mark_done(db, run_id, counters)
|
||||
return
|
||||
|
||||
if blocked and (aborted_by_blocks or produced == 0):
|
||||
reason = (
|
||||
f"backfill-honest-status: {source} остановлен блоками источника — "
|
||||
f"blocked={blocked}, обогащено {enriched} из {attempted} попыток (#2674)"
|
||||
)
|
||||
logger.error("%s run_id=%d", reason, run_id)
|
||||
mark_banned(db, run_id, reason, counters)
|
||||
return
|
||||
|
||||
if produced == 0:
|
||||
reason = (
|
||||
f"backfill-honest-status: {source} без результата — 0 обогащено из "
|
||||
f"{attempted} попыток (failed={counters.get('failed', 0)}, "
|
||||
f"blocked={blocked}) (#2674)"
|
||||
)
|
||||
logger.error("%s run_id=%d", reason, run_id)
|
||||
mark_failed(db, run_id, reason, counters)
|
||||
return
|
||||
|
||||
mark_done(db, run_id, counters)
|
||||
|
||||
|
||||
def mark_cancelled(db: Session, run_id: int) -> bool:
|
||||
"""Set status='cancelled' если currently 'running'. Returns True если cancelled.
|
||||
|
||||
|
|
|
|||
|
|
@ -10,8 +10,9 @@ Legacy listings (older than 2h or outside radius) are never enriched.
|
|||
Solution: single snapshot SELECT at start (guarantees termination), same proxy
|
||||
session path as the detail-phase of `run_avito_city_sweep`
|
||||
(scraper_kit.orchestration.pipeline). Block handling mirrors that phase:
|
||||
rotate IP on every block, abort after max_consecutive_blocks (mark_done not
|
||||
mark_failed -- block is temporary, retry next night via NULL detail_enriched_at).
|
||||
rotate IP on every block, abort after max_consecutive_blocks. Статус оборванного
|
||||
блоками прогона — 'banned' (#2674, runs.mark_backfill_finished): работу он не
|
||||
доделал, остаток снапшота уедет в следующую ночь через NULL detail_enriched_at.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -140,7 +141,8 @@ async def run_avito_detail_backfill(
|
|||
max_consecutive_blocks: int -- abort threshold, default 5.
|
||||
|
||||
Lifecycle: update_heartbeat -> snapshot -> loop with budget guard ->
|
||||
mark_done (incl. partial/block-abort) / mark_failed (exception only).
|
||||
mark_backfill_finished (done / banned при блоках / failed при нуле, #2674);
|
||||
mark_failed напрямую — только при исключении.
|
||||
"""
|
||||
batch_size = int(params.get("batch_size", 800))
|
||||
oblast_batch_size = int(params.get("oblast_batch_size", 100))
|
||||
|
|
@ -306,6 +308,7 @@ async def run_avito_detail_backfill(
|
|||
)
|
||||
|
||||
consecutive_blocks = 0
|
||||
aborted_by_blocks = False
|
||||
do_sleep = False
|
||||
items_since_warm = 0
|
||||
|
||||
|
|
@ -498,6 +501,7 @@ async def run_avito_detail_backfill(
|
|||
counters.enriched,
|
||||
counters.attempted,
|
||||
)
|
||||
aborted_by_blocks = True
|
||||
break
|
||||
# МГТС sticky-IP: один фикс. exit-IP, per-connection ротации нет (проверено:
|
||||
# 6/6 свежих сессий = тот же IP 109.252.125.80; ротация только вручную
|
||||
|
|
@ -570,9 +574,15 @@ async def run_avito_detail_backfill(
|
|||
|
||||
counters.duration_sec = time.monotonic() - start
|
||||
current_counters = counters.to_dict()
|
||||
runs_mod.mark_done(db, run_id, current_counters)
|
||||
runs_mod.mark_backfill_finished(
|
||||
db,
|
||||
run_id,
|
||||
current_counters,
|
||||
source="avito_detail_backfill",
|
||||
aborted_by_blocks=aborted_by_blocks,
|
||||
)
|
||||
logger.info(
|
||||
"avito_detail_backfill: run_id=%d DONE -- attempted=%d enriched=%d "
|
||||
"avito_detail_backfill: run_id=%d FINISHED -- attempted=%d enriched=%d "
|
||||
"blocked=%d gone=%d failed=%d duration=%.1fs",
|
||||
run_id,
|
||||
counters.attempted,
|
||||
|
|
|
|||
|
|
@ -36,10 +36,11 @@ one BrowserFetcher is constructed per run.
|
|||
|
||||
Exception triad differs from Avito:
|
||||
- DomClickBlockedError (QRATOR challenge page OR any browser-fetch failure) --
|
||||
increments consecutive_blocks, abort via mark_done (NOT mark_failed) once
|
||||
max_consecutive_blocks is hit -- a block-abort is an expected operational
|
||||
outcome (QRATOR reputation burn), not a task failure. Mirrors Avito's
|
||||
AvitoBlockedError handling. No IP-rotation/cooldown recovery step exists here
|
||||
increments consecutive_blocks, abort once max_consecutive_blocks is hit.
|
||||
Статус такого прогона — 'banned' (#2674, см. runs.mark_backfill_finished):
|
||||
блок это external constraint, не наш баг, но и НЕ успех — раньше здесь стоял
|
||||
mark_done, и 24 из 30 прогонов с нулём обогащений назывались успешными.
|
||||
No IP-rotation/cooldown recovery step exists here
|
||||
(DomClick uses one dedicated residential proxy, not a rotating pool) -- an
|
||||
aborted run simply retries the remaining backlog next window.
|
||||
- DomClickParseError (__SSR_STATE__ missing/malformed -- schema drift, NOT a
|
||||
|
|
@ -181,7 +182,8 @@ async def run_domclick_detail_backfill(
|
|||
max_consecutive_blocks: int -- abort threshold, default 3.
|
||||
|
||||
Lifecycle: update_heartbeat -> snapshot -> loop with budget guard ->
|
||||
mark_done (incl. partial/block-abort) / mark_failed (exception only).
|
||||
mark_backfill_finished (done / banned при блоках / failed при нуле, #2674);
|
||||
mark_failed напрямую — только при исключении.
|
||||
"""
|
||||
batch_size = int(params.get("batch_size", 200))
|
||||
budget_sec = float(params.get("budget_sec", 3600))
|
||||
|
|
@ -251,6 +253,7 @@ async def run_domclick_detail_backfill(
|
|||
)
|
||||
|
||||
consecutive_blocks = 0
|
||||
aborted_by_blocks = False
|
||||
do_sleep = False
|
||||
|
||||
# Exactly ONE BrowserFetcher per run (no curl fallback for DomClick, see
|
||||
|
|
@ -333,6 +336,7 @@ async def run_domclick_detail_backfill(
|
|||
counters.enriched,
|
||||
counters.attempted,
|
||||
)
|
||||
aborted_by_blocks = True
|
||||
break
|
||||
|
||||
except Exception as e:
|
||||
|
|
@ -354,9 +358,15 @@ async def run_domclick_detail_backfill(
|
|||
|
||||
counters.duration_sec = time.monotonic() - start
|
||||
current_counters = counters.to_dict()
|
||||
runs_mod.mark_done(db, run_id, current_counters)
|
||||
runs_mod.mark_backfill_finished(
|
||||
db,
|
||||
run_id,
|
||||
current_counters,
|
||||
source="domclick_detail_backfill",
|
||||
aborted_by_blocks=aborted_by_blocks,
|
||||
)
|
||||
logger.info(
|
||||
"domclick_detail_backfill: run_id=%d DONE -- attempted=%d enriched=%d "
|
||||
"domclick_detail_backfill: run_id=%d FINISHED -- attempted=%d enriched=%d "
|
||||
"blocked=%d failed=%d duration=%.1fs",
|
||||
run_id,
|
||||
counters.attempted,
|
||||
|
|
|
|||
|
|
@ -12,8 +12,10 @@ offer detail page via curl_cffi AsyncSession (chrome120 + proxy) — mirrors
|
|||
yandex_address_backfill.py which already gets full HTML from Yandex on prod.
|
||||
Parse HTML via YandexDetailScraper.parse (pure, no network). Persist via
|
||||
save_detail_enrichment. Track consecutive parse→None results; abort after
|
||||
max_consecutive_blocks (mark_done, not mark_failed — retry next night via
|
||||
NULL detail_enriched_at).
|
||||
max_consecutive_blocks. Прогон с нулём обогащений теперь 'failed', не 'done'
|
||||
(#2674, runs.mark_backfill_finished): на проде 31 прогон из 52 упирался ровно в
|
||||
этот брейкер (attempted=5 failed=5) и все 31 назывались успешными. Остаток
|
||||
снапшота уедет в следующую ночь через NULL detail_enriched_at.
|
||||
|
||||
Why curl_cffi and not YandexDetailScraper.fetch_detail:
|
||||
fetch_detail uses BaseScraper._http_get (plain httpx, no proxy, no TLS
|
||||
|
|
@ -85,7 +87,8 @@ async def run_yandex_detail_backfill(
|
|||
(possible captcha wall); consecutive None → abort after max_consecutive_blocks.
|
||||
|
||||
Lifecycle: update_heartbeat -> snapshot -> loop with budget guard ->
|
||||
mark_done (incl. partial / consecutive-None abort) / mark_failed (exception only).
|
||||
mark_backfill_finished (done / failed при нуле обогащений, #2674);
|
||||
mark_failed напрямую — только при исключении.
|
||||
"""
|
||||
batch_size = int(params.get("batch_size", 800))
|
||||
budget_sec = float(params.get("budget_sec", 3600))
|
||||
|
|
@ -278,9 +281,11 @@ async def run_yandex_detail_backfill(
|
|||
|
||||
counters.duration_sec = time.monotonic() - start
|
||||
current_counters = counters.to_dict()
|
||||
runs_mod.mark_done(db, run_id, current_counters)
|
||||
runs_mod.mark_backfill_finished(
|
||||
db, run_id, current_counters, source="yandex_detail_backfill"
|
||||
)
|
||||
logger.info(
|
||||
"yandex_detail_backfill: run_id=%d DONE -- attempted=%d enriched=%d "
|
||||
"yandex_detail_backfill: run_id=%d FINISHED -- attempted=%d enriched=%d "
|
||||
"failed=%d duration=%.1fs",
|
||||
run_id,
|
||||
counters.attempted,
|
||||
|
|
|
|||
|
|
@ -138,7 +138,7 @@ async def test_backfill_processes_snapshot_to_completion() -> None:
|
|||
assert result.blocked == 0
|
||||
assert result.failed == 0
|
||||
assert mock_fetch.call_count == 3
|
||||
runs.mark_done.assert_called_once()
|
||||
runs.mark_backfill_finished.assert_called_once()
|
||||
runs.mark_failed.assert_not_called()
|
||||
# #2310 regression guard: kit fetch_detail silently drops the backconnect-
|
||||
# on-403 retry (and kit AvitoScraper can't read scraper_proxy_url at all)
|
||||
|
|
@ -188,13 +188,16 @@ async def test_backfill_build_warmed_session_receives_config() -> None:
|
|||
mock_build.assert_awaited_once()
|
||||
_, build_call_kwargs = mock_build.call_args
|
||||
assert isinstance(build_call_kwargs.get("config"), RealScraperConfig)
|
||||
runs.mark_done.assert_called_once()
|
||||
runs.mark_backfill_finished.assert_called_once()
|
||||
runs.mark_failed.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_backfill_blocked_abort_after_max_consecutive() -> None:
|
||||
"""5 consecutive AvitoBlockedError -> abort, mark_done (NOT mark_failed).
|
||||
"""5 consecutive AvitoBlockedError -> abort с пометкой aborted_by_blocks (#2674).
|
||||
|
||||
Раньше — mark_done; на проде 13 прогонов attempted=5 blocked=5 enriched=0
|
||||
назывались успехом. Теперь флаг обрыва → статус 'banned'.
|
||||
|
||||
#1950 abort-reorder: abort-check ПЕРЕД recovery → на 5-м (аборт-)блоке rotate_ip
|
||||
НЕ дёргается (не тратим recovery на финальном блоке). rotate_ip x4 (блоки 1-4).
|
||||
|
|
@ -226,7 +229,8 @@ async def test_backfill_blocked_abort_after_max_consecutive() -> None:
|
|||
assert result.blocked == 5
|
||||
assert result.attempted == 5
|
||||
assert result.enriched == 0
|
||||
runs.mark_done.assert_called_once()
|
||||
runs.mark_backfill_finished.assert_called_once()
|
||||
assert runs.mark_backfill_finished.call_args.kwargs["aborted_by_blocks"] is True
|
||||
runs.mark_failed.assert_not_called()
|
||||
# abort-check до recovery → 5-й блок абортит без rotate; rotate только на блоках 1-4.
|
||||
assert mock_scraper.return_value._rotate_ip.call_count == 4
|
||||
|
|
@ -266,11 +270,11 @@ async def test_backfill_sigterm_drain_breaks_and_marks_done_partial() -> None:
|
|||
assert result.attempted == 1
|
||||
assert result.enriched == 1
|
||||
assert mock_fetch.call_count == 1
|
||||
runs.mark_done.assert_called_once()
|
||||
runs.mark_backfill_finished.assert_called_once()
|
||||
runs.mark_failed.assert_not_called()
|
||||
runs.mark_cancelled.assert_not_called()
|
||||
# mark_done получил ЧАСТИЧНЫЕ счётчики (attempted=1, а не весь snapshot=3).
|
||||
done_counters = runs.mark_done.call_args.args[2]
|
||||
done_counters = runs.mark_backfill_finished.call_args.args[2]
|
||||
assert done_counters["attempted"] == 1
|
||||
|
||||
|
||||
|
|
@ -293,7 +297,7 @@ async def test_backfill_budget_guard_stops_loop() -> None:
|
|||
await run_avito_detail_backfill(db, run_id=4, params={"batch_size": 5, "budget_sec": 1})
|
||||
|
||||
mock_fetch.assert_not_called()
|
||||
runs.mark_done.assert_called_once()
|
||||
runs.mark_backfill_finished.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -315,7 +319,7 @@ async def test_backfill_top_level_exception_marks_failed() -> None:
|
|||
)
|
||||
|
||||
runs.mark_failed.assert_called_once()
|
||||
runs.mark_done.assert_not_called()
|
||||
runs.mark_backfill_finished.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -350,7 +354,7 @@ async def test_backfill_rotate_ip_called_on_each_block() -> None:
|
|||
assert result.enriched == 1
|
||||
assert result.blocked == 1
|
||||
assert mock_scraper.return_value._rotate_ip.call_count == 1
|
||||
runs.mark_done.assert_called_once()
|
||||
runs.mark_backfill_finished.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -530,7 +534,7 @@ async def test_backfill_fetch_exception_continues() -> None:
|
|||
assert result.enriched == 1
|
||||
assert result.attempted == 2
|
||||
db.rollback.assert_called()
|
||||
runs.mark_done.assert_called_once()
|
||||
runs.mark_backfill_finished.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -587,7 +591,7 @@ async def test_backfill_fetch_timeout_skips_and_continues() -> None:
|
|||
assert result.attempted == 2
|
||||
assert len(call_urls) == 2, "loop должен дойти до второго листинга, а не зависнуть"
|
||||
db.rollback.assert_called()
|
||||
runs.mark_done.assert_called_once()
|
||||
runs.mark_backfill_finished.assert_called_once()
|
||||
runs.mark_failed.assert_not_called()
|
||||
|
||||
|
||||
|
|
@ -631,7 +635,7 @@ async def test_backfill_listing_gone_marks_inactive_no_breaker() -> None:
|
|||
assert result.enriched == 0
|
||||
# breaker НЕ абортил: rotate_ip НЕ дёргался (gone ≠ block), run завершён mark_done.
|
||||
mock_scraper.return_value._rotate_ip.assert_not_called()
|
||||
runs.mark_done.assert_called_once()
|
||||
runs.mark_backfill_finished.assert_called_once()
|
||||
runs.mark_failed.assert_not_called()
|
||||
# UPDATE listings SET is_active = FALSE по row id=1 выполнен (мок db.execute).
|
||||
update_calls = [
|
||||
|
|
@ -683,7 +687,7 @@ async def test_backfill_use_curl_flag_skips_browser_fetcher() -> None:
|
|||
_, kwargs = mock_fetch.call_args
|
||||
assert kwargs.get("browser_fetcher") is None
|
||||
assert result.enriched == 1
|
||||
runs.mark_done.assert_called_once()
|
||||
runs.mark_backfill_finished.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -728,7 +732,7 @@ async def test_backfill_use_curl_false_creates_browser_fetcher() -> None:
|
|||
_, kwargs = mock_fetch.call_args
|
||||
assert kwargs.get("browser_fetcher") is not None
|
||||
assert result.enriched == 1
|
||||
runs.mark_done.assert_called_once()
|
||||
runs.mark_backfill_finished.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -784,5 +788,5 @@ async def test_backfill_use_curl_block_cooldown_research_no_rebuild() -> None:
|
|||
assert mock_build.await_count == 1
|
||||
# changeip-ротация (legacy путь) под use_curl НЕ дёргается.
|
||||
mock_scraper.return_value._rotate_ip.assert_not_called()
|
||||
runs.mark_done.assert_called_once()
|
||||
runs.mark_backfill_finished.assert_called_once()
|
||||
runs.mark_failed.assert_not_called()
|
||||
|
|
|
|||
|
|
@ -165,7 +165,7 @@ async def test_backfill_processes_snapshot_with_cookies_threaded() -> None:
|
|||
_, kwargs = call
|
||||
assert kwargs.get("cookies") == fake_cookies
|
||||
assert kwargs.get("browser_fetcher") is not None
|
||||
runs.mark_done.assert_called_once()
|
||||
runs.mark_backfill_finished.assert_called_once()
|
||||
runs.mark_failed.assert_not_called()
|
||||
|
||||
|
||||
|
|
@ -204,13 +204,17 @@ async def test_backfill_cookies_none_still_proceeds_with_error_alert(caplog) ->
|
|||
assert kwargs.get("cookies") is None
|
||||
assert "кук DomClick нет в БД" in caplog.text
|
||||
assert [r for r in caplog.records if r.levelno >= logging.ERROR]
|
||||
runs.mark_done.assert_called_once()
|
||||
runs.mark_backfill_finished.assert_called_once()
|
||||
runs.mark_failed.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_backfill_blocked_abort_after_max_consecutive() -> None:
|
||||
"""3 consecutive DomClickBlockedError -> abort, mark_done (NOT mark_failed).
|
||||
"""3 consecutive DomClickBlockedError -> abort с пометкой aborted_by_blocks (#2674).
|
||||
|
||||
Раньше здесь стоял mark_done — на проде это дало 16 «успешных» прогонов подряд
|
||||
с attempted=3 blocked=3 enriched=0. Теперь финал уходит в mark_backfill_finished
|
||||
с флагом обрыва → статус 'banned' (ветвление проверено в test_backfill_honest_status).
|
||||
|
||||
No IP-rotation recovery step exists for DomClick (single dedicated proxy) --
|
||||
abort happens on the SAME iteration the threshold is hit, no extra recovery calls.
|
||||
|
|
@ -240,7 +244,8 @@ async def test_backfill_blocked_abort_after_max_consecutive() -> None:
|
|||
assert result.blocked == 3
|
||||
assert result.attempted == 3
|
||||
assert result.enriched == 0
|
||||
runs.mark_done.assert_called_once()
|
||||
runs.mark_backfill_finished.assert_called_once()
|
||||
assert runs.mark_backfill_finished.call_args.kwargs["aborted_by_blocks"] is True
|
||||
runs.mark_failed.assert_not_called()
|
||||
|
||||
|
||||
|
|
@ -279,7 +284,7 @@ async def test_backfill_parse_error_counts_failed_no_abort() -> None:
|
|||
assert result.enriched == 1
|
||||
assert result.attempted == 2
|
||||
assert result.blocked == 0
|
||||
runs.mark_done.assert_called_once()
|
||||
runs.mark_backfill_finished.assert_called_once()
|
||||
runs.mark_failed.assert_not_called()
|
||||
|
||||
|
||||
|
|
@ -314,10 +319,10 @@ async def test_backfill_sigterm_drain_breaks_and_marks_done_partial() -> None:
|
|||
assert result.attempted == 1
|
||||
assert result.enriched == 1
|
||||
assert mock_fetch.call_count == 1
|
||||
runs.mark_done.assert_called_once()
|
||||
runs.mark_backfill_finished.assert_called_once()
|
||||
runs.mark_failed.assert_not_called()
|
||||
runs.mark_cancelled.assert_not_called()
|
||||
done_counters = runs.mark_done.call_args.args[2]
|
||||
done_counters = runs.mark_backfill_finished.call_args.args[2]
|
||||
assert done_counters["attempted"] == 1
|
||||
|
||||
|
||||
|
|
@ -342,7 +347,7 @@ async def test_backfill_budget_guard_stops_loop() -> None:
|
|||
await run_domclick_detail_backfill(db, run_id=7, params={"batch_size": 5, "budget_sec": 1})
|
||||
|
||||
mock_fetch.assert_not_called()
|
||||
runs.mark_done.assert_called_once()
|
||||
runs.mark_backfill_finished.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -366,7 +371,7 @@ async def test_backfill_top_level_exception_marks_failed() -> None:
|
|||
)
|
||||
|
||||
runs.mark_failed.assert_called_once()
|
||||
runs.mark_done.assert_not_called()
|
||||
runs.mark_backfill_finished.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -397,4 +402,4 @@ async def test_backfill_generic_exception_continues_and_rolls_back() -> None:
|
|||
assert result.enriched == 1
|
||||
assert result.attempted == 2
|
||||
db.rollback.assert_called()
|
||||
runs.mark_done.assert_called_once()
|
||||
runs.mark_backfill_finished.assert_called_once()
|
||||
|
|
|
|||
|
|
@ -142,13 +142,17 @@ async def test_backfill_processes_snapshot_to_completion() -> None:
|
|||
assert result.enriched == 3
|
||||
assert result.failed == 0
|
||||
assert session.get.call_count == 3
|
||||
runs.mark_done.assert_called_once()
|
||||
runs.mark_backfill_finished.assert_called_once()
|
||||
runs.mark_failed.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_backfill_parse_none_abort_after_max_consecutive() -> None:
|
||||
"""5 consecutive parse→None results -> abort, mark_done (NOT mark_failed)."""
|
||||
"""5 consecutive parse→None -> abort; финал через mark_backfill_finished (#2674).
|
||||
|
||||
Раньше — mark_done; на проде ровно этот брейкер дал 31 «успешный» прогон из 52
|
||||
(attempted=5 failed=5 enriched=0). Блоков у Яндекса нет → статус 'failed'.
|
||||
"""
|
||||
snapshot = _make_snapshot(10)
|
||||
db = _mock_db(snapshot)
|
||||
runs = MagicMock()
|
||||
|
|
@ -172,7 +176,7 @@ async def test_backfill_parse_none_abort_after_max_consecutive() -> None:
|
|||
assert result.attempted == 5
|
||||
assert result.failed == 5
|
||||
assert result.enriched == 0
|
||||
runs.mark_done.assert_called_once()
|
||||
runs.mark_backfill_finished.assert_called_once()
|
||||
runs.mark_failed.assert_not_called()
|
||||
|
||||
|
||||
|
|
@ -205,7 +209,7 @@ async def test_backfill_parse_none_resets_on_success() -> None:
|
|||
assert result.attempted == 3
|
||||
assert result.enriched == 1
|
||||
assert result.failed == 2
|
||||
runs.mark_done.assert_called_once()
|
||||
runs.mark_backfill_finished.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -233,7 +237,7 @@ async def test_backfill_non200_counts_as_fail_and_aborts() -> None:
|
|||
|
||||
assert result.failed == 5
|
||||
assert result.enriched == 0
|
||||
runs.mark_done.assert_called_once()
|
||||
runs.mark_backfill_finished.assert_called_once()
|
||||
runs.mark_failed.assert_not_called()
|
||||
|
||||
|
||||
|
|
@ -257,7 +261,7 @@ async def test_backfill_budget_guard_stops_loop() -> None:
|
|||
await run_yandex_detail_backfill(db, run_id=6, params={"batch_size": 5, "budget_sec": 1})
|
||||
|
||||
session.get.assert_not_called()
|
||||
runs.mark_done.assert_called_once()
|
||||
runs.mark_backfill_finished.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -277,7 +281,7 @@ async def test_backfill_top_level_exception_marks_failed() -> None:
|
|||
)
|
||||
|
||||
runs.mark_failed.assert_called_once()
|
||||
runs.mark_done.assert_not_called()
|
||||
runs.mark_backfill_finished.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -309,7 +313,7 @@ async def test_backfill_fetch_exception_continues() -> None:
|
|||
assert result.attempted == 2
|
||||
# fetch exception is caught by inner try/except (not DB) — no rollback needed
|
||||
db.rollback.assert_not_called()
|
||||
runs.mark_done.assert_called_once()
|
||||
runs.mark_backfill_finished.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
82
tradein-mvp/backend/tests/test_backfill_honest_status.py
Normal file
82
tradein-mvp/backend/tests/test_backfill_honest_status.py
Normal file
|
|
@ -0,0 +1,82 @@
|
|||
"""#2674 — detail-backfill с нулём обогащений перестаёт называться 'done'.
|
||||
|
||||
Все три backfill'а (avito/yandex/domclick) финализировались одним mark_done.
|
||||
На проде 2026-08-06 это 78 прогонов из 158: avito 23/76 (включая 5 прогонов на
|
||||
1500-1600 попыток без единого обогащения), yandex 31/52, domclick 24/30
|
||||
(494 попытки → 0 обогащено, 63 блока, 431 fail — и все 30 'done').
|
||||
|
||||
Проверяем ровно ветвление mark_backfill_finished — БД замокана.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
|
||||
|
||||
from app.services import scrape_runs as runs_mod
|
||||
|
||||
|
||||
def _finish(counters: dict[str, int], *, aborted: bool = False) -> tuple[str, str | None]:
|
||||
"""Вызвать mark_backfill_finished с замоканными mark_* → (статус, причина)."""
|
||||
calls: list[tuple[str, str | None]] = []
|
||||
with (
|
||||
patch.object(runs_mod, "mark_done", lambda *a, **k: calls.append(("done", None))),
|
||||
patch.object(
|
||||
runs_mod, "mark_failed", lambda db, rid, err, c: calls.append(("failed", err))
|
||||
),
|
||||
patch.object(
|
||||
runs_mod, "mark_banned", lambda db, rid, err, c: calls.append(("banned", err))
|
||||
),
|
||||
):
|
||||
runs_mod.mark_backfill_finished(
|
||||
MagicMock(),
|
||||
1,
|
||||
counters,
|
||||
source="domclick_detail_backfill",
|
||||
aborted_by_blocks=aborted,
|
||||
)
|
||||
assert len(calls) == 1, f"ожидался ровно один финализатор, получено {calls}"
|
||||
return calls[0]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("counters", "aborted", "expected"),
|
||||
[
|
||||
# Прод-факт domclick: 16 прогонов attempted=3 blocked=3 → брейкер оборвал.
|
||||
({"attempted": 3, "enriched": 0, "blocked": 3, "failed": 0}, True, "banned"),
|
||||
# Прод-факт domclick: 100 попыток, все fail, блоки не распознаны (до #2645).
|
||||
({"attempted": 100, "enriched": 0, "blocked": 0, "failed": 100}, False, "failed"),
|
||||
# Прод-факт avito: 1500 попыток, 1499 fail + 1 блок, ноль обогащений.
|
||||
({"attempted": 1500, "enriched": 0, "blocked": 1, "failed": 1499}, False, "banned"),
|
||||
# Прод-факт yandex: 31 прогон attempted=5 failed=5, ключа blocked нет вовсе.
|
||||
({"attempted": 5, "enriched": 0, "failed": 5}, False, "failed"),
|
||||
# Кандидатов не было — честная пустота, это успех.
|
||||
({"attempted": 0, "enriched": 0, "blocked": 0, "failed": 0}, False, "done"),
|
||||
# Частичный прогон: обогатили хоть что-то → успех.
|
||||
({"attempted": 50, "enriched": 12, "blocked": 0, "failed": 38}, False, "done"),
|
||||
# Блоки были, но прогон доработал и обогатил — не бан.
|
||||
({"attempted": 50, "enriched": 12, "blocked": 2, "failed": 36}, False, "done"),
|
||||
# Блок оборвал прогон, хотя часть успели обогатить — работа не доделана.
|
||||
({"attempted": 50, "enriched": 12, "blocked": 5, "failed": 33}, True, "banned"),
|
||||
# avito: 404-«снято с продажи» — тоже результат, а не пустой прогон.
|
||||
({"attempted": 30, "enriched": 0, "gone": 30, "blocked": 0, "failed": 0}, False, "done"),
|
||||
],
|
||||
)
|
||||
def test_status_matches_reality(counters: dict[str, Any], aborted: bool, expected: str) -> None:
|
||||
status, _ = _finish(counters, aborted=aborted)
|
||||
assert status == expected
|
||||
|
||||
|
||||
def test_reason_carries_numbers_and_marker() -> None:
|
||||
"""Причина в scrape_runs.error должна быть читаемой человеком, не пустой."""
|
||||
status, reason = _finish({"attempted": 3, "enriched": 0, "blocked": 3}, aborted=True)
|
||||
assert status == "banned"
|
||||
assert reason is not None
|
||||
assert "backfill-honest-status" in reason
|
||||
assert "domclick_detail_backfill" in reason
|
||||
assert "blocked=3" in reason and "из 3 попыток" in reason
|
||||
Loading…
Add table
Reference in a new issue