bot-backend
c66e18cd13
All checks were successful
Deploy Trade-In / changes (push) Successful in 10s
Deploy Trade-In / build-frontend (push) Has been skipped
Deploy Trade-In / build-browser (push) Has been skipped
Deploy Trade-In / test (push) Successful in 3m0s
Deploy Trade-In / build-backend (push) Successful in 1m36s
Deploy Trade-In / deploy (push) Successful in 1m38s
114 lines
5.5 KiB
Python
114 lines
5.5 KiB
Python
"""#2698 — домовая оценка Авито ходила в сайдкар без прокси пула, и причина отказа терялась.
|
||
|
||
Два независимых дефекта одного пути (backfill_house_imv → BrowserFetcher → POST
|
||
/fetch-json → tradein-browser):
|
||
|
||
1. `BrowserFetcher(source="avito", endpoint=...)` конструировался БЕЗ
|
||
proxy_provider/use_pool/environment — единственный avito-путь без проводки пула
|
||
(avito_city_sweep её подключает, orchestration/pipeline.py). Тело POST уходило без
|
||
"proxy", сайдкар брал env-прокси SCRAPER_PROXY_URL = узел пула id=1
|
||
(provider_affinity='domclick'), который `proxy_pool.acquire('avito')` не выдал бы
|
||
никогда. Прод 03.07-05.08: 35 отказов из 35 попыток в каждом прогоне при живом
|
||
сайдкаре и работающих в те же дни sweep'ах.
|
||
|
||
2. Причина отказа приходила в теле ответа сайдкара ({"error": "browser unavailable
|
||
(proxy may be down)"} / "Page.goto: NS_ERROR_PROXY_BAD_GATEWAY"), а
|
||
`resp.raise_for_status()` её выбрасывал — в houses.imv_error_reason 34 дня лежал
|
||
голый код статуса.
|
||
|
||
Сеть/БД/камуфокс замоканы.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import os
|
||
from typing import Any, ClassVar
|
||
from unittest.mock import AsyncMock, MagicMock
|
||
|
||
import httpx
|
||
import pytest
|
||
|
||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
|
||
|
||
from scraper_kit.browser_fetcher import BrowserFetcher
|
||
|
||
from app.services import house_imv_backfill as hib
|
||
|
||
# ── (1) проводка пула на пути домовой оценки ─────────────────────────────────
|
||
|
||
|
||
class _CapturingFetcher:
|
||
captured: ClassVar[dict[str, Any]] = {}
|
||
|
||
def __init__(self, **kwargs: Any) -> None:
|
||
_CapturingFetcher.captured = kwargs
|
||
|
||
async def __aenter__(self) -> _CapturingFetcher:
|
||
return self
|
||
|
||
async def __aexit__(self, *_: object) -> None:
|
||
return None
|
||
|
||
|
||
async def test_backfill_browser_fetcher_gets_proxy_pool_wiring(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
"""Прод-случай #2698: без этих трёх аргументов сайдкар уходил на env-прокси."""
|
||
_CapturingFetcher.captured = {}
|
||
monkeypatch.setattr(hib.settings, "avito_imv_use_browser_fetcher", True)
|
||
monkeypatch.setattr(hib.settings, "use_proxy_pool_browser", True)
|
||
monkeypatch.setattr(hib.settings, "environment", "production")
|
||
monkeypatch.setattr(hib, "BrowserFetcher", _CapturingFetcher)
|
||
monkeypatch.setattr(hib, "_process_one_house", AsyncMock(return_value="ok"))
|
||
|
||
db = MagicMock()
|
||
db.execute.return_value.mappings.return_value.all.return_value = [
|
||
{"id": 1, "address": "ЕКБ, ул. X, 1", "full_address": None, "lat": 56.8, "lon": 60.6}
|
||
]
|
||
await hib.backfill_house_imv(db, batch_size=1)
|
||
|
||
captured = _CapturingFetcher.captured
|
||
assert captured["source"] == "avito"
|
||
assert captured["proxy_provider"] is not None, "без провайдера пул не подключится"
|
||
assert captured["use_pool"] is True, "флаг пула должен доезжать до фетчера"
|
||
# #2616 шаг 1: иначе прод-отказ «пул пуст» мёртв и мы молча уходим на env-прокси.
|
||
assert captured["environment"] == "production"
|
||
|
||
|
||
# ── (2) причина отказа сайдкара доезжает до вызывающего ──────────────────────
|
||
|
||
|
||
def _client_returning(status: int, body: dict[str, Any], url: str) -> MagicMock:
|
||
resp = httpx.Response(status, json=body, request=httpx.Request("POST", url))
|
||
client = MagicMock()
|
||
client.post = AsyncMock(return_value=resp)
|
||
client.aclose = AsyncMock(return_value=None)
|
||
return client
|
||
|
||
|
||
async def test_fetch_json_error_carries_sidecar_reason() -> None:
|
||
"""503 сайдкара: в тексте ошибки должна быть ПРИЧИНА, а не только код статуса."""
|
||
endpoint = "http://tradein-browser:3000"
|
||
async with BrowserFetcher(source="avito", endpoint=endpoint) as bf:
|
||
bf._client = _client_returning( # type: ignore[assignment]
|
||
503, {"error": "browser unavailable (proxy may be down)"}, f"{endpoint}/fetch-json"
|
||
)
|
||
with pytest.raises(httpx.HTTPStatusError) as exc_info:
|
||
await bf.fetch_json("https://www.avito.ru/web/1/coords/by_address?address=X")
|
||
|
||
message = str(exc_info.value)
|
||
assert "browser unavailable (proxy may be down)" in message
|
||
assert "503" in message
|
||
|
||
|
||
async def test_fetch_error_carries_sidecar_reason() -> None:
|
||
"""Тот же инвариант для /fetch — общий helper, а не заплатка на одном вызове."""
|
||
endpoint = "http://tradein-browser:3000"
|
||
async with BrowserFetcher(source="avito", endpoint=endpoint) as bf:
|
||
bf._client = _client_returning( # type: ignore[assignment]
|
||
500, {"error": "Error: Page.goto: NS_ERROR_PROXY_BAD_GATEWAY"}, f"{endpoint}/fetch"
|
||
)
|
||
with pytest.raises(httpx.HTTPStatusError) as exc_info:
|
||
await bf.fetch("https://www.avito.ru/evaluation/realty")
|
||
|
||
assert "NS_ERROR_PROXY_BAD_GATEWAY" in str(exc_info.value)
|