lekss361/gendesign
1
0
Fork 0
Code Issues 57 Pull requests Projects Releases Packages Wiki Activity Actions

fixup(tradein): merge main (#901 avito browser branch) + real SERP-routing test (#915)

Browse source 
Branch was cut from pre-#901 main, so its avito.py lacked the _browser branch
in _fetch_serp_html → pipeline's scraper._browser was a no-op (reviewer was
right). Merge current main to bring #901's avito.py in; replace the monkeypatch
SERP test with a real _fetch_serp_html call proving browser routing.

Refs #883, #901
This commit is contained in:
bot-backend 2026-05-31 21:07:46 +03:00
parent e19e45b3f4
commit 9fbe8e2717
1 changed files with 14 additions and 45 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

59
tradein-mvp/backend/tests/test_pipeline_browser_routing.py
View file

@ -368,57 +368,26 @@ async def test_fetch_detail_curl_mode_no_browser_fetcher() -> None:
assert result.item_id == "99887766" assert result.item_id == "99887766"
# ── 6. SERP routing: browser-mode — _cffi never touched (refutes #916 concern) ─ # ── 6. SERP routing: real _fetch_serp_html routes through browser ────────────
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_fetch_serp_html_uses_browser_when_browser_set() -> None: async def test_fetch_serp_html_routes_through_browser() -> None:
"""SERP routing: with _browser set and _cffi=None, fetch_around routes SERP """With _browser set and _cffi=None, the REAL _fetch_serp_html returns the
through _browser and never dereferences _cffi (no AssertionError). browser HTML and never dereferences _cffi (no AssertionError). Proves #915
SERP routing on top of #901's browser branch."""
Refutes #916 reviewer concern that _cffi assert always fires in browser-mode. from unittest.mock import AsyncMock
The pipeline sets scraper._browser before calling fetch_around; _fetch_serp_html
is monkey-patched at method level to verify _browser is present and _cffi is
absent at call time — proving the routing contract holds end-to-end without
a network call.
"""
from unittest.mock import AsyncMock, patch
from app.services.scrapers.avito import AvitoScraper from app.services.scrapers.avito import AvitoScraper
scraper = AvitoScraper() scraper = AvitoScraper()
assert scraper._cffi is None # browser-mode: curl session never created assert scraper._cffi is None
scraper._browser = AsyncMock()
scraper._browser.fetch = AsyncMock(return_value="<html><body>real serp listing</body></html>")
mock_browser = AsyncMock() html = await scraper._fetch_serp_html(
# Non-firewall SERP HTML: avoid AvitoBlockedError (_FIREWALL_MARKERS not present) "https://www.avito.ru/ekaterinburg/kvartiry/prodam", page=1
serp_html = "<html><body><div>ok serp result page</div></body></html>" )
mock_browser.fetch = AsyncMock(return_value=serp_html)
scraper._browser = mock_browser
browser_calls: list[str] = [] assert html == "<html><body>real serp listing</body></html>"
scraper._browser.fetch.assert_awaited_once()
async def _patched_fetch_serp(url: str, page: int) -> str | None:
# Core assertion: _browser is set, _cffi is still None in browser-mode
assert scraper._browser is not None, "_browser must be set in browser-mode"
assert scraper._cffi is None, "_cffi must remain None in browser-mode (no AssertionError)"
browser_calls.append(url)
# Delegate to the mock browser (as production code does via browser_fetcher.fetch)
return await scraper._browser.fetch(url)
# Patch at method level — exercises fetch_around → _fetch_serp_html pathway
scraper._fetch_serp_html = _patched_fetch_serp # type: ignore[method-assign]
# Patch _parse_html so 0-cards doesn't obscure the SERP-routing assertion
with patch.object(scraper, "_parse_html", return_value=[]):
try:
await scraper.fetch_around(56.84, 60.60, 1000, pages=1)
except Exception:
# AvitoContentBlockedError on 0-card page=1 is expected — irrelevant to routing
pass
# _browser.fetch was called → SERP routed through browser, not _cffi
assert (
len(browser_calls) >= 1
), "browser.fetch was never called — SERP not routed through browser"
mock_browser.fetch.assert_awaited()
# Reaching here without AssertionError proves _cffi=None did not crash the SERP path