feat(tradein): yandex SERP curl_cffi+cookies transport + rooms×price combos (T4+T5)
This commit is contained in:
lekss361
parent
de6afc99fa
commit
7dbda8eeb6
3 changed files with 536 additions and 26 deletions
|
|
@ -164,5 +164,13 @@ class Settings(BaseSettings):
|
|||
# Сколько раз сменить IP при блоке прежде чем сдаться (на одну страницу).
|
||||
avito_proxy_max_rotations: int = 2
|
||||
|
||||
# ── Yandex SERP cookies (#801/T4) ───────────────────────────────────────
|
||||
# Путь к JSON-файлу с cookies браузера (формат: [{name, value, ...}, ...]).
|
||||
# Если задан и файл существует — cookies передаются в curl_cffi-сессию при
|
||||
# Yandex SERP-запросах; снижает вероятность captcha на datacenter IP.
|
||||
# Пусто / файл не найден = запросы без cookies (не падаем, только warning).
|
||||
# ENV: YANDEX_COOKIES_FILE.
|
||||
yandex_cookies_file: str | None = None
|
||||
|
||||
|
||||
settings = Settings()
|
||||
|
|
|
|||
|
|
@ -8,10 +8,15 @@ Area extraction: DOM concat-text is empty pre-hydration (~82% NULL). Fix: extrac
|
|||
area from the page-level embedded JSON state (script[id="initial_state_script"] or
|
||||
application/json script), keyed by offerId. DOM regex is kept as fallback.
|
||||
|
||||
URL pattern (path-based, no geo radius):
|
||||
https://realty.yandex.ru/{city}/kupit/kvartira/vtorichniy-rynok/?page=N
|
||||
Transport (T4): curl_cffi AsyncSession(impersonate="chrome120") — Chrome TLS
|
||||
fingerprint bypasses Yandex anti-bot gate that returns captcha/shell HTML for plain
|
||||
httpx. Optional cookies loaded from YANDEX_COOKIES_FILE env path.
|
||||
|
||||
Typical: 23 [data-test="OffersSerpItem"] cards per page.
|
||||
Combos (T5): fetch_around_multi_room supports rooms × price-range combinations to
|
||||
work around Yandex SERP ~575-card cap per query. Full combos → ~4000 unique listings.
|
||||
|
||||
URL pattern (path-based, no geo radius):
|
||||
https://realty.yandex.ru/{city}/kupit/kvartira/[{room-slug}/]vtorichniy-rynok/?...
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -20,8 +25,11 @@ import json
|
|||
import logging
|
||||
import re
|
||||
from datetime import date # noqa: F401 (used in type hints / future helpers)
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import urlencode
|
||||
|
||||
from curl_cffi.requests import AsyncSession as _CurlCffiSession
|
||||
from selectolax.parser import HTMLParser, Node
|
||||
|
||||
from app.services.scraper_settings import get_scraper_delay
|
||||
|
|
@ -63,6 +71,61 @@ _STATE_SCRIPT_SELECTORS = [
|
|||
# Area field candidates in a Yandex offer object (tried in order).
|
||||
_AREA_FIELDS = ("totalArea", "area", "spaceTotal", "spaceAll", "squareTotal")
|
||||
|
||||
# Captcha detection markers (checked in first 5000 chars, lowercased).
|
||||
_CAPTCHA_MARKERS = ("/showcaptcha", "smartcaptcha", "checkboxcaptcha")
|
||||
|
||||
# Path segments for room buckets (Yandex URL canonical slugs).
|
||||
# Key matches the rooms label used in combos; value = URL path segment.
|
||||
ROOM_PATH: dict[str, str] = {
|
||||
"studio": "studiya",
|
||||
"1": "odnokomnatnaya",
|
||||
"2": "dvuhkomnatnaya",
|
||||
"3": "trehkomnatnaya",
|
||||
"4+": "4-i-bolee",
|
||||
}
|
||||
|
||||
# Default price ranges (price_min, price_max) in rubles. None = open-ended.
|
||||
DEFAULT_PRICE_RANGES: list[tuple[int | None, int | None]] = [
|
||||
(None, 5_000_000),
|
||||
(5_000_000, 7_000_000),
|
||||
(7_000_000, 10_000_000),
|
||||
(10_000_000, 15_000_000),
|
||||
(15_000_000, 25_000_000),
|
||||
(25_000_000, None),
|
||||
]
|
||||
|
||||
|
||||
def _load_cookies_from_file(path: str | None) -> dict[str, str]:
|
||||
"""Load Yandex cookies from JSON file path (Netscape/browser export format).
|
||||
|
||||
Format: [{name: str, value: str, ...}, ...].
|
||||
Returns empty dict if path is None, missing, or unparseable — not a failure.
|
||||
"""
|
||||
if not path:
|
||||
return {}
|
||||
p = Path(path)
|
||||
if not p.exists():
|
||||
logger.warning("yandex cookies file not found: %s — running without cookies", path)
|
||||
return {}
|
||||
try:
|
||||
raw = json.loads(p.read_text(encoding="utf-8"))
|
||||
result = {
|
||||
item["name"]: item["value"]
|
||||
for item in raw
|
||||
if isinstance(item, dict) and "name" in item and "value" in item
|
||||
}
|
||||
logger.info("yandex: loaded %d cookies from %s", len(result), path)
|
||||
return result
|
||||
except Exception:
|
||||
logger.warning("yandex: failed to parse cookies file %s — running without", path)
|
||||
return {}
|
||||
|
||||
|
||||
def _is_captcha(html: str) -> bool:
|
||||
"""Return True if the response looks like a Yandex captcha page."""
|
||||
head = html[:5000].lower()
|
||||
return any(m in head for m in _CAPTCHA_MARKERS) or "докажите, что вы не робот" in head
|
||||
|
||||
|
||||
def _extract_offer_areas_from_state(html: str) -> dict[str, float]:
|
||||
"""Extract {offer_id: area_m2} map from Yandex SERP embedded JSON state.
|
||||
|
|
@ -165,6 +228,49 @@ class YandexRealtyScraper(BaseScraper):
|
|||
self.city = city
|
||||
# Load global Yandex delay from DB at instantiation
|
||||
self.request_delay_sec = get_scraper_delay(self.name)
|
||||
self._cffi_session: _CurlCffiSession | None = None
|
||||
self._cookies: dict[str, str] = {}
|
||||
|
||||
async def __aenter__(self) -> YandexRealtyScraper: # type: ignore[override]
|
||||
"""Override: Chrome120 TLS impersonation bypasses Yandex anti-bot gate.
|
||||
|
||||
Plain httpx returns captcha/shell-HTML on datacenter IPs.
|
||||
Sibling pattern: yandex_valuation.py.
|
||||
Cookies loaded from YANDEX_COOKIES_FILE settings path (optional).
|
||||
"""
|
||||
# Deferred import to avoid circular / settings-at-import-time issues
|
||||
from app.core.config import settings
|
||||
|
||||
self._cookies = _load_cookies_from_file(settings.yandex_cookies_file)
|
||||
self._cffi_session = _CurlCffiSession(
|
||||
impersonate="chrome120",
|
||||
cookies=self._cookies or {},
|
||||
headers={
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
"Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8",
|
||||
"Accept-Encoding": "gzip, deflate, br",
|
||||
"Connection": "keep-alive",
|
||||
"Upgrade-Insecure-Requests": "1",
|
||||
},
|
||||
)
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *args: object) -> None: # type: ignore[override]
|
||||
if self._cffi_session is not None:
|
||||
await self._cffi_session.close()
|
||||
self._cffi_session = None
|
||||
|
||||
async def _http_get(self, url: str, **kwargs: object) -> object: # type: ignore[override]
|
||||
"""curl_cffi-based GET with Chrome120 impersonation.
|
||||
|
||||
Returns curl_cffi Response (compatible API: .status_code, .text).
|
||||
Caller must check status_code; no automatic retry (BaseScraper.retry
|
||||
decorator is per-method and not inherited cleanly when overridden).
|
||||
"""
|
||||
if self._cffi_session is None:
|
||||
raise RuntimeError("YandexRealtyScraper must be used as async context manager")
|
||||
kwargs.setdefault("timeout", 30)
|
||||
return await self._cffi_session.get(url, **kwargs)
|
||||
|
||||
async def fetch_around(
|
||||
self,
|
||||
|
|
@ -172,11 +278,18 @@ class YandexRealtyScraper(BaseScraper):
|
|||
lon: float,
|
||||
radius_m: int = 1000,
|
||||
page: int = 0,
|
||||
rooms: str | None = None,
|
||||
price_min: int | None = None,
|
||||
price_max: int | None = None,
|
||||
) -> list[ScrapedLot]:
|
||||
"""Fetch ONE page of SERP cards. lat/lon/radius_m ignored (Yandex uses
|
||||
path-based vtorichka URL); kept for BaseScraper compat + logging.
|
||||
"""Fetch ONE page of SERP cards.
|
||||
|
||||
lat/lon/radius_m ignored (Yandex uses path-based vtorichka URL);
|
||||
kept for BaseScraper compat + logging.
|
||||
rooms: room-bucket slug (one of ROOM_PATH keys) or None = all rooms.
|
||||
price_min/price_max: price filters in rubles (None = open-ended).
|
||||
"""
|
||||
url = self._build_url(page=page)
|
||||
url = self._build_url(page=page, rooms=rooms, price_min=price_min, price_max=price_max)
|
||||
try:
|
||||
response = await self._http_get(url)
|
||||
except Exception:
|
||||
|
|
@ -185,14 +298,27 @@ class YandexRealtyScraper(BaseScraper):
|
|||
if response.status_code != 200:
|
||||
logger.warning("yandex serp returned %d for %s", response.status_code, url)
|
||||
return []
|
||||
lots = self._parse_html(response.text, page=page)
|
||||
|
||||
# NBSP → space: parse_rub() buggy on \xa0 in capture groups → price=None (T4)
|
||||
html = response.text.replace("\xa0", " ")
|
||||
|
||||
if _is_captcha(html):
|
||||
logger.warning(
|
||||
"yandex serp: captcha detected on page=%d url=%s — returning empty",
|
||||
page,
|
||||
url,
|
||||
)
|
||||
return []
|
||||
|
||||
lots = self._parse_html(html, page=page)
|
||||
logger.info(
|
||||
"yandex serp page=%d city=%s: %d cards (anchor=(%.4f,%.4f) ignored)",
|
||||
"yandex serp page=%d city=%s rooms=%s price=[%s-%s]: %d cards",
|
||||
page,
|
||||
self.city,
|
||||
rooms or "all",
|
||||
price_min,
|
||||
price_max,
|
||||
len(lots),
|
||||
lat,
|
||||
lon,
|
||||
)
|
||||
await self.sleep_between_requests()
|
||||
return lots
|
||||
|
|
@ -203,34 +329,101 @@ class YandexRealtyScraper(BaseScraper):
|
|||
lon: float,
|
||||
radius_m: int = 1000,
|
||||
max_pages: int = MAX_PAGES,
|
||||
**_legacy_kwargs: Any, # swallow rooms/sorts/pages from old callers
|
||||
rooms_list: list[str] | None = None,
|
||||
price_ranges: list[tuple[int | None, int | None]] | None = None,
|
||||
**_legacy_kwargs: Any, # swallow old callers' kwargs
|
||||
) -> list[ScrapedLot]:
|
||||
"""Fetch up to max_pages of SERP cards, dedup by offer_id (source_id)."""
|
||||
"""Fetch SERP via rooms × price-range combos to bypass SERP ~575-card cap.
|
||||
|
||||
When rooms_list/price_ranges are provided (T5 combos mode), iterates all
|
||||
(room, price_min, price_max) combinations, each up to max_pages pages,
|
||||
deduplicating by source_id/source_url across combos.
|
||||
|
||||
When both are None (legacy mode), falls back to single citywide sweep
|
||||
(original behaviour — up to max_pages pages, no filtering).
|
||||
"""
|
||||
seen: dict[str, ScrapedLot] = {}
|
||||
for page in range(max_pages):
|
||||
lots = await self.fetch_around(lat, lon, radius_m, page=page)
|
||||
if not lots:
|
||||
break # empty page → no more results
|
||||
for lot in lots:
|
||||
key = lot.source_id or lot.source_url
|
||||
if key and key not in seen:
|
||||
seen[key] = lot
|
||||
|
||||
if rooms_list is None and price_ranges is None:
|
||||
# Legacy / citywide mode — no combos
|
||||
combos: list[tuple[str | None, int | None, int | None]] = [(None, None, None)]
|
||||
else:
|
||||
r_list = rooms_list or list(ROOM_PATH.keys())
|
||||
p_ranges = price_ranges or DEFAULT_PRICE_RANGES
|
||||
combos = [(r, lo, hi) for r in r_list for lo, hi in p_ranges]
|
||||
|
||||
for rooms, price_min, price_max in combos:
|
||||
combo_label = _combo_label(rooms, price_min, price_max)
|
||||
for page in range(max_pages):
|
||||
lots = await self.fetch_around(
|
||||
lat,
|
||||
lon,
|
||||
radius_m,
|
||||
page=page,
|
||||
rooms=rooms,
|
||||
price_min=price_min,
|
||||
price_max=price_max,
|
||||
)
|
||||
if not lots:
|
||||
logger.debug(
|
||||
"yandex combos [%s] page=%d: empty — stopping combo",
|
||||
combo_label,
|
||||
page,
|
||||
)
|
||||
break
|
||||
for lot in lots:
|
||||
key = lot.source_id or lot.source_url
|
||||
if key and key not in seen:
|
||||
seen[key] = lot
|
||||
|
||||
logger.info(
|
||||
"yandex serp aggregate: %d unique lots over %d pages (city=%s)",
|
||||
"yandex serp aggregate: %d unique lots (city=%s, %d combos)",
|
||||
len(seen),
|
||||
max_pages,
|
||||
self.city,
|
||||
len(combos),
|
||||
)
|
||||
return list(seen.values())
|
||||
|
||||
def _build_url(self, page: int = 0) -> str:
|
||||
# Yandex paginates 1-based; `page=0` → no param (first page)
|
||||
base = f"{self.base_url}/{self.city}/kupit/kvartira/vtorichniy-rynok/"
|
||||
def _build_url(
|
||||
self,
|
||||
page: int = 0,
|
||||
rooms: str | None = None,
|
||||
price_min: int | None = None,
|
||||
price_max: int | None = None,
|
||||
) -> str:
|
||||
"""Build Yandex SERP URL.
|
||||
|
||||
rooms: key from ROOM_PATH for path-based room bucket; None = all rooms.
|
||||
price_min/price_max: optional price filters in rubles.
|
||||
newFlat=NO_DEAL is added when any price filter is set — prevents Yandex
|
||||
from canonicalizing away /vtorichniy-rynok/ for some room slugs.
|
||||
"""
|
||||
if rooms and rooms in ROOM_PATH:
|
||||
path = f"/{self.city}/kupit/kvartira/{ROOM_PATH[rooms]}/vtorichniy-rynok/"
|
||||
else:
|
||||
path = f"/{self.city}/kupit/kvartira/vtorichniy-rynok/"
|
||||
base = f"{self.base_url}{path}"
|
||||
|
||||
params: dict[str, str | int] = {}
|
||||
if price_min is not None:
|
||||
params["priceMin"] = price_min
|
||||
if price_max is not None:
|
||||
params["priceMax"] = price_max
|
||||
if price_min is not None or price_max is not None:
|
||||
# Guarantee vtorichka — Yandex sometimes drops /vtorichniy-rynok/ on canonicalize
|
||||
params["newFlat"] = "NO_DEAL"
|
||||
if page > 0:
|
||||
return f"{base}?page={page}"
|
||||
params["page"] = page
|
||||
|
||||
if params:
|
||||
return f"{base}?{urlencode(params)}"
|
||||
return base
|
||||
|
||||
def _parse_html(self, html: str, page: int = 0) -> list[ScrapedLot]:
|
||||
# NBSP-fix applied by caller (fetch_around) before _parse_html;
|
||||
# also applied here as a safety net for direct callers (tests, scripts).
|
||||
html = html.replace("\xa0", " ")
|
||||
|
||||
# Extract offer-level area from embedded JSON state first (covers ~82% of
|
||||
# cards where DOM text is empty pre-hydration). Falls back to DOM regex per card.
|
||||
state_areas = _extract_offer_areas_from_state(html)
|
||||
|
|
@ -405,3 +598,17 @@ class YandexRealtyScraper(BaseScraper):
|
|||
if len(urls) >= 5:
|
||||
break
|
||||
return urls
|
||||
|
||||
|
||||
def _combo_label(
|
||||
rooms: str | None,
|
||||
price_min: int | None,
|
||||
price_max: int | None,
|
||||
) -> str:
|
||||
"""Human-readable label for a (rooms, price_min, price_max) combo."""
|
||||
r = rooms or "all-rooms"
|
||||
if price_min is None and price_max is None:
|
||||
return f"{r}/any-price"
|
||||
lo = f"{price_min // 1_000_000}M" if price_min is not None else "0"
|
||||
hi = f"{price_max // 1_000_000}M" if price_max is not None else "inf"
|
||||
return f"{r}/{lo}-{hi}"
|
||||
|
|
|
|||
|
|
@ -2,14 +2,21 @@
|
|||
|
||||
import json
|
||||
from datetime import date
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from app.services.scrapers.base import ScrapedLot
|
||||
from app.services.scrapers.yandex_realty import (
|
||||
DEFAULT_CITY,
|
||||
DEFAULT_PRICE_RANGES,
|
||||
MAX_PAGES,
|
||||
ROOM_PATH,
|
||||
YandexRealtyScraper,
|
||||
_combo_label,
|
||||
_extract_offer_areas_from_state,
|
||||
_is_captcha,
|
||||
_load_cookies_from_file,
|
||||
)
|
||||
|
||||
# Realistic single-card fixture (trimmed but selector-faithful).
|
||||
|
|
@ -339,3 +346,291 @@ def test_extract_offer_areas_empty_when_no_script():
|
|||
html = "<html><body><p>no state here</p></body></html>"
|
||||
areas = _extract_offer_areas_from_state(html)
|
||||
assert areas == {}
|
||||
|
||||
|
||||
# ── PART B: T4 transport + NBSP fix ─────────────────────────────────────────
|
||||
|
||||
|
||||
def test_nbsp_price_fix():
|
||||
"""NBSP (\xa0) in price tokens must not break parse_rub → price_rub must be non-None.
|
||||
|
||||
Yandex uses \xa0 as thousands separator in prices. Without the replace, parse_rub
|
||||
returns None because regex can't capture across NBSP — the card is then skipped.
|
||||
"""
|
||||
# Price with \xa0 thousands separator (e.g. "4\xa0399\xa0000 ₽")
|
||||
nbsp_card_html = """
|
||||
<html><body>
|
||||
<div data-test="OffersSerpItem">
|
||||
<a href="/offer/8888888/">x</a>
|
||||
<div>36,8\xa0м²\xa0·\xa01-комнатная\xa0·\xa08\xa0этаж\xa0из\xa09\xa0·\xa04\xa0399\xa0000\xa0₽</div>
|
||||
</div>
|
||||
</body></html>"""
|
||||
s = YandexRealtyScraper()
|
||||
# _parse_html applies NBSP-fix before parsing — price must be extracted
|
||||
lots = s._parse_html(nbsp_card_html, page=0)
|
||||
assert len(lots) == 1
|
||||
assert lots[0].price_rub is not None
|
||||
assert lots[0].price_rub > 0
|
||||
|
||||
|
||||
def test_is_captcha_detects_showcaptcha():
|
||||
"""HTML containing /showcaptcha must be flagged as captcha."""
|
||||
html = '<html><head><meta http-equiv="refresh" content="0;url=/showcaptcha?..."></head></html>'
|
||||
assert _is_captcha(html) is True
|
||||
|
||||
|
||||
def test_is_captcha_detects_smartcaptcha():
|
||||
"""HTML with smartcaptcha JS is flagged."""
|
||||
html = "<html><body><script src='smartcaptcha.js'></script></body></html>"
|
||||
assert _is_captcha(html) is True
|
||||
|
||||
|
||||
def test_is_captcha_clean_page():
|
||||
"""Normal SERP page is not flagged as captcha."""
|
||||
assert _is_captcha(SINGLE_CARD_HTML) is False
|
||||
|
||||
|
||||
def test_is_captcha_robot_text():
|
||||
"""Russian 'докажите, что вы не робот' message is detected."""
|
||||
html = "<html><body>Докажите, что вы не робот — введите код.</body></html>"
|
||||
assert _is_captcha(html) is True
|
||||
|
||||
|
||||
def test_load_cookies_from_file_missing_path():
|
||||
"""None or missing path → empty dict, no exception."""
|
||||
assert _load_cookies_from_file(None) == {}
|
||||
assert _load_cookies_from_file("/nonexistent/path.json") == {}
|
||||
|
||||
|
||||
def test_load_cookies_from_file_valid(tmp_path):
|
||||
"""Valid Netscape-format cookie JSON → dict with name→value."""
|
||||
data = [
|
||||
{"name": "yandex_login", "value": "user123", "domain": ".yandex.ru"},
|
||||
{"name": "Session_id", "value": "abc456", "domain": ".yandex.ru"},
|
||||
{"missing_name_key": "ignored"},
|
||||
]
|
||||
f = tmp_path / "cookies.json"
|
||||
f.write_text(json.dumps(data), encoding="utf-8")
|
||||
result = _load_cookies_from_file(str(f))
|
||||
assert result == {"yandex_login": "user123", "Session_id": "abc456"}
|
||||
|
||||
|
||||
# ── PART C: T5 combos — _build_url with rooms + price filters ───────────────
|
||||
|
||||
|
||||
def test_build_url_with_rooms():
|
||||
"""Room slug mapped correctly to Yandex path segment."""
|
||||
s = YandexRealtyScraper()
|
||||
url = s._build_url(page=0, rooms="1")
|
||||
assert "/odnokomnatnaya/vtorichniy-rynok/" in url
|
||||
assert "page" not in url
|
||||
|
||||
|
||||
def test_build_url_with_rooms_2():
|
||||
s = YandexRealtyScraper()
|
||||
url = s._build_url(page=0, rooms="2")
|
||||
assert "/dvuhkomnatnaya/vtorichniy-rynok/" in url
|
||||
|
||||
|
||||
def test_build_url_with_rooms_studio():
|
||||
s = YandexRealtyScraper()
|
||||
url = s._build_url(page=0, rooms="studio")
|
||||
assert "/studiya/vtorichniy-rynok/" in url
|
||||
|
||||
|
||||
def test_build_url_with_price_range():
|
||||
"""Price range adds priceMin/priceMax + newFlat=NO_DEAL."""
|
||||
s = YandexRealtyScraper()
|
||||
url = s._build_url(page=0, price_min=5_000_000, price_max=7_000_000)
|
||||
assert "priceMin=5000000" in url
|
||||
assert "priceMax=7000000" in url
|
||||
assert "newFlat=NO_DEAL" in url
|
||||
# No rooms → base citywide path
|
||||
assert "/vtorichniy-rynok/" in url
|
||||
assert "odnokomnatnaya" not in url
|
||||
|
||||
|
||||
def test_build_url_rooms_and_price_and_page():
|
||||
"""All parameters combined: rooms + price + paginated."""
|
||||
s = YandexRealtyScraper()
|
||||
url = s._build_url(page=3, rooms="3", price_min=10_000_000, price_max=15_000_000)
|
||||
assert "/trehkomnatnaya/vtorichniy-rynok/" in url
|
||||
assert "priceMin=10000000" in url
|
||||
assert "priceMax=15000000" in url
|
||||
assert "newFlat=NO_DEAL" in url
|
||||
assert "page=3" in url
|
||||
|
||||
|
||||
def test_build_url_open_ended_price_min_only():
|
||||
"""Only priceMin set → no priceMax param but newFlat present."""
|
||||
s = YandexRealtyScraper()
|
||||
url = s._build_url(price_min=25_000_000)
|
||||
assert "priceMin=25000000" in url
|
||||
assert "priceMax" not in url
|
||||
assert "newFlat=NO_DEAL" in url
|
||||
|
||||
|
||||
def test_build_url_open_ended_price_max_only():
|
||||
"""Only priceMax → no priceMin param."""
|
||||
s = YandexRealtyScraper()
|
||||
url = s._build_url(price_max=5_000_000)
|
||||
assert "priceMax=5000000" in url
|
||||
assert "priceMin" not in url
|
||||
assert "newFlat=NO_DEAL" in url
|
||||
|
||||
|
||||
def test_build_url_unknown_rooms_falls_back_to_citywide():
|
||||
"""Unknown/invalid rooms key falls back to citywide (no room path segment)."""
|
||||
s = YandexRealtyScraper()
|
||||
url = s._build_url(rooms="99-room-penthouse")
|
||||
assert "/kupit/kvartira/vtorichniy-rynok/" in url
|
||||
assert "studiya" not in url
|
||||
assert "odnokomnatnaya" not in url
|
||||
|
||||
|
||||
def test_combos_produce_different_urls():
|
||||
"""T5: rooms × price-ranges produce distinct URLs (no URL collisions)."""
|
||||
s = YandexRealtyScraper()
|
||||
urls = set()
|
||||
rooms_list = list(ROOM_PATH.keys())
|
||||
for rooms in rooms_list:
|
||||
for lo, hi in DEFAULT_PRICE_RANGES:
|
||||
urls.add(s._build_url(page=0, rooms=rooms, price_min=lo, price_max=hi))
|
||||
expected = len(rooms_list) * len(DEFAULT_PRICE_RANGES)
|
||||
assert len(urls) == expected, f"URL collisions: {expected} combos but {len(urls)} unique URLs"
|
||||
|
||||
|
||||
def test_combo_label_all():
|
||||
assert _combo_label(None, None, None) == "all-rooms/any-price"
|
||||
|
||||
|
||||
def test_combo_label_room_and_price():
|
||||
assert _combo_label("1", 5_000_000, 7_000_000) == "1/5M-7M"
|
||||
|
||||
|
||||
def test_combo_label_open_ended():
|
||||
assert _combo_label("4+", 25_000_000, None) == "4+/25M-inf"
|
||||
assert _combo_label("studio", None, 5_000_000) == "studio/0-5M"
|
||||
|
||||
|
||||
# ── PART D: T4 transport — async context manager + _http_get mock ───────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_aenter_creates_cffi_session():
|
||||
"""__aenter__ creates a _CurlCffiSession (Chrome120 impersonation)."""
|
||||
mock_settings = MagicMock()
|
||||
mock_settings.yandex_cookies_file = None
|
||||
|
||||
with patch("app.services.scrapers.yandex_realty._CurlCffiSession") as mock_cls:
|
||||
mock_session = AsyncMock()
|
||||
mock_cls.return_value = mock_session
|
||||
with patch("app.services.scrapers.yandex_realty._load_cookies_from_file", return_value={}):
|
||||
# Patch settings import inside __aenter__ without triggering Settings()
|
||||
with patch.dict("sys.modules", {"app.core.config": MagicMock(settings=mock_settings)}):
|
||||
s = YandexRealtyScraper()
|
||||
result = await s.__aenter__()
|
||||
assert result is s
|
||||
mock_cls.assert_called_once()
|
||||
call_kwargs = mock_cls.call_args
|
||||
assert call_kwargs.kwargs.get("impersonate") == "chrome120"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_aexit_closes_session():
|
||||
"""__aexit__ closes the curl_cffi session and sets it to None."""
|
||||
s = YandexRealtyScraper()
|
||||
mock_session = AsyncMock()
|
||||
s._cffi_session = mock_session
|
||||
await s.__aexit__(None, None, None)
|
||||
mock_session.close.assert_awaited_once()
|
||||
assert s._cffi_session is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_http_get_raises_when_no_context():
|
||||
"""_http_get must raise RuntimeError when called outside async context manager."""
|
||||
s = YandexRealtyScraper()
|
||||
with pytest.raises(RuntimeError, match="async context manager"):
|
||||
await s._http_get("https://realty.yandex.ru/test/")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_around_returns_empty_on_captcha():
|
||||
"""fetch_around returns [] and logs warning when captcha detected."""
|
||||
s = YandexRealtyScraper()
|
||||
mock_resp = MagicMock()
|
||||
mock_resp.status_code = 200
|
||||
mock_resp.text = (
|
||||
"<html><body>Докажите, что вы не робот. "
|
||||
'<a href="/showcaptcha">пройдите проверку</a></body></html>'
|
||||
)
|
||||
s._cffi_session = AsyncMock()
|
||||
s._cffi_session.get = AsyncMock(return_value=mock_resp)
|
||||
|
||||
with patch.object(s, "sleep_between_requests", new_callable=AsyncMock):
|
||||
lots = await s.fetch_around(lat=0.0, lon=0.0)
|
||||
assert lots == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_around_multi_room_dedup():
|
||||
"""fetch_around_multi_room deduplicates offers seen in multiple combos."""
|
||||
s = YandexRealtyScraper()
|
||||
|
||||
# Two combos both return the same offer_id "1234" plus unique ids
|
||||
combo_results: list[list[ScrapedLot]] = []
|
||||
for i in range(2):
|
||||
lot_shared = MagicMock(spec=ScrapedLot)
|
||||
lot_shared.source_id = "shared_offer_1234"
|
||||
lot_shared.source_url = "https://realty.yandex.ru/offer/shared_offer_1234/"
|
||||
|
||||
lot_unique = MagicMock(spec=ScrapedLot)
|
||||
lot_unique.source_id = f"unique_offer_{i}"
|
||||
lot_unique.source_url = f"https://realty.yandex.ru/offer/unique_offer_{i}/"
|
||||
|
||||
combo_results.append([lot_shared, lot_unique])
|
||||
|
||||
call_count = 0
|
||||
|
||||
async def fake_fetch_around(*args, **kwargs):
|
||||
nonlocal call_count
|
||||
idx = call_count
|
||||
call_count += 1
|
||||
if idx < len(combo_results):
|
||||
return combo_results[idx]
|
||||
return []
|
||||
|
||||
with patch.object(s, "fetch_around", side_effect=fake_fetch_around):
|
||||
lots = await s.fetch_around_multi_room(
|
||||
lat=0.0,
|
||||
lon=0.0,
|
||||
rooms_list=["1"],
|
||||
price_ranges=[(None, 5_000_000), (5_000_000, 7_000_000)],
|
||||
max_pages=1,
|
||||
)
|
||||
|
||||
# shared_offer_1234 appears once; 2 unique offers
|
||||
ids = {lot.source_id for lot in lots}
|
||||
assert "shared_offer_1234" in ids
|
||||
assert len(ids) == 3 # shared + 2 unique
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_around_multi_room_legacy_no_combos():
|
||||
"""Legacy mode (no rooms_list/price_ranges) uses single citywide sweep."""
|
||||
s = YandexRealtyScraper()
|
||||
calls: list[dict] = []
|
||||
|
||||
async def fake_fetch_around(
|
||||
lat, lon, radius_m=1000, page=0, rooms=None, price_min=None, price_max=None
|
||||
):
|
||||
calls.append({"page": page, "rooms": rooms, "price_min": price_min})
|
||||
return [] # empty → stops pagination
|
||||
|
||||
with patch.object(s, "fetch_around", side_effect=fake_fetch_around):
|
||||
await s.fetch_around_multi_room(lat=0.0, lon=0.0, max_pages=3)
|
||||
|
||||
# Should call with rooms=None, price_min=None (citywide)
|
||||
assert all(c["rooms"] is None for c in calls)
|
||||
assert all(c["price_min"] is None for c in calls)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue