lekss361
f8779c0812
All checks were successful
Deploy Trade-In / build-frontend (push) Has been skipped
Deploy Trade-In / test (push) Successful in 32s
Deploy Trade-In / build-backend (push) Successful in 51s
Deploy Trade-In / build-browser (push) Has been skipped
Deploy Trade-In / deploy (push) Successful in 42s
Deploy Trade-In / changes (push) Successful in 6s
881 lines
32 KiB
Python
881 lines
32 KiB
Python
"""Yandex.Nedvizhimost scraper (realty.yandex.ru) -- gate-API JSON parser.
|
|
|
|
Transport: system curl --compressed -x <socks5 proxy> (subprocess).
|
|
Proven to fetch gate-API payload in ~17s at 129 KB on prod (2026-06-17).
|
|
|
|
API path: GET https://realty.yandex.ru/gate/react-page/get/
|
|
_pageType=search&_providers=react-search-results-data
|
|
type=SELL&category=APARTMENT&newFlat=NO&rgid=559132 (EKB)
|
|
roomsTotal=<int|STUDIO>&priceMin=...&priceMax=...&page=<1-based>
|
|
|
|
Response shape: response.search.offers.{entities[], pager{page,pageSize,totalItems,totalPages}}
|
|
|
|
Page param is 1-based: page=1 -> first page (pager.page=0 in response).
|
|
page=0 -> API returns error. Paginate from page=1 to totalPages (inclusive).
|
|
|
|
Studio roomsTotal:
|
|
Request: roomsTotal=STUDIO
|
|
Response entities: roomsTotal=None -> mapped to rooms=0 (existing convention).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import json
|
|
import logging
|
|
import math
|
|
from dataclasses import dataclass
|
|
from typing import Any
|
|
from urllib.parse import urlencode
|
|
|
|
from curl_cffi.requests import AsyncSession as _CurlCffiSession
|
|
|
|
from app.services.scraper_settings import get_scraper_delay
|
|
from app.services.scrapers.base import BaseScraper, ScrapedLot
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
DEFAULT_CITY = "ekaterinburg"
|
|
_GATE_MAX_PAGES_CAP = 50
|
|
_EKB_RGID = 559132
|
|
_GATE_URL = "https://realty.yandex.ru/gate/react-page/get/"
|
|
_CURL_UA = (
|
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
|
"(KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36"
|
|
)
|
|
_CURL_STATUS_MARKER = "\n###CURL_HTTP_STATUS:"
|
|
|
|
ROOM_GATE_PARAM: dict[str, str | int] = {
|
|
"studio": "STUDIO",
|
|
"1": 1,
|
|
"2": 2,
|
|
"3": 3,
|
|
"4+": 4,
|
|
}
|
|
|
|
ROOM_PATH: dict[str, str] = {
|
|
"studio": "studiya",
|
|
"1": "odnokomnatnaya",
|
|
"2": "dvuhkomnatnaya",
|
|
"3": "trehkomnatnaya",
|
|
"4+": "4-i-bolee",
|
|
}
|
|
|
|
DEFAULT_PRICE_RANGES: list[tuple[int | None, int | None]] = [
|
|
(None, 5_000_000),
|
|
(5_000_000, 7_000_000),
|
|
(7_000_000, 10_000_000),
|
|
(10_000_000, 15_000_000),
|
|
(15_000_000, 25_000_000),
|
|
(25_000_000, None),
|
|
]
|
|
|
|
_YANDEX_MAX_PRICE = 200_000_000
|
|
_YANDEX_MIN_BRACKET = 50_000
|
|
_GATE_PAGE_SIZE = 20
|
|
_YANDEX_TARPIT_MAX_RETRIES: int = 2
|
|
|
|
|
|
@dataclass
|
|
class _CurlResponse:
|
|
status_code: int
|
|
text: str
|
|
|
|
|
|
def _is_gate_error(payload: dict[str, Any]) -> bool:
|
|
"""Return True when gate-API returned an error object instead of search results."""
|
|
return "error" in payload or "response" not in payload
|
|
|
|
|
|
def _extract_gate_data(
|
|
payload: dict[str, Any],
|
|
) -> tuple[list[dict[str, Any]], dict[str, Any]] | None:
|
|
"""Extract (entities, pager) from valid gate-API response. None on missing path."""
|
|
try:
|
|
offers = payload["response"]["search"]["offers"]
|
|
entities: list[dict[str, Any]] = offers["entities"]
|
|
pager: dict[str, Any] = offers["pager"]
|
|
return entities, pager
|
|
except (KeyError, TypeError):
|
|
return None
|
|
|
|
|
|
def _parse_gate_json(payload: dict[str, Any], page_param: int = 1) -> list[ScrapedLot]:
|
|
"""Parse gate-API JSON payload into a list of ScrapedLot.
|
|
|
|
Verified field mapping (from .issdump/yandex_gate_api_sample.json + live prod 2026-06-17):
|
|
offerId -> source_id
|
|
url (//realty.yandex.ru/offer/) -> source_url (prepend "https:")
|
|
price.value -> price_rub
|
|
price.valuePerPart -> price_per_m2
|
|
area.value -> area_m2
|
|
livingSpace.value -> living_area_m2
|
|
kitchenSpace.value -> raw_payload["kitchen_area_m2"]
|
|
roomsTotal (int | None) -> rooms (None -> 0 for studio)
|
|
floorsOffered[0] -> floor
|
|
floorsTotal -> total_floors
|
|
building.builtYear -> year_built
|
|
building.buildingType -> house_type (raw: "MONOLIT"/"BRICK"/etc.)
|
|
ceilingHeight -> raw_payload["ceiling_height"]
|
|
location.point.latitude/longitude -> lat / lon
|
|
location.geocoderAddress -> address (full, with house number)
|
|
mainImages[] -> photo_urls (prepend "https:", up to 5)
|
|
building.siteId -> house_ext_id (JK linkage)
|
|
listing_segment = "vtorichka" (all requests use newFlat=NO)
|
|
"""
|
|
result = _extract_gate_data(payload)
|
|
if result is None:
|
|
logger.warning("yandex gate: response missing response.search.offers path")
|
|
return []
|
|
|
|
entities, _pager = result
|
|
lots: list[ScrapedLot] = []
|
|
for entity in entities:
|
|
lot = _entity_to_lot(entity, page_param=page_param)
|
|
if lot is not None:
|
|
lots.append(lot)
|
|
return lots
|
|
|
|
|
|
def _entity_to_lot(entity: dict[str, Any], page_param: int = 1) -> ScrapedLot | None:
|
|
"""Convert one gate-API entity dict to ScrapedLot. None on missing required fields."""
|
|
try:
|
|
offer_id = str(entity.get("offerId") or "")
|
|
if not offer_id:
|
|
return None
|
|
|
|
url_raw = entity.get("url") or ""
|
|
source_url = f"https:{url_raw}" if url_raw.startswith("//") else url_raw
|
|
if not source_url:
|
|
source_url = f"https://realty.yandex.ru/offer/{offer_id}/"
|
|
|
|
price_dict = entity.get("price") or {}
|
|
price_val = price_dict.get("value")
|
|
if not price_val or price_val <= 0:
|
|
return None
|
|
price_rub = int(price_val)
|
|
|
|
price_per_m2_raw = price_dict.get("valuePerPart")
|
|
price_per_m2 = int(price_per_m2_raw) if price_per_m2_raw else None
|
|
|
|
area_dict = entity.get("area") or {}
|
|
area_m2_raw = area_dict.get("value")
|
|
area_m2 = float(area_m2_raw) if area_m2_raw is not None else None
|
|
|
|
living_dict = entity.get("livingSpace") or {}
|
|
living_area_raw = living_dict.get("value")
|
|
living_area_m2 = float(living_area_raw) if living_area_raw is not None else None
|
|
|
|
kitchen_dict = entity.get("kitchenSpace") or {}
|
|
kitchen_area_raw = kitchen_dict.get("value")
|
|
|
|
rooms_raw = entity.get("roomsTotal")
|
|
if rooms_raw is None:
|
|
rooms = 0 # studio
|
|
else:
|
|
try:
|
|
rooms = int(rooms_raw)
|
|
except (TypeError, ValueError):
|
|
rooms = None
|
|
|
|
floors_offered = entity.get("floorsOffered") or []
|
|
floor = int(floors_offered[0]) if floors_offered else None
|
|
floors_total_raw = entity.get("floorsTotal")
|
|
total_floors = int(floors_total_raw) if floors_total_raw is not None else None
|
|
|
|
ceiling_height = entity.get("ceilingHeight")
|
|
|
|
building = entity.get("building") or {}
|
|
year_built_raw = building.get("builtYear")
|
|
year_built = int(year_built_raw) if year_built_raw is not None else None
|
|
house_type = building.get("buildingType")
|
|
|
|
site_id = building.get("siteId")
|
|
site_name = building.get("siteName")
|
|
house_source: str | None = None
|
|
house_ext_id: str | None = None
|
|
if site_id:
|
|
house_source = "yandex_realty_nb"
|
|
house_ext_id = str(site_id)
|
|
|
|
location = entity.get("location") or {}
|
|
point = location.get("point") or {}
|
|
lat_raw = point.get("latitude")
|
|
lon_raw = point.get("longitude")
|
|
lat = float(lat_raw) if lat_raw is not None else None
|
|
lon = float(lon_raw) if lon_raw is not None else None
|
|
|
|
address = location.get("geocoderAddress") or location.get("streetAddress") or None
|
|
|
|
photo_urls: list[str] = []
|
|
for img in entity.get("mainImages") or []:
|
|
if img:
|
|
photo_urls.append(f"https:{img}" if img.startswith("//") else img)
|
|
if len(photo_urls) >= 5:
|
|
break
|
|
|
|
return ScrapedLot(
|
|
source="yandex",
|
|
source_url=source_url,
|
|
source_id=offer_id,
|
|
address=address,
|
|
lat=lat,
|
|
lon=lon,
|
|
rooms=rooms,
|
|
area_m2=area_m2,
|
|
living_area_m2=living_area_m2,
|
|
floor=floor,
|
|
total_floors=total_floors,
|
|
year_built=year_built,
|
|
house_type=house_type,
|
|
price_rub=price_rub,
|
|
price_per_m2=price_per_m2,
|
|
photo_urls=photo_urls,
|
|
house_source=house_source,
|
|
house_ext_id=house_ext_id,
|
|
house_url=None,
|
|
listing_segment="vtorichka",
|
|
raw_payload={
|
|
"page_param": page_param,
|
|
"ceiling_height": ceiling_height,
|
|
"kitchen_area_m2": kitchen_area_raw,
|
|
"site_name": site_name,
|
|
"offer_id": offer_id,
|
|
},
|
|
)
|
|
except Exception:
|
|
logger.exception("yandex gate: entity parse failed offer_id=%s", entity.get("offerId"))
|
|
return None
|
|
|
|
|
|
class YandexRealtyScraper(BaseScraper):
|
|
name = "yandex"
|
|
base_url = "https://realty.yandex.ru"
|
|
request_delay_sec = 5.0
|
|
|
|
def __init__(self, city: str = DEFAULT_CITY) -> None:
|
|
super().__init__()
|
|
self.city = city
|
|
self.request_delay_sec = get_scraper_delay(self.name)
|
|
self._cffi_session: _CurlCffiSession | None = None
|
|
self._cookies: dict[str, str] = {}
|
|
|
|
async def __aenter__(self) -> YandexRealtyScraper: # type: ignore[override]
|
|
"""Create curl_cffi session as fallback transport (dev / non-socks5 proxy).
|
|
|
|
Primary transport: system curl subprocess over SOCKS5 (see _http_get).
|
|
gate-API does not require cookies or heavy anti-bot headers.
|
|
"""
|
|
from app.core.config import settings
|
|
|
|
_proxy_url = settings.yandex_proxy_url
|
|
_proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None
|
|
self._cffi_session = _CurlCffiSession(
|
|
impersonate="chrome120",
|
|
cookies={},
|
|
proxies=_proxies,
|
|
headers={
|
|
"Accept": "application/json",
|
|
"X-Requested-With": "XMLHttpRequest",
|
|
"Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8",
|
|
"Accept-Encoding": "gzip, deflate, br",
|
|
},
|
|
)
|
|
return self
|
|
|
|
async def __aexit__(self, *args: object) -> None: # type: ignore[override]
|
|
if self._cffi_session is not None:
|
|
await self._cffi_session.close()
|
|
self._cffi_session = None
|
|
|
|
async def _curl_subprocess_get(self, url: str, proxy: str, timeout: int) -> _CurlResponse:
|
|
"""Fetch via system curl --compressed -x <socks5 proxy>.
|
|
|
|
Appends HTTP status via -w marker and strips it from body.
|
|
curl exit 28 (timeout) -> status_code=0.
|
|
"""
|
|
args = [
|
|
"curl",
|
|
"-sS",
|
|
"-L",
|
|
"--compressed",
|
|
"-x",
|
|
proxy,
|
|
"--max-time",
|
|
str(timeout),
|
|
"-A",
|
|
_CURL_UA,
|
|
"-H",
|
|
"Accept: application/json",
|
|
"-H",
|
|
"X-Requested-With: XMLHttpRequest",
|
|
"-H",
|
|
"Accept-Language: ru-RU,ru;q=0.9,en;q=0.8",
|
|
"-w",
|
|
_CURL_STATUS_MARKER + "%{http_code}",
|
|
url,
|
|
]
|
|
try:
|
|
proc = await asyncio.create_subprocess_exec(
|
|
*args,
|
|
stdout=asyncio.subprocess.PIPE,
|
|
stderr=asyncio.subprocess.PIPE,
|
|
)
|
|
out, err = await asyncio.wait_for(proc.communicate(), timeout=timeout + 10)
|
|
except TimeoutError:
|
|
logger.warning("yandex curl: subprocess timeout url=%s", url)
|
|
return _CurlResponse(status_code=0, text="")
|
|
except Exception:
|
|
logger.exception("yandex curl: subprocess failed url=%s", url)
|
|
return _CurlResponse(status_code=0, text="")
|
|
|
|
body = out.decode("utf-8", "replace")
|
|
status = 0
|
|
marker = _CURL_STATUS_MARKER
|
|
idx = body.rfind(marker)
|
|
if idx != -1:
|
|
try:
|
|
status = int(body[idx + len(marker) :].strip() or "0")
|
|
except ValueError:
|
|
status = 0
|
|
body = body[:idx]
|
|
|
|
if proc.returncode == 28:
|
|
status = 0
|
|
elif proc.returncode not in (0, None) and status == 0:
|
|
logger.warning(
|
|
"yandex curl: rc=%s url=%s err=%s",
|
|
proc.returncode,
|
|
url,
|
|
err.decode("utf-8", "replace")[:200],
|
|
)
|
|
|
|
return _CurlResponse(status_code=status, text=body)
|
|
|
|
async def _http_get(self, url: str, **kwargs: object) -> object: # type: ignore[override]
|
|
"""Route GET to system curl (SOCKS5 proxy) or curl_cffi fallback."""
|
|
from app.core.config import settings
|
|
|
|
proxy = settings.yandex_proxy_url
|
|
timeout = int(kwargs.get("timeout", 60) or 60)
|
|
if proxy and proxy.startswith(("socks5://", "socks5h://")):
|
|
return await self._curl_subprocess_get(url, proxy, timeout)
|
|
|
|
if self._cffi_session is None:
|
|
raise RuntimeError("YandexRealtyScraper must be used as async context manager")
|
|
kwargs.setdefault("timeout", 60)
|
|
return await self._cffi_session.get(url, **kwargs)
|
|
|
|
async def _rotate_ip(self) -> bool:
|
|
"""Rotate mobile proxy IP via changeip URL. Returns True on success."""
|
|
from app.core.config import settings as _settings
|
|
|
|
rotate_url = _settings.yandex_proxy_rotate_url or _settings.avito_proxy_rotate_url
|
|
if not rotate_url:
|
|
return False
|
|
sep = "&" if "?" in rotate_url else "?"
|
|
try:
|
|
async with _CurlCffiSession(timeout=30) as rot:
|
|
await rot.get(f"{rotate_url}{sep}format=json")
|
|
await asyncio.sleep(9)
|
|
logger.info("yandex proxy: IP rotated via changeip")
|
|
return True
|
|
except Exception:
|
|
logger.warning("yandex proxy: IP rotation failed", exc_info=True)
|
|
return False
|
|
|
|
def _build_url(
|
|
self,
|
|
page: int = 1,
|
|
rooms: str | None = None,
|
|
price_min: int | None = None,
|
|
price_max: int | None = None,
|
|
) -> str:
|
|
"""Build gate-API URL. page: 1-based (page=0 -> API error)."""
|
|
params: dict[str, str | int] = {
|
|
"rgid": _EKB_RGID,
|
|
"type": "SELL",
|
|
"category": "APARTMENT",
|
|
"newFlat": "NO",
|
|
"_pageType": "search",
|
|
"_providers": "react-search-results-data",
|
|
"page": page,
|
|
}
|
|
if rooms and rooms in ROOM_GATE_PARAM:
|
|
params["roomsTotal"] = ROOM_GATE_PARAM[rooms]
|
|
if price_min is not None:
|
|
params["priceMin"] = price_min
|
|
if price_max is not None:
|
|
params["priceMax"] = price_max
|
|
return f"{_GATE_URL}?{urlencode(params)}"
|
|
|
|
async def _fetch_page_json(
|
|
self,
|
|
rooms: str | None,
|
|
page: int,
|
|
price_min: int | None,
|
|
price_max: int | None,
|
|
) -> dict[str, Any] | None:
|
|
"""Fetch one gate-API page. Returns parsed JSON dict or None on error/invalid."""
|
|
url = self._build_url(page=page, rooms=rooms, price_min=price_min, price_max=price_max)
|
|
try:
|
|
resp = await self._http_get(url, timeout=60)
|
|
except Exception:
|
|
logger.exception("yandex gate: GET failed url=%s", url)
|
|
return None
|
|
if resp.status_code != 200: # type: ignore[union-attr]
|
|
logger.warning("yandex gate: HTTP %d url=%s", resp.status_code, url) # type: ignore[union-attr]
|
|
return None
|
|
try:
|
|
payload: dict[str, Any] = json.loads(resp.text) # type: ignore[union-attr]
|
|
except (json.JSONDecodeError, ValueError):
|
|
logger.warning("yandex gate: JSON parse failed url=%s", url)
|
|
return None
|
|
if _is_gate_error(payload):
|
|
logger.warning("yandex gate: error response url=%s keys=%s", url, list(payload.keys()))
|
|
return None
|
|
return payload
|
|
|
|
async def fetch_around(
|
|
self,
|
|
lat: float,
|
|
lon: float,
|
|
radius_m: int = 1000,
|
|
page: int = 1,
|
|
rooms: str | None = None,
|
|
price_min: int | None = None,
|
|
price_max: int | None = None,
|
|
) -> list[ScrapedLot]:
|
|
"""Fetch ONE page of gate-API results. lat/lon/radius_m ignored (uses rgid).
|
|
|
|
page: 1-based. Tarpit resilience: status_code==0 or JSON error
|
|
-> rotate IP + retry up to _YANDEX_TARPIT_MAX_RETRIES times.
|
|
Non-tarpit failures (4xx/5xx) are not retried.
|
|
"""
|
|
url = self._build_url(page=page, rooms=rooms, price_min=price_min, price_max=price_max)
|
|
|
|
payload: dict[str, Any] | None = None
|
|
for attempt in range(1 + _YANDEX_TARPIT_MAX_RETRIES):
|
|
try:
|
|
response = await self._http_get(url, timeout=60)
|
|
except Exception:
|
|
logger.exception("yandex gate fetch failed: %s", url)
|
|
return []
|
|
|
|
status = response.status_code # type: ignore[union-attr]
|
|
|
|
if status == 0:
|
|
logger.warning(
|
|
"yandex gate: status=0 (tarpit?) rooms=%s page=%d"
|
|
" -- rotating IP + retry attempt %d",
|
|
rooms,
|
|
page,
|
|
attempt + 1,
|
|
)
|
|
await self._rotate_ip()
|
|
await asyncio.sleep(2)
|
|
continue
|
|
|
|
if status != 200:
|
|
logger.warning(
|
|
"yandex gate: HTTP %d rooms=%s page=%d url=%s", status, rooms, page, url
|
|
)
|
|
return []
|
|
|
|
try:
|
|
payload = json.loads(response.text) # type: ignore[union-attr]
|
|
except (json.JSONDecodeError, ValueError):
|
|
logger.warning(
|
|
"yandex gate: JSON parse failed rooms=%s page=%d -- rotating + retry %d",
|
|
rooms,
|
|
page,
|
|
attempt + 1,
|
|
)
|
|
await self._rotate_ip()
|
|
await asyncio.sleep(2)
|
|
continue
|
|
|
|
if _is_gate_error(payload):
|
|
logger.warning(
|
|
"yandex gate: error payload rooms=%s page=%d keys=%s -- rotating + retry %d",
|
|
rooms,
|
|
page,
|
|
list(payload.keys()),
|
|
attempt + 1,
|
|
)
|
|
await self._rotate_ip()
|
|
await asyncio.sleep(2)
|
|
payload = None
|
|
continue
|
|
|
|
break
|
|
|
|
if payload is None:
|
|
logger.warning(
|
|
"yandex gate: all retries exhausted rooms=%s page=%d -- returning empty",
|
|
rooms,
|
|
page,
|
|
)
|
|
return []
|
|
|
|
lots = _parse_gate_json(payload, page_param=page)
|
|
logger.info(
|
|
"yandex gate page=%d rooms=%s price=[%s-%s]: %d cards",
|
|
page,
|
|
rooms or "all",
|
|
price_min,
|
|
price_max,
|
|
len(lots),
|
|
)
|
|
await self.sleep_between_requests()
|
|
return lots
|
|
|
|
async def fetch_around_multi_room(
|
|
self,
|
|
lat: float,
|
|
lon: float,
|
|
radius_m: int = 1000,
|
|
max_pages: int = _GATE_MAX_PAGES_CAP,
|
|
rooms_list: list[str] | None = None,
|
|
price_ranges: list[tuple[int | None, int | None]] | None = None,
|
|
**_legacy_kwargs: Any,
|
|
) -> list[ScrapedLot]:
|
|
"""Fetch via rooms x price-range combos; paginate each combo to totalPages.
|
|
|
|
Pagination driven by pager.totalPages from first page response.
|
|
Each combo iterates page=1 .. min(totalPages, max_pages, _GATE_MAX_PAGES_CAP).
|
|
Deduplicates by source_id/source_url across all combos.
|
|
Legacy mode (rooms_list=None, price_ranges=None): single citywide sweep.
|
|
"""
|
|
seen: dict[str, ScrapedLot] = {}
|
|
|
|
if rooms_list is None and price_ranges is None:
|
|
combos: list[tuple[str | None, int | None, int | None]] = [(None, None, None)]
|
|
else:
|
|
r_list = rooms_list or list(ROOM_PATH.keys())
|
|
p_ranges = price_ranges or DEFAULT_PRICE_RANGES
|
|
combos = [(r, lo, hi) for r in r_list for lo, hi in p_ranges]
|
|
|
|
for rooms, price_min, price_max in combos:
|
|
combo_label = _combo_label(rooms, price_min, price_max)
|
|
total_pages: int | None = None
|
|
|
|
for page in range(1, max_pages + 1):
|
|
if page == 1:
|
|
payload_p1: dict[str, Any] | None = None
|
|
p1_url = self._build_url(
|
|
page=1, rooms=rooms, price_min=price_min, price_max=price_max
|
|
)
|
|
for _attempt in range(1 + _YANDEX_TARPIT_MAX_RETRIES):
|
|
try:
|
|
resp = await self._http_get(p1_url, timeout=60)
|
|
except Exception:
|
|
logger.exception("yandex gate combo fetch failed combo=%s", combo_label)
|
|
break
|
|
if resp.status_code == 0: # type: ignore[union-attr]
|
|
logger.warning(
|
|
"yandex gate: tarpit combo=%s page=1 -- rotating", combo_label
|
|
)
|
|
await self._rotate_ip()
|
|
await asyncio.sleep(2)
|
|
continue
|
|
if resp.status_code != 200: # type: ignore[union-attr]
|
|
logger.warning(
|
|
"yandex gate: HTTP %d combo=%s page=1",
|
|
resp.status_code,
|
|
combo_label, # type: ignore[union-attr]
|
|
)
|
|
break
|
|
try:
|
|
payload_p1 = json.loads(resp.text) # type: ignore[union-attr]
|
|
except (json.JSONDecodeError, ValueError):
|
|
logger.warning(
|
|
"yandex gate: JSON parse error combo=%s page=1", combo_label
|
|
)
|
|
payload_p1 = None
|
|
await self._rotate_ip()
|
|
await asyncio.sleep(2)
|
|
continue
|
|
if _is_gate_error(payload_p1):
|
|
logger.warning(
|
|
"yandex gate: error payload combo=%s page=1", combo_label
|
|
)
|
|
payload_p1 = None
|
|
break
|
|
break
|
|
|
|
if payload_p1 is None:
|
|
logger.debug(
|
|
"yandex gate combo [%s] page=1: failed -- skipping", combo_label
|
|
)
|
|
break
|
|
|
|
result = _extract_gate_data(payload_p1)
|
|
if result is None:
|
|
break
|
|
_entities_p1, pager_p1 = result
|
|
total_pages = min(
|
|
pager_p1.get("totalPages", 1),
|
|
max_pages,
|
|
_GATE_MAX_PAGES_CAP,
|
|
)
|
|
lots_p1 = _parse_gate_json(payload_p1, page_param=1)
|
|
if not lots_p1:
|
|
logger.debug(
|
|
"yandex gate combo [%s] page=1: empty -- stopping", combo_label
|
|
)
|
|
break
|
|
for lot in lots_p1:
|
|
key = lot.source_id or lot.source_url
|
|
if key and key not in seen:
|
|
seen[key] = lot
|
|
await self.sleep_between_requests()
|
|
if total_pages <= 1:
|
|
break
|
|
continue
|
|
|
|
if total_pages is not None and page > total_pages:
|
|
break
|
|
|
|
lots = await self.fetch_around(
|
|
lat,
|
|
lon,
|
|
radius_m,
|
|
page=page,
|
|
rooms=rooms,
|
|
price_min=price_min,
|
|
price_max=price_max,
|
|
)
|
|
if not lots:
|
|
logger.debug(
|
|
"yandex gate combo [%s] page=%d: empty -- stopping",
|
|
combo_label,
|
|
page,
|
|
)
|
|
break
|
|
for lot in lots:
|
|
key = lot.source_id or lot.source_url
|
|
if key and key not in seen:
|
|
seen[key] = lot
|
|
|
|
logger.info(
|
|
"yandex gate aggregate: %d unique lots (%d combos)",
|
|
len(seen),
|
|
len(combos),
|
|
)
|
|
return list(seen.values())
|
|
|
|
async def fetch_all_secondary(
|
|
self,
|
|
*,
|
|
rooms_buckets: list[str] | None = None,
|
|
price_cap_per_bucket: int = 500,
|
|
max_pages_per_bucket: int = _GATE_MAX_PAGES_CAP,
|
|
concurrency: int = 4,
|
|
on_bucket: Any = None,
|
|
on_progress: Any = None,
|
|
skip_buckets: set[str] | None = None,
|
|
) -> list[ScrapedLot]:
|
|
"""Exhaustive EKB vtorichka load via KOMNATNOST x TSENA partitioning.
|
|
|
|
Uses gate-API JSON (totalItems from pager) instead of DOM state extraction.
|
|
"""
|
|
_buckets = rooms_buckets if rooms_buckets is not None else list(ROOM_PATH.keys())
|
|
seen: dict[str, ScrapedLot] = {}
|
|
|
|
for rooms_key in _buckets:
|
|
logger.info(
|
|
"yandex gate exhaustive: starting rooms=%s (price_cap=%d concurrency=%d)",
|
|
rooms_key,
|
|
price_cap_per_bucket,
|
|
concurrency,
|
|
)
|
|
before = len(seen)
|
|
await self._walk_price_range(
|
|
rooms=rooms_key,
|
|
lo=0,
|
|
hi=_YANDEX_MAX_PRICE,
|
|
seen=seen,
|
|
price_cap_per_bucket=price_cap_per_bucket,
|
|
max_pages_per_bucket=max_pages_per_bucket,
|
|
concurrency=concurrency,
|
|
on_bucket=on_bucket,
|
|
skip_buckets=skip_buckets,
|
|
)
|
|
room_collected = len(seen) - before
|
|
logger.info(
|
|
"yandex gate exhaustive: rooms=%s done -- collected %d (total unique=%d)",
|
|
rooms_key,
|
|
room_collected,
|
|
len(seen),
|
|
)
|
|
if on_progress is not None:
|
|
on_progress(len(seen))
|
|
|
|
logger.info("yandex gate exhaustive: DONE -- total unique=%d lots", len(seen))
|
|
return list(seen.values())
|
|
|
|
async def _walk_price_range(
|
|
self,
|
|
*,
|
|
rooms: str | None,
|
|
lo: int,
|
|
hi: int,
|
|
seen: dict[str, ScrapedLot],
|
|
price_cap_per_bucket: int,
|
|
max_pages_per_bucket: int,
|
|
concurrency: int = 4,
|
|
on_bucket: Any = None,
|
|
skip_buckets: set[str] | None = None,
|
|
_depth: int = 0,
|
|
) -> None:
|
|
"""Recursive binary price-range partitioning.
|
|
|
|
Probe page=1 -> read totalItems from pager.
|
|
totalItems > cap and bracket >= MIN_BRACKET -> split; else paginate leaf.
|
|
Fallback: totalItems=None -> paginate-until-empty (safe degradation).
|
|
skip_buckets: resume checkpoint -- skip leaf buckets already done.
|
|
"""
|
|
_lo_param = lo if lo > 0 else None
|
|
|
|
probe_payload = await self._fetch_page_json(rooms, 1, _lo_param, hi)
|
|
await asyncio.sleep(self.request_delay_sec)
|
|
|
|
total: int | None = None
|
|
probe_lots: list[ScrapedLot] = []
|
|
|
|
if probe_payload is None:
|
|
logger.warning(
|
|
"yandex gate: probe failed rooms=%s [%d, %d] depth=%d -- rotating + retry",
|
|
rooms,
|
|
lo,
|
|
hi,
|
|
_depth,
|
|
)
|
|
rotated = await self._rotate_ip()
|
|
if rotated:
|
|
probe_payload = await self._fetch_page_json(rooms, 1, _lo_param, hi)
|
|
await asyncio.sleep(self.request_delay_sec)
|
|
|
|
if probe_payload is not None:
|
|
result = _extract_gate_data(probe_payload)
|
|
if result is not None:
|
|
_, pager = result
|
|
total = pager.get("totalItems")
|
|
probe_lots = _parse_gate_json(probe_payload, page_param=1)
|
|
else:
|
|
logger.warning(
|
|
"yandex gate: probe data extract failed rooms=%s [%d, %d]", rooms, lo, hi
|
|
)
|
|
|
|
if total is None:
|
|
# Degraded path: paginate-until-empty without knowing total
|
|
logger.warning(
|
|
"yandex gate: totalItems unknown rooms=%s [%d, %d] -- paginate until empty",
|
|
rooms,
|
|
lo,
|
|
hi,
|
|
)
|
|
page = 1
|
|
bucket_key = _combo_label(rooms, lo, hi)
|
|
if skip_buckets and bucket_key in skip_buckets:
|
|
return
|
|
pages_fetched = 0
|
|
while pages_fetched < max_pages_per_bucket:
|
|
payload = await self._fetch_page_json(rooms, page, _lo_param, hi)
|
|
await asyncio.sleep(self.request_delay_sec)
|
|
if payload is None:
|
|
break
|
|
lots = _parse_gate_json(payload, page_param=page)
|
|
if not lots:
|
|
break
|
|
for lot in lots:
|
|
if lot.source_id and lot.source_id not in seen:
|
|
seen[lot.source_id] = lot
|
|
page += 1
|
|
pages_fetched += 1
|
|
if on_bucket is not None:
|
|
on_bucket(bucket_key, len(seen))
|
|
return
|
|
|
|
# Split or paginate
|
|
cap = price_cap_per_bucket
|
|
_min_bracket = 500_000
|
|
bracket = hi - lo
|
|
|
|
if total > cap and bracket >= _min_bracket and _depth < 8:
|
|
mid = (lo + hi) // 2
|
|
await self._walk_price_range(
|
|
rooms=rooms,
|
|
lo=lo,
|
|
hi=mid,
|
|
seen=seen,
|
|
price_cap_per_bucket=cap,
|
|
max_pages_per_bucket=max_pages_per_bucket,
|
|
concurrency=concurrency,
|
|
on_bucket=on_bucket,
|
|
skip_buckets=skip_buckets,
|
|
_depth=_depth + 1,
|
|
)
|
|
await self._walk_price_range(
|
|
rooms=rooms,
|
|
lo=mid + 1,
|
|
hi=hi,
|
|
seen=seen,
|
|
price_cap_per_bucket=cap,
|
|
max_pages_per_bucket=max_pages_per_bucket,
|
|
concurrency=concurrency,
|
|
on_bucket=on_bucket,
|
|
skip_buckets=skip_buckets,
|
|
_depth=_depth + 1,
|
|
)
|
|
return
|
|
|
|
# Leaf bucket: paginate
|
|
bucket_key = _combo_label(rooms, lo, hi)
|
|
if skip_buckets and bucket_key in skip_buckets:
|
|
logger.debug("yandex gate: skip bucket %s (checkpoint)", bucket_key)
|
|
return
|
|
|
|
total_pages = min(
|
|
math.ceil(total / 20),
|
|
_GATE_MAX_PAGES_CAP,
|
|
max_pages_per_bucket,
|
|
)
|
|
logger.info("yandex gate: leaf bucket %s total=%d pages=%d", bucket_key, total, total_pages)
|
|
|
|
# Add probe lots (page 1 already fetched)
|
|
for lot in probe_lots:
|
|
if lot.source_id and lot.source_id not in seen:
|
|
seen[lot.source_id] = lot
|
|
|
|
if total_pages <= 1:
|
|
if on_bucket is not None:
|
|
on_bucket(bucket_key, len(seen))
|
|
return
|
|
|
|
# Paginate pages 2..total_pages with concurrency
|
|
sem = asyncio.Semaphore(concurrency)
|
|
|
|
async def _fetch_leaf_page(pg: int) -> list[ScrapedLot]:
|
|
async with sem:
|
|
payload = await self._fetch_page_json(rooms, pg, _lo_param, hi)
|
|
await asyncio.sleep(self.request_delay_sec)
|
|
if payload is None:
|
|
return []
|
|
return _parse_gate_json(payload, page_param=pg)
|
|
|
|
tasks = [_fetch_leaf_page(pg) for pg in range(2, total_pages + 1)]
|
|
results = await asyncio.gather(*tasks)
|
|
for page_lots in results:
|
|
for lot in page_lots:
|
|
if lot.source_id and lot.source_id not in seen:
|
|
seen[lot.source_id] = lot
|
|
|
|
if on_bucket is not None:
|
|
on_bucket(bucket_key, len(seen))
|
|
|
|
|
|
def _combo_label(rooms: str | None, lo: int | None, hi: int | None) -> str:
|
|
"""Уникальная метка бакета для checkpoint-логики."""
|
|
return f"{rooms or 'any'}:{lo}-{hi}"
|