376 lines
14 KiB
Python
376 lines
14 KiB
Python
"""Yandex.Недвижимость ЕКБ vtorichka sweep — runs locally через residential IP.
|
||
|
||
Why local: Yandex anti-bot (captcha redirect) включается быстрее на datacenter
|
||
IP + httpx fingerprint. Запуск с residential IP + Chrome120 TLS impersonation
|
||
(curl_cffi) + browser session cookies — проходит. Скрипт пишет в prod tradein
|
||
DB через SSH tunnel (localhost:15433 → tradein-postgres docker container).
|
||
|
||
Modes:
|
||
--mode citywide (default): один общий /vtorichniy-rynok/ sweep, ~25 страниц
|
||
× 23 cards = ~575 уникальных. Yandex SERP лимит.
|
||
--mode combos: итерация 5 rooms × 6 price ranges = 30 combos × до 25 pages.
|
||
DB-level дедуп → coverage близко к total (~4000+ vtorichka
|
||
ЕКБ). ~75 min полный прогон.
|
||
|
||
Setup:
|
||
ssh -f -N -L 15433:172.20.0.2:5432 gendesign # tradein-postgres tunnel
|
||
cd tradein-mvp/backend
|
||
export DATABASE_URL='postgresql+psycopg://tradein:<PASS>@localhost:15433/tradein'
|
||
export ENVIRONMENT='local'
|
||
export GLITCHTIP_DSN=''
|
||
|
||
Usage:
|
||
uv run python ../scripts/local-sweep-ekb-yandex.py # citywide
|
||
uv run python ../scripts/local-sweep-ekb-yandex.py --mode combos # full ~4000
|
||
uv run python ../scripts/local-sweep-ekb-yandex.py --mode combos \\
|
||
--rooms 1,2 --price-ranges 5-7,7-10 # subset
|
||
uv run python ../scripts/local-sweep-ekb-yandex.py --pages 1 # smoke
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
# stdlib
|
||
import argparse
|
||
import asyncio
|
||
import json
|
||
import random
|
||
import sys
|
||
import time
|
||
import types
|
||
from pathlib import Path
|
||
from urllib.parse import urlencode
|
||
|
||
# Auto-inject backend/ в sys.path чтобы найти app.* package
|
||
_BACKEND_DIR = Path(__file__).resolve().parent.parent / "backend"
|
||
if str(_BACKEND_DIR) not in sys.path:
|
||
sys.path.insert(0, str(_BACKEND_DIR))
|
||
|
||
# Stub weasyprint (Windows без GTK ломает import chain через trade_in.py)
|
||
for _m in (
|
||
"weasyprint",
|
||
"weasyprint.text",
|
||
"weasyprint.text.ffi",
|
||
"weasyprint.css",
|
||
"weasyprint.css.targets",
|
||
):
|
||
sys.modules[_m] = types.ModuleType(_m)
|
||
sys.modules["weasyprint"].HTML = type("HTML", (), {"__init__": lambda self, *a, **kw: None})
|
||
sys.modules["weasyprint"].CSS = type("CSS", (), {"__init__": lambda self, *a, **kw: None})
|
||
|
||
# app imports после stubs
|
||
from curl_cffi.requests import AsyncSession # noqa: E402
|
||
|
||
from app.core.db import SessionLocal # noqa: E402
|
||
from app.services.scrapers.base import save_listings # noqa: E402
|
||
from app.services.scrapers.yandex_realty import YandexRealtyScraper # noqa: E402
|
||
|
||
DEFAULT_CITY = "ekaterinburg"
|
||
DEFAULT_COOKIES_FILE = Path(__file__).resolve().parent / ".yandex-cookies.json"
|
||
|
||
# Path-сегменты Yandex для room buckets (после канонизации). Ключ — slug для
|
||
# --rooms CLI; значение — segment в URL.
|
||
ROOM_PATH: dict[str, str] = {
|
||
"studio": "studiya",
|
||
"1": "odnokomnatnaya",
|
||
"2": "dvuhkomnatnaya",
|
||
"3": "trehkomnatnaya",
|
||
"4+": "4-i-bolee",
|
||
}
|
||
|
||
# Price ranges (price_min, price_max) в рублях. None — open-ended boundary.
|
||
PRICE_RANGES: list[tuple[int | None, int | None]] = [
|
||
(None, 5_000_000),
|
||
(5_000_000, 7_000_000),
|
||
(7_000_000, 10_000_000),
|
||
(10_000_000, 15_000_000),
|
||
(15_000_000, 25_000_000),
|
||
(25_000_000, None),
|
||
]
|
||
|
||
_CAPTCHA_MARKERS = ("/showcaptcha", "smartcaptcha", "checkboxcaptcha")
|
||
|
||
|
||
def _load_cookies(path: Path) -> dict[str, str]:
|
||
if not path.exists():
|
||
return {}
|
||
raw = json.loads(path.read_text(encoding="utf-8"))
|
||
return {item["name"]: item["value"] for item in raw if "name" in item and "value" in item}
|
||
|
||
|
||
def _is_captcha(html: str) -> bool:
|
||
head = html[:5000].lower()
|
||
return any(m in head for m in _CAPTCHA_MARKERS) or "докажите, что вы не робот" in head
|
||
|
||
|
||
def _build_url(
|
||
city: str,
|
||
page: int = 0,
|
||
rooms: str | None = None,
|
||
price_min: int | None = None,
|
||
price_max: int | None = None,
|
||
) -> str:
|
||
"""Yandex SERP URL. Если rooms задан — path-based бакет; иначе общий vtorichka.
|
||
|
||
newFlat=NO_DEAL принудительно отфильтровывает новостройки (нужно при
|
||
combo, т.к. Yandex canonicalize иногда дропает /vtorichniy-rynok/).
|
||
"""
|
||
if rooms:
|
||
path = f"/{city}/kupit/kvartira/{ROOM_PATH[rooms]}/vtorichniy-rynok/"
|
||
else:
|
||
path = f"/{city}/kupit/kvartira/vtorichniy-rynok/"
|
||
base = f"https://realty.yandex.ru{path}"
|
||
|
||
params: dict[str, str | int] = {}
|
||
if price_min is not None:
|
||
params["priceMin"] = price_min
|
||
if price_max is not None:
|
||
params["priceMax"] = price_max
|
||
if price_min is not None or price_max is not None:
|
||
# Гарантия vtorichka — Yandex иногда дропает /vtorichniy-rynok/ при canonicalize
|
||
params["newFlat"] = "NO_DEAL"
|
||
if page > 0:
|
||
params["page"] = page
|
||
|
||
if params:
|
||
return f"{base}?{urlencode(params)}"
|
||
return base
|
||
|
||
|
||
def _combo_label(rooms: str | None, price_min: int | None, price_max: int | None) -> str:
|
||
r = rooms or "all-rooms"
|
||
if price_min is None and price_max is None:
|
||
return f"{r}/any-price"
|
||
lo = f"{price_min // 1_000_000}M" if price_min else "0"
|
||
hi = f"{price_max // 1_000_000}M" if price_max else "inf"
|
||
return f"{r}/{lo}-{hi}"
|
||
|
||
|
||
async def sweep_one(
|
||
session: AsyncSession,
|
||
parser: YandexRealtyScraper,
|
||
city: str,
|
||
pages: int,
|
||
delay: float,
|
||
rooms: str | None = None,
|
||
price_min: int | None = None,
|
||
price_max: int | None = None,
|
||
) -> tuple[int, int, int]:
|
||
"""Один scan по фиксированному (rooms, price) — пагинация до pages или empty.
|
||
|
||
Returns: (fetched, inserted, updated)
|
||
"""
|
||
label = _combo_label(rooms, price_min, price_max)
|
||
total = cum_ins = cum_upd = 0
|
||
for page in range(pages):
|
||
url = _build_url(city, page=page, rooms=rooms, price_min=price_min, price_max=price_max)
|
||
try:
|
||
resp = await session.get(url, timeout=20)
|
||
except Exception as e: # noqa: BLE001
|
||
print(f" [{label}] page={page} FAIL fetch: {type(e).__name__}: {e}", flush=True)
|
||
await asyncio.sleep(delay)
|
||
continue
|
||
if resp.status_code != 200:
|
||
print(f" [{label}] page={page} HTTP {resp.status_code} — stop combo", flush=True)
|
||
break
|
||
if _is_captcha(resp.text):
|
||
print(
|
||
f" [{label}] page={page} ⚠️ CAPTCHA — stop combo. "
|
||
f"Подожди 30-60min или refresh .yandex-cookies.json",
|
||
flush=True,
|
||
)
|
||
raise RuntimeError("captcha")
|
||
# NBSP → space: yandex_helpers.parse_rub() багует на \xa0 в group(1) → price None
|
||
html = resp.text.replace("\xa0", " ")
|
||
lots = parser._parse_html(html, page=page)
|
||
if not lots:
|
||
print(f" [{label}] page={page}: 0 lots — end of pagination, stop", flush=True)
|
||
break
|
||
db = SessionLocal()
|
||
try:
|
||
ins, upd = save_listings(db, lots)
|
||
cum_ins += ins
|
||
cum_upd += upd
|
||
total += len(lots)
|
||
print(
|
||
f" [{label}] page={page + 1}/{pages}: {len(lots)} lots → "
|
||
f"ins={ins} upd={upd} [combo: {total}/{cum_ins}/{cum_upd}]",
|
||
flush=True,
|
||
)
|
||
except Exception as e: # noqa: BLE001
|
||
print(f" [{label}] page={page} FAIL save: {type(e).__name__}: {e}", flush=True)
|
||
finally:
|
||
db.close()
|
||
if page + 1 < pages:
|
||
await asyncio.sleep(delay * random.uniform(0.85, 1.3))
|
||
return total, cum_ins, cum_upd
|
||
|
||
|
||
async def phase_citywide(pages: int, delay: float, city: str, cookies: dict[str, str]) -> None:
|
||
print(
|
||
f"\n=== PHASE CITYWIDE: Yandex vtorichka (city={city}, {pages} pages, "
|
||
f"delay {delay}s) ===\n",
|
||
flush=True,
|
||
)
|
||
parser = YandexRealtyScraper(city=city)
|
||
async with AsyncSession(impersonate="chrome120", cookies=cookies or {}) as session:
|
||
total, ins, upd = await sweep_one(session, parser, city, pages, delay)
|
||
print(f" CITYWIDE TOTAL: fetched={total} ins={ins} upd={upd}", flush=True)
|
||
|
||
|
||
async def phase_combos(
|
||
pages: int,
|
||
delay: float,
|
||
city: str,
|
||
cookies: dict[str, str],
|
||
rooms_list: list[str],
|
||
price_ranges: list[tuple[int | None, int | None]],
|
||
combo_delay: float,
|
||
) -> None:
|
||
combos = [(r, lo, hi) for r in rooms_list for lo, hi in price_ranges]
|
||
print(
|
||
f"\n=== PHASE COMBOS: {len(combos)} combos × ≤{pages} pages "
|
||
f"(city={city}, delay {delay}s, combo-delay {combo_delay}s) ===",
|
||
flush=True,
|
||
)
|
||
parser = YandexRealtyScraper(city=city)
|
||
grand_fetched = grand_ins = grand_upd = 0
|
||
async with AsyncSession(impersonate="chrome120", cookies=cookies or {}) as session:
|
||
for i, (rooms, lo, hi) in enumerate(combos, 1):
|
||
label = _combo_label(rooms, lo, hi)
|
||
print(f"\n [{i}/{len(combos)}] combo={label}", flush=True)
|
||
try:
|
||
f, ins, upd = await sweep_one(
|
||
session, parser, city, pages, delay,
|
||
rooms=rooms, price_min=lo, price_max=hi,
|
||
)
|
||
except RuntimeError as e:
|
||
if str(e) == "captcha":
|
||
print(" ⚠️ stopping combo run — captcha", flush=True)
|
||
break
|
||
raise
|
||
grand_fetched += f
|
||
grand_ins += ins
|
||
grand_upd += upd
|
||
print(
|
||
f" [{i}/{len(combos)}] {label} done: f={f} i={ins} u={upd} "
|
||
f"[grand: f={grand_fetched} i={grand_ins} u={grand_upd}]",
|
||
flush=True,
|
||
)
|
||
if i < len(combos):
|
||
await asyncio.sleep(combo_delay * random.uniform(0.85, 1.3))
|
||
print(
|
||
f"\n COMBOS GRAND TOTAL: fetched={grand_fetched} "
|
||
f"ins={grand_ins} upd={grand_upd}",
|
||
flush=True,
|
||
)
|
||
|
||
|
||
def _parse_rooms_arg(s: str) -> list[str]:
|
||
if s == "all":
|
||
return list(ROOM_PATH)
|
||
out: list[str] = []
|
||
for tok in s.split(","):
|
||
tok = tok.strip()
|
||
if tok not in ROOM_PATH:
|
||
raise argparse.ArgumentTypeError(
|
||
f"unknown room '{tok}'. Allowed: {sorted(ROOM_PATH)} or 'all'"
|
||
)
|
||
out.append(tok)
|
||
return out
|
||
|
||
|
||
def _parse_price_ranges_arg(s: str) -> list[tuple[int | None, int | None]]:
|
||
if s == "all":
|
||
return list(PRICE_RANGES)
|
||
out: list[tuple[int | None, int | None]] = []
|
||
for tok in s.split(","):
|
||
tok = tok.strip()
|
||
if "-" not in tok:
|
||
raise argparse.ArgumentTypeError(f"range '{tok}' missing '-'")
|
||
lo_s, hi_s = tok.split("-", 1)
|
||
lo = int(float(lo_s) * 1_000_000) if lo_s else None
|
||
hi = int(float(hi_s) * 1_000_000) if hi_s else None
|
||
out.append((lo, hi))
|
||
return out
|
||
|
||
|
||
async def main() -> None:
|
||
p = argparse.ArgumentParser(description="Yandex.Недвижимость ЕКБ sweep")
|
||
p.add_argument(
|
||
"--mode",
|
||
choices=["citywide", "combos"],
|
||
default="citywide",
|
||
help="citywide = один общий vtorichka sweep (~575 уникальных). "
|
||
"combos = rooms × price = ~4000+",
|
||
)
|
||
p.add_argument(
|
||
"--pages",
|
||
type=int,
|
||
default=25,
|
||
help="max pages per sweep (Yandex SERP cap ≈ 25; stops earlier on empty)",
|
||
)
|
||
p.add_argument(
|
||
"--delay",
|
||
type=float,
|
||
default=15,
|
||
help="sec между HTTP requests внутри combo (default 15, residential-safe)",
|
||
)
|
||
p.add_argument(
|
||
"--combo-delay",
|
||
type=float,
|
||
default=20,
|
||
help="sec между combos (default 20, дополнительный jitter combos mode)",
|
||
)
|
||
p.add_argument(
|
||
"--rooms",
|
||
type=_parse_rooms_arg,
|
||
default="all",
|
||
help=f"combos: comma-list из {sorted(ROOM_PATH)} или 'all' (default)",
|
||
)
|
||
p.add_argument(
|
||
"--price-ranges",
|
||
type=_parse_price_ranges_arg,
|
||
default="all",
|
||
help="combos: comma-list 'lo-hi' в млн руб ('-5,5-7,7-10,10-15,15-25,25-') "
|
||
"или 'all' (default). Empty side = open-ended.",
|
||
)
|
||
p.add_argument(
|
||
"--city",
|
||
type=str,
|
||
default=DEFAULT_CITY,
|
||
help=f"Yandex city slug (default '{DEFAULT_CITY}')",
|
||
)
|
||
p.add_argument(
|
||
"--cookies-file",
|
||
type=Path,
|
||
default=DEFAULT_COOKIES_FILE,
|
||
help=f"Browser cookies JSON (default '{DEFAULT_COOKIES_FILE.name}'). "
|
||
f"Empty/missing → start без cookies (риск captcha).",
|
||
)
|
||
args = p.parse_args()
|
||
|
||
cookies = _load_cookies(args.cookies_file)
|
||
if cookies:
|
||
print(f"loaded {len(cookies)} cookies from {args.cookies_file}", flush=True)
|
||
else:
|
||
print(
|
||
f"⚠️ no cookies loaded ({args.cookies_file}, exists={args.cookies_file.exists()}). "
|
||
f"Yandex anti-bot risk.",
|
||
flush=True,
|
||
)
|
||
|
||
t0 = time.perf_counter()
|
||
if args.mode == "citywide":
|
||
await phase_citywide(args.pages, args.delay, args.city, cookies)
|
||
else:
|
||
await phase_combos(
|
||
args.pages, args.delay, args.city, cookies,
|
||
rooms_list=args.rooms,
|
||
price_ranges=args.price_ranges,
|
||
combo_delay=args.combo_delay,
|
||
)
|
||
elapsed = time.perf_counter() - t0
|
||
print(f"\n=== ALL DONE in {elapsed / 60:.1f} min ===", flush=True)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
asyncio.run(main())
|