359 lines
15 KiB
Python
359 lines
15 KiB
Python
"""Cian ЕКБ vtorichka sweep — local residential IP → prod tradein DB через SSH tunnel.
|
||
|
||
Why: ekb.cian.ru SERP отдаёт ВЕСЬ город (без geo-фильтра по lat/lon), поэтому
|
||
"10 anchors" из Avito-эталона тут бесполезны — одинаковый ответ × 10.
|
||
Вместо anchors — proper room-segmented citywide pagination
|
||
(rooms 1/2/3/4 × N pages × ~28 listings), per-page save (visible progress +
|
||
не теряем данные при крэше).
|
||
|
||
Cian реально менее агрессивен чем Avito, но **residential IP всё равно может попасть
|
||
в blacklist при слишком частых запросах** — default delay 15s.
|
||
|
||
Setup (один раз):
|
||
ssh -f -N -L 15433:172.20.0.2:5432 gendesign
|
||
cd tradein-mvp/backend
|
||
$env:DATABASE_URL='postgresql+psycopg://tradein:<PASS>@localhost:15433/tradein'
|
||
$env:ENVIRONMENT='local'; $env:GLITCHTIP_DSN=''
|
||
|
||
Usage:
|
||
uv run python ../scripts/local-sweep-ekb-cian.py # default: 4 rooms × 8 pages
|
||
uv run python ../scripts/local-sweep-ekb-cian.py --pages 20 # больше глубины
|
||
uv run python ../scripts/local-sweep-ekb-cian.py --delay 10 # быстрее (risk)
|
||
uv run python ../scripts/local-sweep-ekb-cian.py --rooms 0 # всё (включая студии — no room param)
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
# stdlib
|
||
import argparse
|
||
import asyncio
|
||
import json
|
||
import sys
|
||
import time
|
||
import types
|
||
from pathlib import Path
|
||
|
||
# Auto-inject backend/ в sys.path чтобы найти app.* package
|
||
_BACKEND_DIR = Path(__file__).resolve().parent.parent / "backend"
|
||
if str(_BACKEND_DIR) not in sys.path:
|
||
sys.path.insert(0, str(_BACKEND_DIR))
|
||
|
||
# Stub weasyprint (Windows без GTK ломает import chain через trade_in.py)
|
||
for _m in (
|
||
"weasyprint",
|
||
"weasyprint.text",
|
||
"weasyprint.text.ffi",
|
||
"weasyprint.css",
|
||
"weasyprint.css.targets",
|
||
):
|
||
sys.modules[_m] = types.ModuleType(_m)
|
||
sys.modules["weasyprint"].HTML = type("HTML", (), {"__init__": lambda self, *a, **kw: None})
|
||
sys.modules["weasyprint"].CSS = type("CSS", (), {"__init__": lambda self, *a, **kw: None})
|
||
|
||
# app imports после stubs
|
||
import logging # noqa: E402
|
||
import re # noqa: E402
|
||
|
||
from app.core.db import SessionLocal # noqa: E402
|
||
from app.services.scrapers.base import ScrapedLot, save_listings # noqa: E402
|
||
from app.services.scrapers.cian import CianScraper # noqa: E402
|
||
|
||
# Шумный WARNING из upstream cian_state_parser ('state extraction failed') — у нас
|
||
# fallback на новый .concat() markup отрабатывает, warning нерелевантен.
|
||
logging.getLogger("app.services.scrapers.cian").setLevel(logging.ERROR)
|
||
|
||
# ── New-markup fallback parser ───────────────────────────────────────────────
|
||
# Cian переключил MFE bootstrap с .push({...}) на .concat([{...},{...}]):
|
||
# window._cianConfig['frontend-serp'] = (window._cianConfig['frontend-serp'] || []).concat([…])
|
||
# Существующий cian_state_parser.py не ловит этот шаблон → 0 lots на свежей выдаче.
|
||
# Чтобы не трогать shared scraper module (правилами запрещено), парсим тут.
|
||
_RE_CONCAT_START = re.compile(
|
||
r"window\._cianConfig\['(?P<mfe>[\w-]+)'\]\s*=\s*"
|
||
r"\(window\._cianConfig\['[\w-]+'\]\s*\|\|\s*\[\]\)\.concat\(\["
|
||
)
|
||
|
||
|
||
def _extract_balanced_array(text: str, start: int) -> str | None:
|
||
"""Из позиции `start` (где '[') вернуть текст массива включая внешние '[' и ']'.
|
||
|
||
JSON-aware: учитывает strings и escapes, чтобы не сбить счётчик '[' ']' на
|
||
скобках внутри string literals.
|
||
"""
|
||
if text[start] != "[":
|
||
return None
|
||
depth = 0
|
||
i = start
|
||
in_str = False
|
||
esc = False
|
||
while i < len(text):
|
||
ch = text[i]
|
||
if in_str:
|
||
if esc:
|
||
esc = False
|
||
elif ch == "\\":
|
||
esc = True
|
||
elif ch == '"':
|
||
in_str = False
|
||
else:
|
||
if ch == '"':
|
||
in_str = True
|
||
elif ch == "[":
|
||
depth += 1
|
||
elif ch == "]":
|
||
depth -= 1
|
||
if depth == 0:
|
||
return text[start : i + 1]
|
||
i += 1
|
||
return None
|
||
|
||
|
||
def _parse_serp_v2(scraper: CianScraper, html: str) -> list[ScrapedLot]:
|
||
"""Fallback парсер для нового шаблона .concat([...]) — ekb.cian.ru/cat.php c 2026-05."""
|
||
state: dict | None = None
|
||
for m in _RE_CONCAT_START.finditer(html):
|
||
if m.group("mfe") != "frontend-serp":
|
||
continue
|
||
# Позиция '[' — это последний символ матча минус 1 (regex кончается на '[')
|
||
arr_start = m.end() - 1
|
||
arr_str = _extract_balanced_array(html, arr_start)
|
||
if arr_str is None:
|
||
continue
|
||
try:
|
||
entries = json.loads(arr_str)
|
||
except json.JSONDecodeError:
|
||
continue
|
||
for entry in entries:
|
||
if (
|
||
isinstance(entry, dict)
|
||
and entry.get("key") == "initialState"
|
||
and isinstance(entry.get("value"), dict)
|
||
):
|
||
state = entry["value"]
|
||
break
|
||
if state is not None:
|
||
break
|
||
if state is None:
|
||
return []
|
||
offers = (state.get("results") or {}).get("offers") or []
|
||
lots: list[ScrapedLot] = []
|
||
for offer in offers:
|
||
# Cian свежий markup: descriptionMinhash = list[int] (multi-hash signatures).
|
||
# ScrapedLot ожидает str → сбрасываем чтобы _offer_to_lot вычислил SHA1 от description.
|
||
if isinstance(offer.get("descriptionMinhash"), list):
|
||
offer["descriptionMinhash"] = None
|
||
lot = scraper._offer_to_lot(offer)
|
||
if lot is not None:
|
||
lots.append(lot)
|
||
return lots
|
||
|
||
|
||
def _load_cookies_file(path: Path) -> dict[str, str]:
|
||
"""Загрузить cookies из JSON-файла.
|
||
|
||
Поддерживает два формата:
|
||
- browser export (list[{name,value,...}]) — DevTools / EditThisCookie / Cookie-Editor
|
||
- plain dict {name: value}
|
||
"""
|
||
raw = json.loads(path.read_text(encoding="utf-8"))
|
||
if isinstance(raw, list):
|
||
return {c["name"]: c["value"] for c in raw if c.get("name") and c.get("value")}
|
||
if isinstance(raw, dict):
|
||
return {str(k): str(v) for k, v in raw.items()}
|
||
raise SystemExit(f"--cookies-file: unknown JSON shape (got {type(raw).__name__})")
|
||
|
||
|
||
async def phase_citywide(
|
||
pages: int,
|
||
delay: float,
|
||
room_segments: tuple[tuple[int, ...] | None, ...],
|
||
cookies_file: Path | None = None,
|
||
start_page: int = 1,
|
||
) -> int:
|
||
"""Room-segmented citywide pagination — per-page save, incremental.
|
||
|
||
room_segments — кортеж из tuple (1,) / (2,) / ... / None (= no room filter).
|
||
None segment даёт смешанную выдачу (студии + всё подряд) — Cian
|
||
не различает их без room param.
|
||
"""
|
||
seg_labels = [
|
||
("r" + str(seg[0]) if seg else "all")
|
||
for seg in room_segments
|
||
]
|
||
print(
|
||
f"\n=== PHASE CITYWIDE: Cian ЕКБ vtorichka "
|
||
f"({len(room_segments)} segments={seg_labels} × {pages} pages, delay {delay}s) ===\n",
|
||
flush=True,
|
||
)
|
||
total = 0
|
||
cum_ins = 0
|
||
cum_upd = 0
|
||
async with CianScraper() as s:
|
||
# Override per-request delay (in addition to between-anchors delay)
|
||
s.request_delay_sec = delay
|
||
# Inject cookies (если есть): прокидываем в curl_cffi session перед warm-up
|
||
if cookies_file is not None:
|
||
cookies = _load_cookies_file(cookies_file)
|
||
assert s._cffi is not None
|
||
s._cffi.cookies.update(cookies)
|
||
print(f" loaded {len(cookies)} cookies from {cookies_file.name}", flush=True)
|
||
# Warm-up: посетить homepage чтобы получить _CIAN_GK / _yasc + не выглядеть как наглый бот
|
||
try:
|
||
assert s._cffi is not None
|
||
r_warm = await s._cffi.get(f"{s.base_url}/")
|
||
print(
|
||
f" warmup GET / → HTTP {r_warm.status_code} "
|
||
f"(cookies={len(list(s._cffi.cookies.jar))})",
|
||
flush=True,
|
||
)
|
||
# Subsequent SERP requests should look like in-site navigation
|
||
s._cffi.headers.update(
|
||
{
|
||
"Referer": f"{s.base_url}/",
|
||
"Sec-Fetch-Site": "same-origin",
|
||
"Sec-Fetch-Mode": "navigate",
|
||
"Sec-Fetch-User": "?1",
|
||
"Sec-Fetch-Dest": "document",
|
||
}
|
||
)
|
||
await asyncio.sleep(min(5.0, delay))
|
||
except Exception as e: # noqa: BLE001
|
||
print(f" warmup FAIL: {type(e).__name__}: {e}", flush=True)
|
||
for seg_idx, seg in enumerate(room_segments):
|
||
seg_label = "r" + str(seg[0]) if seg else "all"
|
||
# --start-page применяется только к первому segment'у — продолжение прерванного sweep
|
||
page_from = start_page if seg_idx == 0 else 1
|
||
for page in range(page_from, pages + 1):
|
||
# Manual fetch+parse чтобы можно было дампнуть HTML при empty results
|
||
try:
|
||
url_ = s._build_url(rooms=seg, page=page)
|
||
assert s._cffi is not None
|
||
resp = await s._cffi.get(url_)
|
||
if resp.status_code != 200:
|
||
print(
|
||
f" seg={seg_label} page={page} HTTP {resp.status_code} → skip",
|
||
flush=True,
|
||
)
|
||
await asyncio.sleep(delay)
|
||
continue
|
||
lots = s._parse_serp_html(resp.text)
|
||
if not lots:
|
||
# Fallback: новый markup .concat([...])
|
||
lots = _parse_serp_v2(s, resp.text)
|
||
if lots:
|
||
print(
|
||
f" (parsed via v2 .concat fallback)",
|
||
flush=True,
|
||
)
|
||
if not lots:
|
||
# Дамп для разбора: captcha vs пустая страница vs изменение markup
|
||
dump_path = Path(f"cian-empty-seg-{seg_label}-p{page}.html")
|
||
dump_path.write_text(resp.text, encoding="utf-8")
|
||
print(f" DUMP: {dump_path} ({len(resp.text)} bytes)", flush=True)
|
||
await s.sleep_between_requests()
|
||
except Exception as e: # noqa: BLE001
|
||
print(
|
||
f" seg={seg_label} page={page} FAIL fetch: "
|
||
f"{type(e).__name__}: {e}",
|
||
flush=True,
|
||
)
|
||
await asyncio.sleep(delay)
|
||
continue
|
||
if not lots:
|
||
print(
|
||
f" seg={seg_label} page={page}: 0 lots — end of segment "
|
||
"(либо captcha)",
|
||
flush=True,
|
||
)
|
||
break
|
||
db = SessionLocal()
|
||
try:
|
||
ins, upd = save_listings(db, lots)
|
||
cum_ins += ins
|
||
cum_upd += upd
|
||
total += len(lots)
|
||
print(
|
||
f" seg={seg_label} page={page}/{pages}: {len(lots)} lots → "
|
||
f"ins={ins} upd={upd} "
|
||
f"[cum: total={total} new={cum_ins} upd={cum_upd}]",
|
||
flush=True,
|
||
)
|
||
except Exception as e: # noqa: BLE001
|
||
print(
|
||
f" seg={seg_label} page={page} FAIL save: "
|
||
f"{type(e).__name__}: {e}",
|
||
flush=True,
|
||
)
|
||
finally:
|
||
db.close()
|
||
# scraper.fetch_around() уже спит request_delay_sec * jitter — extra sleep не нужен
|
||
print(
|
||
f" PHASE CITYWIDE TOTAL: fetched={total} ins={cum_ins} upd={cum_upd}",
|
||
flush=True,
|
||
)
|
||
return total
|
||
|
||
|
||
async def main() -> None:
|
||
p = argparse.ArgumentParser()
|
||
p.add_argument(
|
||
"--pages",
|
||
type=int,
|
||
default=8,
|
||
help="страниц на room-segment (default 8 → ~28×4×8 = ~900 до дедупа)",
|
||
)
|
||
p.add_argument(
|
||
"--delay",
|
||
type=float,
|
||
default=15,
|
||
help="sec между HTTP requests (residential-safe ≥10, default 15)",
|
||
)
|
||
p.add_argument(
|
||
"--rooms",
|
||
type=str,
|
||
default="1,2,3,4",
|
||
help="room segments через запятую (1,2,3,4) или 'all' для одного citywide "
|
||
"запроса без room param (default '1,2,3,4')",
|
||
)
|
||
p.add_argument(
|
||
"--cookies-file",
|
||
type=Path,
|
||
default=None,
|
||
help="JSON-файл с Cian cookies (browser-export или {name:value}). "
|
||
"Опционально — для обхода captcha. См. scripts/.cian-cookies.json",
|
||
)
|
||
p.add_argument(
|
||
"--start-page",
|
||
type=int,
|
||
default=1,
|
||
help="С какой страницы начать ПЕРВЫЙ сегмент (для resume прерванного sweep). "
|
||
"Остальные segments идут с p=1. Default 1.",
|
||
)
|
||
args = p.parse_args()
|
||
|
||
# Parse rooms argument
|
||
if args.rooms.strip().lower() == "all":
|
||
segments: tuple[tuple[int, ...] | None, ...] = (None,)
|
||
else:
|
||
try:
|
||
nums = [int(x.strip()) for x in args.rooms.split(",") if x.strip()]
|
||
except ValueError as e:
|
||
raise SystemExit(f"--rooms must be comma-separated ints or 'all': {e}") from e
|
||
segments = tuple((n,) for n in nums)
|
||
|
||
cookies_file: Path | None = args.cookies_file
|
||
if cookies_file is not None and not cookies_file.exists():
|
||
raise SystemExit(f"--cookies-file not found: {cookies_file}")
|
||
|
||
t0 = time.perf_counter()
|
||
await phase_citywide(
|
||
args.pages,
|
||
args.delay,
|
||
segments,
|
||
cookies_file=cookies_file,
|
||
start_page=args.start_page,
|
||
)
|
||
elapsed = time.perf_counter() - t0
|
||
print(f"\n=== ALL DONE in {elapsed / 60:.1f} min ===", flush=True)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
asyncio.run(main())
|