423 lines
22 KiB
Python
423 lines
22 KiB
Python
"""Cian Valuation backfill через real Chrome (Playwright) с auth cookies.
|
||
|
||
Iterates listings без свежей external_valuations записи, открывает Calculator URL
|
||
с реальным address+area+rooms, ждёт когда state.estimation.sale.data populated,
|
||
extract'ит sale/rent/chart/houseId, INSERT'ит в external_valuations.
|
||
|
||
Requires:
|
||
pip install playwright "psycopg[binary]"
|
||
python -m playwright install chromium
|
||
SSH tunnel to tradein-postgres on localhost:15433
|
||
Env: TRADEIN_DB_DSN, COOKIE_ENCRYPTION_KEY
|
||
|
||
Run:
|
||
python C:\\Users\\user\\cian-backfill\\playwright_valuation.py --batch 10 --delay 3 --wait 8
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import asyncio
|
||
import hashlib
|
||
import json
|
||
import os
|
||
import re
|
||
import signal
|
||
import sys
|
||
import time
|
||
import urllib.parse
|
||
from datetime import datetime, timezone
|
||
|
||
import psycopg
|
||
from playwright.async_api import async_playwright
|
||
|
||
# JS — wait + extract когда valuation populated (или timeout)
|
||
EXTRACT_JS = """
|
||
() => {
|
||
const arr = window._cianConfig && window._cianConfig['valuation-for-agent-frontend'];
|
||
if (!arr) return { error: 'no MFE', captcha: location.href.includes('captcha') };
|
||
const entry = arr.find(e => e && e.key === 'initialState');
|
||
if (!entry || !entry.value) return { error: 'no initialState' };
|
||
const v = entry.value;
|
||
// unwrap state.data контейнеры в чистые объекты (Cian шаблон — { data, isFetching, isError })
|
||
const unwrap = (o) => (o && typeof o === 'object' && 'data' in o ? o.data : o);
|
||
return {
|
||
isAuth: v.user && v.user.isAuthenticated,
|
||
user_id_cian: v.user && v.user.userId,
|
||
isFetching: (v.estimation && v.estimation.sale && v.estimation.sale.isFetching) ||
|
||
(v.estimationChart && v.estimationChart.isFetching),
|
||
sale: v.estimation && v.estimation.sale && v.estimation.sale.data,
|
||
rent: v.estimation && v.estimation.rent && v.estimation.rent.data,
|
||
chart_obj: v.estimationChart && v.estimationChart.data,
|
||
house: unwrap(v.houseInfo),
|
||
management_company: unwrap(v.managementCompany),
|
||
nearby_houses_info: unwrap(v.nearbyHousesInfo),
|
||
market_trend: unwrap(v.marketTrand), // sic — typo в Cian API
|
||
offers_history: unwrap(v.offersHistory),
|
||
precise_filters: v.preciseFilters,
|
||
geocode: unwrap(v.geocode),
|
||
services: v.services,
|
||
house_id: v.filters && v.filters.houseId,
|
||
filters_address: v.filters && v.filters.address,
|
||
filters_house_id: v.filters && v.filters.houseId,
|
||
// FULL initialState — для raw_payload (полный snapshot для будущего парсинга)
|
||
full_state: v,
|
||
};
|
||
}
|
||
"""
|
||
|
||
WAIT_JS = """
|
||
() => {
|
||
const arr = window._cianConfig && window._cianConfig['valuation-for-agent-frontend'];
|
||
if (!arr) return false;
|
||
const entry = arr.find(e => e && e.key === 'initialState');
|
||
if (!entry || !entry.value) return false;
|
||
const v = entry.value;
|
||
const sale = v.estimation && v.estimation.sale;
|
||
// Готово когда либо есть data, либо isError, и не isFetching
|
||
if (sale && sale.data && sale.data.price) return true;
|
||
if (sale && sale.isError) return true;
|
||
return false;
|
||
}
|
||
"""
|
||
|
||
# Cian floor enum
|
||
_FLOOR_SPECIAL = {1: "floorOne", 2: "floorTwo"}
|
||
|
||
|
||
def cian_floor_enum(floor: int, total_floors: int | None) -> str:
|
||
if floor in _FLOOR_SPECIAL:
|
||
return _FLOOR_SPECIAL[floor]
|
||
if total_floors and floor == total_floors:
|
||
return "floorLast"
|
||
return "floorOther"
|
||
|
||
|
||
def compute_cache_key(address, total_area, rooms, floor, repair, deal):
|
||
norm = f"{address.strip().lower()}|{total_area}|{rooms}|{floor}|{repair}|{deal}"
|
||
return hashlib.sha256(norm.encode()).hexdigest()
|
||
|
||
|
||
def parse_change_pct(value_str: str) -> float | None:
|
||
"""'6,4%' / '-1.2%' → 6.4 / -1.2"""
|
||
if not value_str:
|
||
return None
|
||
m = re.search(r"-?\d+[\.,]?\d*", value_str)
|
||
if not m:
|
||
return None
|
||
return float(m.group(0).replace(",", "."))
|
||
|
||
|
||
def normalize_address(addr: str) -> str:
|
||
"""listings.address типа 'улица Машинная, 1В/4 · р-н Ленинский' → 'Екатеринбург, улица Машинная, 1В/4'.
|
||
|
||
1. отрезаем ' · р-н ...' postfix
|
||
2. подставляем 'Екатеринбург, ' если города нет в строке
|
||
"""
|
||
if not addr:
|
||
return addr
|
||
# cut at middle-dot (Cian district separator)
|
||
addr = re.split(r"\s*[·•]\s*", addr)[0].strip().rstrip(",")
|
||
if "екатеринбург" not in addr.lower():
|
||
addr = f"Екатеринбург, {addr}"
|
||
return addr
|
||
|
||
|
||
def build_calc_url(address: str, area: float, rooms: int, floor: int, total_floors: int | None) -> str:
|
||
params = {
|
||
"address": normalize_address(address),
|
||
"totalArea": str(area),
|
||
"roomsCount": str(rooms),
|
||
"valuationType": "sale",
|
||
"floor[0]": cian_floor_enum(floor, total_floors),
|
||
"repairType[0]": "repairTypeCosmetic",
|
||
}
|
||
return f"https://www.cian.ru/kalkulator-nedvizhimosti/?{urllib.parse.urlencode(params, safe='[]')}"
|
||
|
||
|
||
def main() -> int:
|
||
ap = argparse.ArgumentParser()
|
||
ap.add_argument("--batch", type=int, default=10)
|
||
ap.add_argument("--delay", type=float, default=3.0, help="seconds между listings")
|
||
ap.add_argument("--wait", type=float, default=8.0, help="max seconds wait until valuation data")
|
||
ap.add_argument("--max", type=int, default=5500)
|
||
ap.add_argument("--headed", action="store_true")
|
||
args = ap.parse_args()
|
||
|
||
dsn = os.environ.get("TRADEIN_DB_DSN")
|
||
if not dsn:
|
||
sys.exit("FATAL: TRADEIN_DB_DSN env required")
|
||
enc_key = os.environ.get("COOKIE_ENCRYPTION_KEY")
|
||
if not enc_key:
|
||
sys.exit("FATAL: COOKIE_ENCRYPTION_KEY env required")
|
||
pg_dsn = dsn.replace("postgresql+psycopg://", "postgresql://")
|
||
|
||
interrupted = False
|
||
|
||
def on_sigint(s, f):
|
||
nonlocal interrupted
|
||
print("\n!! Ctrl+C — finishing current item")
|
||
interrupted = True
|
||
signal.signal(signal.SIGINT, on_sigint)
|
||
|
||
async def run() -> None:
|
||
nonlocal interrupted
|
||
cum_ok = cum_fail = cum_skipped = 0
|
||
start = time.time()
|
||
last_id = 0
|
||
|
||
def load_cookies() -> tuple[list, datetime | None]:
|
||
with psycopg.connect(pg_dsn) as conn:
|
||
with conn.cursor() as cur:
|
||
cur.execute(
|
||
"""
|
||
SELECT pgp_sym_decrypt(cookies_encrypted, %s)::text, uploaded_at
|
||
FROM cian_session_cookies
|
||
WHERE expires_at_estimate > NOW() AND last_invalid_at IS NULL
|
||
ORDER BY uploaded_at DESC LIMIT 1
|
||
""",
|
||
(enc_key,),
|
||
)
|
||
r = cur.fetchone()
|
||
if not r:
|
||
return [], None
|
||
jar = json.loads(r[0])
|
||
out = []
|
||
for name, val in jar.items():
|
||
out.append({
|
||
"name": name, "value": val,
|
||
"domain": ".cian.ru", "path": "/",
|
||
"secure": True, "sameSite": "Lax",
|
||
})
|
||
return out, r[1]
|
||
|
||
cookies_from_db, cookies_uploaded_at = load_cookies()
|
||
if not cookies_from_db:
|
||
sys.exit("FATAL: no valid cookies in cian_session_cookies")
|
||
print(f"[init] loaded {len(cookies_from_db)} cookies (uploaded {cookies_uploaded_at})")
|
||
|
||
profile_dir = os.environ.get("CIAN_PROFILE_DIR", r"C:\Users\user\cian-backfill\chrome_profile")
|
||
os.makedirs(profile_dir, exist_ok=True)
|
||
|
||
async with async_playwright() as p:
|
||
ctx = await p.chromium.launch_persistent_context(
|
||
user_data_dir=profile_dir,
|
||
headless=not args.headed,
|
||
user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||
viewport={"width": 1366, "height": 900},
|
||
locale="ru-RU",
|
||
args=["--disable-blink-features=AutomationControlled"],
|
||
)
|
||
# First launch: inject cookies из БД. Subsequent launches: cookies уже в profile.
|
||
existing = await ctx.cookies("https://www.cian.ru")
|
||
has_auth = any(c.get("name") == "DMIR_AUTH" for c in existing)
|
||
if not has_auth and cookies_from_db:
|
||
await ctx.add_cookies(cookies_from_db)
|
||
print(f"[init] injected {len(cookies_from_db)} cookies into fresh profile")
|
||
else:
|
||
print(f"[init] reusing profile cookies ({len(existing)} present, DMIR_AUTH={'yes' if has_auth else 'no'})")
|
||
page = ctx.pages[0] if ctx.pages else await ctx.new_page()
|
||
page.set_default_navigation_timeout(45_000)
|
||
page.set_default_timeout(45_000)
|
||
|
||
processed = 0
|
||
while not interrupted and processed < args.max:
|
||
# 1. Fetch batch of listings WITH полный address + geo
|
||
with psycopg.connect(pg_dsn) as conn:
|
||
with conn.cursor() as cur:
|
||
cur.execute(
|
||
"""
|
||
SELECT l.id, l.address, l.area_m2, l.rooms, l.floor, l.total_floors
|
||
FROM listings l
|
||
WHERE l.source = 'cian'
|
||
AND l.address IS NOT NULL
|
||
AND length(l.address) > 15 -- хотя бы город+улица+номер
|
||
AND l.area_m2 IS NOT NULL
|
||
AND l.rooms IS NOT NULL
|
||
AND l.floor IS NOT NULL
|
||
AND l.total_floors IS NOT NULL
|
||
AND l.id > %s
|
||
AND NOT EXISTS (
|
||
SELECT 1 FROM external_valuations ev
|
||
WHERE ev.source='cian_valuation'
|
||
AND ev.listing_id = l.id
|
||
AND ev.expires_at > NOW()
|
||
)
|
||
ORDER BY l.id
|
||
LIMIT %s
|
||
""",
|
||
(last_id, args.batch),
|
||
)
|
||
rows = cur.fetchall()
|
||
if not rows:
|
||
print(f"\n[done] drained — no more listings")
|
||
break
|
||
print(f"\n[batch] {len(rows)} listings (cum: ok={cum_ok} fail={cum_fail} skipped={cum_skipped})")
|
||
|
||
with psycopg.connect(pg_dsn, autocommit=False) as conn:
|
||
for (lid, addr, area, rooms, floor, total_floors) in rows:
|
||
if interrupted:
|
||
break
|
||
processed += 1
|
||
last_id = lid
|
||
t0 = time.time()
|
||
try:
|
||
area_f = float(area)
|
||
rooms_i = int(rooms)
|
||
floor_i = int(floor)
|
||
total_floors_i = int(total_floors) if total_floors else None
|
||
url = build_calc_url(addr, area_f, rooms_i, floor_i, total_floors_i)
|
||
await page.goto(url, wait_until="domcontentloaded")
|
||
# Click submit button "Оценить" — без этого Cian не вычисляет
|
||
try:
|
||
btn = page.get_by_test_id("next-button")
|
||
await btn.click(timeout=5000)
|
||
except Exception:
|
||
pass # button может отсутствовать если URL params не валидны — fallback
|
||
try:
|
||
await page.wait_for_function(WAIT_JS, timeout=int(args.wait * 1000))
|
||
except Exception:
|
||
pass # fall through и попробуем extract — может data есть с задержкой
|
||
data = await page.evaluate(EXTRACT_JS)
|
||
if data.get("error"):
|
||
if data.get("captcha"):
|
||
print(f" [{lid}] CAPTCHA — STOP, please solve in --headed mode")
|
||
interrupted = True
|
||
break
|
||
cum_fail += 1
|
||
print(f" [{lid}] err={data['error']} {time.time() - t0:.1f}s")
|
||
continue
|
||
if not data.get("isAuth"):
|
||
# 1. Try fresher cookies из БД
|
||
fresh, uploaded = load_cookies()
|
||
if fresh and uploaded and (cookies_uploaded_at is None or uploaded > cookies_uploaded_at):
|
||
print(f" [{lid}] auth lost — reloading fresh cookies (uploaded {uploaded})")
|
||
await ctx.clear_cookies()
|
||
await ctx.add_cookies(fresh)
|
||
cookies_uploaded_at = uploaded
|
||
await page.goto(url, wait_until="domcontentloaded")
|
||
try:
|
||
await page.wait_for_function(WAIT_JS, timeout=int(args.wait * 1000))
|
||
except Exception:
|
||
pass
|
||
data = await page.evaluate(EXTRACT_JS)
|
||
# 2. Если headed — даём user'у залогиниться вручную в открытом окне
|
||
if not data.get("isAuth") and args.headed:
|
||
print(f"\n [{lid}] !! AUTH LOST !!")
|
||
print(f" → В открытом Chrome окне открой www.cian.ru, ЗАЛОГИНЬСЯ.")
|
||
print(f" → После успешного логина нажми Enter здесь чтобы продолжить.")
|
||
print(f" → Или Ctrl+C чтобы остановить.")
|
||
try:
|
||
# blocking input — пользователь логинится сам
|
||
loop = asyncio.get_event_loop()
|
||
await loop.run_in_executor(None, input, " >>> press Enter when logged in: ")
|
||
except (EOFError, KeyboardInterrupt):
|
||
interrupted = True
|
||
break
|
||
# retry same listing
|
||
await page.goto(url, wait_until="domcontentloaded")
|
||
try:
|
||
await page.wait_for_function(WAIT_JS, timeout=int(args.wait * 1000))
|
||
except Exception:
|
||
pass
|
||
data = await page.evaluate(EXTRACT_JS)
|
||
# 3. Headless и всё ещё no auth → STOP
|
||
if not data.get("isAuth"):
|
||
print(f" [{lid}] not auth — STOP. Запусти с --headed и login manually.")
|
||
interrupted = True
|
||
break
|
||
sale = data.get("sale")
|
||
if not sale or not sale.get("price"):
|
||
cum_skipped += 1
|
||
print(f" [{lid}] no sale data (address not matched?) {time.time() - t0:.1f}s")
|
||
continue
|
||
rent = data.get("rent") or {}
|
||
chart_obj = data.get("chart_obj") or {}
|
||
chart_points = (chart_obj.get("chartData") or {}).get("data") or []
|
||
chart_title = chart_obj.get("title") or {}
|
||
chart_change_dir = chart_title.get("change") # 'increase'/'decrease'/'neutral'
|
||
chart_change_pct = parse_change_pct(chart_title.get("changeValue") or "")
|
||
house_id = data.get("house_id")
|
||
cache_key = compute_cache_key(addr, area_f, rooms_i, floor_i, "cosmetic", "sale")
|
||
|
||
with conn.cursor() as cur:
|
||
cur.execute(
|
||
"""
|
||
INSERT INTO external_valuations (
|
||
source, cache_key, address,
|
||
total_area, rooms_count, floor, total_floors,
|
||
repair_type, deal_type,
|
||
price_rub, accuracy,
|
||
low_price, high_price,
|
||
house_id, listing_id,
|
||
sale_price_rub, sale_accuracy,
|
||
rent_price_rub, rent_accuracy,
|
||
chart, chart_change_pct, chart_change_direction,
|
||
external_house_id, filters_hash,
|
||
raw_payload, fetched_at, expires_at
|
||
) VALUES (
|
||
'cian_valuation', %s, %s,
|
||
CAST(%s AS numeric), %s, %s, %s,
|
||
'cosmetic', 'sale',
|
||
CAST(%s AS numeric), CAST(%s AS numeric),
|
||
CAST(%s AS numeric), CAST(%s AS numeric),
|
||
NULL, CAST(%s AS bigint),
|
||
CAST(%s AS numeric), CAST(%s AS numeric),
|
||
CAST(%s AS numeric), CAST(%s AS numeric),
|
||
CAST(%s AS jsonb), CAST(%s AS numeric), %s,
|
||
%s, NULL,
|
||
CAST(%s AS jsonb), NOW(), NOW() + INTERVAL '30 days'
|
||
)
|
||
ON CONFLICT (source, cache_key) DO UPDATE SET
|
||
price_rub = EXCLUDED.price_rub,
|
||
accuracy = EXCLUDED.accuracy,
|
||
low_price = EXCLUDED.low_price,
|
||
high_price = EXCLUDED.high_price,
|
||
listing_id = COALESCE(EXCLUDED.listing_id, external_valuations.listing_id),
|
||
sale_price_rub = EXCLUDED.sale_price_rub,
|
||
sale_accuracy = EXCLUDED.sale_accuracy,
|
||
rent_price_rub = EXCLUDED.rent_price_rub,
|
||
rent_accuracy = EXCLUDED.rent_accuracy,
|
||
chart = EXCLUDED.chart,
|
||
chart_change_pct = EXCLUDED.chart_change_pct,
|
||
chart_change_direction = EXCLUDED.chart_change_direction,
|
||
external_house_id = EXCLUDED.external_house_id,
|
||
raw_payload = EXCLUDED.raw_payload,
|
||
fetched_at = NOW(),
|
||
expires_at = NOW() + INTERVAL '30 days'
|
||
""",
|
||
(
|
||
cache_key, addr,
|
||
area_f, rooms_i, floor_i, total_floors_i,
|
||
sale.get("price"), (sale.get("accuracy") or 0) / 100,
|
||
sale.get("priceFrom"), sale.get("priceTo"),
|
||
lid,
|
||
sale.get("price"), (sale.get("accuracy") or 0) / 100,
|
||
rent.get("price"), (rent.get("accuracy") or 0) / 100 if rent.get("accuracy") else None,
|
||
json.dumps(chart_points), chart_change_pct,
|
||
chart_change_dir if chart_change_dir in ("increase", "decrease", "neutral") else None,
|
||
house_id,
|
||
json.dumps(data.get("full_state") or {"sale": sale, "rent": rent, "chart": chart_obj, "house_id": house_id}),
|
||
),
|
||
)
|
||
conn.commit()
|
||
cum_ok += 1
|
||
print(f" [{lid}] OK sale={sale.get('price'):,} rent={rent.get('price')} chart={len(chart_points)} chg={chart_change_pct}%/{chart_change_dir} houseId={house_id} {time.time() - t0:.1f}s")
|
||
except Exception as exc:
|
||
conn.rollback()
|
||
cum_fail += 1
|
||
print(f" [{lid}] ERR {type(exc).__name__}: {str(exc)[:140]} {time.time() - t0:.1f}s")
|
||
await asyncio.sleep(args.delay)
|
||
|
||
await ctx.close()
|
||
|
||
print()
|
||
print(f"== Summary ==")
|
||
print(f" ok={cum_ok} fail={cum_fail} skipped_no_data={cum_skipped}")
|
||
print(f" elapsed={time.time() - start:.1f}s finished={datetime.now(timezone.utc).isoformat(timespec='seconds')}")
|
||
|
||
asyncio.run(run())
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|