"""Cian Valuation backfill через real Chrome (Playwright) с auth cookies. Iterates listings без свежей external_valuations записи, открывает Calculator URL с реальным address+area+rooms, ждёт когда state.estimation.sale.data populated, extract'ит sale/rent/chart/houseId, INSERT'ит в external_valuations. Requires: pip install playwright "psycopg[binary]" python -m playwright install chromium SSH tunnel to tradein-postgres on localhost:15433 Env: TRADEIN_DB_DSN, COOKIE_ENCRYPTION_KEY Run: python C:\\Users\\user\\cian-backfill\\playwright_valuation.py --batch 10 --delay 3 --wait 8 """ from __future__ import annotations import argparse import asyncio import hashlib import json import os import re import signal import sys import time import urllib.parse from datetime import datetime, timezone import psycopg from playwright.async_api import async_playwright # JS — wait + extract когда valuation populated (или timeout) EXTRACT_JS = """ () => { const arr = window._cianConfig && window._cianConfig['valuation-for-agent-frontend']; if (!arr) return { error: 'no MFE', captcha: location.href.includes('captcha') }; const entry = arr.find(e => e && e.key === 'initialState'); if (!entry || !entry.value) return { error: 'no initialState' }; const v = entry.value; // unwrap state.data контейнеры в чистые объекты (Cian шаблон — { data, isFetching, isError }) const unwrap = (o) => (o && typeof o === 'object' && 'data' in o ? o.data : o); return { isAuth: v.user && v.user.isAuthenticated, user_id_cian: v.user && v.user.userId, isFetching: (v.estimation && v.estimation.sale && v.estimation.sale.isFetching) || (v.estimationChart && v.estimationChart.isFetching), sale: v.estimation && v.estimation.sale && v.estimation.sale.data, rent: v.estimation && v.estimation.rent && v.estimation.rent.data, chart_obj: v.estimationChart && v.estimationChart.data, house: unwrap(v.houseInfo), management_company: unwrap(v.managementCompany), nearby_houses_info: unwrap(v.nearbyHousesInfo), market_trend: unwrap(v.marketTrand), // sic — typo в Cian API offers_history: unwrap(v.offersHistory), precise_filters: v.preciseFilters, geocode: unwrap(v.geocode), services: v.services, house_id: v.filters && v.filters.houseId, filters_address: v.filters && v.filters.address, filters_house_id: v.filters && v.filters.houseId, // FULL initialState — для raw_payload (полный snapshot для будущего парсинга) full_state: v, }; } """ WAIT_JS = """ () => { const arr = window._cianConfig && window._cianConfig['valuation-for-agent-frontend']; if (!arr) return false; const entry = arr.find(e => e && e.key === 'initialState'); if (!entry || !entry.value) return false; const v = entry.value; const sale = v.estimation && v.estimation.sale; // Готово когда либо есть data, либо isError, и не isFetching if (sale && sale.data && sale.data.price) return true; if (sale && sale.isError) return true; return false; } """ # Cian floor enum _FLOOR_SPECIAL = {1: "floorOne", 2: "floorTwo"} def cian_floor_enum(floor: int, total_floors: int | None) -> str: if floor in _FLOOR_SPECIAL: return _FLOOR_SPECIAL[floor] if total_floors and floor == total_floors: return "floorLast" return "floorOther" def compute_cache_key(address, total_area, rooms, floor, repair, deal): norm = f"{address.strip().lower()}|{total_area}|{rooms}|{floor}|{repair}|{deal}" return hashlib.sha256(norm.encode()).hexdigest() def parse_change_pct(value_str: str) -> float | None: """'6,4%' / '-1.2%' → 6.4 / -1.2""" if not value_str: return None m = re.search(r"-?\d+[\.,]?\d*", value_str) if not m: return None return float(m.group(0).replace(",", ".")) def normalize_address(addr: str) -> str: """listings.address типа 'улица Машинная, 1В/4 · р-н Ленинский' → 'Екатеринбург, улица Машинная, 1В/4'. 1. отрезаем ' · р-н ...' postfix 2. подставляем 'Екатеринбург, ' если города нет в строке """ if not addr: return addr # cut at middle-dot (Cian district separator) addr = re.split(r"\s*[·•]\s*", addr)[0].strip().rstrip(",") if "екатеринбург" not in addr.lower(): addr = f"Екатеринбург, {addr}" return addr def build_calc_url(address: str, area: float, rooms: int, floor: int, total_floors: int | None) -> str: params = { "address": normalize_address(address), "totalArea": str(area), "roomsCount": str(rooms), "valuationType": "sale", "floor[0]": cian_floor_enum(floor, total_floors), "repairType[0]": "repairTypeCosmetic", } return f"https://www.cian.ru/kalkulator-nedvizhimosti/?{urllib.parse.urlencode(params, safe='[]')}" def main() -> int: ap = argparse.ArgumentParser() ap.add_argument("--batch", type=int, default=10) ap.add_argument("--delay", type=float, default=3.0, help="seconds между listings") ap.add_argument("--wait", type=float, default=8.0, help="max seconds wait until valuation data") ap.add_argument("--max", type=int, default=5500) ap.add_argument("--headed", action="store_true") args = ap.parse_args() dsn = os.environ.get("TRADEIN_DB_DSN") if not dsn: sys.exit("FATAL: TRADEIN_DB_DSN env required") enc_key = os.environ.get("COOKIE_ENCRYPTION_KEY") if not enc_key: sys.exit("FATAL: COOKIE_ENCRYPTION_KEY env required") pg_dsn = dsn.replace("postgresql+psycopg://", "postgresql://") interrupted = False def on_sigint(s, f): nonlocal interrupted print("\n!! Ctrl+C — finishing current item") interrupted = True signal.signal(signal.SIGINT, on_sigint) async def run() -> None: nonlocal interrupted cum_ok = cum_fail = cum_skipped = 0 start = time.time() last_id = 0 def load_cookies() -> tuple[list, datetime | None]: with psycopg.connect(pg_dsn) as conn: with conn.cursor() as cur: cur.execute( """ SELECT pgp_sym_decrypt(cookies_encrypted, %s)::text, uploaded_at FROM cian_session_cookies WHERE expires_at_estimate > NOW() AND last_invalid_at IS NULL ORDER BY uploaded_at DESC LIMIT 1 """, (enc_key,), ) r = cur.fetchone() if not r: return [], None jar = json.loads(r[0]) out = [] for name, val in jar.items(): out.append({ "name": name, "value": val, "domain": ".cian.ru", "path": "/", "secure": True, "sameSite": "Lax", }) return out, r[1] cookies_from_db, cookies_uploaded_at = load_cookies() if not cookies_from_db: sys.exit("FATAL: no valid cookies in cian_session_cookies") print(f"[init] loaded {len(cookies_from_db)} cookies (uploaded {cookies_uploaded_at})") profile_dir = os.environ.get("CIAN_PROFILE_DIR", r"C:\Users\user\cian-backfill\chrome_profile") os.makedirs(profile_dir, exist_ok=True) async with async_playwright() as p: ctx = await p.chromium.launch_persistent_context( user_data_dir=profile_dir, headless=not args.headed, user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", viewport={"width": 1366, "height": 900}, locale="ru-RU", args=["--disable-blink-features=AutomationControlled"], ) # First launch: inject cookies из БД. Subsequent launches: cookies уже в profile. existing = await ctx.cookies("https://www.cian.ru") has_auth = any(c.get("name") == "DMIR_AUTH" for c in existing) if not has_auth and cookies_from_db: await ctx.add_cookies(cookies_from_db) print(f"[init] injected {len(cookies_from_db)} cookies into fresh profile") else: print(f"[init] reusing profile cookies ({len(existing)} present, DMIR_AUTH={'yes' if has_auth else 'no'})") page = ctx.pages[0] if ctx.pages else await ctx.new_page() page.set_default_navigation_timeout(45_000) page.set_default_timeout(45_000) processed = 0 while not interrupted and processed < args.max: # 1. Fetch batch of listings WITH полный address + geo with psycopg.connect(pg_dsn) as conn: with conn.cursor() as cur: cur.execute( """ SELECT l.id, l.address, l.area_m2, l.rooms, l.floor, l.total_floors FROM listings l WHERE l.source = 'cian' AND l.address IS NOT NULL AND length(l.address) > 15 -- хотя бы город+улица+номер AND l.area_m2 IS NOT NULL AND l.rooms IS NOT NULL AND l.floor IS NOT NULL AND l.total_floors IS NOT NULL AND l.id > %s AND NOT EXISTS ( SELECT 1 FROM external_valuations ev WHERE ev.source='cian_valuation' AND ev.listing_id = l.id AND ev.expires_at > NOW() ) ORDER BY l.id LIMIT %s """, (last_id, args.batch), ) rows = cur.fetchall() if not rows: print(f"\n[done] drained — no more listings") break print(f"\n[batch] {len(rows)} listings (cum: ok={cum_ok} fail={cum_fail} skipped={cum_skipped})") with psycopg.connect(pg_dsn, autocommit=False) as conn: for (lid, addr, area, rooms, floor, total_floors) in rows: if interrupted: break processed += 1 last_id = lid t0 = time.time() try: area_f = float(area) rooms_i = int(rooms) floor_i = int(floor) total_floors_i = int(total_floors) if total_floors else None url = build_calc_url(addr, area_f, rooms_i, floor_i, total_floors_i) await page.goto(url, wait_until="domcontentloaded") # Click submit button "Оценить" — без этого Cian не вычисляет try: btn = page.get_by_test_id("next-button") await btn.click(timeout=5000) except Exception: pass # button может отсутствовать если URL params не валидны — fallback try: await page.wait_for_function(WAIT_JS, timeout=int(args.wait * 1000)) except Exception: pass # fall through и попробуем extract — может data есть с задержкой data = await page.evaluate(EXTRACT_JS) if data.get("error"): if data.get("captcha"): print(f" [{lid}] CAPTCHA — STOP, please solve in --headed mode") interrupted = True break cum_fail += 1 print(f" [{lid}] err={data['error']} {time.time() - t0:.1f}s") continue if not data.get("isAuth"): # 1. Try fresher cookies из БД fresh, uploaded = load_cookies() if fresh and uploaded and (cookies_uploaded_at is None or uploaded > cookies_uploaded_at): print(f" [{lid}] auth lost — reloading fresh cookies (uploaded {uploaded})") await ctx.clear_cookies() await ctx.add_cookies(fresh) cookies_uploaded_at = uploaded await page.goto(url, wait_until="domcontentloaded") try: await page.wait_for_function(WAIT_JS, timeout=int(args.wait * 1000)) except Exception: pass data = await page.evaluate(EXTRACT_JS) # 2. Если headed — даём user'у залогиниться вручную в открытом окне if not data.get("isAuth") and args.headed: print(f"\n [{lid}] !! AUTH LOST !!") print(f" → В открытом Chrome окне открой www.cian.ru, ЗАЛОГИНЬСЯ.") print(f" → После успешного логина нажми Enter здесь чтобы продолжить.") print(f" → Или Ctrl+C чтобы остановить.") try: # blocking input — пользователь логинится сам loop = asyncio.get_event_loop() await loop.run_in_executor(None, input, " >>> press Enter when logged in: ") except (EOFError, KeyboardInterrupt): interrupted = True break # retry same listing await page.goto(url, wait_until="domcontentloaded") try: await page.wait_for_function(WAIT_JS, timeout=int(args.wait * 1000)) except Exception: pass data = await page.evaluate(EXTRACT_JS) # 3. Headless и всё ещё no auth → STOP if not data.get("isAuth"): print(f" [{lid}] not auth — STOP. Запусти с --headed и login manually.") interrupted = True break sale = data.get("sale") if not sale or not sale.get("price"): cum_skipped += 1 print(f" [{lid}] no sale data (address not matched?) {time.time() - t0:.1f}s") continue rent = data.get("rent") or {} chart_obj = data.get("chart_obj") or {} chart_points = (chart_obj.get("chartData") or {}).get("data") or [] chart_title = chart_obj.get("title") or {} chart_change_dir = chart_title.get("change") # 'increase'/'decrease'/'neutral' chart_change_pct = parse_change_pct(chart_title.get("changeValue") or "") house_id = data.get("house_id") cache_key = compute_cache_key(addr, area_f, rooms_i, floor_i, "cosmetic", "sale") with conn.cursor() as cur: cur.execute( """ INSERT INTO external_valuations ( source, cache_key, address, total_area, rooms_count, floor, total_floors, repair_type, deal_type, price_rub, accuracy, low_price, high_price, house_id, listing_id, sale_price_rub, sale_accuracy, rent_price_rub, rent_accuracy, chart, chart_change_pct, chart_change_direction, external_house_id, filters_hash, raw_payload, fetched_at, expires_at ) VALUES ( 'cian_valuation', %s, %s, CAST(%s AS numeric), %s, %s, %s, 'cosmetic', 'sale', CAST(%s AS numeric), CAST(%s AS numeric), CAST(%s AS numeric), CAST(%s AS numeric), NULL, CAST(%s AS bigint), CAST(%s AS numeric), CAST(%s AS numeric), CAST(%s AS numeric), CAST(%s AS numeric), CAST(%s AS jsonb), CAST(%s AS numeric), %s, %s, NULL, CAST(%s AS jsonb), NOW(), NOW() + INTERVAL '30 days' ) ON CONFLICT (source, cache_key) DO UPDATE SET price_rub = EXCLUDED.price_rub, accuracy = EXCLUDED.accuracy, low_price = EXCLUDED.low_price, high_price = EXCLUDED.high_price, listing_id = COALESCE(EXCLUDED.listing_id, external_valuations.listing_id), sale_price_rub = EXCLUDED.sale_price_rub, sale_accuracy = EXCLUDED.sale_accuracy, rent_price_rub = EXCLUDED.rent_price_rub, rent_accuracy = EXCLUDED.rent_accuracy, chart = EXCLUDED.chart, chart_change_pct = EXCLUDED.chart_change_pct, chart_change_direction = EXCLUDED.chart_change_direction, external_house_id = EXCLUDED.external_house_id, raw_payload = EXCLUDED.raw_payload, fetched_at = NOW(), expires_at = NOW() + INTERVAL '30 days' """, ( cache_key, addr, area_f, rooms_i, floor_i, total_floors_i, sale.get("price"), (sale.get("accuracy") or 0) / 100, sale.get("priceFrom"), sale.get("priceTo"), lid, sale.get("price"), (sale.get("accuracy") or 0) / 100, rent.get("price"), (rent.get("accuracy") or 0) / 100 if rent.get("accuracy") else None, json.dumps(chart_points), chart_change_pct, chart_change_dir if chart_change_dir in ("increase", "decrease", "neutral") else None, house_id, json.dumps(data.get("full_state") or {"sale": sale, "rent": rent, "chart": chart_obj, "house_id": house_id}), ), ) conn.commit() cum_ok += 1 print(f" [{lid}] OK sale={sale.get('price'):,} rent={rent.get('price')} chart={len(chart_points)} chg={chart_change_pct}%/{chart_change_dir} houseId={house_id} {time.time() - t0:.1f}s") except Exception as exc: conn.rollback() cum_fail += 1 print(f" [{lid}] ERR {type(exc).__name__}: {str(exc)[:140]} {time.time() - t0:.1f}s") await asyncio.sleep(args.delay) await ctx.close() print() print(f"== Summary ==") print(f" ok={cum_ok} fail={cum_fail} skipped_no_data={cum_skipped}") print(f" elapsed={time.time() - start:.1f}s finished={datetime.now(timezone.utc).isoformat(timespec='seconds')}") asyncio.run(run()) return 0 if __name__ == "__main__": sys.exit(main())