From 136202dc800a101422d380c616cbfb0cf98a57d9 Mon Sep 17 00:00:00 2001 From: lekss361 Date: Sun, 24 May 2026 20:00:59 +0000 Subject: [PATCH] =?UTF-8?q?feat(tradein/scripts):=20local=20Playwright=20r?= =?UTF-8?q?unners=20=D0=B4=D0=BB=D1=8F=20Cian=20backfill=20(bypass=20serve?= =?UTF-8?q?r-IP=20anti-bot)=20(#553)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tradein-mvp/scripts/local-cian/README.md | 87 ++++ .../scripts/local-cian/playwright_history.py | 256 +++++++++++ .../local-cian/playwright_valuation.py | 423 ++++++++++++++++++ .../local-cian/upload_cookies_local.py | 65 +++ 4 files changed, 831 insertions(+) create mode 100644 tradein-mvp/scripts/local-cian/README.md create mode 100644 tradein-mvp/scripts/local-cian/playwright_history.py create mode 100644 tradein-mvp/scripts/local-cian/playwright_valuation.py create mode 100644 tradein-mvp/scripts/local-cian/upload_cookies_local.py diff --git a/tradein-mvp/scripts/local-cian/README.md b/tradein-mvp/scripts/local-cian/README.md new file mode 100644 index 00000000..df70898c --- /dev/null +++ b/tradein-mvp/scripts/local-cian/README.md @@ -0,0 +1,87 @@ +# Local Cian backfill scrapers (Playwright, real browser) + +Cian блокирует server-IP scraping anti-bot. Эти скрипты запускаются с **локальной машины разработчика** +через SSH-tunnel к prod tradein-postgres — обходят bot detection, используют real Chrome с persistent profile. + +## Когда использовать + +- Backfill `offer_price_history` для активных Cian listings (history scraper, надёжно) +- Best-effort backfill `external_valuations` для домов где Calculator отдаёт оценку без agent-wizard + +## Prerequisites + +```bash +pip install playwright "psycopg[binary]" sqlalchemy +python -m playwright install chromium +``` + +## Setup + +### 1. SSH tunnel к tradein-postgres (отдельное окно, держать открытым) + +```powershell +$IP = (ssh gendesign 'docker inspect tradein-postgres --format "{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}"').Trim() +ssh -L "15433:${IP}:5432" gendesign -N +``` + +### 2. Env vars (creds из vault `meta/00_credentials.md`) + +```powershell +$env:TRADEIN_DB_DSN = "postgresql+psycopg://tradein:@localhost:15433/tradein" +$env:COOKIE_ENCRYPTION_KEY = (ssh gendesign 'grep ^COOKIE_ENCRYPTION_KEY= /opt/gendesign/tradein-mvp/.env.runtime | cut -d= -f2').Trim() +``` + +### 3. Залить Cian cookies (один раз / по мере истечения) + +1. Login на cian.ru в обычном Chrome. +2. DevTools → Application → Cookies → www.cian.ru → скопировать `DMIR_AUTH` (httpOnly) + `_CIAN_GK`. +3. Вставить значения в `upload_cookies_local.py` (поля с плейсхолдером ``). +4. Вставить реальный Cian userId в `ACCOUNT_USER_ID` (найти: DevTools → Network → XHR → `/v1/get-my-id/` → `userId`). +5. Run: + +```powershell +python upload_cookies_local.py +``` + +## Usage + +### History backfill (production-ready, не блокируется) + +```powershell +python -u playwright_history.py --batch 20 --delay 2 2>&1 | Tee-Object -FilePath cian_history.log +``` + +- Темп: ~25 listings/min +- Идёт по `id > last_id` ASC pagination +- Записывает в `offer_price_history` (UNIQUE listing_id + change_time) +- Скорость дрейна 5к листингов: ~3-4 часа + +### Valuation backfill (best-effort, partial) + +```powershell +python -u playwright_valuation.py --batch 10 --delay 5 --wait 8 2>&1 | Tee-Object -FilePath cian_valuation.log +``` + +- Использует persistent profile `./chrome_profile/` для сохранения сессии между runs + (переопределяется через `$env:CIAN_PROFILE_DIR`) +- На `not auth` в `--headed` mode — ждёт manual login в открытом окне +- Skip листинги где Cian не отдаёт оценку (~70% таких — новостройки, нет аналогов) + +### Если auth теряется + +Playwright Chrome имеет другой fingerprint чем твой обычный Chrome → Cian может invalidate сессию. Решения: + +- Запустить с `--headed` → login manually в окне. +- Re-upload cookies через `upload_cookies_local.py`. + +## Pre-existing knowledge + +См. vault `decisions/Decision_Cian_History_Valuation_Activation_May24.md` — full 6-PR sequence +(#523/#525/#530/#535/#539/#543) + open follow-ups. + +## Limitations + +- Calculator (`external_valuations`) собирается только для домов с готовой оценкой в + `_cianConfig['valuation-for-agent-frontend'].initialState.estimation`. Полный wizard-flow не автоматизируется. +- `DMIR_AUTH` expires ~30-60 min — нужно периодически перезаливать cookies. +- Не покрывает Cian listings вне нашего SERP (старые не-active, нестандартные region/category). diff --git a/tradein-mvp/scripts/local-cian/playwright_history.py b/tradein-mvp/scripts/local-cian/playwright_history.py new file mode 100644 index 00000000..7ec481ec --- /dev/null +++ b/tradein-mvp/scripts/local-cian/playwright_history.py @@ -0,0 +1,256 @@ +"""Cian history backfill через real Chrome (Playwright) — обходит anti-bot detection. + +Iterates Cian listings без offer_price_history, открывает в headless Chrome, +extract'ит defaultState.offerData.priceChanges через page.evaluate, пишет в БД. + +Requires: + pip install playwright "sqlalchemy>=2" "psycopg[binary]" + playwright install chromium + +Run: + $env:TRADEIN_DB_DSN = "postgresql+psycopg://tradein:PASS@localhost:15433/tradein" + python C:\\Users\\user\\cian-backfill\\playwright_history.py --batch 20 --delay 2 + +Flags: + --headed показать браузер (debug) + --batch N листингов за батч (default 20) + --delay S seconds между listings (default 2) + --max N hard cap (default 5500) +""" +from __future__ import annotations + +import argparse +import asyncio +import os +import signal +import sys +import time +from datetime import datetime, timezone + +import json + +import psycopg +from playwright.async_api import async_playwright + +EXTRACT_JS = """ +() => { + if (!window._cianConfig || !window._cianConfig['frontend-offer-card']) { + return { error: 'no _cianConfig', captcha: location.href.includes('captcha') }; + } + const arr = window._cianConfig['frontend-offer-card']; + const def = arr.find(e => e && e.key === 'defaultState'); + if (!def || !def.value) { + return { error: 'no defaultState' }; + } + const v = def.value; + const offerData = v.offerData || {}; + const offer = offerData.offer || {}; + // priceChanges живёт в offerData.priceChanges (НЕ в offerData.offer.priceChanges) + const priceChanges = offerData.priceChanges || offer.priceChanges || v.priceChanges || []; + // bargainTerms.price — текущая цена + const price_rub = (offer.bargainTerms && offer.bargainTerms.price) || null; + // Также собрать ключи offerData чтобы видеть что доступно + return { + cian_id: offer.id || null, + price_rub, + price_changes: priceChanges, + _offerData_keys: Object.keys(offerData).slice(0, 25), + _offer_keys: Object.keys(offer).slice(0, 25), + }; +} +""" + + +def parse_change_time(s): + """Parse 'YYYY-MM-DDTHH:MM:SS' or ISO → timestamptz; return raw string if parse fails.""" + if not s: + return None + return s # psycopg позволяет передать строку → postgres парсит как timestamptz + + +def upsert_changes(cur, listing_id: int, price_rub: int | None, changes: list) -> int: + """Insert priceChanges array; return count inserted (ignoring conflicts).""" + if not changes: + return 0 + rows = [] + prev_price = None + for c in changes: + ct = c.get("changeTime") or c.get("date") + # priceData.price — основной формат 2026+; fallback на старые поля + pd = c.get("priceData") if isinstance(c.get("priceData"), dict) else None + pr = (pd.get("price") if pd else None) or c.get("price") or c.get("priceRub") + if not ct or not pr: + continue + diff_pct = None + if prev_price is not None and prev_price > 0: + diff_pct = round((pr - prev_price) / prev_price * 100, 2) + rows.append((listing_id, ct, pr, diff_pct, "cian")) + prev_price = pr + if not rows: + return 0 + cur.executemany( + """ + INSERT INTO offer_price_history (listing_id, change_time, price_rub, diff_percent, source) + VALUES (%s, %s, %s, %s, %s) + ON CONFLICT (listing_id, change_time) DO NOTHING + """, + rows, + ) + return cur.rowcount or 0 + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--batch", type=int, default=20) + ap.add_argument("--delay", type=float, default=2.0) + ap.add_argument("--max", type=int, default=5500) + ap.add_argument("--headed", action="store_true") + args = ap.parse_args() + + dsn = os.environ.get("TRADEIN_DB_DSN") + if not dsn: + sys.exit("FATAL: TRADEIN_DB_DSN env required") + # psycopg native dsn (без +psycopg для sqlalchemy) + pg_dsn = dsn.replace("postgresql+psycopg://", "postgresql://") + + interrupted = False + + def on_sigint(s, f): + nonlocal interrupted + print("\n!! Ctrl+C — finishing current item") + interrupted = True + signal.signal(signal.SIGINT, on_sigint) + + async def run() -> None: + nonlocal interrupted + cum_ok = cum_fail = cum_changes = cum_with_history = 0 + start = time.time() + last_id = 0 # paginate forward — избегаем loop когда changes=0 + + # Optional: load cookies из БД для авторизованной сессии + cookies_from_db: list = [] + try: + with psycopg.connect(pg_dsn) as conn: + with conn.cursor() as cur: + enc_key = os.environ.get("COOKIE_ENCRYPTION_KEY") + if enc_key: + cur.execute( + """ + SELECT pgp_sym_decrypt(cookies_encrypted, %s)::text + FROM cian_session_cookies + WHERE expires_at_estimate > NOW() AND last_invalid_at IS NULL + ORDER BY uploaded_at DESC LIMIT 1 + """, + (enc_key,), + ) + r = cur.fetchone() + if r: + jar = json.loads(r[0]) + for name, val in jar.items(): + cookies_from_db.append({ + "name": name, "value": val, + "domain": ".cian.ru", "path": "/", + "secure": True, "sameSite": "Lax", + }) + print(f"[init] loaded {len(cookies_from_db)} cookies from DB") + except Exception as exc: + print(f"[init] cookies load skipped: {exc}") + + async with async_playwright() as p: + browser = await p.chromium.launch(headless=not args.headed) + ctx = await browser.new_context( + user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", + viewport={"width": 1366, "height": 900}, + locale="ru-RU", + ) + if cookies_from_db: + await ctx.add_cookies(cookies_from_db) + page = await ctx.new_page() + page.set_default_navigation_timeout(45_000) + page.set_default_timeout(45_000) + + processed = 0 + while not interrupted and processed < args.max: + # 1. Fetch batch — paginate id > last_id чтобы не зацикливаться + with psycopg.connect(pg_dsn) as conn: + with conn.cursor() as cur: + cur.execute( + """ + SELECT l.id, l.source_url + FROM listings l + LEFT JOIN offer_price_history oph ON oph.listing_id = l.id + WHERE l.source = 'cian' + AND l.source_url IS NOT NULL + AND oph.listing_id IS NULL + AND l.id > %s + ORDER BY l.id + LIMIT %s + """, + (last_id, args.batch), + ) + rows = cur.fetchall() + if not rows: + print(f"\n[done] drained — no more listings") + break + print(f"\n[batch] {len(rows)} listings to process (cum: ok={cum_ok} fail={cum_fail} changes={cum_changes})") + + # 2. Process each + with psycopg.connect(pg_dsn, autocommit=False) as conn: + for (lid, url) in rows: + if interrupted: + break + processed += 1 + t0 = time.time() + try: + await page.goto(url, wait_until="domcontentloaded") + # wait for _cianConfig populated + await page.wait_for_function( + "() => window._cianConfig && window._cianConfig['frontend-offer-card'] && window._cianConfig['frontend-offer-card'].some(e => e.key === 'defaultState')", + timeout=15_000, + ) + data = await page.evaluate(EXTRACT_JS) + if data.get("error"): + if data.get("captcha"): + print(f" [{lid}] CAPTCHA — please solve in browser (use --headed)") + interrupted = True + break + cum_fail += 1 + print(f" [{lid}] {data['error']} {time.time() - t0:.1f}s") + continue + changes = data.get("price_changes") or [] + with conn.cursor() as cur: + inserted = upsert_changes(cur, lid, data.get("price_rub"), changes) + conn.commit() + cum_ok += 1 + cum_changes += inserted + if len(changes) > 0: + cum_with_history += 1 + print(f" [{lid}] OK cian_id={data.get('cian_id')} price={data.get('price_rub')} changes={len(changes)} inserted={inserted} {time.time() - t0:.1f}s") + # На первом обращении и затем каждые 50 — dump для diagnostics + if processed == 1 or processed % 50 == 0: + print(f" [diag] offerData_keys={data.get('_offerData_keys')}") + print(f" [diag] offer_keys={data.get('_offer_keys')}") + # Dump shape priceChanges первые 3 раза + при первом успехе + if (changes and processed <= 3) or (changes and inserted > 0 and cum_changes == inserted): + print(f" [diag] price_changes_sample={changes[0]}") + last_id = lid # advance pagination cursor even on 0-change result + except Exception as exc: + conn.rollback() + cum_fail += 1 + print(f" [{lid}] ERR {type(exc).__name__}: {str(exc)[:120]} {time.time() - t0:.1f}s") + await asyncio.sleep(args.delay) + + await browser.close() + + print() + print(f"== Summary ==") + print(f" processed_ok={cum_ok} fail={cum_fail}") + print(f" listings_with_history={cum_with_history} price_changes_inserted={cum_changes}") + print(f" elapsed={time.time() - start:.1f}s finished={datetime.now(timezone.utc).isoformat(timespec='seconds')}") + + asyncio.run(run()) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tradein-mvp/scripts/local-cian/playwright_valuation.py b/tradein-mvp/scripts/local-cian/playwright_valuation.py new file mode 100644 index 00000000..984e2b1e --- /dev/null +++ b/tradein-mvp/scripts/local-cian/playwright_valuation.py @@ -0,0 +1,423 @@ +"""Cian Valuation backfill через real Chrome (Playwright) с auth cookies. + +Iterates listings без свежей external_valuations записи, открывает Calculator URL +с реальным address+area+rooms, ждёт когда state.estimation.sale.data populated, +extract'ит sale/rent/chart/houseId, INSERT'ит в external_valuations. + +Requires: + pip install playwright "psycopg[binary]" + python -m playwright install chromium + SSH tunnel to tradein-postgres on localhost:15433 + Env: TRADEIN_DB_DSN, COOKIE_ENCRYPTION_KEY + +Run: + python C:\\Users\\user\\cian-backfill\\playwright_valuation.py --batch 10 --delay 3 --wait 8 +""" +from __future__ import annotations + +import argparse +import asyncio +import hashlib +import json +import os +import re +import signal +import sys +import time +import urllib.parse +from datetime import datetime, timezone + +import psycopg +from playwright.async_api import async_playwright + +# JS — wait + extract когда valuation populated (или timeout) +EXTRACT_JS = """ +() => { + const arr = window._cianConfig && window._cianConfig['valuation-for-agent-frontend']; + if (!arr) return { error: 'no MFE', captcha: location.href.includes('captcha') }; + const entry = arr.find(e => e && e.key === 'initialState'); + if (!entry || !entry.value) return { error: 'no initialState' }; + const v = entry.value; + // unwrap state.data контейнеры в чистые объекты (Cian шаблон — { data, isFetching, isError }) + const unwrap = (o) => (o && typeof o === 'object' && 'data' in o ? o.data : o); + return { + isAuth: v.user && v.user.isAuthenticated, + user_id_cian: v.user && v.user.userId, + isFetching: (v.estimation && v.estimation.sale && v.estimation.sale.isFetching) || + (v.estimationChart && v.estimationChart.isFetching), + sale: v.estimation && v.estimation.sale && v.estimation.sale.data, + rent: v.estimation && v.estimation.rent && v.estimation.rent.data, + chart_obj: v.estimationChart && v.estimationChart.data, + house: unwrap(v.houseInfo), + management_company: unwrap(v.managementCompany), + nearby_houses_info: unwrap(v.nearbyHousesInfo), + market_trend: unwrap(v.marketTrand), // sic — typo в Cian API + offers_history: unwrap(v.offersHistory), + precise_filters: v.preciseFilters, + geocode: unwrap(v.geocode), + services: v.services, + house_id: v.filters && v.filters.houseId, + filters_address: v.filters && v.filters.address, + filters_house_id: v.filters && v.filters.houseId, + // FULL initialState — для raw_payload (полный snapshot для будущего парсинга) + full_state: v, + }; +} +""" + +WAIT_JS = """ +() => { + const arr = window._cianConfig && window._cianConfig['valuation-for-agent-frontend']; + if (!arr) return false; + const entry = arr.find(e => e && e.key === 'initialState'); + if (!entry || !entry.value) return false; + const v = entry.value; + const sale = v.estimation && v.estimation.sale; + // Готово когда либо есть data, либо isError, и не isFetching + if (sale && sale.data && sale.data.price) return true; + if (sale && sale.isError) return true; + return false; +} +""" + +# Cian floor enum +_FLOOR_SPECIAL = {1: "floorOne", 2: "floorTwo"} + + +def cian_floor_enum(floor: int, total_floors: int | None) -> str: + if floor in _FLOOR_SPECIAL: + return _FLOOR_SPECIAL[floor] + if total_floors and floor == total_floors: + return "floorLast" + return "floorOther" + + +def compute_cache_key(address, total_area, rooms, floor, repair, deal): + norm = f"{address.strip().lower()}|{total_area}|{rooms}|{floor}|{repair}|{deal}" + return hashlib.sha256(norm.encode()).hexdigest() + + +def parse_change_pct(value_str: str) -> float | None: + """'6,4%' / '-1.2%' → 6.4 / -1.2""" + if not value_str: + return None + m = re.search(r"-?\d+[\.,]?\d*", value_str) + if not m: + return None + return float(m.group(0).replace(",", ".")) + + +def normalize_address(addr: str) -> str: + """listings.address типа 'улица Машинная, 1В/4 · р-н Ленинский' → 'Екатеринбург, улица Машинная, 1В/4'. + + 1. отрезаем ' · р-н ...' postfix + 2. подставляем 'Екатеринбург, ' если города нет в строке + """ + if not addr: + return addr + # cut at middle-dot (Cian district separator) + addr = re.split(r"\s*[·•]\s*", addr)[0].strip().rstrip(",") + if "екатеринбург" not in addr.lower(): + addr = f"Екатеринбург, {addr}" + return addr + + +def build_calc_url(address: str, area: float, rooms: int, floor: int, total_floors: int | None) -> str: + params = { + "address": normalize_address(address), + "totalArea": str(area), + "roomsCount": str(rooms), + "valuationType": "sale", + "floor[0]": cian_floor_enum(floor, total_floors), + "repairType[0]": "repairTypeCosmetic", + } + return f"https://www.cian.ru/kalkulator-nedvizhimosti/?{urllib.parse.urlencode(params, safe='[]')}" + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--batch", type=int, default=10) + ap.add_argument("--delay", type=float, default=3.0, help="seconds между listings") + ap.add_argument("--wait", type=float, default=8.0, help="max seconds wait until valuation data") + ap.add_argument("--max", type=int, default=5500) + ap.add_argument("--headed", action="store_true") + args = ap.parse_args() + + dsn = os.environ.get("TRADEIN_DB_DSN") + if not dsn: + sys.exit("FATAL: TRADEIN_DB_DSN env required") + enc_key = os.environ.get("COOKIE_ENCRYPTION_KEY") + if not enc_key: + sys.exit("FATAL: COOKIE_ENCRYPTION_KEY env required") + pg_dsn = dsn.replace("postgresql+psycopg://", "postgresql://") + + interrupted = False + + def on_sigint(s, f): + nonlocal interrupted + print("\n!! Ctrl+C — finishing current item") + interrupted = True + signal.signal(signal.SIGINT, on_sigint) + + async def run() -> None: + nonlocal interrupted + cum_ok = cum_fail = cum_skipped = 0 + start = time.time() + last_id = 0 + + def load_cookies() -> tuple[list, datetime | None]: + with psycopg.connect(pg_dsn) as conn: + with conn.cursor() as cur: + cur.execute( + """ + SELECT pgp_sym_decrypt(cookies_encrypted, %s)::text, uploaded_at + FROM cian_session_cookies + WHERE expires_at_estimate > NOW() AND last_invalid_at IS NULL + ORDER BY uploaded_at DESC LIMIT 1 + """, + (enc_key,), + ) + r = cur.fetchone() + if not r: + return [], None + jar = json.loads(r[0]) + out = [] + for name, val in jar.items(): + out.append({ + "name": name, "value": val, + "domain": ".cian.ru", "path": "/", + "secure": True, "sameSite": "Lax", + }) + return out, r[1] + + cookies_from_db, cookies_uploaded_at = load_cookies() + if not cookies_from_db: + sys.exit("FATAL: no valid cookies in cian_session_cookies") + print(f"[init] loaded {len(cookies_from_db)} cookies (uploaded {cookies_uploaded_at})") + + profile_dir = os.environ.get("CIAN_PROFILE_DIR", r"C:\Users\user\cian-backfill\chrome_profile") + os.makedirs(profile_dir, exist_ok=True) + + async with async_playwright() as p: + ctx = await p.chromium.launch_persistent_context( + user_data_dir=profile_dir, + headless=not args.headed, + user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", + viewport={"width": 1366, "height": 900}, + locale="ru-RU", + args=["--disable-blink-features=AutomationControlled"], + ) + # First launch: inject cookies из БД. Subsequent launches: cookies уже в profile. + existing = await ctx.cookies("https://www.cian.ru") + has_auth = any(c.get("name") == "DMIR_AUTH" for c in existing) + if not has_auth and cookies_from_db: + await ctx.add_cookies(cookies_from_db) + print(f"[init] injected {len(cookies_from_db)} cookies into fresh profile") + else: + print(f"[init] reusing profile cookies ({len(existing)} present, DMIR_AUTH={'yes' if has_auth else 'no'})") + page = ctx.pages[0] if ctx.pages else await ctx.new_page() + page.set_default_navigation_timeout(45_000) + page.set_default_timeout(45_000) + + processed = 0 + while not interrupted and processed < args.max: + # 1. Fetch batch of listings WITH полный address + geo + with psycopg.connect(pg_dsn) as conn: + with conn.cursor() as cur: + cur.execute( + """ + SELECT l.id, l.address, l.area_m2, l.rooms, l.floor, l.total_floors + FROM listings l + WHERE l.source = 'cian' + AND l.address IS NOT NULL + AND length(l.address) > 15 -- хотя бы город+улица+номер + AND l.area_m2 IS NOT NULL + AND l.rooms IS NOT NULL + AND l.floor IS NOT NULL + AND l.total_floors IS NOT NULL + AND l.id > %s + AND NOT EXISTS ( + SELECT 1 FROM external_valuations ev + WHERE ev.source='cian_valuation' + AND ev.listing_id = l.id + AND ev.expires_at > NOW() + ) + ORDER BY l.id + LIMIT %s + """, + (last_id, args.batch), + ) + rows = cur.fetchall() + if not rows: + print(f"\n[done] drained — no more listings") + break + print(f"\n[batch] {len(rows)} listings (cum: ok={cum_ok} fail={cum_fail} skipped={cum_skipped})") + + with psycopg.connect(pg_dsn, autocommit=False) as conn: + for (lid, addr, area, rooms, floor, total_floors) in rows: + if interrupted: + break + processed += 1 + last_id = lid + t0 = time.time() + try: + area_f = float(area) + rooms_i = int(rooms) + floor_i = int(floor) + total_floors_i = int(total_floors) if total_floors else None + url = build_calc_url(addr, area_f, rooms_i, floor_i, total_floors_i) + await page.goto(url, wait_until="domcontentloaded") + # Click submit button "Оценить" — без этого Cian не вычисляет + try: + btn = page.get_by_test_id("next-button") + await btn.click(timeout=5000) + except Exception: + pass # button может отсутствовать если URL params не валидны — fallback + try: + await page.wait_for_function(WAIT_JS, timeout=int(args.wait * 1000)) + except Exception: + pass # fall through и попробуем extract — может data есть с задержкой + data = await page.evaluate(EXTRACT_JS) + if data.get("error"): + if data.get("captcha"): + print(f" [{lid}] CAPTCHA — STOP, please solve in --headed mode") + interrupted = True + break + cum_fail += 1 + print(f" [{lid}] err={data['error']} {time.time() - t0:.1f}s") + continue + if not data.get("isAuth"): + # 1. Try fresher cookies из БД + fresh, uploaded = load_cookies() + if fresh and uploaded and (cookies_uploaded_at is None or uploaded > cookies_uploaded_at): + print(f" [{lid}] auth lost — reloading fresh cookies (uploaded {uploaded})") + await ctx.clear_cookies() + await ctx.add_cookies(fresh) + cookies_uploaded_at = uploaded + await page.goto(url, wait_until="domcontentloaded") + try: + await page.wait_for_function(WAIT_JS, timeout=int(args.wait * 1000)) + except Exception: + pass + data = await page.evaluate(EXTRACT_JS) + # 2. Если headed — даём user'у залогиниться вручную в открытом окне + if not data.get("isAuth") and args.headed: + print(f"\n [{lid}] !! AUTH LOST !!") + print(f" → В открытом Chrome окне открой www.cian.ru, ЗАЛОГИНЬСЯ.") + print(f" → После успешного логина нажми Enter здесь чтобы продолжить.") + print(f" → Или Ctrl+C чтобы остановить.") + try: + # blocking input — пользователь логинится сам + loop = asyncio.get_event_loop() + await loop.run_in_executor(None, input, " >>> press Enter when logged in: ") + except (EOFError, KeyboardInterrupt): + interrupted = True + break + # retry same listing + await page.goto(url, wait_until="domcontentloaded") + try: + await page.wait_for_function(WAIT_JS, timeout=int(args.wait * 1000)) + except Exception: + pass + data = await page.evaluate(EXTRACT_JS) + # 3. Headless и всё ещё no auth → STOP + if not data.get("isAuth"): + print(f" [{lid}] not auth — STOP. Запусти с --headed и login manually.") + interrupted = True + break + sale = data.get("sale") + if not sale or not sale.get("price"): + cum_skipped += 1 + print(f" [{lid}] no sale data (address not matched?) {time.time() - t0:.1f}s") + continue + rent = data.get("rent") or {} + chart_obj = data.get("chart_obj") or {} + chart_points = (chart_obj.get("chartData") or {}).get("data") or [] + chart_title = chart_obj.get("title") or {} + chart_change_dir = chart_title.get("change") # 'increase'/'decrease'/'neutral' + chart_change_pct = parse_change_pct(chart_title.get("changeValue") or "") + house_id = data.get("house_id") + cache_key = compute_cache_key(addr, area_f, rooms_i, floor_i, "cosmetic", "sale") + + with conn.cursor() as cur: + cur.execute( + """ + INSERT INTO external_valuations ( + source, cache_key, address, + total_area, rooms_count, floor, total_floors, + repair_type, deal_type, + price_rub, accuracy, + low_price, high_price, + house_id, listing_id, + sale_price_rub, sale_accuracy, + rent_price_rub, rent_accuracy, + chart, chart_change_pct, chart_change_direction, + external_house_id, filters_hash, + raw_payload, fetched_at, expires_at + ) VALUES ( + 'cian_valuation', %s, %s, + CAST(%s AS numeric), %s, %s, %s, + 'cosmetic', 'sale', + CAST(%s AS numeric), CAST(%s AS numeric), + CAST(%s AS numeric), CAST(%s AS numeric), + NULL, CAST(%s AS bigint), + CAST(%s AS numeric), CAST(%s AS numeric), + CAST(%s AS numeric), CAST(%s AS numeric), + CAST(%s AS jsonb), CAST(%s AS numeric), %s, + %s, NULL, + CAST(%s AS jsonb), NOW(), NOW() + INTERVAL '30 days' + ) + ON CONFLICT (source, cache_key) DO UPDATE SET + price_rub = EXCLUDED.price_rub, + accuracy = EXCLUDED.accuracy, + low_price = EXCLUDED.low_price, + high_price = EXCLUDED.high_price, + listing_id = COALESCE(EXCLUDED.listing_id, external_valuations.listing_id), + sale_price_rub = EXCLUDED.sale_price_rub, + sale_accuracy = EXCLUDED.sale_accuracy, + rent_price_rub = EXCLUDED.rent_price_rub, + rent_accuracy = EXCLUDED.rent_accuracy, + chart = EXCLUDED.chart, + chart_change_pct = EXCLUDED.chart_change_pct, + chart_change_direction = EXCLUDED.chart_change_direction, + external_house_id = EXCLUDED.external_house_id, + raw_payload = EXCLUDED.raw_payload, + fetched_at = NOW(), + expires_at = NOW() + INTERVAL '30 days' + """, + ( + cache_key, addr, + area_f, rooms_i, floor_i, total_floors_i, + sale.get("price"), (sale.get("accuracy") or 0) / 100, + sale.get("priceFrom"), sale.get("priceTo"), + lid, + sale.get("price"), (sale.get("accuracy") or 0) / 100, + rent.get("price"), (rent.get("accuracy") or 0) / 100 if rent.get("accuracy") else None, + json.dumps(chart_points), chart_change_pct, + chart_change_dir if chart_change_dir in ("increase", "decrease", "neutral") else None, + house_id, + json.dumps(data.get("full_state") or {"sale": sale, "rent": rent, "chart": chart_obj, "house_id": house_id}), + ), + ) + conn.commit() + cum_ok += 1 + print(f" [{lid}] OK sale={sale.get('price'):,} rent={rent.get('price')} chart={len(chart_points)} chg={chart_change_pct}%/{chart_change_dir} houseId={house_id} {time.time() - t0:.1f}s") + except Exception as exc: + conn.rollback() + cum_fail += 1 + print(f" [{lid}] ERR {type(exc).__name__}: {str(exc)[:140]} {time.time() - t0:.1f}s") + await asyncio.sleep(args.delay) + + await ctx.close() + + print() + print(f"== Summary ==") + print(f" ok={cum_ok} fail={cum_fail} skipped_no_data={cum_skipped}") + print(f" elapsed={time.time() - start:.1f}s finished={datetime.now(timezone.utc).isoformat(timespec='seconds')}") + + asyncio.run(run()) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tradein-mvp/scripts/local-cian/upload_cookies_local.py b/tradein-mvp/scripts/local-cian/upload_cookies_local.py new file mode 100644 index 00000000..dd7215b8 --- /dev/null +++ b/tradein-mvp/scripts/local-cian/upload_cookies_local.py @@ -0,0 +1,65 @@ +"""Локальный upload Cian cookies в tradein-postgres через SSH tunnel. + +Не требует docker exec на проде. Требует: + pip install "psycopg[binary]" + SSH tunnel поднят к tradein-postgres (см. README рядом) + Env: TRADEIN_DB_DSN, COOKIE_ENCRYPTION_KEY +""" +import json +import os +import sys + +import psycopg + +dsn = os.environ.get("TRADEIN_DB_DSN") +key = os.environ.get("COOKIE_ENCRYPTION_KEY") +if not dsn: + sys.exit("FATAL: TRADEIN_DB_DSN env not set") +if not key: + sys.exit("FATAL: COOKIE_ENCRYPTION_KEY env not set") + +cookies = { + "DMIR_AUTH": "", + "_CIAN_GK": "", + "_yasc": "", + "_ym_isad": "", + "uxfb_card_satisfaction": "", +} + +# account_user_id — ваш Cian user ID (найти: DevTools → XHR → /v1/get-my-id/ → userId). +# Используется как ключ в cian_session_cookies. Для локального backfill +# достаточно любого уникального числа (например ваш реальный Cian userId). +ACCOUNT_USER_ID = 999999999 # TODO: замените на реальный Cian userId + +with psycopg.connect(dsn) as conn: + with conn.cursor() as cur: + cur.execute(""" + INSERT INTO cian_session_cookies ( + account_user_id, + cookies_encrypted, + expires_at_estimate, + uploaded_at, + notes + ) VALUES ( + CAST(%s AS bigint), + pgp_sym_encrypt(%s, %s), + NOW() + INTERVAL '30 days', + NOW(), + 'manual local upload via SSH tunnel' + ) + ON CONFLICT (account_user_id) DO UPDATE SET + cookies_encrypted = EXCLUDED.cookies_encrypted, + expires_at_estimate = EXCLUDED.expires_at_estimate, + uploaded_at = NOW(), + last_invalid_at = NULL + """, (ACCOUNT_USER_ID, json.dumps(cookies), key)) + conn.commit() + cur.execute( + "SELECT account_user_id, uploaded_at, expires_at_estimate, " + "octet_length(cookies_encrypted) AS enc_bytes " + "FROM cian_session_cookies WHERE account_user_id = CAST(%s AS bigint)", + (ACCOUNT_USER_ID,), + ) + row = cur.fetchone() + print(f"OK saved: user_id={row[0]} uploaded={row[1]} expires={row[2]} " + f"enc_bytes={row[3]}")