lekss361
35f5c3426b
All checks were successful
Deploy Trade-In / changes (push) Successful in 12s
Deploy Trade-In / build-frontend (push) Has been skipped
Deploy Trade-In / build-browser (push) Has been skipped
Deploy Trade-In / test (push) Successful in 4m50s
Deploy Trade-In / build-backend (push) Successful in 1m34s
Deploy Trade-In / deploy (push) Successful in 1m34s
307 lines
14 KiB
Python
307 lines
14 KiB
Python
"""Cian session cookie management — load/save/verify encrypted cookies.
|
||
|
||
Used by cian_valuation.py (Stage 7) to authenticate Valuation Calculator API.
|
||
Cookies stored encrypted (pgp_sym_encrypt) in cian_session_cookies table.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import logging
|
||
from typing import Any
|
||
|
||
from curl_cffi.requests import AsyncSession
|
||
|
||
# #2306: extract_state migrated to scraper_kit (byte-identical to legacy
|
||
# cian_state_parser module, deleted #2397 финальный шаг E — golden-parity против
|
||
# него уже недоступна, kit — единственный живой путь).
|
||
from scraper_kit.cian_state_parser import extract_state
|
||
from sqlalchemy import text
|
||
from sqlalchemy.orm import Session
|
||
|
||
from app.core.config import settings
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
# Cookies критичные для Cian auth — фильтр перед сохранением.
|
||
# Список обновлён по реальному DevTools-дампу из logged-in сессии cian.ru (2026-05-23).
|
||
# Старые записи оставлены как fallback (backward compat).
|
||
CIAN_REQUIRED_COOKIES: set[str] = {
|
||
# --- Cian auth & session (critical) ---
|
||
"DMIR_AUTH", # Cian auth token (httpOnly) — основной auth-сигнал
|
||
"_CIAN_GK", # Cian session GUID
|
||
# --- Yandex Metrika (обычно присутствуют, не critical) ---
|
||
"_ym_uid",
|
||
"_ym_d",
|
||
"_ym_isad",
|
||
"_ym_visorc",
|
||
"_yasc", # Yandex anti-spam
|
||
# --- Cian UX state ---
|
||
"uxfb_card_satisfaction",
|
||
# --- Cian session IDs (fallback / legacy deployments) ---
|
||
"_cian_visitor_session_id",
|
||
"_cian_app_session_id",
|
||
"_cian_uid",
|
||
# --- Misc (legacy / optional) ---
|
||
"_ga", # Google Analytics
|
||
"tlsr_id", # legacy fingerprint
|
||
"cf_clearance", # Cloudflare bot check
|
||
"session-id", # generic session cookie
|
||
"anti_bot",
|
||
"csrftoken",
|
||
}
|
||
|
||
_VALUATION_TEST_URL = (
|
||
"https://www.cian.ru/kalkulator-nedvizhimosti/"
|
||
"?address=%D0%A1%D0%B2%D0%B5%D1%80%D0%B4%D0%BB%D0%BE%D0%B2%D1%81%D0%BA%D0%B0%D1%8F"
|
||
"%20%D0%BE%D0%B1%D0%BB%2C%20%D0%95%D0%BA%D0%B0%D1%82%D0%B5%D1%80%D0%B8%D0%BD%D0%B1%D1%83%D1%80%D0%B3"
|
||
"&totalArea=40&roomsCount=1&valuationType=sale"
|
||
"&floor%5B0%5D=floorOther&repairType%5B0%5D=repairTypeCosmetic"
|
||
)
|
||
|
||
_MFE_VALUATION = "valuation-for-agent-frontend"
|
||
# #639 (2026-05): auth-сигнал (user.isAuthenticated/userId) уехал из valuation-MFE
|
||
# (там user теперь всегда anonymous) в site-header MFE `header-frontend`, который
|
||
# присутствует на любой cian-странице. Verify читает auth именно оттуда.
|
||
_MFE_AUTH = "header-frontend"
|
||
|
||
# Returned by verify_session when a TLS/bot-fingerprint ban (HTTP 403) is detected.
|
||
# Callers that check `if state is None` will NOT treat this as "expired cookies" —
|
||
# the sentinel is truthy (non-None), so cookie-refresh alerts are suppressed.
|
||
# Callers that need to distinguish ban from valid auth should check state.get("_ban").
|
||
VERIFY_BAN_SENTINEL: dict[str, Any] = {"_ban": True}
|
||
|
||
# audit-scrapers finding 4: verify_session раньше сводило 5xx / сетевой сбой /
|
||
# смену вёрстки к тому же None, что и реальный логаут (401 / isAuthenticated=false) —
|
||
# вызывающие (_cian_pre_claim, admin upload/auto-login) реагировали "куки протухли,
|
||
# перезалей" там, где куки ни при чём (Cian недоступен ИЛИ scraper_kit.cian_state_parser
|
||
# больше не находит header-frontend initialState). Два отдельных сигнала ниже НЕ
|
||
# триггерят "cookies expired" алерт у вызывающих.
|
||
|
||
# Cian источник недоступен прямо сейчас (5xx-ответ ИЛИ сетевой/транспортный сбой —
|
||
# timeout, DNS, connection reset). Cookies могут быть абсолютно валидны — просто
|
||
# нечем было их проверить. Retry позже, БЕЗ пометки session invalid.
|
||
VERIFY_SOURCE_UNAVAILABLE_SENTINEL: dict[str, Any] = {"_source_unavailable": True}
|
||
|
||
# HTTP 200 получен, но ожидаемый auth-state (header-frontend/initialState с
|
||
# user.isAuthenticated) не найден/не распарсился — Cian изменил вёрстку/MFE-схему.
|
||
# Это engineering-проблема (extract_state/_MFE_AUTH нужно обновить), НЕ протухшие
|
||
# cookies — переставлять куки здесь бесполезно.
|
||
VERIFY_MARKUP_CHANGED_SENTINEL: dict[str, Any] = {"_markup_changed": True}
|
||
|
||
|
||
def _classify_verify_response(
|
||
status_code: int,
|
||
html: str | None,
|
||
) -> dict[str, Any] | None:
|
||
"""Pure classifier — maps (status_code, html) to verify_session outcome.
|
||
|
||
Returns:
|
||
VERIFY_BAN_SENTINEL — 403/TLS ban (cookies могут быть в порядке,
|
||
блокирует сервер)
|
||
VERIFY_SOURCE_UNAVAILABLE_SENTINEL — 5xx/иной non-200 без содержимого —
|
||
источник недоступен, НЕ cookies
|
||
VERIFY_MARKUP_CHANGED_SENTINEL — HTTP 200, но auth-state не найден/не
|
||
распарсился — вёрстка/схема изменилась
|
||
None — 401 ИЛИ isAuthenticated=false — cookies
|
||
ДЕЙСТВИТЕЛЬНО протухли/разлогинены
|
||
state dict — authenticated successfully
|
||
"""
|
||
if status_code == 403:
|
||
return VERIFY_BAN_SENTINEL
|
||
if status_code == 401:
|
||
return None
|
||
if status_code != 200 or html is None:
|
||
return VERIFY_SOURCE_UNAVAILABLE_SENTINEL
|
||
state = extract_state(html, mfe=_MFE_AUTH, key="initialState")
|
||
if state is None:
|
||
return VERIFY_MARKUP_CHANGED_SENTINEL
|
||
user = state.get("user", {}) or {}
|
||
if not user.get("isAuthenticated"):
|
||
return None
|
||
return state
|
||
|
||
|
||
async def verify_session(cookies: dict[str, str]) -> dict[str, Any] | None:
|
||
"""Hit Cian Valuation Calculator with cookies — return parsed state if authenticated.
|
||
|
||
Uses curl_cffi with impersonate='chrome120' (same as prod scrapers) to avoid
|
||
TLS-fingerprint bans that httpx would trigger.
|
||
|
||
Returns (проверяй через `is`, НЕ `==` — это sentinel-объекты):
|
||
state dict — authenticated (user.isAuthenticated + userId)
|
||
VERIFY_BAN_SENTINEL — HTTP 403 TLS/bot ban; cookies могут быть
|
||
валидны — НЕ триггерить cookie-refresh alert
|
||
VERIFY_SOURCE_UNAVAILABLE_SENTINEL — 5xx/network/timeout; источник недоступен,
|
||
НЕ триггерить cookie-refresh alert, retry позже
|
||
VERIFY_MARKUP_CHANGED_SENTINEL — HTTP 200 но auth-state не распарсился;
|
||
Cian изменил вёрстку — НЕ cookie-проблема,
|
||
нужен engineering-фикс extract_state/_MFE_AUTH
|
||
None — HTTP 401 или isAuthenticated=false; cookies
|
||
ДЕЙСТВИТЕЛЬНО протухли — здесь и только здесь
|
||
имеет смысл просить re-upload
|
||
|
||
Никогда не логирует сырые значения cookies.
|
||
"""
|
||
try:
|
||
# proxies: mobile-proxy egress (#806) — Cian блокирует datacenter-IP даже
|
||
# при валидных DMIR_AUTH cookies. Без прокси verify всегда вернёт 403.
|
||
# Пусто (env не задан) → прямое подключение (dev/no-op).
|
||
_proxy_url = settings.cian_proxy_url
|
||
_proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None
|
||
async with AsyncSession(
|
||
impersonate="chrome120",
|
||
cookies=cookies,
|
||
timeout=20.0,
|
||
proxies=_proxies,
|
||
) as session:
|
||
resp = await session.get(_VALUATION_TEST_URL, allow_redirects=True)
|
||
status = resp.status_code
|
||
html: str | None = resp.text if status == 200 else None
|
||
|
||
result = _classify_verify_response(status, html)
|
||
|
||
if result is VERIFY_BAN_SENTINEL:
|
||
logger.warning(
|
||
"Cian cookies verify: HTTP 403 TLS/bot ban — cookies NOT marked expired"
|
||
)
|
||
elif result is VERIFY_SOURCE_UNAVAILABLE_SENTINEL:
|
||
logger.warning(
|
||
"Cian cookies verify: source unavailable (status=%d) — "
|
||
"cookies NOT marked expired, retry later",
|
||
status,
|
||
)
|
||
elif result is VERIFY_MARKUP_CHANGED_SENTINEL:
|
||
logger.error(
|
||
"Cian cookies verify: HTTP 200 but auth-state not found/parseable "
|
||
"(mfe=%s) — markup/schema changed, cookies NOT marked expired",
|
||
_MFE_AUTH,
|
||
)
|
||
elif result is None:
|
||
logger.warning("Cian cookies verify: expired/unauthenticated (status=%d)", status)
|
||
else:
|
||
user = result.get("user", {}) or {}
|
||
logger.info("Cian cookies verified — userId=%s", user.get("userId"))
|
||
|
||
return result
|
||
except Exception as exc:
|
||
# Сетевой/транспортный сбой (timeout, DNS, connection reset и т.п.) — источник
|
||
# недоступен, НЕ признак протухших cookies (finding 4). Раньше здесь везде
|
||
# возвращался None, конфлируя с реальным логаутом.
|
||
logger.warning("Cian cookies verify: transport/network error — %s", exc)
|
||
return VERIFY_SOURCE_UNAVAILABLE_SENTINEL
|
||
|
||
|
||
def save_session(
|
||
db: Session,
|
||
account_user_id: int,
|
||
cookies: dict[str, str],
|
||
ttl_days: int = 30,
|
||
) -> None:
|
||
"""Encrypt cookies via pgp_sym_encrypt and UPSERT into cian_session_cookies.
|
||
|
||
Uses settings.cookie_encryption_key as the encryption secret.
|
||
Никогда не логирует сырые значения cookies.
|
||
"""
|
||
cookies_json = json.dumps(cookies)
|
||
db.execute(
|
||
text("""
|
||
INSERT INTO cian_session_cookies (
|
||
account_user_id,
|
||
cookies_encrypted,
|
||
expires_at_estimate,
|
||
uploaded_at
|
||
) VALUES (
|
||
CAST(:uid AS bigint),
|
||
pgp_sym_encrypt(:cookies_json, :key),
|
||
NOW() + (CAST(:ttl_days AS int) || ' days')::interval,
|
||
NOW()
|
||
)
|
||
ON CONFLICT (account_user_id) DO UPDATE SET
|
||
cookies_encrypted = EXCLUDED.cookies_encrypted,
|
||
expires_at_estimate = EXCLUDED.expires_at_estimate,
|
||
uploaded_at = NOW(),
|
||
last_invalid_at = NULL
|
||
"""),
|
||
{
|
||
"uid": account_user_id,
|
||
"cookies_json": cookies_json,
|
||
"key": settings.cookie_encryption_key,
|
||
"ttl_days": ttl_days,
|
||
},
|
||
)
|
||
db.commit()
|
||
logger.info(
|
||
"Cian cookies saved for userId=%s (count=%d, ttl=%d days)",
|
||
account_user_id,
|
||
len(cookies),
|
||
ttl_days,
|
||
)
|
||
|
||
|
||
def load_session(db: Session) -> dict[str, str] | None:
|
||
"""Load most-recently-uploaded valid Cian cookies (decrypt).
|
||
|
||
Returns dict[cookie_name, cookie_value] или None если нет валидной session.
|
||
Выбирает только записи где expires_at_estimate > NOW() и сессия не
|
||
была инвалидирована после последнего upload'а.
|
||
"""
|
||
row = (
|
||
db.execute(
|
||
text("""
|
||
SELECT
|
||
account_user_id,
|
||
pgp_sym_decrypt(cookies_encrypted, :key)::text AS cookies_json,
|
||
expires_at_estimate
|
||
FROM cian_session_cookies
|
||
WHERE expires_at_estimate > NOW()
|
||
AND (last_invalid_at IS NULL OR last_invalid_at < uploaded_at)
|
||
ORDER BY uploaded_at DESC
|
||
LIMIT 1
|
||
"""),
|
||
{"key": settings.cookie_encryption_key},
|
||
)
|
||
.mappings()
|
||
.first()
|
||
)
|
||
|
||
if row is None:
|
||
logger.warning("No valid Cian session cookies in DB")
|
||
return None
|
||
|
||
cookies: dict[str, str] = json.loads(row["cookies_json"])
|
||
|
||
# Обновляем last_used_at — не критично, игнорируем ошибки.
|
||
try:
|
||
db.execute(
|
||
text(
|
||
"UPDATE cian_session_cookies SET last_used_at = NOW()"
|
||
" WHERE account_user_id = CAST(:uid AS bigint)"
|
||
),
|
||
{"uid": row["account_user_id"]},
|
||
)
|
||
db.commit()
|
||
except Exception as exc:
|
||
logger.warning(
|
||
"Failed to update last_used_at for userId=%s: %s", row["account_user_id"], exc
|
||
)
|
||
|
||
logger.info(
|
||
"Cian cookies loaded for userId=%s (count=%d)",
|
||
row["account_user_id"],
|
||
len(cookies),
|
||
)
|
||
return cookies
|
||
|
||
|
||
def mark_session_invalid(db: Session, account_user_id: int) -> None:
|
||
"""Flag session как expired/invalid (например после 401 во время scrape)."""
|
||
db.execute(
|
||
text(
|
||
"UPDATE cian_session_cookies SET last_invalid_at = NOW()"
|
||
" WHERE account_user_id = CAST(:uid AS bigint)"
|
||
),
|
||
{"uid": account_user_id},
|
||
)
|
||
db.commit()
|
||
logger.warning("Cian session marked invalid for userId=%s", account_user_id)
|