diff --git a/tradein-mvp/backend/app/api/v1/admin.py b/tradein-mvp/backend/app/api/v1/admin.py index a38c47bb..f2c0aa1c 100644 --- a/tradein-mvp/backend/app/api/v1/admin.py +++ b/tradein-mvp/backend/app/api/v1/admin.py @@ -16,6 +16,7 @@ from sqlalchemy.orm import Session from app.core.config import settings from app.core.db import get_db +from app.services import cian_session as cian_session_svc from app.services.geocoder import geocode from app.services.scrapers.avito import AvitoScraper from app.services.scrapers.base import save_listings @@ -233,3 +234,72 @@ async def geocode_missing( "skipped": skipped, "remaining": int(remaining or 0), } + + +@router.post("/scrape/cian/upload-cookies", status_code=200, dependencies=[Depends(require_admin)]) +async def upload_cian_cookies( + cookies: dict[str, str], + db: Annotated[Session, Depends(get_db)], +) -> dict: + """Upload Cian session cookies для аутентификации Valuation Calculator. + + Body: JSON object вида { "_ym_uid": "...", "_cian_visitor_session_id": "...", ... } + Cookies экспортируются из Chrome DevTools → Application → Cookies → www.cian.ru + или через расширение Cookie-Editor → Export → JSON (object format). + + Шаги: + 1. Фильтрует payload по CIAN_REQUIRED_COOKIES. + 2. Верифицирует через GET /kalkulator-nedvizhimosti/ — проверяет isAuthenticated. + 3. Сохраняет зашифрованно (pgp_sym_encrypt) в cian_session_cookies. + + Returns: {"ok": true, "userId": , "cookieCount": } + """ + if not settings.cookie_encryption_key: + raise HTTPException(status_code=503, detail="COOKIE_ENCRYPTION_KEY not configured") + + cleaned = {k: v for k, v in cookies.items() if k in cian_session_svc.CIAN_REQUIRED_COOKIES} + if not cleaned: + raise HTTPException( + status_code=400, + detail=( + f"No recognized Cian cookies in payload. " + f"Expected any of: {sorted(cian_session_svc.CIAN_REQUIRED_COOKIES)}" + ), + ) + + state = await cian_session_svc.verify_session(cleaned) + if state is None: + raise HTTPException( + status_code=401, + detail="Cookies invalid or session not authenticated on cian.ru", + ) + + user_id = state.get("user", {}).get("userId") + if not user_id: + raise HTTPException(status_code=400, detail="Authenticated state missing userId") + + cian_session_svc.save_session(db, account_user_id=int(user_id), cookies=cleaned) + return {"ok": True, "userId": user_id, "cookieCount": len(cleaned)} + + +@router.get("/scrape/cian/test-auth", status_code=200, dependencies=[Depends(require_admin)]) +async def test_cian_auth( + db: Annotated[Session, Depends(get_db)], +) -> dict: + """Проверить что текущие сохранённые Cian cookies ещё валидны. + + Returns: {"authenticated": bool, "userId": , "reason": } + """ + if not settings.cookie_encryption_key: + return {"authenticated": False, "userId": None, "reason": "encryption_key_not_configured"} + + cookies = cian_session_svc.load_session(db) + if cookies is None: + return {"authenticated": False, "userId": None, "reason": "no_session_in_db"} + + state = await cian_session_svc.verify_session(cookies) + if state is None: + return {"authenticated": False, "userId": None, "reason": "session_expired_or_invalid"} + + user_id = state.get("user", {}).get("userId") + return {"authenticated": True, "userId": user_id, "reason": None} diff --git a/tradein-mvp/backend/app/core/config.py b/tradein-mvp/backend/app/core/config.py index dcf7b42c..890bad97 100644 --- a/tradein-mvp/backend/app/core/config.py +++ b/tradein-mvp/backend/app/core/config.py @@ -29,5 +29,9 @@ class Settings(BaseSettings): # Пусто = мониторинг выключен (dev). В prod — env GLITCHTIP_DSN из .env.runtime. glitchtip_dsn: str | None = None + # Ключ шифрования для pgp_sym_encrypt (Cian session cookies). + # Задаётся через env COOKIE_ENCRYPTION_KEY. Пусто = шифрование не работает. + cookie_encryption_key: str = "" + settings = Settings() diff --git a/tradein-mvp/backend/app/services/cian_session.py b/tradein-mvp/backend/app/services/cian_session.py new file mode 100644 index 00000000..616fcbb5 --- /dev/null +++ b/tradein-mvp/backend/app/services/cian_session.py @@ -0,0 +1,196 @@ +"""Cian session cookie management — load/save/verify encrypted cookies. + +Used by cian_valuation.py (Stage 7) to authenticate Valuation Calculator API. +Cookies stored encrypted (pgp_sym_encrypt) in cian_session_cookies table. +""" +from __future__ import annotations + +import json +import logging +from typing import Any + +import httpx +from sqlalchemy import text +from sqlalchemy.orm import Session + +from app.core.config import settings +from app.services.scrapers.cian_state_parser import extract_state + +logger = logging.getLogger(__name__) + +# Cookies критичные для Cian auth — фильтр перед сохранением. +# Conservative set подтверждён в Stage 4 (probe через DevTools на live session). +CIAN_REQUIRED_COOKIES: set[str] = { + "_ym_uid", + "_ym_d", + "_ym_isad", + "_cian_visitor_session_id", + "_cian_app_session_id", + "_cian_uid", + "_ga", + "tlsr_id", + "cf_clearance", + "session-id", + "anti_bot", + "csrftoken", +} + +_VALUATION_TEST_URL = ( + "https://www.cian.ru/kalkulator-nedvizhimosti/" + "?address=%D0%A1%D0%B2%D0%B5%D1%80%D0%B4%D0%BB%D0%BE%D0%B2%D1%81%D0%BA%D0%B0%D1%8F" + "%20%D0%BE%D0%B1%D0%BB%2C%20%D0%95%D0%BA%D0%B0%D1%82%D0%B5%D1%80%D0%B8%D0%BD%D0%B1%D1%83%D1%80%D0%B3" + "&totalArea=40&roomsCount=1&valuationType=sale" + "&floor%5B0%5D=floorOther&repairType%5B0%5D=repairTypeCosmetic" +) + +_MFE_VALUATION = "valuation-for-agent-frontend" + + +async def verify_session(cookies: dict[str, str]) -> dict[str, Any] | None: + """Hit Cian Valuation Calculator with cookies — return parsed state if authenticated. + + Returns state dict containing user.isAuthenticated + userId, or None if + cookies are invalid/expired or state extraction fails. + Никогда не логирует сырые значения cookies. + """ + try: + async with httpx.AsyncClient( + cookies=cookies, + timeout=20.0, + follow_redirects=True, + ) as client: + resp = await client.get(_VALUATION_TEST_URL) + resp.raise_for_status() + state = extract_state(resp.text, mfe=_MFE_VALUATION, key="initialState") + if state is None: + logger.warning( + "Cian cookies verify: state extraction failed (mfe=%s)", _MFE_VALUATION + ) + return None + user = state.get("user", {}) + if not user.get("isAuthenticated"): + logger.warning("Cian cookies verify: user.isAuthenticated=false") + return None + logger.info("Cian cookies verified — userId=%s", user.get("userId")) + return state + except httpx.HTTPStatusError as exc: + logger.warning( + "Cian cookies verify HTTP error: %s %s", + exc.response.status_code, + exc.request.url, + ) + return None + except Exception as exc: + logger.warning("Cian cookies verify failed: %s", exc) + return None + + +def save_session( + db: Session, + account_user_id: int, + cookies: dict[str, str], + ttl_days: int = 30, +) -> None: + """Encrypt cookies via pgp_sym_encrypt and UPSERT into cian_session_cookies. + + Uses settings.cookie_encryption_key as the encryption secret. + Никогда не логирует сырые значения cookies. + """ + cookies_json = json.dumps(cookies) + db.execute( + text(""" + INSERT INTO cian_session_cookies ( + account_user_id, + cookies_encrypted, + expires_at_estimate, + uploaded_at + ) VALUES ( + CAST(:uid AS bigint), + pgp_sym_encrypt(:cookies_json, :key), + NOW() + (CAST(:ttl_days AS int) || ' days')::interval, + NOW() + ) + ON CONFLICT (account_user_id) DO UPDATE SET + cookies_encrypted = EXCLUDED.cookies_encrypted, + expires_at_estimate = EXCLUDED.expires_at_estimate, + uploaded_at = NOW(), + last_invalid_at = NULL + """), + { + "uid": account_user_id, + "cookies_json": cookies_json, + "key": settings.cookie_encryption_key, + "ttl_days": ttl_days, + }, + ) + db.commit() + logger.info( + "Cian cookies saved for userId=%s (count=%d, ttl=%d days)", + account_user_id, + len(cookies), + ttl_days, + ) + + +def load_session(db: Session) -> dict[str, str] | None: + """Load most-recently-uploaded valid Cian cookies (decrypt). + + Returns dict[cookie_name, cookie_value] или None если нет валидной session. + Выбирает только записи где expires_at_estimate > NOW() и сессия не + была инвалидирована после последнего upload'а. + """ + row = db.execute( + text(""" + SELECT + account_user_id, + pgp_sym_decrypt(cookies_encrypted, :key)::text AS cookies_json, + expires_at_estimate + FROM cian_session_cookies + WHERE expires_at_estimate > NOW() + AND (last_invalid_at IS NULL OR last_invalid_at < uploaded_at) + ORDER BY uploaded_at DESC + LIMIT 1 + """), + {"key": settings.cookie_encryption_key}, + ).mappings().first() + + if row is None: + logger.warning("No valid Cian session cookies in DB") + return None + + cookies: dict[str, str] = json.loads(row["cookies_json"]) + + # Обновляем last_used_at — не критично, игнорируем ошибки. + try: + db.execute( + text( + "UPDATE cian_session_cookies SET last_used_at = NOW()" + " WHERE account_user_id = CAST(:uid AS bigint)" + ), + {"uid": row["account_user_id"]}, + ) + db.commit() + except Exception as exc: + logger.warning( + "Failed to update last_used_at for userId=%s: %s", row["account_user_id"], exc + ) + + logger.info( + "Cian cookies loaded for userId=%s (count=%d)", + row["account_user_id"], + len(cookies), + ) + return cookies + + +def mark_session_invalid(db: Session, account_user_id: int) -> None: + """Flag session как expired/invalid (например после 401 во время scrape).""" + db.execute( + text( + "UPDATE cian_session_cookies SET last_invalid_at = NOW()" + " WHERE account_user_id = CAST(:uid AS bigint)" + ), + {"uid": account_user_id}, + ) + db.commit() + logger.warning("Cian session marked invalid for userId=%s", account_user_id) diff --git a/tradein-mvp/backend/tests/test_cian_session.py b/tradein-mvp/backend/tests/test_cian_session.py new file mode 100644 index 00000000..82906028 --- /dev/null +++ b/tradein-mvp/backend/tests/test_cian_session.py @@ -0,0 +1,238 @@ +"""Tests for cian_session — cookie management service.""" +from __future__ import annotations + +import json +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from app.services.cian_session import ( + CIAN_REQUIRED_COOKIES, + load_session, + mark_session_invalid, + save_session, + verify_session, +) + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture() +def mock_db() -> MagicMock: + db = MagicMock() + # Default: no row found (load_session → None) + db.execute.return_value.mappings.return_value.first.return_value = None + return db + + +# --------------------------------------------------------------------------- +# CIAN_REQUIRED_COOKIES +# --------------------------------------------------------------------------- + + +def test_required_cookies_set_not_empty() -> None: + assert isinstance(CIAN_REQUIRED_COOKIES, set) + assert len(CIAN_REQUIRED_COOKIES) >= 5 + + +def test_required_cookies_contains_key_names() -> None: + assert "_cian_visitor_session_id" in CIAN_REQUIRED_COOKIES + assert "_ym_uid" in CIAN_REQUIRED_COOKIES + assert "_cian_uid" in CIAN_REQUIRED_COOKIES + assert "tlsr_id" in CIAN_REQUIRED_COOKIES + assert "cf_clearance" in CIAN_REQUIRED_COOKIES + + +# --------------------------------------------------------------------------- +# save_session +# --------------------------------------------------------------------------- + + +def test_save_session_calls_pgp_sym_encrypt(mock_db: MagicMock) -> None: + save_session(mock_db, account_user_id=12345, cookies={"_ym_uid": "abc"}) + args, _ = mock_db.execute.call_args + sql_text = str(args[0]) + assert "pgp_sym_encrypt" in sql_text + assert "cian_session_cookies" in sql_text + + +def test_save_session_uses_cast_not_colon_colon(mock_db: MagicMock) -> None: + """Verify psycopg v3 compatibility — no ::bigint syntax in SQL.""" + save_session(mock_db, account_user_id=99, cookies={"csrftoken": "x"}) + args, _ = mock_db.execute.call_args + sql_text = str(args[0]) + # CAST(:x AS type) is required; ::type would fail with psycopg v3 named params + assert "CAST(" in sql_text + assert "::bigint" not in sql_text + + +def test_save_session_commits(mock_db: MagicMock) -> None: + save_session(mock_db, account_user_id=12345, cookies={"_ym_uid": "abc"}) + assert mock_db.commit.called + + +def test_save_session_on_conflict_do_update(mock_db: MagicMock) -> None: + save_session(mock_db, account_user_id=1, cookies={"_ym_uid": "v"}) + args, _ = mock_db.execute.call_args + sql_text = str(args[0]) + assert "ON CONFLICT" in sql_text + assert "DO UPDATE" in sql_text + + +def test_save_session_passes_correct_params(mock_db: MagicMock) -> None: + save_session(mock_db, account_user_id=777, cookies={"_ga": "GA1.2.x"}, ttl_days=14) + _, kwargs = mock_db.execute.call_args + params = kwargs if kwargs else mock_db.execute.call_args[0][1] + # params могут передаваться как второй позиционный аргумент + call_args = mock_db.execute.call_args + params = call_args[0][1] if len(call_args[0]) > 1 else call_args[1].get("params", {}) + assert params["uid"] == 777 + assert params["ttl_days"] == 14 + cookies_payload = json.loads(params["cookies_json"]) + assert cookies_payload == {"_ga": "GA1.2.x"} + + +# --------------------------------------------------------------------------- +# load_session +# --------------------------------------------------------------------------- + + +def test_load_session_returns_none_when_empty(mock_db: MagicMock) -> None: + result = load_session(mock_db) + assert result is None + + +def test_load_session_decodes_json(mock_db: MagicMock) -> None: + mock_row = { + "account_user_id": 12345, + "cookies_json": json.dumps({"_ym_uid": "abc", "_cian_uid": "def"}), + "expires_at_estimate": None, + } + mock_db.execute.return_value.mappings.return_value.first.return_value = mock_row + result = load_session(mock_db) + assert result == {"_ym_uid": "abc", "_cian_uid": "def"} + + +def test_load_session_updates_last_used_at(mock_db: MagicMock) -> None: + mock_row = { + "account_user_id": 42, + "cookies_json": json.dumps({"session-id": "s"}), + "expires_at_estimate": None, + } + mock_db.execute.return_value.mappings.return_value.first.return_value = mock_row + load_session(mock_db) + # Должно быть минимум 2 вызова execute: SELECT + UPDATE last_used_at + assert mock_db.execute.call_count >= 2 + + +# --------------------------------------------------------------------------- +# mark_session_invalid +# --------------------------------------------------------------------------- + + +def test_mark_session_invalid_updates_table(mock_db: MagicMock) -> None: + mark_session_invalid(mock_db, account_user_id=12345) + args, _ = mock_db.execute.call_args + sql = str(args[0]) + assert "last_invalid_at" in sql + assert "cian_session_cookies" in sql + + +def test_mark_session_invalid_commits(mock_db: MagicMock) -> None: + mark_session_invalid(mock_db, account_user_id=99) + assert mock_db.commit.called + + +def test_mark_session_invalid_uses_cast(mock_db: MagicMock) -> None: + mark_session_invalid(mock_db, account_user_id=5) + args, _ = mock_db.execute.call_args + sql = str(args[0]) + assert "CAST(" in sql + assert "::bigint" not in sql + + +# --------------------------------------------------------------------------- +# verify_session (async) +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_verify_session_returns_none_when_state_missing(monkeypatch: pytest.MonkeyPatch) -> None: + """extract_state returns None → verify_session returns None.""" + monkeypatch.setattr( + "app.services.cian_session.extract_state", + lambda html, mfe, key: None, + ) + + async def fake_get(url: str, **kwargs): # noqa: ARG001 + class FakeResp: + text = "" + def raise_for_status(self) -> None: + pass + return FakeResp() + + with patch("httpx.AsyncClient") as mock_client_cls: + mock_client = AsyncMock() + mock_client.__aenter__ = AsyncMock(return_value=mock_client) + mock_client.__aexit__ = AsyncMock(return_value=None) + mock_client.get = AsyncMock(return_value=_make_fake_resp("")) + mock_client_cls.return_value = mock_client + + result = await verify_session({"_ym_uid": "x"}) + assert result is None + + +@pytest.mark.asyncio +async def test_verify_session_returns_none_when_not_authenticated(monkeypatch: pytest.MonkeyPatch) -> None: + """state.user.isAuthenticated is false → verify_session returns None.""" + monkeypatch.setattr( + "app.services.cian_session.extract_state", + lambda html, mfe, key: {"user": {"isAuthenticated": False, "userId": None}}, + ) + + with patch("httpx.AsyncClient") as mock_client_cls: + mock_client = AsyncMock() + mock_client.__aenter__ = AsyncMock(return_value=mock_client) + mock_client.__aexit__ = AsyncMock(return_value=None) + mock_client.get = AsyncMock(return_value=_make_fake_resp("")) + mock_client_cls.return_value = mock_client + + result = await verify_session({"_ym_uid": "x"}) + assert result is None + + +@pytest.mark.asyncio +async def test_verify_session_returns_state_when_authenticated(monkeypatch: pytest.MonkeyPatch) -> None: + """state.user.isAuthenticated is true → returns state dict.""" + expected_state = {"user": {"isAuthenticated": True, "userId": 102963817}} + monkeypatch.setattr( + "app.services.cian_session.extract_state", + lambda html, mfe, key: expected_state, + ) + + with patch("httpx.AsyncClient") as mock_client_cls: + mock_client = AsyncMock() + mock_client.__aenter__ = AsyncMock(return_value=mock_client) + mock_client.__aexit__ = AsyncMock(return_value=None) + mock_client.get = AsyncMock(return_value=_make_fake_resp("state here")) + mock_client_cls.return_value = mock_client + + result = await verify_session({"_ym_uid": "x", "_cian_uid": "y"}) + assert result is not None + assert result["user"]["isAuthenticated"] is True + assert result["user"]["userId"] == 102963817 + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_fake_resp(text: str) -> MagicMock: + resp = MagicMock() + resp.text = text + resp.raise_for_status = MagicMock() + return resp diff --git a/tradein-mvp/scripts/upload-cian-cookies.sh b/tradein-mvp/scripts/upload-cian-cookies.sh new file mode 100644 index 00000000..57be0858 --- /dev/null +++ b/tradein-mvp/scripts/upload-cian-cookies.sh @@ -0,0 +1,50 @@ +#!/bin/bash +# Upload Cian session cookies to backend for Valuation Calculator auth. +# +# Usage: ./scripts/upload-cian-cookies.sh [cookies.json] +# +# cookies.json — JSON object { cookie_name: cookie_value, ... } +# Экспортируется из Chrome DevTools → Application → Cookies → www.cian.ru +# или через расширение Cookie-Editor → Export → JSON (object format). +# +# Переменные окружения: +# TRADEIN_ADMIN_TOKEN — admin token (или берётся из .env.runtime) +# TRADEIN_BASE_URL — base URL (default: https://gendsgn.ru/trade-in) + +set -euo pipefail + +COOKIES_FILE="${1:-cookies.json}" + +if [[ ! -f "$COOKIES_FILE" ]]; then + echo "Error: cookies file not found: $COOKIES_FILE" >&2 + echo "Usage: $0 cookies.json" >&2 + exit 1 +fi + +# Загружаем токен из env или .env.runtime +ADMIN_TOKEN="${TRADEIN_ADMIN_TOKEN:-}" +if [[ -z "$ADMIN_TOKEN" ]] && [[ -f ".env.runtime" ]]; then + ADMIN_TOKEN="$(grep '^TRADEIN_ADMIN_TOKEN=' .env.runtime 2>/dev/null | cut -d= -f2- || true)" +fi + +if [[ -z "$ADMIN_TOKEN" ]]; then + echo "Error: TRADEIN_ADMIN_TOKEN not set (env or .env.runtime)" >&2 + exit 1 +fi + +BASE_URL="${TRADEIN_BASE_URL:-https://gendsgn.ru/trade-in}" + +echo "Uploading Cian cookies from $COOKIES_FILE to $BASE_URL ..." +echo + +curl -fsSL -X POST "$BASE_URL/api/v1/admin/scrape/cian/upload-cookies" \ + -H "X-Admin-Token: $ADMIN_TOKEN" \ + -H "Content-Type: application/json" \ + -d "@$COOKIES_FILE" \ + | python3 -m json.tool + +echo +echo "Verifying session:" +curl -fsSL "$BASE_URL/api/v1/admin/scrape/cian/test-auth" \ + -H "X-Admin-Token: $ADMIN_TOKEN" \ + | python3 -m json.tool