feat(tradein): cian_session cookies management + admin endpoints (Wave 4 Worker A) (#457)
This commit is contained in:
parent
962daf2d47
commit
2dd76ea5e8
7 changed files with 1206 additions and 0 deletions
|
|
@ -16,6 +16,7 @@ from sqlalchemy.orm import Session
|
|||
|
||||
from app.core.config import settings
|
||||
from app.core.db import get_db
|
||||
from app.services import cian_session as cian_session_svc
|
||||
from app.services.geocoder import geocode
|
||||
from app.services.scrapers.avito import AvitoScraper
|
||||
from app.services.scrapers.base import save_listings
|
||||
|
|
@ -233,3 +234,72 @@ async def geocode_missing(
|
|||
"skipped": skipped,
|
||||
"remaining": int(remaining or 0),
|
||||
}
|
||||
|
||||
|
||||
@router.post("/scrape/cian/upload-cookies", status_code=200, dependencies=[Depends(require_admin)])
|
||||
async def upload_cian_cookies(
|
||||
cookies: dict[str, str],
|
||||
db: Annotated[Session, Depends(get_db)],
|
||||
) -> dict:
|
||||
"""Upload Cian session cookies для аутентификации Valuation Calculator.
|
||||
|
||||
Body: JSON object вида { "_ym_uid": "...", "_cian_visitor_session_id": "...", ... }
|
||||
Cookies экспортируются из Chrome DevTools → Application → Cookies → www.cian.ru
|
||||
или через расширение Cookie-Editor → Export → JSON (object format).
|
||||
|
||||
Шаги:
|
||||
1. Фильтрует payload по CIAN_REQUIRED_COOKIES.
|
||||
2. Верифицирует через GET /kalkulator-nedvizhimosti/ — проверяет isAuthenticated.
|
||||
3. Сохраняет зашифрованно (pgp_sym_encrypt) в cian_session_cookies.
|
||||
|
||||
Returns: {"ok": true, "userId": <int>, "cookieCount": <int>}
|
||||
"""
|
||||
if not settings.cookie_encryption_key:
|
||||
raise HTTPException(status_code=503, detail="COOKIE_ENCRYPTION_KEY not configured")
|
||||
|
||||
cleaned = {k: v for k, v in cookies.items() if k in cian_session_svc.CIAN_REQUIRED_COOKIES}
|
||||
if not cleaned:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=(
|
||||
f"No recognized Cian cookies in payload. "
|
||||
f"Expected any of: {sorted(cian_session_svc.CIAN_REQUIRED_COOKIES)}"
|
||||
),
|
||||
)
|
||||
|
||||
state = await cian_session_svc.verify_session(cleaned)
|
||||
if state is None:
|
||||
raise HTTPException(
|
||||
status_code=401,
|
||||
detail="Cookies invalid or session not authenticated on cian.ru",
|
||||
)
|
||||
|
||||
user_id = state.get("user", {}).get("userId")
|
||||
if not user_id:
|
||||
raise HTTPException(status_code=400, detail="Authenticated state missing userId")
|
||||
|
||||
cian_session_svc.save_session(db, account_user_id=int(user_id), cookies=cleaned)
|
||||
return {"ok": True, "userId": user_id, "cookieCount": len(cleaned)}
|
||||
|
||||
|
||||
@router.get("/scrape/cian/test-auth", status_code=200, dependencies=[Depends(require_admin)])
|
||||
async def test_cian_auth(
|
||||
db: Annotated[Session, Depends(get_db)],
|
||||
) -> dict:
|
||||
"""Проверить что текущие сохранённые Cian cookies ещё валидны.
|
||||
|
||||
Returns: {"authenticated": bool, "userId": <int|null>, "reason": <str|null>}
|
||||
"""
|
||||
if not settings.cookie_encryption_key:
|
||||
return {"authenticated": False, "userId": None, "reason": "encryption_key_not_configured"}
|
||||
|
||||
cookies = cian_session_svc.load_session(db)
|
||||
if cookies is None:
|
||||
return {"authenticated": False, "userId": None, "reason": "no_session_in_db"}
|
||||
|
||||
state = await cian_session_svc.verify_session(cookies)
|
||||
if state is None:
|
||||
return {"authenticated": False, "userId": None, "reason": "session_expired_or_invalid"}
|
||||
|
||||
user_id = state.get("user", {}).get("userId")
|
||||
return {"authenticated": True, "userId": user_id, "reason": None}
|
||||
|
|
|
|||
|
|
@ -29,5 +29,9 @@ class Settings(BaseSettings):
|
|||
# Пусто = мониторинг выключен (dev). В prod — env GLITCHTIP_DSN из .env.runtime.
|
||||
glitchtip_dsn: str | None = None
|
||||
|
||||
# Ключ шифрования для pgp_sym_encrypt (Cian session cookies).
|
||||
# Задаётся через env COOKIE_ENCRYPTION_KEY. Пусто = шифрование не работает.
|
||||
cookie_encryption_key: str = ""
|
||||
|
||||
|
||||
settings = Settings()
|
||||
|
|
|
|||
196
tradein-mvp/backend/app/services/cian_session.py
Normal file
196
tradein-mvp/backend/app/services/cian_session.py
Normal file
|
|
@ -0,0 +1,196 @@
|
|||
"""Cian session cookie management — load/save/verify encrypted cookies.
|
||||
|
||||
Used by cian_valuation.py (Stage 7) to authenticate Valuation Calculator API.
|
||||
Cookies stored encrypted (pgp_sym_encrypt) in cian_session_cookies table.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.core.config import settings
|
||||
from app.services.scrapers.cian_state_parser import extract_state
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Cookies критичные для Cian auth — фильтр перед сохранением.
|
||||
# Conservative set подтверждён в Stage 4 (probe через DevTools на live session).
|
||||
CIAN_REQUIRED_COOKIES: set[str] = {
|
||||
"_ym_uid",
|
||||
"_ym_d",
|
||||
"_ym_isad",
|
||||
"_cian_visitor_session_id",
|
||||
"_cian_app_session_id",
|
||||
"_cian_uid",
|
||||
"_ga",
|
||||
"tlsr_id",
|
||||
"cf_clearance",
|
||||
"session-id",
|
||||
"anti_bot",
|
||||
"csrftoken",
|
||||
}
|
||||
|
||||
_VALUATION_TEST_URL = (
|
||||
"https://www.cian.ru/kalkulator-nedvizhimosti/"
|
||||
"?address=%D0%A1%D0%B2%D0%B5%D1%80%D0%B4%D0%BB%D0%BE%D0%B2%D1%81%D0%BA%D0%B0%D1%8F"
|
||||
"%20%D0%BE%D0%B1%D0%BB%2C%20%D0%95%D0%BA%D0%B0%D1%82%D0%B5%D1%80%D0%B8%D0%BD%D0%B1%D1%83%D1%80%D0%B3"
|
||||
"&totalArea=40&roomsCount=1&valuationType=sale"
|
||||
"&floor%5B0%5D=floorOther&repairType%5B0%5D=repairTypeCosmetic"
|
||||
)
|
||||
|
||||
_MFE_VALUATION = "valuation-for-agent-frontend"
|
||||
|
||||
|
||||
async def verify_session(cookies: dict[str, str]) -> dict[str, Any] | None:
|
||||
"""Hit Cian Valuation Calculator with cookies — return parsed state if authenticated.
|
||||
|
||||
Returns state dict containing user.isAuthenticated + userId, or None if
|
||||
cookies are invalid/expired or state extraction fails.
|
||||
Никогда не логирует сырые значения cookies.
|
||||
"""
|
||||
try:
|
||||
async with httpx.AsyncClient(
|
||||
cookies=cookies,
|
||||
timeout=20.0,
|
||||
follow_redirects=True,
|
||||
) as client:
|
||||
resp = await client.get(_VALUATION_TEST_URL)
|
||||
resp.raise_for_status()
|
||||
state = extract_state(resp.text, mfe=_MFE_VALUATION, key="initialState")
|
||||
if state is None:
|
||||
logger.warning(
|
||||
"Cian cookies verify: state extraction failed (mfe=%s)", _MFE_VALUATION
|
||||
)
|
||||
return None
|
||||
user = state.get("user", {})
|
||||
if not user.get("isAuthenticated"):
|
||||
logger.warning("Cian cookies verify: user.isAuthenticated=false")
|
||||
return None
|
||||
logger.info("Cian cookies verified — userId=%s", user.get("userId"))
|
||||
return state
|
||||
except httpx.HTTPStatusError as exc:
|
||||
logger.warning(
|
||||
"Cian cookies verify HTTP error: %s %s",
|
||||
exc.response.status_code,
|
||||
exc.request.url,
|
||||
)
|
||||
return None
|
||||
except Exception as exc:
|
||||
logger.warning("Cian cookies verify failed: %s", exc)
|
||||
return None
|
||||
|
||||
|
||||
def save_session(
|
||||
db: Session,
|
||||
account_user_id: int,
|
||||
cookies: dict[str, str],
|
||||
ttl_days: int = 30,
|
||||
) -> None:
|
||||
"""Encrypt cookies via pgp_sym_encrypt and UPSERT into cian_session_cookies.
|
||||
|
||||
Uses settings.cookie_encryption_key as the encryption secret.
|
||||
Никогда не логирует сырые значения cookies.
|
||||
"""
|
||||
cookies_json = json.dumps(cookies)
|
||||
db.execute(
|
||||
text("""
|
||||
INSERT INTO cian_session_cookies (
|
||||
account_user_id,
|
||||
cookies_encrypted,
|
||||
expires_at_estimate,
|
||||
uploaded_at
|
||||
) VALUES (
|
||||
CAST(:uid AS bigint),
|
||||
pgp_sym_encrypt(:cookies_json, :key),
|
||||
NOW() + (CAST(:ttl_days AS int) || ' days')::interval,
|
||||
NOW()
|
||||
)
|
||||
ON CONFLICT (account_user_id) DO UPDATE SET
|
||||
cookies_encrypted = EXCLUDED.cookies_encrypted,
|
||||
expires_at_estimate = EXCLUDED.expires_at_estimate,
|
||||
uploaded_at = NOW(),
|
||||
last_invalid_at = NULL
|
||||
"""),
|
||||
{
|
||||
"uid": account_user_id,
|
||||
"cookies_json": cookies_json,
|
||||
"key": settings.cookie_encryption_key,
|
||||
"ttl_days": ttl_days,
|
||||
},
|
||||
)
|
||||
db.commit()
|
||||
logger.info(
|
||||
"Cian cookies saved for userId=%s (count=%d, ttl=%d days)",
|
||||
account_user_id,
|
||||
len(cookies),
|
||||
ttl_days,
|
||||
)
|
||||
|
||||
|
||||
def load_session(db: Session) -> dict[str, str] | None:
|
||||
"""Load most-recently-uploaded valid Cian cookies (decrypt).
|
||||
|
||||
Returns dict[cookie_name, cookie_value] или None если нет валидной session.
|
||||
Выбирает только записи где expires_at_estimate > NOW() и сессия не
|
||||
была инвалидирована после последнего upload'а.
|
||||
"""
|
||||
row = db.execute(
|
||||
text("""
|
||||
SELECT
|
||||
account_user_id,
|
||||
pgp_sym_decrypt(cookies_encrypted, :key)::text AS cookies_json,
|
||||
expires_at_estimate
|
||||
FROM cian_session_cookies
|
||||
WHERE expires_at_estimate > NOW()
|
||||
AND (last_invalid_at IS NULL OR last_invalid_at < uploaded_at)
|
||||
ORDER BY uploaded_at DESC
|
||||
LIMIT 1
|
||||
"""),
|
||||
{"key": settings.cookie_encryption_key},
|
||||
).mappings().first()
|
||||
|
||||
if row is None:
|
||||
logger.warning("No valid Cian session cookies in DB")
|
||||
return None
|
||||
|
||||
cookies: dict[str, str] = json.loads(row["cookies_json"])
|
||||
|
||||
# Обновляем last_used_at — не критично, игнорируем ошибки.
|
||||
try:
|
||||
db.execute(
|
||||
text(
|
||||
"UPDATE cian_session_cookies SET last_used_at = NOW()"
|
||||
" WHERE account_user_id = CAST(:uid AS bigint)"
|
||||
),
|
||||
{"uid": row["account_user_id"]},
|
||||
)
|
||||
db.commit()
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"Failed to update last_used_at for userId=%s: %s", row["account_user_id"], exc
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"Cian cookies loaded for userId=%s (count=%d)",
|
||||
row["account_user_id"],
|
||||
len(cookies),
|
||||
)
|
||||
return cookies
|
||||
|
||||
|
||||
def mark_session_invalid(db: Session, account_user_id: int) -> None:
|
||||
"""Flag session как expired/invalid (например после 401 во время scrape)."""
|
||||
db.execute(
|
||||
text(
|
||||
"UPDATE cian_session_cookies SET last_invalid_at = NOW()"
|
||||
" WHERE account_user_id = CAST(:uid AS bigint)"
|
||||
),
|
||||
{"uid": account_user_id},
|
||||
)
|
||||
db.commit()
|
||||
logger.warning("Cian session marked invalid for userId=%s", account_user_id)
|
||||
324
tradein-mvp/backend/app/services/scrapers/cian_detail.py
Normal file
324
tradein-mvp/backend/app/services/scrapers/cian_detail.py
Normal file
|
|
@ -0,0 +1,324 @@
|
|||
"""Cian.ru detail page scraper — single offer enrichment.
|
||||
|
||||
Different from SERP (cian.py):
|
||||
- URL: https://ekb.cian.ru/sale/flat/<offer_id>/ or https://www.cian.ru/sale/flat/<offer_id>/
|
||||
- MFE: 'frontend-offer-card'
|
||||
- State KEY: 'defaultState' (NOT 'initialState' — SERP uses initialState)
|
||||
- Sister containers in same _cianConfig: bti, priceChanges, stats, agent, newObject
|
||||
|
||||
Parses ~88 offer fields + sister data → DetailEnrichment dataclass.
|
||||
Stage 5 of CianScraper v1.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from curl_cffi.requests import AsyncSession
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.services.scrapers.cian_state_parser import extract_all_states, extract_state
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class DetailEnrichment:
|
||||
"""Result of cian_detail.fetch_detail() — fields для UPDATE listings + sister rows."""
|
||||
|
||||
# Core listing enrichment (extend existing listings row)
|
||||
cian_id: int | None = None
|
||||
windows_view_type: str | None = None # 'yard' / 'street' / 'yardAndStreet'
|
||||
separate_wcs_count: int | None = None
|
||||
combined_wcs_count: int | None = None
|
||||
ceiling_height: float | None = None # meters
|
||||
repair_type: str | None = None # 'cosmetic' / 'design' / 'no'
|
||||
views_total: int | None = None # Cian stats.totalViewsFormattedString → int
|
||||
views_today: int | None = None
|
||||
|
||||
# BTI (вторичка only — primary buildings don't have БТИ)
|
||||
bti_data: dict[str, Any] | None = None # raw bti.houseData snapshot
|
||||
|
||||
# Price history (offer.priceChanges)
|
||||
price_changes: list[dict[str, Any]] = field(default_factory=list)
|
||||
|
||||
# Agent profile (offer.agent.* expanded)
|
||||
agent_profile: dict[str, Any] | None = None
|
||||
|
||||
# Newbuilding link (if offer is from a newbuilding)
|
||||
newbuilding_data: dict[str, Any] | None = None
|
||||
|
||||
# Raw payload для debugging / future-extraction
|
||||
raw_offer: dict[str, Any] | None = None
|
||||
raw_sister_states: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
async def fetch_detail(
|
||||
offer_url: str,
|
||||
*,
|
||||
session: AsyncSession | None = None,
|
||||
) -> DetailEnrichment | None:
|
||||
"""Fetch detail page and extract all containers.
|
||||
|
||||
Args:
|
||||
offer_url: e.g. 'https://ekb.cian.ru/sale/flat/327830237/'
|
||||
session: optional shared curl_cffi session (для batched scrapes)
|
||||
|
||||
Returns: DetailEnrichment, or None если fetch / parse failed.
|
||||
"""
|
||||
close_session = False
|
||||
if session is None:
|
||||
session = AsyncSession(
|
||||
impersonate="chrome120",
|
||||
timeout=25.0,
|
||||
headers={
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
"Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8",
|
||||
},
|
||||
)
|
||||
close_session = True
|
||||
|
||||
try:
|
||||
resp = await session.get(offer_url, allow_redirects=True)
|
||||
if resp.status_code != 200:
|
||||
logger.warning("Cian detail fetch %s → HTTP %d", offer_url, resp.status_code)
|
||||
return None
|
||||
html = resp.text
|
||||
|
||||
# Primary state container — defaultState in frontend-offer-card MFE
|
||||
# NOTE: detail pages use 'defaultState', SERP uses 'initialState'
|
||||
offer_state = extract_state(html, mfe="frontend-offer-card", key="defaultState")
|
||||
if offer_state is None:
|
||||
logger.warning("Cian detail %s: defaultState extraction failed", offer_url)
|
||||
return None
|
||||
|
||||
offer_data = offer_state.get("offerData", {})
|
||||
offer = offer_data.get("offer", {})
|
||||
|
||||
result = DetailEnrichment(
|
||||
cian_id=offer.get("cianId") or offer.get("id"),
|
||||
windows_view_type=_extract_windows_view(offer),
|
||||
separate_wcs_count=offer.get("separateWcsCount"),
|
||||
combined_wcs_count=offer.get("combinedWcsCount"),
|
||||
ceiling_height=_parse_float(offer.get("ceilingHeight")),
|
||||
repair_type=_extract_repair_type(offer),
|
||||
raw_offer=offer,
|
||||
)
|
||||
|
||||
# Stats (views) — from stats key in offerData or top-level state
|
||||
stats = offer_data.get("stats") or offer_state.get("stats") or {}
|
||||
result.views_total = _parse_views(stats.get("totalViewsFormattedString"))
|
||||
result.views_today = _parse_views(stats.get("todayViewsFormattedString"))
|
||||
|
||||
# Price changes — from offer.priceChanges or state-level priceChanges
|
||||
result.price_changes = _extract_price_changes(offer, offer_state)
|
||||
|
||||
# Agent profile expansion
|
||||
result.agent_profile = _extract_agent_profile(offer)
|
||||
|
||||
# Sister state containers: bti, newObject, plus raw stash of everything else
|
||||
all_states = extract_all_states(html)
|
||||
offer_card_states = all_states.get("frontend-offer-card", {})
|
||||
|
||||
bti_state = offer_card_states.get("bti")
|
||||
if bti_state:
|
||||
result.bti_data = bti_state.get("houseData")
|
||||
result.raw_sister_states["bti"] = bti_state
|
||||
|
||||
# Newbuilding link (if applicable)
|
||||
newbuilding = offer.get("newbuilding")
|
||||
if newbuilding and newbuilding.get("id"):
|
||||
result.newbuilding_data = newbuilding
|
||||
|
||||
# Stash all non-defaultState sister containers for debugging / future-extraction
|
||||
for k, v in offer_card_states.items():
|
||||
if k != "defaultState":
|
||||
result.raw_sister_states[k] = v
|
||||
|
||||
return result
|
||||
|
||||
finally:
|
||||
if close_session:
|
||||
await session.close()
|
||||
|
||||
|
||||
# ── Field extractors ──────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _extract_windows_view(offer: dict[str, Any]) -> str | None:
|
||||
return offer.get("windowsViewType")
|
||||
|
||||
|
||||
def _extract_repair_type(offer: dict[str, Any]) -> str | None:
|
||||
# Cian: 'cosmetic' / 'design' / 'no' / 'euro' / ...
|
||||
return offer.get("repairType")
|
||||
|
||||
|
||||
def _parse_float(value: Any) -> float | None:
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
return float(value)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def _parse_views(formatted_str: str | None) -> int | None:
|
||||
"""Cian's stats.totalViewsFormattedString — '1 234' → 1234."""
|
||||
if not formatted_str:
|
||||
return None
|
||||
try:
|
||||
return int(str(formatted_str).replace(" ", "").replace(",", ""))
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def _extract_price_changes(offer: dict[str, Any], state: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
"""offer.priceChanges or state.priceChanges — normalize to common form."""
|
||||
raw = offer.get("priceChanges") or state.get("priceChanges") or []
|
||||
if not isinstance(raw, list):
|
||||
return []
|
||||
result = []
|
||||
for entry in raw:
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
result.append(
|
||||
{
|
||||
"change_time": entry.get("changeTime") or entry.get("date"),
|
||||
"price_rub": entry.get("price") or entry.get("priceRub"),
|
||||
"diff_percent": entry.get("diffPercent") or entry.get("diff_percent"),
|
||||
}
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def _extract_agent_profile(offer: dict[str, Any]) -> dict[str, Any] | None:
|
||||
"""offer.agent or offer.agency or offer.user — extract canonical profile shape."""
|
||||
agent = offer.get("agent") or {}
|
||||
user = offer.get("user") or {}
|
||||
if not agent and not user:
|
||||
return None
|
||||
return {
|
||||
"ext_agent_id": agent.get("id") or user.get("cianUserId"),
|
||||
"company_name": agent.get("companyName") or user.get("companyName"),
|
||||
"is_pro": agent.get("isPro") or user.get("isPro"),
|
||||
"skills": agent.get("skills") or [],
|
||||
"account_type": agent.get("accountType") or user.get("accountType"),
|
||||
}
|
||||
|
||||
|
||||
# ── Save helpers ──────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def save_detail_enrichment(db: Session, listing_id: int, enrichment: DetailEnrichment) -> None:
|
||||
"""Persist DetailEnrichment to DB:
|
||||
|
||||
- UPDATE listings SET windows_view_type, ceiling_height, ... WHERE id = listing_id
|
||||
- INSERT INTO offer_price_history rows (if price_changes non-empty)
|
||||
- INSERT INTO agents (if agent_profile) and link via agent_id_fk
|
||||
- bti_data не сохраняется здесь — это задача Stage 6 (houses), где есть house_id
|
||||
|
||||
Idempotency: re-running безопасно (UPSERT / ON CONFLICT DO NOTHING semantics).
|
||||
"""
|
||||
# Extend listings row with detail-enriched fields
|
||||
db.execute(
|
||||
text("""
|
||||
UPDATE listings SET
|
||||
windows_view_type = COALESCE(:wvt, windows_view_type),
|
||||
separate_wcs_count = COALESCE(:swc, separate_wcs_count),
|
||||
combined_wcs_count = COALESCE(:cwc, combined_wcs_count),
|
||||
ceiling_height = COALESCE(:ch, ceiling_height),
|
||||
repair_type = COALESCE(:rt, repair_type),
|
||||
views_total = COALESCE(:vt, views_total),
|
||||
views_today = COALESCE(:vd, views_today)
|
||||
WHERE id = CAST(:lid AS bigint)
|
||||
"""),
|
||||
{
|
||||
"lid": listing_id,
|
||||
"wvt": enrichment.windows_view_type,
|
||||
"swc": enrichment.separate_wcs_count,
|
||||
"cwc": enrichment.combined_wcs_count,
|
||||
"ch": enrichment.ceiling_height,
|
||||
"rt": enrichment.repair_type,
|
||||
"vt": enrichment.views_total,
|
||||
"vd": enrichment.views_today,
|
||||
},
|
||||
)
|
||||
|
||||
# Price changes — INSERT new rows, de-dup by listing_id + change_time
|
||||
for change in enrichment.price_changes:
|
||||
if not change.get("change_time") or not change.get("price_rub"):
|
||||
continue
|
||||
db.execute(
|
||||
text("""
|
||||
INSERT INTO offer_price_history
|
||||
(listing_id, change_time, price_rub, diff_percent, source)
|
||||
VALUES (
|
||||
CAST(:lid AS bigint),
|
||||
CAST(:ct AS timestamptz),
|
||||
CAST(:price AS numeric),
|
||||
CAST(:diff AS numeric),
|
||||
'cian'
|
||||
)
|
||||
ON CONFLICT DO NOTHING
|
||||
"""),
|
||||
{
|
||||
"lid": listing_id,
|
||||
"ct": change["change_time"],
|
||||
"price": change["price_rub"],
|
||||
"diff": change.get("diff_percent"),
|
||||
},
|
||||
)
|
||||
|
||||
# Agent profile UPSERT — store agent row and link to listing
|
||||
if enrichment.agent_profile and enrichment.agent_profile.get("ext_agent_id"):
|
||||
ap = enrichment.agent_profile
|
||||
row = db.execute(
|
||||
text("""
|
||||
INSERT INTO agents
|
||||
(ext_source, ext_agent_id, company_name, is_pro, skills, account_type)
|
||||
VALUES (
|
||||
'cian',
|
||||
:eai,
|
||||
:cn,
|
||||
:ip,
|
||||
CAST(:sk AS jsonb),
|
||||
:at
|
||||
)
|
||||
ON CONFLICT (ext_source, ext_agent_id) DO UPDATE SET
|
||||
company_name = COALESCE(EXCLUDED.company_name, agents.company_name),
|
||||
is_pro = COALESCE(EXCLUDED.is_pro, agents.is_pro),
|
||||
skills = COALESCE(EXCLUDED.skills, agents.skills),
|
||||
account_type = COALESCE(EXCLUDED.account_type, agents.account_type)
|
||||
RETURNING id
|
||||
"""),
|
||||
{
|
||||
"eai": str(ap["ext_agent_id"]),
|
||||
"cn": ap.get("company_name"),
|
||||
"ip": ap.get("is_pro"),
|
||||
"sk": json.dumps(ap.get("skills") or []),
|
||||
"at": ap.get("account_type"),
|
||||
},
|
||||
)
|
||||
agent_db_id = row.scalar_one_or_none()
|
||||
if agent_db_id is not None:
|
||||
db.execute(
|
||||
text("""
|
||||
UPDATE listings
|
||||
SET agent_id_fk = CAST(:aid AS bigint)
|
||||
WHERE id = CAST(:lid AS bigint)
|
||||
AND agent_id_fk IS NULL
|
||||
"""),
|
||||
{"aid": agent_db_id, "lid": listing_id},
|
||||
)
|
||||
|
||||
db.commit()
|
||||
logger.info(
|
||||
"Cian detail enrichment saved for listing_id=%s (price_changes=%d, agent=%s)",
|
||||
listing_id,
|
||||
len(enrichment.price_changes),
|
||||
bool(enrichment.agent_profile),
|
||||
)
|
||||
324
tradein-mvp/backend/tests/test_cian_detail.py
Normal file
324
tradein-mvp/backend/tests/test_cian_detail.py
Normal file
|
|
@ -0,0 +1,324 @@
|
|||
"""Tests for cian_detail.py — Stage 5 of CianScraper v1."""
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from app.services.scrapers.cian_detail import (
|
||||
DetailEnrichment,
|
||||
_extract_agent_profile,
|
||||
_extract_price_changes,
|
||||
_parse_views,
|
||||
fetch_detail,
|
||||
)
|
||||
|
||||
|
||||
# ── _parse_views ──────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_parse_views_space_formatted():
|
||||
assert _parse_views("1 234") == 1234
|
||||
|
||||
|
||||
def test_parse_views_plain():
|
||||
assert _parse_views("567") == 567
|
||||
|
||||
|
||||
def test_parse_views_none():
|
||||
assert _parse_views(None) is None
|
||||
|
||||
|
||||
def test_parse_views_invalid():
|
||||
assert _parse_views("invalid") is None
|
||||
|
||||
|
||||
def test_parse_views_comma_formatted():
|
||||
assert _parse_views("1,234") == 1234
|
||||
|
||||
|
||||
# ── _extract_price_changes ────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_extract_price_changes_normalizes():
|
||||
offer = {
|
||||
"priceChanges": [
|
||||
{"changeTime": "2026-04-01T00:00:00Z", "price": 5000000, "diffPercent": -2.5},
|
||||
{"changeTime": "2026-04-10T00:00:00Z", "price": 4900000, "diffPercent": -2.0},
|
||||
]
|
||||
}
|
||||
result = _extract_price_changes(offer, {})
|
||||
assert len(result) == 2
|
||||
assert result[0]["change_time"] == "2026-04-01T00:00:00Z"
|
||||
assert result[0]["price_rub"] == 5000000
|
||||
assert result[0]["diff_percent"] == -2.5
|
||||
|
||||
|
||||
def test_extract_price_changes_handles_empty_offer():
|
||||
assert _extract_price_changes({}, {}) == []
|
||||
|
||||
|
||||
def test_extract_price_changes_handles_none_value():
|
||||
assert _extract_price_changes({"priceChanges": None}, {}) == []
|
||||
|
||||
|
||||
def test_extract_price_changes_falls_back_to_state():
|
||||
state = {
|
||||
"priceChanges": [
|
||||
{"changeTime": "2026-03-01T00:00:00Z", "price": 6000000, "diffPercent": 0},
|
||||
]
|
||||
}
|
||||
result = _extract_price_changes({}, state)
|
||||
assert len(result) == 1
|
||||
assert result[0]["price_rub"] == 6000000
|
||||
|
||||
|
||||
def test_extract_price_changes_skips_non_dict_entries():
|
||||
offer = {"priceChanges": [{"changeTime": "2026-04-01T00:00:00Z", "price": 5000000}, "bad"]}
|
||||
result = _extract_price_changes(offer, {})
|
||||
assert len(result) == 1
|
||||
|
||||
|
||||
# ── _extract_agent_profile ────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_extract_agent_profile_from_agent_key():
|
||||
offer = {
|
||||
"agent": {
|
||||
"id": 12345,
|
||||
"companyName": "Новосёл",
|
||||
"isPro": True,
|
||||
"skills": [{"id": 1, "name": "Продажа"}],
|
||||
"accountType": "specialist",
|
||||
}
|
||||
}
|
||||
profile = _extract_agent_profile(offer)
|
||||
assert profile is not None
|
||||
assert profile["ext_agent_id"] == 12345
|
||||
assert profile["company_name"] == "Новосёл"
|
||||
assert profile["is_pro"] is True
|
||||
assert profile["account_type"] == "specialist"
|
||||
|
||||
|
||||
def test_extract_agent_profile_from_user_key():
|
||||
offer = {
|
||||
"user": {
|
||||
"cianUserId": 99999,
|
||||
"companyName": "Агентство Икс",
|
||||
"isPro": False,
|
||||
"accountType": "agency",
|
||||
}
|
||||
}
|
||||
profile = _extract_agent_profile(offer)
|
||||
assert profile is not None
|
||||
assert profile["ext_agent_id"] == 99999
|
||||
assert profile["company_name"] == "Агентство Икс"
|
||||
|
||||
|
||||
def test_extract_agent_profile_returns_none_when_no_agent():
|
||||
assert _extract_agent_profile({}) is None
|
||||
assert _extract_agent_profile({"agent": {}, "user": {}}) is None
|
||||
|
||||
|
||||
# ── DetailEnrichment defaults ─────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_dataclass_defaults():
|
||||
enrich = DetailEnrichment()
|
||||
assert enrich.cian_id is None
|
||||
assert enrich.price_changes == []
|
||||
assert enrich.raw_sister_states == {}
|
||||
assert enrich.bti_data is None
|
||||
assert enrich.agent_profile is None
|
||||
assert enrich.newbuilding_data is None
|
||||
|
||||
|
||||
# ── fetch_detail integration (mocked) ────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_detail_returns_none_on_no_state(monkeypatch):
|
||||
"""If state extraction returns None → fetch_detail returns None gracefully."""
|
||||
monkeypatch.setattr(
|
||||
"app.services.scrapers.cian_detail.extract_state",
|
||||
lambda html, mfe, key: None,
|
||||
)
|
||||
|
||||
response = MagicMock()
|
||||
response.status_code = 200
|
||||
response.text = "<html></html>"
|
||||
|
||||
session = MagicMock()
|
||||
session.get = AsyncMock(return_value=response)
|
||||
session.close = AsyncMock()
|
||||
|
||||
result = await fetch_detail("https://ekb.cian.ru/sale/flat/123/", session=session)
|
||||
assert result is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_detail_returns_none_on_non_200(monkeypatch):
|
||||
"""Non-200 HTTP response → returns None, no exception raised."""
|
||||
response = MagicMock()
|
||||
response.status_code = 403
|
||||
response.text = ""
|
||||
|
||||
session = MagicMock()
|
||||
session.get = AsyncMock(return_value=response)
|
||||
session.close = AsyncMock()
|
||||
|
||||
result = await fetch_detail("https://ekb.cian.ru/sale/flat/456/", session=session)
|
||||
assert result is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_detail_extracts_core_fields(monkeypatch):
|
||||
"""Verify core fields are extracted correctly from mocked state."""
|
||||
fake_state = {
|
||||
"offerData": {
|
||||
"offer": {
|
||||
"cianId": 327830237,
|
||||
"windowsViewType": "yardAndStreet",
|
||||
"separateWcsCount": 1,
|
||||
"combinedWcsCount": 0,
|
||||
"ceilingHeight": 2.65,
|
||||
"repairType": "cosmetic",
|
||||
"priceChanges": [
|
||||
{
|
||||
"changeTime": "2026-04-01T00:00:00Z",
|
||||
"price": 5500000,
|
||||
"diffPercent": -1.8,
|
||||
}
|
||||
],
|
||||
"agent": {
|
||||
"id": 7654321,
|
||||
"companyName": "Первичка Плюс",
|
||||
"isPro": True,
|
||||
"skills": [],
|
||||
"accountType": "specialist",
|
||||
},
|
||||
},
|
||||
"stats": {
|
||||
"totalViewsFormattedString": "1 042",
|
||||
"todayViewsFormattedString": "7",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
monkeypatch.setattr(
|
||||
"app.services.scrapers.cian_detail.extract_state",
|
||||
lambda html, mfe, key: fake_state if key == "defaultState" else None,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"app.services.scrapers.cian_detail.extract_all_states",
|
||||
lambda html: {},
|
||||
)
|
||||
|
||||
response = MagicMock()
|
||||
response.status_code = 200
|
||||
response.text = "<html></html>"
|
||||
|
||||
session = MagicMock()
|
||||
session.get = AsyncMock(return_value=response)
|
||||
session.close = AsyncMock()
|
||||
|
||||
result = await fetch_detail("https://ekb.cian.ru/sale/flat/327830237/", session=session)
|
||||
assert result is not None
|
||||
assert result.cian_id == 327830237
|
||||
assert result.windows_view_type == "yardAndStreet"
|
||||
assert result.separate_wcs_count == 1
|
||||
assert result.combined_wcs_count == 0
|
||||
assert result.ceiling_height == 2.65
|
||||
assert result.repair_type == "cosmetic"
|
||||
assert result.views_total == 1042
|
||||
assert result.views_today == 7
|
||||
assert len(result.price_changes) == 1
|
||||
assert result.price_changes[0]["price_rub"] == 5500000
|
||||
assert result.agent_profile is not None
|
||||
assert result.agent_profile["ext_agent_id"] == 7654321
|
||||
assert result.agent_profile["company_name"] == "Первичка Плюс"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_detail_extracts_bti_from_sister_states(monkeypatch):
|
||||
"""BTI sister state is captured into bti_data."""
|
||||
fake_state = {
|
||||
"offerData": {
|
||||
"offer": {
|
||||
"cianId": 111111,
|
||||
}
|
||||
}
|
||||
}
|
||||
fake_all_states = {
|
||||
"frontend-offer-card": {
|
||||
"bti": {
|
||||
"houseData": {
|
||||
"seriesName": "П-44Т",
|
||||
"floorsCount": 17,
|
||||
"flatCount": 120,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
monkeypatch.setattr(
|
||||
"app.services.scrapers.cian_detail.extract_state",
|
||||
lambda html, mfe, key: fake_state if key == "defaultState" else None,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"app.services.scrapers.cian_detail.extract_all_states",
|
||||
lambda html: fake_all_states,
|
||||
)
|
||||
|
||||
response = MagicMock()
|
||||
response.status_code = 200
|
||||
response.text = "<html></html>"
|
||||
|
||||
session = MagicMock()
|
||||
session.get = AsyncMock(return_value=response)
|
||||
session.close = AsyncMock()
|
||||
|
||||
result = await fetch_detail("https://ekb.cian.ru/sale/flat/111111/", session=session)
|
||||
assert result is not None
|
||||
assert result.bti_data is not None
|
||||
assert result.bti_data["seriesName"] == "П-44Т"
|
||||
assert "bti" in result.raw_sister_states
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_detail_extracts_newbuilding_link(monkeypatch):
|
||||
"""Newbuilding reference is captured from offer.newbuilding."""
|
||||
fake_state = {
|
||||
"offerData": {
|
||||
"offer": {
|
||||
"cianId": 222222,
|
||||
"newbuilding": {
|
||||
"id": 54016,
|
||||
"name": "Екатерининский парк",
|
||||
"url": "https://zhk-ekaterininskiy-park-ekb-i.cian.ru/",
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
monkeypatch.setattr(
|
||||
"app.services.scrapers.cian_detail.extract_state",
|
||||
lambda html, mfe, key: fake_state if key == "defaultState" else None,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"app.services.scrapers.cian_detail.extract_all_states",
|
||||
lambda html: {},
|
||||
)
|
||||
|
||||
response = MagicMock()
|
||||
response.status_code = 200
|
||||
response.text = "<html></html>"
|
||||
|
||||
session = MagicMock()
|
||||
session.get = AsyncMock(return_value=response)
|
||||
session.close = AsyncMock()
|
||||
|
||||
result = await fetch_detail("https://ekb.cian.ru/sale/flat/222222/", session=session)
|
||||
assert result is not None
|
||||
assert result.newbuilding_data is not None
|
||||
assert result.newbuilding_data["id"] == 54016
|
||||
238
tradein-mvp/backend/tests/test_cian_session.py
Normal file
238
tradein-mvp/backend/tests/test_cian_session.py
Normal file
|
|
@ -0,0 +1,238 @@
|
|||
"""Tests for cian_session — cookie management service."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from app.services.cian_session import (
|
||||
CIAN_REQUIRED_COOKIES,
|
||||
load_session,
|
||||
mark_session_invalid,
|
||||
save_session,
|
||||
verify_session,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def mock_db() -> MagicMock:
|
||||
db = MagicMock()
|
||||
# Default: no row found (load_session → None)
|
||||
db.execute.return_value.mappings.return_value.first.return_value = None
|
||||
return db
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CIAN_REQUIRED_COOKIES
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_required_cookies_set_not_empty() -> None:
|
||||
assert isinstance(CIAN_REQUIRED_COOKIES, set)
|
||||
assert len(CIAN_REQUIRED_COOKIES) >= 5
|
||||
|
||||
|
||||
def test_required_cookies_contains_key_names() -> None:
|
||||
assert "_cian_visitor_session_id" in CIAN_REQUIRED_COOKIES
|
||||
assert "_ym_uid" in CIAN_REQUIRED_COOKIES
|
||||
assert "_cian_uid" in CIAN_REQUIRED_COOKIES
|
||||
assert "tlsr_id" in CIAN_REQUIRED_COOKIES
|
||||
assert "cf_clearance" in CIAN_REQUIRED_COOKIES
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# save_session
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_save_session_calls_pgp_sym_encrypt(mock_db: MagicMock) -> None:
|
||||
save_session(mock_db, account_user_id=12345, cookies={"_ym_uid": "abc"})
|
||||
args, _ = mock_db.execute.call_args
|
||||
sql_text = str(args[0])
|
||||
assert "pgp_sym_encrypt" in sql_text
|
||||
assert "cian_session_cookies" in sql_text
|
||||
|
||||
|
||||
def test_save_session_uses_cast_not_colon_colon(mock_db: MagicMock) -> None:
|
||||
"""Verify psycopg v3 compatibility — no ::bigint syntax in SQL."""
|
||||
save_session(mock_db, account_user_id=99, cookies={"csrftoken": "x"})
|
||||
args, _ = mock_db.execute.call_args
|
||||
sql_text = str(args[0])
|
||||
# CAST(:x AS type) is required; ::type would fail with psycopg v3 named params
|
||||
assert "CAST(" in sql_text
|
||||
assert "::bigint" not in sql_text
|
||||
|
||||
|
||||
def test_save_session_commits(mock_db: MagicMock) -> None:
|
||||
save_session(mock_db, account_user_id=12345, cookies={"_ym_uid": "abc"})
|
||||
assert mock_db.commit.called
|
||||
|
||||
|
||||
def test_save_session_on_conflict_do_update(mock_db: MagicMock) -> None:
|
||||
save_session(mock_db, account_user_id=1, cookies={"_ym_uid": "v"})
|
||||
args, _ = mock_db.execute.call_args
|
||||
sql_text = str(args[0])
|
||||
assert "ON CONFLICT" in sql_text
|
||||
assert "DO UPDATE" in sql_text
|
||||
|
||||
|
||||
def test_save_session_passes_correct_params(mock_db: MagicMock) -> None:
|
||||
save_session(mock_db, account_user_id=777, cookies={"_ga": "GA1.2.x"}, ttl_days=14)
|
||||
_, kwargs = mock_db.execute.call_args
|
||||
params = kwargs if kwargs else mock_db.execute.call_args[0][1]
|
||||
# params могут передаваться как второй позиционный аргумент
|
||||
call_args = mock_db.execute.call_args
|
||||
params = call_args[0][1] if len(call_args[0]) > 1 else call_args[1].get("params", {})
|
||||
assert params["uid"] == 777
|
||||
assert params["ttl_days"] == 14
|
||||
cookies_payload = json.loads(params["cookies_json"])
|
||||
assert cookies_payload == {"_ga": "GA1.2.x"}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# load_session
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_load_session_returns_none_when_empty(mock_db: MagicMock) -> None:
|
||||
result = load_session(mock_db)
|
||||
assert result is None
|
||||
|
||||
|
||||
def test_load_session_decodes_json(mock_db: MagicMock) -> None:
|
||||
mock_row = {
|
||||
"account_user_id": 12345,
|
||||
"cookies_json": json.dumps({"_ym_uid": "abc", "_cian_uid": "def"}),
|
||||
"expires_at_estimate": None,
|
||||
}
|
||||
mock_db.execute.return_value.mappings.return_value.first.return_value = mock_row
|
||||
result = load_session(mock_db)
|
||||
assert result == {"_ym_uid": "abc", "_cian_uid": "def"}
|
||||
|
||||
|
||||
def test_load_session_updates_last_used_at(mock_db: MagicMock) -> None:
|
||||
mock_row = {
|
||||
"account_user_id": 42,
|
||||
"cookies_json": json.dumps({"session-id": "s"}),
|
||||
"expires_at_estimate": None,
|
||||
}
|
||||
mock_db.execute.return_value.mappings.return_value.first.return_value = mock_row
|
||||
load_session(mock_db)
|
||||
# Должно быть минимум 2 вызова execute: SELECT + UPDATE last_used_at
|
||||
assert mock_db.execute.call_count >= 2
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# mark_session_invalid
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_mark_session_invalid_updates_table(mock_db: MagicMock) -> None:
|
||||
mark_session_invalid(mock_db, account_user_id=12345)
|
||||
args, _ = mock_db.execute.call_args
|
||||
sql = str(args[0])
|
||||
assert "last_invalid_at" in sql
|
||||
assert "cian_session_cookies" in sql
|
||||
|
||||
|
||||
def test_mark_session_invalid_commits(mock_db: MagicMock) -> None:
|
||||
mark_session_invalid(mock_db, account_user_id=99)
|
||||
assert mock_db.commit.called
|
||||
|
||||
|
||||
def test_mark_session_invalid_uses_cast(mock_db: MagicMock) -> None:
|
||||
mark_session_invalid(mock_db, account_user_id=5)
|
||||
args, _ = mock_db.execute.call_args
|
||||
sql = str(args[0])
|
||||
assert "CAST(" in sql
|
||||
assert "::bigint" not in sql
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# verify_session (async)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_verify_session_returns_none_when_state_missing(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""extract_state returns None → verify_session returns None."""
|
||||
monkeypatch.setattr(
|
||||
"app.services.cian_session.extract_state",
|
||||
lambda html, mfe, key: None,
|
||||
)
|
||||
|
||||
async def fake_get(url: str, **kwargs): # noqa: ARG001
|
||||
class FakeResp:
|
||||
text = "<html></html>"
|
||||
def raise_for_status(self) -> None:
|
||||
pass
|
||||
return FakeResp()
|
||||
|
||||
with patch("httpx.AsyncClient") as mock_client_cls:
|
||||
mock_client = AsyncMock()
|
||||
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
|
||||
mock_client.__aexit__ = AsyncMock(return_value=None)
|
||||
mock_client.get = AsyncMock(return_value=_make_fake_resp("<html></html>"))
|
||||
mock_client_cls.return_value = mock_client
|
||||
|
||||
result = await verify_session({"_ym_uid": "x"})
|
||||
assert result is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_verify_session_returns_none_when_not_authenticated(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""state.user.isAuthenticated is false → verify_session returns None."""
|
||||
monkeypatch.setattr(
|
||||
"app.services.cian_session.extract_state",
|
||||
lambda html, mfe, key: {"user": {"isAuthenticated": False, "userId": None}},
|
||||
)
|
||||
|
||||
with patch("httpx.AsyncClient") as mock_client_cls:
|
||||
mock_client = AsyncMock()
|
||||
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
|
||||
mock_client.__aexit__ = AsyncMock(return_value=None)
|
||||
mock_client.get = AsyncMock(return_value=_make_fake_resp("<html></html>"))
|
||||
mock_client_cls.return_value = mock_client
|
||||
|
||||
result = await verify_session({"_ym_uid": "x"})
|
||||
assert result is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_verify_session_returns_state_when_authenticated(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""state.user.isAuthenticated is true → returns state dict."""
|
||||
expected_state = {"user": {"isAuthenticated": True, "userId": 102963817}}
|
||||
monkeypatch.setattr(
|
||||
"app.services.cian_session.extract_state",
|
||||
lambda html, mfe, key: expected_state,
|
||||
)
|
||||
|
||||
with patch("httpx.AsyncClient") as mock_client_cls:
|
||||
mock_client = AsyncMock()
|
||||
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
|
||||
mock_client.__aexit__ = AsyncMock(return_value=None)
|
||||
mock_client.get = AsyncMock(return_value=_make_fake_resp("<html>state here</html>"))
|
||||
mock_client_cls.return_value = mock_client
|
||||
|
||||
result = await verify_session({"_ym_uid": "x", "_cian_uid": "y"})
|
||||
assert result is not None
|
||||
assert result["user"]["isAuthenticated"] is True
|
||||
assert result["user"]["userId"] == 102963817
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_fake_resp(text: str) -> MagicMock:
|
||||
resp = MagicMock()
|
||||
resp.text = text
|
||||
resp.raise_for_status = MagicMock()
|
||||
return resp
|
||||
50
tradein-mvp/scripts/upload-cian-cookies.sh
Normal file
50
tradein-mvp/scripts/upload-cian-cookies.sh
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
#!/bin/bash
|
||||
# Upload Cian session cookies to backend for Valuation Calculator auth.
|
||||
#
|
||||
# Usage: ./scripts/upload-cian-cookies.sh [cookies.json]
|
||||
#
|
||||
# cookies.json — JSON object { cookie_name: cookie_value, ... }
|
||||
# Экспортируется из Chrome DevTools → Application → Cookies → www.cian.ru
|
||||
# или через расширение Cookie-Editor → Export → JSON (object format).
|
||||
#
|
||||
# Переменные окружения:
|
||||
# TRADEIN_ADMIN_TOKEN — admin token (или берётся из .env.runtime)
|
||||
# TRADEIN_BASE_URL — base URL (default: https://gendsgn.ru/trade-in)
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
COOKIES_FILE="${1:-cookies.json}"
|
||||
|
||||
if [[ ! -f "$COOKIES_FILE" ]]; then
|
||||
echo "Error: cookies file not found: $COOKIES_FILE" >&2
|
||||
echo "Usage: $0 cookies.json" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Загружаем токен из env или .env.runtime
|
||||
ADMIN_TOKEN="${TRADEIN_ADMIN_TOKEN:-}"
|
||||
if [[ -z "$ADMIN_TOKEN" ]] && [[ -f ".env.runtime" ]]; then
|
||||
ADMIN_TOKEN="$(grep '^TRADEIN_ADMIN_TOKEN=' .env.runtime 2>/dev/null | cut -d= -f2- || true)"
|
||||
fi
|
||||
|
||||
if [[ -z "$ADMIN_TOKEN" ]]; then
|
||||
echo "Error: TRADEIN_ADMIN_TOKEN not set (env or .env.runtime)" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
BASE_URL="${TRADEIN_BASE_URL:-https://gendsgn.ru/trade-in}"
|
||||
|
||||
echo "Uploading Cian cookies from $COOKIES_FILE to $BASE_URL ..."
|
||||
echo
|
||||
|
||||
curl -fsSL -X POST "$BASE_URL/api/v1/admin/scrape/cian/upload-cookies" \
|
||||
-H "X-Admin-Token: $ADMIN_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "@$COOKIES_FILE" \
|
||||
| python3 -m json.tool
|
||||
|
||||
echo
|
||||
echo "Verifying session:"
|
||||
curl -fsSL "$BASE_URL/api/v1/admin/scrape/cian/test-auth" \
|
||||
-H "X-Admin-Token: $ADMIN_TOKEN" \
|
||||
| python3 -m json.tool
|
||||
Loading…
Add table
Reference in a new issue