feat(tradein): Cian browser auto-login — POST /admin/scrape/cian/auto-login (#639)
Variant B: headless camoufox логинится email+паролем (без SMS), извлекает auth-cookies → verify → save_session. Browser-сервис получает generic POST /login (url/selectors/success_cookie в body), Cian-специфика — в config (env-overridable). Авто-refresh в scheduler — follow-up PR после ручной валидации логина на проде.
This commit is contained in:
parent
639d3b0cea
commit
1f18a2ccdc
4 changed files with 321 additions and 0 deletions
|
|
@ -317,6 +317,79 @@ async def upload_cian_cookies(
|
|||
return {"ok": True, "userId": user_id, "cookieCount": len(cleaned)}
|
||||
|
||||
|
||||
class CianAutoLoginRequest(BaseModel):
|
||||
email: str | None = None # override settings.cian_login_email
|
||||
password: str | None = None # override settings.cian_login_password
|
||||
|
||||
|
||||
@router.post("/scrape/cian/auto-login", status_code=200)
|
||||
async def cian_auto_login(
|
||||
db: Annotated[Session, Depends(get_db)],
|
||||
body: CianAutoLoginRequest | None = None,
|
||||
) -> dict:
|
||||
"""Browser auto-login на cian.ru (Variant B, #639) → извлекает cookies → save_session.
|
||||
|
||||
Headless camoufox (tradein-browser /login) логинится email+паролем, забирает
|
||||
auth-cookies, фильтрует по CIAN_REQUIRED_COOKIES, верифицирует isAuthenticated,
|
||||
сохраняет зашифрованно. Креды из settings.cian_login_* или из body (override).
|
||||
|
||||
Returns: {"ok": true, "userId": <int>, "cookieCount": <int>}
|
||||
"""
|
||||
if not settings.cookie_encryption_key:
|
||||
raise HTTPException(status_code=503, detail="COOKIE_ENCRYPTION_KEY not configured")
|
||||
|
||||
email = (body.email if body else None) or settings.cian_login_email
|
||||
password = (body.password if body else None) or settings.cian_login_password
|
||||
if not email or not password:
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="CIAN_LOGIN_EMAIL/PASSWORD not configured",
|
||||
)
|
||||
|
||||
from app.services.scrapers.browser_fetcher import BrowserFetcher
|
||||
|
||||
try:
|
||||
async with BrowserFetcher() as fetcher:
|
||||
raw_cookies = await fetcher.login(
|
||||
url=settings.cian_login_url,
|
||||
email=email,
|
||||
password=password,
|
||||
email_selector=settings.cian_login_email_selector,
|
||||
password_selector=settings.cian_login_password_selector,
|
||||
submit_selector=settings.cian_login_submit_selector,
|
||||
success_cookie=settings.cian_login_success_cookie,
|
||||
pre_click_selector=settings.cian_login_pre_click_selector,
|
||||
wait_ms=settings.cian_login_wait_ms,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.error("cian-auto-login: browser login failed: %s", type(exc).__name__)
|
||||
raise HTTPException(status_code=502, detail=f"Browser login failed: {exc}") from exc
|
||||
|
||||
cleaned = {k: v for k, v in raw_cookies.items() if k in cian_session_svc.CIAN_REQUIRED_COOKIES}
|
||||
if not cleaned:
|
||||
raise HTTPException(
|
||||
status_code=502,
|
||||
detail=(
|
||||
"Login returned no recognized Cian cookies "
|
||||
"(selectors may be stale — check CIAN_LOGIN_*_SELECTOR)"
|
||||
),
|
||||
)
|
||||
|
||||
state = await cian_session_svc.verify_session(cleaned)
|
||||
if state is None:
|
||||
raise HTTPException(
|
||||
status_code=401,
|
||||
detail="Logged in but session not authenticated (cookies rejected by cian.ru)",
|
||||
)
|
||||
|
||||
user_id = state.get("user", {}).get("userId")
|
||||
if not user_id:
|
||||
raise HTTPException(status_code=400, detail="Authenticated state missing userId")
|
||||
|
||||
cian_session_svc.save_session(db, account_user_id=int(user_id), cookies=cleaned)
|
||||
return {"ok": True, "userId": user_id, "cookieCount": len(cleaned)}
|
||||
|
||||
|
||||
@router.get("/scrape/cian/test-auth", status_code=200)
|
||||
async def test_cian_auth(
|
||||
db: Annotated[Session, Depends(get_db)],
|
||||
|
|
|
|||
|
|
@ -213,6 +213,21 @@ class Settings(BaseSettings):
|
|||
# ENV: YANDEX_COOKIES_FILE.
|
||||
yandex_cookies_file: str | None = None
|
||||
|
||||
# ── #639: Cian browser auto-login (Variant B) ────────────────────────────
|
||||
# Headless-браузер логинится в cian.ru email+паролем → извлекает auth-cookies →
|
||||
# save_session. Подтверждено: вход без SMS/OTP. Креды пусты в dev → endpoint 503.
|
||||
# Все селекторы env-overridable: при смене DOM cian тюним через .env.runtime БЕЗ
|
||||
# редеплоя кода (restart backend достаточно). ENV: CIAN_LOGIN_EMAIL, CIAN_LOGIN_PASSWORD, ...
|
||||
cian_login_email: str | None = None
|
||||
cian_login_password: str | None = None
|
||||
cian_login_url: str = "https://www.cian.ru/zhalobi-i-otzivi/"
|
||||
cian_login_pre_click_selector: str = ""
|
||||
cian_login_email_selector: str = "input[name='username']"
|
||||
cian_login_password_selector: str = "input[type='password']"
|
||||
cian_login_submit_selector: str = "button[type='submit']"
|
||||
cian_login_success_cookie: str = "DMIR_AUTH"
|
||||
cian_login_wait_ms: int = 4000
|
||||
|
||||
# ── #884/#905: BrowserFetcher — HTTP-клиент к tradein-browser контейнеру ────
|
||||
# scraper_fetch_mode: "curl_cffi" (дефолт, текущее поведение) или "browser"
|
||||
# (HTTP POST к tradein-browser /fetch эндпоинту). Phase 1+.
|
||||
|
|
|
|||
|
|
@ -86,6 +86,54 @@ class BrowserFetcher:
|
|||
await asyncio.sleep(_RETRY_SLEEP_S)
|
||||
return await self._post_fetch(url)
|
||||
|
||||
async def login(
|
||||
self,
|
||||
*,
|
||||
url: str,
|
||||
email: str,
|
||||
password: str,
|
||||
email_selector: str,
|
||||
password_selector: str,
|
||||
submit_selector: str,
|
||||
success_cookie: str,
|
||||
pre_click_selector: str = "",
|
||||
wait_ms: int | None = None,
|
||||
) -> dict[str, str]:
|
||||
"""Логинится через tradein-browser /login и возвращает cookies как dict name→value.
|
||||
|
||||
Использует увеличенный таймаут (60s) — логин медленнее обычного fetch.
|
||||
При HTTPError / TransportError делает одну повторную попытку.
|
||||
|
||||
Returns:
|
||||
Плоский словарь {cookie_name: cookie_value}.
|
||||
"""
|
||||
assert self._client is not None, "BrowserFetcher: используй как async context manager"
|
||||
assert self._endpoint is not None, "BrowserFetcher: endpoint не задан"
|
||||
|
||||
body: dict[str, object] = {
|
||||
"url": url,
|
||||
"email": email,
|
||||
"password": password,
|
||||
"email_selector": email_selector,
|
||||
"password_selector": password_selector,
|
||||
"submit_selector": submit_selector,
|
||||
"success_cookie": success_cookie,
|
||||
"pre_click_selector": pre_click_selector,
|
||||
}
|
||||
if wait_ms is not None:
|
||||
body["wait_ms"] = wait_ms
|
||||
|
||||
try:
|
||||
return await self._post_login(body)
|
||||
except (httpx.HTTPError, httpx.TransportError) as exc:
|
||||
logger.warning(
|
||||
"BrowserFetcher: ошибка login-запроса (%s), retry через %.1fs",
|
||||
type(exc).__name__,
|
||||
_RETRY_SLEEP_S,
|
||||
)
|
||||
await asyncio.sleep(_RETRY_SLEEP_S)
|
||||
return await self._post_login(body)
|
||||
|
||||
# ── internal ───────────────────────────────────────────────────────────────
|
||||
|
||||
async def _post_fetch(self, url: str) -> str:
|
||||
|
|
@ -102,3 +150,31 @@ class BrowserFetcher:
|
|||
html = data["html"]
|
||||
logger.debug("BrowserFetcher: fetch OK url=%r html_len=%d", url, len(html))
|
||||
return html
|
||||
|
||||
async def _post_login(self, body: dict[str, object]) -> dict[str, str]:
|
||||
"""Один HTTP POST к /login эндпоинту сервиса."""
|
||||
assert self._client is not None
|
||||
assert self._endpoint is not None
|
||||
|
||||
resp = await self._client.post(
|
||||
f"{self._endpoint}/login",
|
||||
json=body,
|
||||
timeout=httpx.Timeout(60.0),
|
||||
)
|
||||
if resp.status_code == 502:
|
||||
data = resp.json()
|
||||
has_screenshot = bool(data.get("screenshot_b64"))
|
||||
logger.debug(
|
||||
"BrowserFetcher: login 502 — has_screenshot=%s page_url=%r",
|
||||
has_screenshot,
|
||||
data.get("page_url"),
|
||||
)
|
||||
raise RuntimeError(
|
||||
f"browser login failed: {data.get('error')} url={data.get('page_url')}"
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
raw: list[dict[str, object]] = data["cookies"]
|
||||
result = {str(c["name"]): str(c["value"]) for c in raw}
|
||||
logger.info("BrowserFetcher: login OK cookie_count=%d", len(result))
|
||||
return result
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@
|
|||
|
||||
GET /health → {"status": "ok"}
|
||||
POST /fetch → {"url": "..."} → {"html": "..."}
|
||||
POST /login → {"url": "...", "email": "...", "password": "...", ...} → {"cookies": [...]}
|
||||
|
||||
Такой подход выбран потому, что ``camoufox.server.launch_server`` (Playwright
|
||||
WS-сервер) несовместим с современными версиями playwright (1.45–1.60):
|
||||
|
|
@ -25,6 +26,7 @@ WS-сервер) несовместим с современными версия
|
|||
"""
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import logging
|
||||
import os
|
||||
from urllib.parse import urlparse
|
||||
|
|
@ -252,6 +254,160 @@ def _is_browser_crash(exc: BaseException) -> bool:
|
|||
)
|
||||
|
||||
|
||||
# ── login handler ──────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class LoginError(Exception):
|
||||
"""Исключение с диагностическим payload для /login эндпоинта."""
|
||||
|
||||
def __init__(self, payload: dict) -> None:
|
||||
super().__init__(str(payload))
|
||||
self.payload = payload
|
||||
|
||||
|
||||
async def login_handler(request: web.Request) -> web.Response:
|
||||
"""POST /login {...} → {"cookies": [...]}
|
||||
|
||||
Выполняет вход на указанную страницу (email + пароль), ждёт появления
|
||||
success_cookie, возвращает полный список cookies браузерного контекста.
|
||||
Запросы сериализованы через lock (один за раз — один браузер).
|
||||
"""
|
||||
try:
|
||||
body = await request.json()
|
||||
except Exception:
|
||||
return web.json_response({"error": "invalid JSON body"}, status=400)
|
||||
|
||||
missing = [f for f in ("url", "email", "password", "email_selector",
|
||||
"password_selector", "submit_selector") if not body.get(f)]
|
||||
if missing:
|
||||
return web.json_response({"error": f"missing required fields: {missing}"}, status=400)
|
||||
|
||||
assert _fetch_lock is not None, "_fetch_lock not initialised"
|
||||
|
||||
async with _fetch_lock:
|
||||
try:
|
||||
cookies = await _do_login(body)
|
||||
except LoginError as exc:
|
||||
return web.json_response(exc.payload, status=502)
|
||||
except Exception as exc:
|
||||
logger.error(
|
||||
"tradein-browser: login error url=%r: %s: %s",
|
||||
body.get("url"),
|
||||
type(exc).__name__,
|
||||
exc,
|
||||
)
|
||||
return web.json_response({"error": f"{type(exc).__name__}: {exc}"}, status=502)
|
||||
|
||||
return web.json_response({"cookies": cookies})
|
||||
|
||||
|
||||
async def _do_login(params: dict) -> list[dict]:
|
||||
"""Одна попытка логина; при краше браузера — перезапускает и повторяет."""
|
||||
try:
|
||||
return await _login_once(params)
|
||||
except LoginError:
|
||||
raise # диагностический payload идёт напрямую в хендлер
|
||||
except Exception as exc:
|
||||
if _is_browser_crash(exc):
|
||||
logger.warning(
|
||||
"tradein-browser: краш браузера (%s) при логине, перезапуск + retry",
|
||||
type(exc).__name__,
|
||||
)
|
||||
await _relaunch_browser()
|
||||
return await _login_once(params)
|
||||
raise
|
||||
|
||||
|
||||
async def _login_once(params: dict) -> list[dict]:
|
||||
"""Открывает страницу, заполняет форму, нажимает submit, ждёт success_cookie."""
|
||||
global _page_counter
|
||||
|
||||
browser = _browser
|
||||
assert browser is not None, "browser not launched"
|
||||
|
||||
url: str = params["url"]
|
||||
email: str = params["email"]
|
||||
password: str = params["password"]
|
||||
email_selector: str = params["email_selector"]
|
||||
password_selector: str = params["password_selector"]
|
||||
submit_selector: str = params["submit_selector"]
|
||||
pre_click_selector: str = params.get("pre_click_selector", "") or ""
|
||||
success_cookie: str = params.get("success_cookie", "")
|
||||
wait_ms: int = params.get("wait_ms") or BROWSER_WAIT_MS
|
||||
|
||||
page = await browser.new_page() # type: ignore[attr-defined]
|
||||
try:
|
||||
await page.goto(url, timeout=BROWSER_NAV_TIMEOUT_MS, wait_until="domcontentloaded") # type: ignore[attr-defined]
|
||||
await page.wait_for_timeout(wait_ms) # type: ignore[attr-defined]
|
||||
|
||||
if pre_click_selector:
|
||||
try:
|
||||
await page.click(pre_click_selector, timeout=15000) # type: ignore[attr-defined]
|
||||
await page.wait_for_timeout(1500) # type: ignore[attr-defined]
|
||||
except Exception as pre_exc:
|
||||
logger.warning(
|
||||
"tradein-browser: pre_click_selector=%r не найден (не фатально): %s",
|
||||
pre_click_selector,
|
||||
pre_exc,
|
||||
)
|
||||
|
||||
await page.fill(email_selector, email, timeout=15000) # type: ignore[attr-defined]
|
||||
await page.fill(password_selector, password, timeout=15000) # type: ignore[attr-defined]
|
||||
await page.click(submit_selector, timeout=15000) # type: ignore[attr-defined]
|
||||
|
||||
# Ждём появления success_cookie до ~20s
|
||||
cookie_found = False
|
||||
if success_cookie:
|
||||
for _ in range(40): # 40 × 500ms = 20s
|
||||
await page.wait_for_timeout(500) # type: ignore[attr-defined]
|
||||
ctx_cookies = await page.context.cookies() # type: ignore[attr-defined]
|
||||
if any(c.get("name") == success_cookie for c in ctx_cookies):
|
||||
cookie_found = True
|
||||
break
|
||||
logger.info(
|
||||
"tradein-browser: login success_cookie=%r present=%s",
|
||||
success_cookie,
|
||||
cookie_found,
|
||||
)
|
||||
|
||||
cookies: list[dict] = await page.context.cookies() # type: ignore[attr-defined]
|
||||
logger.info(
|
||||
"tradein-browser: login OK url=%r email_selector=%r cookie_count=%d",
|
||||
url,
|
||||
email_selector,
|
||||
len(cookies),
|
||||
)
|
||||
except Exception as exc:
|
||||
screenshot_b64 = ""
|
||||
try:
|
||||
screenshot_bytes: bytes = await page.screenshot() # type: ignore[attr-defined]
|
||||
screenshot_b64 = base64.b64encode(screenshot_bytes).decode()
|
||||
except Exception:
|
||||
pass
|
||||
page_url = ""
|
||||
try:
|
||||
page_url = page.url # type: ignore[attr-defined]
|
||||
except Exception:
|
||||
pass
|
||||
await page.close() # type: ignore[attr-defined]
|
||||
raise LoginError({
|
||||
"error": f"{type(exc).__name__}: {exc}",
|
||||
"page_url": page_url,
|
||||
"screenshot_b64": screenshot_b64,
|
||||
}) from exc
|
||||
|
||||
await page.close() # type: ignore[attr-defined]
|
||||
_page_counter += 1
|
||||
if _page_counter >= BROWSER_RECYCLE_PAGES:
|
||||
logger.info(
|
||||
"tradein-browser: recycle threshold (%d) достигнут после login, перезапуск браузера",
|
||||
BROWSER_RECYCLE_PAGES,
|
||||
)
|
||||
await _relaunch_browser()
|
||||
|
||||
return cookies
|
||||
|
||||
|
||||
# ── entrypoint ─────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
|
|
@ -261,6 +417,7 @@ def build_app() -> web.Application:
|
|||
app.on_cleanup.append(_on_cleanup)
|
||||
app.router.add_get("/health", health_handler)
|
||||
app.router.add_post("/fetch", fetch_handler)
|
||||
app.router.add_post("/login", login_handler)
|
||||
return app
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue