diff --git a/tradein-mvp/backend/app/api/v1/admin.py b/tradein-mvp/backend/app/api/v1/admin.py index 22f41677..a4c686d0 100644 --- a/tradein-mvp/backend/app/api/v1/admin.py +++ b/tradein-mvp/backend/app/api/v1/admin.py @@ -317,6 +317,79 @@ async def upload_cian_cookies( return {"ok": True, "userId": user_id, "cookieCount": len(cleaned)} +class CianAutoLoginRequest(BaseModel): + email: str | None = None # override settings.cian_login_email + password: str | None = None # override settings.cian_login_password + + +@router.post("/scrape/cian/auto-login", status_code=200) +async def cian_auto_login( + db: Annotated[Session, Depends(get_db)], + body: CianAutoLoginRequest | None = None, +) -> dict: + """Browser auto-login на cian.ru (Variant B, #639) → извлекает cookies → save_session. + + Headless camoufox (tradein-browser /login) логинится email+паролем, забирает + auth-cookies, фильтрует по CIAN_REQUIRED_COOKIES, верифицирует isAuthenticated, + сохраняет зашифрованно. Креды из settings.cian_login_* или из body (override). + + Returns: {"ok": true, "userId": , "cookieCount": } + """ + if not settings.cookie_encryption_key: + raise HTTPException(status_code=503, detail="COOKIE_ENCRYPTION_KEY not configured") + + email = (body.email if body else None) or settings.cian_login_email + password = (body.password if body else None) or settings.cian_login_password + if not email or not password: + raise HTTPException( + status_code=503, + detail="CIAN_LOGIN_EMAIL/PASSWORD not configured", + ) + + from app.services.scrapers.browser_fetcher import BrowserFetcher + + try: + async with BrowserFetcher() as fetcher: + raw_cookies = await fetcher.login( + url=settings.cian_login_url, + email=email, + password=password, + email_selector=settings.cian_login_email_selector, + password_selector=settings.cian_login_password_selector, + submit_selector=settings.cian_login_submit_selector, + success_cookie=settings.cian_login_success_cookie, + pre_click_selector=settings.cian_login_pre_click_selector, + wait_ms=settings.cian_login_wait_ms, + ) + except Exception as exc: + logger.error("cian-auto-login: browser login failed: %s", type(exc).__name__) + raise HTTPException(status_code=502, detail=f"Browser login failed: {exc}") from exc + + cleaned = {k: v for k, v in raw_cookies.items() if k in cian_session_svc.CIAN_REQUIRED_COOKIES} + if not cleaned: + raise HTTPException( + status_code=502, + detail=( + "Login returned no recognized Cian cookies " + "(selectors may be stale — check CIAN_LOGIN_*_SELECTOR)" + ), + ) + + state = await cian_session_svc.verify_session(cleaned) + if state is None: + raise HTTPException( + status_code=401, + detail="Logged in but session not authenticated (cookies rejected by cian.ru)", + ) + + user_id = state.get("user", {}).get("userId") + if not user_id: + raise HTTPException(status_code=400, detail="Authenticated state missing userId") + + cian_session_svc.save_session(db, account_user_id=int(user_id), cookies=cleaned) + return {"ok": True, "userId": user_id, "cookieCount": len(cleaned)} + + @router.get("/scrape/cian/test-auth", status_code=200) async def test_cian_auth( db: Annotated[Session, Depends(get_db)], diff --git a/tradein-mvp/backend/app/core/config.py b/tradein-mvp/backend/app/core/config.py index b1b92443..d3f67c79 100644 --- a/tradein-mvp/backend/app/core/config.py +++ b/tradein-mvp/backend/app/core/config.py @@ -213,6 +213,21 @@ class Settings(BaseSettings): # ENV: YANDEX_COOKIES_FILE. yandex_cookies_file: str | None = None + # ── #639: Cian browser auto-login (Variant B) ──────────────────────────── + # Headless-браузер логинится в cian.ru email+паролем → извлекает auth-cookies → + # save_session. Подтверждено: вход без SMS/OTP. Креды пусты в dev → endpoint 503. + # Все селекторы env-overridable: при смене DOM cian тюним через .env.runtime БЕЗ + # редеплоя кода (restart backend достаточно). ENV: CIAN_LOGIN_EMAIL, CIAN_LOGIN_PASSWORD, ... + cian_login_email: str | None = None + cian_login_password: str | None = None + cian_login_url: str = "https://www.cian.ru/zhalobi-i-otzivi/" + cian_login_pre_click_selector: str = "" + cian_login_email_selector: str = "input[name='username']" + cian_login_password_selector: str = "input[type='password']" + cian_login_submit_selector: str = "button[type='submit']" + cian_login_success_cookie: str = "DMIR_AUTH" + cian_login_wait_ms: int = 4000 + # ── #884/#905: BrowserFetcher — HTTP-клиент к tradein-browser контейнеру ──── # scraper_fetch_mode: "curl_cffi" (дефолт, текущее поведение) или "browser" # (HTTP POST к tradein-browser /fetch эндпоинту). Phase 1+. diff --git a/tradein-mvp/backend/app/services/scrapers/browser_fetcher.py b/tradein-mvp/backend/app/services/scrapers/browser_fetcher.py index 730673db..c7b7328a 100644 --- a/tradein-mvp/backend/app/services/scrapers/browser_fetcher.py +++ b/tradein-mvp/backend/app/services/scrapers/browser_fetcher.py @@ -86,6 +86,54 @@ class BrowserFetcher: await asyncio.sleep(_RETRY_SLEEP_S) return await self._post_fetch(url) + async def login( + self, + *, + url: str, + email: str, + password: str, + email_selector: str, + password_selector: str, + submit_selector: str, + success_cookie: str, + pre_click_selector: str = "", + wait_ms: int | None = None, + ) -> dict[str, str]: + """Логинится через tradein-browser /login и возвращает cookies как dict name→value. + + Использует увеличенный таймаут (60s) — логин медленнее обычного fetch. + При HTTPError / TransportError делает одну повторную попытку. + + Returns: + Плоский словарь {cookie_name: cookie_value}. + """ + assert self._client is not None, "BrowserFetcher: используй как async context manager" + assert self._endpoint is not None, "BrowserFetcher: endpoint не задан" + + body: dict[str, object] = { + "url": url, + "email": email, + "password": password, + "email_selector": email_selector, + "password_selector": password_selector, + "submit_selector": submit_selector, + "success_cookie": success_cookie, + "pre_click_selector": pre_click_selector, + } + if wait_ms is not None: + body["wait_ms"] = wait_ms + + try: + return await self._post_login(body) + except (httpx.HTTPError, httpx.TransportError) as exc: + logger.warning( + "BrowserFetcher: ошибка login-запроса (%s), retry через %.1fs", + type(exc).__name__, + _RETRY_SLEEP_S, + ) + await asyncio.sleep(_RETRY_SLEEP_S) + return await self._post_login(body) + # ── internal ─────────────────────────────────────────────────────────────── async def _post_fetch(self, url: str) -> str: @@ -102,3 +150,31 @@ class BrowserFetcher: html = data["html"] logger.debug("BrowserFetcher: fetch OK url=%r html_len=%d", url, len(html)) return html + + async def _post_login(self, body: dict[str, object]) -> dict[str, str]: + """Один HTTP POST к /login эндпоинту сервиса.""" + assert self._client is not None + assert self._endpoint is not None + + resp = await self._client.post( + f"{self._endpoint}/login", + json=body, + timeout=httpx.Timeout(60.0), + ) + if resp.status_code == 502: + data = resp.json() + has_screenshot = bool(data.get("screenshot_b64")) + logger.debug( + "BrowserFetcher: login 502 — has_screenshot=%s page_url=%r", + has_screenshot, + data.get("page_url"), + ) + raise RuntimeError( + f"browser login failed: {data.get('error')} url={data.get('page_url')}" + ) + resp.raise_for_status() + data = resp.json() + raw: list[dict[str, object]] = data["cookies"] + result = {str(c["name"]): str(c["value"]) for c in raw} + logger.info("BrowserFetcher: login OK cookie_count=%d", len(result)) + return result diff --git a/tradein-mvp/browser/server.py b/tradein-mvp/browser/server.py index 83226b0c..afd0b710 100644 --- a/tradein-mvp/browser/server.py +++ b/tradein-mvp/browser/server.py @@ -6,6 +6,7 @@ GET /health → {"status": "ok"} POST /fetch → {"url": "..."} → {"html": "..."} + POST /login → {"url": "...", "email": "...", "password": "...", ...} → {"cookies": [...]} Такой подход выбран потому, что ``camoufox.server.launch_server`` (Playwright WS-сервер) несовместим с современными версиями playwright (1.45–1.60): @@ -25,6 +26,7 @@ WS-сервер) несовместим с современными версия """ import asyncio +import base64 import logging import os from urllib.parse import urlparse @@ -252,6 +254,160 @@ def _is_browser_crash(exc: BaseException) -> bool: ) +# ── login handler ────────────────────────────────────────────────────────────── + + +class LoginError(Exception): + """Исключение с диагностическим payload для /login эндпоинта.""" + + def __init__(self, payload: dict) -> None: + super().__init__(str(payload)) + self.payload = payload + + +async def login_handler(request: web.Request) -> web.Response: + """POST /login {...} → {"cookies": [...]} + + Выполняет вход на указанную страницу (email + пароль), ждёт появления + success_cookie, возвращает полный список cookies браузерного контекста. + Запросы сериализованы через lock (один за раз — один браузер). + """ + try: + body = await request.json() + except Exception: + return web.json_response({"error": "invalid JSON body"}, status=400) + + missing = [f for f in ("url", "email", "password", "email_selector", + "password_selector", "submit_selector") if not body.get(f)] + if missing: + return web.json_response({"error": f"missing required fields: {missing}"}, status=400) + + assert _fetch_lock is not None, "_fetch_lock not initialised" + + async with _fetch_lock: + try: + cookies = await _do_login(body) + except LoginError as exc: + return web.json_response(exc.payload, status=502) + except Exception as exc: + logger.error( + "tradein-browser: login error url=%r: %s: %s", + body.get("url"), + type(exc).__name__, + exc, + ) + return web.json_response({"error": f"{type(exc).__name__}: {exc}"}, status=502) + + return web.json_response({"cookies": cookies}) + + +async def _do_login(params: dict) -> list[dict]: + """Одна попытка логина; при краше браузера — перезапускает и повторяет.""" + try: + return await _login_once(params) + except LoginError: + raise # диагностический payload идёт напрямую в хендлер + except Exception as exc: + if _is_browser_crash(exc): + logger.warning( + "tradein-browser: краш браузера (%s) при логине, перезапуск + retry", + type(exc).__name__, + ) + await _relaunch_browser() + return await _login_once(params) + raise + + +async def _login_once(params: dict) -> list[dict]: + """Открывает страницу, заполняет форму, нажимает submit, ждёт success_cookie.""" + global _page_counter + + browser = _browser + assert browser is not None, "browser not launched" + + url: str = params["url"] + email: str = params["email"] + password: str = params["password"] + email_selector: str = params["email_selector"] + password_selector: str = params["password_selector"] + submit_selector: str = params["submit_selector"] + pre_click_selector: str = params.get("pre_click_selector", "") or "" + success_cookie: str = params.get("success_cookie", "") + wait_ms: int = params.get("wait_ms") or BROWSER_WAIT_MS + + page = await browser.new_page() # type: ignore[attr-defined] + try: + await page.goto(url, timeout=BROWSER_NAV_TIMEOUT_MS, wait_until="domcontentloaded") # type: ignore[attr-defined] + await page.wait_for_timeout(wait_ms) # type: ignore[attr-defined] + + if pre_click_selector: + try: + await page.click(pre_click_selector, timeout=15000) # type: ignore[attr-defined] + await page.wait_for_timeout(1500) # type: ignore[attr-defined] + except Exception as pre_exc: + logger.warning( + "tradein-browser: pre_click_selector=%r не найден (не фатально): %s", + pre_click_selector, + pre_exc, + ) + + await page.fill(email_selector, email, timeout=15000) # type: ignore[attr-defined] + await page.fill(password_selector, password, timeout=15000) # type: ignore[attr-defined] + await page.click(submit_selector, timeout=15000) # type: ignore[attr-defined] + + # Ждём появления success_cookie до ~20s + cookie_found = False + if success_cookie: + for _ in range(40): # 40 × 500ms = 20s + await page.wait_for_timeout(500) # type: ignore[attr-defined] + ctx_cookies = await page.context.cookies() # type: ignore[attr-defined] + if any(c.get("name") == success_cookie for c in ctx_cookies): + cookie_found = True + break + logger.info( + "tradein-browser: login success_cookie=%r present=%s", + success_cookie, + cookie_found, + ) + + cookies: list[dict] = await page.context.cookies() # type: ignore[attr-defined] + logger.info( + "tradein-browser: login OK url=%r email_selector=%r cookie_count=%d", + url, + email_selector, + len(cookies), + ) + except Exception as exc: + screenshot_b64 = "" + try: + screenshot_bytes: bytes = await page.screenshot() # type: ignore[attr-defined] + screenshot_b64 = base64.b64encode(screenshot_bytes).decode() + except Exception: + pass + page_url = "" + try: + page_url = page.url # type: ignore[attr-defined] + except Exception: + pass + await page.close() # type: ignore[attr-defined] + raise LoginError({ + "error": f"{type(exc).__name__}: {exc}", + "page_url": page_url, + "screenshot_b64": screenshot_b64, + }) from exc + + await page.close() # type: ignore[attr-defined] + _page_counter += 1 + if _page_counter >= BROWSER_RECYCLE_PAGES: + logger.info( + "tradein-browser: recycle threshold (%d) достигнут после login, перезапуск браузера", + BROWSER_RECYCLE_PAGES, + ) + await _relaunch_browser() + + return cookies + + # ── entrypoint ───────────────────────────────────────────────────────────────── @@ -261,6 +417,7 @@ def build_app() -> web.Application: app.on_cleanup.append(_on_cleanup) app.router.add_get("/health", health_handler) app.router.add_post("/fetch", fetch_handler) + app.router.add_post("/login", login_handler) return app