diff --git a/tradein-mvp/backend/app/services/scrapers/avito_detail.py b/tradein-mvp/backend/app/services/scrapers/avito_detail.py index c6373df6..4f9319d8 100644 --- a/tradein-mvp/backend/app/services/scrapers/avito_detail.py +++ b/tradein-mvp/backend/app/services/scrapers/avito_detail.py @@ -70,9 +70,7 @@ _INT_RE = re.compile(r"\d+") _FLOOR_SPLIT_RE = re.compile(r"(\d+)\s+из\s+(\d+)") OWNERS_RE = re.compile(r"^(\d+)\s+собственник(?:\s+или\s+больше)?", re.IGNORECASE) -OWNER_CHANGE_RE = re.compile( - r"смена собственника\s+(\d+)\s+(\w+)\s+(\d{4})", re.IGNORECASE -) +OWNER_CHANGE_RE = re.compile(r"смена собственника\s+(\d+)\s+(\w+)\s+(\d{4})", re.IGNORECASE) ENCUMBRANCES_RE = re.compile(r"Не\s+найдены\s+ограничения", re.IGNORECASE) REGISTRY_MATCH_RE = re.compile(r"Совпадают\s+площадь", re.IGNORECASE) @@ -138,10 +136,10 @@ _REPAIR_MAP: dict[str, str] = { # Русские значения из Avito HTML → каноничный enum (needs_repair/standard/good/excellent) # Маппинг выровнен по repair_state_normalizer.normalize_repair_state(). "косметический": "standard", - "евро": "good", - "дизайнерский": "excellent", - "требуется": "needs_repair", - "без ремонта": "needs_repair", + "евро": "good", + "дизайнерский": "excellent", + "требуется": "needs_repair", + "без ремонта": "needs_repair", } _SALE_TYPE_MAP: dict[str, str] = { @@ -240,20 +238,31 @@ async def fetch_detail( ValueError — если item_id не извлечён из HTML. """ own_session = cffi_session is None - session = cffi_session or AsyncSession( - impersonate="chrome120", - timeout=25, - headers={ - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", - "Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8", - "Cache-Control": "max-age=0", - "Sec-Fetch-Dest": "document", - "Sec-Fetch-Mode": "navigate", - "Sec-Fetch-Site": "none", - "Sec-Fetch-User": "?1", - "Upgrade-Insecure-Requests": "1", - }, - ) + if cffi_session is None: + # Mobile proxy wiring (#806 follow-up): own-session path (no shared session passed, + # e.g. admin endpoint). Caller-provided sessions (scrape_pipeline) already have proxy + # from AvitoScraper.__aenter__ — don't override. proxy=None → прямое подключение. + from app.core.config import settings + + _proxy_url = settings.scraper_proxy_url + _proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None + session = AsyncSession( + impersonate="chrome120", + timeout=25, + proxies=_proxies, + headers={ + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8", + "Cache-Control": "max-age=0", + "Sec-Fetch-Dest": "document", + "Sec-Fetch-Mode": "navigate", + "Sec-Fetch-Site": "none", + "Sec-Fetch-User": "?1", + "Upgrade-Insecure-Requests": "1", + }, + ) + else: + session = cffi_session full_url = item_url if item_url.startswith("http") else urljoin(AVITO_BASE, item_url) try: @@ -263,9 +272,7 @@ async def fetch_detail( if response.status_code == 429: raise AvitoRateLimitedError(f"Avito detail HTTP 429 for {full_url}") if response.status_code != 200: - raise ValueError( - f"avito detail HTTP {response.status_code} for {full_url}" - ) + raise ValueError(f"avito detail HTTP {response.status_code} for {full_url}") enrichment = parse_detail_html(response.text, full_url) return enrichment finally: @@ -570,9 +577,7 @@ def save_detail_enrichment(db: Session, e: DetailEnrichment) -> bool: db.commit() found = result.rowcount > 0 if not found: - logger.warning( - "save_detail_enrichment: listing not found in DB for item_id=%s", e.item_id - ) + logger.warning("save_detail_enrichment: listing not found in DB for item_id=%s", e.item_id) else: logger.info("save_detail_enrichment: updated listing item_id=%s", e.item_id) return found diff --git a/tradein-mvp/backend/app/services/scrapers/avito_imv.py b/tradein-mvp/backend/app/services/scrapers/avito_imv.py index f1caa3be..9b56d321 100644 --- a/tradein-mvp/backend/app/services/scrapers/avito_imv.py +++ b/tradein-mvp/backend/app/services/scrapers/avito_imv.py @@ -396,14 +396,21 @@ async def evaluate_via_imv( try: from curl_cffi.requests import AsyncSession as CffiAsyncSession + from app.core.config import settings + _own_session = False if cffi_session is None: # Зеркалим production-набор из AvitoScraper.__aenter__ (avito.py:150-162): # chrome120 TLS + document-заголовки + timeout. Затем warm-up GET для # seed anti-bot cookies — bare-session XHR Avito банит на server-IP. + # Mobile proxy wiring (#806 follow-up): только для own-session (переданная + # сессия уже с прокси от AvitoScraper). proxy=None → прямое подключение. + _proxy_url = settings.scraper_proxy_url + _proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None cffi_session = CffiAsyncSession( impersonate="chrome120", timeout=_HTTP_TIMEOUT_SEC, + proxies=_proxies, headers=_DOC_HEADERS, ) _own_session = True @@ -496,9 +503,7 @@ async def _geocode(session: Any, address: str) -> IMVGeo: normalized = data_a.get("normalizedAddress") if not lat or not lon or not normalized: - raise IMVAddressNotFoundError( - f"Avito /coords/by_address не вернул point для {address!r}" - ) + raise IMVAddressNotFoundError(f"Avito /coords/by_address не вернул point для {address!r}") # Step B — rich JWT url_b = f"{AVITO_BASE}{POSITION_ENDPOINT}" @@ -767,9 +772,7 @@ def save_imv_placement_history( "removed_date": item.removed_date, "exposure_days": item.exposure_days, "raw_payload": ( - json.dumps(item.raw_payload, ensure_ascii=False) - if item.raw_payload - else None + json.dumps(item.raw_payload, ensure_ascii=False) if item.raw_payload else None ), }, ) diff --git a/tradein-mvp/backend/app/services/scrapers/cian_newbuilding.py b/tradein-mvp/backend/app/services/scrapers/cian_newbuilding.py index cc702d3f..2ae7127e 100644 --- a/tradein-mvp/backend/app/services/scrapers/cian_newbuilding.py +++ b/tradein-mvp/backend/app/services/scrapers/cian_newbuilding.py @@ -13,6 +13,7 @@ Sister state containers extracted from same MFE initialState top-level keys: Stage 6 of CianScraper v1. """ + from __future__ import annotations import json @@ -22,6 +23,7 @@ from typing import Any from curl_cffi.requests import AsyncSession # type: ignore[import-untyped] +from app.core.config import settings from app.services.scrapers.cian_state_parser import extract_all_states, extract_state logger = logging.getLogger(__name__) @@ -92,9 +94,14 @@ async def fetch_newbuilding( """ close_session = False if session is None: + # Mobile proxy wiring (#806 follow-up): Cian ЖК страница — datacenter-IP бан. + # proxy=None → прямое подключение (dev без прокси). + _proxy_url = settings.scraper_proxy_url + _proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None session = AsyncSession( impersonate="chrome120", timeout=30.0, + proxies=_proxies, headers={ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8", @@ -255,19 +262,21 @@ def _extract_chart(rv_state: dict[str, Any]) -> list[dict[str, Any]]: for label, value in zip(labels, values, strict=False): if label is None or value is None: continue - points.append({ - "month_date": str(label)[:10], # YYYY-MM-DD - "room_count": "all", # aggregate (initial state) - "prices_type": "price", # total price (not priceSqm) - "period": "halfYear", # default period shown - "price_per_sqm": None, # chart shows total price, not sqm - }) + points.append( + { + "month_date": str(label)[:10], # YYYY-MM-DD + "room_count": "all", # aggregate (initial state) + "prices_type": "price", # total price (not priceSqm) + "period": "halfYear", # default period shown + "price_per_sqm": None, # chart shows total price, not sqm + } + ) # Store raw value as price_per_sqm placeholder for DB # (actual sqm price would require area context) # Use the value as-is; caller / downstream can enrich - points[-1]["price_per_sqm"] = float(value) if isinstance( - value, (int, float) - ) else None + points[-1]["price_per_sqm"] = ( + float(value) if isinstance(value, int | float) else None + ) if points: return points @@ -286,13 +295,17 @@ def _extract_chart(rv_state: dict[str, Any]) -> list[dict[str, Any]]: price_val = float(price) except (TypeError, ValueError): pass - points.append({ - "month_date": str(month)[:10], - "room_count": str(entry.get("roomType") or entry.get("roomCount") or "all"), - "prices_type": str(entry.get("pricesType") or entry.get("priceType") or "price"), - "period": str(entry.get("period") or "halfYear"), - "price_per_sqm": price_val, - }) + points.append( + { + "month_date": str(month)[:10], + "room_count": str(entry.get("roomType") or entry.get("roomCount") or "all"), + "prices_type": str( + entry.get("pricesType") or entry.get("priceType") or "price" + ), + "period": str(entry.get("period") or "halfYear"), + "price_per_sqm": price_val, + } + ) return points return [] @@ -380,10 +393,12 @@ def _extract_nested_offers(offers_state: dict[str, Any]) -> list[dict[str, Any]] continue for layout in layouts: if isinstance(layout, dict): - result.append({ - "room_count": room_type, - **layout, - }) + result.append( + { + "room_count": room_type, + **layout, + } + ) return result @@ -537,9 +552,7 @@ def save_newbuilding_enrichment( "hid": house_id, "cs": check.get("check_status"), "cn": check.get("check_name"), - "det": json.dumps( - check.get("details") or [], ensure_ascii=False - ), + "det": json.dumps(check.get("details") or [], ensure_ascii=False), }, ) @@ -580,9 +593,13 @@ async def resolve_cian_zhk_url( """ close_session = False if session is None: + # Mobile proxy wiring (#806 follow-up): resolve ЖК URL через мобильный прокси. + _proxy_url = settings.scraper_proxy_url + _proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None session = AsyncSession( impersonate="chrome120", timeout=15.0, + proxies=_proxies, headers={ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8", @@ -595,9 +612,7 @@ async def resolve_cian_zhk_url( resp = await session.get(fallback_url, allow_redirects=True) final_url = str(resp.url) if final_url and final_url != fallback_url: - logger.debug( - "resolve_cian_zhk_url id=%s → %s", cian_internal_house_id, final_url - ) + logger.debug("resolve_cian_zhk_url id=%s → %s", cian_internal_house_id, final_url) return final_url # Redirect not followed or same URL — return fallback as canonical if resp.status_code == 200: @@ -609,9 +624,7 @@ async def resolve_cian_zhk_url( ) return None except Exception as exc: - logger.warning( - "resolve_cian_zhk_url id=%s failed: %s", cian_internal_house_id, exc - ) + logger.warning("resolve_cian_zhk_url id=%s failed: %s", cian_internal_house_id, exc) return None finally: if close_session: diff --git a/tradein-mvp/backend/app/services/scrapers/cian_valuation.py b/tradein-mvp/backend/app/services/scrapers/cian_valuation.py index f2fe97f9..e5a4dd36 100644 --- a/tradein-mvp/backend/app/services/scrapers/cian_valuation.py +++ b/tradein-mvp/backend/app/services/scrapers/cian_valuation.py @@ -17,6 +17,7 @@ State shape (sec 24.3–24.6 Schema_Cian_SERP_Inventory): Used as 6th evaluation source в estimator.py (Stage 9). """ + from __future__ import annotations import hashlib @@ -30,6 +31,7 @@ from curl_cffi.requests import AsyncSession from sqlalchemy import text from sqlalchemy.orm import Session +from app.core.config import settings from app.services.cian_session import load_session, mark_session_invalid from app.services.scrapers.cian_state_parser import extract_state @@ -173,11 +175,16 @@ async def estimate_via_cian_valuation( url = f"{VALUATION_BASE_URL}?{urlencode(params, safe='[]')}" # 4. Fetch с TLS fingerprint (curl_cffi, impersonate chrome120) + # Mobile proxy wiring (#806 follow-up): Cian валюация — такой же datacenter-бан риск + # как SERP. Используем scraper_proxy_url. proxy=None → прямое подключение (dev). + _proxy_url = settings.scraper_proxy_url + _proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None try: async with AsyncSession( impersonate="chrome120", cookies=cookies, timeout=25.0, + proxies=_proxies, headers={"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"}, ) as session: resp = await session.get(url, allow_redirects=True) @@ -294,16 +301,19 @@ def _parse_valuation_state(state: dict[str, Any]) -> CianValuationResult: # date: unix_ms → ISO string "YYYY-MM-DD" date_ms = entry.get("date") date_str: str | None = None - if isinstance(date_ms, (int, float)) and date_ms > 0: + if isinstance(date_ms, int | float) and date_ms > 0: import datetime + date_str = datetime.datetime.fromtimestamp( date_ms / 1000, tz=datetime.UTC ).strftime("%Y-%m-%d") - result.chart.append({ - "month_date": date_str or "", - "price": _parse_num(entry.get("price")), - "price_formatted": entry.get("priceFormatted"), - }) + result.chart.append( + { + "month_date": date_str or "", + "price": _parse_num(entry.get("price")), + "price_formatted": entry.get("priceFormatted"), + } + ) # --- houseInfo: 15 items о доме --- house_info_wrapper = state.get("houseInfo") or {} @@ -347,8 +357,9 @@ def _parse_change_pct(value_str: str) -> float | None: def _load_from_cache(db: Session, cache_key: str) -> CianValuationResult | None: """SELECT from external_valuations where cache_key + source='cian_valuation' + not expired.""" - row = db.execute( - text(""" + row = ( + db.execute( + text(""" SELECT raw_payload, sale_price_rub, @@ -369,8 +380,11 @@ def _load_from_cache(db: Session, cache_key: str) -> CianValuationResult | None: ORDER BY fetched_at DESC LIMIT 1 """), - {"ck": cache_key}, - ).mappings().first() + {"ck": cache_key}, + ) + .mappings() + .first() + ) if row is None: return None diff --git a/tradein-mvp/backend/app/services/scrapers/yandex_realty.py b/tradein-mvp/backend/app/services/scrapers/yandex_realty.py index 893bb63d..80531abd 100644 --- a/tradein-mvp/backend/app/services/scrapers/yandex_realty.py +++ b/tradein-mvp/backend/app/services/scrapers/yandex_realty.py @@ -242,9 +242,15 @@ class YandexRealtyScraper(BaseScraper): from app.core.config import settings self._cookies = _load_cookies_from_file(settings.yandex_cookies_file) + # Mobile proxy wiring (#806 follow-up): Yandex blocks datacenter IPs with + # captcha/shell HTML even with Chrome TLS fingerprint. Route via mobile proxy. + # proxy=None → curl_cffi ходит напрямую (dev без прокси — не падаем). + _proxy_url = settings.scraper_proxy_url + _proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None self._cffi_session = _CurlCffiSession( impersonate="chrome120", cookies=self._cookies or {}, + proxies=_proxies, headers={ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8", diff --git a/tradein-mvp/backend/app/services/scrapers/yandex_valuation.py b/tradein-mvp/backend/app/services/scrapers/yandex_valuation.py index 5760382c..91033dd4 100644 --- a/tradein-mvp/backend/app/services/scrapers/yandex_valuation.py +++ b/tradein-mvp/backend/app/services/scrapers/yandex_valuation.py @@ -165,7 +165,13 @@ class YandexValuationScraper(BaseScraper): # Override: Yandex valuation endpoint gates SSR data on Chrome TLS # fingerprint. Plain httpx returns shell HTML (CSR-only). # Sibling: yandex_realty.py / scripts/local-sweep-ekb-yandex.py - self._cffi_session = _CurlCffiSession(impersonate="chrome120") + # Mobile proxy wiring (#806 follow-up): route via mobile proxy to avoid + # datacenter-IP blocks. proxy=None → прямое подключение (dev). + from app.core.config import settings + + _proxy_url = settings.scraper_proxy_url + _proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None + self._cffi_session = _CurlCffiSession(impersonate="chrome120", proxies=_proxies) return self async def __aexit__(self, *args: object) -> None: # type: ignore[override] diff --git a/tradein-mvp/backend/tests/test_scraper_proxy.py b/tradein-mvp/backend/tests/test_scraper_proxy.py index 108e2b01..5e1aca70 100644 --- a/tradein-mvp/backend/tests/test_scraper_proxy.py +++ b/tradein-mvp/backend/tests/test_scraper_proxy.py @@ -1,10 +1,12 @@ -"""Tests for scraper proxy wiring (#806). +"""Tests for scraper proxy wiring (#806 + follow-up). Covers: - scraper_proxy_url property: SCRAPER_PROXY_URL takes precedence over AVITO_PROXY_URL - AVITO_PROXY_URL fallback works when SCRAPER_PROXY_URL is absent - None/empty → direct connection (no proxies dict) - _avito_proxies() helper returns correct dict shape + - per-scraper proxy wiring: YandexRealty, YandexValuation, CianValuation, + CianNewbuilding (fetch_newbuilding + resolve_cian_zhk_url), AvitoIMV, AvitoDetail """ from __future__ import annotations @@ -14,12 +16,20 @@ import os os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test") from types import SimpleNamespace -from unittest.mock import patch +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest -def _mock_settings(scraper_proxy_url: str | None) -> SimpleNamespace: - """Minimal settings stand-in with only the fields the proxy helpers read.""" - return SimpleNamespace(scraper_proxy_url=scraper_proxy_url) +def _mock_settings( + scraper_proxy_url: str | None, + yandex_cookies_file: str | None = None, +) -> SimpleNamespace: + """Minimal settings stand-in with fields read by proxy helpers and scraper __aenter__.""" + return SimpleNamespace( + scraper_proxy_url=scraper_proxy_url, + yandex_cookies_file=yandex_cookies_file, + ) # ── Settings.scraper_proxy_url property ────────────────────────────────────── @@ -140,3 +150,280 @@ def test_settings_ignores_misnamed_scraper_proxy_url_env(monkeypatch): monkeypatch.delenv("AVITO_PROXY_URL", raising=False) monkeypatch.setenv("SCRAPER_PROXY_URL_ENV", "http://wrong-name:1111") assert _fresh_settings().scraper_proxy_url is None + + +# ── Per-scraper proxy wiring (follow-up: yandex/valuation/imv/newbuilding) ──── +# Each test: patch settings.scraper_proxy_url, mock AsyncSession constructor, +# invoke __aenter__ / create-session path, assert proxies= kwarg is forwarded. +# proxy=None path: assert proxies kwarg is None / not passed as non-None. + + +_PROXY_URL = "http://mobile.proxy:8080" +_EXPECTED_PROXIES = {"http": _PROXY_URL, "https": _PROXY_URL} + + +# ── YandexRealtyScraper ────────────────────────────────────────────────────── + + +@pytest.mark.asyncio +async def test_yandex_realty_session_receives_proxies(monkeypatch): + """YandexRealtyScraper.__aenter__ forwards scraper_proxy_url as proxies= kwarg. + + settings is imported deferred inside __aenter__ via `from app.core.config import settings`, + so we patch app.core.config.settings (module attribute) which the deferred import reads. + """ + from app.services.scrapers.yandex_realty import YandexRealtyScraper + + mock_session = MagicMock() + mock_session.__aenter__ = AsyncMock(return_value=mock_session) + mock_session.__aexit__ = AsyncMock(return_value=None) + mock_session.close = AsyncMock() + + captured_kwargs: dict = {} + + def _fake_session(*args, **kwargs): + captured_kwargs.update(kwargs) + return mock_session + + monkeypatch.setattr("app.services.scrapers.yandex_realty._CurlCffiSession", _fake_session) + with patch("app.core.config.settings", _mock_settings(_PROXY_URL)): + scraper = YandexRealtyScraper() + await scraper.__aenter__() + + assert captured_kwargs.get("proxies") == _EXPECTED_PROXIES + + +@pytest.mark.asyncio +async def test_yandex_realty_session_no_proxies_when_none(monkeypatch): + """YandexRealtyScraper.__aenter__: proxies=None when scraper_proxy_url is None.""" + from app.services.scrapers.yandex_realty import YandexRealtyScraper + + mock_session = MagicMock() + mock_session.close = AsyncMock() + captured_kwargs: dict = {} + + def _fake_session(*args, **kwargs): + captured_kwargs.update(kwargs) + return mock_session + + monkeypatch.setattr("app.services.scrapers.yandex_realty._CurlCffiSession", _fake_session) + with patch("app.core.config.settings", _mock_settings(None)): + scraper = YandexRealtyScraper() + await scraper.__aenter__() + + assert captured_kwargs.get("proxies") is None + + +# ── YandexValuationScraper ─────────────────────────────────────────────────── + + +@pytest.mark.asyncio +async def test_yandex_valuation_session_receives_proxies(): + """YandexValuationScraper.__aenter__ forwards proxies= kwarg.""" + from app.services.scrapers.yandex_valuation import YandexValuationScraper + + mock_session = MagicMock() + mock_session.close = AsyncMock() + captured_kwargs: dict = {} + + def _fake_session(*args, **kwargs): + captured_kwargs.update(kwargs) + return mock_session + + with patch("app.services.scrapers.yandex_valuation._CurlCffiSession", _fake_session): + with patch("app.core.config.settings", _mock_settings(_PROXY_URL)): + scraper = YandexValuationScraper() + await scraper.__aenter__() + + assert captured_kwargs.get("proxies") == _EXPECTED_PROXIES + + +@pytest.mark.asyncio +async def test_yandex_valuation_session_no_proxies_when_none(): + """YandexValuationScraper.__aenter__: proxies=None when no proxy configured.""" + from app.services.scrapers.yandex_valuation import YandexValuationScraper + + mock_session = MagicMock() + mock_session.close = AsyncMock() + captured_kwargs: dict = {} + + def _fake_session(*args, **kwargs): + captured_kwargs.update(kwargs) + return mock_session + + with patch("app.services.scrapers.yandex_valuation._CurlCffiSession", _fake_session): + with patch("app.core.config.settings", _mock_settings(None)): + scraper = YandexValuationScraper() + await scraper.__aenter__() + + assert captured_kwargs.get("proxies") is None + + +# ── CianNewbuilding — fetch_newbuilding ─────────────────────────────────────── + + +@pytest.mark.asyncio +async def test_cian_newbuilding_own_session_receives_proxies(): + """fetch_newbuilding creates own session with proxies= when no session passed.""" + from app.services.scrapers import cian_newbuilding + + mock_session = MagicMock() + mock_resp = MagicMock() + mock_resp.status_code = 200 + mock_resp.text = "" # empty HTML → extract_state returns None → result None (ok) + mock_session.get = AsyncMock(return_value=mock_resp) + mock_session.close = AsyncMock() + + captured_kwargs: dict = {} + + def _fake_session(*args, **kwargs): + captured_kwargs.update(kwargs) + return mock_session + + with patch.object(cian_newbuilding, "AsyncSession", _fake_session): + with patch.object(cian_newbuilding, "settings", _mock_settings(_PROXY_URL)): + await cian_newbuilding.fetch_newbuilding("https://zhk-test.cian.ru/") + + assert captured_kwargs.get("proxies") == _EXPECTED_PROXIES + + +@pytest.mark.asyncio +async def test_cian_newbuilding_shared_session_not_recreated(): + """fetch_newbuilding does NOT create own session when one is passed.""" + from app.services.scrapers import cian_newbuilding + + mock_session = MagicMock() + mock_resp = MagicMock() + mock_resp.status_code = 200 + mock_resp.text = "" + mock_session.get = AsyncMock(return_value=mock_resp) + mock_session.close = AsyncMock() + + session_ctor_calls = [] + + def _fake_session(*args, **kwargs): + session_ctor_calls.append(kwargs) + return MagicMock() + + with patch.object(cian_newbuilding, "AsyncSession", _fake_session): + with patch.object(cian_newbuilding, "settings", _mock_settings(_PROXY_URL)): + await cian_newbuilding.fetch_newbuilding( + "https://zhk-test.cian.ru/", session=mock_session + ) + + # No own session should have been created (shared session was passed) + assert session_ctor_calls == [] + + +# ── AvitoDetail — fetch_detail own-session ─────────────────────────────────── + + +@pytest.mark.asyncio +async def test_avito_detail_own_session_receives_proxies(): + """fetch_detail creates own session with proxies= when no cffi_session passed.""" + from app.services.scrapers import avito_detail + + mock_session = MagicMock() + mock_resp = MagicMock() + mock_resp.status_code = 200 + # Minimal HTML that parse_detail_html won't raise ValueError (item_id required) + mock_resp.text = "" + mock_session.get = AsyncMock(return_value=mock_resp) + mock_session.close = AsyncMock() + + captured_kwargs: dict = {} + + def _fake_session(*args, **kwargs): + captured_kwargs.update(kwargs) + return mock_session + + with patch.object(avito_detail, "AsyncSession", _fake_session): + with patch("app.core.config.settings", _mock_settings(_PROXY_URL)): + try: + await avito_detail.fetch_detail("/ekaterinburg/kvartiry/test-1234") + except ValueError: + pass # parse_detail_html raises ValueError (no item_id) — that's ok + + assert captured_kwargs.get("proxies") == _EXPECTED_PROXIES + + +@pytest.mark.asyncio +async def test_avito_detail_shared_session_not_patched(): + """fetch_detail does NOT build own session when cffi_session is provided.""" + from app.services.scrapers import avito_detail + + mock_session = MagicMock() + mock_resp = MagicMock() + mock_resp.status_code = 200 + mock_resp.text = "" + mock_session.get = AsyncMock(return_value=mock_resp) + mock_session.close = AsyncMock() + + ctor_calls = [] + + def _fake_session(*args, **kwargs): + ctor_calls.append(kwargs) + return MagicMock() + + with patch.object(avito_detail, "AsyncSession", _fake_session): + with patch("app.core.config.settings", _mock_settings(_PROXY_URL)): + try: + await avito_detail.fetch_detail( + "/ekaterinburg/kvartiry/test-1234", + cffi_session=mock_session, + ) + except ValueError: + pass + + # No own session created — shared session was used as-is + assert ctor_calls == [] + + +# ── AvitoIMV — evaluate_via_imv own-session ────────────────────────────────── + + +@pytest.mark.asyncio +async def test_avito_imv_own_session_receives_proxies(): + """evaluate_via_imv creates own session with proxies= when none passed. + + `CffiAsyncSession` is imported locally inside evaluate_via_imv as + `from curl_cffi.requests import AsyncSession as CffiAsyncSession`, so we + patch `curl_cffi.requests.AsyncSession` (the source) to intercept it. + """ + from app.services.scrapers import avito_imv + + mock_session = MagicMock() + mock_warmup_resp = MagicMock() + mock_warmup_resp.status_code = 200 + mock_geo_resp = MagicMock() + mock_geo_resp.status_code = 200 + mock_geo_resp.json = MagicMock(return_value={"result": {"point": None}}) + mock_session.get = AsyncMock(return_value=mock_warmup_resp) + mock_session.post = AsyncMock(return_value=mock_geo_resp) + mock_session.close = AsyncMock() + + captured_kwargs: dict = {} + + def _fake_session(*args, **kwargs): + captured_kwargs.update(kwargs) + return mock_session + + # Patch the source so the local `from curl_cffi.requests import AsyncSession` picks it up + with patch("curl_cffi.requests.AsyncSession", _fake_session): + with patch("app.core.config.settings", _mock_settings(_PROXY_URL)): + try: + await avito_imv.evaluate_via_imv( + address="ул. Тургенева, 4", + rooms=2, + area_m2=50.0, + floor=3, + floor_at_home=9, + house_type="panel", + renovation_type="cosmetic", + has_balcony=False, + has_loggia=False, + ) + except Exception: + pass # network/parse failures expected in unit test + + assert captured_kwargs.get("proxies") == _EXPECTED_PROXIES