fix(tradein): route yandex/valuation/imv/newbuilding scrapers через scraper_proxy_url #860
7 changed files with 413 additions and 79 deletions
|
|
@ -70,9 +70,7 @@ _INT_RE = re.compile(r"\d+")
|
||||||
_FLOOR_SPLIT_RE = re.compile(r"(\d+)\s+из\s+(\d+)")
|
_FLOOR_SPLIT_RE = re.compile(r"(\d+)\s+из\s+(\d+)")
|
||||||
|
|
||||||
OWNERS_RE = re.compile(r"^(\d+)\s+собственник(?:\s+или\s+больше)?", re.IGNORECASE)
|
OWNERS_RE = re.compile(r"^(\d+)\s+собственник(?:\s+или\s+больше)?", re.IGNORECASE)
|
||||||
OWNER_CHANGE_RE = re.compile(
|
OWNER_CHANGE_RE = re.compile(r"смена собственника\s+(\d+)\s+(\w+)\s+(\d{4})", re.IGNORECASE)
|
||||||
r"смена собственника\s+(\d+)\s+(\w+)\s+(\d{4})", re.IGNORECASE
|
|
||||||
)
|
|
||||||
ENCUMBRANCES_RE = re.compile(r"Не\s+найдены\s+ограничения", re.IGNORECASE)
|
ENCUMBRANCES_RE = re.compile(r"Не\s+найдены\s+ограничения", re.IGNORECASE)
|
||||||
REGISTRY_MATCH_RE = re.compile(r"Совпадают\s+площадь", re.IGNORECASE)
|
REGISTRY_MATCH_RE = re.compile(r"Совпадают\s+площадь", re.IGNORECASE)
|
||||||
|
|
||||||
|
|
@ -138,10 +136,10 @@ _REPAIR_MAP: dict[str, str] = {
|
||||||
# Русские значения из Avito HTML → каноничный enum (needs_repair/standard/good/excellent)
|
# Русские значения из Avito HTML → каноничный enum (needs_repair/standard/good/excellent)
|
||||||
# Маппинг выровнен по repair_state_normalizer.normalize_repair_state().
|
# Маппинг выровнен по repair_state_normalizer.normalize_repair_state().
|
||||||
"косметический": "standard",
|
"косметический": "standard",
|
||||||
"евро": "good",
|
"евро": "good",
|
||||||
"дизайнерский": "excellent",
|
"дизайнерский": "excellent",
|
||||||
"требуется": "needs_repair",
|
"требуется": "needs_repair",
|
||||||
"без ремонта": "needs_repair",
|
"без ремонта": "needs_repair",
|
||||||
}
|
}
|
||||||
|
|
||||||
_SALE_TYPE_MAP: dict[str, str] = {
|
_SALE_TYPE_MAP: dict[str, str] = {
|
||||||
|
|
@ -240,20 +238,31 @@ async def fetch_detail(
|
||||||
ValueError — если item_id не извлечён из HTML.
|
ValueError — если item_id не извлечён из HTML.
|
||||||
"""
|
"""
|
||||||
own_session = cffi_session is None
|
own_session = cffi_session is None
|
||||||
session = cffi_session or AsyncSession(
|
if cffi_session is None:
|
||||||
impersonate="chrome120",
|
# Mobile proxy wiring (#806 follow-up): own-session path (no shared session passed,
|
||||||
timeout=25,
|
# e.g. admin endpoint). Caller-provided sessions (scrape_pipeline) already have proxy
|
||||||
headers={
|
# from AvitoScraper.__aenter__ — don't override. proxy=None → прямое подключение.
|
||||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
from app.core.config import settings
|
||||||
"Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8",
|
|
||||||
"Cache-Control": "max-age=0",
|
_proxy_url = settings.scraper_proxy_url
|
||||||
"Sec-Fetch-Dest": "document",
|
_proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None
|
||||||
"Sec-Fetch-Mode": "navigate",
|
session = AsyncSession(
|
||||||
"Sec-Fetch-Site": "none",
|
impersonate="chrome120",
|
||||||
"Sec-Fetch-User": "?1",
|
timeout=25,
|
||||||
"Upgrade-Insecure-Requests": "1",
|
proxies=_proxies,
|
||||||
},
|
headers={
|
||||||
)
|
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||||
|
"Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8",
|
||||||
|
"Cache-Control": "max-age=0",
|
||||||
|
"Sec-Fetch-Dest": "document",
|
||||||
|
"Sec-Fetch-Mode": "navigate",
|
||||||
|
"Sec-Fetch-Site": "none",
|
||||||
|
"Sec-Fetch-User": "?1",
|
||||||
|
"Upgrade-Insecure-Requests": "1",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
session = cffi_session
|
||||||
|
|
||||||
full_url = item_url if item_url.startswith("http") else urljoin(AVITO_BASE, item_url)
|
full_url = item_url if item_url.startswith("http") else urljoin(AVITO_BASE, item_url)
|
||||||
try:
|
try:
|
||||||
|
|
@ -263,9 +272,7 @@ async def fetch_detail(
|
||||||
if response.status_code == 429:
|
if response.status_code == 429:
|
||||||
raise AvitoRateLimitedError(f"Avito detail HTTP 429 for {full_url}")
|
raise AvitoRateLimitedError(f"Avito detail HTTP 429 for {full_url}")
|
||||||
if response.status_code != 200:
|
if response.status_code != 200:
|
||||||
raise ValueError(
|
raise ValueError(f"avito detail HTTP {response.status_code} for {full_url}")
|
||||||
f"avito detail HTTP {response.status_code} for {full_url}"
|
|
||||||
)
|
|
||||||
enrichment = parse_detail_html(response.text, full_url)
|
enrichment = parse_detail_html(response.text, full_url)
|
||||||
return enrichment
|
return enrichment
|
||||||
finally:
|
finally:
|
||||||
|
|
@ -570,9 +577,7 @@ def save_detail_enrichment(db: Session, e: DetailEnrichment) -> bool:
|
||||||
db.commit()
|
db.commit()
|
||||||
found = result.rowcount > 0
|
found = result.rowcount > 0
|
||||||
if not found:
|
if not found:
|
||||||
logger.warning(
|
logger.warning("save_detail_enrichment: listing not found in DB for item_id=%s", e.item_id)
|
||||||
"save_detail_enrichment: listing not found in DB for item_id=%s", e.item_id
|
|
||||||
)
|
|
||||||
else:
|
else:
|
||||||
logger.info("save_detail_enrichment: updated listing item_id=%s", e.item_id)
|
logger.info("save_detail_enrichment: updated listing item_id=%s", e.item_id)
|
||||||
return found
|
return found
|
||||||
|
|
|
||||||
|
|
@ -396,14 +396,21 @@ async def evaluate_via_imv(
|
||||||
try:
|
try:
|
||||||
from curl_cffi.requests import AsyncSession as CffiAsyncSession
|
from curl_cffi.requests import AsyncSession as CffiAsyncSession
|
||||||
|
|
||||||
|
from app.core.config import settings
|
||||||
|
|
||||||
_own_session = False
|
_own_session = False
|
||||||
if cffi_session is None:
|
if cffi_session is None:
|
||||||
# Зеркалим production-набор из AvitoScraper.__aenter__ (avito.py:150-162):
|
# Зеркалим production-набор из AvitoScraper.__aenter__ (avito.py:150-162):
|
||||||
# chrome120 TLS + document-заголовки + timeout. Затем warm-up GET для
|
# chrome120 TLS + document-заголовки + timeout. Затем warm-up GET для
|
||||||
# seed anti-bot cookies — bare-session XHR Avito банит на server-IP.
|
# seed anti-bot cookies — bare-session XHR Avito банит на server-IP.
|
||||||
|
# Mobile proxy wiring (#806 follow-up): только для own-session (переданная
|
||||||
|
# сессия уже с прокси от AvitoScraper). proxy=None → прямое подключение.
|
||||||
|
_proxy_url = settings.scraper_proxy_url
|
||||||
|
_proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None
|
||||||
cffi_session = CffiAsyncSession(
|
cffi_session = CffiAsyncSession(
|
||||||
impersonate="chrome120",
|
impersonate="chrome120",
|
||||||
timeout=_HTTP_TIMEOUT_SEC,
|
timeout=_HTTP_TIMEOUT_SEC,
|
||||||
|
proxies=_proxies,
|
||||||
headers=_DOC_HEADERS,
|
headers=_DOC_HEADERS,
|
||||||
)
|
)
|
||||||
_own_session = True
|
_own_session = True
|
||||||
|
|
@ -496,9 +503,7 @@ async def _geocode(session: Any, address: str) -> IMVGeo:
|
||||||
normalized = data_a.get("normalizedAddress")
|
normalized = data_a.get("normalizedAddress")
|
||||||
|
|
||||||
if not lat or not lon or not normalized:
|
if not lat or not lon or not normalized:
|
||||||
raise IMVAddressNotFoundError(
|
raise IMVAddressNotFoundError(f"Avito /coords/by_address не вернул point для {address!r}")
|
||||||
f"Avito /coords/by_address не вернул point для {address!r}"
|
|
||||||
)
|
|
||||||
|
|
||||||
# Step B — rich JWT
|
# Step B — rich JWT
|
||||||
url_b = f"{AVITO_BASE}{POSITION_ENDPOINT}"
|
url_b = f"{AVITO_BASE}{POSITION_ENDPOINT}"
|
||||||
|
|
@ -767,9 +772,7 @@ def save_imv_placement_history(
|
||||||
"removed_date": item.removed_date,
|
"removed_date": item.removed_date,
|
||||||
"exposure_days": item.exposure_days,
|
"exposure_days": item.exposure_days,
|
||||||
"raw_payload": (
|
"raw_payload": (
|
||||||
json.dumps(item.raw_payload, ensure_ascii=False)
|
json.dumps(item.raw_payload, ensure_ascii=False) if item.raw_payload else None
|
||||||
if item.raw_payload
|
|
||||||
else None
|
|
||||||
),
|
),
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -13,6 +13,7 @@ Sister state containers extracted from same MFE initialState top-level keys:
|
||||||
|
|
||||||
Stage 6 of CianScraper v1.
|
Stage 6 of CianScraper v1.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import json
|
import json
|
||||||
|
|
@ -22,6 +23,7 @@ from typing import Any
|
||||||
|
|
||||||
from curl_cffi.requests import AsyncSession # type: ignore[import-untyped]
|
from curl_cffi.requests import AsyncSession # type: ignore[import-untyped]
|
||||||
|
|
||||||
|
from app.core.config import settings
|
||||||
from app.services.scrapers.cian_state_parser import extract_all_states, extract_state
|
from app.services.scrapers.cian_state_parser import extract_all_states, extract_state
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
@ -92,9 +94,14 @@ async def fetch_newbuilding(
|
||||||
"""
|
"""
|
||||||
close_session = False
|
close_session = False
|
||||||
if session is None:
|
if session is None:
|
||||||
|
# Mobile proxy wiring (#806 follow-up): Cian ЖК страница — datacenter-IP бан.
|
||||||
|
# proxy=None → прямое подключение (dev без прокси).
|
||||||
|
_proxy_url = settings.scraper_proxy_url
|
||||||
|
_proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None
|
||||||
session = AsyncSession(
|
session = AsyncSession(
|
||||||
impersonate="chrome120",
|
impersonate="chrome120",
|
||||||
timeout=30.0,
|
timeout=30.0,
|
||||||
|
proxies=_proxies,
|
||||||
headers={
|
headers={
|
||||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||||
"Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8",
|
"Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8",
|
||||||
|
|
@ -255,19 +262,21 @@ def _extract_chart(rv_state: dict[str, Any]) -> list[dict[str, Any]]:
|
||||||
for label, value in zip(labels, values, strict=False):
|
for label, value in zip(labels, values, strict=False):
|
||||||
if label is None or value is None:
|
if label is None or value is None:
|
||||||
continue
|
continue
|
||||||
points.append({
|
points.append(
|
||||||
"month_date": str(label)[:10], # YYYY-MM-DD
|
{
|
||||||
"room_count": "all", # aggregate (initial state)
|
"month_date": str(label)[:10], # YYYY-MM-DD
|
||||||
"prices_type": "price", # total price (not priceSqm)
|
"room_count": "all", # aggregate (initial state)
|
||||||
"period": "halfYear", # default period shown
|
"prices_type": "price", # total price (not priceSqm)
|
||||||
"price_per_sqm": None, # chart shows total price, not sqm
|
"period": "halfYear", # default period shown
|
||||||
})
|
"price_per_sqm": None, # chart shows total price, not sqm
|
||||||
|
}
|
||||||
|
)
|
||||||
# Store raw value as price_per_sqm placeholder for DB
|
# Store raw value as price_per_sqm placeholder for DB
|
||||||
# (actual sqm price would require area context)
|
# (actual sqm price would require area context)
|
||||||
# Use the value as-is; caller / downstream can enrich
|
# Use the value as-is; caller / downstream can enrich
|
||||||
points[-1]["price_per_sqm"] = float(value) if isinstance(
|
points[-1]["price_per_sqm"] = (
|
||||||
value, (int, float)
|
float(value) if isinstance(value, int | float) else None
|
||||||
) else None
|
)
|
||||||
if points:
|
if points:
|
||||||
return points
|
return points
|
||||||
|
|
||||||
|
|
@ -286,13 +295,17 @@ def _extract_chart(rv_state: dict[str, Any]) -> list[dict[str, Any]]:
|
||||||
price_val = float(price)
|
price_val = float(price)
|
||||||
except (TypeError, ValueError):
|
except (TypeError, ValueError):
|
||||||
pass
|
pass
|
||||||
points.append({
|
points.append(
|
||||||
"month_date": str(month)[:10],
|
{
|
||||||
"room_count": str(entry.get("roomType") or entry.get("roomCount") or "all"),
|
"month_date": str(month)[:10],
|
||||||
"prices_type": str(entry.get("pricesType") or entry.get("priceType") or "price"),
|
"room_count": str(entry.get("roomType") or entry.get("roomCount") or "all"),
|
||||||
"period": str(entry.get("period") or "halfYear"),
|
"prices_type": str(
|
||||||
"price_per_sqm": price_val,
|
entry.get("pricesType") or entry.get("priceType") or "price"
|
||||||
})
|
),
|
||||||
|
"period": str(entry.get("period") or "halfYear"),
|
||||||
|
"price_per_sqm": price_val,
|
||||||
|
}
|
||||||
|
)
|
||||||
return points
|
return points
|
||||||
|
|
||||||
return []
|
return []
|
||||||
|
|
@ -380,10 +393,12 @@ def _extract_nested_offers(offers_state: dict[str, Any]) -> list[dict[str, Any]]
|
||||||
continue
|
continue
|
||||||
for layout in layouts:
|
for layout in layouts:
|
||||||
if isinstance(layout, dict):
|
if isinstance(layout, dict):
|
||||||
result.append({
|
result.append(
|
||||||
"room_count": room_type,
|
{
|
||||||
**layout,
|
"room_count": room_type,
|
||||||
})
|
**layout,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
@ -537,9 +552,7 @@ def save_newbuilding_enrichment(
|
||||||
"hid": house_id,
|
"hid": house_id,
|
||||||
"cs": check.get("check_status"),
|
"cs": check.get("check_status"),
|
||||||
"cn": check.get("check_name"),
|
"cn": check.get("check_name"),
|
||||||
"det": json.dumps(
|
"det": json.dumps(check.get("details") or [], ensure_ascii=False),
|
||||||
check.get("details") or [], ensure_ascii=False
|
|
||||||
),
|
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -580,9 +593,13 @@ async def resolve_cian_zhk_url(
|
||||||
"""
|
"""
|
||||||
close_session = False
|
close_session = False
|
||||||
if session is None:
|
if session is None:
|
||||||
|
# Mobile proxy wiring (#806 follow-up): resolve ЖК URL через мобильный прокси.
|
||||||
|
_proxy_url = settings.scraper_proxy_url
|
||||||
|
_proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None
|
||||||
session = AsyncSession(
|
session = AsyncSession(
|
||||||
impersonate="chrome120",
|
impersonate="chrome120",
|
||||||
timeout=15.0,
|
timeout=15.0,
|
||||||
|
proxies=_proxies,
|
||||||
headers={
|
headers={
|
||||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||||
"Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8",
|
"Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8",
|
||||||
|
|
@ -595,9 +612,7 @@ async def resolve_cian_zhk_url(
|
||||||
resp = await session.get(fallback_url, allow_redirects=True)
|
resp = await session.get(fallback_url, allow_redirects=True)
|
||||||
final_url = str(resp.url)
|
final_url = str(resp.url)
|
||||||
if final_url and final_url != fallback_url:
|
if final_url and final_url != fallback_url:
|
||||||
logger.debug(
|
logger.debug("resolve_cian_zhk_url id=%s → %s", cian_internal_house_id, final_url)
|
||||||
"resolve_cian_zhk_url id=%s → %s", cian_internal_house_id, final_url
|
|
||||||
)
|
|
||||||
return final_url
|
return final_url
|
||||||
# Redirect not followed or same URL — return fallback as canonical
|
# Redirect not followed or same URL — return fallback as canonical
|
||||||
if resp.status_code == 200:
|
if resp.status_code == 200:
|
||||||
|
|
@ -609,9 +624,7 @@ async def resolve_cian_zhk_url(
|
||||||
)
|
)
|
||||||
return None
|
return None
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
logger.warning(
|
logger.warning("resolve_cian_zhk_url id=%s failed: %s", cian_internal_house_id, exc)
|
||||||
"resolve_cian_zhk_url id=%s failed: %s", cian_internal_house_id, exc
|
|
||||||
)
|
|
||||||
return None
|
return None
|
||||||
finally:
|
finally:
|
||||||
if close_session:
|
if close_session:
|
||||||
|
|
|
||||||
|
|
@ -17,6 +17,7 @@ State shape (sec 24.3–24.6 Schema_Cian_SERP_Inventory):
|
||||||
|
|
||||||
Used as 6th evaluation source в estimator.py (Stage 9).
|
Used as 6th evaluation source в estimator.py (Stage 9).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import hashlib
|
import hashlib
|
||||||
|
|
@ -30,6 +31,7 @@ from curl_cffi.requests import AsyncSession
|
||||||
from sqlalchemy import text
|
from sqlalchemy import text
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app.core.config import settings
|
||||||
from app.services.cian_session import load_session, mark_session_invalid
|
from app.services.cian_session import load_session, mark_session_invalid
|
||||||
from app.services.scrapers.cian_state_parser import extract_state
|
from app.services.scrapers.cian_state_parser import extract_state
|
||||||
|
|
||||||
|
|
@ -173,11 +175,16 @@ async def estimate_via_cian_valuation(
|
||||||
url = f"{VALUATION_BASE_URL}?{urlencode(params, safe='[]')}"
|
url = f"{VALUATION_BASE_URL}?{urlencode(params, safe='[]')}"
|
||||||
|
|
||||||
# 4. Fetch с TLS fingerprint (curl_cffi, impersonate chrome120)
|
# 4. Fetch с TLS fingerprint (curl_cffi, impersonate chrome120)
|
||||||
|
# Mobile proxy wiring (#806 follow-up): Cian валюация — такой же datacenter-бан риск
|
||||||
|
# как SERP. Используем scraper_proxy_url. proxy=None → прямое подключение (dev).
|
||||||
|
_proxy_url = settings.scraper_proxy_url
|
||||||
|
_proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None
|
||||||
try:
|
try:
|
||||||
async with AsyncSession(
|
async with AsyncSession(
|
||||||
impersonate="chrome120",
|
impersonate="chrome120",
|
||||||
cookies=cookies,
|
cookies=cookies,
|
||||||
timeout=25.0,
|
timeout=25.0,
|
||||||
|
proxies=_proxies,
|
||||||
headers={"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"},
|
headers={"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"},
|
||||||
) as session:
|
) as session:
|
||||||
resp = await session.get(url, allow_redirects=True)
|
resp = await session.get(url, allow_redirects=True)
|
||||||
|
|
@ -294,16 +301,19 @@ def _parse_valuation_state(state: dict[str, Any]) -> CianValuationResult:
|
||||||
# date: unix_ms → ISO string "YYYY-MM-DD"
|
# date: unix_ms → ISO string "YYYY-MM-DD"
|
||||||
date_ms = entry.get("date")
|
date_ms = entry.get("date")
|
||||||
date_str: str | None = None
|
date_str: str | None = None
|
||||||
if isinstance(date_ms, (int, float)) and date_ms > 0:
|
if isinstance(date_ms, int | float) and date_ms > 0:
|
||||||
import datetime
|
import datetime
|
||||||
|
|
||||||
date_str = datetime.datetime.fromtimestamp(
|
date_str = datetime.datetime.fromtimestamp(
|
||||||
date_ms / 1000, tz=datetime.UTC
|
date_ms / 1000, tz=datetime.UTC
|
||||||
).strftime("%Y-%m-%d")
|
).strftime("%Y-%m-%d")
|
||||||
result.chart.append({
|
result.chart.append(
|
||||||
"month_date": date_str or "",
|
{
|
||||||
"price": _parse_num(entry.get("price")),
|
"month_date": date_str or "",
|
||||||
"price_formatted": entry.get("priceFormatted"),
|
"price": _parse_num(entry.get("price")),
|
||||||
})
|
"price_formatted": entry.get("priceFormatted"),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
# --- houseInfo: 15 items о доме ---
|
# --- houseInfo: 15 items о доме ---
|
||||||
house_info_wrapper = state.get("houseInfo") or {}
|
house_info_wrapper = state.get("houseInfo") or {}
|
||||||
|
|
@ -347,8 +357,9 @@ def _parse_change_pct(value_str: str) -> float | None:
|
||||||
|
|
||||||
def _load_from_cache(db: Session, cache_key: str) -> CianValuationResult | None:
|
def _load_from_cache(db: Session, cache_key: str) -> CianValuationResult | None:
|
||||||
"""SELECT from external_valuations where cache_key + source='cian_valuation' + not expired."""
|
"""SELECT from external_valuations where cache_key + source='cian_valuation' + not expired."""
|
||||||
row = db.execute(
|
row = (
|
||||||
text("""
|
db.execute(
|
||||||
|
text("""
|
||||||
SELECT
|
SELECT
|
||||||
raw_payload,
|
raw_payload,
|
||||||
sale_price_rub,
|
sale_price_rub,
|
||||||
|
|
@ -369,8 +380,11 @@ def _load_from_cache(db: Session, cache_key: str) -> CianValuationResult | None:
|
||||||
ORDER BY fetched_at DESC
|
ORDER BY fetched_at DESC
|
||||||
LIMIT 1
|
LIMIT 1
|
||||||
"""),
|
"""),
|
||||||
{"ck": cache_key},
|
{"ck": cache_key},
|
||||||
).mappings().first()
|
)
|
||||||
|
.mappings()
|
||||||
|
.first()
|
||||||
|
)
|
||||||
|
|
||||||
if row is None:
|
if row is None:
|
||||||
return None
|
return None
|
||||||
|
|
|
||||||
|
|
@ -242,9 +242,15 @@ class YandexRealtyScraper(BaseScraper):
|
||||||
from app.core.config import settings
|
from app.core.config import settings
|
||||||
|
|
||||||
self._cookies = _load_cookies_from_file(settings.yandex_cookies_file)
|
self._cookies = _load_cookies_from_file(settings.yandex_cookies_file)
|
||||||
|
# Mobile proxy wiring (#806 follow-up): Yandex blocks datacenter IPs with
|
||||||
|
# captcha/shell HTML even with Chrome TLS fingerprint. Route via mobile proxy.
|
||||||
|
# proxy=None → curl_cffi ходит напрямую (dev без прокси — не падаем).
|
||||||
|
_proxy_url = settings.scraper_proxy_url
|
||||||
|
_proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None
|
||||||
self._cffi_session = _CurlCffiSession(
|
self._cffi_session = _CurlCffiSession(
|
||||||
impersonate="chrome120",
|
impersonate="chrome120",
|
||||||
cookies=self._cookies or {},
|
cookies=self._cookies or {},
|
||||||
|
proxies=_proxies,
|
||||||
headers={
|
headers={
|
||||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||||
"Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8",
|
"Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8",
|
||||||
|
|
|
||||||
|
|
@ -165,7 +165,13 @@ class YandexValuationScraper(BaseScraper):
|
||||||
# Override: Yandex valuation endpoint gates SSR data on Chrome TLS
|
# Override: Yandex valuation endpoint gates SSR data on Chrome TLS
|
||||||
# fingerprint. Plain httpx returns shell HTML (CSR-only).
|
# fingerprint. Plain httpx returns shell HTML (CSR-only).
|
||||||
# Sibling: yandex_realty.py / scripts/local-sweep-ekb-yandex.py
|
# Sibling: yandex_realty.py / scripts/local-sweep-ekb-yandex.py
|
||||||
self._cffi_session = _CurlCffiSession(impersonate="chrome120")
|
# Mobile proxy wiring (#806 follow-up): route via mobile proxy to avoid
|
||||||
|
# datacenter-IP blocks. proxy=None → прямое подключение (dev).
|
||||||
|
from app.core.config import settings
|
||||||
|
|
||||||
|
_proxy_url = settings.scraper_proxy_url
|
||||||
|
_proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None
|
||||||
|
self._cffi_session = _CurlCffiSession(impersonate="chrome120", proxies=_proxies)
|
||||||
return self
|
return self
|
||||||
|
|
||||||
async def __aexit__(self, *args: object) -> None: # type: ignore[override]
|
async def __aexit__(self, *args: object) -> None: # type: ignore[override]
|
||||||
|
|
|
||||||
|
|
@ -1,10 +1,12 @@
|
||||||
"""Tests for scraper proxy wiring (#806).
|
"""Tests for scraper proxy wiring (#806 + follow-up).
|
||||||
|
|
||||||
Covers:
|
Covers:
|
||||||
- scraper_proxy_url property: SCRAPER_PROXY_URL takes precedence over AVITO_PROXY_URL
|
- scraper_proxy_url property: SCRAPER_PROXY_URL takes precedence over AVITO_PROXY_URL
|
||||||
- AVITO_PROXY_URL fallback works when SCRAPER_PROXY_URL is absent
|
- AVITO_PROXY_URL fallback works when SCRAPER_PROXY_URL is absent
|
||||||
- None/empty → direct connection (no proxies dict)
|
- None/empty → direct connection (no proxies dict)
|
||||||
- _avito_proxies() helper returns correct dict shape
|
- _avito_proxies() helper returns correct dict shape
|
||||||
|
- per-scraper proxy wiring: YandexRealty, YandexValuation, CianValuation,
|
||||||
|
CianNewbuilding (fetch_newbuilding + resolve_cian_zhk_url), AvitoIMV, AvitoDetail
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
@ -14,12 +16,20 @@ import os
|
||||||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
|
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
|
||||||
|
|
||||||
from types import SimpleNamespace
|
from types import SimpleNamespace
|
||||||
from unittest.mock import patch
|
from unittest.mock import AsyncMock, MagicMock, patch
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
|
||||||
def _mock_settings(scraper_proxy_url: str | None) -> SimpleNamespace:
|
def _mock_settings(
|
||||||
"""Minimal settings stand-in with only the fields the proxy helpers read."""
|
scraper_proxy_url: str | None,
|
||||||
return SimpleNamespace(scraper_proxy_url=scraper_proxy_url)
|
yandex_cookies_file: str | None = None,
|
||||||
|
) -> SimpleNamespace:
|
||||||
|
"""Minimal settings stand-in with fields read by proxy helpers and scraper __aenter__."""
|
||||||
|
return SimpleNamespace(
|
||||||
|
scraper_proxy_url=scraper_proxy_url,
|
||||||
|
yandex_cookies_file=yandex_cookies_file,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
# ── Settings.scraper_proxy_url property ──────────────────────────────────────
|
# ── Settings.scraper_proxy_url property ──────────────────────────────────────
|
||||||
|
|
@ -140,3 +150,280 @@ def test_settings_ignores_misnamed_scraper_proxy_url_env(monkeypatch):
|
||||||
monkeypatch.delenv("AVITO_PROXY_URL", raising=False)
|
monkeypatch.delenv("AVITO_PROXY_URL", raising=False)
|
||||||
monkeypatch.setenv("SCRAPER_PROXY_URL_ENV", "http://wrong-name:1111")
|
monkeypatch.setenv("SCRAPER_PROXY_URL_ENV", "http://wrong-name:1111")
|
||||||
assert _fresh_settings().scraper_proxy_url is None
|
assert _fresh_settings().scraper_proxy_url is None
|
||||||
|
|
||||||
|
|
||||||
|
# ── Per-scraper proxy wiring (follow-up: yandex/valuation/imv/newbuilding) ────
|
||||||
|
# Each test: patch settings.scraper_proxy_url, mock AsyncSession constructor,
|
||||||
|
# invoke __aenter__ / create-session path, assert proxies= kwarg is forwarded.
|
||||||
|
# proxy=None path: assert proxies kwarg is None / not passed as non-None.
|
||||||
|
|
||||||
|
|
||||||
|
_PROXY_URL = "http://mobile.proxy:8080"
|
||||||
|
_EXPECTED_PROXIES = {"http": _PROXY_URL, "https": _PROXY_URL}
|
||||||
|
|
||||||
|
|
||||||
|
# ── YandexRealtyScraper ──────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_yandex_realty_session_receives_proxies(monkeypatch):
|
||||||
|
"""YandexRealtyScraper.__aenter__ forwards scraper_proxy_url as proxies= kwarg.
|
||||||
|
|
||||||
|
settings is imported deferred inside __aenter__ via `from app.core.config import settings`,
|
||||||
|
so we patch app.core.config.settings (module attribute) which the deferred import reads.
|
||||||
|
"""
|
||||||
|
from app.services.scrapers.yandex_realty import YandexRealtyScraper
|
||||||
|
|
||||||
|
mock_session = MagicMock()
|
||||||
|
mock_session.__aenter__ = AsyncMock(return_value=mock_session)
|
||||||
|
mock_session.__aexit__ = AsyncMock(return_value=None)
|
||||||
|
mock_session.close = AsyncMock()
|
||||||
|
|
||||||
|
captured_kwargs: dict = {}
|
||||||
|
|
||||||
|
def _fake_session(*args, **kwargs):
|
||||||
|
captured_kwargs.update(kwargs)
|
||||||
|
return mock_session
|
||||||
|
|
||||||
|
monkeypatch.setattr("app.services.scrapers.yandex_realty._CurlCffiSession", _fake_session)
|
||||||
|
with patch("app.core.config.settings", _mock_settings(_PROXY_URL)):
|
||||||
|
scraper = YandexRealtyScraper()
|
||||||
|
await scraper.__aenter__()
|
||||||
|
|
||||||
|
assert captured_kwargs.get("proxies") == _EXPECTED_PROXIES
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_yandex_realty_session_no_proxies_when_none(monkeypatch):
|
||||||
|
"""YandexRealtyScraper.__aenter__: proxies=None when scraper_proxy_url is None."""
|
||||||
|
from app.services.scrapers.yandex_realty import YandexRealtyScraper
|
||||||
|
|
||||||
|
mock_session = MagicMock()
|
||||||
|
mock_session.close = AsyncMock()
|
||||||
|
captured_kwargs: dict = {}
|
||||||
|
|
||||||
|
def _fake_session(*args, **kwargs):
|
||||||
|
captured_kwargs.update(kwargs)
|
||||||
|
return mock_session
|
||||||
|
|
||||||
|
monkeypatch.setattr("app.services.scrapers.yandex_realty._CurlCffiSession", _fake_session)
|
||||||
|
with patch("app.core.config.settings", _mock_settings(None)):
|
||||||
|
scraper = YandexRealtyScraper()
|
||||||
|
await scraper.__aenter__()
|
||||||
|
|
||||||
|
assert captured_kwargs.get("proxies") is None
|
||||||
|
|
||||||
|
|
||||||
|
# ── YandexValuationScraper ───────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_yandex_valuation_session_receives_proxies():
|
||||||
|
"""YandexValuationScraper.__aenter__ forwards proxies= kwarg."""
|
||||||
|
from app.services.scrapers.yandex_valuation import YandexValuationScraper
|
||||||
|
|
||||||
|
mock_session = MagicMock()
|
||||||
|
mock_session.close = AsyncMock()
|
||||||
|
captured_kwargs: dict = {}
|
||||||
|
|
||||||
|
def _fake_session(*args, **kwargs):
|
||||||
|
captured_kwargs.update(kwargs)
|
||||||
|
return mock_session
|
||||||
|
|
||||||
|
with patch("app.services.scrapers.yandex_valuation._CurlCffiSession", _fake_session):
|
||||||
|
with patch("app.core.config.settings", _mock_settings(_PROXY_URL)):
|
||||||
|
scraper = YandexValuationScraper()
|
||||||
|
await scraper.__aenter__()
|
||||||
|
|
||||||
|
assert captured_kwargs.get("proxies") == _EXPECTED_PROXIES
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_yandex_valuation_session_no_proxies_when_none():
|
||||||
|
"""YandexValuationScraper.__aenter__: proxies=None when no proxy configured."""
|
||||||
|
from app.services.scrapers.yandex_valuation import YandexValuationScraper
|
||||||
|
|
||||||
|
mock_session = MagicMock()
|
||||||
|
mock_session.close = AsyncMock()
|
||||||
|
captured_kwargs: dict = {}
|
||||||
|
|
||||||
|
def _fake_session(*args, **kwargs):
|
||||||
|
captured_kwargs.update(kwargs)
|
||||||
|
return mock_session
|
||||||
|
|
||||||
|
with patch("app.services.scrapers.yandex_valuation._CurlCffiSession", _fake_session):
|
||||||
|
with patch("app.core.config.settings", _mock_settings(None)):
|
||||||
|
scraper = YandexValuationScraper()
|
||||||
|
await scraper.__aenter__()
|
||||||
|
|
||||||
|
assert captured_kwargs.get("proxies") is None
|
||||||
|
|
||||||
|
|
||||||
|
# ── CianNewbuilding — fetch_newbuilding ───────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_cian_newbuilding_own_session_receives_proxies():
|
||||||
|
"""fetch_newbuilding creates own session with proxies= when no session passed."""
|
||||||
|
from app.services.scrapers import cian_newbuilding
|
||||||
|
|
||||||
|
mock_session = MagicMock()
|
||||||
|
mock_resp = MagicMock()
|
||||||
|
mock_resp.status_code = 200
|
||||||
|
mock_resp.text = "" # empty HTML → extract_state returns None → result None (ok)
|
||||||
|
mock_session.get = AsyncMock(return_value=mock_resp)
|
||||||
|
mock_session.close = AsyncMock()
|
||||||
|
|
||||||
|
captured_kwargs: dict = {}
|
||||||
|
|
||||||
|
def _fake_session(*args, **kwargs):
|
||||||
|
captured_kwargs.update(kwargs)
|
||||||
|
return mock_session
|
||||||
|
|
||||||
|
with patch.object(cian_newbuilding, "AsyncSession", _fake_session):
|
||||||
|
with patch.object(cian_newbuilding, "settings", _mock_settings(_PROXY_URL)):
|
||||||
|
await cian_newbuilding.fetch_newbuilding("https://zhk-test.cian.ru/")
|
||||||
|
|
||||||
|
assert captured_kwargs.get("proxies") == _EXPECTED_PROXIES
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_cian_newbuilding_shared_session_not_recreated():
|
||||||
|
"""fetch_newbuilding does NOT create own session when one is passed."""
|
||||||
|
from app.services.scrapers import cian_newbuilding
|
||||||
|
|
||||||
|
mock_session = MagicMock()
|
||||||
|
mock_resp = MagicMock()
|
||||||
|
mock_resp.status_code = 200
|
||||||
|
mock_resp.text = ""
|
||||||
|
mock_session.get = AsyncMock(return_value=mock_resp)
|
||||||
|
mock_session.close = AsyncMock()
|
||||||
|
|
||||||
|
session_ctor_calls = []
|
||||||
|
|
||||||
|
def _fake_session(*args, **kwargs):
|
||||||
|
session_ctor_calls.append(kwargs)
|
||||||
|
return MagicMock()
|
||||||
|
|
||||||
|
with patch.object(cian_newbuilding, "AsyncSession", _fake_session):
|
||||||
|
with patch.object(cian_newbuilding, "settings", _mock_settings(_PROXY_URL)):
|
||||||
|
await cian_newbuilding.fetch_newbuilding(
|
||||||
|
"https://zhk-test.cian.ru/", session=mock_session
|
||||||
|
)
|
||||||
|
|
||||||
|
# No own session should have been created (shared session was passed)
|
||||||
|
assert session_ctor_calls == []
|
||||||
|
|
||||||
|
|
||||||
|
# ── AvitoDetail — fetch_detail own-session ───────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_avito_detail_own_session_receives_proxies():
|
||||||
|
"""fetch_detail creates own session with proxies= when no cffi_session passed."""
|
||||||
|
from app.services.scrapers import avito_detail
|
||||||
|
|
||||||
|
mock_session = MagicMock()
|
||||||
|
mock_resp = MagicMock()
|
||||||
|
mock_resp.status_code = 200
|
||||||
|
# Minimal HTML that parse_detail_html won't raise ValueError (item_id required)
|
||||||
|
mock_resp.text = "<html><body></body></html>"
|
||||||
|
mock_session.get = AsyncMock(return_value=mock_resp)
|
||||||
|
mock_session.close = AsyncMock()
|
||||||
|
|
||||||
|
captured_kwargs: dict = {}
|
||||||
|
|
||||||
|
def _fake_session(*args, **kwargs):
|
||||||
|
captured_kwargs.update(kwargs)
|
||||||
|
return mock_session
|
||||||
|
|
||||||
|
with patch.object(avito_detail, "AsyncSession", _fake_session):
|
||||||
|
with patch("app.core.config.settings", _mock_settings(_PROXY_URL)):
|
||||||
|
try:
|
||||||
|
await avito_detail.fetch_detail("/ekaterinburg/kvartiry/test-1234")
|
||||||
|
except ValueError:
|
||||||
|
pass # parse_detail_html raises ValueError (no item_id) — that's ok
|
||||||
|
|
||||||
|
assert captured_kwargs.get("proxies") == _EXPECTED_PROXIES
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_avito_detail_shared_session_not_patched():
|
||||||
|
"""fetch_detail does NOT build own session when cffi_session is provided."""
|
||||||
|
from app.services.scrapers import avito_detail
|
||||||
|
|
||||||
|
mock_session = MagicMock()
|
||||||
|
mock_resp = MagicMock()
|
||||||
|
mock_resp.status_code = 200
|
||||||
|
mock_resp.text = "<html><body></body></html>"
|
||||||
|
mock_session.get = AsyncMock(return_value=mock_resp)
|
||||||
|
mock_session.close = AsyncMock()
|
||||||
|
|
||||||
|
ctor_calls = []
|
||||||
|
|
||||||
|
def _fake_session(*args, **kwargs):
|
||||||
|
ctor_calls.append(kwargs)
|
||||||
|
return MagicMock()
|
||||||
|
|
||||||
|
with patch.object(avito_detail, "AsyncSession", _fake_session):
|
||||||
|
with patch("app.core.config.settings", _mock_settings(_PROXY_URL)):
|
||||||
|
try:
|
||||||
|
await avito_detail.fetch_detail(
|
||||||
|
"/ekaterinburg/kvartiry/test-1234",
|
||||||
|
cffi_session=mock_session,
|
||||||
|
)
|
||||||
|
except ValueError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
# No own session created — shared session was used as-is
|
||||||
|
assert ctor_calls == []
|
||||||
|
|
||||||
|
|
||||||
|
# ── AvitoIMV — evaluate_via_imv own-session ──────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_avito_imv_own_session_receives_proxies():
|
||||||
|
"""evaluate_via_imv creates own session with proxies= when none passed.
|
||||||
|
|
||||||
|
`CffiAsyncSession` is imported locally inside evaluate_via_imv as
|
||||||
|
`from curl_cffi.requests import AsyncSession as CffiAsyncSession`, so we
|
||||||
|
patch `curl_cffi.requests.AsyncSession` (the source) to intercept it.
|
||||||
|
"""
|
||||||
|
from app.services.scrapers import avito_imv
|
||||||
|
|
||||||
|
mock_session = MagicMock()
|
||||||
|
mock_warmup_resp = MagicMock()
|
||||||
|
mock_warmup_resp.status_code = 200
|
||||||
|
mock_geo_resp = MagicMock()
|
||||||
|
mock_geo_resp.status_code = 200
|
||||||
|
mock_geo_resp.json = MagicMock(return_value={"result": {"point": None}})
|
||||||
|
mock_session.get = AsyncMock(return_value=mock_warmup_resp)
|
||||||
|
mock_session.post = AsyncMock(return_value=mock_geo_resp)
|
||||||
|
mock_session.close = AsyncMock()
|
||||||
|
|
||||||
|
captured_kwargs: dict = {}
|
||||||
|
|
||||||
|
def _fake_session(*args, **kwargs):
|
||||||
|
captured_kwargs.update(kwargs)
|
||||||
|
return mock_session
|
||||||
|
|
||||||
|
# Patch the source so the local `from curl_cffi.requests import AsyncSession` picks it up
|
||||||
|
with patch("curl_cffi.requests.AsyncSession", _fake_session):
|
||||||
|
with patch("app.core.config.settings", _mock_settings(_PROXY_URL)):
|
||||||
|
try:
|
||||||
|
await avito_imv.evaluate_via_imv(
|
||||||
|
address="ул. Тургенева, 4",
|
||||||
|
rooms=2,
|
||||||
|
area_m2=50.0,
|
||||||
|
floor=3,
|
||||||
|
floor_at_home=9,
|
||||||
|
house_type="panel",
|
||||||
|
renovation_type="cosmetic",
|
||||||
|
has_balcony=False,
|
||||||
|
has_loggia=False,
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
pass # network/parse failures expected in unit test
|
||||||
|
|
||||||
|
assert captured_kwargs.get("proxies") == _EXPECTED_PROXIES
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue