lekss361/gendesign
1
0
Fork 0
Code Issues 57 Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request 'refactor(tradein/scrapers): удалить 6 осиротевших legacy scraper-модулей + тест-хирургия (#2397 Part E2)' (#2407) from chore/tradein-partE2-delete-orphaned-scrapers into main
All checks were successful
Deploy Trade-In / build-backend (push) Successful in 54s
Details
Deploy Trade-In / changes (push) Successful in 11s
Details
Deploy Trade-In / build-frontend (push) Has been skipped
Details
Deploy Trade-In / build-browser (push) Has been skipped
Details
Deploy Trade-In / test (push) Successful in 1m1s
Details
Deploy Trade-In / deploy (push) Successful in 1m46s
Details

Browse source
This commit is contained in:
lekss361 2026-07-04 11:49:44 +00:00
commit eafa305ac8
36 changed files with 161 additions and 13256 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
tradein-mvp/backend/app/services/cian_price_history.py
View file

@ -15,9 +15,10 @@ import logging
import time
from dataclasses import dataclass, field
# #2306: fetch_detail/save_detail_enrichment migrated to scraper_kit (golden-parity
# with legacy app.services.scrapers.cian_detail, см. tests/test_scraper_kit_cian_golden_parity.py
# + tests/test_scraper_kit_pricehistory_session_parity.py). RealScraperConfig — тот же
# #2306: fetch_detail/save_detail_enrichment migrated to scraper_kit (byte-identical
# golden-parity была доказана против legacy app.services.scrapers.cian_detail до его
# удаления, #2397 Part E2 — см. tests/test_scraper_kit_pricehistory_session_parity.py
# для оставшейся extract_state/ScrapedLot parity). RealScraperConfig — тот же
# read-only адаптер над settings, что и остальные kit-инжекции (#2131) — сохраняет
# proxy-поведение (config.cian_proxy_url) идентичным прежнему прямому импорту settings.
from scraper_kit.providers.cian.detail import fetch_detail, save_detail_enrichment

1392
tradein-mvp/backend/app/services/scrapers/avito_houses.py
View file

File diff suppressed because it is too large Load diff

1185
tradein-mvp/backend/app/services/scrapers/cian.py
View file

File diff suppressed because it is too large Load diff

464
tradein-mvp/backend/app/services/scrapers/cian_detail.py
View file

@ -1,464 +0,0 @@
"""Cian.ru detail page scraper — single offer enrichment.
Different from SERP (cian.py):
- URL: https://ekb.cian.ru/sale/flat/<offer_id>/ or https://www.cian.ru/sale/flat/<offer_id>/
- MFE: 'frontend-offer-card'
- State KEY: 'defaultState' (NOT 'initialState' — SERP uses initialState)
- Sister containers in same _cianConfig: bti, priceChanges, stats, agent, newObject
Parses ~88 offer fields + sister data → DetailEnrichment dataclass.
Stage 5 of CianScraper v1.
"""
from __future__ import annotations
import json
import logging
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Any
from curl_cffi.requests import AsyncSession
from sqlalchemy import text
from sqlalchemy.orm import Session
from app.core.config import settings
from app.services.scrapers.cian_state_parser import extract_all_states, extract_state
from app.services.scrapers.repair_state_normalizer import (
infer_repair_state_from_text,
normalize_repair_state,
)
from app.services.scrapers.snapshot_writer import upsert_listing_snapshot
if TYPE_CHECKING:
from app.services.scrapers.browser_fetcher import BrowserFetcher
logger = logging.getLogger(__name__)
# offer_price_history.diff_percent — NUMERIC(8,2) (миграция
# 131_fix_diff_percent_overflow.sql). Прямой INSERT ниже читает diff_percent из
# скрапленного Cian JSON и не проходит через БД-триггер record_listing_price_change()
# (тот срабатывает только на UPDATE listings) — гарбадж/экстремальное значение
# с редизайна source-страницы иначе переполняет колонку.
_DIFF_PERCENT_BOUND = 999999.99
def _clamp_diff_percent(value: float | int | None) -> float | None:
"""Клампит diff_percent в границы NUMERIC(8,2) (зеркало migration 131 trigger-clamp)."""
if value is None or isinstance(value, bool):
return None
try:
numeric = float(value)
except (TypeError, ValueError):
return None
return min(max(numeric, -_DIFF_PERCENT_BOUND), _DIFF_PERCENT_BOUND)
@dataclass
class DetailEnrichment:
"""Result of cian_detail.fetch_detail() — fields для UPDATE listings + sister rows."""
# Core listing enrichment (extend existing listings row)
cian_id: int | None = None
windows_view_type: str | None = None # 'yard' / 'street' / 'yardAndStreet'
separate_wcs_count: int | None = None
combined_wcs_count: int | None = None
ceiling_height: float | None = None # meters — offer.building.ceilingHeight (#1791)
repair_type: str | None = None # raw Cian: offer.repairType OR offer.decoration (#1791)
repair_state: str | None = None # enum: needs_repair/standard/good/excellent
kitchen_area_m2: float | None = None # offer.kitchenArea (м²)
description: str | None = None # offer.description (текст объявления)
views_total: int | None = None # Cian stats.totalViewsFormattedString → int
views_today: int | None = None
# BTI (вторичка only — primary buildings don't have БТИ)
bti_data: dict[str, Any] | None = None # raw bti.houseData snapshot
# Price history (offer.priceChanges)
price_changes: list[dict[str, Any]] = field(default_factory=list)
# Agent profile (offer.agent.* expanded)
agent_profile: dict[str, Any] | None = None
# Newbuilding link (if offer is from a newbuilding)
newbuilding_data: dict[str, Any] | None = None
# Raw payload для debugging / future-extraction
raw_offer: dict[str, Any] | None = None
raw_sister_states: dict[str, Any] = field(default_factory=dict)
async def fetch_detail(
offer_url: str,
*,
session: AsyncSession | None = None,
browser_fetcher: BrowserFetcher | None = None,
) -> DetailEnrichment | None:
"""Fetch detail page and extract all containers.
Args:
offer_url: e.g. 'https://ekb.cian.ru/sale/flat/327830237/'
session: optional shared curl_cffi session (для batched scrapes).
Ignored when browser_fetcher is provided.
browser_fetcher: optional BrowserFetcher instance (camoufox HTTP service).
When provided, fetches JS-rendered HTML via the browser service instead of
curl_cffi — enables priceChanges extraction which requires JS rendering.
Caller is responsible for the context-manager lifecycle of the fetcher.
Returns: DetailEnrichment, or None если fetch / parse failed.
"""
if browser_fetcher is not None:
# Browser path: get fully JS-rendered HTML; same parse path follows.
try:
html = await browser_fetcher.fetch(offer_url)
except Exception as exc:
logger.warning("Cian detail browser fetch failed %s: %s", offer_url, exc)
return None
else:
# curl_cffi path (legacy, back-compat).
close_session = False
if session is None:
# proxies: mobile-proxy egress (#806) — без прокси datacenter-IP блокируется Cian.
# Пусто (env не задан) → прямое подключение (dev/no-op).
_proxy_url = settings.cian_proxy_url
_proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None
session = AsyncSession(
impersonate="chrome120",
timeout=25.0,
proxies=_proxies,
headers={
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8",
},
)
close_session = True
try:
resp = await session.get(offer_url, allow_redirects=True)
if resp.status_code != 200:
logger.warning("Cian detail fetch %s → HTTP %d", offer_url, resp.status_code)
return None
html = resp.text
finally:
if close_session:
await session.close()
# ── Shared parse path (identical regardless of HTML source) ──────────────
# Primary state container — defaultState in frontend-offer-card MFE
# NOTE: detail pages use 'defaultState', SERP uses 'initialState'
offer_state = extract_state(html, mfe="frontend-offer-card", key="defaultState")
if offer_state is None:
logger.warning("Cian detail %s: defaultState extraction failed", offer_url)
return None
offer_data = offer_state.get("offerData", {})
offer = offer_data.get("offer", {})
result = DetailEnrichment(
cian_id=offer.get("cianId") or offer.get("id"),
windows_view_type=_extract_windows_view(offer),
separate_wcs_count=offer.get("separateWcsCount"),
combined_wcs_count=offer.get("combinedWcsCount"),
ceiling_height=_extract_ceiling_height(offer),
repair_type=_extract_repair_type(offer),
repair_state=_extract_repair_state(offer),
kitchen_area_m2=_parse_float(offer.get("kitchenArea")),
description=offer.get("description") or None,
raw_offer=offer,
)
# Stats (views) — from stats key in offerData or top-level state
stats = offer_data.get("stats") or offer_state.get("stats") or {}
result.views_total = _parse_views(stats.get("totalViewsFormattedString"))
result.views_today = _parse_views(stats.get("todayViewsFormattedString"))
# Price changes — browser HTML: offerData.priceChanges (priceData.price format)
# curl_cffi HTML: offer.priceChanges or state.priceChanges (flat price field)
result.price_changes = _extract_price_changes(offer, offer_data, offer_state)
# Agent profile expansion
result.agent_profile = _extract_agent_profile(offer)
# Sister state containers: bti, newObject, plus raw stash of everything else
all_states = extract_all_states(html)
offer_card_states = all_states.get("frontend-offer-card", {})
bti_state = offer_card_states.get("bti")
if bti_state:
result.bti_data = bti_state.get("houseData")
result.raw_sister_states["bti"] = bti_state
# Newbuilding link (if applicable)
newbuilding = offer.get("newbuilding")
if newbuilding and newbuilding.get("id"):
result.newbuilding_data = newbuilding
# Stash all non-defaultState sister containers for debugging / future-extraction
for k, v in offer_card_states.items():
if k != "defaultState":
result.raw_sister_states[k] = v
return result
# ── Field extractors ──────────────────────────────────────────────────────────
def _extract_windows_view(offer: dict[str, Any]) -> str | None:
return offer.get("windowsViewType")
def _extract_ceiling_height(offer: dict[str, Any]) -> float | None:
"""Высота потолков (м).
На detail-странице (defaultState) значение лежит в ``offer.building.ceilingHeight``,
а НЕ в ``offer.ceilingHeight`` (последнее всегда null) — #1791. Top-level оставлен
как fallback на случай иной структуры карточки.
"""
building = offer.get("building") or {}
return _parse_float(building.get("ceilingHeight") or offer.get("ceilingHeight"))
def _extract_repair_type(offer: dict[str, Any]) -> str | None:
# Cian: 'cosmetic' / 'design' / 'no' / 'euro' / ...
# На detail-странице структурное поле чаще приходит как `decoration`
# ('preFine'/'fine'/...), а `repairType` пустое (#1791) — читаем оба.
return offer.get("repairType") or offer.get("decoration")
def _extract_repair_state(offer: dict[str, Any]) -> str | None:
"""Нормализованный enum из repairType/decoration → needs_repair/standard/good/excellent.
Если структурное поле пустое — fallback на инференс из описания (#622).
Структурное значение всегда в приоритете.
"""
state = normalize_repair_state(offer.get("repairType") or offer.get("decoration"))
if state is None:
state = infer_repair_state_from_text(offer.get("description"))
return state
def _parse_float(value: Any) -> float | None:
if value is None:
return None
try:
return float(value)
except (ValueError, TypeError):
return None
def _parse_views(formatted_str: str | None) -> int | None:
"""Cian's stats.totalViewsFormattedString — '1 234' → 1234."""
if not formatted_str:
return None
try:
return int(str(formatted_str).replace(" ", "").replace(",", ""))
except (ValueError, TypeError):
return None
def _extract_price_changes(
offer: dict[str, Any],
offer_data: dict[str, Any],
state: dict[str, Any],
) -> list[dict[str, Any]]:
"""Normalize priceChanges from any of the known Cian locations/formats.
Lookup priority:
1. offer.priceChanges — curl_cffi flat format: {changeTime, price, diffPercent}
2. offerData.priceChanges — browser-rendered format: {changeTime, priceData:{price}}
3. state.priceChanges — top-level fallback (rare)
Normalizes to: {change_time, price_rub, diff_percent}.
"""
raw = (
offer.get("priceChanges")
or offer_data.get("priceChanges")
or state.get("priceChanges")
or []
)
if not isinstance(raw, list):
return []
result = []
for entry in raw:
if not isinstance(entry, dict):
continue
# price_rub: flat field (curl_cffi) OR nested priceData.price (browser)
price_data = entry.get("priceData") or {}
price_rub = entry.get("price") or entry.get("priceRub") or price_data.get("price")
result.append(
{
"change_time": entry.get("changeTime") or entry.get("date"),
"price_rub": price_rub,
"diff_percent": entry.get("diffPercent") or entry.get("diff_percent"),
}
)
return result
def _extract_agent_profile(offer: dict[str, Any]) -> dict[str, Any] | None:
"""offer.agent or offer.agency or offer.user — extract canonical profile shape."""
agent = offer.get("agent") or {}
user = offer.get("user") or {}
if not agent and not user:
return None
return {
"ext_agent_id": agent.get("id") or user.get("cianUserId"),
"company_name": agent.get("companyName") or user.get("companyName"),
"is_pro": agent.get("isPro") or user.get("isPro"),
"skills": agent.get("skills") or [],
"account_type": agent.get("accountType") or user.get("accountType"),
}
# ── Save helpers ──────────────────────────────────────────────────────────────
def save_detail_enrichment(db: Session, listing_id: int, enrichment: DetailEnrichment) -> None:
"""Persist DetailEnrichment to DB:
- UPDATE listings SET windows_view_type, ceiling_height, ... WHERE id = listing_id
- INSERT INTO offer_price_history rows (if price_changes non-empty)
- INSERT INTO agents (if agent_profile) and link via agent_id_fk
- bti_data не сохраняется здесь — это задача Stage 6 (houses), где есть house_id
Idempotency: re-running безопасно (UPSERT / ON CONFLICT DO NOTHING semantics).
"""
# Extend listings row with detail-enriched fields
db.execute(
text("""
UPDATE listings SET
windows_view_type = COALESCE(:wvt, windows_view_type),
separate_wcs_count = COALESCE(:swc, separate_wcs_count),
combined_wcs_count = COALESCE(:cwc, combined_wcs_count),
ceiling_height = COALESCE(:ch, ceiling_height),
repair_type = COALESCE(:rt, repair_type),
repair_state = COALESCE(:rs, repair_state),
kitchen_area_m2 = COALESCE(CAST(:ka AS double precision), kitchen_area_m2),
description = COALESCE(:descr, description),
views_total = COALESCE(:vt, views_total),
views_today = COALESCE(:vd, views_today),
detail_enriched_at = NOW()
WHERE id = CAST(:lid AS bigint)
"""),
{
"lid": listing_id,
"wvt": enrichment.windows_view_type,
"swc": enrichment.separate_wcs_count,
"cwc": enrichment.combined_wcs_count,
"ch": enrichment.ceiling_height,
"rt": enrichment.repair_type,
"rs": enrichment.repair_state,
"ka": enrichment.kitchen_area_m2,
"descr": enrichment.description,
"vt": enrichment.views_total,
"vd": enrichment.views_today,
},
)
# Snapshot: point-in-time observation в listings_snapshots.
# Берём price_rub / price_per_m2 из listings строки (только что обновлённой).
# Используем COALESCE — на случай если listings.price_rub NULL (не должно быть, но safe).
_snap_row = db.execute(
text("""
SELECT price_rub, price_per_m2
FROM listings
WHERE id = CAST(:lid AS bigint)
"""),
{"lid": listing_id},
).fetchone()
_snap_written = False
if _snap_row is not None and _snap_row.price_rub is not None:
try:
with db.begin_nested():
upsert_listing_snapshot(
db,
listing_id=listing_id,
price_rub=int(_snap_row.price_rub),
price_per_m2=int(_snap_row.price_per_m2) if _snap_row.price_per_m2 else None,
run_id=None,
status="active",
)
_snap_written = True
except Exception as exc:
logger.warning(
"save_detail_enrichment: snapshot write failed listing_id=%s: %s",
listing_id,
exc,
)
# Price changes — INSERT new rows, de-dup by listing_id + change_time
for change in enrichment.price_changes:
if not change.get("change_time") or not change.get("price_rub"):
continue
db.execute(
text("""
INSERT INTO offer_price_history
(listing_id, change_time, price_rub, diff_percent, source)
VALUES (
CAST(:lid AS bigint),
CAST(:ct AS timestamptz),
CAST(:price AS numeric),
CAST(:diff AS numeric),
'cian'
)
ON CONFLICT ON CONSTRAINT offer_price_history_listing_change_uq DO NOTHING
"""),
{
"lid": listing_id,
"ct": change["change_time"],
"price": change["price_rub"],
"diff": _clamp_diff_percent(change.get("diff_percent")),
},
)
# Agent profile UPSERT — store agent row and link to listing
if enrichment.agent_profile and enrichment.agent_profile.get("ext_agent_id"):
ap = enrichment.agent_profile
row = db.execute(
text("""
INSERT INTO agents
(ext_source, ext_agent_id, company_name, is_pro, skills, account_type)
VALUES (
'cian',
:eai,
:cn,
:ip,
CAST(:sk AS jsonb),
:at
)
ON CONFLICT (ext_source, ext_agent_id) DO UPDATE SET
company_name = COALESCE(EXCLUDED.company_name, agents.company_name),
is_pro = COALESCE(EXCLUDED.is_pro, agents.is_pro),
skills = COALESCE(EXCLUDED.skills, agents.skills),
account_type = COALESCE(EXCLUDED.account_type, agents.account_type)
RETURNING id
"""),
{
"eai": str(ap["ext_agent_id"]),
"cn": ap.get("company_name"),
"ip": ap.get("is_pro"),
"sk": json.dumps(ap.get("skills") or []),
"at": ap.get("account_type"),
},
)
agent_db_id = row.scalar_one_or_none()
if agent_db_id is not None:
db.execute(
text("""
UPDATE listings
SET agent_id_fk = CAST(:aid AS bigint)
WHERE id = CAST(:lid AS bigint)
AND agent_id_fk IS NULL
"""),
{"aid": agent_db_id, "lid": listing_id},
)
db.commit()
logger.info(
"Cian detail enrichment saved for listing_id=%s (price_changes=%d, agent=%s, snapshot=%s)",
listing_id,
len(enrichment.price_changes),
bool(enrichment.agent_profile),
_snap_written,
)

667
tradein-mvp/backend/app/services/scrapers/domclick.py
View file

@ -1,667 +0,0 @@
"""DomClick.ru scraper — вторичка через JSON BFF API (#1968).
Стратегия: GET https://bff-search-web.domclick.ru/api/offers/v1?... через
BrowserFetcher(source="domclick") (generic provider → shared mobile proxy).
QRATOR банит прямые datacenter-запросы, но пропускает через mobile proxy.
Ответ BrowserFetcher содержит JSON, обёрнутый в HTML (<pre> или bare body).
Парсинг: _extract_json() вытаскивает первый {...} из ответа.
Стратегия нумерации комнат (ROOM_BUCKETS):
"st" — студии (force rooms=0 в ScrapedLot)
"1"–"4" — соответственно
"5+" — 5 и более комнат
Пагинация: offset 0, 20, 40, …, cap=2000. Если snippetsCount > 2000
для бакета, рекурсивно делим по цене (binary split) до тех пор пока
каждая ветка укладывается в cap.
# TODO Layer B (#1846 follow-up): backfill renovation/wallType/priceHistory
через detail-эндпоинт после первоначального сбора.
"""
from __future__ import annotations
import hashlib
import json
import logging
from datetime import date
from typing import Any
from urllib.parse import urlencode
from app.services.scraper_settings import get_scraper_delay
from app.services.scrapers.base import BaseScraper, ScrapedLot
from app.services.scrapers.domclick_exceptions import DomClickBlockedError
from app.services.scrapers.repair_state_normalizer import infer_repair_state_from_text
logger = logging.getLogger(__name__)
# ── API constants ─────────────────────────────────────────────────────────────
_BFF_BASE = "https://bff-search-web.domclick.ru"
_EKB_ADDRESS_GUID = "0d475b79-88de-4054-818c-37d8f9d0d440"
_EKB_AREA_ID = "20561"
# Buckets to sweep — порядок влияет на логи.
ROOM_BUCKETS: tuple[str, ...] = ("st", "1", "2", "3", "4", "5+")
OFFSET_CAP: int = 2000 # max offset принятый BFF API
PAGE_SIZE: int = 20 # items per page (жёстко задан API)
# Потолок цены для binary split. 1 млрд ₽ — безопасно выше любой ЕКБ-квартиры;
# выбран так чтобы НЕ отбрасывать листинги дороже потолка при сплите unbounded-бакета
# (родительский count не ограничен сверху → clamped _lte должен покрывать весь хвост).
LTE_MAX: int = 1_000_000_000
MIN_PRICE_SPAN: int = 100_000 # ниже этого span прекращаем делить
# ── QRATOR block detection ────────────────────────────────────────────────────
_QRATOR_MARKERS: tuple[str, ...] = (
"qrator",
"bot_mitigation",
"система защиты",
"403 | домклик",
"captcha",
"access denied",
)
# ── EKB geo guard ─────────────────────────────────────────────────────────────
_EKB_LAT_MIN: float = 56.6
_EKB_LAT_MAX: float = 57.0
_EKB_LON_MIN: float = 60.2
_EKB_LON_MAX: float = 60.9
_EKB_REGION_NAME: str = "Екатеринбург"
# ── JSON extraction ───────────────────────────────────────────────────────────
def _extract_json(html: str) -> dict[str, Any]:
"""Извлекает JSON-объект из ответа BrowserFetcher.
BrowserFetcher оборачивает JSON в HTML-страницу двумя способами:
1. Bare body: ``{"result": ...}`` напрямую в <body>.
2. <pre>-wrapped: ``<html><body><pre>{"result": ...}</pre></body></html>``.
Стратегия: найти первый '{' и последний '}' — работает для обоих вариантов.
Raises:
DomClickBlockedError: если ответ содержит QRATOR/captcha маркеры.
ValueError: если JSON не найден или не является dict.
"""
# Сканируем ВЕСЬ ответ (а не только первые 4096B): block-маркер может
# стоять за пределами head в крупных challenge-страницах.
html_lower = html.lower()
if any(m in html_lower for m in _QRATOR_MARKERS):
raise DomClickBlockedError(
f"DomClick BFF: QRATOR block page detected (markers checked: {_QRATOR_MARKERS[:2]})"
)
start = html.find("{")
end = html.rfind("}")
if start == -1 or end == -1 or end <= start:
raise ValueError(f"No JSON object found in BFF response (len={len(html)})")
data = json.loads(html[start : end + 1])
if not isinstance(data, dict):
raise ValueError(f"BFF response JSON is not a dict: {type(data)}")
return data # type: ignore[return-value]
# ── URL builders ──────────────────────────────────────────────────────────────
def _build_offers_url(
rooms: str,
price_gte: int | None,
price_lte: int | None,
offset: int,
) -> str:
"""Строит URL для GET /api/offers/v1 с пагинацией.
urlencode кодирует "5+" → "5%2B" (literal '+' в query string = space → reject).
"""
params: list[tuple[str, str]] = [
("address", _EKB_ADDRESS_GUID),
("aids", _EKB_AREA_ID),
("deal_type", "sale"),
("category", "living"),
("offer_type", "flat"),
("rooms", rooms),
("sort", "qi"),
("sort_dir", "desc"),
("offset", str(offset)),
("limit", str(PAGE_SIZE)),
]
if price_gte is not None:
params.append(("sale_price__gte", str(price_gte)))
if price_lte is not None:
params.append(("sale_price__lte", str(price_lte)))
return f"{_BFF_BASE}/api/offers/v1?{urlencode(params)}"
def _build_count_url(
rooms: str,
price_gte: int | None,
price_lte: int | None,
) -> str:
"""Строит URL для GET /api/offers/count/v1 (без offset/limit)."""
params: list[tuple[str, str]] = [
("address", _EKB_ADDRESS_GUID),
("aids", _EKB_AREA_ID),
("deal_type", "sale"),
("category", "living"),
("offer_type", "flat"),
("rooms", rooms),
("sort", "qi"),
("sort_dir", "desc"),
]
if price_gte is not None:
params.append(("sale_price__gte", str(price_gte)))
if price_lte is not None:
params.append(("sale_price__lte", str(price_lte)))
return f"{_BFF_BASE}/api/offers/count/v1?{urlencode(params)}"
def _parse_publish_date(iso: str | None) -> date | None:
"""Парсит ISO8601 дату публикации → date. None при ошибке."""
if not iso:
return None
try:
return date.fromisoformat(iso[:10])
except (ValueError, TypeError):
return None
def _extract_agency_name(seller: dict[str, Any]) -> str | None:
"""Извлечь читаемое имя агентства/агента из seller-блока BFF offer.
seller.company / seller.agent могут быть строкой ИЛИ dict (defensive —
реальная форма поля варьируется). Приоритет: company → agent. Для dict
пробуем типичные name-поля. Возвращает None если ничего читаемого нет.
"""
for key in ("company", "agent"):
val = seller.get(key)
if isinstance(val, str):
name = val.strip()
if name:
return name
elif isinstance(val, dict):
for name_key in ("name", "title", "fullName"):
nm = val.get(name_key)
if isinstance(nm, str) and nm.strip():
return nm.strip()
return None
# ── DomClickScraper ──────────────────────────────────────────────────────────
class DomClickScraper(BaseScraper):
"""DomClick вторичка через JSON BFF API. Источник = 'domklik'.
Использует BrowserFetcher(source="domclick") (generic provider → shared mobile
proxy), который обходит QRATOR. Прямые curl/httpx-запросы с datacenter-IP
блокируются QRATOR.
Основной метод: fetch_city(city_id, rooms, pages).
fetch_around() не реализован: DomClick не поддерживает geo-radius в URL.
Counters (публичные после fetch_city):
parse_failures — офферы с ошибкой маппинга
geo_filtered — офферы вне ЕКБ bbox или с неверным offerRegionName
blocked — True если sweep был прерван QRATOR-блоком
fetch_errors — не-block ошибки извлечения JSON (truncated/garbled/bad shape)
"""
name = "domklik"
source = "domklik"
base_url = "https://domclick.ru"
# Консервативная задержка между страницами (fallback до init)
request_delay_sec = 8.0
def __init__(self) -> None:
super().__init__()
self.request_delay_sec = get_scraper_delay(self.name)
self.parse_failures: int = 0
self.geo_filtered: int = 0
self.blocked: bool = False
# Не-block ошибки извлечения JSON (truncated/garbled response, неверная
# структура). В отличие от parse_failures (per-item), это per-fetch ошибки,
# которые ограничивают сбор бакета. Учитываются в honest-status pipeline.
self.fetch_errors: int = 0
async def __aenter__(self) -> DomClickScraper:
await super().__aenter__()
return self
# ── fetch_around — stub ───────────────────────────────────────────────────
async def fetch_around(self, lat: float, lon: float, radius_m: int = 1000) -> list[ScrapedLot]:
"""DomClick не поддерживает geo-radius. Используй fetch_city()."""
raise NotImplementedError(
"DomClick не поддерживает geo-radius; используй fetch_city(city_id=...)"
)
# ── Основной метод ────────────────────────────────────────────────────────
async def fetch_city(
self,
city_id: int,
rooms: list[int] | None = None,
pages: int = 100,
) -> list[ScrapedLot]:
"""Citywide sweep через BFF JSON API.
Аргументы city_id и rooms принимаются для совместимости сигнатуры с
вызывающим кодом (run_domclick_city_sweep), но:
- city_id — vestigial (EКБ захардкожен через GUID).
- rooms — игнорируется; всегда обходятся все ROOM_BUCKETS внутри.
Args:
city_id: игнорируется (EKB захардкожен).
rooms: игнорируется (ROOM_BUCKETS перебирается всегда).
pages: максимальное число страниц на бакет (safety cap).
Returns:
Дедуплицированный по source_id список ScrapedLot.
"""
# TODO Layer B (#1846 follow-up): backfill renovation/wallType/priceHistory
from app.services.scrapers.browser_fetcher import BrowserFetcher
out_lots: list[ScrapedLot] = []
seen_ids: set[str] = set()
async with BrowserFetcher(source="domclick") as fetcher:
for bucket in ROOM_BUCKETS:
logger.info(
"domklik: BFF sweep rooms=%r city_id=%d pages_cap=%d",
bucket,
city_id,
pages,
)
try:
await self._sweep_bucket(
fetcher=fetcher,
rooms=bucket,
price_gte=None,
price_lte=None,
seen_ids=seen_ids,
out_lots=out_lots,
pages=pages,
)
except DomClickBlockedError:
self.blocked = True
logger.error(
"domklik: QRATOR block during rooms=%r — aborting all buckets",
bucket,
)
break
except (ValueError, TypeError) as exc:
# Defensive: bucket-level ошибка не должна убивать весь sweep.
# _count/_paginate уже глотают эти ошибки per-fetch (fetch_errors++),
# но если что-то всё же всплыло — переходим к следующему бакету.
self.fetch_errors += 1
logger.warning(
"domklik: bucket rooms=%r failed (%s) — skipping to next bucket",
bucket,
exc,
exc_info=True,
)
continue
logger.info(
"domklik: fetch_city done city_id=%d total=%d "
"parse_failures=%d geo_filtered=%d fetch_errors=%d blocked=%s",
city_id,
len(out_lots),
self.parse_failures,
self.geo_filtered,
self.fetch_errors,
self.blocked,
)
return out_lots
# ── Internal sweep helpers ────────────────────────────────────────────────
async def _count(
self,
fetcher: Any,
rooms: str,
price_gte: int | None,
price_lte: int | None,
) -> int:
"""Запрашивает snippetsCount для данного бакета (rooms + price range).
DomClickBlockedError пробрасывается наверх (abort sweep). Прочие ошибки
извлечения JSON (truncated/garbled/bad shape) → fetch_errors++ и return 0
(бакет пропускается, sweep продолжается).
"""
url = _build_count_url(rooms, price_gte, price_lte)
logger.debug("domklik: count url=%s", url)
html = await fetcher.fetch(url)
try:
data = _extract_json(html)
except DomClickBlockedError:
raise
except (ValueError, TypeError, AttributeError):
self.fetch_errors += 1
logger.warning(
"domklik: _count JSON extract failed rooms=%r price=[%s,%s] — skip bucket",
rooms,
price_gte,
price_lte,
exc_info=True,
)
return 0
# {"result": null} → None, {"snippetsCount": null} → None: harden обе.
res = data.get("result") or {}
raw = res.get("snippetsCount")
return int(raw) if raw else 0
async def _sweep_bucket(
self,
fetcher: Any,
rooms: str,
price_gte: int | None,
price_lte: int | None,
seen_ids: set[str],
out_lots: list[ScrapedLot],
pages: int,
) -> None:
"""Рекурсивный sweep бакета (rooms, price_gte, price_lte).
Если snippetsCount > OFFSET_CAP — делим диапазон цен пополам и
рекурсируем в каждую половину. Остановка рекурсии:
- span <= MIN_PRICE_SPAN → пагинируем как есть (с предупреждением о truncation)
- snippetsCount == 0 → пропускаем
- snippetsCount <= OFFSET_CAP → пагинируем
Raises:
DomClickBlockedError: если QRATOR-блок — propagate наверх.
"""
count = await self._count(fetcher, rooms, price_gte, price_lte)
if count == 0:
return
if count > OFFSET_CAP:
_gte = price_gte if price_gte is not None else 0
_lte = price_lte if price_lte is not None else LTE_MAX
span = _lte - _gte
if span <= MIN_PRICE_SPAN:
logger.warning(
"domklik: rooms=%r price=[%s,%s] count=%d > cap=%d "
"but span=%d <= min=%d — paginating as-is (bucket truncated at %d)",
rooms,
price_gte,
price_lte,
count,
OFFSET_CAP,
span,
MIN_PRICE_SPAN,
OFFSET_CAP,
)
await self._paginate(
fetcher, rooms, price_gte, price_lte, seen_ids, out_lots, pages
)
return
mid = (_gte + _lte) // 2
logger.debug(
"domklik: rooms=%r count=%d > cap=%d — price-split [%d,%d] → [%d,%d]+[%d,%d]",
rooms,
count,
OFFSET_CAP,
_gte,
_lte,
_gte,
mid,
mid + 1,
_lte,
)
await self._sweep_bucket(fetcher, rooms, _gte, mid, seen_ids, out_lots, pages)
await self._sweep_bucket(fetcher, rooms, mid + 1, _lte, seen_ids, out_lots, pages)
else:
await self._paginate(fetcher, rooms, price_gte, price_lte, seen_ids, out_lots, pages)
async def _paginate(
self,
fetcher: Any,
rooms: str,
price_gte: int | None,
price_lte: int | None,
seen_ids: set[str],
out_lots: list[ScrapedLot],
pages: int,
) -> None:
"""Пагинирует один бакет (rooms, price range) до исчерпания или cap.
DomClickBlockedError пробрасывается наверх (abort sweep). Прочие ошибки
извлечения JSON (truncated/garbled/bad shape) → fetch_errors++ и break
(трактуем как конец бакета — уже собранные lots сохраняются).
Raises:
DomClickBlockedError: propagate из _extract_json при QRATOR-блоке.
"""
force_rooms = 0 if rooms == "st" else None
page_idx = 0
while page_idx < pages:
offset = page_idx * PAGE_SIZE
if offset >= OFFSET_CAP:
break
url = _build_offers_url(rooms, price_gte, price_lte, offset)
logger.debug("domklik: offers url=%s", url)
html = await fetcher.fetch(url)
try:
data = _extract_json(html)
except DomClickBlockedError:
raise
except (ValueError, TypeError, AttributeError):
self.fetch_errors += 1
logger.warning(
"domklik: _paginate JSON extract failed rooms=%r offset=%d "
"— ending bucket (lots so far preserved)",
rooms,
offset,
exc_info=True,
)
break
# {"result": null} → None; harden до .get("items").
items: list[dict[str, Any]] = (data.get("result") or {}).get("items") or []
if not items:
break
for item in items:
lot = self._map_item(item, force_rooms=force_rooms)
if lot is None:
continue
if not self._is_geo_ok(item):
self.geo_filtered += 1
continue
key = lot.source_id or lot.source_url
if key in seen_ids:
continue
seen_ids.add(key)
out_lots.append(lot)
page_idx += 1
if page_idx < pages and len(items) == PAGE_SIZE:
await self.sleep_between_requests()
elif not items or len(items) < PAGE_SIZE:
break
def _is_geo_ok(self, item: dict[str, Any]) -> bool:
"""Гео-гард: пропускает только листинги ЕКБ в bbox.
aids=20561 даёт чистый ЕКБ, но гард оставляем как defensive проверку.
"""
region = item.get("offerRegionName", "")
if region != _EKB_REGION_NAME:
return False
loc = item.get("location") or {}
lat = loc.get("lat")
lon = loc.get("lon")
if lat is None or lon is None:
return False
return (
_EKB_LAT_MIN <= float(lat) <= _EKB_LAT_MAX
and _EKB_LON_MIN <= float(lon) <= _EKB_LON_MAX
)
def _map_item(
self, item: dict[str, Any], *, force_rooms: int | None = None
) -> ScrapedLot | None:
"""Маппинг offer-item из BFF API → ScrapedLot.
Возвращает None если price <= 0 или при ошибке парсинга.
"""
try:
price = item.get("price", 0) or 0
if price <= 0:
return None
item_id = item.get("id")
source_id = str(item_id) if item_id is not None else None
source_url: str = item.get("path", "") or ""
loc = item.get("location") or {}
lat: float | None = loc.get("lat")
lon: float | None = loc.get("lon")
address_obj = item.get("address") or {}
address: str | None = address_obj.get("displayName")
# 0 — sentinel «неизвестно» для area/floor/total_floors/buildYear
# (DomClick эмитит 0 для under-construction / unknown) → None.
obj_info = item.get("objectInfo") or {}
area_raw = obj_info.get("area")
area_m2: float | None = float(area_raw) if area_raw else None
floor_raw = obj_info.get("floor")
floor: int | None = int(floor_raw) if floor_raw else None
house = item.get("house") or {}
floors_raw = house.get("floors")
total_floors: int | None = int(floors_raw) if floors_raw else None
year_raw = house.get("buildYear")
year_built: int | None = int(year_raw) if year_raw else None
# rooms: 0 — ВАЛИДНОЕ значение (студия), поэтому проверка is not None,
# а не truthy. force_rooms=0 (st-бакет) перебивает objectInfo.
if force_rooms is not None:
rooms: int | None = force_rooms
else:
rooms_raw = obj_info.get("rooms")
rooms = int(rooms_raw) if rooms_raw is not None else None
square_price_raw = item.get("squarePrice")
price_per_m2: int | None = int(square_price_raw) if square_price_raw else None
if price_per_m2 is None and area_m2 and area_m2 > 0:
price_per_m2 = int(price / area_m2)
# ── Описание + repair-инференс + minhash (Layer A enrichment) ─────
# DomClick search не отдаёт renovation-enum: repair_state выводим из
# текста описания (как cian #622 fallback — «евроремонт»/«без отделки»).
# description_minhash — sha1 от описания (mirror cian) для cross-source
# дедупа.
description_raw = item.get("description")
description: str | None = None
if isinstance(description_raw, str):
description = description_raw.strip() or None
repair_state: str | None = None
description_minhash: str | None = None
if description:
repair_state = infer_repair_state_from_text(description)
description_minhash = hashlib.sha1(
description.lower().encode("utf-8", errors="replace")
).hexdigest()[:32]
# ── Продавец / агентство ──────────────────────────────────────────
seller = item.get("seller")
if not isinstance(seller, dict):
seller = {}
agency_name = _extract_agency_name(seller)
flat_complex = item.get("flatComplex") or {}
raw_payload: dict[str, Any] = {
"isRosreestrApproved": item.get("isRosreestrApproved"),
"squarePrice": square_price_raw,
"lastPriceHistoryState": item.get("lastPriceHistoryState"),
"flatComplex": {
"name": flat_complex.get("name"),
"id": flat_complex.get("id"),
"slug": flat_complex.get("slug"),
}
if flat_complex
else None,
"updatedDate": item.get("updatedDate"),
}
# ── Доп. SERP-поля (Layer A enrichment) — мержим в raw_payload, ────
# сбрасывая None-значения чтобы payload оставался компактным; ничего
# из существующих ключей не теряем.
seller_compact = {
k: seller.get(k) for k in ("agent", "company") if seller.get(k) is not None
}
_enrich: dict[str, Any] = {
"is_sber_collateral": item.get("isSberCollateral"),
"has_discount": item.get("hasDiscount"),
"discount_value": item.get("discountValue"),
"duplicates_offer_count": item.get("duplicatesOfferCount"),
"seller": seller_compact or None,
}
raw_payload.update({k: v for k, v in _enrich.items() if v is not None})
publish_date = _parse_publish_date(item.get("publishedDate"))
return ScrapedLot(
source="domklik",
source_url=source_url,
source_id=source_id,
address=address,
lat=lat,
lon=lon,
rooms=rooms,
area_m2=area_m2,
floor=floor,
total_floors=total_floors,
year_built=year_built,
price_rub=int(price),
price_per_m2=price_per_m2,
listing_segment="vtorichka",
publish_date=publish_date,
description=description,
repair_state=repair_state,
description_minhash=description_minhash,
agency_name=agency_name,
raw_payload=raw_payload,
)
except Exception:
self.parse_failures += 1
logger.warning(
"domklik: _map_item failed for item id=%r (parse_failures=%d)",
item.get("id"),
self.parse_failures,
exc_info=True,
)
return None
# ── Convenience runner (для Celery tasks) ────────────────────────────────────
async def scrape_domclick_city(
city_id: int,
rooms: list[int] | None = None,
pages: int = 100,
) -> list[ScrapedLot]:
"""Удобная точка входа для вызова из Celery tasks.
Пример::
import asyncio
lots = asyncio.run(scrape_domclick_city(city_id=4, rooms=None))
"""
async with DomClickScraper() as scraper:
return await scraper.fetch_city(city_id=city_id, rooms=rooms, pages=pages)

27
tradein-mvp/backend/app/services/scrapers/domclick_exceptions.py
View file

@ -1,27 +0,0 @@
"""DomClick-specific exceptions для anti-bot detection."""
class DomClickBlockedError(Exception):
"""DomClick BFF вернул QRATOR block-страницу (HTTP 200 + block HTML).
QRATOR (qrator.net) — WAF/DDoS-защита domclick.ru. Блокирует datacenter-IP
и возвращает HTML с маркерами: "qrator", "bot_mitigation", "система защиты",
"403 | домклик", "captcha", "access denied".
Обходится через shared mobile proxy (BrowserFetcher(source="domclick") →
generic provider → мобильный egress).
"""
class DomClickParseError(Exception):
"""DomClick detail-карточка получена (HTTP 200, не block), но SSR-стейт
не извлекается / не парсится.
Возникает в Layer B (domclick_detail.py) когда:
- `window.__SSR_STATE__` не найден в HTML (но это и не challenge-страница);
- object-литерал не сбалансирован по скобкам;
- json.loads падает после санитайза bare `undefined` → `null`.
В отличие от DomClickBlockedError (anti-bot), это сигнал дрейфа схемы карточки
или повреждённого ответа — оркестратор считает его parse-failure, а не блоком.
"""

1107
tradein-mvp/backend/app/services/scrapers/yandex_realty.py
View file

File diff suppressed because it is too large Load diff

112
tradein-mvp/backend/tests/scrapers/test_admin_avito_kit_parity.py
View file

@ -1,112 +0,0 @@
"""Parity-proof: admin.py Avito debug-роуты (#2305, Group A) — legacy vs scraper_kit.
`app/api/v1/admin.py::scrape_avito_house` теперь зовёт `scraper_kit.providers.avito.houses`
вместо `app.services.scrapers.avito_houses` (тот же `/scrape` root-роут —
`scraper_kit.providers.avito.serp.AvitoScraper`). Каждая пара функций ниже — чистое,
offline-тестируемое parsing-ядро, которое реально используется соответствующим
debug-эндпоинтом (не сам HTTP-fetch — см. `tests/support/README.md`: "Live network/DB
в parity-тестах избегайте").
- `_parse_house_page` — `scrape_avito_house` → `fetch_house_catalog` парсит виджет
`housePage` этой функцией.
- `_extract_rooms_from_title` / `_extract_area_from_title` — использует
`AvitoScraper` (SERP-парсинг `/scrape` root-роута) для title-fallback полей.
Легаси `app.services.scrapers.avito_imv` (`scrape_avito_imv` → `evaluate_via_imv` →
`_parse_price`) удалён (#2277 финальный шаг scraper_kit-миграции, 0 runtime
importers) — parity-проверка `_parse_price` убрана вместе с ним; kit-сторона
`_parse_price` покрыта отдельно (`tests/scrapers/test_avito_imv_kit_parity.py` /
`tests/test_avito_imv_parse.py`).
"""
from __future__ import annotations
import dataclasses
import os
# Соседние модули app.services.scrapers.* импортируют app.core.config при импорте
# пакета — фиктивный DSN мостит офлайн-запуск (mirror test_avito_detail_kit_parity.py).
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
from scraper_kit.providers.avito.houses import _parse_house_page as kit_parse_house_page
from scraper_kit.providers.avito.serp import (
_extract_area_from_title as kit_extract_area_from_title,
)
from scraper_kit.providers.avito.serp import (
_extract_rooms_from_title as kit_extract_rooms_from_title,
)
from app.services.scrapers.avito import _extract_area_from_title as legacy_extract_area_from_title
from app.services.scrapers.avito import (
_extract_rooms_from_title as legacy_extract_rooms_from_title,
)
from app.services.scrapers.avito_houses import _parse_house_page as legacy_parse_house_page
from tests.support.parity import assert_parity
_HOUSE_WIDGET = {
"props": {
"developmentData": {
"avitoId": 3171365,
"id": "abc123hash",
"title": "ЖК Тестовый",
"address": "ул. Тестовая, 1",
"fullAddress": "Екатеринбург, ул. Тестовая, 1",
"coords": {"lat": 56.8, "lng": 60.6},
"aboutDevelopment": {
"expandParams": {
"items": [
{
"params": [
{"type": "Год постройки", "value": "2020"},
{"type": "Этажей", "value": "16"},
{"type": "Пассажирский лифт", "value": "2"},
]
}
]
}
},
"ratingBadge": {"info": {"score": 4.5, "scoreString": "4.5"}},
"ratingSummaryStat": {"reviewCount": 10, "ratingStat": []},
"mapPreview": {"objects": "школа рядом", "distance": "5 мин", "pins": []},
"developer": {"name": "ООО Застройщик", "key": "dev1"},
}
}
}
def test_avito_house_page_parse_are_class_distinct() -> None:
"""Sanity: legacy/kit HouseInfo — разные классы, обычный `==` даёт False."""
legacy_result = legacy_parse_house_page(_HOUSE_WIDGET)
kit_result = kit_parse_house_page(_HOUSE_WIDGET)
assert type(legacy_result) is not type(kit_result)
assert legacy_result != kit_result
assert dataclasses.asdict(legacy_result) == dataclasses.asdict(kit_result)
def test_parity_avito_house_page() -> None:
"""`scrape_avito_house` (admin.py) → fetch_house_catalog → _parse_house_page."""
assert_parity(
legacy_fn=legacy_parse_house_page,
kit_fn=kit_parse_house_page,
fixtures=[(_HOUSE_WIDGET,)],
)
def test_parity_avito_serp_title_extract() -> None:
"""`/scrape` root-роут (admin.py) → AvitoScraper SERP title-fallback fields."""
titles = [
"2-к. квартира, 54.3 м², 3/9 эт.",
"Студия, 25 м², 1/5 эт.",
"3-к. кв., 75,5 м², 12/16 эт.",
]
assert_parity(
legacy_fn=legacy_extract_rooms_from_title,
kit_fn=kit_extract_rooms_from_title,
fixtures=titles,
)
assert_parity(
legacy_fn=legacy_extract_area_from_title,
kit_fn=kit_extract_area_from_title,
fixtures=titles,
)

51
tradein-mvp/backend/tests/scrapers/test_admin_cian_kit_parity.py
View file

@ -1,51 +0,0 @@
"""Parity-proof: admin.py `scrape_cian_detail` (#2305, Group A) — legacy vs scraper_kit.
`app/api/v1/admin.py::scrape_cian_detail` теперь зовёт
`scraper_kit.providers.cian.detail.fetch_detail/save_detail_enrichment` вместо
`app.services.scrapers.cian_detail`. `_extract_price_changes` — чистая,
offline-тестируемая normalize-функция, которую `fetch_detail` вызывает при
сборке `DetailEnrichment.price_changes` (см. `tests/support/README.md`:
"Live network/DB в parity-тестах избегайте").
"""
from __future__ import annotations
import os
# cian_detail.py импортирует app.core.config при module-level import — фиктивный
# DSN мостит офлайн-запуск (mirror test_avito_detail_kit_parity.py).
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
from scraper_kit.providers.cian.detail import _extract_price_changes as kit_extract_price_changes
from app.services.scrapers.cian_detail import (
_extract_price_changes as legacy_extract_price_changes,
)
from tests.support.parity import assert_parity
# Три реальных формы priceChanges: curl_cffi flat, browser-rendered nested, оба-пусто.
_OFFER_FLAT = {
"priceChanges": [
{"changeTime": "2026-01-15T00:00:00", "price": 9000000, "diffPercent": -2.5},
{"changeTime": "2026-02-01T00:00:00", "priceRub": 8800000, "diff_percent": -2.2},
]
}
_OFFER_DATA_NESTED = {
"priceChanges": [
{"changeTime": "2026-01-15T00:00:00", "priceData": {"price": 9000000}},
]
}
_STATE_EMPTY: dict[str, object] = {}
def test_parity_cian_detail_price_changes() -> None:
"""`scrape_cian_detail` (admin.py) → fetch_detail → _extract_price_changes."""
assert_parity(
legacy_fn=legacy_extract_price_changes,
kit_fn=kit_extract_price_changes,
fixtures=[
(_OFFER_FLAT, {}, _STATE_EMPTY),
({}, _OFFER_DATA_NESTED, _STATE_EMPTY),
({}, {}, _STATE_EMPTY),
],
)

99
tradein-mvp/backend/tests/scrapers/test_admin_yandex_kit_parity.py
View file

@ -1,99 +0,0 @@
"""Parity-proof: admin.py Yandex debug-роуты (#2305, Group A) — legacy vs scraper_kit.
`app/api/v1/admin.py::scrape_yandex_detail` / `scrape_yandex_valuation` теперь
зовут `scraper_kit.providers.yandex.{detail,valuation}` вместо
`app.services.scrapers.yandex_{detail,valuation}` (тот же `/scrape` root-роут —
`scraper_kit.providers.yandex.serp.YandexRealtyScraper`). Каждая функция ниже —
чистое, offline-тестируемое parsing-ядро (см. `tests/support/README.md`:
"Live network/DB в parity-тестах избегайте").
- `_entity_to_lot` — `/scrape` root-роут → `YandexRealtyScraper.fetch_around`
конвертирует gate-API entity → ScrapedLot этой функцией (также доказывает
ScrapedLot-контракт, общий с `save_listings`/`scripts/ingest_domclick_jsonl.py`).
Легаси `app.services.scrapers.yandex_detail` / `yandex_valuation` удалены (#2277
финальный шаг scraper_kit-миграции, 0 runtime importers) — `_parse_card_fields` /
`_parse_house_meta` parity-проверки убраны отсюда вместе с ними; kit-side coverage
для обеих функций сохранена standalone в `tests/test_yandex_detail_structural.py` /
`tests/test_yandex_valuation.py`. `yandex_realty` остаётся легаси (не мигрирован) —
его `_entity_to_lot` parity-проверка не тронута.
"""
from __future__ import annotations
import os
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
from scraper_kit.providers.yandex.serp import _entity_to_lot as kit_entity_to_lot
from app.services.scrapers.yandex_realty import _entity_to_lot as legacy_entity_to_lot
from tests.support.parity import assert_parity
# gate-API entity (realty.yandex.ru), без creationDate/author — исключает
# time-dependent поля (days_on_market) из фикстуры (deterministic offline).
_YANDEX_ENTITY = {
"offerId": "12345",
"url": "//realty.yandex.ru/offer/12345",
"price": {"value": 8500000, "valuePerPart": 157000},
"area": {"value": 54.3},
"livingSpace": {"value": 32.1},
"kitchenSpace": {"value": 9.5},
"roomsTotal": 2,
"floorsOffered": [5],
"floorsTotal": 9,
"ceilingHeight": 2.7,
"building": {"builtYear": 2015, "buildingType": "MONOLIT"},
"location": {
"point": {"latitude": 56.83, "longitude": 60.6},
"geocoderAddress": "Екатеринбург, ул. Тестовая, 1",
},
"mainImages": ["//avatars.mds.yandex.net/img1.jpg"],
}
def test_yandex_entity_to_lot_are_class_distinct() -> None:
"""Sanity: legacy/kit ScrapedLot — разные классы (pydantic), обычный `==` даёт False."""
legacy_result = legacy_entity_to_lot(_YANDEX_ENTITY)
kit_result = kit_entity_to_lot(_YANDEX_ENTITY)
assert legacy_result is not None
assert kit_result is not None
assert type(legacy_result) is not type(kit_result)
assert legacy_result.model_dump() == kit_result.model_dump()
def test_parity_yandex_serp_entity_to_lot() -> None:
"""`/scrape` root-роут (admin.py) → YandexRealtyScraper.fetch_around → _entity_to_lot.
Также доказывает ScrapedLot-контракт, общий с `save_listings`
(используется и `/scrape`, и `scripts/ingest_domclick_jsonl.py`).
"""
assert_parity(
legacy_fn=legacy_entity_to_lot,
kit_fn=kit_entity_to_lot,
fixtures=[(_YANDEX_ENTITY,)],
)
def test_parity_harness_catches_yandex_entity_divergence() -> None:
"""Harness реально ЛОВИТ расхождение — мутируем price_rub в kit-результате."""
import pytest
from tests.support.parity import ParityMismatchError
legacy_result = legacy_entity_to_lot(_YANDEX_ENTITY)
kit_result = kit_entity_to_lot(_YANDEX_ENTITY)
assert legacy_result is not None
assert kit_result is not None
# ScrapedLot — pydantic BaseModel (НЕ dataclass) → model_copy, не dataclasses.replace.
mutated_kit_result = kit_result.model_copy(update={"price_rub": kit_result.price_rub + 1})
with pytest.raises(ParityMismatchError) as exc_info:
assert_parity(
legacy_fn=lambda: legacy_result,
kit_fn=lambda: mutated_kit_result,
fixtures=[()],
)
assert "price_rub" in str(exc_info.value)

25
tradein-mvp/backend/tests/scrapers/test_avito_unix_date_tz_consistency.py
View file

@ -3,7 +3,7 @@
До фикса было 3 независимые копии с разными timezone:
- avito.py (inline в `_build_sort_timestamp_map`) — MSK (документировано #726)
- avito_houses.py — UTC
- avito_houses.py (легаси, удалён #2397 Part E2) — UTC
- avito_imv.py (легаси, удалён #2277) — naive/system-local
(`datetime.fromtimestamp(ts)` без tz)
@ -17,6 +17,11 @@ avito_imv_getdata.json: removedDate=1756077885 → UTC 2025-08-24, MSK 2025-08-2
kit-эквивалент `scraper_kit.providers.avito.imv`, который использует ТОТ ЖЕ паттерн
(`from scraper_kit.providers.avito.shared import _unix_to_date`) — регрессия
"3 независимые копии с разными TZ" остаётся под наблюдением и на kit-стороне.
Легаси `app.services.scrapers.avito_houses` удалён (#2397 Part E2, 0 runtime
importers) — houses-ветка consolidation-проверки переведена на kit-эквивалент
`scraper_kit.providers.avito.houses`, который использует ТОТ ЖЕ паттерн
(`from scraper_kit.providers.avito.shared import RUS_MONTHS, _unix_to_date`).
"""
from __future__ import annotations
@ -26,10 +31,11 @@ from datetime import date
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
from scraper_kit.providers.avito import houses as kit_avito_houses
from scraper_kit.providers.avito import imv as kit_avito_imv
from scraper_kit.providers.avito import shared as kit_avito_shared
from app.services.scrapers import avito, avito_houses, avito_shared
from app.services.scrapers import avito, avito_shared
# Живой пример из fixtures/avito_imv_getdata.json (placementHistory.items[0].removedDate) —
# именно тот timestamp, на котором UTC и MSK расходились в date() ДО фикса
@ -44,14 +50,15 @@ def test_shared_unix_to_date_uses_msk() -> None:
def test_avito_houses_reexports_shared_unix_to_date() -> None:
"""avito_houses.py больше не держит свою копию — импортирует общую функцию."""
assert avito_houses._unix_to_date is avito_shared._unix_to_date
assert avito_houses._unix_to_date(_EPOCH_SECONDS) == _EXPECTED_MSK_DATE
"""kit houses.py не держит свою копию — импортирует общую функцию (тот же паттерн,
что и легаси avito_houses.py до удаления, #2397 Part E2)."""
assert kit_avito_houses._unix_to_date is kit_avito_shared._unix_to_date
assert kit_avito_houses._unix_to_date(_EPOCH_SECONDS) == _EXPECTED_MSK_DATE
def test_kit_avito_imv_reexports_shared_unix_to_date() -> None:
"""kit imv.py не держит свою копию — импортирует общую функцию (тот же паттерн,
что и легаси avito_houses.py/avito_shared.py после фикса #2129)."""
что и kit houses.py/avito_shared.py после фикса #2129)."""
assert kit_avito_imv._unix_to_date is kit_avito_shared._unix_to_date
assert kit_avito_imv._unix_to_date(_EPOCH_SECONDS) == _EXPECTED_MSK_DATE
@ -66,11 +73,11 @@ def test_avito_serp_sort_timestamp_matches_shared_date() -> None:
)
ts_map = avito._build_sort_timestamp_map(html)
assert ts_map["8043936560"] == _EXPECTED_MSK_DATE
# Легаси avito/avito_houses/avito_shared + kit avito_imv все согласованы на одном
# Легаси avito/avito_shared + kit avito_houses/avito_imv все согласованы на одном
# моменте времени:
assert (
ts_map["8043936560"]
== avito_houses._unix_to_date(_EPOCH_SECONDS)
== kit_avito_houses._unix_to_date(_EPOCH_SECONDS)
== kit_avito_imv._unix_to_date(_EPOCH_SECONDS)
== avito_shared._unix_to_date(_EPOCH_SECONDS)
)
@ -81,7 +88,7 @@ def test_rus_months_single_source() -> None:
from app.services.scrapers import avito_detail
assert avito_detail.RUS_MONTHS is avito_shared.RUS_MONTHS
assert avito_houses.RUS_MONTHS is avito_shared.RUS_MONTHS
assert kit_avito_houses.RUS_MONTHS is kit_avito_shared.RUS_MONTHS
assert avito_shared.RUS_MONTHS["декабря"] == 12

792
tradein-mvp/backend/tests/scrapers/test_cian_exhaustive.py
View file

@ -1,792 +0,0 @@
"""Тесты exhaustive-загрузки Cian: fetch_all_secondary + _walk_price_range.
Мокаем сетевые вызовы (_fetch_page_html, _extract_total_offers, sleep_between_requests)
для детерминированного контроля поведения без реального HTTP.
"""
from __future__ import annotations
import os
from unittest.mock import AsyncMock, MagicMock, patch
# Settings requires DATABASE_URL at import time — set dummy DSN before any app import.
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
import logging
import pytest
from app.services.scrapers.base import ScrapedLot
from app.services.scrapers.cian import (
_CIAN_OFFERS_PER_PAGE,
_MAX_PRICE,
_MIN_BRACKET,
CianScraper,
)
from app.services.scrapers.price_brackets import get_price_seed_brackets
_SEED = get_price_seed_brackets()
def _make_lot(source_id: str, price: int = 3_000_000) -> ScrapedLot:
"""Минимальный ScrapedLot для теста."""
return ScrapedLot(
source="cian",
source_url=f"https://ekb.cian.ru/sale/flat/{source_id}/",
source_id=source_id,
address="Екатеринбург",
price_rub=price,
)
def _make_html_stub(total_offers: int, lot_ids: list[str]) -> str:
"""Заглушка HTML — не парсится напрямую; используется через мокнутые методы."""
return f"<html>total={total_offers} ids={lot_ids}</html>"
@pytest.fixture
def scraper() -> CianScraper:
"""CianScraper без реального __aenter__ (без HTTP warm-up)."""
s = CianScraper.__new__(CianScraper)
s.name = "cian"
s.base_url = "https://ekb.cian.ru"
s.request_delay_sec = 0.0
s._browser = MagicMock() # не None → assert self._browser is not None пройдёт
return s
@pytest.mark.asyncio
async def test_fetch_all_secondary_splits_on_cap(scraper: CianScraper) -> None:
"""totalOffers > cap → бакет разбивается; в итоге оба суб-бакета пагинируются."""
# Настройка: rooms=(1,), один бакет [0, None]
# Probe [0, None]: totalOffers=3000 > cap=1400 → split на [0, MAX_PRICE]
# Probe [0, MAX_PRICE]: totalOffers=3000 > 1400 → split на [0, mid] и [mid+1, MAX_PRICE]
# Probe [0, mid] (mid = MAX_PRICE//2 = 100_000_000): totalOffers=800 <= cap → пагинировать
# Probe [mid+1, MAX_PRICE]: totalOffers=700 <= cap → пагинировать
mid = _MAX_PRICE // 2
call_log: list[tuple] = []
async def fake_fetch_page_html(
rooms: tuple, page: int, min_price: int | None, max_price: int | None
) -> str:
call_log.append((rooms, page, min_price, max_price))
return f"<html>page={page} min={min_price} max={max_price}</html>"
def fake_extract_total_offers(html: str) -> int | None:
# Парсим min/max из fake html
import re
min_m = re.search(r"min=(\w+)", html)
max_m = re.search(r"max=(\w+)", html)
min_p = int(min_m.group(1)) if min_m and min_m.group(1) != "None" else 0
max_p = int(max_m.group(1)) if max_m and max_m.group(1) != "None" else _MAX_PRICE
# Имитируем разделение: весь диапазон > 1400, левая и правая половины <= 1400
if max_p is None or max_p >= _MAX_PRICE:
if min_p == 0:
return 3000 # весь диапазон — нужно split
if max_p == mid:
return 800 # левая половина — пагинировать
if min_p == mid + 1:
return 700 # правая половина — пагинировать
return 3000 # всё остальное — split
def fake_parse_serp_html(html: str) -> list[ScrapedLot]:
import re
page_m = re.search(r"page=(\d+)", html)
min_m = re.search(r"min=(\w+)", html)
max_m = re.search(r"max=(\w+)", html)
page = int(page_m.group(1)) if page_m else 1
min_p = min_m.group(1) if min_m else "0"
max_p = max_m.group(1) if max_m else "MAX"
# Возвращаем по 2 лота на первой странице каждого пагинируемого бакета
if page == 1:
return [
_make_lot(f"lot_{min_p}_{max_p}_p{page}_1"),
_make_lot(f"lot_{min_p}_{max_p}_p{page}_2"),
]
return [] # страница 2+ — пусто → early stop
async def fake_sleep(*_: object, **__: object) -> None:
pass
with (
patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html),
patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers),
patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html),
patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock),
patch.object(scraper, "_rotate_ip", return_value=False),
):
lots = await scraper.fetch_all_secondary(
rooms_buckets=[(1,)],
price_cap_per_bucket=1400,
)
# Должны получить лоты из ОБОИХ суб-бакетов после split
assert len(lots) > 0, "Должны получить лоты после разделения бакета"
# Проверяем что был probe с большим диапазоном И с sub-бакетами
probed_ranges = [(min_p, max_p) for (_, page, min_p, max_p) in call_log if page == 1]
# Должен быть probe [0, None] или [0, MAX] — начальный probe всего диапазона
initial_probes = [r for r in probed_ranges if r[0] is None or r[0] == 0]
assert (
len(initial_probes) >= 1
), f"Ожидался probe всего диапазона, probed_ranges={probed_ranges}"
@pytest.mark.asyncio
async def test_fetch_all_secondary_paginates_when_under_cap(scraper: CianScraper) -> None:
"""totalOffers <= cap → пагинирует бакет полностью (страница за страницей)."""
pages_fetched: list[int] = []
async def fake_fetch_page_html(
rooms: tuple, page: int, min_price: int | None, max_price: int | None
) -> str:
pages_fetched.append(page)
return f"<html>page={page}</html>"
def fake_extract_total_offers(html: str) -> int | None:
return 56 # 56 офферов → ceil(56/28) = 2 страницы
def fake_parse_serp_html(html: str) -> list[ScrapedLot]:
import re
page_m = re.search(r"page=(\d+)", html)
page = int(page_m.group(1)) if page_m else 1
if page <= 2:
return [_make_lot(f"lot_p{page}_{i}") for i in range(28)]
return []
with (
patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html),
patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers),
patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html),
patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock),
patch.object(scraper, "_rotate_ip", return_value=False),
):
lots = await scraper.fetch_all_secondary(
rooms_buckets=[(2,)],
price_cap_per_bucket=1400,
)
# 28 лотов на стр 1 + 28 на стр 2 = 56, дедуп по source_id
assert len(lots) == 56, f"Ожидалось 56 уникальных лотов, получено {len(lots)}"
# Должно быть 2 страницы пагинации (page=1 probe + page=2)
assert 2 in pages_fetched, "Должна быть запрошена страница 2"
@pytest.mark.asyncio
async def test_fetch_all_secondary_min_bracket_guard(scraper: CianScraper) -> None:
"""Если hi - lo < MIN_BRACKET → пагинируем как есть (не делим бесконечно)."""
call_count = 0
async def fake_fetch_page_html(
rooms: tuple, page: int, min_price: int | None, max_price: int | None
) -> str:
nonlocal call_count
call_count += 1
if call_count > 100:
raise RuntimeError("Бесконечная рекурсия — MIN_BRACKET guard не сработал")
return f"<html>page={page}</html>"
def fake_extract_total_offers(html: str) -> int | None:
# Всегда > cap → без guard делилось бы бесконечно
return 5000
def fake_parse_serp_html(html: str) -> list[ScrapedLot]:
return [] # пустой → early stop в пагинации
with (
patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html),
patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers),
patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html),
patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock),
patch.object(scraper, "_rotate_ip", return_value=False),
):
# Запускаем с очень узким диапазоном — guard должен остановить деление
await scraper._walk_price_range(
rooms=(1,),
lo=1_000_000,
hi=1_000_000 + _MIN_BRACKET - 1, # bracket < MIN_BRACKET
seen={},
price_cap_per_bucket=1400,
max_pages_per_bucket=54,
)
# Guard сработал: нет RuntimeError (бесконечная рекурсия не возникла).
# Параллельная пагинация запускает до max_pages_per_bucket=54 страниц единовременно
# (все страницы бакета параллельны), поэтому call_count может быть до 54.
# Важно что нет рекурсии (bracket < MIN_BRACKET → один проход пагинации, не деление).
assert call_count <= 55, f"Слишком много вызовов ({call_count}) — guard не сработал"
@pytest.mark.asyncio
async def test_fetch_all_secondary_on_bucket_called_per_leaf(scraper: CianScraper) -> None:
"""on_bucket вызывается после каждого leaf-бакета с лотами бакета."""
# totalOffers=56 ≤ cap → leaf-бакет, пагинируется параллельно 2 страницы
bucket_calls: list[tuple[str, list[ScrapedLot]]] = []
def fake_on_bucket(bucket_key: str, lots: list[ScrapedLot]) -> None:
bucket_calls.append((bucket_key, list(lots)))
async def fake_fetch_page_html(
rooms: tuple, page: int, min_price: int | None, max_price: int | None
) -> str:
return f"<html>page={page}</html>"
def fake_extract_total_offers(html: str) -> int | None:
return 56 # ceil(56/28) = 2 страницы
def fake_parse_serp_html(html: str) -> list[ScrapedLot]:
import re
page_m = re.search(r"page=(\d+)", html)
page = int(page_m.group(1)) if page_m else 1
return [_make_lot(f"lot_p{page}_{i}") for i in range(10)]
with (
patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html),
patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers),
patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html),
patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock),
patch.object(scraper, "_rotate_ip", return_value=False),
):
await scraper.fetch_all_secondary(
rooms_buckets=[(1,)],
price_cap_per_bucket=1400,
concurrency=3,
on_bucket=fake_on_bucket,
)
# on_bucket вызван по разу на каждый seed-брекет (все непустые leaf-бакеты).
assert len(bucket_calls) == len(
_SEED
), f"Ожидалось {len(_SEED)} вызовов on_bucket, получено {len(bucket_calls)}"
key, lots = bucket_calls[0]
# Ключ бакета содержит room_label:lo:hi
assert "room1" in key, f"bucket_key должен содержать room1, got {key!r}"
# Лоты из обеих страниц переданы в on_bucket
assert len(lots) == 20, f"Ожидалось 20 лотов в on_bucket (2 стр × 10), получено {len(lots)}"
@pytest.mark.asyncio
async def test_fetch_all_secondary_on_bucket_cancel_stops_run(scraper: CianScraper) -> None:
"""Если on_bucket кидает RuntimeError('cancelled') — прогон прерывается."""
def cancel_on_bucket(bucket_key: str, lots: list[ScrapedLot]) -> None:
raise RuntimeError("cancelled")
async def fake_fetch_page_html(
rooms: tuple, page: int, min_price: int | None, max_price: int | None
) -> str:
return f"<html>page={page}</html>"
def fake_extract_total_offers(html: str) -> int | None:
return 28 # один лист → 1 страница
def fake_parse_serp_html(html: str) -> list[ScrapedLot]:
return [_make_lot("lot_1")]
with (
patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html),
patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers),
patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html),
patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock),
patch.object(scraper, "_rotate_ip", return_value=False),
):
with pytest.raises(RuntimeError, match="cancelled"):
await scraper.fetch_all_secondary(
rooms_buckets=[(1,), (2,)],
price_cap_per_bucket=1400,
on_bucket=cancel_on_bucket,
)
@pytest.mark.asyncio
async def test_fetch_all_secondary_concurrent_pages_deduped(scraper: CianScraper) -> None:
"""Параллельная пагинация не ломает дедупликацию по source_id."""
fetch_pages: list[int] = []
async def fake_fetch_page_html(
rooms: tuple, page: int, min_price: int | None, max_price: int | None
) -> str:
fetch_pages.append(page)
return f"<html>page={page}</html>"
def fake_extract_total_offers(html: str) -> int | None:
return 84 # ceil(84/28) = 3 страницы
def fake_parse_serp_html(html: str) -> list[ScrapedLot]:
import re
page_m = re.search(r"page=(\d+)", html)
page = int(page_m.group(1)) if page_m else 1
# Страница 1 и 2 дают уникальные лоты; страница 3 дублирует страницу 1
if page == 3:
return [_make_lot(f"lot_p1_{i}") for i in range(10)] # дубликаты
return [_make_lot(f"lot_p{page}_{i}") for i in range(10)]
with (
patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html),
patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers),
patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html),
patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock),
patch.object(scraper, "_rotate_ip", return_value=False),
):
lots = await scraper.fetch_all_secondary(
rooms_buckets=[(1,)],
price_cap_per_bucket=1400,
concurrency=5,
)
# Страницы 1+2 = 20 уникальных, страница 3 дублирует → итого 20
assert len(lots) == 20, f"Ожидалось 20 уникальных лотов (дедуп), получено {len(lots)}"
# Все 3 страницы запрошены (параллельно или нет — нам важен факт)
assert 3 in fetch_pages or 3 in [
p for p in fetch_pages
], f"Страница 3 не была запрошена, fetch_pages={fetch_pages}"
@pytest.mark.asyncio
async def test_skip_buckets_skips_pagination_and_on_bucket(scraper: CianScraper) -> None:
"""skip_buckets: для скипнутого бакета пагинация не вызывается, on_bucket не вызывается."""
fetch_calls: list[tuple] = []
on_bucket_calls: list[str] = []
async def fake_fetch_page_html(
rooms: tuple, page: int, min_price: int | None, max_price: int | None
) -> str:
fetch_calls.append((rooms, page, min_price, max_price))
return f"<html>page={page}</html>"
def fake_extract_total_offers(html: str) -> int | None:
return 28 # один leaf-бакет, 1 страница
def fake_parse_serp_html(html: str) -> list[ScrapedLot]:
return [_make_lot("lot_1")]
def fake_on_bucket(bucket_key: str, lots: list[ScrapedLot]) -> None:
on_bucket_calls.append(bucket_key)
# Скипаем ВСЕ seed-бакеты комнатности room1. Закрытые → ':<hi-1>', открытый → ':open'.
skip_set: set[str] = set()
for br_lo, br_hi in _SEED:
hi_repr = "open" if br_hi is None else str(br_hi - 1)
skip_set.add(f"room1:{br_lo}:{hi_repr}")
with (
patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html),
patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers),
patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html),
patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock),
patch.object(scraper, "_rotate_ip", return_value=False),
):
lots = await scraper.fetch_all_secondary(
rooms_buckets=[(1,)],
price_cap_per_bucket=1400,
on_bucket=fake_on_bucket,
skip_buckets=skip_set,
)
# Probe (page=1) ВСЁ РАВНО вызывается на каждый seed-брекет (нужен для split-решения),
# но НЕ должно быть страниц пагинации (все бакеты в skip_buckets).
# Probe = page=1; пагинация пропущена ДО fetch остальных страниц → ровно len(_SEED) probe.
assert len(fetch_calls) == len(
_SEED
), f"Ожидалось {len(_SEED)} probe-вызовов, получено {len(fetch_calls)}: {fetch_calls}"
assert all(c[1] == 1 for c in fetch_calls), "Все вызовы должны быть probe page=1"
# on_bucket не вызывается для skip-бакетов
assert (
len(on_bucket_calls) == 0
), f"on_bucket не должен вызываться для skip-бакетов, got: {on_bucket_calls}"
# lots пустой — ничего не собрали (пагинация пропущена)
assert len(lots) == 0, f"Ожидалось 0 лотов для skip-бакетов, got {len(lots)}"
@pytest.mark.asyncio
async def test_on_bucket_receives_key_and_lots(scraper: CianScraper) -> None:
"""on_bucket получает (bucket_key, lots) — bucket_key в формате room_label:lo:hi."""
received: list[tuple[str, int]] = [] # (key, len(lots))
def fake_on_bucket(bucket_key: str, lots: list[ScrapedLot]) -> None:
received.append((bucket_key, len(lots)))
async def fake_fetch_page_html(
rooms: tuple, page: int, min_price: int | None, max_price: int | None
) -> str:
return f"<html>page={page}</html>"
def fake_extract_total_offers(html: str) -> int | None:
return 28 # 1 страница
def fake_parse_serp_html(html: str) -> list[ScrapedLot]:
import re
page_m = re.search(r"page=(\d+)", html)
page = int(page_m.group(1)) if page_m else 1
return [_make_lot(f"lot_p{page}_{i}") for i in range(5)]
with (
patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html),
patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers),
patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html),
patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock),
patch.object(scraper, "_rotate_ip", return_value=False),
):
await scraper.fetch_all_secondary(
rooms_buckets=[(2,)],
price_cap_per_bucket=1400,
on_bucket=fake_on_bucket,
)
assert len(received) == len(
_SEED
), f"Ожидалось {len(_SEED)} вызовов on_bucket, получено {len(received)}"
keys = [k for (k, _c) in received]
# Закрытые брекеты: hi = br_hi - 1. Открытый (последний): ':open'.
first_lo, first_hi = _SEED[0]
assert first_hi is not None
assert keys[0] == f"room2:{first_lo}:{first_hi - 1}", f"Неверный ключ: {keys[0]!r}"
last_lo, last_hi = _SEED[-1]
assert last_hi is None
assert keys[-1] == f"room2:{last_lo}:open", f"Открытый ключ неверен: {keys[-1]!r}"
assert all(count == 5 for (_k, count) in received)
# ── secondary_only фильтр ─────────────────────────────────────────────────────
@pytest.mark.asyncio
async def test_secondary_only_drops_novostroyki(scraper: CianScraper) -> None:
"""secondary_only=True: новостройки (listing_segment=='novostroyki') не попадают
в on_bucket и в итоговый список; вторичка и None-сегмент — пропускаются.
"""
on_bucket_calls: list[list[ScrapedLot]] = []
def fake_on_bucket(bucket_key: str, lots: list[ScrapedLot]) -> None:
on_bucket_calls.append(list(lots))
# Микс лотов: 2 новостройки + 2 вторички + 1 None-сегмент
mixed_lots = [
ScrapedLot(
source="cian",
source_url="https://ekb.cian.ru/sale/flat/nb_1/",
source_id="nb_1",
address="ЖК Новый",
price_rub=5_000_000,
listing_segment="novostroyki",
),
ScrapedLot(
source="cian",
source_url="https://ekb.cian.ru/sale/flat/nb_2/",
source_id="nb_2",
address="ЖК Новый 2",
price_rub=6_000_000,
listing_segment="novostroyki",
),
ScrapedLot(
source="cian",
source_url="https://ekb.cian.ru/sale/flat/vt_1/",
source_id="vt_1",
address="ул. Ленина",
price_rub=3_500_000,
listing_segment="vtorichka",
),
ScrapedLot(
source="cian",
source_url="https://ekb.cian.ru/sale/flat/vt_2/",
source_id="vt_2",
address="ул. Мира",
price_rub=4_000_000,
listing_segment="vtorichka",
),
ScrapedLot(
source="cian",
source_url="https://ekb.cian.ru/sale/flat/none_seg/",
source_id="none_seg",
address="ул. Пушкина",
price_rub=3_000_000,
listing_segment=None,
),
]
page_call_count = 0
async def fake_fetch_page_html(
rooms: tuple, page: int, min_price: int | None, max_price: int | None
) -> str:
nonlocal page_call_count
page_call_count += 1
return f"<html>page={page}</html>"
def fake_extract_total_offers(html: str) -> int | None:
return 5 # <= cap → leaf-бакет
def fake_parse_serp_html(html: str) -> list[ScrapedLot]:
# Первая страница отдаёт весь микс; вторая — пусто
import re
page_m = re.search(r"page=(\d+)", html)
page = int(page_m.group(1)) if page_m else 1
return mixed_lots if page == 1 else []
with (
patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html),
patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers),
patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html),
patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock),
patch.object(scraper, "_rotate_ip", return_value=False),
):
lots = await scraper.fetch_all_secondary(
rooms_buckets=[(1,)],
price_cap_per_bucket=1400,
secondary_only=True,
on_bucket=fake_on_bucket,
)
# Итоговый список: только vtorichka + None-сегмент (3 лота), без новостроек
assert len(lots) == 3, f"Ожидалось 3 лота (без novostroyki), получено {len(lots)}"
lot_ids = {lot.source_id for lot in lots}
assert "nb_1" not in lot_ids, "nb_1 (novostroyki) не должен присутствовать"
assert "nb_2" not in lot_ids, "nb_2 (novostroyki) не должен присутствовать"
assert "vt_1" in lot_ids, "vt_1 (vtorichka) должен присутствовать"
assert "vt_2" in lot_ids, "vt_2 (vtorichka) должен присутствовать"
assert "none_seg" in lot_ids, "none_seg (listing_segment=None) должен присутствовать"
# on_bucket вызывается на каждый seed-брекет (тот же микс на page=1 каждого),
# но новостройки отфильтрованы в каждом вызове.
assert len(on_bucket_calls) == len(
_SEED
), f"Ожидалось {len(_SEED)} вызовов on_bucket, получено {len(on_bucket_calls)}"
for call_lots in on_bucket_calls:
bucket_ids = {lot.source_id for lot in call_lots}
assert "nb_1" not in bucket_ids
assert "nb_2" not in bucket_ids
assert len(call_lots) == 3
@pytest.mark.asyncio
async def test_secondary_only_false_passes_all(scraper: CianScraper) -> None:
"""secondary_only=False: новостройки НЕ отфильтровываются — все лоты проходят."""
mixed_lots = [
ScrapedLot(
source="cian",
source_url="https://ekb.cian.ru/sale/flat/nb_1/",
source_id="nb_1",
address="ЖК Новый",
price_rub=5_000_000,
listing_segment="novostroyki",
),
ScrapedLot(
source="cian",
source_url="https://ekb.cian.ru/sale/flat/vt_1/",
source_id="vt_1",
address="ул. Ленина",
price_rub=3_500_000,
listing_segment="vtorichka",
),
]
async def fake_fetch_page_html(
rooms: tuple, page: int, min_price: int | None, max_price: int | None
) -> str:
return f"<html>page={page}</html>"
def fake_extract_total_offers(html: str) -> int | None:
return 2
def fake_parse_serp_html(html: str) -> list[ScrapedLot]:
import re
page_m = re.search(r"page=(\d+)", html)
page = int(page_m.group(1)) if page_m else 1
return mixed_lots if page == 1 else []
with (
patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html),
patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers),
patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html),
patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock),
patch.object(scraper, "_rotate_ip", return_value=False),
):
lots = await scraper.fetch_all_secondary(
rooms_buckets=[(1,)],
price_cap_per_bucket=1400,
secondary_only=False,
)
assert len(lots) == 2, f"Ожидалось 2 лота (фильтр выключен), получено {len(lots)}"
lot_ids = {lot.source_id for lot in lots}
assert "nb_1" in lot_ids, "nb_1 должен присутствовать при secondary_only=False"
assert "vt_1" in lot_ids
# ── #1393: tail-loss warning при превышении page cap ─────────────────────────
@pytest.mark.asyncio
async def test_walk_price_range_warns_on_page_cap_tail_loss(
scraper: CianScraper, caplog: pytest.LogCaptureFixture
) -> None:
"""#1393: когда totalOffers > max_pages_per_bucket × offers_per_page,
_walk_price_range логирует WARNING с числом потерянных офферов.
Сценарий: bracket < _MIN_BRACKET (guard срабатывает → force-paginate),
totalOffers = max_pages * offers_per_page + 50 → tail_loss = 50.
"""
max_pages_cap = 5 # нарочно мало, чтобы быстро дойти до cap
offers_per_page = _CIAN_OFFERS_PER_PAGE # 28
# total > cap (2000 > любой price_cap_per_bucket=1400) И bracket < _MIN_BRACKET
# → force-paginate path; pages_needed = ceil(2000/28) = 72 > max_pages_cap=5
total_offers = max_pages_cap * offers_per_page + 50 # 190; tail_loss=50
async def fake_fetch_page_html(
rooms: tuple, page: int, min_price: int | None, max_price: int | None
) -> str:
return f"<html>page={page}</html>"
def fake_extract_total_offers(html: str) -> int | None:
return total_offers
def fake_parse_serp_html(html: str) -> list[ScrapedLot]:
import re
page_m = re.search(r"page=(\d+)", html)
page = int(page_m.group(1)) if page_m else 1
return [_make_lot(f"lot_p{page}_{i}") for i in range(offers_per_page)]
with (
patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html),
patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers),
patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html),
patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock),
patch.object(scraper, "_rotate_ip", return_value=False),
caplog.at_level(logging.WARNING, logger="app.services.scrapers.cian"),
):
await scraper._walk_price_range(
rooms=(1,),
# Узкий диапазон: bracket < _MIN_BRACKET → force-paginate guard
lo=1_000_000,
hi=1_000_000 + _MIN_BRACKET - 1,
seen={},
price_cap_per_bucket=1400, # total(190) < 1400 → нет split, только leaf
max_pages_per_bucket=max_pages_cap,
)
# Должно быть предупреждение о tail-loss
tail_warnings = [
r for r in caplog.records if "tail loss" in r.message and r.levelno == logging.WARNING
]
assert tail_warnings, (
f"Ожидалось WARNING о tail loss, но записей нет. "
f"Все warnings: {[r.message for r in caplog.records if r.levelno == logging.WARNING]}"
)
# Сообщение должно содержать число потерянных офферов (50)
assert any(
"50" in r.message for r in tail_warnings
), f"WARNING должен содержать tail_loss=50, got: {[r.message for r in tail_warnings]}"
@pytest.mark.asyncio
async def test_walk_price_range_no_warning_when_within_page_cap(
scraper: CianScraper, caplog: pytest.LogCaptureFixture
) -> None:
"""#1393: когда totalOffers <= max_pages_per_bucket × offers_per_page,
tail-loss WARNING не должен эмитироваться (данные собраны полностью).
"""
max_pages_cap = 54
total_offers = 56 # ceil(56/28) = 2 страницы < 54 cap → нет потерь
async def fake_fetch_page_html(
rooms: tuple, page: int, min_price: int | None, max_price: int | None
) -> str:
return f"<html>page={page}</html>"
def fake_extract_total_offers(html: str) -> int | None:
return total_offers
def fake_parse_serp_html(html: str) -> list[ScrapedLot]:
import re
page_m = re.search(r"page=(\d+)", html)
page = int(page_m.group(1)) if page_m else 1
return [_make_lot(f"lot_p{page}_{i}") for i in range(10)]
with (
patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html),
patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers),
patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html),
patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock),
patch.object(scraper, "_rotate_ip", return_value=False),
caplog.at_level(logging.WARNING, logger="app.services.scrapers.cian"),
):
await scraper._walk_price_range(
rooms=(1,),
lo=0,
hi=_MAX_PRICE,
seen={},
price_cap_per_bucket=1400,
max_pages_per_bucket=max_pages_cap,
)
tail_warnings = [
r for r in caplog.records if "tail loss" in r.message and r.levelno == logging.WARNING
]
assert not tail_warnings, (
f"Не ожидалось WARNING о tail loss при total={total_offers} <= page cap, "
f"но получено: {[r.message for r in tail_warnings]}"
)
@pytest.mark.asyncio
async def test_walk_price_range_open_bracket_no_split(scraper: CianScraper) -> None:
"""Открытый брекет (hi=None) не делится даже при total > cap — leaf без maxprice.
bucket_key оканчивается на ':open'; maxprice в запросах всегда None (без потолка).
"""
on_bucket_keys: list[str] = []
seen_max: list[int | None] = []
async def fake_fetch_page_html(
rooms: tuple, page: int, min_price: int | None, max_price: int | None
) -> str:
seen_max.append(max_price)
return f"<html>page={page}</html>"
def fake_extract_total_offers(html: str) -> int | None:
return 5000 # > cap, но открытый брекет делить нельзя → leaf
def fake_parse_serp_html(html: str) -> list[ScrapedLot]:
import re
page_m = re.search(r"page=(\d+)", html)
page = int(page_m.group(1)) if page_m else 1
return [_make_lot(f"lot_p{page}")] if page == 1 else []
with (
patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html),
patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers),
patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html),
patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock),
patch.object(scraper, "_rotate_ip", return_value=False),
):
await scraper._walk_price_range(
rooms=(1,),
lo=250_000_000,
hi=None,
seen={},
price_cap_per_bucket=1400,
max_pages_per_bucket=54,
on_bucket=lambda k, lots: on_bucket_keys.append(k),
)
# Открытый брекет: ровно один leaf-бакет с ключом ':open', maxprice всегда None.
assert on_bucket_keys == [
"room1:250000000:open"
], f"Открытый брекет должен дать один ':open'-бакет, got {on_bucket_keys}"
assert all(
m is None for m in seen_max
), f"Открытый брекет: maxprice всегда None (без потолка), got {seen_max}"

630
tradein-mvp/backend/tests/scrapers/test_domclick.py
View file

@ -1,630 +0,0 @@
"""Unit-тесты для DomClick BFF scraper (#1968).
Тестируем:
- _extract_json: bare + <pre>-wrapped → dict; QRATOR-маркер → DomClickBlockedError
- _build_offers_url / _build_count_url: корректные параметры, "5+" → "5%2B",
offers имеет offset/limit, count — нет
- _map_item: маппинг realistic offer dict → ScrapedLot; price=0 → None; studio
- _is_geo_ok: гео-гард по bbox + offerRegionName
- _sweep_bucket: price-split при count > OFFSET_CAP
- DomClickScraper: source/name, fetch_around raises NotImplementedError
"""
from __future__ import annotations
import asyncio
from typing import Any
import pytest
from app.services.scrapers.domclick import (
LTE_MAX,
OFFSET_CAP,
DomClickScraper,
_build_count_url,
_build_offers_url,
_extract_json,
)
from app.services.scrapers.domclick_exceptions import DomClickBlockedError
# ── _extract_json ─────────────────────────────────────────────────────────────
def test_extract_json_bare_body() -> None:
"""Bare JSON body (без HTML-обёртки) корректно парсится."""
html = '{"result": {"items": [], "snippetsCount": 42}}'
data = _extract_json(html)
assert data["result"]["snippetsCount"] == 42
def test_extract_json_pre_wrapped() -> None:
"""JSON в <pre>-теге корректно парсится."""
html = (
"<html><head><title>DomClick</title></head>"
'<body><pre>{"result": {"items": [{"id": 1}]}}</pre></body></html>'
)
data = _extract_json(html)
assert data["result"]["items"][0]["id"] == 1
def test_extract_json_qrator_marker_raises() -> None:
"""HTML с QRATOR-маркером вызывает DomClickBlockedError."""
blocked_html = (
"<html><head><title>QRATOR protection</title></head>"
"<body>Access denied by bot_mitigation system</body></html>"
)
with pytest.raises(DomClickBlockedError):
_extract_json(blocked_html)
def test_extract_json_captcha_raises() -> None:
"""HTML с captcha-маркером вызывает DomClickBlockedError."""
html = "<html><body>Please solve the captcha to continue</body></html>"
with pytest.raises(DomClickBlockedError):
_extract_json(html)
def test_extract_json_no_json_raises() -> None:
"""HTML без JSON-объекта вызывает ValueError."""
with pytest.raises(ValueError, match="No JSON"):
_extract_json("<html><body>plain text</body></html>")
# ── _build_offers_url ────────────────────────────────────────────────────────
def test_build_offers_url_basic_params() -> None:
url = _build_offers_url("1", None, None, 0)
assert "deal_type=sale" in url
assert "category=living" in url
assert "offer_type=flat" in url
assert "rooms=1" in url
assert "offset=0" in url
assert "limit=20" in url
assert "sale_price__gte" not in url
assert "sale_price__lte" not in url
def test_build_offers_url_5plus_encoded() -> None:
"""rooms='5+' должен кодироваться как '5%2B' в URL."""
url = _build_offers_url("5+", None, None, 0)
assert "5%2B" in url
# literal '+' в URL не должно быть для значения rooms
assert "rooms=5+" not in url
def test_build_offers_url_with_price_range() -> None:
url = _build_offers_url("2", 3_000_000, 6_000_000, 40)
assert "sale_price__gte=3000000" in url
assert "sale_price__lte=6000000" in url
assert "offset=40" in url
def test_build_offers_url_ekb_guid_present() -> None:
url = _build_offers_url("3", None, None, 0)
assert "0d475b79-88de-4054-818c-37d8f9d0d440" in url
assert "aids=20561" in url
def test_build_offers_url_bff_host() -> None:
url = _build_offers_url("1", None, None, 0)
assert url.startswith("https://bff-search-web.domclick.ru/api/offers/v1?")
# ── _build_count_url ─────────────────────────────────────────────────────────
def test_build_count_url_no_offset_limit() -> None:
"""count url не должен содержать offset и limit."""
url = _build_count_url("2", None, None)
assert "offset" not in url
assert "limit" not in url
def test_build_count_url_5plus_encoded() -> None:
url = _build_count_url("5+", None, None)
assert "5%2B" in url
assert "rooms=5+" not in url
def test_build_count_url_with_price() -> None:
url = _build_count_url("1", 1_000_000, 4_000_000)
assert "sale_price__gte=1000000" in url
assert "sale_price__lte=4000000" in url
def test_build_count_url_bff_host() -> None:
url = _build_count_url("1", None, None)
assert url.startswith("https://bff-search-web.domclick.ru/api/offers/count/v1?")
# ── _map_item → ScrapedLot ────────────────────────────────────────────────────
_REALISTIC_ITEM: dict[str, Any] = {
"id": 2075671636,
"path": "https://domclick.ru/card/sale__flat__2075671636",
"location": {"lat": 56.838, "lon": 60.612},
"address": {"displayName": "Екатеринбург, улица Малышева, 1"},
"objectInfo": {"area": 53.5, "rooms": 2, "floor": 5, "isApartment": False},
"house": {"floors": 15, "buildYear": 2003},
"price": 5_200_000,
"squarePrice": 97196,
"flatComplex": {"id": 12345, "name": "ЖК Тест", "slug": "zhk-test"},
"isRosreestrApproved": True,
"publishedDate": "2026-06-01T10:00:00+05:00",
"updatedDate": "2026-06-15T12:00:00+05:00",
"lastPriceHistoryState": {"direction": "DECREASED"},
"offerRegionName": "Екатеринбург",
}
def test_map_item_basic_fields() -> None:
scraper = DomClickScraper.__new__(DomClickScraper)
scraper.parse_failures = 0
lot = scraper._map_item(_REALISTIC_ITEM)
assert lot is not None
assert lot.source == "domklik"
assert lot.source_id == "2075671636"
assert lot.source_url == "https://domclick.ru/card/sale__flat__2075671636"
assert lot.lat == pytest.approx(56.838)
assert lot.lon == pytest.approx(60.612)
assert lot.address == "Екатеринбург, улица Малышева, 1"
assert lot.rooms == 2
assert lot.area_m2 == pytest.approx(53.5)
assert lot.floor == 5
assert lot.total_floors == 15
assert lot.year_built == 2003
assert lot.price_rub == 5_200_000
assert lot.price_per_m2 == 97196
assert lot.listing_segment == "vtorichka"
def test_map_item_publish_date() -> None:
scraper = DomClickScraper.__new__(DomClickScraper)
scraper.parse_failures = 0
from datetime import date
lot = scraper._map_item(_REALISTIC_ITEM)
assert lot is not None
assert lot.publish_date == date(2026, 6, 1)
def test_map_item_raw_payload() -> None:
scraper = DomClickScraper.__new__(DomClickScraper)
scraper.parse_failures = 0
lot = scraper._map_item(_REALISTIC_ITEM)
assert lot is not None
assert lot.raw_payload is not None
assert lot.raw_payload["isRosreestrApproved"] is True
assert lot.raw_payload["flatComplex"]["name"] == "ЖК Тест"
assert lot.raw_payload["lastPriceHistoryState"]["direction"] == "DECREASED"
def test_map_item_studio_force_rooms_zero() -> None:
"""force_rooms=0 (студия-бакет) перебивает objectInfo.rooms."""
scraper = DomClickScraper.__new__(DomClickScraper)
scraper.parse_failures = 0
item = {**_REALISTIC_ITEM, "objectInfo": {"area": 28.0, "rooms": 1, "floor": 3}}
lot = scraper._map_item(item, force_rooms=0)
assert lot is not None
assert lot.rooms == 0
def test_map_item_zero_price_returns_none() -> None:
scraper = DomClickScraper.__new__(DomClickScraper)
scraper.parse_failures = 0
item = {**_REALISTIC_ITEM, "price": 0}
assert scraper._map_item(item) is None
def test_map_item_zero_sentinels_become_none() -> None:
"""buildYear/area/floor/total_floors == 0 (sentinel «неизвестно») → None.
rooms=0 НЕ затрагивается (валидная студия), но здесь force_rooms не задан и
objectInfo.rooms=2, так что rooms остаётся 2.
"""
scraper = DomClickScraper.__new__(DomClickScraper)
scraper.parse_failures = 0
item = {
**_REALISTIC_ITEM,
"objectInfo": {"area": 0, "rooms": 2, "floor": 0},
"house": {"floors": 0, "buildYear": 0},
}
lot = scraper._map_item(item)
assert lot is not None
assert lot.area_m2 is None
assert lot.floor is None
assert lot.total_floors is None
assert lot.year_built is None
assert lot.rooms == 2 # rooms 0-sentinel НЕ применяется
def test_map_item_rooms_zero_studio_preserved() -> None:
"""force_rooms=0 даёт rooms=0 (студия) — 0 здесь валиден, не sentinel."""
scraper = DomClickScraper.__new__(DomClickScraper)
scraper.parse_failures = 0
item = {**_REALISTIC_ITEM, "objectInfo": {"area": 24.0, "rooms": 0, "floor": 2}}
lot = scraper._map_item(item, force_rooms=0)
assert lot is not None
assert lot.rooms == 0
def test_map_item_price_per_m2_fallback() -> None:
"""Если squarePrice отсутствует, price_per_m2 вычисляется из price / area_m2."""
scraper = DomClickScraper.__new__(DomClickScraper)
scraper.parse_failures = 0
item = {**_REALISTIC_ITEM, "squarePrice": None}
lot = scraper._map_item(item)
assert lot is not None
assert lot.price_per_m2 == int(5_200_000 / 53.5)
# ── _map_item Layer A enrichment (description / repair / minhash / seller) ─────
_VALID_REPAIR_ENUM = {"needs_repair", "standard", "good", "excellent"}
def test_map_item_description_repair_minhash() -> None:
"""Offer с описанием → description set, repair_state выведен, minhash len 32."""
scraper = DomClickScraper.__new__(DomClickScraper)
scraper.parse_failures = 0
item = {
**_REALISTIC_ITEM,
"description": "Продаётся светлая квартира с евроремонтом, рядом метро.",
}
lot = scraper._map_item(item)
assert lot is not None
assert lot.description == "Продаётся светлая квартира с евроремонтом, рядом метро."
# «евроремонт» → good (см. repair_state_normalizer._TEXT_PATTERNS)
assert lot.repair_state == "good"
assert lot.repair_state in _VALID_REPAIR_ENUM
assert lot.description_minhash is not None
assert len(lot.description_minhash) == 32
def test_map_item_description_stripped() -> None:
"""description триммится; пустой/whitespace → None."""
scraper = DomClickScraper.__new__(DomClickScraper)
scraper.parse_failures = 0
item = {**_REALISTIC_ITEM, "description": " Без отделки, требуется ремонт. "}
lot = scraper._map_item(item)
assert lot is not None
assert lot.description == "Без отделки, требуется ремонт."
# «без отделк» → needs_repair
assert lot.repair_state == "needs_repair"
def test_map_item_enrichment_raw_payload() -> None:
"""Новые SERP-поля попадают в raw_payload (is_sber_collateral и т.д.)."""
scraper = DomClickScraper.__new__(DomClickScraper)
scraper.parse_failures = 0
item = {
**_REALISTIC_ITEM,
"isSberCollateral": True,
"hasDiscount": True,
"discountValue": 2.3,
"duplicatesOfferCount": 3,
"seller": {"company": "Этажи", "agent": "Иван Петров"},
}
lot = scraper._map_item(item)
assert lot is not None
assert lot.raw_payload is not None
assert lot.raw_payload["is_sber_collateral"] is True
assert lot.raw_payload["has_discount"] is True
assert lot.raw_payload["discount_value"] == 2.3
assert lot.raw_payload["duplicates_offer_count"] == 3
assert lot.raw_payload["seller"] == {"company": "Этажи", "agent": "Иван Петров"}
# agency_name извлекается из seller.company (приоритет company → agent)
assert lot.agency_name == "Этажи"
def test_map_item_agency_name_from_agent_dict() -> None:
"""seller.company отсутствует, agent — dict с name → agency_name = name."""
scraper = DomClickScraper.__new__(DomClickScraper)
scraper.parse_failures = 0
item = {**_REALISTIC_ITEM, "seller": {"agent": {"name": "Мария С."}}}
lot = scraper._map_item(item)
assert lot is not None
assert lot.agency_name == "Мария С."
assert lot.raw_payload is not None
assert lot.raw_payload["seller"] == {"agent": {"name": "Мария С."}}
def test_map_item_no_description_no_crash() -> None:
"""Offer без description: не падает, description/repair_state/minhash = None."""
scraper = DomClickScraper.__new__(DomClickScraper)
scraper.parse_failures = 0
# _REALISTIC_ITEM не содержит description/seller/enrichment-полей
lot = scraper._map_item(_REALISTIC_ITEM)
assert lot is not None
assert lot.description is None
assert lot.repair_state is None
assert lot.description_minhash is None
assert lot.agency_name is None
def test_map_item_enrichment_preserves_existing_raw_payload_and_drops_none() -> None:
"""Существующие raw_payload-ключи не теряются; None-enrich-ключи отброшены."""
scraper = DomClickScraper.__new__(DomClickScraper)
scraper.parse_failures = 0
lot = scraper._map_item(_REALISTIC_ITEM)
assert lot is not None
assert lot.raw_payload is not None
# Существующие ключи на месте
assert lot.raw_payload["isRosreestrApproved"] is True
assert lot.raw_payload["flatComplex"]["name"] == "ЖК Тест"
assert lot.raw_payload["lastPriceHistoryState"]["direction"] == "DECREASED"
assert "updatedDate" in lot.raw_payload
# None-значные enrich-ключи отсутствуют (компактный payload)
assert "is_sber_collateral" not in lot.raw_payload
assert "has_discount" not in lot.raw_payload
assert "discount_value" not in lot.raw_payload
assert "duplicates_offer_count" not in lot.raw_payload
assert "seller" not in lot.raw_payload
# ── _is_geo_ok — гео-гард ────────────────────────────────────────────────────
def test_is_geo_ok_valid_ekb_item() -> None:
scraper = DomClickScraper.__new__(DomClickScraper)
assert scraper._is_geo_ok(_REALISTIC_ITEM) is True
def test_is_geo_ok_wrong_region_name() -> None:
scraper = DomClickScraper.__new__(DomClickScraper)
item = {**_REALISTIC_ITEM, "offerRegionName": "Москва"}
assert scraper._is_geo_ok(item) is False
def test_is_geo_ok_out_of_bbox_lat() -> None:
scraper = DomClickScraper.__new__(DomClickScraper)
item = {**_REALISTIC_ITEM, "location": {"lat": 55.5, "lon": 60.6}}
assert scraper._is_geo_ok(item) is False
def test_is_geo_ok_out_of_bbox_lon() -> None:
scraper = DomClickScraper.__new__(DomClickScraper)
item = {**_REALISTIC_ITEM, "location": {"lat": 56.8, "lon": 61.5}}
assert scraper._is_geo_ok(item) is False
def test_is_geo_ok_missing_coords() -> None:
scraper = DomClickScraper.__new__(DomClickScraper)
item = {**_REALISTIC_ITEM, "location": {"lat": None, "lon": None}}
assert scraper._is_geo_ok(item) is False
# ── _sweep_bucket price-split ────────────────────────────────────────────────
async def test_sweep_bucket_price_split_boundaries(monkeypatch: pytest.MonkeyPatch) -> None:
"""_sweep_bucket: top-level unbounded split даёт contiguous non-overlapping ranges.
rooms="2" count>cap один раз, потом ≤cap → ровно 2 leaf-бакета, покрывающих
[0, LTE_MAX] без перекрытия и без зазора: [(0, mid), (mid+1, LTE_MAX)].
"""
call_count: dict[str, int] = {"n": 0}
paginate_calls: list[tuple[str, int | None, int | None]] = []
async def fake_count(
self: DomClickScraper,
fetcher: object,
rooms: str,
price_gte: int | None,
price_lte: int | None,
) -> int:
call_count["n"] += 1
# Первый вызов (unbounded) → over cap → price-split; остальные → под cap
return OFFSET_CAP + 1 if call_count["n"] == 1 else 1
async def fake_paginate(
self: DomClickScraper,
fetcher: object,
rooms: str,
price_gte: int | None,
price_lte: int | None,
seen_ids: set[str],
out_lots: list,
pages: int,
) -> None:
paginate_calls.append((rooms, price_gte, price_lte))
monkeypatch.setattr(DomClickScraper, "_count", fake_count)
monkeypatch.setattr(DomClickScraper, "_paginate", fake_paginate)
scraper = DomClickScraper.__new__(DomClickScraper)
scraper.parse_failures = 0
scraper.geo_filtered = 0
scraper.blocked = False
scraper.fetch_errors = 0
scraper.request_delay_sec = 0.0
await scraper._sweep_bucket(
fetcher=object(),
rooms="2",
price_gte=None,
price_lte=None,
seen_ids=set(),
out_lots=[],
pages=10,
)
# 1 count call (over cap) + 2 recursive count calls (≤ cap) = 3 total
assert call_count["n"] == 3
# Both sub-buckets paginated, с точными границами split'а unbounded-диапазона.
mid = (0 + LTE_MAX) // 2
assert paginate_calls == [("2", 0, mid), ("2", mid + 1, LTE_MAX)]
# Sanity: contiguous (нет зазора) + non-overlapping + покрывают весь хвост.
assert paginate_calls[0][2] + 1 == paginate_calls[1][1] # mid + 1
assert paginate_calls[0][1] == 0
assert paginate_calls[1][2] == LTE_MAX
async def test_sweep_bucket_zero_count_no_paginate(monkeypatch: pytest.MonkeyPatch) -> None:
"""_sweep_bucket при count==0 не вызывает _paginate."""
paginate_called = False
async def fake_count(self: DomClickScraper, *_args: object, **_kwargs: object) -> int:
return 0
async def fake_paginate(self: DomClickScraper, *_args: object, **_kwargs: object) -> None:
nonlocal paginate_called
paginate_called = True
monkeypatch.setattr(DomClickScraper, "_count", fake_count)
monkeypatch.setattr(DomClickScraper, "_paginate", fake_paginate)
scraper = DomClickScraper.__new__(DomClickScraper)
scraper.parse_failures = 0
scraper.geo_filtered = 0
scraper.blocked = False
scraper.fetch_errors = 0
scraper.request_delay_sec = 0.0
await scraper._sweep_bucket(
fetcher=object(),
rooms="3",
price_gte=None,
price_lte=None,
seen_ids=set(),
out_lots=[],
pages=5,
)
assert paginate_called is False
# ── _paginate resilience — mid-bucket parse error ────────────────────────────
class _FakeFetcher:
"""Стаб BrowserFetcher: возвращает заранее заданные ответы по порядку вызовов."""
def __init__(self, responses: list[str]) -> None:
self._responses = responses
self._idx = 0
async def fetch(self, _url: str) -> str:
resp = self._responses[min(self._idx, len(self._responses) - 1)]
self._idx += 1
return resp
async def test_paginate_mid_bucket_parse_error_preserves_lots() -> None:
"""Garbled JSON на page 2 → bucket breaks, page-1 lots survive, fetch_errors++.
Page 1 — валидный JSON с 20 items (PAGE_SIZE) → продолжаем; page 2 — мусор
(ValueError из _extract_json) → fetch_errors++ и break, уже собранные lots целы.
"""
import json
page1_items = [
{
"id": 1000 + i,
"path": f"https://domclick.ru/card/sale__flat__{1000 + i}",
"location": {"lat": 56.84, "lon": 60.6},
"address": {"displayName": "Екатеринбург, тест"},
"objectInfo": {"area": 40.0, "rooms": 1, "floor": 3},
"house": {"floors": 9, "buildYear": 2005},
"price": 4_000_000 + i,
"squarePrice": 100000,
"offerRegionName": "Екатеринбург",
}
for i in range(20)
]
page1 = json.dumps({"result": {"items": page1_items}})
garbled = "<html><body><pre>{not valid json at all</pre></body></html>"
fetcher = _FakeFetcher([page1, garbled])
scraper = DomClickScraper.__new__(DomClickScraper)
scraper.parse_failures = 0
scraper.geo_filtered = 0
scraper.blocked = False
scraper.fetch_errors = 0
scraper.request_delay_sec = 0.0
out_lots: list = []
await scraper._paginate(
fetcher=fetcher,
rooms="1",
price_gte=None,
price_lte=None,
seen_ids=set(),
out_lots=out_lots,
pages=10,
)
assert len(out_lots) == 20 # page-1 lots сохранены
assert scraper.fetch_errors == 1 # page-2 garbled посчитан
async def test_fetch_city_bucket_error_continues_to_next_bucket(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Ошибка в одном бакете НЕ убивает весь sweep — остальные бакеты отрабатывают."""
swept: list[str] = []
async def fake_sweep_bucket(
self: DomClickScraper,
fetcher: object,
rooms: str,
price_gte: int | None,
price_lte: int | None,
seen_ids: set[str],
out_lots: list,
pages: int,
) -> None:
swept.append(rooms)
if rooms == "2":
raise ValueError("garbled bucket")
# Заглушка BrowserFetcher через async context manager
class _BFStub:
async def __aenter__(self) -> _BFStub:
return self
async def __aexit__(self, *_a: object) -> None:
return None
async def fetch(self, _url: str) -> str:
return "{}"
import app.services.scrapers.browser_fetcher as bf_mod
monkeypatch.setattr(DomClickScraper, "_sweep_bucket", fake_sweep_bucket)
monkeypatch.setattr(bf_mod, "BrowserFetcher", lambda **_k: _BFStub())
scraper = DomClickScraper.__new__(DomClickScraper)
scraper.parse_failures = 0
scraper.geo_filtered = 0
scraper.blocked = False
scraper.fetch_errors = 0
scraper.request_delay_sec = 0.0
lots = await scraper.fetch_city(city_id=4, rooms=None, pages=5)
# Все 6 бакетов посещены несмотря на ошибку в "2"
assert swept == ["st", "1", "2", "3", "4", "5+"]
assert scraper.fetch_errors == 1
assert scraper.blocked is False
assert lots == []
# ── DomClickScraper — class attrs ─────────────────────────────────────────────
def test_scraper_source_name() -> None:
scraper = DomClickScraper.__new__(DomClickScraper)
scraper.parse_failures = 0
scraper.geo_filtered = 0
scraper.blocked = False
scraper.request_delay_sec = 0.0
assert scraper.name == "domklik"
assert scraper.source == "domklik"
assert scraper.base_url == "https://domclick.ru"
def test_fetch_around_raises() -> None:
scraper = DomClickScraper.__new__(DomClickScraper)
with pytest.raises(NotImplementedError, match="geo-radius"):
asyncio.run(scraper.fetch_around(56.8, 60.6))

7
tradein-mvp/backend/tests/scrapers/test_domclick_detail.py
View file

@ -24,9 +24,10 @@ test_cian_detail.py — в проекте нет live-DB фикстуры, ид
scraper_kit-миграции, 0 runtime importers) — тесты переведены на kit-эквивалент
`scraper_kit.providers.domclick.detail` без изменения покрытия. Exceptions тоже
переведены на `scraper_kit.domclick_exceptions` (kit держит свою независимую копию,
`parse_detail_html`(kit) поднимает именно её — legacy `app.services.scrapers.
domclick_exceptions` остаётся живым модулем для `domclick.py`, но здесь больше не
подходящий класс для `pytest.raises`).
`parse_detail_html`(kit) поднимает именно её). Legacy `app.services.scrapers.
domclick`/`domclick_exceptions` удалены целиком (#2397 Part E2, 0 runtime
importers) — `scraper_kit.domclick_exceptions` остаётся единственным живым классом
для `pytest.raises`.
"""
from __future__ import annotations

381
tradein-mvp/backend/tests/test_avito_houses_dedup.py
View file

@ -1,381 +0,0 @@
"""Tests for avito_houses dedup fix — _persist_house делегирует в match_or_create_house.
Проверяет:
1. Два HouseInfo с разными ext_id но одинаковым адресом и координатами
→ _persist_house возвращает ОДИН house_id (не плодит дубль).
2. Enrichment-поля (rating_score, developer_name) реально записываются в UPDATE.
3. _persist_house не вызывает db.commit().
4. upsert_house и _upsert_house_with_url делегируют в _persist_house.
5. #1820: два id-less URL (ext_id=0, без адреса) НЕ создают house_sources(avito,"0").
Использует тот же паттерн mock-DB что в test_matching.py и test_same_building_match.py.
"""
import json
import os
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
from unittest.mock import MagicMock, patch
from app.services.scrapers.avito_houses import (
HouseCatalogEnrichment,
HouseInfo,
_persist_house,
_upsert_house_with_url,
save_house_catalog_enrichment,
upsert_house,
)
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _make_house(ext_id: int = 111, address: str = "ул. Ленина, 5") -> HouseInfo:
return HouseInfo(
ext_id=ext_id,
ext_id_hash="hash_" + str(ext_id),
short_address=address,
full_address="Свердловская обл., Екатеринбург, " + address,
lat=56.8378,
lon=60.5946,
year_built=2010,
house_type="monolith",
total_floors=16,
passenger_elevators=2,
cargo_elevators=1,
has_concierge=True,
closed_yard=False,
material_floors="concrete",
hot_water="central",
parking_type="underground",
has_playground=True,
developer_name="ПИК",
developer_key="pik",
infrastructure_summary="Школа, детсад",
infrastructure_walk_distance="5 мин",
rating_score=4.5,
rating_string="4.5 из 5",
reviews_count=120,
rating_distribution=[{"score": 5, "count": 80}],
map_pins=[{"type": "metro", "name": "Уральская"}],
raw_characteristics=[{"title": "Год постройки", "value": "2010"}],
)
def _make_db_for_persist(canonical_id: int = 42) -> MagicMock:
"""Mock DB для _persist_house: match_or_create_house ходит через db.execute,
поэтому патчим match_or_create_house целиком и проверяем вызов UPDATE.
"""
db = MagicMock()
# db.execute для UPDATE houses SET ... — просто возвращаем None (rowcount не важен)
db.execute.return_value = MagicMock()
return db
# ---------------------------------------------------------------------------
# Test 1: два HouseInfo с разными ext_id, одинаковый адрес → один house_id
# ---------------------------------------------------------------------------
def test_persist_house_same_address_returns_same_id():
"""Два HouseInfo с разными ext_id но одинаковым адресом → один canonical house_id.
Моделируем случай: первый вызов (ext_id=111) → match_or_create_house вернул 42 (new).
Второй вызов (ext_id=222, тот же адрес) → match_or_create_house вернул 42 (fingerprint).
Оба _persist_house должны вернуть 42.
"""
db = MagicMock()
db.execute.return_value = MagicMock()
h1 = _make_house(ext_id=111)
h2 = _make_house(ext_id=222) # другой ext_id, тот же адрес/координаты
with patch(
"app.services.scrapers.avito_houses.match_or_create_house",
side_effect=[(42, 1.0, "new"), (42, 0.9, "fingerprint")],
) as mock_match:
id1 = _persist_house(db, h1, "https://avito.ru/house/111")
id2 = _persist_house(db, h2, "https://avito.ru/house/222")
assert id1 == 42, f"Первый вызов должен вернуть 42, получили {id1}"
assert id2 == 42, f"Второй вызов должен вернуть тот же 42, получили {id2}"
assert id1 == id2, "Оба вызова должны вернуть один house_id (нет дубля)"
assert mock_match.call_count == 2
# ---------------------------------------------------------------------------
# Test 2: enrichment-поля записываются в UPDATE
# ---------------------------------------------------------------------------
def test_persist_house_enrichment_fields_written():
"""Enrichment-поля (rating_score, developer_name) передаются в UPDATE-запрос."""
db = MagicMock()
db.execute.return_value = MagicMock()
h = _make_house(ext_id=111)
with patch(
"app.services.scrapers.avito_houses.match_or_create_house",
return_value=(42, 1.0, "source_exact"),
):
_persist_house(db, h, "https://avito.ru/house/111")
# db.execute должен быть вызван 1 раз (UPDATE houses SET ...)
assert (
db.execute.call_count == 1
), f"Ожидался 1 вызов db.execute (UPDATE), получили {db.execute.call_count}"
# Проверяем параметры bind
call_args = db.execute.call_args
sql_obj = call_args[0][0]
bind = call_args[0][1]
assert "UPDATE houses" in str(sql_obj), "SQL должен содержать UPDATE houses"
assert bind["rating_score"] == 4.5, f"rating_score={bind.get('rating_score')!r}"
assert bind["developer_name"] == "ПИК", f"developer_name={bind.get('developer_name')!r}"
assert bind["house_id"] == 42, f"house_id={bind.get('house_id')!r}"
# avito_id_hash должен быть передан
assert bind["avito_id_hash"] == "hash_111", f"avito_id_hash={bind.get('avito_id_hash')!r}"
# rating_distribution должен быть JSON-строкой
rd = bind.get("rating_distribution")
assert isinstance(rd, str), f"rating_distribution должен быть str (JSON), получили {type(rd)}"
assert json.loads(rd) == [{"score": 5, "count": 80}]
# ---------------------------------------------------------------------------
# Test 3: _persist_house не вызывает db.commit()
# ---------------------------------------------------------------------------
def test_persist_house_no_commit():
"""_persist_house не должен вызывать db.commit() — коммитит вызывающий."""
db = MagicMock()
db.execute.return_value = MagicMock()
h = _make_house()
with patch(
"app.services.scrapers.avito_houses.match_or_create_house",
return_value=(7, 1.0, "new"),
):
_persist_house(db, h, "")
db.commit.assert_not_called()
# ---------------------------------------------------------------------------
# Test 4: upsert_house делегирует в _persist_house (с пустым url)
# ---------------------------------------------------------------------------
def test_upsert_house_delegates_to_persist_house():
"""upsert_house(db, h) → _persist_house(db, h, "") → возвращает house_id."""
db = MagicMock()
db.execute.return_value = MagicMock()
h = _make_house()
with patch(
"app.services.scrapers.avito_houses.match_or_create_house",
return_value=(55, 1.0, "new"),
):
result = upsert_house(db, h)
assert result == 55
# Проверяем что UPDATE был вызван с url="" (пустая строка — нет url в этом пути)
bind = db.execute.call_args[0][1]
assert bind["url"] == "", f"url должен быть пустой строкой, получили {bind.get('url')!r}"
# ---------------------------------------------------------------------------
# Test 5: _upsert_house_with_url делегирует в _persist_house (с реальным url)
# ---------------------------------------------------------------------------
def test_upsert_house_with_url_delegates_to_persist_house():
"""_upsert_house_with_url(db, h, url) → _persist_house(db, h, url) → house_id."""
db = MagicMock()
db.execute.return_value = MagicMock()
h = _make_house()
url = "https://www.avito.ru/catalog/houses/ekb/dom-1234"
with patch(
"app.services.scrapers.avito_houses.match_or_create_house",
return_value=(99, 1.0, "new"),
):
result = _upsert_house_with_url(db, h, url)
assert result == 99
# Проверяем что правильный url передан в UPDATE
bind = db.execute.call_args[0][1]
assert bind["url"] == url, f"url должен быть {url!r}, получили {bind.get('url')!r}"
# ---------------------------------------------------------------------------
# Test 6: match_or_create_house вызывается с правильными аргументами
# ---------------------------------------------------------------------------
def test_persist_house_calls_match_with_correct_args():
"""_persist_house передаёт правильные аргументы в match_or_create_house."""
db = MagicMock()
db.execute.return_value = MagicMock()
h = _make_house(ext_id=333)
url = "https://avito.ru/333"
with patch(
"app.services.scrapers.avito_houses.match_or_create_house",
return_value=(10, 1.0, "new"),
) as mock_match:
_persist_house(db, h, url)
mock_match.assert_called_once_with(
db,
ext_source="avito",
ext_id="333",
address=h.short_address or h.full_address,
lat=h.lat,
lon=h.lon,
year_built=h.year_built,
source_url=url,
)
# ---------------------------------------------------------------------------
# Test 7: COALESCE — адрес/координаты не затираются через UPDATE параметры
# (проверяем что bind содержит address/lat/lon для COALESCE в SQL)
# ---------------------------------------------------------------------------
def test_persist_house_coalesce_fields_in_bind():
"""UPDATE передаёт address/lat/lon в bind для COALESCE(houses.<col>, CAST(:x AS ...))."""
db = MagicMock()
db.execute.return_value = MagicMock()
h = _make_house()
with patch(
"app.services.scrapers.avito_houses.match_or_create_house",
return_value=(1, 1.0, "new"),
):
_persist_house(db, h, "")
bind = db.execute.call_args[0][1]
assert bind["address"] == h.short_address
assert bind["lat"] == h.lat
assert bind["lon"] == h.lon
assert bind["year_built"] == h.year_built
# ---------------------------------------------------------------------------
# Test 7b: #2003 — house_class передаётся в UPDATE bind (COALESCE fill-only)
# ---------------------------------------------------------------------------
def test_persist_house_house_class_in_bind():
"""UPDATE передаёт house_class в bind для COALESCE(CAST(:house_class AS text), houses.*)."""
db = MagicMock()
db.execute.return_value = MagicMock()
h = _make_house()
h.house_class = "comfort"
with patch(
"app.services.scrapers.avito_houses.match_or_create_house",
return_value=(1, 1.0, "new"),
):
_persist_house(db, h, "")
bind = db.execute.call_args[0][1]
assert bind["house_class"] == "comfort"
sql_obj = db.execute.call_args[0][0]
assert "house_class" in str(sql_obj)
def test_persist_house_house_class_none_does_not_null_via_coalesce():
"""house_class=None в bind — SQL использует COALESCE(new, existing), не голый overwrite."""
db = MagicMock()
db.execute.return_value = MagicMock()
h = _make_house() # house_class остаётся дефолтным None
with patch(
"app.services.scrapers.avito_houses.match_or_create_house",
return_value=(1, 1.0, "new"),
):
_persist_house(db, h, "")
bind = db.execute.call_args[0][1]
assert bind["house_class"] is None
sql_obj = str(db.execute.call_args[0][0])
# SQL должен явно защищать существующее значение через COALESCE, а не
# безусловный overwrite (который занулил бы значение от другого источника).
assert "COALESCE(CAST(:house_class AS text), houses.house_class)" in sql_obj
# ---------------------------------------------------------------------------
# Test 8: #1820 — ext_id=0 graceful-skip гард
# Два id-less URL (ext_id=0, без адреса) → match_or_create_house НЕ вызывается,
# house_sources("avito","0") НЕ создаётся, возвращается sentinel dict с house_id=0.
# ---------------------------------------------------------------------------
def test_save_house_catalog_enrichment_skips_zero_ext_id_no_address():
"""#1820: ext_id=0 + нет адреса → гард пропускает персистирование canonical house."""
db = MagicMock()
# Создаём два id-less graceful-skip enrichment (как _parse_development_or_skip возвращает)
enrichment_a = HouseCatalogEnrichment(
house_url="https://www.avito.ru/catalog/novostroyki/dom-A/",
house=HouseInfo(ext_id=0),
)
enrichment_b = HouseCatalogEnrichment(
house_url="https://www.avito.ru/catalog/novostroyki/dom-B/",
house=HouseInfo(ext_id=0),
)
with patch(
"app.services.scrapers.avito_houses.match_or_create_house",
) as mock_match:
result_a = save_house_catalog_enrichment(db, enrichment_a)
result_b = save_house_catalog_enrichment(db, enrichment_b)
# match_or_create_house НЕ должен вызываться — гард сработал до него
mock_match.assert_not_called()
# Возвращается sentinel dict с house_id=0, не плодятся дубли
assert result_a["house_id"] == 0, f"ожидали house_id=0, получили {result_a['house_id']}"
assert result_b["house_id"] == 0, f"ожидали house_id=0, получили {result_b['house_id']}"
# db.commit() НЕ вызывается (нет реальной транзакции)
db.commit.assert_not_called()
def test_save_house_catalog_enrichment_with_address_persists_despite_zero_ext_id():
"""#1820: ext_id=0 НО есть адрес → НЕ пропускаем (developmentPage с адресом, без avitoId)."""
db = MagicMock()
db.execute.return_value = MagicMock()
enrichment = HouseCatalogEnrichment(
house_url="https://www.avito.ru/catalog/novostroyki/dom-C/",
house=HouseInfo(ext_id=0, short_address="ул. Ленина, 1"),
)
with patch(
"app.services.scrapers.avito_houses.match_or_create_house",
return_value=(77, 0.9, "fingerprint"),
) as mock_match:
result = save_house_catalog_enrichment(db, enrichment)
# Адрес есть → персистим через fingerprint tier
mock_match.assert_called_once()
assert result["house_id"] == 77

790
tradein-mvp/backend/tests/test_avito_houses_parse.py
View file

@ -1,790 +0,0 @@
"""Offline smoke for avito_houses parser.
Uses minimal JSON fixture mimicking Avito __preloadedState__ structure.
Не требует сети, БД или curl_cffi — тестирует только чистый парсер.
"""
from datetime import date
from pathlib import Path
import pytest
from app.services.scrapers.avito_houses import (
HouseCatalogEnrichment,
HouseInfo,
HouseReview,
MiniSerpListing,
PlacementHistoryItem,
RecommendationItem,
SellerInfo,
_extract_preloaded_state,
_parse_ru_date,
parse_houses_state,
)
FIXTURES_DIR = Path(__file__).parent / "fixtures"
# ---------------------------------------------------------------------------
# Minimal __preloadedState__ fixture (Python dict — не raw HTML)
# ---------------------------------------------------------------------------
MINIMAL_STATE: dict = {
"data": {
"data": {
"page": {
"placeholders": [
{"type": "breadcrumbs"},
{"type": "title"},
{"type": "gallery", "props": {}},
{
"type": "housePage",
"props": {
"developmentData": {
"avitoId": 3171365,
"id": "MTE3LjM3MDg5MA",
"title": "Akademika Postovskogo 17a",
"address": "ул. Постовского, 17а",
"fullAddress": (
"Свердловская обл., Екатеринбург, ул. Постовского, 17а"
),
"coords": {"lat": 56.790699, "lng": 60.580191},
"aboutDevelopment": {
"expandParams": {
"items": [
{
"title": "Параметры дома",
"params": [
{"type": "Год постройки", "value": "2019"},
{"type": "Этажей", "value": "25"},
{"type": "Тип дома", "value": "Монолитный"},
{"type": "Пассажирский лифт", "value": "2"},
{"type": "Консьерж", "value": "Да"},
{"type": "Закрытый двор", "value": "Да"},
{"type": "Детская площадка", "value": "Да"},
],
},
]
}
},
"developer": {"name": "Атомстройкомплекс", "key": "atomstroy"},
"mapPreview": {
"distance": "в 5 минутах ходьбы",
"objects": "школа, парк, аптека",
"pins": [{"lat": 56.79, "lng": 60.58, "type": "school"}],
},
"ratingBadge": {"info": {"score": 4.666, "scoreString": "4,7"}},
"ratingSummaryStat": {
"ratingStat": [
{"score": 5, "count": 4},
{"score": 4, "count": 2},
],
"reviewCount": 6,
},
}
},
},
{
"type": "reviews",
"props": {
"entries": [
{
"type": "rating",
"value": {
"id": 12345,
"author": {"title": "Иван"},
"reviewTitle": "Хорошо",
"score": 5,
"modelExperience": "Живу в своей квартире",
"rated": "9 октября 2025",
"textSections": [
{"title": "", "text": "Общий отзыв"},
{"title": "Преимущества", "text": "Парк рядом"},
{"title": "Недостатки", "text": "Парковки мало"},
],
},
},
{"type": "pages", "value": {"nextPageUrl": "/reviews?page=2"}},
]
},
},
{
"type": "miniSerp",
"props": {
"items": [
{
"id": 7986882804,
"title": "3-к. квартира",
"description": "Описание",
"url": "/ekaterinburg/kvartiry/test_7986882804",
"price": "11 990 000 ₽",
"geo": {"content": "Чкаловская, 2,7 км", "colors": ["#cf2734"]},
"seller": {
"name": "DOMRF66",
"type": "Компания",
"from": "На Авито с июля 2012",
"url": "/brands/domrf66",
"logos": [],
},
},
]
},
},
{
"type": "housePlacementHistory",
"props": {
"items": [
{
"id": 7000000001,
"title": "1-к. квартира",
"startPrice": 5000000,
"startPriceDate": 1697000000,
"lastPrice": 4900000,
"lastPriceDate": 1715000000,
"exposure": 210,
"itemImage": {},
},
]
},
},
{"type": "uxFeedback"},
{
"type": "recommendations",
"props": {
"items": [
{
"title": "Похожая",
"priceRange": "от 6 млн",
"additionalInfo": ["Адрес 1"],
"url": "/rec1",
"images": [{"208x156": "https://avito.ru/img1.jpg"}],
},
]
},
},
{"type": "padding"},
]
}
}
}
}
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def test_ru_date_parse() -> None:
assert _parse_ru_date("9 октября 2025") == date(2025, 10, 9)
assert _parse_ru_date("31 декабря 2020") == date(2020, 12, 31)
assert _parse_ru_date("1 января 2000") == date(2000, 1, 1)
assert _parse_ru_date(None) is None
assert _parse_ru_date("bogus") is None
assert _parse_ru_date("") is None
# ---------------------------------------------------------------------------
# Main parse test
# ---------------------------------------------------------------------------
def test_parse_minimal_state() -> None:
e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
assert isinstance(e, HouseCatalogEnrichment)
assert e.house_url == "/catalog/houses/ekaterinburg/test/3171365"
def test_house_info() -> None:
e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
h = e.house
assert isinstance(h, HouseInfo)
assert h.ext_id == 3171365
assert h.ext_id_hash == "MTE3LjM3MDg5MA"
assert h.title == "Akademika Postovskogo 17a"
assert h.short_address == "ул. Постовского, 17а"
assert h.full_address == "Свердловская обл., Екатеринбург, ул. Постовского, 17а"
assert h.lat == 56.790699
assert h.lon == 60.580191
def test_house_expand_params() -> None:
e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
h = e.house
assert h.year_built == 2019
assert h.total_floors == 25
assert h.house_type == "monolith"
assert h.passenger_elevators == 2
assert h.has_concierge is True
assert h.closed_yard is True
assert h.has_playground is True
def test_house_developer_and_rating() -> None:
e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
h = e.house
assert h.developer_name == "Атомстройкомплекс"
assert h.developer_key == "atomstroy"
assert h.rating_score == 4.666
assert h.rating_string == "4,7"
assert h.reviews_count == 6
assert len(h.rating_distribution) == 2
def test_house_map_and_infra() -> None:
e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
h = e.house
assert h.infrastructure_walk_distance == "в 5 минутах ходьбы"
assert h.infrastructure_summary == "школа, парк, аптека"
assert len(h.map_pins) == 1
assert h.map_pins[0]["type"] == "school"
def test_house_raw_characteristics() -> None:
e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
# raw_characteristics = полный expandParams items (1 категория)
assert len(e.house.raw_characteristics) == 1
assert e.house.raw_characteristics[0]["title"] == "Параметры дома"
def test_reviews() -> None:
e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
assert len(e.reviews) == 1
assert e.reviews_next_page_url == "/reviews?page=2"
r = e.reviews[0]
assert isinstance(r, HouseReview)
assert r.ext_review_id == 12345
assert r.author_name == "Иван"
assert r.review_title == "Хорошо"
assert r.score == 5
assert r.model_experience == "Живу в своей квартире"
assert r.rated_date == date(2025, 10, 9)
assert r.text_main == "Общий отзыв"
assert r.text_pros == "Парк рядом"
assert r.text_cons == "Парковки мало"
assert r.raw_payload is not None
def test_mini_serp() -> None:
e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
assert len(e.mini_serp) == 1
m = e.mini_serp[0]
assert isinstance(m, MiniSerpListing)
assert m.ext_item_id == 7986882804
assert m.price_rub == 11990000
assert m.metro_text == "Чкаловская, 2,7 км"
assert m.metro_color == "#cf2734"
assert m.description == "Описание"
def test_seller_extraction() -> None:
e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
m = e.mini_serp[0]
assert m.seller is not None
s = m.seller
assert isinstance(s, SellerInfo)
assert s.ext_seller_id == "domrf66"
assert s.name == "DOMRF66"
assert s.seller_type == "Компания"
assert s.on_platform_since == "На Авито с июля 2012"
assert s.profile_url == "/brands/domrf66"
def test_placement_history() -> None:
e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
assert len(e.placement_history) == 1
p = e.placement_history[0]
assert isinstance(p, PlacementHistoryItem)
assert p.ext_item_id == "7000000001"
assert p.title == "1-к. квартира"
assert p.start_price == 5000000
assert p.last_price == 4900000
assert p.exposure_days == 210
# removed_date ВСЕГДА None — виджет не отдаёт!
assert p.removed_date is None
# raw_payload не содержит itemImage (скипается)
assert p.raw_payload is not None
assert "itemImage" not in p.raw_payload
# start_price_date и last_price_date — из unix timestamp
assert isinstance(p.start_price_date, date)
assert isinstance(p.last_price_date, date)
def test_recommendations() -> None:
e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
assert len(e.recommendations) == 1
rec = e.recommendations[0]
assert isinstance(rec, RecommendationItem)
assert rec.title == "Похожая"
assert rec.price_text == "от 6 млн"
assert rec.address == "Адрес 1"
assert rec.url == "/rec1"
assert rec.image_url == "https://avito.ru/img1.jpg"
def test_missing_optional_widgets() -> None:
"""Парсер не падает если опциональные виджеты отсутствуют."""
state_minimal: dict = {
"data": {
"data": {
"page": {
"placeholders": [
{
"type": "housePage",
"props": {
"developmentData": {
"avitoId": 9999,
"coords": {"lat": 56.0, "lng": 60.0},
"aboutDevelopment": {"expandParams": {"items": []}},
"developer": {},
"mapPreview": {},
"ratingBadge": {"info": {}},
"ratingSummaryStat": {},
}
},
},
]
}
}
}
}
e = parse_houses_state(state_minimal, "/catalog/houses/test/9999")
assert e.house.ext_id == 9999
assert e.reviews == []
assert e.mini_serp == []
assert e.placement_history == []
assert e.recommendations == []
assert e.reviews_next_page_url is None
def test_invalid_state_raises() -> None:
"""ValueError если placeholders не найден."""
with pytest.raises(ValueError, match="placeholders"):
parse_houses_state({}, "/catalog/houses/test/1")
def test_missing_house_page_raises() -> None:
"""ValueError если housePage виджет отсутствует (вторичка — /catalog/houses/)."""
state: dict = {
"data": {
"data": {
"page": {
"placeholders": [
{"type": "breadcrumbs"},
]
}
}
}
}
with pytest.raises(ValueError, match="housePage"):
parse_houses_state(state, "/catalog/houses/test/1")
def test_novostroyka_minimal_development_page() -> None:
"""Новостройка: виджет 'developmentPage' парсится в partial HouseInfo без ValueError."""
state: dict = {
"data": {
"data": {
"page": {
"placeholders": [
{"type": "reNavigation"},
{
"type": "developmentPage",
"props": {
"developmentPage": {
"developmentData": {
"avitoId": 12879380,
"id": "MGDKHN70PQ",
"title": "ЖК Береговой",
"address": "ул. Щербакова, жилые дома",
"fullAddress": "",
"coords": {"lat": 56.77005, "lng": 60.67244},
"developer": {"name": "Талан", "key": "talan-1"},
"mapPreview": {
"distance": "в 15 минутах ходьбы",
"objects": "школа, детский сад",
"pins": [{"lat": 56.77, "lng": 60.67}],
},
"aboutDevelopment": {
"tabs": [
{
"title": "Параметры",
"expandParams": {
"items": [
{
"params": [
{
"type": "Тип дома",
"value": "Монолит",
},
{
"type": "Этажность",
"value": "от 4 до 24",
},
{
"type": (
"Пассажирских лифтов"
),
"value": "42",
},
{
"type": "Грузовых лифтов",
"value": "60",
},
]
}
]
},
}
]
},
}
}
},
},
{"type": "uxFeedback"},
]
}
}
}
}
url = "/catalog/novostroyki/ekaterinburg/zhk_beregovoy/12879380"
e = parse_houses_state(state, url)
assert isinstance(e, HouseCatalogEnrichment)
assert e.house_url == url
h = e.house
assert h.ext_id == 12879380
assert h.ext_id_hash == "MGDKHN70PQ"
assert h.title == "ЖК Береговой"
assert h.short_address == "ул. Щербакова, жилые дома"
assert h.lat == 56.77005
assert h.lon == 60.67244
assert h.house_type == "monolith" # "Монолит" → monolith
assert h.passenger_elevators == 42
assert h.cargo_elevators == 60
# Этажность = диапазон "от 4 до 24" → int-каст None (только в raw_characteristics)
assert h.total_floors is None
assert h.year_built is None # дом строится, года постройки нет
assert h.developer_name == "Талан"
assert h.developer_key == "talan-1"
assert h.infrastructure_summary == "школа, детский сад"
assert h.infrastructure_walk_distance == "в 15 минутах ходьбы"
assert len(h.map_pins) == 1
# Нет рейтинга/отзывов/истории у новостройки
assert h.rating_score is None
assert e.reviews == []
assert e.placement_history == []
assert e.recommendations == []
def test_novostroyka_no_development_page_graceful_skip() -> None:
"""Новостройка без developmentPage/housePage → graceful skip (НЕ ValueError)."""
state: dict = {
"data": {
"data": {
"page": {
"placeholders": [
{"type": "reNavigation"},
{"type": "uxFeedback"},
]
}
}
}
}
url = "/catalog/novostroyki/ekaterinburg/zhk_unknown/55667788"
e = parse_houses_state(state, url)
assert isinstance(e, HouseCatalogEnrichment)
assert e.house_url == url
# ext_id извлечён из хвоста URL
assert e.house.ext_id == 55667788
assert e.reviews == []
assert e.placement_history == []
def test_novostroyka_real_fixture_end_to_end() -> None:
"""Реальная фикстура новостройки: parse не кидает ValueError, отдаёт partial-обогащение."""
html = (FIXTURES_DIR / "avito_novostroyka_12879380.html").read_text(encoding="utf-8")
state = _extract_preloaded_state(html)
assert state is not None, "URL-encoded __preloadedState__ должен распарситься"
url = "/catalog/novostroyki/ekaterinburg/semeynyy_kvartal_beregovoy/12879380"
e = parse_houses_state(state, url)
assert isinstance(e, HouseCatalogEnrichment)
h = e.house
assert h.ext_id == 12879380
assert h.ext_id_hash # base64 internal id
assert h.house_type == "monolith"
assert h.passenger_elevators is not None and h.passenger_elevators > 0
assert h.lat is not None and h.lon is not None
assert h.developer_name # застройщик присутствует
# Новостройка: нет отзывов/истории/года постройки
assert h.year_built is None
assert e.reviews == []
assert e.placement_history == []
def test_house_type_normalization() -> None:
"""Все варианты типа дома нормализуются корректно."""
from app.services.scrapers.avito_houses import _normalize_house_type
assert _normalize_house_type("Монолитный") == "monolith"
assert _normalize_house_type("Панельный") == "panel"
assert _normalize_house_type("Кирпичный") == "brick"
assert _normalize_house_type("Монолитно-кирпичный") == "monolith_brick"
assert _normalize_house_type("Блочный") == "block"
assert _normalize_house_type("Деревянный") == "wood"
assert _normalize_house_type("Неизвестный") == "other"
assert _normalize_house_type(None) is None
def test_house_class_normalization() -> None:
"""Все варианты класса жилья нормализуются корректно (#2003)."""
from app.services.scrapers.avito_houses import _normalize_house_class
assert _normalize_house_class("Комфорт") == "comfort"
assert _normalize_house_class("Бизнес") == "business"
assert _normalize_house_class("Премиум") == "premium"
assert _normalize_house_class("Эконом") == "economy"
assert _normalize_house_class("Комфорт+") == "comfort_plus"
assert _normalize_house_class("Элит") == "elite"
# Регистронезависимость + пробелы по краям
assert _normalize_house_class(" комфорт ") == "comfort"
# Неизвестное значение → None (НЕ "other" — house_class точный ANY(...) фильтр)
assert _normalize_house_class("Неизвестный класс") is None
assert _normalize_house_class(None) is None
assert _normalize_house_class("") is None
def test_house_class_parsed_from_novostroyka_expand_params() -> None:
""" "Класс жилья" из developmentPage expandParams → HouseInfo.house_class (#2003)."""
state: dict = {
"data": {
"data": {
"page": {
"placeholders": [
{
"type": "developmentPage",
"props": {
"developmentPage": {
"developmentData": {
"avitoId": 42,
"coords": {"lat": 56.8, "lng": 60.6},
"developer": {},
"mapPreview": {},
"aboutDevelopment": {
"tabs": [
{
"expandParams": {
"items": [
{
"params": [
{
"type": "Класс жилья",
"value": "Комфорт",
},
]
}
]
}
}
]
},
}
}
},
},
]
}
}
}
}
e = parse_houses_state(state, "/catalog/novostroyki/ekaterinburg/test/42")
assert e.house.house_class == "comfort"
def test_house_class_absent_on_secondary_listing() -> None:
"""Вторичка (housePage) не отдаёт "Класс жилья" → house_class остаётся None."""
e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
assert e.house.house_class is None
def test_strip_price() -> None:
"""_strip_price корректно извлекает числа из строк Avito."""
from app.services.scrapers.avito_houses import _strip_price
assert _strip_price("11 990 000 ₽") == 11990000
assert _strip_price("5 000 000 ₽") == 5000000
assert _strip_price(None) is None
assert _strip_price("") is None
def test_mini_serp_no_seller() -> None:
"""Объявление без продавца не вызывает ошибку."""
state: dict = {
"data": {
"data": {
"page": {
"placeholders": [
{
"type": "housePage",
"props": {
"developmentData": {
"avitoId": 111,
"coords": {"lat": 56.0, "lng": 60.0},
"aboutDevelopment": {"expandParams": {"items": []}},
"developer": {},
"mapPreview": {},
"ratingBadge": {"info": {}},
"ratingSummaryStat": {},
}
},
},
{
"type": "miniSerp",
"props": {
"items": [
{
"id": 12345,
"title": "1-к.",
"url": "/test",
"price": "3 000 000 ₽",
"geo": {"content": "Центр", "colors": []},
},
]
},
},
]
}
}
}
}
e = parse_houses_state(state, "/catalog/houses/test/111")
assert len(e.mini_serp) == 1
assert e.mini_serp[0].seller is None
assert e.mini_serp[0].price_rub == 3000000
# ---------------------------------------------------------------------------
# Real-HTML fixture (regression for issue #566 — state-blob encoding/shape change)
# ---------------------------------------------------------------------------
#
# Fixture: реальная страница /catalog/houses/ekaterinburg/ul_yumasheva_6/115188,
# захвачена curl_cffi chrome120 (2026-05-28). Avito сменил формат __preloadedState__
# на URL-encoded литерал + обернул виджеты в placeholders[0]['widgets'] +
# вложил developmentData под props['housePage']. Этот тест ловит регрессию декода/shape.
def test_extract_preloaded_state_from_real_html() -> None:
"""_extract_preloaded_state декодирует текущий URL-encoded __preloadedState__."""
html = (FIXTURES_DIR / "avito_houses_sample.html").read_text(encoding="utf-8")
state = _extract_preloaded_state(html)
assert state is not None, "URL-encoded __preloadedState__ должен распарситься"
# Avito SSR обёртка — успешный ответ
assert state["data"]["status"] == 200
# Inner path до placeholders сохранился
placeholders = state["data"]["data"]["page"]["placeholders"]
assert isinstance(placeholders, list) and placeholders, "placeholders непустой массив"
def test_parse_real_fixture_end_to_end() -> None:
"""Фикстура HTML → _extract_preloaded_state → parse_houses_state → HouseInfo.ext_id."""
html = (FIXTURES_DIR / "avito_houses_sample.html").read_text(encoding="utf-8")
state = _extract_preloaded_state(html)
assert state is not None
e = parse_houses_state(state, "/catalog/houses/ekaterinburg/ul_yumasheva_6/115188")
assert isinstance(e, HouseCatalogEnrichment)
# housePage распарсился — ext_id (avitoId) присутствует и совпадает с URL
assert e.house.ext_id == 115188
assert e.house.ext_id_hash # base64 internal id присутствует
# expandParams распарсились (год постройки реальной фикстуры — 2016)
assert e.house.year_built == 2016
assert e.house.total_floors == 17
assert e.house.house_type == "monolith"
assert e.house.lat is not None and e.house.lon is not None
def test_extract_preloaded_state_jsparse_fallback() -> None:
"""Старая форма JSON.parse("...") всё ещё декодируется (fallback-ветка)."""
inner = '{\\"data\\":{\\"data\\":{\\"page\\":{\\"placeholders\\":[]}}}}'
html = f'<script>window.__preloadedState__ = JSON.parse("{inner}");</script>'
state = _extract_preloaded_state(html)
assert state is not None
assert state["data"]["data"]["page"]["placeholders"] == []
def test_extract_preloaded_state_missing_returns_none() -> None:
"""Нет маркера → None (а не исключение)."""
assert _extract_preloaded_state("<html><body>no state here</body></html>") is None
# ---------------------------------------------------------------------------
# Real-HTML fixture (regression for issue #1789 — MFE props nesting)
# ---------------------------------------------------------------------------
#
# Fixture: реальная страница /catalog/houses/ekaterinburg/ul_soni_morozovoy_190/572538,
# захвачена браузер-сервисом (2026-06-19). Avito перенёс данные виджетов на уровень
# глубже — под ключ с именем виджета:
# reviews: props['reviews']['entries'] (было props['entries'])
# miniSerp: props['miniSerp']['items'] (было props['items'])
# housePlacementHistory: props['housePlacementHistory']['items'] (было props['items'])
# recommendations: props['recommendations']['items'] (было props['items'])
# Из-за этого reviews/placement/seller-обогащение НЕ долетало в БД (0 строк).
# Этот тест ловит регрессию шейпа props.
def test_parse_mfe_fixture_widgets_reach_db() -> None:
"""Фикстура MFE: reviews/placement/miniSerp/house_type/year_built распарсены непусто."""
html = (FIXTURES_DIR / "avito_house_572538.html").read_text(encoding="utf-8")
state = _extract_preloaded_state(html)
assert state is not None, "URL-encoded __preloadedState__ должен распарситься"
e = parse_houses_state(state, "/catalog/houses/ekaterinburg/ul_soni_morozovoy_190/572538")
# housePage — тип дома и год постройки долетают
assert e.house.ext_id == 572538
assert e.house.house_type is not None, "house_type должен парситься из expandParams"
assert e.house.house_type == "brick"
assert e.house.year_built == 1994
assert e.house.reviews_count == 3
# reviews — >=1 отзыв с текстом, автором, оценкой
assert len(e.reviews) >= 1, "reviews должны распарситься из props['reviews']['entries']"
r = e.reviews[0]
assert r.ext_review_id, "у отзыва должен быть id"
assert r.author_name, "у отзыва должен быть автор"
assert r.score is not None, "у отзыва должна быть оценка"
assert r.text_main, "у отзыва должен быть основной текст (textSections)"
# placement — >=1 item из props['housePlacementHistory']['items']
assert len(e.placement_history) >= 1, "placement должен распарситься"
p = e.placement_history[0]
assert p.ext_item_id, "у placement item должен быть id"
assert p.start_price is not None
assert p.last_price is not None
assert isinstance(p.last_price_date, date)
# miniSerp — >=1 объявление из props['miniSerp']['items']
assert len(e.mini_serp) >= 1, "miniSerp должен распарситься"
m = e.mini_serp[0]
assert m.ext_item_id
assert m.price_rub is not None

6
tradein-mvp/backend/tests/test_backfill_wave2.py
View file

@ -234,9 +234,10 @@ async def test_backfill_yandex_addresses_error_on_non_200():
@pytest.mark.asyncio
async def test_cian_price_history_saves_changes():
"""Happy path: fetch returns DetailEnrichment with price_changes → saved."""
from scraper_kit.providers.cian.detail import DetailEnrichment
from app.services.cian_price_history import backfill_cian_price_history
from app.services.scraper_adapters import RealScraperConfig
from app.services.scrapers.cian_detail import DetailEnrichment
enrichment = DetailEnrichment(
cian_id=123456,
@ -290,9 +291,10 @@ async def test_cian_price_history_saves_changes():
@pytest.mark.asyncio
async def test_cian_price_history_skips_no_changes():
"""Fetch succeeds but no price_changes → skipped (not error)."""
from scraper_kit.providers.cian.detail import DetailEnrichment
from app.services.cian_price_history import backfill_cian_price_history
from app.services.scraper_adapters import RealScraperConfig
from app.services.scrapers.cian_detail import DetailEnrichment
enrichment = DetailEnrichment(cian_id=111, price_changes=[])

830
tradein-mvp/backend/tests/test_cian_detail.py
View file

@ -1,830 +0,0 @@
"""Tests for cian_detail.py — Stage 5 of CianScraper v1."""
from __future__ import annotations
from pathlib import Path
from unittest.mock import AsyncMock, MagicMock
import pytest
from app.services.scrapers.cian_detail import (
DetailEnrichment,
_extract_agent_profile,
_extract_ceiling_height,
_extract_price_changes,
_extract_repair_state,
_extract_repair_type,
_parse_views,
fetch_detail,
save_detail_enrichment,
)
_FIXTURE_DIR = Path(__file__).parent / "fixtures"
# ── _parse_views ──────────────────────────────────────────────────────────────
def test_parse_views_space_formatted():
assert _parse_views("1 234") == 1234
def test_parse_views_plain():
assert _parse_views("567") == 567
def test_parse_views_none():
assert _parse_views(None) is None
def test_parse_views_invalid():
assert _parse_views("invalid") is None
def test_parse_views_comma_formatted():
assert _parse_views("1,234") == 1234
# ── _extract_repair_state — структурное поле в приоритете, fallback на текст (#622) ──
def test_repair_state_structured_wins_over_description():
"""repairType присутствует → используется он, описание игнорируется."""
offer = {
"repairType": "cosmetic", # → standard
"description": "Шикарный дизайнерский ремонт", # инференс дал бы excellent
}
assert _extract_repair_state(offer) == "standard"
def test_repair_state_falls_back_to_description_when_structured_missing():
"""repairType отсутствует → инференс из описания."""
offer = {
"repairType": None,
"description": "Сделан свежий евроремонт, заезжай и живи",
}
assert _extract_repair_state(offer) == "good"
def test_repair_state_none_when_both_missing():
"""Нет structured и нет сигнала в тексте → None (не фабрикуем)."""
offer = {"description": "Светлая квартира рядом с парком"}
assert _extract_repair_state(offer) is None
# ── _extract_ceiling_height — offer.building.ceilingHeight, не offer.ceilingHeight (#1791) ──
def test_ceiling_height_from_building_node():
"""На detail-странице высота лежит в offer.building.ceilingHeight."""
offer = {"ceilingHeight": None, "building": {"ceilingHeight": "3.0"}}
assert _extract_ceiling_height(offer) == 3.0
def test_ceiling_height_falls_back_to_top_level():
"""Если building.ceilingHeight отсутствует — fallback на offer.ceilingHeight."""
offer = {"ceilingHeight": "2.75", "building": {}}
assert _extract_ceiling_height(offer) == 2.75
def test_ceiling_height_none_when_absent_everywhere():
assert _extract_ceiling_height({"building": {}}) is None
assert _extract_ceiling_height({}) is None
# ── _extract_repair_type / state — offer.decoration fallback (#1791) ──────────
def test_repair_type_prefers_repair_type_then_decoration():
assert _extract_repair_type({"repairType": "euro", "decoration": "cosmetic"}) == "euro"
assert _extract_repair_type({"repairType": None, "decoration": "preFine"}) == "preFine"
assert _extract_repair_type({}) is None
def test_repair_state_from_decoration_when_repair_type_missing():
"""offer.decoration='preFine' → needs_repair (через нормализатор)."""
offer = {"repairType": None, "decoration": "preFine", "description": ""}
assert _extract_repair_state(offer) == "needs_repair"
# ── _extract_price_changes ────────────────────────────────────────────────────
def test_extract_price_changes_normalizes():
"""curl_cffi flat format: {changeTime, price, diffPercent}."""
offer = {
"priceChanges": [
{"changeTime": "2026-04-01T00:00:00Z", "price": 5000000, "diffPercent": -2.5},
{"changeTime": "2026-04-10T00:00:00Z", "price": 4900000, "diffPercent": -2.0},
]
}
result = _extract_price_changes(offer, {}, {})
assert len(result) == 2
assert result[0]["change_time"] == "2026-04-01T00:00:00Z"
assert result[0]["price_rub"] == 5000000
assert result[0]["diff_percent"] == -2.5
def test_extract_price_changes_handles_empty_offer():
assert _extract_price_changes({}, {}, {}) == []
def test_extract_price_changes_handles_none_value():
assert _extract_price_changes({"priceChanges": None}, {}, {}) == []
def test_extract_price_changes_falls_back_to_state():
state = {
"priceChanges": [
{"changeTime": "2026-03-01T00:00:00Z", "price": 6000000, "diffPercent": 0},
]
}
result = _extract_price_changes({}, {}, state)
assert len(result) == 1
assert result[0]["price_rub"] == 6000000
def test_extract_price_changes_skips_non_dict_entries():
offer = {"priceChanges": [{"changeTime": "2026-04-01T00:00:00Z", "price": 5000000}, "bad"]}
result = _extract_price_changes(offer, {}, {})
assert len(result) == 1
def test_extract_price_changes_offer_data_browser_format():
"""Browser-rendered HTML: offerData.priceChanges with nested priceData.price."""
offer_data = {
"priceChanges": [
{
"changeTime": "2026-06-05T15:32:41Z",
"priceData": {"currency": "rur", "price": 13300000},
},
{
"changeTime": "2026-06-03T08:05:04Z",
"priceData": {"currency": "rur", "price": 13600000},
},
]
}
result = _extract_price_changes({}, offer_data, {})
assert len(result) == 2
assert result[0]["change_time"] == "2026-06-05T15:32:41Z"
assert result[0]["price_rub"] == 13300000
assert result[1]["price_rub"] == 13600000
def test_extract_price_changes_offer_wins_over_offer_data():
"""offer.priceChanges takes priority over offerData.priceChanges (flat format wins)."""
offer = {
"priceChanges": [
{"changeTime": "2026-04-01T00:00:00Z", "price": 5000000},
]
}
offer_data = {
"priceChanges": [
{"changeTime": "2026-06-01T00:00:00Z", "priceData": {"price": 9000000}},
]
}
result = _extract_price_changes(offer, offer_data, {})
assert len(result) == 1
assert result[0]["price_rub"] == 5000000
# ── _extract_agent_profile ────────────────────────────────────────────────────
def test_extract_agent_profile_from_agent_key():
offer = {
"agent": {
"id": 12345,
"companyName": "Новосёл",
"isPro": True,
"skills": [{"id": 1, "name": "Продажа"}],
"accountType": "specialist",
}
}
profile = _extract_agent_profile(offer)
assert profile is not None
assert profile["ext_agent_id"] == 12345
assert profile["company_name"] == "Новосёл"
assert profile["is_pro"] is True
assert profile["account_type"] == "specialist"
def test_extract_agent_profile_from_user_key():
offer = {
"user": {
"cianUserId": 99999,
"companyName": "Агентство Икс",
"isPro": False,
"accountType": "agency",
}
}
profile = _extract_agent_profile(offer)
assert profile is not None
assert profile["ext_agent_id"] == 99999
assert profile["company_name"] == "Агентство Икс"
def test_extract_agent_profile_returns_none_when_no_agent():
assert _extract_agent_profile({}) is None
assert _extract_agent_profile({"agent": {}, "user": {}}) is None
# ── DetailEnrichment defaults ─────────────────────────────────────────────────
def test_dataclass_defaults():
enrich = DetailEnrichment()
assert enrich.cian_id is None
assert enrich.price_changes == []
assert enrich.raw_sister_states == {}
assert enrich.bti_data is None
assert enrich.agent_profile is None
assert enrich.newbuilding_data is None
# ── fetch_detail integration (mocked) ────────────────────────────────────────
@pytest.mark.asyncio
async def test_fetch_detail_returns_none_on_no_state(monkeypatch):
"""If state extraction returns None → fetch_detail returns None gracefully."""
monkeypatch.setattr(
"app.services.scrapers.cian_detail.extract_state",
lambda html, mfe, key: None,
)
response = MagicMock()
response.status_code = 200
response.text = "<html></html>"
session = MagicMock()
session.get = AsyncMock(return_value=response)
session.close = AsyncMock()
result = await fetch_detail("https://ekb.cian.ru/sale/flat/123/", session=session)
assert result is None
@pytest.mark.asyncio
async def test_fetch_detail_returns_none_on_non_200(monkeypatch):
"""Non-200 HTTP response → returns None, no exception raised."""
response = MagicMock()
response.status_code = 403
response.text = ""
session = MagicMock()
session.get = AsyncMock(return_value=response)
session.close = AsyncMock()
result = await fetch_detail("https://ekb.cian.ru/sale/flat/456/", session=session)
assert result is None
@pytest.mark.asyncio
async def test_fetch_detail_extracts_core_fields(monkeypatch):
"""Verify core fields are extracted correctly from mocked state."""
fake_state = {
"offerData": {
"offer": {
"cianId": 327830237,
"windowsViewType": "yardAndStreet",
"separateWcsCount": 1,
"combinedWcsCount": 0,
"ceilingHeight": 2.65,
"repairType": "cosmetic",
"priceChanges": [
{
"changeTime": "2026-04-01T00:00:00Z",
"price": 5500000,
"diffPercent": -1.8,
}
],
"agent": {
"id": 7654321,
"companyName": "Первичка Плюс",
"isPro": True,
"skills": [],
"accountType": "specialist",
},
},
"stats": {
"totalViewsFormattedString": "1 042",
"todayViewsFormattedString": "7",
},
}
}
monkeypatch.setattr(
"app.services.scrapers.cian_detail.extract_state",
lambda html, mfe, key: fake_state if key == "defaultState" else None,
)
monkeypatch.setattr(
"app.services.scrapers.cian_detail.extract_all_states",
lambda html: {},
)
response = MagicMock()
response.status_code = 200
response.text = "<html></html>"
session = MagicMock()
session.get = AsyncMock(return_value=response)
session.close = AsyncMock()
result = await fetch_detail("https://ekb.cian.ru/sale/flat/327830237/", session=session)
assert result is not None
assert result.cian_id == 327830237
assert result.windows_view_type == "yardAndStreet"
assert result.separate_wcs_count == 1
assert result.combined_wcs_count == 0
assert result.ceiling_height == 2.65
assert result.repair_type == "cosmetic"
assert result.views_total == 1042
assert result.views_today == 7
assert len(result.price_changes) == 1
assert result.price_changes[0]["price_rub"] == 5500000
assert result.agent_profile is not None
assert result.agent_profile["ext_agent_id"] == 7654321
assert result.agent_profile["company_name"] == "Первичка Плюс"
@pytest.mark.asyncio
async def test_fetch_detail_extracts_bti_from_sister_states(monkeypatch):
"""BTI sister state is captured into bti_data."""
fake_state = {
"offerData": {
"offer": {
"cianId": 111111,
}
}
}
fake_all_states = {
"frontend-offer-card": {
"bti": {
"houseData": {
"seriesName": "П-44Т",
"floorsCount": 17,
"flatCount": 120,
}
}
}
}
monkeypatch.setattr(
"app.services.scrapers.cian_detail.extract_state",
lambda html, mfe, key: fake_state if key == "defaultState" else None,
)
monkeypatch.setattr(
"app.services.scrapers.cian_detail.extract_all_states",
lambda html: fake_all_states,
)
response = MagicMock()
response.status_code = 200
response.text = "<html></html>"
session = MagicMock()
session.get = AsyncMock(return_value=response)
session.close = AsyncMock()
result = await fetch_detail("https://ekb.cian.ru/sale/flat/111111/", session=session)
assert result is not None
assert result.bti_data is not None
assert result.bti_data["seriesName"] == "П-44Т"
assert "bti" in result.raw_sister_states
@pytest.mark.asyncio
async def test_fetch_detail_extracts_newbuilding_link(monkeypatch):
"""Newbuilding reference is captured from offer.newbuilding."""
fake_state = {
"offerData": {
"offer": {
"cianId": 222222,
"newbuilding": {
"id": 54016,
"name": "Екатерининский парк",
"url": "https://zhk-ekaterininskiy-park-ekb-i.cian.ru/",
},
}
}
}
monkeypatch.setattr(
"app.services.scrapers.cian_detail.extract_state",
lambda html, mfe, key: fake_state if key == "defaultState" else None,
)
monkeypatch.setattr(
"app.services.scrapers.cian_detail.extract_all_states",
lambda html: {},
)
response = MagicMock()
response.status_code = 200
response.text = "<html></html>"
session = MagicMock()
session.get = AsyncMock(return_value=response)
session.close = AsyncMock()
result = await fetch_detail("https://ekb.cian.ru/sale/flat/222222/", session=session)
assert result is not None
assert result.newbuilding_data is not None
assert result.newbuilding_data["id"] == 54016
# ── kitchen_area_m2 extraction ────────────────────────────────────────────────
@pytest.mark.asyncio
async def test_fetch_detail_extracts_kitchen_area(monkeypatch):
"""kitchenArea из offer → kitchen_area_m2 в DetailEnrichment."""
fake_state = {
"offerData": {
"offer": {
"cianId": 333333,
"kitchenArea": 9.5,
}
}
}
monkeypatch.setattr(
"app.services.scrapers.cian_detail.extract_state",
lambda html, mfe, key: fake_state if key == "defaultState" else None,
)
monkeypatch.setattr(
"app.services.scrapers.cian_detail.extract_all_states",
lambda html: {},
)
response = MagicMock()
response.status_code = 200
response.text = "<html></html>"
session = MagicMock()
session.get = AsyncMock(return_value=response)
session.close = AsyncMock()
result = await fetch_detail("https://ekb.cian.ru/sale/flat/333333/", session=session)
assert result is not None
assert result.kitchen_area_m2 == 9.5
@pytest.mark.asyncio
async def test_fetch_detail_kitchen_area_none_when_missing(monkeypatch):
"""Если kitchenArea отсутствует в offer — kitchen_area_m2 остаётся None."""
fake_state = {
"offerData": {
"offer": {
"cianId": 444444,
}
}
}
monkeypatch.setattr(
"app.services.scrapers.cian_detail.extract_state",
lambda html, mfe, key: fake_state if key == "defaultState" else None,
)
monkeypatch.setattr(
"app.services.scrapers.cian_detail.extract_all_states",
lambda html: {},
)
response = MagicMock()
response.status_code = 200
response.text = "<html></html>"
session = MagicMock()
session.get = AsyncMock(return_value=response)
session.close = AsyncMock()
result = await fetch_detail("https://ekb.cian.ru/sale/flat/444444/", session=session)
assert result is not None
assert result.kitchen_area_m2 is None
@pytest.mark.asyncio
async def test_fetch_detail_extracts_description(monkeypatch):
"""offer.description → DetailEnrichment.description (для persist в listings)."""
fake_state = {
"offerData": {
"offer": {
"cianId": 555555,
"description": "Продаётся уютная квартира с видом на парк.",
}
}
}
monkeypatch.setattr(
"app.services.scrapers.cian_detail.extract_state",
lambda html, mfe, key: fake_state if key == "defaultState" else None,
)
monkeypatch.setattr(
"app.services.scrapers.cian_detail.extract_all_states",
lambda html: {},
)
response = MagicMock()
response.status_code = 200
response.text = "<html></html>"
session = MagicMock()
session.get = AsyncMock(return_value=response)
session.close = AsyncMock()
result = await fetch_detail("https://ekb.cian.ru/sale/flat/555555/", session=session)
assert result is not None
assert result.description == "Продаётся уютная квартира с видом на парк."
# ── save_detail_enrichment — kitchen_area_m2 persisted ───────────────────────
def test_save_detail_enrichment_writes_kitchen_area():
"""save_detail_enrichment передаёт kitchen_area_m2 в UPDATE listings."""
db = MagicMock()
# Мокаем fetchone для snapshot-запроса — price_rub=None → snapshot не пишется
db.execute.return_value.fetchone.return_value = None
enrichment = DetailEnrichment(
windows_view_type=None,
separate_wcs_count=None,
combined_wcs_count=None,
ceiling_height=None,
repair_type=None,
repair_state=None,
kitchen_area_m2=8.3,
views_total=None,
views_today=None,
)
save_detail_enrichment(db, listing_id=999, enrichment=enrichment)
# Проверяем что UPDATE listings вызван с параметром ka=8.3
update_call_params = db.execute.call_args_list[0][0][1]
assert update_call_params["ka"] == 8.3
def test_save_detail_enrichment_kitchen_area_none_passthrough():
"""Если kitchen_area_m2 = None — параметр ka=None, COALESCE сохраняет существующее."""
db = MagicMock()
db.execute.return_value.fetchone.return_value = None
enrichment = DetailEnrichment(kitchen_area_m2=None)
save_detail_enrichment(db, listing_id=1001, enrichment=enrichment)
update_call_params = db.execute.call_args_list[0][0][1]
assert update_call_params["ka"] is None
# ── save_detail_enrichment — description + detail_enriched_at stamp ───────────
def test_save_detail_enrichment_writes_description():
"""save_detail_enrichment передаёт description в UPDATE listings."""
db = MagicMock()
db.execute.return_value.fetchone.return_value = None
enrichment = DetailEnrichment(description="Светлая квартира рядом с парком")
save_detail_enrichment(db, listing_id=1002, enrichment=enrichment)
update_call_params = db.execute.call_args_list[0][0][1]
assert update_call_params["descr"] == "Светлая квартира рядом с парком"
def test_save_detail_enrichment_stamps_detail_enriched_at():
"""UPDATE listings содержит detail_enriched_at = NOW() — иначе строка
переобогащается каждый run (detail_enriched_at IS NULL filter)."""
db = MagicMock()
db.execute.return_value.fetchone.return_value = None
enrichment = DetailEnrichment(kitchen_area_m2=8.3)
save_detail_enrichment(db, listing_id=1003, enrichment=enrichment)
update_sql = str(db.execute.call_args_list[0][0][0])
assert "detail_enriched_at = NOW()" in update_sql
# ── save_detail_enrichment — diff_percent клампится перед INSERT (#2277) ──────
# offer_price_history.diff_percent — NUMERIC(8,2) (миграция 131). Этот прямой
# INSERT не проходит через БД-триггер record_listing_price_change() (тот
# срабатывает только на UPDATE listings) — garbage/экстремальный diff_percent
# из скрапленного Cian JSON иначе переполняет колонку.
def _insert_diff_param(db: MagicMock) -> float | None:
insert_calls = [c for c in db.execute.call_args_list if "offer_price_history" in str(c[0][0])]
assert len(insert_calls) == 1
return insert_calls[0][0][1]["diff"]
def test_save_detail_enrichment_clamps_extreme_positive_diff():
db = MagicMock()
db.execute.return_value.fetchone.return_value = None
enrichment = DetailEnrichment(
price_changes=[
{"change_time": "2026-04-01T00:00:00Z", "price_rub": 5000000, "diff_percent": 5000000}
]
)
save_detail_enrichment(db, listing_id=2001, enrichment=enrichment)
assert _insert_diff_param(db) == 999999.99
def test_save_detail_enrichment_clamps_extreme_negative_diff():
db = MagicMock()
db.execute.return_value.fetchone.return_value = None
enrichment = DetailEnrichment(
price_changes=[
{
"change_time": "2026-04-01T00:00:00Z",
"price_rub": 5000000,
"diff_percent": -5000000,
}
]
)
save_detail_enrichment(db, listing_id=2002, enrichment=enrichment)
assert _insert_diff_param(db) == -999999.99
def test_save_detail_enrichment_normal_diff_unchanged():
db = MagicMock()
db.execute.return_value.fetchone.return_value = None
enrichment = DetailEnrichment(
price_changes=[
{"change_time": "2026-04-01T00:00:00Z", "price_rub": 5000000, "diff_percent": -2.5}
]
)
save_detail_enrichment(db, listing_id=2003, enrichment=enrichment)
assert _insert_diff_param(db) == -2.5
def test_save_detail_enrichment_none_diff_unchanged():
db = MagicMock()
db.execute.return_value.fetchone.return_value = None
enrichment = DetailEnrichment(
price_changes=[
{"change_time": "2026-04-01T00:00:00Z", "price_rub": 5000000, "diff_percent": None}
]
)
save_detail_enrichment(db, listing_id=2004, enrichment=enrichment)
assert _insert_diff_param(db) is None
# ── fetch_detail browser_fetcher param ───────────────────────────────────────
@pytest.mark.asyncio
async def test_fetch_detail_browser_fetcher_uses_browser_html(monkeypatch):
"""browser_fetcher provided → HTML comes from BrowserFetcher.fetch, not curl_cffi.
Simulates the browser-rendered format where priceChanges is at offerData level
with nested priceData.price (verified on prod 2026-06-16).
"""
fake_state = {
"offerData": {
"offer": {
"cianId": 328647045,
},
# Browser-rendered: priceChanges at offerData level, priceData.price format
"priceChanges": [
{
"changeTime": "2026-06-05T15:32:41Z",
"priceData": {"currency": "rur", "price": 13300000},
},
{
"changeTime": "2026-06-03T08:05:04Z",
"priceData": {"currency": "rur", "price": 13600000},
},
],
}
}
monkeypatch.setattr(
"app.services.scrapers.cian_detail.extract_state",
lambda html, mfe, key: fake_state if key == "defaultState" else None,
)
monkeypatch.setattr(
"app.services.scrapers.cian_detail.extract_all_states",
lambda html: {},
)
# Mock BrowserFetcher: fetch() returns HTML tagged so we can verify it was called.
mock_bf = MagicMock()
mock_bf.fetch = AsyncMock(return_value="<html>browser-rendered</html>")
result = await fetch_detail(
"https://ekb.cian.ru/sale/flat/328647045/",
browser_fetcher=mock_bf,
)
assert result is not None
assert result.cian_id == 328647045
# price_changes populated from browser-rendered HTML via offerData.priceChanges
assert len(result.price_changes) == 2
assert result.price_changes[0]["price_rub"] == 13300000
assert result.price_changes[0]["change_time"] == "2026-06-05T15:32:41Z"
assert result.price_changes[1]["price_rub"] == 13600000
# BrowserFetcher.fetch was called with the offer URL
mock_bf.fetch.assert_called_once_with("https://ekb.cian.ru/sale/flat/328647045/")
@pytest.mark.asyncio
async def test_fetch_detail_browser_fetcher_no_curl_session_used(monkeypatch):
"""When browser_fetcher is given, curl_cffi AsyncSession is NOT instantiated."""
fake_state = {"offerData": {"offer": {"cianId": 555555}}}
monkeypatch.setattr(
"app.services.scrapers.cian_detail.extract_state",
lambda html, mfe, key: fake_state if key == "defaultState" else None,
)
monkeypatch.setattr(
"app.services.scrapers.cian_detail.extract_all_states",
lambda html: {},
)
# Patch AsyncSession so we can assert it was NOT called.
mock_session_cls = MagicMock()
monkeypatch.setattr(
"app.services.scrapers.cian_detail.AsyncSession",
mock_session_cls,
)
mock_bf = MagicMock()
mock_bf.fetch = AsyncMock(return_value="<html></html>")
result = await fetch_detail(
"https://ekb.cian.ru/sale/flat/555555/",
browser_fetcher=mock_bf,
)
assert result is not None
mock_session_cls.assert_not_called()
@pytest.mark.asyncio
async def test_fetch_detail_browser_fetcher_returns_none_on_exception(monkeypatch):
"""If BrowserFetcher.fetch raises, fetch_detail returns None (no crash)."""
mock_bf = MagicMock()
mock_bf.fetch = AsyncMock(side_effect=RuntimeError("browser service unavailable"))
result = await fetch_detail(
"https://ekb.cian.ru/sale/flat/999999/",
browser_fetcher=mock_bf,
)
assert result is None
@pytest.mark.asyncio
async def test_fetch_detail_browser_fetcher_state_missing_returns_none(monkeypatch):
"""browser HTML parsed but no defaultState → fetch_detail returns None."""
monkeypatch.setattr(
"app.services.scrapers.cian_detail.extract_state",
lambda html, mfe, key: None,
)
mock_bf = MagicMock()
mock_bf.fetch = AsyncMock(return_value="<html>no state here</html>")
result = await fetch_detail(
"https://ekb.cian.ru/sale/flat/777777/",
browser_fetcher=mock_bf,
)
assert result is None
# ── Real-card fixture (#1791) — снята с www.cian.ru/sale/flat/330982715/ ──────
# Регрессия на field-drift: ceiling/kitchen/description должны извлекаться из
# актуальной structure detail-страницы (defaultState, concat-формат).
@pytest.mark.asyncio
async def test_fetch_detail_real_fixture_extracts_drifted_fields():
"""На реальной карточке ceiling берётся из offer.building.ceilingHeight,
kitchen/description — из offer top-level. До #1791 ceiling был 0%."""
html = (_FIXTURE_DIR / "cian_flat_330982715.html").read_text(encoding="utf-8")
response = MagicMock()
response.status_code = 200
response.text = html
session = MagicMock()
session.get = AsyncMock(return_value=response)
session.close = AsyncMock()
result = await fetch_detail("https://www.cian.ru/sale/flat/330982715/", session=session)
assert result is not None
# ceiling_height — РАНЬШЕ был None (читали offer.ceilingHeight=null);
# теперь из offer.building.ceilingHeight=3.0
assert result.ceiling_height == 3.0
# kitchen_area_m2 — offer.kitchenArea='14.5' (строка → float)
assert result.kitchen_area_m2 == 14.5
# description — непустой текст объявления
assert result.description is not None
assert len(result.description) > 50
assert result.cian_id is not None

792
tradein-mvp/backend/tests/test_cian_serp_scraper.py
View file

@ -1,792 +0,0 @@
"""Smoke тесты для cian.py SERP scraper (Stage 3 рефактор).
Проверяют парсинг 137 полей из синтетического Redux state без сетевых запросов.
Fixture формирует минимальный JS push() conforming to _cianConfig pattern.
"""
import json
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from app.services.scrapers.cian import CianScraper, _format_address
# ── Fixture helpers ──────────────────────────────────────────────────────────
def _make_cian_serp_html(offers: list[dict]) -> str:
"""Обернуть offers в валидный _cianConfig['frontend-serp'].push() HTML."""
state = {
"results": {
"totalOffers": len(offers),
"offers": offers,
}
}
state_json = json.dumps(state, ensure_ascii=False)
return f"""<html><body>
<script>
window._cianConfig['frontend-serp'] = window._cianConfig['frontend-serp'] || [];
window._cianConfig['frontend-serp'].push({{
key: 'initialState',
value: {state_json},
priority: 10,
filter: null
}});
</script>
</body></html>"""
def _make_full_offer(
cian_id: int = 328739308,
price_rur: int = 6500000,
rooms: int = 2,
total_area: str = "52.0",
living_area: str = "35.0",
kitchen_area: str = "11.5",
floor: int = 7,
floors_count: int = 16,
year_built: int = 2010,
material_type: str = "panel",
lat: float = 56.8385,
lng: float = 60.5940,
cadastral_number: str = "66:41:0204016:1234",
building_cadastral_number: str = "66:41:0204016:100",
description: str = "Продаётся отличная квартира",
description_minhash: str | list[int] = "abc123def456abc1",
newbuilding_id: int = 0,
has_phones: bool = True,
is_homeowner: bool = True,
is_pro: bool = False,
bargain_allowed: bool = True,
sale_type: str = "free",
) -> dict:
"""Синтетический Cian offer с полным набором полей (137 полей subset)."""
offer: dict = {
"cianId": cian_id,
"id": cian_id,
"fullUrl": f"https://ekb.cian.ru/sale/flat/{cian_id}/",
"category": "flatSale",
"dealType": "sale",
"offerType": "flat",
"flatType": "rooms",
"isApartments": False,
"roomsCount": rooms,
"status": "published",
# Area
"totalArea": total_area,
"livingArea": living_area,
"kitchenArea": kitchen_area,
"balconiesCount": 1,
"loggiasCount": 0,
"bedroomsCount": None,
"hasFurniture": False,
"isSoldFurnished": False,
"decoration": None,
# Floor / building
"floorNumber": floor,
"building": {
"floorsCount": floors_count,
"materialType": material_type,
"buildYear": year_built,
"deadline": None,
"parking": None,
},
# Cadastral
"cadastralNumber": cadastral_number,
"buildingCadastralNumber": building_cadastral_number,
# Price
"bargainTerms": {
"priceRur": price_rur,
"price": price_rur,
"currency": "rur",
"saleType": sale_type,
"mortgageAllowed": True,
"bargainAllowed": bargain_allowed,
"priceType": "all",
},
"formattedShortPrice": f"{price_rur // 1000000} млн ₽",
# Geo
"geo": {
"coordinates": {"lat": lat, "lng": lng},
"address": [
{"type": "location", "name": "Свердловская область"},
{"type": "city", "name": "Екатеринбург"},
{"type": "street", "name": "улица Постовского"},
{"type": "house", "name": "17А"},
],
"undergrounds": [
{
"name": "Геологическая",
"time": 10,
"transportType": "walk",
"lineId": 10,
"lineColor": "FF0000",
"isDefault": True,
"id": 100,
}
],
"districts": [],
},
# Newbuilding
"newbuilding": {"id": newbuilding_id, "name": ""},
"buildersIds": [],
# Seller
"phones": [{"number": "9012345678", "countryCode": "7"}] if has_phones else [],
"userId": 13332612,
"cianUserId": 13332612,
"publishedUserId": 13332612,
"isByHomeowner": is_homeowner,
"isByCommercialOwner": False,
"isCianPartner": False,
"isPro": is_pro,
"isCalltrackingEnabled": False,
# Description
"description": description,
"descriptionMinhash": description_minhash,
"isDuplicatedDescription": False,
# Time
"addedTimestamp": 1716451200, # 2024-05-23 00:00:00 UTC
"added": "вчера",
# Photos
"photos": [
{
"id": 1001,
"fullUrl": f"https://images.cdn-cian.ru/images/{cian_id}-1.jpg",
"thumbnailUrl": f"https://images.cdn-cian.ru/images/{cian_id}-2.jpg",
}
],
# Verification
"isRosreestrChecked": True,
"isLayoutApproved": None,
# Misc
"isNew": False,
"isTop3": False,
"isPremium": False,
}
return offer
# ── Tests ────────────────────────────────────────────────────────────────────
def test_cian_serp_parses_full_offer():
"""Базовый тест: полный offer → ScrapedLot с ключевыми полями."""
offer = _make_full_offer()
html = _make_cian_serp_html([offer])
scraper = CianScraper()
lots = scraper._parse_serp_html(html)
assert len(lots) == 1
lot = lots[0]
assert lot.source == "cian"
assert lot.source_id == "328739308"
assert lot.source_url == "https://ekb.cian.ru/sale/flat/328739308/"
def test_cian_serp_coords_no_jitter():
"""Координаты берутся точно из state — NO anchor jitter."""
offer = _make_full_offer(lat=56.8385, lng=60.5940)
html = _make_cian_serp_html([offer])
scraper = CianScraper()
lots = scraper._parse_serp_html(html)
assert len(lots) == 1
lot = lots[0]
assert lot.lat == pytest.approx(56.8385, abs=1e-9)
assert lot.lon == pytest.approx(60.5940, abs=1e-9)
def test_cian_serp_kitchen_area_in_raw_payload():
"""Сырой kitchenArea (str) остаётся в raw_payload (#2008 промоутит float
в колонку ScrapedLot.kitchen_area_m2 отдельно — см. соответств. тест)."""
offer = _make_full_offer(kitchen_area="11.5")
html = _make_cian_serp_html([offer])
scraper = CianScraper()
lots = scraper._parse_serp_html(html)
lot = lots[0]
assert lot.raw_payload is not None
assert lot.raw_payload["kitchen_area_m2"] == "11.5"
def test_cian_serp_description_minhash():
"""description_minhash берётся из Cian state, если есть."""
offer = _make_full_offer(
description_minhash="cian_minhash_abc123",
description="Тест описания",
)
html = _make_cian_serp_html([offer])
scraper = CianScraper()
lots = scraper._parse_serp_html(html)
lot = lots[0]
assert lot.description_minhash == "cian_minhash_abc123"
def test_cian_serp_description_minhash_fallback():
"""Если Cian не дал minhash — вычисляем SHA1 из description."""
offer = _make_full_offer(description_minhash="", description="Уникальное описание квартиры")
offer["descriptionMinhash"] = "" # явно пусто
html = _make_cian_serp_html([offer])
scraper = CianScraper()
lots = scraper._parse_serp_html(html)
lot = lots[0]
assert lot.description_minhash is not None
assert len(lot.description_minhash) == 32 # SHA1 hex [:32]
def test_cian_serp_description_minhash_list_int():
"""descriptionMinhash как list[int] (новый формат Cian ~2026-05-23) → строка через запятую.
Порядок элементов сохраняется как-есть — sorted() НЕ применяется,
т.к. порядок LSH-band'ов значим для dedup/cross-source matching.
Offer должен быть принят (не отброшен) — возвращает ScrapedLot, не None.
"""
minhash_list = [8439545, 5781654, 555882, 407717]
offer = _make_full_offer(
description_minhash=minhash_list, # type: ignore[arg-type]
description="Продаётся квартира с видом на парк",
)
html = _make_cian_serp_html([offer])
scraper = CianScraper()
lots = scraper._parse_serp_html(html)
assert len(lots) == 1, "offer с list[int] minhash должен быть принят, не отброшен"
lot = lots[0]
assert lot.description_minhash == "8439545,5781654,555882,407717"
def test_cian_serp_phones():
"""PII purge (#2217): даже если raw offer содержит phones, lot.phones пуст —
скрейпер больше не читает offer.get("phones")."""
offer = _make_full_offer(has_phones=True)
html = _make_cian_serp_html([offer])
scraper = CianScraper()
lots = scraper._parse_serp_html(html)
lot = lots[0]
assert isinstance(lot.phones, list)
assert lot.phones == []
def test_cian_serp_house_ext_id_newbuilding():
"""Новостройка с кадастром дома → anchor = building cad (#1773), не nb_id.
_make_full_offer несёт building_cadastral_number, поэтому same-building anchor
предпочитает кадастр дома (надёжнее, чем nb_id, который группирует весь ЖК)."""
offer = _make_full_offer(newbuilding_id=54016)
html = _make_cian_serp_html([offer])
scraper = CianScraper()
lots = scraper._parse_serp_html(html)
lot = lots[0]
assert lot.house_source == "cian_building_cad"
assert lot.house_ext_id == "66:41:0204016:100"
assert lot.listing_segment == "novostroyki"
def test_cian_serp_house_ext_id_newbuilding_nb_id_fallback():
"""Без кадастра дома anchor падает на nb_id (ЖК) — back-compat fallback."""
offer = _make_full_offer(newbuilding_id=54016, building_cadastral_number="")
offer["buildingCadastralNumber"] = None
html = _make_cian_serp_html([offer])
scraper = CianScraper()
lots = scraper._parse_serp_html(html)
lot = lots[0]
assert lot.house_source == "cian_newbuilding"
assert lot.house_ext_id == "54016"
assert lot.listing_segment == "novostroyki"
def test_cian_serp_vtorichka_no_house_ext_id():
"""Вторичка (newbuilding.id == 0) — house_ext_id None."""
offer = _make_full_offer(newbuilding_id=0)
html = _make_cian_serp_html([offer])
scraper = CianScraper()
lots = scraper._parse_serp_html(html)
lot = lots[0]
assert lot.house_source == "cian"
assert lot.house_ext_id is None
assert lot.listing_segment == "vtorichka"
def test_cian_serp_metro_stations():
"""metro_stations — список dicts с name/time/mode/line_color."""
offer = _make_full_offer()
html = _make_cian_serp_html([offer])
scraper = CianScraper()
lots = scraper._parse_serp_html(html)
lot = lots[0]
assert isinstance(lot.metro_stations, list)
assert len(lot.metro_stations) == 1
station = lot.metro_stations[0]
assert station["name"] == "Геологическая"
assert station["time"] == 10
assert station["mode"] == "walk"
assert station["line_color"] == "FF0000"
def test_cian_serp_cadastral_numbers():
"""Кадастровые номера квартиры и здания."""
offer = _make_full_offer(
cadastral_number="66:41:0204016:1234",
building_cadastral_number="66:41:0204016:100",
)
html = _make_cian_serp_html([offer])
scraper = CianScraper()
lots = scraper._parse_serp_html(html)
lot = lots[0]
assert lot.cadastral_number == "66:41:0204016:1234"
assert lot.building_cadastral_number == "66:41:0204016:100"
# kadastr_num (base field) тоже заполнен
assert lot.kadastr_num == "66:41:0204016:1234"
def test_cian_serp_is_homeowner_seller():
"""is_homeowner и is_pro_seller из offer."""
offer = _make_full_offer(is_homeowner=True, is_pro=False)
html = _make_cian_serp_html([offer])
scraper = CianScraper()
lots = scraper._parse_serp_html(html)
lot = lots[0]
assert lot.is_homeowner is True
assert lot.is_pro_seller is False
def test_cian_serp_bargain_and_sale_type():
"""bargain_allowed и sale_type из bargainTerms."""
offer = _make_full_offer(bargain_allowed=True, sale_type="alternative")
html = _make_cian_serp_html([offer])
scraper = CianScraper()
lots = scraper._parse_serp_html(html)
lot = lots[0]
assert lot.bargain_allowed is True
assert lot.sale_type == "alternative"
def test_cian_serp_building_fields():
"""Поля здания: year_built, house_type, total_floors."""
offer = _make_full_offer(year_built=1988, material_type="brick", floors_count=9)
html = _make_cian_serp_html([offer])
scraper = CianScraper()
lots = scraper._parse_serp_html(html)
lot = lots[0]
assert lot.year_built == 1988
assert lot.house_type == "brick"
assert lot.total_floors == 9
def test_cian_serp_promotes_class_b_fields():
"""#2008: kitchen_area_m2 / mortgage_available / is_apartments /
is_rosreestr_checked промоутятся в колонки ScrapedLot (Class B —
дашборд/matching, НЕ valuation)."""
# _make_full_offer: mortgageAllowed=True, isApartments=False, isRosreestrChecked=True
offer = _make_full_offer(kitchen_area="11.5")
html = _make_cian_serp_html([offer])
scraper = CianScraper()
lots = scraper._parse_serp_html(html)
lot = lots[0]
assert lot.kitchen_area_m2 == pytest.approx(11.5)
assert lot.mortgage_available is True
assert lot.is_apartments is False
assert lot.is_rosreestr_checked is True
def test_cian_serp_house_type_normalized_monolith_brick():
"""#2008 (Class A): cian materialType 'monolithBrick' → канон 'monolith_brick';
сырьё остаётся в raw_payload['raw_material_type']."""
offer = _make_full_offer(material_type="monolithBrick")
html = _make_cian_serp_html([offer])
scraper = CianScraper()
lots = scraper._parse_serp_html(html)
lot = lots[0]
assert lot.house_type == "monolith_brick"
assert lot.raw_payload is not None
assert lot.raw_payload["raw_material_type"] == "monolithBrick"
def test_cian_serp_house_type_block_synonyms_normalized():
"""#2008: gasSilicate/aerocrete/foamConcrete Block + stalin → канон."""
scraper = CianScraper()
for raw, canon in (
("gasSilicateBlock", "block"),
("aerocreteBlock", "block"),
("foamConcreteBlock", "block"),
("stalin", "brick"), # «сталинка» = кирпич
):
offer = _make_full_offer(material_type=raw)
lots = scraper._parse_serp_html(_make_cian_serp_html([offer]))
assert lots[0].house_type == canon, f"{raw} → {lots[0].house_type} != {canon}"
def test_cian_serp_house_type_canonical_passthrough():
"""#2008: уже-каноничные monolith/panel остаются неизменными (idempotency)."""
scraper = CianScraper()
for raw in ("monolith", "panel", "brick", "block", "wood"):
offer = _make_full_offer(material_type=raw)
lots = scraper._parse_serp_html(_make_cian_serp_html([offer]))
assert lots[0].house_type == raw
def test_cian_serp_house_type_unknown_becomes_none():
"""#2008: неизвестный materialType → None (нейтрально для estimator soft-penalty)."""
offer = _make_full_offer(material_type="someUnknownMaterial")
lots = CianScraper()._parse_serp_html(_make_cian_serp_html([offer]))
assert lots[0].house_type is None
def test_cian_serp_living_area():
"""living_area_m2 из offer.livingArea."""
offer = _make_full_offer(living_area="35.0")
html = _make_cian_serp_html([offer])
scraper = CianScraper()
lots = scraper._parse_serp_html(html)
lot = lots[0]
assert lot.living_area_m2 == pytest.approx(35.0)
def test_cian_serp_state_missing_returns_empty():
"""Если state нет в HTML — возвращаем пустой список."""
html = "<html><body><p>no cian state here</p></body></html>"
scraper = CianScraper()
lots = scraper._parse_serp_html(html)
assert lots == []
def test_cian_serp_skips_offer_without_price():
"""Offer без цены — пропускаем."""
offer = _make_full_offer()
del offer["bargainTerms"]["priceRur"]
del offer["bargainTerms"]["price"]
html = _make_cian_serp_html([offer])
scraper = CianScraper()
lots = scraper._parse_serp_html(html)
assert lots == []
def test_cian_serp_multiple_offers():
"""Несколько offers → список ScrapedLot."""
offers = [_make_full_offer(cian_id=100 + i) for i in range(5)]
html = _make_cian_serp_html(offers)
scraper = CianScraper()
lots = scraper._parse_serp_html(html)
assert len(lots) == 5
ids = [lot.source_id for lot in lots]
assert sorted(ids) == ["100", "101", "102", "103", "104"]
def test_cian_serp_photo_urls():
"""photo_urls — полные URL из offer.photos[].fullUrl."""
offer = _make_full_offer(cian_id=999)
html = _make_cian_serp_html([offer])
scraper = CianScraper()
lots = scraper._parse_serp_html(html)
lot = lots[0]
assert len(lot.photo_urls) == 1
assert "cdn-cian.ru" in lot.photo_urls[0]
assert "999-1.jpg" in lot.photo_urls[0]
def test_cian_serp_no_coords_does_not_crash():
"""Если координаты отсутствуют — lot.lat/lon = None, не краш."""
offer = _make_full_offer()
del offer["geo"]["coordinates"]
html = _make_cian_serp_html([offer])
scraper = CianScraper()
lots = scraper._parse_serp_html(html)
assert len(lots) == 1
assert lots[0].lat is None
assert lots[0].lon is None
# ── Transport: BrowserFetcher (camoufox) миграция (#1806) ────────────────────
@pytest.mark.asyncio
async def test_cian_aenter_opens_browser_fetcher() -> None:
"""__aenter__ открывает ОДНУ BrowserFetcher(source="cian") сессию (camoufox)."""
mock_browser = MagicMock()
mock_browser.__aenter__ = AsyncMock(return_value=mock_browser)
mock_browser.__aexit__ = AsyncMock(return_value=None)
captured: dict = {}
def _fake_browser(*args, **kwargs):
captured["source"] = kwargs.get("source") if kwargs else (args[0] if args else None)
return mock_browser
scraper = CianScraper()
with patch("app.services.scrapers.cian.BrowserFetcher", _fake_browser):
await scraper.__aenter__()
mock_browser.__aenter__.assert_awaited_once()
assert scraper._browser is mock_browser
assert captured["source"] == "cian"
await scraper.__aexit__(None, None, None)
mock_browser.__aexit__.assert_awaited_once()
assert scraper._browser is None
@pytest.mark.asyncio
async def test_cian_fetch_page_html_uses_browser_fetcher() -> None:
"""_fetch_page_html тянет HTML через BrowserFetcher.fetch, не через curl_cffi."""
scraper = CianScraper.__new__(CianScraper)
scraper.name = "cian"
scraper.base_url = "https://ekb.cian.ru"
scraper.request_delay_sec = 0.0
fetched_urls: list[str] = []
async def fake_fetch(url: str) -> str:
fetched_urls.append(url)
return "<html>serp-page</html>"
scraper._browser = MagicMock()
scraper._browser.fetch = AsyncMock(side_effect=fake_fetch)
html = await scraper._fetch_page_html((2,), 1, None, None)
assert html == "<html>serp-page</html>"
assert len(fetched_urls) == 1
assert "cat.php" in fetched_urls[0]
@pytest.mark.asyncio
async def test_cian_fetch_page_html_returns_none_on_browser_error() -> None:
"""Если BrowserFetcher.fetch падает — _fetch_page_html возвращает None (не краш)."""
scraper = CianScraper.__new__(CianScraper)
scraper.name = "cian"
scraper.base_url = "https://ekb.cian.ru"
scraper.request_delay_sec = 0.0
scraper._browser = MagicMock()
scraper._browser.fetch = AsyncMock(side_effect=RuntimeError("browser down"))
html = await scraper._fetch_page_html((1,), 2, 1_000_000, 5_000_000)
assert html is None
@pytest.mark.asyncio
async def test_cian_fetch_around_parses_browser_html() -> None:
"""fetch_around парсит лоты из SSR-HTML, отданного BrowserFetcher."""
offer = _make_full_offer(cian_id=777)
serp_html = _make_cian_serp_html([offer])
scraper = CianScraper.__new__(CianScraper)
scraper.name = "cian"
scraper.base_url = "https://ekb.cian.ru"
scraper.request_delay_sec = 0.0
scraper._browser = MagicMock()
scraper._browser.fetch = AsyncMock(return_value=serp_html)
with patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock):
lots = await scraper.fetch_around(56.84, 60.60, rooms=(2,), page=1)
assert len(lots) == 1
assert lots[0].source_id == "777"
scraper._browser.fetch.assert_awaited_once()
@pytest.mark.asyncio
async def test_cian_rotate_ip_is_noop() -> None:
"""_rotate_ip после миграции — no-op, всегда False (прокси ротирует IP сам)."""
scraper = CianScraper.__new__(CianScraper)
result = await scraper._rotate_ip()
assert result is False
# ── Address formatter tests ──────────────────────────────────────────────────
def test_format_address_standard():
"""Стандартный geo.address[]."""
parts = [
{"type": "location", "name": "Свердловская область"},
{"type": "city", "name": "Екатеринбург"},
{"type": "district", "name": "Чкаловский"},
{"type": "street", "name": "улица Постовского"},
{"type": "house", "name": "17А"},
{"type": "metro", "name": "Геологическая"},
]
result = _format_address(parts)
# location и metro пропускаем
assert "Свердловская" not in result
assert "Геологическая" not in result
assert "Екатеринбург" in result
assert "17А" in result
def test_format_address_empty():
"""Пустой address[] → fallback string."""
assert _format_address([]) == "Екатеринбург (Cian)"
# ── #1773: новостройка без part{type:"house"} → восстановление номера дома ─────
def _make_newbuilding_houseless_offer(
cian_id: int = 330047129,
newbuilding_id: int = 54016,
newbuilding_name: str = "Суходольский квартал",
building_cadastral_number: str | None = "66:41:0313010:11396",
building_house_number: str | None = None,
) -> dict:
"""Cian-оффер новостройки: geo.address[] БЕЗ {type:"house"} (ЖК Суходольский,
ул. Евгения Савкова 29, cian 330047129 — реальный кейс из #1773)."""
offer = _make_full_offer(
cian_id=cian_id,
newbuilding_id=newbuilding_id,
building_cadastral_number=building_cadastral_number or "",
)
# geo.address[] без части house — корень бага: «…улица Евгения Савкова» без «29»
offer["geo"]["address"] = [
{"type": "location", "name": "Свердловская область"},
{"type": "city", "name": "Екатеринбург"},
{"type": "district", "name": "Академический"},
{"type": "street", "name": "улица Евгения Савкова"},
]
offer["newbuilding"] = {"id": newbuilding_id, "name": newbuilding_name}
building = offer.setdefault("building", {})
if building_house_number is not None:
building["houseNumber"] = building_house_number
if building_cadastral_number is None:
offer["buildingCadastralNumber"] = None
return offer
def test_cian_newbuilding_cad_number_anchor():
"""Новостройка с кадастром дома → house_source/house_ext_id = building cad
(same-building anchor #1773), адрес дополнен кадастром."""
offer = _make_newbuilding_houseless_offer(
building_cadastral_number="66:41:0313010:11396",
)
html = _make_cian_serp_html([offer])
scraper = CianScraper()
lots = scraper._parse_serp_html(html)
assert len(lots) == 1
lot = lots[0]
# Кадастр дома предпочтён как anchor-ключ вместо nb_id (ЖК целиком)
assert lot.house_source == "cian_building_cad"
assert lot.house_ext_id == "66:41:0313010:11396"
assert lot.building_cadastral_number == "66:41:0313010:11396"
assert lot.listing_segment == "novostroyki"
# geo.address[] был без house → кадастр дописан в адрес как якорь
assert "Евгения Савкова" in (lot.address or "")
assert "66:41:0313010:11396" in (lot.address or "")
def test_cian_newbuilding_recovers_house_from_building_number():
"""building.houseNumber имеет приоритет (a): «29» восстановлен в адрес,
кадастр остаётся anchor-ключом."""
offer = _make_newbuilding_houseless_offer(
building_cadastral_number="66:41:0313010:11396",
building_house_number="29",
)
html = _make_cian_serp_html([offer])
scraper = CianScraper()
lots = scraper._parse_serp_html(html)
lot = lots[0]
addr = lot.address or ""
assert "Евгения Савкова" in addr
assert "29" in addr # номер дома восстановлен из building.houseNumber
# анкор по-прежнему кадастр дома (надёжнее nb_id)
assert lot.house_ext_id == "66:41:0313010:11396"
def test_cian_newbuilding_recovers_house_from_name_when_no_cad():
"""Без кадастра дома: дом-токен «29» вытягивается из названия ЖК (приоритет c),
anchor падает на nb_id."""
offer = _make_newbuilding_houseless_offer(
newbuilding_name="ЖК на Савкова 29",
building_cadastral_number=None,
)
html = _make_cian_serp_html([offer])
scraper = CianScraper()
lots = scraper._parse_serp_html(html)
lot = lots[0]
assert lot.building_cadastral_number is None
# нет кадастра → anchor = nb_id (ЖК)
assert lot.house_source == "cian_newbuilding"
assert lot.house_ext_id == "54016"
addr = lot.address or ""
assert "29" in addr # дом восстановлен из названия ЖК
def test_cian_newbuilding_no_house_no_fabrication():
"""Ни building, ни кадастра, ни номера в названии → номер НЕ выдумываем."""
offer = _make_newbuilding_houseless_offer(
newbuilding_name="Суходольский квартал", # без числа-хвоста
building_cadastral_number=None,
)
html = _make_cian_serp_html([offer])
scraper = CianScraper()
lots = scraper._parse_serp_html(html)
lot = lots[0]
addr = lot.address or ""
assert "Евгения Савкова" in addr
# никаких фейковых catch-all номеров (например 7019) в адресе
assert not addr.rstrip().endswith("7019")
def test_format_address_vtorichka_houseless_untouched():
"""Вторичка с house-less адресом НЕ трогается — номер не выдумываем."""
parts = [
{"type": "city", "name": "Екатеринбург"},
{"type": "street", "name": "улица Евгения Савкова"},
]
result = _format_address(
parts,
building={"houseNumber": "29"},
building_cadastral_number="66:41:0313010:11396",
newbuilding={"id": 0, "name": ""},
is_newbuilding=False,
)
# вторичка: is_newbuilding=False → ни «29», ни кадастр не дописываются
assert "29" not in result
assert "66:41" not in result

15
tradein-mvp/backend/tests/test_domclick_sweep.py
View file

@ -6,21 +6,25 @@ cancel, honest mark_done/mark_failed statuses) удалён вместе с
теперь в `test_scraper_kit_pipeline_parity2.py::test_domclick_city_sweep_{done,
blocked_failed}` (kit `run_domclick_city_sweep`).
`app.services.scrapers.domclick` удалён (#2397 Part E2, 0 runtime-импортёров) —
`_map_item` ретаргетирован на kit `scraper_kit.providers.domclick.serp.DomClickScraper`
(единственный живой путь).
Остались:
- DomClickCitySweepCounters (dataclass, из scraper_kit) — defaults + to_dict
- DomClickScraper._map_item — BFF offer-item → ScrapedLot маппинг (не зависит
от оркестрации, `app.services.scrapers.domclick` живёт до Part E2)
- DomClickScraper._map_item (kit) — BFF offer-item → ScrapedLot маппинг (не
зависит от оркестрации)
"""
from __future__ import annotations
import os
from types import SimpleNamespace
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
import pytest
from app.services.scrapers.domclick import DomClickScraper
from scraper_kit.providers.domclick.serp import DomClickScraper
# ── DomClickCitySweepCounters ───────────────────────────────────────────────
@ -54,7 +58,8 @@ def test_domclick_sweep_counters_to_dict_all_keys() -> None:
def test_map_item_basic_mapping() -> None:
"""DomClickScraper._map_item маппит BFF offer-item → ScrapedLot корректно."""
scraper = DomClickScraper()
config = SimpleNamespace(browser_http_endpoint="http://tradein-browser:9000")
scraper = DomClickScraper(config, delay_provider=lambda _name: 8.0)
item = {
"id": 111222333,
"path": "https://domclick.ru/card/sale__flat__111222333",

189
tradein-mvp/backend/tests/test_houses_rating_filter_entries_1789.py
View file

@ -1,189 +0,0 @@
"""#1789 — rating_score из reviews.filterEntries[type==score] (fallback).
Avito перенёс сводный рейтинг из housePage.developmentData.ratingPreview в виджет
reviews.filterEntries[{type: 'score', value: {scoreFloat, scoreString, reviewCount, ratingStat}}].
Тест 1: когда housePage НЕ даёт рейтинг (ratingBadge пуст), rating берётся из
reviews.filterEntries — проверяем rating_score/rating_string/reviews_count/
rating_distribution.
Тест 2: housePage даёт рейтинг (старый путь ratingBadge) → filterEntries НЕ
перезаписывает (приоритет housePage).
Тест 3: новостройка (developmentPage) без reviews-виджета → graceful, нет краша.
Без сети, без БД.
"""
from __future__ import annotations
import os
import pytest
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
from app.services.scrapers.avito_houses import parse_houses_state
def _make_state(
*,
house_page_rating: dict | None = None,
reviews_filter_entries: list | None = None,
) -> dict:
"""Минимальный __preloadedState__ с housePage и опциональными reviews.
house_page_rating — если задан, проставляем в developmentData (старый путь).
reviews_filter_entries — если задан, проставляем в reviews.props.reviews.filterEntries
"""
dev_data: dict = {
"avitoId": 555666,
"id": "aGFzaA==",
"title": "Test House",
"address": "ул. Тестовая, 1",
"coords": {"lat": 56.8, "lng": 60.6},
"aboutDevelopment": {"expandParams": {"items": []}},
"developer": {},
"mapPreview": {},
}
if house_page_rating is not None:
dev_data.update(house_page_rating)
placeholders: list = [
{
"type": "housePage",
"props": {"developmentData": dev_data},
},
]
if reviews_filter_entries is not None:
placeholders.append(
{
"type": "reviews",
"props": {
# MFE-форма: данные под ключом "reviews" внутри props
"reviews": {
"entries": [],
"filterEntries": reviews_filter_entries,
}
},
}
)
return {"data": {"data": {"page": {"placeholders": placeholders}}}}
# ---------------------------------------------------------------------------
# Тест 1: рейтинг из reviews.filterEntries (housePage пуст)
# ---------------------------------------------------------------------------
def test_rating_from_filter_entries_when_housepage_empty() -> None:
"""Когда housePage не даёт рейтинг — берём из reviews.filterEntries."""
state = _make_state(
reviews_filter_entries=[
{
"type": "score",
"value": {
"scoreFloat": 4.666,
"scoreString": "4,7",
"reviewCount": 3,
"ratingStat": [
{"score": 5, "count": 2},
{"score": 4, "count": 1},
],
},
}
]
)
e = parse_houses_state(state, "/catalog/houses/ekaterinburg/test/555666")
h = e.house
assert h.rating_score == pytest.approx(4.666, rel=1e-3)
assert h.rating_string == "4,7"
assert h.reviews_count == 3
assert len(h.rating_distribution) == 2
assert h.rating_distribution[0]["score"] == 5
def test_rating_filter_entries_float_type() -> None:
"""scoreFloat из filterEntries конвертируется в float (не остаётся int/str)."""
state = _make_state(
reviews_filter_entries=[
{
"type": "score",
"value": {"scoreFloat": 5, "scoreString": "5,0", "reviewCount": 1},
}
]
)
e = parse_houses_state(state, "/catalog/houses/ekaterinburg/test/555666")
assert isinstance(e.house.rating_score, float)
assert e.house.rating_score == 5.0
# ---------------------------------------------------------------------------
# Тест 2: housePage даёт рейтинг → filterEntries НЕ перезаписывает
# ---------------------------------------------------------------------------
def test_housepage_rating_takes_priority_over_filter_entries() -> None:
"""Если housePage.ratingBadge уже заполнен — filterEntries не перезаписывает."""
state = _make_state(
house_page_rating={
"ratingBadge": {"info": {"score": 3.9, "scoreString": "3,9"}},
"ratingSummaryStat": {"reviewCount": 10, "ratingStat": []},
},
reviews_filter_entries=[
{
"type": "score",
"value": {
"scoreFloat": 4.9,
"scoreString": "4,9",
"reviewCount": 99,
"ratingStat": [],
},
}
],
)
e = parse_houses_state(state, "/catalog/houses/ekaterinburg/test/555666")
h = e.house
# Значение из housePage должно сохраниться
assert h.rating_score == pytest.approx(3.9, rel=1e-3)
assert h.rating_string == "3,9"
assert h.reviews_count == 10
# ---------------------------------------------------------------------------
# Тест 3: нет reviews-виджета вообще → None, не падаем
# ---------------------------------------------------------------------------
def test_no_reviews_widget_rating_is_none() -> None:
"""Без виджета reviews и без ratingBadge → rating_score=None, graceful."""
state = _make_state() # нет reviews_filter_entries
e = parse_houses_state(state, "/catalog/houses/ekaterinburg/test/555666")
h = e.house
assert h.rating_score is None
assert h.rating_string is None
assert h.reviews_count is None
# ---------------------------------------------------------------------------
# Тест 4: filterEntries без entry type==score → rating остаётся None
# ---------------------------------------------------------------------------
def test_filter_entries_without_score_type() -> None:
"""filterEntries без type==score → рейтинг не заполняется."""
state = _make_state(
reviews_filter_entries=[
{"type": "something_else", "value": {"scoreFloat": 9.9}},
]
)
e = parse_houses_state(state, "/catalog/houses/ekaterinburg/test/555666")
assert e.house.rating_score is None

12
tradein-mvp/backend/tests/test_pipeline_browser_routing.py
View file

@ -12,6 +12,10 @@ Legacy `run_avito_pipeline` browser/curl-mode integration tests (было ран
удалены вместе с `app.services.scrape_pipeline` (#2397 Part E1) — browser-routing
покрытие на уровне `fetch_detail`/`fetch_house_catalog`/`AvitoScraper._fetch_serp_html`
ниже остаётся нетронутым.
`fetch_house_catalog` ретаргетирован на `scraper_kit.providers.avito.houses` (#2397
Part E2) — legacy `app.services.scrapers.avito_houses` удалён (0 runtime-импортёров,
kit — единственный живой путь).
"""
from __future__ import annotations
@ -26,9 +30,11 @@ import pytest
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
from scraper_kit.avito_exceptions import AvitoBlockedError as KitAvitoBlockedError
from scraper_kit.providers.avito.houses import fetch_house_catalog
from app.services.scrapers.avito_detail import fetch_detail
from app.services.scrapers.avito_exceptions import AvitoBlockedError
from app.services.scrapers.avito_houses import fetch_house_catalog
FIXTURES = Path(__file__).parent / "fixtures"
@ -205,10 +211,10 @@ async def test_fetch_house_catalog_browser_no_curl_used() -> None:
@pytest.mark.asyncio
async def test_fetch_house_catalog_browser_firewall_raises() -> None:
"""Firewall HTML через browser path → AvitoBlockedError."""
"""Firewall HTML через browser path → AvitoBlockedError (kit)."""
mock_fetcher = make_mock_fetcher(FIREWALL_HTML)
with pytest.raises(AvitoBlockedError, match="browser-mode"):
with pytest.raises(KitAvitoBlockedError, match="browser-mode"):
await fetch_house_catalog(
"/catalog/houses/ekaterinburg/ul_test/9999",
browser_fetcher=mock_fetcher,

282
tradein-mvp/backend/tests/test_scraper_kit_bisection_parity.py
View file

@ -1,14 +1,15 @@
"""Golden-parity бисекции: старый боевой обход `app.services.scrapers.<provider>`
"""Golden-parity бисекции: старый боевой обход `app.services.scrapers.avito`
≡ новый единый движок `scraper_kit.pricing.walk_price_range` (#2134).
Четыре независимые копии адаптивной бисекции «комнатность × цена» были
консолидированы в один движок (`scraper_kit/pricing/bisection.py`). Провайдеры
(avito/cian/yandex/domclick) теперь передают в движок свои probe/paginate + свой
`BisectionConfig`. Этот тест доказывает, что unification НЕ изменил ОБХОД: для
каждого провайдера при одинаковом входе (детерминированный мок счётчика по
диапазону) старая боевая реализация и новая (движок) посещают ИДЕНТИЧНУЮ
последовательность бакетов — probe-запросы, leaf-пагинации, деления, ранние
возвраты на нуле, probe-fail-ветки.
cian/domclick/yandex_realty удалены (#2397 Part E2, 0 runtime-импортёров) —
их legacy-vs-kit секции обхода убраны отсюда вместе с ними (легаси-эталон
исчез). `avito` остаётся легаси (не мигрирован) — его parity-проверка ниже
доказывает, что unification НЕ изменил ОБХОД: при одинаковом входе
(детерминированный мок счётчика по диапазону) старая боевая реализация и
новая (движок) посещают ИДЕНТИЧНУЮ последовательность бакетов — probe-запросы,
leaf-пагинации, деления, ранние возвраты на нуле, probe-fail-ветки.
Механика: на ОБЕ реализации (старую и новую) ставятся ОДИНАКОВЫЕ моки на
низкоуровневые методы (probe-fetch + leaf-пагинатор), которые (а) возвращают
@ -22,7 +23,7 @@ import os
from collections.abc import Callable
from types import SimpleNamespace
from typing import Any
from unittest.mock import AsyncMock, MagicMock, patch
from unittest.mock import AsyncMock, patch
import pytest
@ -30,14 +31,8 @@ os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/
from scraper_kit.price_brackets import get_price_seed_brackets
from scraper_kit.providers.avito.serp import AvitoScraper as NewAvito
from scraper_kit.providers.cian.serp import CianScraper as NewCian
from scraper_kit.providers.domclick.serp import DomClickScraper as NewDomClick
from scraper_kit.providers.yandex.serp import YandexRealtyScraper as NewYandex
from app.services.scrapers.avito import AvitoScraper as OldAvito
from app.services.scrapers.cian import CianScraper as OldCian
from app.services.scrapers.domclick import DomClickScraper as OldDomClick
from app.services.scrapers.yandex_realty import YandexRealtyScraper as OldYandex
# ── Оракул счётчиков ─────────────────────────────────────────────────────────
@ -55,7 +50,7 @@ def make_oracle(
span считается по клампнутым границам (None-lo → 0, None-hi → open_ceiling),
чтобы широкие брекеты делились, а узкие уходили в leaf. `zeros` форсит 0 (ранний
возврат / leaf-на-нуле у yandex), `nones` форсит None (probe-fail).
возврат), `nones` форсит None (probe-fail).
"""
def oracle(lo: Any, hi: Any) -> int | None:
@ -71,10 +66,10 @@ def make_oracle(
return oracle
# ── Провайдер-специфичные «сборщики событий» ─────────────────────────────────
# ── avito «сборщик событий» ──────────────────────────────────────────────────
#
# Каждый ставит одинаковые моки на старый/новый экземпляр и возвращает список
# событий обхода. `("probe", lo, hi)` — запрос счётчика бакета; `("leaf", ...)` —
# Ставит одинаковые моки на старый/новый экземпляр и возвращает список событий
# обхода. `("probe", lo, hi)` — запрос счётчика бакета; `("leaf", ...)` —
# пагинация leaf-бакета.
@ -91,81 +86,6 @@ def _patch_avito(scraper: Any, oracle: Oracle, events: list[Any]) -> None:
scraper._rotate_ip = AsyncMock(return_value=False)
def _patch_cian(scraper: Any, oracle: Oracle, events: list[Any]) -> None:
holder: dict[str, tuple[Any, Any]] = {}
def fph(rooms: Any, page: int, min_price: Any, max_price: Any) -> str:
events.append(("probe", min_price, max_price))
holder["b"] = (min_price, max_price)
return "H"
def eto(html: str) -> int | None:
return oracle(*holder["b"])
def leaf(**kw: Any) -> None:
events.append(("leaf", kw["lo"], kw["hi"], kw["total"]))
scraper._fetch_page_html = AsyncMock(side_effect=fph)
scraper._extract_total_offers = MagicMock(side_effect=eto)
scraper._paginate_leaf_bucket = AsyncMock(side_effect=leaf)
scraper._rotate_ip = AsyncMock(return_value=False)
scraper.sleep_between_requests = AsyncMock()
def _patch_domclick(scraper: Any, oracle: Oracle, events: list[Any]) -> None:
def count(fetcher: Any, rooms: Any, gte: Any, lte: Any) -> int:
events.append(("probe", gte, lte))
val = oracle(gte, lte)
return val if val is not None else 0
def paginate(
fetcher: Any, rooms: Any, gte: Any, lte: Any, seen_ids: Any, out_lots: Any, pages: Any
) -> None:
events.append(("leaf", gte, lte))
scraper._count = AsyncMock(side_effect=count)
scraper._paginate = AsyncMock(side_effect=paginate)
def _patch_yandex(
scraper: Any, oracle: Oracle, events: list[Any], *, record_pages: bool = False
) -> None:
"""Yandex: leaf-пагинация встроена в walk (нет отдельного метода-хука).
Стабим `_fetch_page_json` (page=1 отдаёт payload с totalItems из оракула;
page>1 — пустой payload) и передаём `on_bucket`, который пишет завершение
leaf/degraded бакета. probe = page-1-fetch; leaf-страницы 2..N идут через
asyncio.gather (порядок недетерминирован) — их НЕ пишем в норм-режиме; факт
leaf-обхода фиксируется через on_bucket. Для degraded-теста (пагинация
последовательная) `record_pages=True` пишет все страницы.
"""
def gate(total: int | None, entities: list[Any] | None = None) -> dict[str, Any]:
pager: dict[str, Any] = {}
if total is not None:
pager["totalItems"] = total
return {"response": {"search": {"offers": {"entities": entities or [], "pager": pager}}}}
def fpj(rooms: Any, page: int, price_min: Any, price_max: Any, new_flat: str = "NO") -> Any:
if page == 1:
events.append(("probe", price_min, price_max))
return gate(oracle(price_min, price_max))
if record_pages:
events.append(("page", price_min, price_max, page))
return gate(0, entities=[])
scraper._fetch_page_json = AsyncMock(side_effect=fpj)
scraper._rotate_ip = AsyncMock(return_value=False)
scraper.request_delay_sec = 0.0
def _yandex_on_bucket(events: list[Any]) -> Callable[[str, int], None]:
def on_bucket(bucket_key: str, count: int) -> None:
events.append(("bucket", bucket_key))
return on_bucket
# ── Конструкторы старых/новых экземпляров (offline, без DB) ───────────────────
_KIT_CONFIG = SimpleNamespace(
@ -185,42 +105,14 @@ def _old_avito() -> Any:
return OldAvito()
def _old_cian() -> Any:
with patch("app.services.scrapers.cian.get_scraper_delay", return_value=5.0):
return OldCian()
# ── avito: probe → split → leaf ───────────────────────────────────────────────
def _old_yandex() -> Any:
with patch("app.services.scrapers.yandex_realty.get_scraper_delay", return_value=5.0):
return OldYandex()
def _old_domclick() -> Any:
with patch("app.services.scrapers.domclick.get_scraper_delay", return_value=8.0):
return OldDomClick()
# ── avito / cian: одинаковый скелет (probe → split → leaf) ────────────────────
# Провайдер-специфичные kwargs для _walk_price_range (avito идентифицирует бакет
# room_slug/room_label; cian — кортежем rooms).
_AVITO_WALK_EXTRA = {"room_slug": "room1", "room_label": "1к"}
_CIAN_WALK_EXTRA = {"rooms": (1,)}
@pytest.mark.asyncio
@pytest.mark.parametrize(
"old_factory, new_factory, patch_fn, walk_extra",
[
(_old_avito, lambda: NewAvito(config=_KIT_CONFIG), _patch_avito, _AVITO_WALK_EXTRA),
(_old_cian, lambda: NewCian(config=_KIT_CONFIG), _patch_cian, _CIAN_WALK_EXTRA),
],
)
async def test_bisection_parity_avito_cian(
old_factory: Any, new_factory: Any, patch_fn: Any, walk_extra: dict[str, Any]
) -> None:
"""avito/cian: split-дерево + leaf + ранний возврат на нуле — обход идентичен.
async def test_bisection_parity_avito() -> None:
"""avito: split-дерево + leaf + ранний возврат на нуле — обход идентичен.
Оракул: divisor=1000, cap=1400 → брекет 8М делится 3 уровня (8М→4М→2М→1М-leaf);
один под-брекет форсим в 0 (ранний возврат без leaf).
@ -237,7 +129,7 @@ async def test_bisection_parity_avito_cian(
async def run(scraper: Any) -> list[Any]:
events: list[Any] = []
patch_fn(scraper, oracle, events)
_patch_avito(scraper, oracle, events)
for lo, hi in brackets:
await scraper._walk_price_range(
lo=lo,
@ -249,12 +141,12 @@ async def test_bisection_parity_avito_cian(
secondary_only=True,
on_bucket=None,
skip_buckets=None,
**walk_extra,
**_AVITO_WALK_EXTRA,
)
return events
old_events = await run(old_factory())
new_events = await run(new_factory())
old_events = await run(_old_avito())
new_events = await run(NewAvito(config=_KIT_CONFIG))
assert old_events == new_events
# Санити: обход реально нетривиален (несколько probe + несколько leaf).
assert sum(1 for e in old_events if e[0] == "leaf") >= 4
@ -300,140 +192,18 @@ async def test_bisection_parity_avito_probe_fail() -> None:
assert ("leaf", 25_000_000, None, 100) in old_events
# ── domclick: count-endpoint + None-both-ends + split открытого корня ─────────
@pytest.mark.asyncio
async def test_bisection_parity_domclick() -> None:
"""domclick: unbounded-корень (None,None) клампится к [0,1e9] и делится;
count-endpoint; span<=MIN_PRICE_SPAN строго → leaf. Обход идентичен старому."""
oracle = make_oracle(divisor=100_000, open_ceiling=1_000_000_000)
async def run(scraper: Any) -> list[Any]:
events: list[Any] = []
_patch_domclick(scraper, oracle, events)
await scraper._sweep_bucket(
fetcher=None,
rooms="1",
price_gte=None,
price_lte=None,
seen_ids=set(),
out_lots=[],
pages=100,
)
return events
old_events = await run(_old_domclick())
new_events = await run(NewDomClick(_make_domclick_config()))
assert old_events == new_events
assert sum(1 for e in old_events if e[0] == "leaf") >= 4
# Корень (None,None) обязательно пробится первым.
assert old_events[0] == ("probe", None, None)
def _make_domclick_config() -> Any:
return SimpleNamespace(browser_http_endpoint="http://tradein-browser:9000")
# ── yandex: max_depth=8, min_bracket=500k, count-based on_bucket ──────────────
@pytest.mark.asyncio
async def test_bisection_parity_yandex_normal() -> None:
"""yandex: split-дерево (порог 500k, cap 500) + leaf-on-bucket + zero-as-leaf.
Оракул divisor=1000, cap=500 → 4М→2М→1М→500k-leaf. Один под-брекет=0
(yandex вызывает on_bucket даже на нуле — treat_zero_as_leaf).
"""
oracle = make_oracle(
divisor=1000,
open_ceiling=30_000_000,
zeros=frozenset({(2_000_000, 2_999_999)}),
)
brackets: list[tuple[int, int | None]] = [(0, 3_999_999), (25_000_000, None)]
async def run(scraper: Any) -> list[Any]:
events: list[Any] = []
_patch_yandex(scraper, oracle, events)
for lo, hi in brackets:
await scraper._walk_price_range(
rooms="1",
lo=lo,
hi=hi,
seen={},
price_cap_per_bucket=500,
max_pages_per_bucket=50,
concurrency=4,
on_bucket=_yandex_on_bucket(events),
skip_buckets=None,
)
return events
old_events = await run(_old_yandex())
new_events = await run(NewYandex(config=_KIT_CONFIG))
assert old_events == new_events
assert sum(1 for e in old_events if e[0] == "bucket") >= 4
assert sum(1 for e in old_events if e[0] == "probe") >= 6
@pytest.mark.asyncio
async def test_bisection_parity_yandex_degraded() -> None:
"""yandex probe-fail (totalItems неизвестен) → degraded paginate-until-empty.
Оракул форсит None на бакете → probe page1 payload без totalItems → total=None
→ degraded последовательно пагинирует. record_pages=True фиксирует все страницы.
"""
oracle = make_oracle(
divisor=1000,
open_ceiling=30_000_000,
nones=frozenset({(None, 3_999_999)}),
)
async def run(scraper: Any) -> list[Any]:
events: list[Any] = []
_patch_yandex(scraper, oracle, events, record_pages=True)
await scraper._walk_price_range(
rooms="1",
lo=0,
hi=3_999_999,
seen={},
price_cap_per_bucket=500,
max_pages_per_bucket=50,
concurrency=4,
on_bucket=_yandex_on_bucket(events),
skip_buckets=None,
)
return events
old_events = await run(_old_yandex())
new_events = await run(NewYandex(config=_KIT_CONFIG))
assert old_events == new_events
# degraded: probe page1 + затем последовательная пагинация страниц 1..K до пустой.
assert ("probe", None, 3_999_999) in old_events
assert any(e[0] == "bucket" for e in old_events)
# ── Full seed-grid обход (все seed-брекеты одной комнатности) ─────────────────
@pytest.mark.asyncio
@pytest.mark.parametrize(
"old_factory, new_factory, patch_fn, walk_extra",
[
(_old_avito, lambda: NewAvito(config=_KIT_CONFIG), _patch_avito, _AVITO_WALK_EXTRA),
(_old_cian, lambda: NewCian(config=_KIT_CONFIG), _patch_cian, _CIAN_WALK_EXTRA),
],
)
async def test_bisection_parity_full_seed_grid(
old_factory: Any, new_factory: Any, patch_fn: Any, walk_extra: dict[str, Any]
) -> None:
async def test_bisection_parity_full_seed_grid() -> None:
"""Полный обход всех seed-брекетов (как в fetch_all_secondary) — включая
открытый catch-all — идентичен между старым и новым для avito/cian."""
открытый catch-all — идентичен между старым и новым для avito."""
oracle = make_oracle(divisor=1500, open_ceiling=300_000_000)
async def run(scraper: Any) -> list[Any]:
events: list[Any] = []
patch_fn(scraper, oracle, events)
_patch_avito(scraper, oracle, events)
for br_lo, br_hi in get_price_seed_brackets():
walk_hi = br_hi - 1 if br_hi is not None else None
await scraper._walk_price_range(
@ -446,10 +216,10 @@ async def test_bisection_parity_full_seed_grid(
secondary_only=True,
on_bucket=None,
skip_buckets=None,
**walk_extra,
**_AVITO_WALK_EXTRA,
)
return events
old_events = await run(old_factory())
new_events = await run(new_factory())
old_events = await run(_old_avito())
new_events = await run(NewAvito(config=_KIT_CONFIG))
assert old_events == new_events

331
tradein-mvp/backend/tests/test_scraper_kit_cian_golden_parity.py
View file

@ -1,20 +1,22 @@
"""Golden-parity: `scraper_kit.providers.cian.*` парсинг ≡ `app.services.scrapers.cian*`.
"""Golden-parity: `scraper_kit.providers.cian.*` парсинг ≡ ещё живым `app.services.scrapers.cian*`.
Strangler-инвариант (#2133 cian): новая scraper_kit-копия cian-провайдера должна давать
БАЙТ-ИДЕНТИЧНЫЙ результат парсинга старому боевому коду на одинаковом входе. Развязка
(settings→ScraperConfig, get_scraper_delay→delay_provider, cian_session→локальная копия)
НЕ меняет распарсенные данные.
Гоняем ОБА модуля на одних фикстурах и сравниваем результат:
- SERP `_parse_serp_html` (Redux state → list[ScrapedLot]) на синтетическом _cianConfig
push-HTML (вторичка + новостройка + студия), сравнение model_dump();
- SERP `_offer_to_lot` пооферно + `_format_address` (pure, восстановление дома ЖК);
- `_extract_total_offers` (totalOffers из state);
- `extract_state` — app-версия ≡ kit-версия на одном HTML (общий парсер state);
- detail `fetch_detail` (defaultState + bti sister → DetailEnrichment, dataclasses.asdict);
- valuation `_parse_valuation_state` (estimation/chart/houseInfo → CianValuationResult);
- newbuilding pure-экстракторы (_extract_chart / _extract_reliability_checks /
_extract_nested_offers / _extract_zhk_url_from_serp).
`app.services.scrapers.cian` / `cian_detail` удалены (#2397 Part E2, 0 runtime-
импортёров — admin.py/cian_history_backfill.py уже на kit). Parity-тесты SERP
`_parse_serp_html`/`_offer_to_lot`/`_extract_total_offers`/`_format_address` и
detail `fetch_detail` удалены вместе с ними (легаси-эталон исчез).
Оставшееся здесь сравнивает kit ≡ ЕЩЁ ЖИВЫЕ легаси-модули:
- `extract_state` — `app.services.scrapers.cian_state_parser` (живой, общий парсер state);
- valuation `_parse_valuation_state` / `estimate_via_cian_valuation` —
`app.services.scrapers.cian_valuation` (живой);
- newbuilding pure-экстракторы — `app.services.scrapers.cian_newbuilding` (живой);
- kit mandatory-config guards (`estimate_via_cian_valuation`, `cian.session`) — kit-only;
- strangler-guard: kit-провайдер не импортирует `app.*`.
Идентичность результата = kit-парсер верно скопирован (развязка не задела логику).
"""
@ -27,17 +29,17 @@ import json
import os
from types import SimpleNamespace
from typing import Any
from unittest.mock import AsyncMock, MagicMock, patch
from unittest.mock import MagicMock
import pytest
# Старый app.services.scrapers.cian* импортирует app.core.config.settings=Settings(),
# которому нужен DATABASE_URL. Офлайн-парсинг БД не трогает — фиктивный DSN достаточен.
# Старый app.services.scrapers.cian_state_parser/cian_valuation/cian_newbuilding
# импортируют app.core.config.settings=Settings(), которому нужен DATABASE_URL.
# Офлайн-парсинг БД не трогает — фиктивный DSN достаточен.
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
# НОВЫЙ (scraper_kit) cian-провайдер.
from scraper_kit.cian_state_parser import extract_state as new_extract_state
from scraper_kit.providers.cian.detail import fetch_detail as new_fetch_detail
from scraper_kit.providers.cian.newbuilding import _extract_chart as new_extract_chart
from scraper_kit.providers.cian.newbuilding import (
_extract_nested_offers as new_extract_nested_offers,
@ -48,8 +50,6 @@ from scraper_kit.providers.cian.newbuilding import (
from scraper_kit.providers.cian.newbuilding import (
_extract_zhk_url_from_serp as new_extract_zhk_url,
)
from scraper_kit.providers.cian.serp import CianScraper as NewCianScraper
from scraper_kit.providers.cian.serp import _format_address as new_format_address
from scraper_kit.providers.cian.valuation import (
_parse_valuation_state as new_parse_valuation,
)
@ -62,10 +62,7 @@ from scraper_kit.providers.cian.valuation import (
# #2306's cian_price_history.py (см. app/services/scraper_adapters.py).
from app.services.scraper_adapters import RealScraperConfig
# СТАРЫЙ (app.services.scrapers) cian-провайдер.
from app.services.scrapers.cian import CianScraper as OldCianScraper
from app.services.scrapers.cian import _format_address as old_format_address
from app.services.scrapers.cian_detail import fetch_detail as old_fetch_detail
# СТАРЫЙ (app.services.scrapers) — ещё живые cian-модули (cian.py/cian_detail.py удалены).
from app.services.scrapers.cian_newbuilding import _extract_chart as old_extract_chart
from app.services.scrapers.cian_newbuilding import (
_extract_nested_offers as old_extract_nested_offers,
@ -85,14 +82,11 @@ from app.services.scrapers.cian_valuation import (
)
from tests.support.parity import assert_parity
# ── Fixtures: SERP offers → _cianConfig push HTML ────────────────────────────
_CONFIG = SimpleNamespace(glitchtip_dsn=None)
# ── extract_state parity (state-парсер, живой в cian_state_parser.py) ────────
def _serp_html(offers: list[dict[str, Any]]) -> str:
"""Обернуть offers в валидный _cianConfig['frontend-serp'].push() HTML."""
state = {"results": {"totalOffers": len(offers), "offers": offers}}
def _serp_html_for_state(state: dict[str, Any]) -> str:
"""Обернуть произвольный state в валидный _cianConfig['frontend-serp'].push() HTML."""
state_json = json.dumps(state, ensure_ascii=False)
return (
"<html><body><script>\n"
@ -103,293 +97,16 @@ def _serp_html(offers: list[dict[str, Any]]) -> str:
)
def _vtorichka_offer() -> dict[str, Any]:
return {
"cianId": 328739308,
"id": 328739308,
"fullUrl": "https://ekb.cian.ru/sale/flat/328739308/",
"offerType": "flat",
"flatType": "rooms",
"category": "flatSale",
"isApartments": False,
"roomsCount": 2,
"totalArea": "52.0",
"livingArea": "35.0",
"kitchenArea": "11.5",
"balconiesCount": 1,
"loggiasCount": 0,
"bedroomsCount": None,
"hasFurniture": False,
"isSoldFurnished": False,
"decoration": "euro",
"floorNumber": 7,
"building": {
"floorsCount": 16,
"materialType": "monolithBrick",
"buildYear": 2010,
"parking": {"type": "underground"},
"totalArea": "3400",
},
"cadastralNumber": "66:41:0204016:1234",
"buildingCadastralNumber": "66:41:0204016:100",
"bargainTerms": {
"priceRur": 6500000,
"price": 6500000,
"saleType": "free",
"mortgageAllowed": True,
"bargainAllowed": True,
},
"geo": {
"coordinates": {"lat": 56.8385, "lng": 60.5940},
"address": [
{"type": "location", "fullName": "Свердловская область"},
{"type": "raion", "fullName": "Кировский"},
{"type": "street", "fullName": "улица Постовского"},
{"type": "house", "fullName": "17А"},
{"type": "metro", "fullName": "Геологическая"},
],
"undergrounds": [
{
"name": "Геологическая",
"time": 10,
"transportType": "walk",
"lineId": 10,
"lineColor": "FF0000",
"isDefault": True,
}
],
},
"newbuilding": {"id": 0, "name": ""},
"phones": [{"number": "9012345678", "countryCode": "7"}],
"userId": 13332612,
"publishedUserId": 13332612,
"isByHomeowner": True,
"isPro": False,
"isRosreestrChecked": True,
"description": "Продаётся отличная квартира с евроремонтом",
"descriptionMinhash": "abc123def456abc1",
"addedTimestamp": 1716451200,
"photos": [{"fullUrl": "https://images.cdn-cian.ru/images/328739308-1.jpg"}],
}
def _newbuilding_offer() -> dict[str, Any]:
"""Новостройка без части house в geo.address[] — жмёт _recover_newbuilding_house."""
return {
"cianId": 411112222,
"id": 411112222,
"fullUrl": "https://ekb.cian.ru/sale/flat/411112222/",
"offerType": "flat",
"flatType": "rooms",
"isApartments": False,
"roomsCount": 1,
"totalArea": "38.0",
"floorNumber": 12,
"building": {
"floorsCount": 25,
"materialType": "monolith",
"buildYear": None,
"deadline": {"year": 2026},
},
"buildingCadastralNumber": "66:41:0204016:11396",
"bargainTerms": {"priceRur": 5200000, "price": 5200000, "mortgageAllowed": True},
"geo": {
"coordinates": {"lat": 56.8000, "lng": 60.6000},
"address": [
{"type": "location", "fullName": "Свердловская область"},
{"type": "street", "fullName": "улица Новая"},
],
},
"newbuilding": {"id": 48853, "name": "ЖК Парковый квартал 29А"},
"isFromDeveloper": True,
"photos": [],
}
def _studio_offer() -> dict[str, Any]:
return {
"cianId": 500011111,
"id": 500011111,
"offerType": "flat",
"flatType": "studio",
"roomsCount": None,
"totalArea": "24.5",
"floorNumber": 3,
"building": {"floorsCount": 9, "materialType": "panel", "buildYear": 1980},
"bargainTerms": {"priceRur": 3100000, "price": 3100000},
"geo": {
"coordinates": {"lat": 56.81, "lng": 60.61},
"address": [
{"type": "street", "fullName": "улица Ленина"},
{"type": "house", "fullName": "5"},
],
},
"newbuilding": {"id": 0},
"photos": [],
}
_OFFERS = [_vtorichka_offer(), _newbuilding_offer(), _studio_offer()]
def _make_old() -> OldCianScraper:
with patch("app.services.scrapers.cian.get_scraper_delay", return_value=5.0):
return OldCianScraper()
def _make_new() -> NewCianScraper:
return NewCianScraper(config=_CONFIG)
def _dump(lots: list[Any]) -> list[dict[str, Any]]:
return [lot.model_dump() for lot in lots]
# ── SERP _parse_serp_html parity ─────────────────────────────────────────────
def test_serp_parse_serp_html_parity() -> None:
"""Redux offers → list[ScrapedLot] идентичны (вторичка + новостройка + студия)."""
html = _serp_html(_OFFERS)
old_lots = _make_old()._parse_serp_html(html)
new_lots = _make_new()._parse_serp_html(html)
assert len(old_lots) == len(new_lots) == 3
assert _dump(old_lots) == _dump(new_lots)
def test_serp_offer_to_lot_parity_per_offer() -> None:
"""Пооферный _offer_to_lot: model_dump идентичен для каждого offer."""
old, new = _make_old(), _make_new()
for offer in _OFFERS:
old_lot = old._offer_to_lot(offer)
new_lot = new._offer_to_lot(offer)
assert (old_lot is None) == (new_lot is None)
if old_lot is not None and new_lot is not None:
assert old_lot.model_dump() == new_lot.model_dump()
def test_serp_extract_total_offers_parity() -> None:
html = _serp_html(_OFFERS)
assert _make_old()._extract_total_offers(html) == _make_new()._extract_total_offers(html)
def test_format_address_parity() -> None:
"""_format_address (pure) — вкл. восстановление дома для ЖК без части house."""
building = {"houseNumber": None}
nb = {"name": "ЖК Парковый квартал 29А к1"}
cases = [
# (address_parts, is_newbuilding kwargs)
([], {}),
(
[
{"type": "location", "fullName": "Свердловская область"},
{"type": "street", "fullName": "улица Мира"},
{"type": "house", "fullName": "12"},
],
{},
),
(
[
{"type": "location", "fullName": "Свердловская область"},
{"type": "street", "fullName": "улица Новая"},
],
{
"building": building,
"building_cadastral_number": "66:41:0204016:11396",
"newbuilding": nb,
"is_newbuilding": True,
},
),
]
for parts, kwargs in cases:
assert old_format_address(parts, **kwargs) == new_format_address(parts, **kwargs)
def test_extract_state_parity() -> None:
"""app.extract_state ≡ scraper_kit.extract_state на SERP-фикстуре."""
html = _serp_html(_OFFERS)
html = _serp_html_for_state({"results": {"totalOffers": 1, "offers": []}})
old_state = old_extract_state(html, mfe="frontend-serp", key="initialState")
new_state = new_extract_state(html, mfe="frontend-serp", key="initialState")
assert old_state == new_state
assert new_state is not None
# ── detail fetch_detail parity ───────────────────────────────────────────────
def _detail_html() -> str:
"""defaultState + bti sister → _cianConfig['frontend-offer-card'] push HTML."""
default_state = {
"offerData": {
"offer": {
"cianId": 327830237,
"windowsViewType": "yardAndStreet",
"separateWcsCount": 1,
"combinedWcsCount": 0,
"building": {"ceilingHeight": 2.65},
"repairType": "cosmetic",
"kitchenArea": "11.5",
"description": "Отличная квартира",
"priceChanges": [
{"changeTime": "2026-04-01T00:00:00Z", "price": 5500000, "diffPercent": -1.8}
],
"agent": {
"id": 7654321,
"companyName": "Первичка Плюс",
"isPro": True,
"skills": [],
"accountType": "specialist",
},
"newbuilding": {"id": 0},
},
"stats": {
"totalViewsFormattedString": "1 042",
"todayViewsFormattedString": "7",
},
}
}
bti_state = {"houseData": {"series": "1-464", "wallMaterial": "brick"}}
ds_json = json.dumps(default_state, ensure_ascii=False)
bti_json = json.dumps(bti_state, ensure_ascii=False)
return (
"<html><body><script>\n"
"window._cianConfig['frontend-offer-card'] = "
"window._cianConfig['frontend-offer-card'] || [];\n"
"window._cianConfig['frontend-offer-card'].push({"
f"key: 'defaultState', value: {ds_json}, priority: 10, filter: null"
"});\n"
"window._cianConfig['frontend-offer-card'].push({"
f"key: 'bti', value: {bti_json}, priority: 5, filter: null"
"});\n</script></body></html>"
)
def _mock_session(html: str) -> MagicMock:
response = MagicMock()
response.status_code = 200
response.text = html
session = MagicMock()
session.get = AsyncMock(return_value=response)
session.close = AsyncMock()
return session
@pytest.mark.asyncio
async def test_detail_fetch_parity() -> None:
"""fetch_detail (defaultState + bti) → DetailEnrichment идентичен (все поля)."""
html = _detail_html()
url = "https://ekb.cian.ru/sale/flat/327830237/"
old_res = await old_fetch_detail(url, session=_mock_session(html))
new_res = await new_fetch_detail(url, session=_mock_session(html))
assert old_res is not None
assert new_res is not None
assert dataclasses.asdict(old_res) == dataclasses.asdict(new_res)
# ── valuation _parse_valuation_state parity ──────────────────────────────────
# ── valuation _parse_valuation_state parity (живой cian_valuation.py) ────────
def _valuation_state() -> dict[str, Any]:
@ -595,7 +312,7 @@ def test_kit_cian_session_config_requirement_asymmetry() -> None:
db2.execute.assert_called_once()
# ── newbuilding pure-экстракторы parity ──────────────────────────────────────
# ── newbuilding pure-экстракторы parity (живой cian_newbuilding.py) ──────────
def test_newbuilding_extractors_parity() -> None:

183
tradein-mvp/backend/tests/test_scraper_kit_domclick_golden_parity.py
View file

@ -1,183 +0,0 @@
"""Golden-parity: `scraper_kit.providers.domclick.serp` ≡ `app.services.scrapers.domclick`.
Strangler-инвариант (#2133): новая scraper_kit-копия domclick-провайдера должна
давать БАЙТ-ИДЕНТИЧНЫЙ результат парсинга старому боевому коду на одинаковом входе.
Развязка (get_scraper_delay→delay_provider, base/exceptions/repair-normalizer→
scraper_kit.*) НЕ меняет распарсенные данные.
Гоняем ОБА модуля на одной BFF-JSON-фикстуре и сравниваем:
- `_extract_json` (JSON из HTML-обёртки + QRATOR-block detection);
- `_map_item` (offer → ScrapedLot): студия rooms=0, sentinel 0→None, agency;
- `_is_geo_ok` (bbox + offerRegionName guard);
- url-билдеры и чистые хелперы (_build_offers_url/_build_count_url/
_parse_publish_date/_extract_agency_name).
Идентичность результата = kit-парсер верно скопирован (развязка не задела логику).
"""
from __future__ import annotations
import json
import os
from pathlib import Path
from types import SimpleNamespace
from typing import Any
from unittest.mock import patch
# Старый app.services.scrapers.domclick тянет app.core.config.settings=Settings(),
# которому нужен DATABASE_URL. Офлайн-парсинг БД не трогает — фиктивный DSN достаточен.
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
# НОВЫЙ (scraper_kit) — ruff сортирует scraper_kit (first-party) выше app-импортов.
from scraper_kit.providers.domclick.serp import DomClickScraper as NewDomClickScraper
from scraper_kit.providers.domclick.serp import _build_count_url as new_build_count_url
from scraper_kit.providers.domclick.serp import _build_offers_url as new_build_offers_url
from scraper_kit.providers.domclick.serp import _extract_agency_name as new_extract_agency
from scraper_kit.providers.domclick.serp import _extract_json as new_extract_json
from scraper_kit.providers.domclick.serp import _parse_publish_date as new_parse_publish_date
from app.services.scrapers.domclick import DomClickScraper as OldDomClickScraper
from app.services.scrapers.domclick import _build_count_url as old_build_count_url
from app.services.scrapers.domclick import _build_offers_url as old_build_offers_url
from app.services.scrapers.domclick import _extract_agency_name as old_extract_agency
from app.services.scrapers.domclick import _extract_json as old_extract_json
from app.services.scrapers.domclick import _parse_publish_date as old_parse_publish_date
_FIXTURES = Path(__file__).parent / "fixtures"
_BFF_JSON = (_FIXTURES / "domclick_bff_offers_sample.json").read_text(encoding="utf-8")
_BFF_ITEMS: list[dict[str, Any]] = json.loads(_BFF_JSON)["result"]["items"]
def _make_old() -> OldDomClickScraper:
"""Старый DomClickScraper без DB: patch get_scraper_delay."""
with patch("app.services.scrapers.domclick.get_scraper_delay", return_value=8.0):
return OldDomClickScraper()
def _make_new() -> NewDomClickScraper:
"""Новый DomClickScraper с инжектируемым ScraperConfig + delay_provider.
Парсинг-тесты не дёргают fetch_city → browser_http_endpoint не используется,
но конструктор требует config (duck-typed namespace достаточен).
"""
config = SimpleNamespace(browser_http_endpoint="http://tradein-browser:9000")
return NewDomClickScraper(config, delay_provider=lambda _name: 8.0)
# ── _extract_json parity ──────────────────────────────────────────────────────
def test_extract_json_parity_bare() -> None:
"""_extract_json на bare JSON-теле идентичен."""
assert old_extract_json(_BFF_JSON) == new_extract_json(_BFF_JSON)
def test_extract_json_parity_pre_wrapped() -> None:
"""_extract_json на <pre>-обёрнутом JSON идентичен (реальная форма ответа)."""
wrapped = f"<html><body><pre>{_BFF_JSON}</pre></body></html>"
assert old_extract_json(wrapped) == new_extract_json(wrapped)
def test_extract_json_parity_qrator_block() -> None:
"""QRATOR block-страница → DomClickBlockedError в обоих (идентичное поведение)."""
from scraper_kit.domclick_exceptions import DomClickBlockedError as NewBlocked
from app.services.scrapers.domclick_exceptions import DomClickBlockedError as OldBlocked
block_html = "<html><body>Система защиты QRATOR: captcha</body></html>"
old_raised = new_raised = False
try:
old_extract_json(block_html)
except OldBlocked:
old_raised = True
try:
new_extract_json(block_html)
except NewBlocked:
new_raised = True
assert old_raised and new_raised
# ── _map_item parity (offer → ScrapedLot) ─────────────────────────────────────
def test_map_item_parity_all_items() -> None:
"""Каждый offer-item → ScrapedLot идентичен (студия rooms=0, sentinel 0→None)."""
old = _make_old()
new = _make_new()
for item in _BFF_ITEMS:
# force_rooms=0 для студий (st-бакет) — проверяем обе ветки маппинга.
for force_rooms in (None, 0):
old_lot = old._map_item(item, force_rooms=force_rooms)
new_lot = new._map_item(item, force_rooms=force_rooms)
if old_lot is None or new_lot is None:
assert old_lot is None and new_lot is None
continue
assert old_lot.model_dump() == new_lot.model_dump()
# Счётчик parse-ошибок эволюционирует одинаково.
assert old.parse_failures == new.parse_failures
# ── _is_geo_ok parity (bbox + region guard) ───────────────────────────────────
def test_is_geo_ok_parity_all_items() -> None:
"""Гео-гард (offerRegionName + bbox) идентичен на всех items."""
old = _make_old()
new = _make_new()
for item in _BFF_ITEMS:
assert old._is_geo_ok(item) == new._is_geo_ok(item)
# ── url-билдеры + чистые хелперы ──────────────────────────────────────────────
def test_url_builders_parity() -> None:
"""_build_offers_url / _build_count_url идентичны (вкл. '5+' → '5%2B')."""
cases: list[tuple[str, int | None, int | None]] = [
("st", None, None),
("2", 3_000_000, 8_000_000),
("5+", 15_000_000, None),
("1", None, 5_000_000),
]
for rooms, gte, lte in cases:
assert old_build_count_url(rooms, gte, lte) == new_build_count_url(rooms, gte, lte)
for offset in (0, 20, 1980):
assert old_build_offers_url(rooms, gte, lte, offset) == new_build_offers_url(
rooms, gte, lte, offset
)
def test_parse_publish_date_parity() -> None:
"""_parse_publish_date идентичен на валидных/битых ISO-строках."""
for iso in [None, "", "2026-06-28T09:15:00+05:00", "2026-06-27", "not-a-date", "2026-13-40"]:
assert old_parse_publish_date(iso) == new_parse_publish_date(iso)
def test_extract_agency_name_parity() -> None:
"""_extract_agency_name идентичен на строковых/dict/пустых seller-блоках."""
sellers: list[dict[str, Any]] = [
{},
{"company": "АН Этажи", "agent": "Иванов"},
{"company": " ", "agent": {"name": "Резерв"}},
{"agent": {"fullName": "Петров П."}},
{"company": {"title": "СберРиелт"}},
{"agent": 12345},
]
for seller in sellers:
assert old_extract_agency(seller) == new_extract_agency(seller)
# ── strangler-guard: kit-провайдер не импортирует app.* ───────────────────────
def test_kit_domclick_has_no_app_imports() -> None:
"""Ни один модуль scraper_kit.providers.domclick.* не импортирует app.* (развязка)."""
import inspect
from scraper_kit.providers.domclick import serp
source = inspect.getsource(serp)
for line in source.splitlines():
stripped = line.strip()
assert not stripped.startswith("from app"), f"serp: {line!r}"
assert not stripped.startswith("import app"), f"serp: {line!r}"

60
tradein-mvp/backend/tests/test_scraper_kit_group_c_backfill_kit_parity.py
View file

@ -32,16 +32,21 @@ Migrates legacy scraper imports of these files to their `scraper_kit` equivalent
This file proves the DELTA introduced by the #2310 migration specifically —
general avito/cian-provider parsing golden-parity is already covered by
`tests/test_scraper_kit_avito_golden_parity.py` and
`tests/test_scraper_kit_avito_golden_parity.py` and (while it existed)
`tests/test_scraper_kit_cian_golden_parity.py`. Uses `tests/support/parity.assert_parity`
per the epic's parity-gate convention (issue #2304).
`test_cian_fetch_detail_via_browser_fetcher_parity` (legacy `app.services.scrapers.
cian_detail.fetch_detail` vs kit, browser_fetcher call shape) removed (#2397 Part E2)
— `app.services.scrapers.cian_detail` deleted, 0 runtime importers (cian_history_backfill.py
already calls the kit `fetch_detail` exclusively, proven by
`test_cian_history_backfill_browser_fetcher_uses_settings_endpoint` below).
"""
from __future__ import annotations
import asyncio
import dataclasses
import json
import os
from typing import Any
from unittest.mock import AsyncMock, MagicMock, patch
@ -53,12 +58,10 @@ os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/
from scraper_kit.avito_exceptions import AvitoBlockedError as KitAvitoBlockedError
from scraper_kit.providers.avito.detail import fetch_detail as kit_avito_fetch_detail
from scraper_kit.providers.avito.serp import AvitoScraper as KitAvitoScraper
from scraper_kit.providers.cian.detail import fetch_detail as kit_cian_fetch_detail
from app.core.config import settings
from app.services.scraper_adapters import RealScraperConfig
from app.services.scrapers.avito_detail import fetch_detail as legacy_avito_fetch_detail
from app.services.scrapers.cian_detail import fetch_detail as legacy_cian_fetch_detail
from tests.support.parity import assert_parity
# ── avito fetch_detail: backconnect config-gate (avito_detail_backfill.py delta) ──
@ -144,30 +147,7 @@ def test_avito_scraper_rotate_ip_reads_injected_config() -> None:
assert scraper._config.avito_proxy_rotate_url == settings.avito_proxy_rotate_url
# ── cian fetch_detail via browser_fetcher (cian_history_backfill.py call shape) ──
def _cian_detail_html() -> str:
default_state = {
"offerData": {
"offer": {
"cianId": 777,
"priceChanges": [
{"changeTime": "2026-06-01T00:00:00Z", "price": 5900000, "diffPercent": -0.5}
],
},
"stats": {},
}
}
ds_json = json.dumps(default_state, ensure_ascii=False)
return (
"<html><body><script>\n"
"window._cianConfig['frontend-offer-card'] = "
"window._cianConfig['frontend-offer-card'] || [];\n"
"window._cianConfig['frontend-offer-card'].push({"
f"key: 'defaultState', value: {ds_json}, priority: 10, filter: null"
"});\n</script></body></html>"
)
# ── avito fetch_detail via browser_fetcher (avito_detail_backfill.py browser_mode) ──
def _fake_browser_fetcher(html: str) -> Any:
@ -176,30 +156,6 @@ def _fake_browser_fetcher(html: str) -> Any:
return bf
def test_cian_fetch_detail_via_browser_fetcher_parity() -> None:
"""cian_history_backfill.py's listings block ALWAYS calls fetch_detail(url,
browser_fetcher=bf) — never the curl_cffi/config own-session path exercised by
tests/test_scraper_kit_pricehistory_session_parity.py (cian_price_history.py's
call shape). Proves parity holds for THIS call shape too."""
html = _cian_detail_html()
url = "https://ekb.cian.ru/sale/flat/777/"
def _run_legacy() -> Any:
return dataclasses.asdict(
asyncio.run(legacy_cian_fetch_detail(url, browser_fetcher=_fake_browser_fetcher(html)))
)
def _run_kit() -> Any:
return dataclasses.asdict(
asyncio.run(kit_cian_fetch_detail(url, browser_fetcher=_fake_browser_fetcher(html)))
)
assert_parity(legacy_fn=_run_legacy, kit_fn=_run_kit, fixtures=[()])
# ── avito fetch_detail via browser_fetcher (avito_detail_backfill.py browser_mode) ──
def test_avito_fetch_detail_via_browser_fetcher_parity() -> None:
"""browser_mode branch of avito_detail_backfill.py calls fetch_detail(url,
browser_fetcher=browser_fetcher) — config is never read on this path (early

131
tradein-mvp/backend/tests/test_scraper_kit_pricehistory_session_parity.py
View file

@ -17,32 +17,31 @@ cian-provider / ScrapedLot golden-parity is already covered by
`tests/test_scraper_kit_cian_golden_parity.py` and
`tests/test_scraper_kit_base_golden_parity.py`. Uses `tests/support/parity.assert_parity`
per the epic's parity-gate convention (issue #2304).
`test_fetch_detail_with_config_injection_parity` / `test_fetch_detail_own_session_proxy_
wiring_parity` (legacy `app.services.scrapers.cian_detail.fetch_detail` vs kit) removed
(#2397 Part E2) — `cian_detail` deleted, 0 runtime importers. `extract_state`
(`cian_state_parser`, still legacy-live) and `ScrapedLot` (`base`, still legacy-live)
parity below is unaffected.
"""
from __future__ import annotations
import asyncio
import dataclasses
import json
import os
from typing import Any
from unittest.mock import AsyncMock, MagicMock, patch
# Старый app.services.scrapers.cian* / app.core.config.settings требует DATABASE_URL
# при импорте (тот же паттерн, что в test_scraper_kit_cian_golden_parity.py).
# Старый app.services.scrapers.cian_state_parser/base / app.core.config.settings
# требует DATABASE_URL при импорте (тот же паттерн, что в
# test_scraper_kit_cian_golden_parity.py).
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
# Kit — то, на что теперь ссылаются мигрированные файлы (#2306).
from scraper_kit.base import ScrapedLot as KitScrapedLot
from scraper_kit.cian_state_parser import extract_state as kit_extract_state
from scraper_kit.providers.cian.detail import fetch_detail as kit_fetch_detail
from app.core.config import settings
from app.services.scraper_adapters import RealScraperConfig
# Legacy — эталон сравнения (эти модули НЕ меняются в #2306).
# Legacy — эталон сравнения (эти модули НЕ меняются в #2306, остаются live).
from app.services.scrapers.base import ScrapedLot as LegacyScrapedLot
from app.services.scrapers.cian_detail import fetch_detail as legacy_fetch_detail
from app.services.scrapers.cian_state_parser import extract_state as legacy_extract_state
from tests.support.parity import assert_parity
@ -108,113 +107,3 @@ def test_scraped_lot_model_dump_parity() -> None:
kit_fn=lambda kw: KitScrapedLot(**kw).model_dump(),
fixtures=[kwargs],
)
# ── fetch_detail / DetailEnrichment parity (cian_price_history.py dependency) ──
def _detail_html() -> str:
default_state = {
"offerData": {
"offer": {
"cianId": 555111,
"priceChanges": [
{"changeTime": "2026-05-01T00:00:00Z", "price": 6100000, "diffPercent": -1.2}
],
},
"stats": {},
}
}
ds_json = json.dumps(default_state, ensure_ascii=False)
return (
"<html><body><script>\n"
"window._cianConfig['frontend-offer-card'] = "
"window._cianConfig['frontend-offer-card'] || [];\n"
"window._cianConfig['frontend-offer-card'].push({"
f"key: 'defaultState', value: {ds_json}, priority: 10, filter: null"
"});\n</script></body></html>"
)
def _mock_session(html: str) -> MagicMock:
response = MagicMock()
response.status_code = 200
response.text = html
session = MagicMock()
session.get = AsyncMock(return_value=response)
session.close = AsyncMock()
return session
def _run_legacy_fetch(url: str, html: str) -> Any:
return dataclasses.asdict(asyncio.run(legacy_fetch_detail(url, session=_mock_session(html))))
def _run_kit_fetch(url: str, html: str) -> Any:
# Точно тот вызов, что теперь делает cian_price_history.backfill_cian_price_history:
# config=RealScraperConfig() добавлен по миграции (#2306); session= общий mock здесь
# только чтобы избежать сетевого I/O — сам config не участвует в session-пути, так
# что этот тест доказывает: инжекция config= не меняет результат парсинга.
result = asyncio.run(
kit_fetch_detail(url, config=RealScraperConfig(), session=_mock_session(html))
)
return dataclasses.asdict(result)
def test_fetch_detail_with_config_injection_parity() -> None:
html = _detail_html()
url = "https://ekb.cian.ru/sale/flat/555111/"
assert_parity(
legacy_fn=_run_legacy_fetch,
kit_fn=_run_kit_fetch,
fixtures=[(url, html)],
)
def test_fetch_detail_own_session_proxy_wiring_parity(monkeypatch: Any) -> None:
"""cian_price_history calls fetch_detail WITHOUT session= (own-session curl_cffi path).
Kit fetch_detail without config= silently drops the proxy (direct connection) — a
real behaviour regression risk (#806: datacenter-IP gets banned by Cian without a
proxy). This proves config=RealScraperConfig() (our actual call site) constructs
AsyncSession(proxies=...) identically to the legacy direct-settings path.
#2362 (F4b): kit's own-session branch now builds the session via
`scraper_kit.providers._base.build_curl_cffi_session()` instead of constructing
`AsyncSession` directly in `detail.py` — so the spy patches `AsyncSession` where
the constructor call now actually happens (`_base`), not in `detail` anymore.
The asserted kwargs (proxies=...) are unchanged.
"""
monkeypatch.setattr(settings, "cian_proxy_url_env", "http://test-proxy.local:8080")
captured: dict[str, dict[str, Any]] = {}
def _make_fake_session(name: str) -> Any:
class _FakeSession:
def __init__(self, **kwargs: Any) -> None:
captured[name] = kwargs
async def get(self, *_args: Any, **_kwargs: Any) -> Any:
response = MagicMock()
response.status_code = 200
response.text = _detail_html()
return response
async def close(self) -> None:
pass
return _FakeSession
url = "https://ekb.cian.ru/sale/flat/1/"
with patch("app.services.scrapers.cian_detail.AsyncSession", _make_fake_session("legacy")):
asyncio.run(legacy_fetch_detail(url))
with patch("scraper_kit.providers._base.AsyncSession", _make_fake_session("kit")):
asyncio.run(kit_fetch_detail(url, config=RealScraperConfig()))
assert captured["legacy"].get("proxies") == captured["kit"].get("proxies")
assert captured["kit"]["proxies"] == {
"http": "http://test-proxy.local:8080",
"https": "http://test-proxy.local:8080",
}

237
tradein-mvp/backend/tests/test_scraper_kit_yandex_golden_parity.py
View file

@ -1,4 +1,5 @@
"""Golden-parity: `scraper_kit.providers.yandex.*` парсинг ≡ `app.services.scrapers.yandex_*`.
"""Golden-parity: `scraper_kit.providers.yandex.*` парсинг ≡ ещё живым
`app.services.scrapers.yandex_*`.
Strangler-инвариант (#2133): новая scraper_kit-копия yandex-провайдера должна давать
БАЙТ-ИДЕНТИЧНЫЙ результат парсинга старому боевому коду на одинаковом входе.
@ -6,34 +7,34 @@ Strangler-инвариант (#2133): новая scraper_kit-копия yandex-
house_type_normalizer→providers.yandex.shared, base/browser_fetcher/helpers→scraper_kit.*)
НЕ меняет распарсенные данные.
Гоняем ОБА модуля на одних фикстурах и сравниваем результат:
- SERP `_entity_to_lot` (gate-API entity → ScrapedLot) на репрезентативных entity;
- SERP `_parse_gate_json` (payload → list[ScrapedLot]) с новостройкой/вторичкой;
- `normalize_house_type` (SCREAMING/camelCase → канон);
- newbuilding `parse` (ЖК-лендинг → YandexNewbuildingInfo) на синтетике;
- `_build_url` (gate-API URL builder) — детерминированность.
`app.services.scrapers.yandex_realty` удалён (#2397 Part E2, 0 runtime-импортёров).
SERP `_entity_to_lot`/`_parse_gate_json`/`_build_url` parity-тесты (сравнение с
`YandexRealtyScraper`) удалены вместе с ним (легаси-эталон исчез).
Идентичность результата = kit-парсер верно скопирован (развязка не задела логику).
Оставшееся сравнивает kit ≡ ЕЩЁ ЖИВЫЕ легаси-модули:
- `normalize_house_type` — `app.services.scrapers.house_type_normalizer` (живой);
- newbuilding `parse` (ЖК-лендинг → YandexNewbuildingInfo) —
`app.services.scrapers.yandex_newbuilding` (живой);
- strangler-guard: kit-провайдер не импортирует `app.*`.
Легаси `app.services.scrapers.yandex_detail` / `yandex_valuation` удалены (#2277
финальный шаг scraper_kit-миграции, 0 runtime importers) — detail/valuation
parity-проверки убраны отсюда вместе с ними. Kit-side coverage для detail/valuation
parity-проверки убраны отсюда ранее. Kit-side coverage для detail/valuation
`.parse()` сохранена без изменений в `tests/test_yandex_detail.py` /
`tests/test_yandex_detail_structural.py` / `tests/test_yandex_valuation.py` /
`tests/test_yandex_valuation_save.py` (переведены на kit-эквиваленты тем же PR).
`yandex_realty` / `yandex_newbuilding` / `house_type_normalizer` остаются легаси
(не мигрированы) — их parity-проверки не тронуты.
Идентичность результата = kit-парсер верно скопирован (развязка не задела логику).
"""
from __future__ import annotations
import inspect
import os
from types import SimpleNamespace
from typing import Any
# Старый app.services.scrapers.yandex_* импортирует app.core.config.settings=Settings(),
# которому нужен DATABASE_URL. Офлайн-парсинг БД не трогает — фиктивный DSN достаточен.
# Старый app.services.scrapers.house_type_normalizer/yandex_newbuilding импортирует
# app.core.config.settings=Settings(), которому нужен DATABASE_URL. Офлайн-парсинг
# БД не трогает — фиктивный DSN достаточен.
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
# НОВЫЙ (scraper_kit) и СТАРЫЙ (app.services.scrapers) yandex-парсеры — оба в одном
@ -41,9 +42,6 @@ os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/
from scraper_kit.providers.yandex.newbuilding import (
YandexNewbuildingScraper as NewNewbuildingScraper,
)
from scraper_kit.providers.yandex.serp import YandexRealtyScraper as NewRealtyScraper
from scraper_kit.providers.yandex.serp import _entity_to_lot as new_entity_to_lot
from scraper_kit.providers.yandex.serp import _parse_gate_json as new_parse_gate_json
from scraper_kit.providers.yandex.shared import normalize_house_type as new_normalize_house_type
from app.services.scrapers.house_type_normalizer import (
@ -52,192 +50,6 @@ from app.services.scrapers.house_type_normalizer import (
from app.services.scrapers.yandex_newbuilding import (
YandexNewbuildingScraper as OldNewbuildingScraper,
)
from app.services.scrapers.yandex_realty import YandexRealtyScraper as OldRealtyScraper
from app.services.scrapers.yandex_realty import _entity_to_lot as old_entity_to_lot
from app.services.scrapers.yandex_realty import _parse_gate_json as old_parse_gate_json
# Инжектируемый конфиг для kit-скрапперов (duck-typed namespace, structurally
# удовлетворяет scraper_kit.contracts.ScraperConfig в объёме читаемых полей).
_KIT_CONFIG = SimpleNamespace(
scraper_proxy_url=None,
yandex_proxy_rotate_url=None,
avito_proxy_rotate_url=None,
)
# ── Fixtures: gate-API entities (mirror test_yandex_realty_serp) ─────────────
_ENTITY_FULL: dict = {
"offerId": 4740475460451078271,
"url": "//realty.yandex.ru/offer/4740475460451078271",
"price": {"currency": "RUR", "value": 4500000, "valuePerPart": 119048},
"area": {"value": 37.8, "unit": "SQUARE_METER"},
"livingSpace": {"value": 20, "unit": "SQUARE_METER"},
"kitchenSpace": {"value": 12.7, "unit": "SQUARE_METER"},
"roomsTotal": 1,
"floorsOffered": [8],
"floorsTotal": 25,
"ceilingHeight": 2.7,
"building": {
"builtYear": 2016,
"buildingType": "MONOLIT",
"siteId": 280938,
"siteName": "Peremen",
},
"location": {
"geocoderAddress": "Ekaterinburg, ulitsa Evgenia Savkova, 8",
"point": {"latitude": 56.79512, "longitude": 60.49186},
},
"mainImages": [
"//avatars.mds.yandex.net/get-realty-offers/11904018/abc/main",
"//avatars.mds.yandex.net/get-realty-offers/14717586/def/main",
],
}
_ENTITY_STUDIO: dict = {
"offerId": 9999999,
"url": "//realty.yandex.ru/offer/9999999",
"price": {"value": 3200000, "valuePerPart": 114285},
"area": {"value": 28.0},
"livingSpace": None,
"kitchenSpace": None,
"roomsTotal": None, # studio -> rooms=0
"floorsOffered": [2],
"floorsTotal": 9,
"ceilingHeight": None,
"building": {"builtYear": 2010, "buildingType": "PANEL", "siteId": None, "siteName": None},
"location": {
"geocoderAddress": "Ekaterinburg, ulitsa Lenina, 5",
"point": {"latitude": 56.838, "longitude": 60.597},
},
"mainImages": [],
}
_ENTITY_RICH_OWNER: dict = {
"offerId": 5500000000000000001,
"url": "//realty.yandex.ru/offer/5500000000000000001",
"creationDate": "2026-05-24T15:10:27Z",
"description": "Продаётся светлая квартира с видом на парк.",
"author": {"category": "OWNER", "agentName": "Иван", "organization": None},
"price": {"value": 7200000, "valuePerPart": 150000, "trend": "DECREASED", "previous": 7500000},
"predictions": {"predictedPrice": {"value": "7554000", "min": "7100000", "max": "8000000"}},
"area": {"value": 48.0},
"roomsTotal": 2,
"floorsOffered": [5],
"floorsTotal": 12,
"building": {"builtYear": 2018, "buildingType": "MONOLIT"},
"location": {
"geocoderAddress": "Ekaterinburg, ulitsa Mira, 10",
"point": {"latitude": 56.84, "longitude": 60.61},
"metroList": [
{"name": "Ploshchad 1905 goda", "timeToMetro": 7},
{"name": "Geologicheskaya", "timeToMetro": 12},
],
},
"mainImages": [],
}
_ENTITY_RICH_AGENCY: dict = {
"offerId": 5500000000000000002,
"url": "//realty.yandex.ru/offer/5500000000000000002",
"creationDate": "2026-06-01T09:00:00Z",
"description": "Агентская продажа.",
"author": {"category": "AGENCY", "agentName": "Петров", "organization": "Агентство «Диал»"},
"price": {"value": 9900000, "trend": "UNCHANGED"},
"area": {"value": 60.0},
"roomsTotal": 3,
"floorsOffered": [3],
"floorsTotal": 9,
"building": {"builtYear": 2012, "buildingType": "BRICK"},
"location": {
"geocoderAddress": "Ekaterinburg, ulitsa Lenina, 1",
"point": {"latitude": 56.83, "longitude": 60.6},
"metro": {"name": "Dinamo", "timeToMetro": 4},
},
"mainImages": [],
}
# #2235: novostroyka offer whose gate-API `url` is a developer/redirect landing
# (shared by many flats). Both parsers must canonicalise it to offer_id-based url.
_ENTITY_PARTNER_URL: dict = {
"offerId": 7416316697684447350,
"url": "http://na100.pro/go.php?link=LtWgA6kWd3xjlrUBCLl6VGjo",
"price": {"value": 7790000, "valuePerPart": 130000},
"area": {"value": 60.0},
"roomsTotal": 2,
"floorsOffered": [5],
"floorsTotal": 16,
"building": {"builtYear": 2024, "buildingType": "MONOLIT", "siteId": 999001},
"location": {
"geocoderAddress": "Ekaterinburg, ulitsa Titova, 1",
"point": {"latitude": 56.8, "longitude": 60.6},
},
"mainImages": [],
}
_ALL_ENTITIES = [
_ENTITY_FULL,
_ENTITY_STUDIO,
_ENTITY_RICH_OWNER,
_ENTITY_RICH_AGENCY,
_ENTITY_PARTNER_URL,
]
def _make_gate_payload(entities: list, pager: dict | None = None) -> dict:
if pager is None:
pager = {"page": 0, "pageSize": 20, "totalItems": len(entities), "totalPages": 1}
return {"response": {"search": {"offers": {"entities": entities, "pager": pager}}}}
def _dump_lots(lots: list[Any]) -> list[dict[str, Any]]:
return [lot.model_dump() for lot in lots]
# ── SERP _entity_to_lot parity ───────────────────────────────────────────────
def test_entity_to_lot_parity_all_entities() -> None:
"""gate-API entity → ScrapedLot идентичен на всех репрезентативных entity."""
for entity in _ALL_ENTITIES:
old_lot = old_entity_to_lot(entity)
new_lot = new_entity_to_lot(entity)
assert (old_lot is None) == (new_lot is None)
assert old_lot is not None and new_lot is not None
assert old_lot.model_dump() == new_lot.model_dump()
def test_entity_to_lot_parity_new_flat_segments() -> None:
"""new_flat=YES/NO → listing_segment идентичен (новостройка/вторичка)."""
for new_flat in ("NO", "YES"):
old_lot = old_entity_to_lot(_ENTITY_FULL, new_flat=new_flat)
new_lot = new_entity_to_lot(_ENTITY_FULL, new_flat=new_flat)
assert old_lot is not None and new_lot is not None
assert old_lot.model_dump() == new_lot.model_dump()
def test_entity_to_lot_parity_ceiling_out_of_range() -> None:
"""ceilingHeight мусор (18 / 1.6) → одинаково дропается в None (#2007)."""
for bad_ceiling in (18, 1.6):
entity = {**_ENTITY_FULL, "ceilingHeight": bad_ceiling}
old_lot = old_entity_to_lot(entity)
new_lot = new_entity_to_lot(entity)
assert old_lot is not None and new_lot is not None
assert old_lot.model_dump() == new_lot.model_dump()
# ── SERP _parse_gate_json parity ─────────────────────────────────────────────
def test_parse_gate_json_parity() -> None:
"""payload → list[ScrapedLot] идентичен (вторичка и новостройка)."""
payload = _make_gate_payload(_ALL_ENTITIES)
for new_flat in ("NO", "YES"):
old_lots = old_parse_gate_json(payload, page_param=2, new_flat=new_flat)
new_lots = new_parse_gate_json(payload, page_param=2, new_flat=new_flat)
assert len(old_lots) == len(new_lots) > 0
assert _dump_lots(old_lots) == _dump_lots(new_lots)
# ── normalize_house_type parity ──────────────────────────────────────────────
@ -296,23 +108,6 @@ def test_newbuilding_parse_parity() -> None:
assert old_res.model_dump() == new_res.model_dump()
# ── _build_url parity ────────────────────────────────────────────────────────
def test_build_url_parity() -> None:
"""gate-API URL builder идентичен на репрезентативных комбо."""
old_scraper = OldRealtyScraper()
new_scraper = NewRealtyScraper(config=_KIT_CONFIG)
combos = [
dict(page=1),
dict(page=3, rooms="2", price_min=5_000_000, price_max=7_000_000),
dict(page=1, rooms="studio", new_flat="YES"),
dict(page=2, rooms="4+", price_min=25_000_000),
]
for combo in combos:
assert old_scraper._build_url(**combo) == new_scraper._build_url(**combo)
# ── strangler-guard: kit-провайдер не импортирует app.* ──────────────────────

31
tradein-mvp/backend/tests/test_scraper_proxy.py
View file

@ -177,30 +177,13 @@ _EXPECTED_PROXIES = {"http": _PROXY_URL, "https": _PROXY_URL}
# ── YandexRealtyScraper ──────────────────────────────────────────────────────
@pytest.mark.asyncio
async def test_yandex_realty_opens_browser_fetcher(monkeypatch):
"""YandexRealtyScraper.__aenter__ opens ONE BrowserFetcher (camoufox) session.
Transport moved off the proxied curl_cffi session to BrowserFetcher (camoufox);
the mobile proxy is applied server-side by the tradein-browser service, not via a
proxies= kwarg here.
"""
from app.services.scrapers.yandex_realty import YandexRealtyScraper
mock_browser = MagicMock()
mock_browser.__aenter__ = AsyncMock(return_value=mock_browser)
mock_browser.__aexit__ = AsyncMock(return_value=None)
monkeypatch.setattr(
"app.services.scrapers.yandex_realty.BrowserFetcher", lambda *a, **k: mock_browser
)
scraper = YandexRealtyScraper()
await scraper.__aenter__()
mock_browser.__aenter__.assert_awaited_once()
assert scraper._browser is mock_browser
#
# `test_yandex_realty_opens_browser_fetcher` (legacy `app.services.scrapers.
# yandex_realty.YandexRealtyScraper.__aenter__` opens ONE BrowserFetcher session)
# removed (#2397 Part E2) — `yandex_realty` deleted, 0 runtime importers. Kit
# equivalent already covered (opens exactly one BrowserFetcher session via
# `build_browser_fetcher`) by `test_kit_serp_proxy_pool.py::
# test_yandex_serp_no_pool_by_default` (and the pool-wiring variants alongside it).
# ── YandexValuationScraper ───────────────────────────────────────────────────

19
tradein-mvp/backend/tests/test_snapshot_writer.py
View file

@ -9,14 +9,23 @@
5. Ошибка snapshot не прерывает основной INSERT в save_listings.
Используем unittest.mock — без реальной БД.
`app.services.scrapers.cian_detail` удалён (#2397 Part E2, 0 runtime-импортёров) —
`DetailEnrichment`/`save_detail_enrichment` ретаргетированы на
`scraper_kit.providers.cian.detail` (byte-identical strangler-копия; `snapshot_writer`
тестируется по-прежнему через живой `app.services.scrapers.snapshot_writer` —
kit и app-версии этого модуля идентичны, kit's `save_detail_enrichment` импортирует
`upsert_listing_snapshot` из `scraper_kit.snapshot_writer`).
"""
from __future__ import annotations
from contextlib import contextmanager
from datetime import date
from unittest.mock import MagicMock, patch
from app.services.scrapers.cian_detail import DetailEnrichment, save_detail_enrichment
from scraper_kit.providers.cian.detail import DetailEnrichment, save_detail_enrichment
from app.services.scrapers.snapshot_writer import upsert_listing_snapshot
# ── Helpers ───────────────────────────────────────────────────────────────────
@ -220,7 +229,7 @@ def test_save_detail_enrichment_snapshot_failure_does_not_abort():
db = _mock_db_detail(price_rub=4_000_000)
with patch(
"app.services.scrapers.cian_detail.upsert_listing_snapshot",
"scraper_kit.providers.cian.detail.upsert_listing_snapshot",
side_effect=RuntimeError("db snapshot blip"),
):
enrichment = DetailEnrichment(ceiling_height=2.5)
@ -261,9 +270,9 @@ def test_save_detail_enrichment_oph_on_conflict_constraint():
sqls = _get_all_sqls(db)
oph_sqls = [s for s in sqls if "offer_price_history" in s]
assert oph_sqls, "INSERT offer_price_history не найден"
assert "offer_price_history_listing_change_uq" in oph_sqls[0], (
"ON CONFLICT должен ссылаться на UNIQUE constraint"
)
assert (
"offer_price_history_listing_change_uq" in oph_sqls[0]
), "ON CONFLICT должен ссылаться на UNIQUE constraint"
def test_save_detail_enrichment_skips_price_change_without_change_time():

7
tradein-mvp/backend/tests/test_yandex_city_sweep.py
View file

@ -11,7 +11,9 @@ Part E1) — эквивалентная regression-coverage теперь в
Coverage:
- YandexCitySweepCounters корректно заполняются (dataclass shape)
- Combos watchdog timeout существенно больше ANCHOR_TIMEOUT_SEC (arithmetic-only)
- Combos watchdog timeout существенно больше ANCHOR_TIMEOUT_SEC (arithmetic-only;
`DEFAULT_PRICE_RANGES`/`ROOM_PATH` — kit `scraper_kit.providers.yandex.serp`,
ретаргетировано с удалённого `app.services.scrapers.yandex_realty`, #2397 Part E2)
- Admin endpoints (start/cancel/list-runs) — kit sweep вызван с DI
(config/matcher/enrichment)
"""
@ -194,8 +196,7 @@ def test_combos_sweep_timeout_substantially_larger_than_anchor_timeout() -> None
_YANDEX_COMBOS_PER_FETCH_S,
ANCHOR_TIMEOUT_SEC,
)
from app.services.scrapers.yandex_realty import DEFAULT_PRICE_RANGES, ROOM_PATH
from scraper_kit.providers.yandex.serp import DEFAULT_PRICE_RANGES, ROOM_PATH
# Параметры prod-режима
rooms_list = list(ROOM_PATH.keys()) # 5 комнатностей

1363
tradein-mvp/backend/tests/test_yandex_realty_serp.py
View file

File diff suppressed because it is too large Load diff

13
tradein-mvp/backend/tests/test_yandex_scrapers_delay_wiring.py
View file

@ -1,8 +1,10 @@
"""Smoke: each Yandex scraper instance reads request_delay_sec from scraper_settings.
Легаси `app.services.scrapers.yandex_detail` / `yandex_valuation` удалены (#2277
финальный шаг scraper_kit-миграции). `yandex_realty` / `yandex_newbuilding` пока
остаются легаси (не мигрированы) — их delay-wiring тесты не тронуты.
финальный шаг scraper_kit-миграции). `yandex_newbuilding` пока остаётся легаси (не
мигрирован) — его delay-wiring тест не тронут. `yandex_realty` удалён (#2397 Part
E2, 0 runtime-импортёров) — его delay-wiring тест убран (kit `YandexRealtyScraper`
delay-provider wiring покрыт `test_kit_serp_proxy_pool.py`).
`yandex_detail`/`yandex_valuation` переведены на kit-эквиваленты, которые используют
инжектируемый `delay_provider=` (strangler DI, #2133) вместо прямого импорта
`get_scraper_delay` — тест адаптирован под реальный контракт kit-класса.
@ -15,7 +17,6 @@ from scraper_kit.providers.yandex.detail import YandexDetailScraper
from scraper_kit.providers.yandex.valuation import YandexValuationScraper
from app.services.scrapers.yandex_newbuilding import YandexNewbuildingScraper
from app.services.scrapers.yandex_realty import YandexRealtyScraper
_KIT_CONFIG = SimpleNamespace(
scraper_proxy_url=None,
@ -24,12 +25,6 @@ _KIT_CONFIG = SimpleNamespace(
)
def test_yandex_realty_picks_up_delay():
with patch("app.services.scrapers.yandex_realty.get_scraper_delay", return_value=9.5):
s = YandexRealtyScraper()
assert s.request_delay_sec == 9.5
def test_yandex_detail_picks_up_delay():
"""Kit YandexDetailScraper: delay_provider callable вызывается с self.name."""
s = YandexDetailScraper(delay_provider=lambda _name: 9.5)

359
tradein-mvp/scripts/local-sweep-ekb-cian.py
View file

@ -1,359 +0,0 @@
"""Cian ЕКБ vtorichka sweep — local residential IP → prod tradein DB через SSH tunnel.
Why: ekb.cian.ru SERP отдаёт ВЕСЬ город (без geo-фильтра по lat/lon), поэтому
"10 anchors" из Avito-эталона тут бесполезны — одинаковый ответ × 10.
Вместо anchors — proper room-segmented citywide pagination
(rooms 1/2/3/4 × N pages × ~28 listings), per-page save (visible progress +
не теряем данные при крэше).
Cian реально менее агрессивен чем Avito, но **residential IP всё равно может попасть
в blacklist при слишком частых запросах** — default delay 15s.
Setup (один раз):
ssh -f -N -L 15433:172.20.0.2:5432 gendesign
cd tradein-mvp/backend
$env:DATABASE_URL='postgresql+psycopg://tradein:<PASS>@localhost:15433/tradein'
$env:ENVIRONMENT='local'; $env:GLITCHTIP_DSN=''
Usage:
uv run python ../scripts/local-sweep-ekb-cian.py # default: 4 rooms × 8 pages
uv run python ../scripts/local-sweep-ekb-cian.py --pages 20 # больше глубины
uv run python ../scripts/local-sweep-ekb-cian.py --delay 10 # быстрее (risk)
uv run python ../scripts/local-sweep-ekb-cian.py --rooms 0 # всё (включая студии — no room param)
"""
from __future__ import annotations
# stdlib
import argparse
import asyncio
import json
import sys
import time
import types
from pathlib import Path
# Auto-inject backend/ в sys.path чтобы найти app.* package
_BACKEND_DIR = Path(__file__).resolve().parent.parent / "backend"
if str(_BACKEND_DIR) not in sys.path:
sys.path.insert(0, str(_BACKEND_DIR))
# Stub weasyprint (Windows без GTK ломает import chain через trade_in.py)
for _m in (
"weasyprint",
"weasyprint.text",
"weasyprint.text.ffi",
"weasyprint.css",
"weasyprint.css.targets",
):
sys.modules[_m] = types.ModuleType(_m)
sys.modules["weasyprint"].HTML = type("HTML", (), {"__init__": lambda self, *a, **kw: None})
sys.modules["weasyprint"].CSS = type("CSS", (), {"__init__": lambda self, *a, **kw: None})
# app imports после stubs
import logging # noqa: E402
import re # noqa: E402
from app.core.db import SessionLocal # noqa: E402
from app.services.scrapers.base import ScrapedLot, save_listings # noqa: E402
from app.services.scrapers.cian import CianScraper # noqa: E402
# Шумный WARNING из upstream cian_state_parser ('state extraction failed') — у нас
# fallback на новый .concat() markup отрабатывает, warning нерелевантен.
logging.getLogger("app.services.scrapers.cian").setLevel(logging.ERROR)
# ── New-markup fallback parser ───────────────────────────────────────────────
# Cian переключил MFE bootstrap с .push({...}) на .concat([{...},{...}]):
# window._cianConfig['frontend-serp'] = (window._cianConfig['frontend-serp'] || []).concat([…])
# Существующий cian_state_parser.py не ловит этот шаблон → 0 lots на свежей выдаче.
# Чтобы не трогать shared scraper module (правилами запрещено), парсим тут.
_RE_CONCAT_START = re.compile(
r"window\._cianConfig\['(?P<mfe>[\w-]+)'\]\s*=\s*"
r"\(window\._cianConfig\['[\w-]+'\]\s*\|\|\s*\[\]\)\.concat\(\["
)
def _extract_balanced_array(text: str, start: int) -> str | None:
"""Из позиции `start` (где '[') вернуть текст массива включая внешние '[' и ']'.
JSON-aware: учитывает strings и escapes, чтобы не сбить счётчик '[' ']' на
скобках внутри string literals.
"""
if text[start] != "[":
return None
depth = 0
i = start
in_str = False
esc = False
while i < len(text):
ch = text[i]
if in_str:
if esc:
esc = False
elif ch == "\\":
esc = True
elif ch == '"':
in_str = False
else:
if ch == '"':
in_str = True
elif ch == "[":
depth += 1
elif ch == "]":
depth -= 1
if depth == 0:
return text[start : i + 1]
i += 1
return None
def _parse_serp_v2(scraper: CianScraper, html: str) -> list[ScrapedLot]:
"""Fallback парсер для нового шаблона .concat([...]) — ekb.cian.ru/cat.php c 2026-05."""
state: dict | None = None
for m in _RE_CONCAT_START.finditer(html):
if m.group("mfe") != "frontend-serp":
continue
# Позиция '[' — это последний символ матча минус 1 (regex кончается на '[')
arr_start = m.end() - 1
arr_str = _extract_balanced_array(html, arr_start)
if arr_str is None:
continue
try:
entries = json.loads(arr_str)
except json.JSONDecodeError:
continue
for entry in entries:
if (
isinstance(entry, dict)
and entry.get("key") == "initialState"
and isinstance(entry.get("value"), dict)
):
state = entry["value"]
break
if state is not None:
break
if state is None:
return []
offers = (state.get("results") or {}).get("offers") or []
lots: list[ScrapedLot] = []
for offer in offers:
# Cian свежий markup: descriptionMinhash = list[int] (multi-hash signatures).
# ScrapedLot ожидает str → сбрасываем чтобы _offer_to_lot вычислил SHA1 от description.
if isinstance(offer.get("descriptionMinhash"), list):
offer["descriptionMinhash"] = None
lot = scraper._offer_to_lot(offer)
if lot is not None:
lots.append(lot)
return lots
def _load_cookies_file(path: Path) -> dict[str, str]:
"""Загрузить cookies из JSON-файла.
Поддерживает два формата:
- browser export (list[{name,value,...}]) — DevTools / EditThisCookie / Cookie-Editor
- plain dict {name: value}
"""
raw = json.loads(path.read_text(encoding="utf-8"))
if isinstance(raw, list):
return {c["name"]: c["value"] for c in raw if c.get("name") and c.get("value")}
if isinstance(raw, dict):
return {str(k): str(v) for k, v in raw.items()}
raise SystemExit(f"--cookies-file: unknown JSON shape (got {type(raw).__name__})")
async def phase_citywide(
pages: int,
delay: float,
room_segments: tuple[tuple[int, ...] | None, ...],
cookies_file: Path | None = None,
start_page: int = 1,
) -> int:
"""Room-segmented citywide pagination — per-page save, incremental.
room_segments — кортеж из tuple (1,) / (2,) / ... / None (= no room filter).
None segment даёт смешанную выдачу (студии + всё подряд) — Cian
не различает их без room param.
"""
seg_labels = [
("r" + str(seg[0]) if seg else "all")
for seg in room_segments
]
print(
f"\n=== PHASE CITYWIDE: Cian ЕКБ vtorichka "
f"({len(room_segments)} segments={seg_labels} × {pages} pages, delay {delay}s) ===\n",
flush=True,
)
total = 0
cum_ins = 0
cum_upd = 0
async with CianScraper() as s:
# Override per-request delay (in addition to between-anchors delay)
s.request_delay_sec = delay
# Inject cookies (если есть): прокидываем в curl_cffi session перед warm-up
if cookies_file is not None:
cookies = _load_cookies_file(cookies_file)
assert s._cffi is not None
s._cffi.cookies.update(cookies)
print(f" loaded {len(cookies)} cookies from {cookies_file.name}", flush=True)
# Warm-up: посетить homepage чтобы получить _CIAN_GK / _yasc + не выглядеть как наглый бот
try:
assert s._cffi is not None
r_warm = await s._cffi.get(f"{s.base_url}/")
print(
f" warmup GET / → HTTP {r_warm.status_code} "
f"(cookies={len(list(s._cffi.cookies.jar))})",
flush=True,
)
# Subsequent SERP requests should look like in-site navigation
s._cffi.headers.update(
{
"Referer": f"{s.base_url}/",
"Sec-Fetch-Site": "same-origin",
"Sec-Fetch-Mode": "navigate",
"Sec-Fetch-User": "?1",
"Sec-Fetch-Dest": "document",
}
)
await asyncio.sleep(min(5.0, delay))
except Exception as e: # noqa: BLE001
print(f" warmup FAIL: {type(e).__name__}: {e}", flush=True)
for seg_idx, seg in enumerate(room_segments):
seg_label = "r" + str(seg[0]) if seg else "all"
# --start-page применяется только к первому segment'у — продолжение прерванного sweep
page_from = start_page if seg_idx == 0 else 1
for page in range(page_from, pages + 1):
# Manual fetch+parse чтобы можно было дампнуть HTML при empty results
try:
url_ = s._build_url(rooms=seg, page=page)
assert s._cffi is not None
resp = await s._cffi.get(url_)
if resp.status_code != 200:
print(
f" seg={seg_label} page={page} HTTP {resp.status_code} → skip",
flush=True,
)
await asyncio.sleep(delay)
continue
lots = s._parse_serp_html(resp.text)
if not lots:
# Fallback: новый markup .concat([...])
lots = _parse_serp_v2(s, resp.text)
if lots:
print(
f" (parsed via v2 .concat fallback)",
flush=True,
)
if not lots:
# Дамп для разбора: captcha vs пустая страница vs изменение markup
dump_path = Path(f"cian-empty-seg-{seg_label}-p{page}.html")
dump_path.write_text(resp.text, encoding="utf-8")
print(f" DUMP: {dump_path} ({len(resp.text)} bytes)", flush=True)
await s.sleep_between_requests()
except Exception as e: # noqa: BLE001
print(
f" seg={seg_label} page={page} FAIL fetch: "
f"{type(e).__name__}: {e}",
flush=True,
)
await asyncio.sleep(delay)
continue
if not lots:
print(
f" seg={seg_label} page={page}: 0 lots — end of segment "
"(либо captcha)",
flush=True,
)
break
db = SessionLocal()
try:
ins, upd = save_listings(db, lots)
cum_ins += ins
cum_upd += upd
total += len(lots)
print(
f" seg={seg_label} page={page}/{pages}: {len(lots)} lots → "
f"ins={ins} upd={upd} "
f"[cum: total={total} new={cum_ins} upd={cum_upd}]",
flush=True,
)
except Exception as e: # noqa: BLE001
print(
f" seg={seg_label} page={page} FAIL save: "
f"{type(e).__name__}: {e}",
flush=True,
)
finally:
db.close()
# scraper.fetch_around() уже спит request_delay_sec * jitter — extra sleep не нужен
print(
f" PHASE CITYWIDE TOTAL: fetched={total} ins={cum_ins} upd={cum_upd}",
flush=True,
)
return total
async def main() -> None:
p = argparse.ArgumentParser()
p.add_argument(
"--pages",
type=int,
default=8,
help="страниц на room-segment (default 8 → ~28×4×8 = ~900 до дедупа)",
)
p.add_argument(
"--delay",
type=float,
default=15,
help="sec между HTTP requests (residential-safe ≥10, default 15)",
)
p.add_argument(
"--rooms",
type=str,
default="1,2,3,4",
help="room segments через запятую (1,2,3,4) или 'all' для одного citywide "
"запроса без room param (default '1,2,3,4')",
)
p.add_argument(
"--cookies-file",
type=Path,
default=None,
help="JSON-файл с Cian cookies (browser-export или {name:value}). "
"Опционально — для обхода captcha. См. scripts/.cian-cookies.json",
)
p.add_argument(
"--start-page",
type=int,
default=1,
help="С какой страницы начать ПЕРВЫЙ сегмент (для resume прерванного sweep). "
"Остальные segments идут с p=1. Default 1.",
)
args = p.parse_args()
# Parse rooms argument
if args.rooms.strip().lower() == "all":
segments: tuple[tuple[int, ...] | None, ...] = (None,)
else:
try:
nums = [int(x.strip()) for x in args.rooms.split(",") if x.strip()]
except ValueError as e:
raise SystemExit(f"--rooms must be comma-separated ints or 'all': {e}") from e
segments = tuple((n,) for n in nums)
cookies_file: Path | None = args.cookies_file
if cookies_file is not None and not cookies_file.exists():
raise SystemExit(f"--cookies-file not found: {cookies_file}")
t0 = time.perf_counter()
await phase_citywide(
args.pages,
args.delay,
segments,
cookies_file=cookies_file,
start_page=args.start_page,
)
elapsed = time.perf_counter() - t0
print(f"\n=== ALL DONE in {elapsed / 60:.1f} min ===", flush=True)
if __name__ == "__main__":
asyncio.run(main())

376
tradein-mvp/scripts/local-sweep-ekb-yandex.py
View file

@ -1,376 +0,0 @@
"""Yandex.Недвижимость ЕКБ vtorichka sweep — runs locally через residential IP.
Why local: Yandex anti-bot (captcha redirect) включается быстрее на datacenter
IP + httpx fingerprint. Запуск с residential IP + Chrome120 TLS impersonation
(curl_cffi) + browser session cookies — проходит. Скрипт пишет в prod tradein
DB через SSH tunnel (localhost:15433 → tradein-postgres docker container).
Modes:
--mode citywide (default): один общий /vtorichniy-rynok/ sweep, ~25 страниц
× 23 cards = ~575 уникальных. Yandex SERP лимит.
--mode combos: итерация 5 rooms × 6 price ranges = 30 combos × до 25 pages.
DB-level дедуп → coverage близко к total (~4000+ vtorichka
ЕКБ). ~75 min полный прогон.
Setup:
ssh -f -N -L 15433:172.20.0.2:5432 gendesign # tradein-postgres tunnel
cd tradein-mvp/backend
export DATABASE_URL='postgresql+psycopg://tradein:<PASS>@localhost:15433/tradein'
export ENVIRONMENT='local'
export GLITCHTIP_DSN=''
Usage:
uv run python ../scripts/local-sweep-ekb-yandex.py # citywide
uv run python ../scripts/local-sweep-ekb-yandex.py --mode combos # full ~4000
uv run python ../scripts/local-sweep-ekb-yandex.py --mode combos \\
--rooms 1,2 --price-ranges 5-7,7-10 # subset
uv run python ../scripts/local-sweep-ekb-yandex.py --pages 1 # smoke
"""
from __future__ import annotations
# stdlib
import argparse
import asyncio
import json
import random
import sys
import time
import types
from pathlib import Path
from urllib.parse import urlencode
# Auto-inject backend/ в sys.path чтобы найти app.* package
_BACKEND_DIR = Path(__file__).resolve().parent.parent / "backend"
if str(_BACKEND_DIR) not in sys.path:
sys.path.insert(0, str(_BACKEND_DIR))
# Stub weasyprint (Windows без GTK ломает import chain через trade_in.py)
for _m in (
"weasyprint",
"weasyprint.text",
"weasyprint.text.ffi",
"weasyprint.css",
"weasyprint.css.targets",
):
sys.modules[_m] = types.ModuleType(_m)
sys.modules["weasyprint"].HTML = type("HTML", (), {"__init__": lambda self, *a, **kw: None})
sys.modules["weasyprint"].CSS = type("CSS", (), {"__init__": lambda self, *a, **kw: None})
# app imports после stubs
from curl_cffi.requests import AsyncSession # noqa: E402
from app.core.db import SessionLocal # noqa: E402
from app.services.scrapers.base import save_listings # noqa: E402
from app.services.scrapers.yandex_realty import YandexRealtyScraper # noqa: E402
DEFAULT_CITY = "ekaterinburg"
DEFAULT_COOKIES_FILE = Path(__file__).resolve().parent / ".yandex-cookies.json"
# Path-сегменты Yandex для room buckets (после канонизации). Ключ — slug для
# --rooms CLI; значение — segment в URL.
ROOM_PATH: dict[str, str] = {
"studio": "studiya",
"1": "odnokomnatnaya",
"2": "dvuhkomnatnaya",
"3": "trehkomnatnaya",
"4+": "4-i-bolee",
}
# Price ranges (price_min, price_max) в рублях. None — open-ended boundary.
PRICE_RANGES: list[tuple[int | None, int | None]] = [
(None, 5_000_000),
(5_000_000, 7_000_000),
(7_000_000, 10_000_000),
(10_000_000, 15_000_000),
(15_000_000, 25_000_000),
(25_000_000, None),
]
_CAPTCHA_MARKERS = ("/showcaptcha", "smartcaptcha", "checkboxcaptcha")
def _load_cookies(path: Path) -> dict[str, str]:
if not path.exists():
return {}
raw = json.loads(path.read_text(encoding="utf-8"))
return {item["name"]: item["value"] for item in raw if "name" in item and "value" in item}
def _is_captcha(html: str) -> bool:
head = html[:5000].lower()
return any(m in head for m in _CAPTCHA_MARKERS) or "докажите, что вы не робот" in head
def _build_url(
city: str,
page: int = 0,
rooms: str | None = None,
price_min: int | None = None,
price_max: int | None = None,
) -> str:
"""Yandex SERP URL. Если rooms задан — path-based бакет; иначе общий vtorichka.
newFlat=NO_DEAL принудительно отфильтровывает новостройки (нужно при
combo, т.к. Yandex canonicalize иногда дропает /vtorichniy-rynok/).
"""
if rooms:
path = f"/{city}/kupit/kvartira/{ROOM_PATH[rooms]}/vtorichniy-rynok/"
else:
path = f"/{city}/kupit/kvartira/vtorichniy-rynok/"
base = f"https://realty.yandex.ru{path}"
params: dict[str, str | int] = {}
if price_min is not None:
params["priceMin"] = price_min
if price_max is not None:
params["priceMax"] = price_max
if price_min is not None or price_max is not None:
# Гарантия vtorichka — Yandex иногда дропает /vtorichniy-rynok/ при canonicalize
params["newFlat"] = "NO_DEAL"
if page > 0:
params["page"] = page
if params:
return f"{base}?{urlencode(params)}"
return base
def _combo_label(rooms: str | None, price_min: int | None, price_max: int | None) -> str:
r = rooms or "all-rooms"
if price_min is None and price_max is None:
return f"{r}/any-price"
lo = f"{price_min // 1_000_000}M" if price_min else "0"
hi = f"{price_max // 1_000_000}M" if price_max else "inf"
return f"{r}/{lo}-{hi}"
async def sweep_one(
session: AsyncSession,
parser: YandexRealtyScraper,
city: str,
pages: int,
delay: float,
rooms: str | None = None,
price_min: int | None = None,
price_max: int | None = None,
) -> tuple[int, int, int]:
"""Один scan по фиксированному (rooms, price) — пагинация до pages или empty.
Returns: (fetched, inserted, updated)
"""
label = _combo_label(rooms, price_min, price_max)
total = cum_ins = cum_upd = 0
for page in range(pages):
url = _build_url(city, page=page, rooms=rooms, price_min=price_min, price_max=price_max)
try:
resp = await session.get(url, timeout=20)
except Exception as e: # noqa: BLE001
print(f" [{label}] page={page} FAIL fetch: {type(e).__name__}: {e}", flush=True)
await asyncio.sleep(delay)
continue
if resp.status_code != 200:
print(f" [{label}] page={page} HTTP {resp.status_code} — stop combo", flush=True)
break
if _is_captcha(resp.text):
print(
f" [{label}] page={page} ⚠️ CAPTCHA — stop combo. "
f"Подожди 30-60min или refresh .yandex-cookies.json",
flush=True,
)
raise RuntimeError("captcha")
# NBSP → space: yandex_helpers.parse_rub() багует на \xa0 в group(1) → price None
html = resp.text.replace("\xa0", " ")
lots = parser._parse_html(html, page=page)
if not lots:
print(f" [{label}] page={page}: 0 lots — end of pagination, stop", flush=True)
break
db = SessionLocal()
try:
ins, upd = save_listings(db, lots)
cum_ins += ins
cum_upd += upd
total += len(lots)
print(
f" [{label}] page={page + 1}/{pages}: {len(lots)} lots → "
f"ins={ins} upd={upd} [combo: {total}/{cum_ins}/{cum_upd}]",
flush=True,
)
except Exception as e: # noqa: BLE001
print(f" [{label}] page={page} FAIL save: {type(e).__name__}: {e}", flush=True)
finally:
db.close()
if page + 1 < pages:
await asyncio.sleep(delay * random.uniform(0.85, 1.3))
return total, cum_ins, cum_upd
async def phase_citywide(pages: int, delay: float, city: str, cookies: dict[str, str]) -> None:
print(
f"\n=== PHASE CITYWIDE: Yandex vtorichka (city={city}, {pages} pages, "
f"delay {delay}s) ===\n",
flush=True,
)
parser = YandexRealtyScraper(city=city)
async with AsyncSession(impersonate="chrome120", cookies=cookies or {}) as session:
total, ins, upd = await sweep_one(session, parser, city, pages, delay)
print(f" CITYWIDE TOTAL: fetched={total} ins={ins} upd={upd}", flush=True)
async def phase_combos(
pages: int,
delay: float,
city: str,
cookies: dict[str, str],
rooms_list: list[str],
price_ranges: list[tuple[int | None, int | None]],
combo_delay: float,
) -> None:
combos = [(r, lo, hi) for r in rooms_list for lo, hi in price_ranges]
print(
f"\n=== PHASE COMBOS: {len(combos)} combos × ≤{pages} pages "
f"(city={city}, delay {delay}s, combo-delay {combo_delay}s) ===",
flush=True,
)
parser = YandexRealtyScraper(city=city)
grand_fetched = grand_ins = grand_upd = 0
async with AsyncSession(impersonate="chrome120", cookies=cookies or {}) as session:
for i, (rooms, lo, hi) in enumerate(combos, 1):
label = _combo_label(rooms, lo, hi)
print(f"\n [{i}/{len(combos)}] combo={label}", flush=True)
try:
f, ins, upd = await sweep_one(
session, parser, city, pages, delay,
rooms=rooms, price_min=lo, price_max=hi,
)
except RuntimeError as e:
if str(e) == "captcha":
print(" ⚠️ stopping combo run — captcha", flush=True)
break
raise
grand_fetched += f
grand_ins += ins
grand_upd += upd
print(
f" [{i}/{len(combos)}] {label} done: f={f} i={ins} u={upd} "
f"[grand: f={grand_fetched} i={grand_ins} u={grand_upd}]",
flush=True,
)
if i < len(combos):
await asyncio.sleep(combo_delay * random.uniform(0.85, 1.3))
print(
f"\n COMBOS GRAND TOTAL: fetched={grand_fetched} "
f"ins={grand_ins} upd={grand_upd}",
flush=True,
)
def _parse_rooms_arg(s: str) -> list[str]:
if s == "all":
return list(ROOM_PATH)
out: list[str] = []
for tok in s.split(","):
tok = tok.strip()
if tok not in ROOM_PATH:
raise argparse.ArgumentTypeError(
f"unknown room '{tok}'. Allowed: {sorted(ROOM_PATH)} or 'all'"
)
out.append(tok)
return out
def _parse_price_ranges_arg(s: str) -> list[tuple[int | None, int | None]]:
if s == "all":
return list(PRICE_RANGES)
out: list[tuple[int | None, int | None]] = []
for tok in s.split(","):
tok = tok.strip()
if "-" not in tok:
raise argparse.ArgumentTypeError(f"range '{tok}' missing '-'")
lo_s, hi_s = tok.split("-", 1)
lo = int(float(lo_s) * 1_000_000) if lo_s else None
hi = int(float(hi_s) * 1_000_000) if hi_s else None
out.append((lo, hi))
return out
async def main() -> None:
p = argparse.ArgumentParser(description="Yandex.Недвижимость ЕКБ sweep")
p.add_argument(
"--mode",
choices=["citywide", "combos"],
default="citywide",
help="citywide = один общий vtorichka sweep (~575 уникальных). "
"combos = rooms × price = ~4000+",
)
p.add_argument(
"--pages",
type=int,
default=25,
help="max pages per sweep (Yandex SERP cap ≈ 25; stops earlier on empty)",
)
p.add_argument(
"--delay",
type=float,
default=15,
help="sec между HTTP requests внутри combo (default 15, residential-safe)",
)
p.add_argument(
"--combo-delay",
type=float,
default=20,
help="sec между combos (default 20, дополнительный jitter combos mode)",
)
p.add_argument(
"--rooms",
type=_parse_rooms_arg,
default="all",
help=f"combos: comma-list из {sorted(ROOM_PATH)} или 'all' (default)",
)
p.add_argument(
"--price-ranges",
type=_parse_price_ranges_arg,
default="all",
help="combos: comma-list 'lo-hi' в млн руб ('-5,5-7,7-10,10-15,15-25,25-') "
"или 'all' (default). Empty side = open-ended.",
)
p.add_argument(
"--city",
type=str,
default=DEFAULT_CITY,
help=f"Yandex city slug (default '{DEFAULT_CITY}')",
)
p.add_argument(
"--cookies-file",
type=Path,
default=DEFAULT_COOKIES_FILE,
help=f"Browser cookies JSON (default '{DEFAULT_COOKIES_FILE.name}'). "
f"Empty/missing → start без cookies (риск captcha).",
)
args = p.parse_args()
cookies = _load_cookies(args.cookies_file)
if cookies:
print(f"loaded {len(cookies)} cookies from {args.cookies_file}", flush=True)
else:
print(
f"⚠️ no cookies loaded ({args.cookies_file}, exists={args.cookies_file.exists()}). "
f"Yandex anti-bot risk.",
flush=True,
)
t0 = time.perf_counter()
if args.mode == "citywide":
await phase_citywide(args.pages, args.delay, args.city, cookies)
else:
await phase_combos(
args.pages, args.delay, args.city, cookies,
rooms_list=args.rooms,
price_ranges=args.price_ranges,
combo_delay=args.combo_delay,
)
elapsed = time.perf_counter() - t0
print(f"\n=== ALL DONE in {elapsed / 60:.1f} min ===", flush=True)
if __name__ == "__main__":
asyncio.run(main())

443
tradein-mvp/scripts/local-sweep-ekb.py
View file

@ -1,443 +0,0 @@
"""Full ЕКБ sweep + houses enrichment — runs locally через residential IP.
Why: prod VPS Beget IP заблокирован Avito (datacenter fingerprint → 429).
Local IP (домашний ISP) = residential → проходит. Скрипт пишет в prod DB
через SSH tunnel (localhost:15433 → tradein-postgres docker container).
Setup:
ssh -f -N -L 15433:172.20.0.2:5432 gendesign # tradein-postgres tunnel
cd tradein-mvp/backend
export DATABASE_URL='postgresql+psycopg://tradein:<PASS>@localhost:15433/tradein'
Usage:
uv run python ../scripts/local-sweep-ekb.py # defaults
uv run python ../scripts/local-sweep-ekb.py --pages 3 --detail-top 20 --delay 30
uv run python ../scripts/local-sweep-ekb.py --skip-search # только detail + houses
uv run python ../scripts/local-sweep-ekb.py --skip-houses # без houses
"""
from __future__ import annotations
# stdlib
import argparse
import asyncio
import sys
import time
import types
from pathlib import Path
from urllib.parse import urlparse
# Auto-inject backend/ в sys.path чтобы найти app.* package
# (скрипт лежит в tradein-mvp/scripts/, app — в tradein-mvp/backend/)
_BACKEND_DIR = Path(__file__).resolve().parent.parent / "backend"
if str(_BACKEND_DIR) not in sys.path:
sys.path.insert(0, str(_BACKEND_DIR))
# Stub weasyprint (Windows без GTK ломает import chain через trade_in.py)
for _m in (
"weasyprint",
"weasyprint.text",
"weasyprint.text.ffi",
"weasyprint.css",
"weasyprint.css.targets",
):
sys.modules[_m] = types.ModuleType(_m)
sys.modules["weasyprint"].HTML = type("HTML", (), {"__init__": lambda self, *a, **kw: None})
sys.modules["weasyprint"].CSS = type("CSS", (), {"__init__": lambda self, *a, **kw: None})
# app imports после stubs
from sqlalchemy import text # noqa: E402
from app.core.db import SessionLocal # noqa: E402
from app.services.scrapers.avito import AvitoScraper # noqa: E402
class CityWideAvitoScraper(AvitoScraper):
"""Subclass без geo-фильтра — search всего города (ЕКБ) через pagination.
Override `_build_web_url` чтобы НЕ добавлять `geoCoords`/`radius` — Avito
отдаёт ВСЕ объявления в городе sorted by date, до 100 pages (cap ~5000).
"""
def _build_web_url(self, lat: float, lon: float, radius_km: int, page: int = 1) -> str: # noqa: ARG002
from urllib.parse import urlencode
params = {"s": 104, "p": page} # s=104 sort by date
return f"{self.base_url}/ekaterinburg/kvartiry/prodam-ASgBAgICAUSSA8YQ?{urlencode(params)}"
# Avito URL slugs для room-filter (extracted из real search page HTML 2026-05-23)
ROOM_SLUGS: list[tuple[str, str]] = [
("студии", "studii-ASgBAgICAkSSA8YQygj~WA"),
("1-комн.", "1-komnatnye-ASgBAgICAkSSA8YQygiAWQ"),
("2-комн.", "2-komnatnye-ASgBAgICAkSSA8YQygiCWQ"),
("3-комн.", "3-komnatnye-ASgBAgICAkSSA8YQygiEWQ"),
("4-комн.", "4-komnatnye-ASgBAgICAkSSA8YQygiGWQ"),
("5+комн.", "5-komnatnye-ASgBAgICAkSSA8YQygiIWQ"),
("своб.планир.", "svobodnaya_planirovka-ASgBAgICAkSSA8YQygj8zzI"),
]
class CityRoomsAvitoScraper(AvitoScraper):
"""Subclass с URL slug filter по комнатности — для splitting когда total > 5000.
URL pattern: /ekaterinburg/kvartiry/prodam/<slug>?s=104&p=N
"""
def __init__(self, slug: str) -> None:
super().__init__()
self.room_slug = slug
def _build_web_url(self, lat: float, lon: float, radius_km: int, page: int = 1) -> str: # noqa: ARG002
from urllib.parse import urlencode
params = {"s": 104, "p": page}
return f"{self.base_url}/ekaterinburg/kvartiry/prodam/{self.room_slug}?{urlencode(params)}"
from app.services.scrapers.avito_detail import ( # noqa: E402
fetch_detail,
save_detail_enrichment,
)
from app.services.scrapers.avito_houses import ( # noqa: E402
fetch_house_catalog,
save_house_catalog_enrichment,
)
from app.services.scrapers.base import save_listings # noqa: E402
# 10 anchors покрывающих центральные/северные/южные/восточные/западные районы ЕКБ
ANCHORS: list[tuple[str, float, float]] = [
("Центр", 56.8400, 60.6050),
("ЮЗ", 56.7950, 60.5300),
("Уралмаш", 56.8970, 60.6100),
("Академический", 56.7700, 60.5500),
("Пионерский", 56.8650, 60.6200),
("Эльмаш", 56.8567, 60.6700),
("Втузгородок", 56.8400, 60.6650),
("ВИЗ", 56.8500, 60.5650),
("Юго-Запад", 56.7770, 60.5800),
("Сортировка", 56.8950, 60.5600),
]
async def phase_citywide(pages: int, delay: float) -> int:
"""City-wide pagination — ВСЕ объявления ЕКБ (no geo filter), per-page save.
Per-page save важно: visible прогресс + не теряем данные если crash mid-way.
"""
print(
f"\n=== PHASE CITYWIDE: full ЕКБ pagination ({pages} pages, delay {delay}s) ===\n",
flush=True,
)
total = 0
cum_ins = 0
cum_upd = 0
async with CityWideAvitoScraper() as s:
for page in range(1, pages + 1):
url = s._build_web_url(0.0, 0.0, 1, page=page)
try:
assert s._cffi is not None
response = await s._cffi.get(url)
if response.status_code != 200:
print(f" ⚠️ page={page} HTTP {response.status_code} — stop", flush=True)
break
lots = s._parse_html(response.text, source_url_base=url)
except Exception as e: # noqa: BLE001
print(f" page={page} FAIL fetch: {type(e).__name__}: {e}", flush=True)
await asyncio.sleep(delay)
continue
if not lots:
print(f" page={page}: 0 lots — end of pagination", flush=True)
break
db = SessionLocal()
try:
ins, upd = save_listings(db, lots)
cum_ins += ins
cum_upd += upd
total += len(lots)
print(
f" page={page}/{pages}: {len(lots)} lots → ins={ins} upd={upd} "
f"[cum: total={total} new={cum_ins} upd={cum_upd}]",
flush=True,
)
except Exception as e: # noqa: BLE001
print(f" page={page} FAIL save: {type(e).__name__}: {e}", flush=True)
finally:
db.close()
if page < pages:
await asyncio.sleep(delay)
print(f" PHASE CITYWIDE TOTAL: fetched={total} ins={cum_ins} upd={cum_upd}", flush=True)
return total
async def phase_byrooms(pages: int, delay: float) -> int:
"""Rooms-split sweep — 7 categories × pages, per-page save.
Для total > 5000 (Avito search cap) split по комнатности позволяет получить
все listings (каждый segment <5000).
"""
print(
f"\n=== PHASE BYROOMS: 7 категорий × {pages} pages, delay {delay}s ===\n",
flush=True,
)
grand_total = 0
grand_ins = 0
grand_upd = 0
import random
rate_limit_hits = 0
for ci, (name, slug) in enumerate(ROOM_SLUGS, 1):
print(f"\n--- [{ci}/{len(ROOM_SLUGS)}] Категория: {name} (slug={slug[:30]}...) ---", flush=True)
cat_total = 0
cat_ins = 0
cat_upd = 0
async with CityRoomsAvitoScraper(slug) as s:
for page in range(1, pages + 1):
url = s._build_web_url(0.0, 0.0, 1, page=page)
try:
assert s._cffi is not None
response = await s._cffi.get(url)
if response.status_code == 429:
# Exponential backoff на rate-limit: 60s × 2^N (max 30 min)
rate_limit_hits += 1
backoff = min(60 * (2 ** (rate_limit_hits - 1)), 1800)
print(f" {name} p={page} HTTP 429 (hit #{rate_limit_hits}) → backoff {backoff}s", flush=True)
await asyncio.sleep(backoff)
# retry same page after backoff
try:
response = await s._cffi.get(url)
if response.status_code != 200:
print(f" {name} p={page} retry HTTP {response.status_code} — skip page", flush=True)
await asyncio.sleep(delay * 2)
continue
except Exception as e: # noqa: BLE001
print(f" {name} p={page} retry FAIL: {type(e).__name__}: {e}", flush=True)
continue
elif response.status_code != 200:
print(f" {name} p={page} HTTP {response.status_code} — skip page", flush=True)
await asyncio.sleep(delay)
continue
lots = s._parse_html(response.text, source_url_base=url)
except Exception as e: # noqa: BLE001
print(f" {name} p={page} FAIL fetch: {type(e).__name__}: {e}", flush=True)
await asyncio.sleep(delay)
continue
if not lots:
print(f" {name} page={page}: 0 lots — end of category", flush=True)
break
db = SessionLocal()
try:
ins, upd = save_listings(db, lots)
cat_ins += ins
cat_upd += upd
cat_total += len(lots)
print(
f" {name} p={page}/{pages}: {len(lots)} lots → ins={ins} upd={upd} "
f"[cat: tot={cat_total} new={cat_ins}]",
flush=True,
)
except Exception as e: # noqa: BLE001
print(f" {name} page={page} FAIL save: {type(e).__name__}: {e}", flush=True)
finally:
db.close()
if page < pages:
# Jitter: ±30% от delay чтобы выглядеть менее robotic
jittered = delay * random.uniform(0.7, 1.3)
await asyncio.sleep(jittered)
print(
f" --- {name} TOTAL: fetched={cat_total} ins={cat_ins} upd={cat_upd} ---",
flush=True,
)
grand_total += cat_total
grand_ins += cat_ins
grand_upd += cat_upd
print(
f"\n PHASE BYROOMS GRAND TOTAL: fetched={grand_total} new={grand_ins} updated={grand_upd}",
flush=True,
)
return grand_total
async def phase_a_search(pages: int, delay: float) -> int:
"""Search всех anchors × pages, save listings."""
print(
f"\n=== PHASE A: search ({len(ANCHORS)} anchors × {pages} pages, "
f"delay {delay}s) ===\n",
flush=True,
)
total = 0
for i, (name, lat, lon) in enumerate(ANCHORS, 1):
print(f"[{i}/{len(ANCHORS)}] {name} ({lat},{lon})", flush=True)
try:
async with AvitoScraper() as s:
lots = await s.fetch_around(
lat, lon, 1500, pages=pages, delay_override_sec=delay
)
except Exception as e: # noqa: BLE001
print(f" FAIL fetch: {type(e).__name__}: {e}", flush=True)
await asyncio.sleep(delay)
continue
print(f" fetched {len(lots)} lots", flush=True)
if not lots:
print(" ⚠️ 0 lots — possible 429, stop phase A", flush=True)
break
db = SessionLocal()
try:
ins, upd = save_listings(db, lots)
print(f" saved: ins={ins} upd={upd}", flush=True)
total += len(lots)
except Exception as e: # noqa: BLE001
print(f" FAIL save: {type(e).__name__}: {e}", flush=True)
finally:
db.close()
if i < len(ANCHORS):
await asyncio.sleep(delay)
return total
async def phase_b_detail(top_n: int, delay: float) -> tuple[int, set[str]]:
"""Detail enrichment для top-N recent listings без detail.
Returns: (enriched_count, set of unique house_catalog_url paths extracted)
"""
print(
f"\n=== PHASE B: detail enrichment (top {top_n}, delay {delay}s) ===\n",
flush=True,
)
db = SessionLocal()
try:
rows = db.execute(
text(
"""
SELECT source_url, source_id FROM listings
WHERE source='avito' AND detail_enriched_at IS NULL
AND source_url IS NOT NULL AND price_rub > 0
ORDER BY scraped_at DESC NULLS LAST
LIMIT :limit
"""
),
{"limit": top_n},
).mappings().all()
finally:
db.close()
print(f" queue: {len(rows)} listings to enrich", flush=True)
enriched = 0
failed = 0
house_urls: set[str] = set()
for i, row in enumerate(rows, 1):
url = row["source_url"]
path = urlparse(url).path if url.startswith("http") else url
print(f"[{i}/{len(rows)}] detail {row['source_id']}", flush=True)
try:
enr = await fetch_detail(path)
db = SessionLocal()
try:
ok = save_detail_enrichment(db, enr)
print(
f" enriched: ok={ok} "
f"rooms={enr.rooms} area={enr.area_m2} "
f"kitchen={enr.kitchen_area_m2} owners={enr.owners_count} "
f"house_url={enr.house_catalog_url}",
flush=True,
)
if ok:
enriched += 1
# Извлекаем house path (strip query string + filter только /catalog/houses/)
if enr.house_catalog_url:
p = urlparse(enr.house_catalog_url).path
if p.startswith("/catalog/houses/"):
house_urls.add(p)
finally:
db.close()
except Exception as e: # noqa: BLE001
print(f" FAIL: {type(e).__name__}: {e}", flush=True)
failed += 1
if i < len(rows):
await asyncio.sleep(delay)
print(f" total: enriched={enriched} failed={failed} unique_houses_extracted={len(house_urls)}", flush=True)
return enriched, house_urls
async def phase_c_houses(delay: float, extra_urls: set[str] | None = None) -> int:
"""Houses enrichment — unique house_url из listings (DB) + extra_urls (из Phase B in-memory).
save_detail_enrichment не пишет house_url в listings (баг Stage 2b),
поэтому Phase C принимает extra_urls собранные Phase B вручную.
"""
print("\n=== PHASE C: houses enrichment ===\n", flush=True)
db = SessionLocal()
try:
rows = db.execute(
text(
"""
SELECT DISTINCT house_url FROM listings
WHERE source='avito' AND house_url IS NOT NULL
AND house_id_fk IS NULL
ORDER BY house_url
"""
)
).fetchall()
finally:
db.close()
urls_from_db = {r[0] for r in rows if r[0]}
urls = urls_from_db | (extra_urls or set())
print(f" queue: {len(urls)} unique houses (db={len(urls_from_db)} +extras={len(extra_urls or set())})", flush=True)
if not urls:
print(" no houses to enrich (skip)", flush=True)
return 0
enriched = 0
failed = 0
for i, url in enumerate(sorted(urls), 1):
path = urlparse(url).path if url.startswith("http") else url
print(f"[{i}/{len(urls)}] {path}", flush=True)
try:
enr = await fetch_house_catalog(path)
db = SessionLocal()
try:
counters = save_house_catalog_enrichment(db, enr)
print(f" saved: {counters}", flush=True)
enriched += 1
finally:
db.close()
except Exception as e: # noqa: BLE001
print(f" FAIL: {type(e).__name__}: {e}", flush=True)
failed += 1
if i < len(urls):
await asyncio.sleep(delay)
print(f" total: enriched={enriched} failed={failed}", flush=True)
return enriched
async def main() -> None:
p = argparse.ArgumentParser()
p.add_argument(
"--mode",
choices=["anchors", "citywide", "byrooms", "both"],
default="anchors",
help="anchors=10 anchors; citywide=full pagination без geo; byrooms=7 категорий комнат (для total>5000); both=anchors+citywide",
)
p.add_argument("--pages", type=int, default=5, help="pages per anchor (anchors mode) или total pages (citywide)")
p.add_argument("--detail-top", type=int, default=50, help="top-N для detail enrichment")
p.add_argument("--delay", type=float, default=30, help="sec между HTTP requests")
p.add_argument("--skip-search", action="store_true")
p.add_argument("--skip-detail", action="store_true")
p.add_argument("--skip-houses", action="store_true")
args = p.parse_args()
t0 = time.perf_counter()
if not args.skip_search:
if args.mode in ("anchors", "both"):
await phase_a_search(args.pages, args.delay)
if args.mode in ("citywide", "both"):
cw_pages = args.pages if args.mode == "citywide" else 100
await phase_citywide(cw_pages, args.delay)
if args.mode == "byrooms":
await phase_byrooms(args.pages, args.delay)
extracted_house_urls: set[str] = set()
if not args.skip_detail:
_, extracted_house_urls = await phase_b_detail(args.detail_top, args.delay)
if not args.skip_houses:
await phase_c_houses(args.delay, extra_urls=extracted_house_urls)
elapsed = time.perf_counter() - t0
print(f"\n=== ALL DONE in {elapsed / 60:.1f} min ===", flush=True)
if __name__ == "__main__":
asyncio.run(main())