lekss361/gendesign
1
0
Fork 0
Code Issues 57 Pull requests Projects Releases Packages Wiki Activity Actions

chore(tradein/scrapers): удалить весь legacy scrapers/ каталог — final E (#2397, #2277) #2419

Merged
lekss361 merged 1 commit from chore/tradein-finalE-delete-scrapers-catalog into main 2026-07-04 13:11:14 +00:00
Conversation 0 Commits 1 Files changed 81 +221 -19861
81 changed files with 221 additions and 19861 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
tradein-mvp/backend/app/services/cian_price_history.py
View file

@ -16,11 +16,12 @@ import time
from dataclasses import dataclass, field
# #2306: fetch_detail/save_detail_enrichment migrated to scraper_kit (byte-identical
# golden-parity была доказана против legacy app.services.scrapers.cian_detail до его
# удаления, #2397 Part E2 — см. tests/test_scraper_kit_pricehistory_session_parity.py
# для оставшейся extract_state/ScrapedLot parity). RealScraperConfig — тот же
# read-only адаптер над settings, что и остальные kit-инжекции (#2131) — сохраняет
# proxy-поведение (config.cian_proxy_url) идентичным прежнему прямому импорту settings.
# golden-parity была доказана против legacy cian_detail-модуля до его удаления,
# #2397 Part E2; extract_state/ScrapedLot parity-тесты убраны вместе с остальным
# legacy scrapers-каталогом, #2397 финальный шаг E — kit единственный живой путь).
# RealScraperConfig — тот же read-only адаптер над settings, что и остальные
# kit-инжекции (#2131) — сохраняет proxy-поведение (config.cian_proxy_url)
# идентичным прежнему прямому импорту settings.
from scraper_kit.providers.cian.detail import fetch_detail, save_detail_enrichment
from sqlalchemy import text
from sqlalchemy.orm import Session

4
tradein-mvp/backend/app/services/cian_session.py
View file

@ -13,8 +13,8 @@ from typing import Any
from curl_cffi.requests import AsyncSession
# #2306: extract_state migrated to scraper_kit (byte-identical to legacy
# app.services.scrapers.cian_state_parser, см. tests/test_scraper_kit_cian_golden_parity.py
# + tests/test_scraper_kit_pricehistory_session_parity.py).
# cian_state_parser module, deleted #2397 финальный шаг E — golden-parity против
# него уже недоступна, kit — единственный живой путь).
from scraper_kit.cian_state_parser import extract_state
from sqlalchemy import text
from sqlalchemy.orm import Session

0
tradein-mvp/backend/app/services/scrapers/__init__.py
View file

2169
tradein-mvp/backend/app/services/scrapers/avito.py
View file

File diff suppressed because it is too large Load diff

1075
tradein-mvp/backend/app/services/scrapers/avito_detail.py
View file

File diff suppressed because it is too large Load diff

36
tradein-mvp/backend/app/services/scrapers/avito_exceptions.py
View file

@ -1,36 +0,0 @@
"""Avito-specific exceptions для anti-bot detection."""
class AvitoError(Exception):
"""Base для всех Avito scraper exceptions."""
class AvitoBlockedError(AvitoError):
"""HTTP 403 от Avito — IP-level block detected."""
class AvitoRateLimitedError(AvitoError):
"""HTTP 429 от Avito — rate limit triggered."""
class AvitoListingGoneError(AvitoError):
"""Объявление удалено / вернуло 404 (страница «Ошибка 404» вместо item-view).
Отдельный сигнал от AvitoBlockedError/AvitoRateLimitedError: 404 — НЕ анти-бот
deflect, а мёртвый листинг (его незачем ротировать/ретраить). Наследует общий
AvitoError, но НЕ AvitoBlockedError — чтобы backfill-ловушка
`except (AvitoBlockedError, AvitoRateLimitedError)` его НЕ перехватывала и не крутила
consecutive-block breaker на мёртвых координатных дырах (run 458: 5 dead → abort).
"""
class AvitoContentBlockedError(AvitoBlockedError):
"""HTTP 200 от Avito, но 0 карточек на первой странице — content-block / captcha.
Наследует AvitoBlockedError, чтобы существующие except-блоки на AvitoBlockedError
продолжали ловить этот сигнал (mark_banned, pipeline abort и т.д.).
"""
class AvitoParseError(AvitoError):
"""Cannot parse Avito HTML structure (selector changes)."""

79
tradein-mvp/backend/app/services/scrapers/avito_shared.py
View file

@ -1,79 +0,0 @@
"""avito_shared.py — общие константы/хелперы для avito*.py модулей.
Единая точка правды для:
- `RUS_MONTHS` — рус. месяц (родительный падеж) → номер, для парсинга дат вида
"9 октября 2025" (было продублировано в avito.py/avito_detail.py/avito_houses.py).
- `_unix_to_date` — unix-epoch (в СЕКУНДАХ) → `date`, в таймзоне Europe/Moscow.
TZ-обоснование (issue #2129): было 3 независимые копии конвертации unix→date с
РАЗНЫМИ timezone — MSK (avito.py, документировано issue #726), UTC (avito_houses.py),
naive/system-local (avito_imv.py). Один и тот же epoch давал разные `date` в
SERP / houses / IMV путях для одного и того же объявления.
Почему MSK — канонический выбор (не UTC, не local):
- Avito — российская площадка, отображает даты объявлений (`item-date` на SERP,
`publishDate`, история цен `startPriceDate`/`lastPriceDate`/`removedDate`) в
Europe/Moscow. Объявление, поднятое в 00:00–03:00 MSK, при наивной UTC-конвертации
получило бы дату на день раньше, чем видно на самом Avito (обоснование #726).
- avito_houses.py и avito_imv.py используются на одних и тех же данных
(`housePlacementHistory` widget / IMV `placementHistory` — идентичные поля
startPriceDate/lastPriceDate/removedDate, см. fixtures/avito_imv_getdata.json) —
единственный смысл держать их согласованными с уже принятым MSK-решением из #726.
- Проверено на живой fixture: startPriceDate/lastPriceDate — всегда полночь UTC
(00:00:00), поэтому MSK vs UTC не меняет date для них. `removedDate`/`publishDate`
несут реальное время суток — там UTC/naive-local и MSK ДЕЙСТВИТЕЛЬНО расходятся
(пример из fixture: removedDate=1756077885 → UTC 2025-08-24, MSK 2025-08-25).
Это единственный реальный сдвиг данных от этого фикса: `avito_imv_evaluations` /
`house_placement_history`, поля `removed_date`/`publish_date`, у ~12.5% строк
(события в 21:00–23:59 UTC/local) дата сдвинется на +1 день при следующем прогоне
скрейпера. Схему/уже сохранённые строки не трогаем — это единственно правильное
представление на будущее.
- Fixed-offset `timezone(timedelta(hours=3))`, а не `zoneinfo("Europe/Moscow")` —
не зависит от системной tzdata в контейнере; Москва не переходит на DST с 2014,
так что fixed offset корректен всегда.
"""
from __future__ import annotations
import logging
from datetime import date, datetime, timedelta, timezone
logger = logging.getLogger(__name__)
# Europe/Moscow, UTC+3, без перехода на летнее время (с 2014).
MSK = timezone(timedelta(hours=3))
# Русские месяцы (родительный падеж) для парсинга дат вида "9 октября 2025".
RUS_MONTHS: dict[str, int] = {
"января": 1,
"февраля": 2,
"марта": 3,
"апреля": 4,
"мая": 5,
"июня": 6,
"июля": 7,
"августа": 8,
"сентября": 9,
"октября": 10,
"ноября": 11,
"декабря": 12,
}
def _unix_to_date(ts: int | float | None) -> date | None:
"""Конвертирует unix-epoch (в СЕКУНДАХ) в `date`, в таймзоне Europe/Moscow (MSK).
Возвращает `None` для `None`/`0` (Avito использует 0 как sentinel «нет даты») и
для значений, которые не конвертируются в валидную дату.
ВАЖНО: ожидает epoch в СЕКУНДАХ. Avito SERP-поле `sortTimeStamp` отдаёт epoch в
МИЛЛИСЕКУНДАХ — вызывающий код обязан поделить на 1000 до вызова (см.
`avito.py::_build_sort_timestamp_map`).
"""
if not ts:
return None
try:
return datetime.fromtimestamp(float(ts), tz=MSK).date()
except (OSError, OverflowError, ValueError):
logger.warning("Не удалось конвертировать unix timestamp: %r", ts)
return None

830
tradein-mvp/backend/app/services/scrapers/base.py
View file

@ -1,830 +0,0 @@
"""Базовый framework для парсеров объявлений (Avito / Cian / DomKlik / ...).
Общие задачи:
- httpx.AsyncClient с realistic browser headers + UA rotation
- tenacity retry с exp backoff
- Sleep между запросами (anti-ban)
- ScrapedLot — единая Pydantic схема всех источников
- Запись в `listings` Postgres с дедупом по dedup_hash
- После INSERT/UPDATE — hook в matching service для регистрации в
`listing_sources` и привязки к каноническому `houses` через `house_sources`.
Каждый конкретный парсер (avito.py, cian.py, ...) наследуется от BaseScraper
и реализует `_fetch_page()` и `_parse_listings()`.
"""
from __future__ import annotations
import asyncio
import hashlib
import logging
import random
from abc import ABC, abstractmethod
from datetime import date, datetime
from typing import Any
from zoneinfo import ZoneInfo
import httpx
import psycopg.errors
from pydantic import BaseModel, Field
from sqlalchemy import text
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session
from tenacity import retry, stop_after_attempt, wait_exponential
from app.services.matching import match_or_create_house, upsert_listing_source
from app.services.scrapers.snapshot_writer import upsert_listing_snapshot
logger = logging.getLogger(__name__)
_MSK = ZoneInfo("Europe/Moscow")
# ── Realistic browser User-Agents (Apr 2026 versions) ──────────────────────
_USER_AGENTS: list[str] = [
# Chrome on macOS
"Mozilla/5.0 (Macintosh; Intel Mac OS X 14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36", # noqa: E501
# Chrome on Windows
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36", # noqa: E501
# Firefox on macOS
"Mozilla/5.0 (Macintosh; Intel Mac OS X 14.5; rv:127.0) Gecko/20100101 Firefox/127.0",
# Safari on macOS
"Mozilla/5.0 (Macintosh; Intel Mac OS X 14_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Safari/605.1.15", # noqa: E501
# Edge on Windows
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36 Edg/132.0.0.0", # noqa: E501
]
def random_user_agent() -> str:
return random.choice(_USER_AGENTS)
# ── Унифицированная схема результата ────────────────────────────────────────
class ScrapedLot(BaseModel):
"""Результат парсинга одного объявления.
Все источники приводятся к этому виду перед записью в Postgres.
Поля optional если источник их не отдаёт.
"""
source: str # 'avito' / 'cian' / 'domklik' / ...
source_url: str # ссылка на оригинал — кликается из UI
source_id: str | None = None # внутренний id источника (для update)
# Локация — должна быть хотя бы address ИЛИ (lat, lon)
address: str | None = None
lat: float | None = None
lon: float | None = None
# Параметры квартиры
rooms: int | None = None # 0 = студия
area_m2: float | None = None
floor: int | None = None
total_floors: int | None = None
year_built: int | None = None
house_type: str | None = None
repair_state: str | None = None
has_balcony: bool | None = None
kadastr_num: str | None = None
# Detail-поля кухни/потолка (#2007). Обычно из detail-страницы (avito_detail,
# cian_detail, yandex_detail), но yandex SERP отдаёт их прямо в карточке —
# промоутим из raw_payload в колонки для дашборда покрытия и matching.
kitchen_area_m2: float | None = None
ceiling_height_m: float | None = None
# Class B promote-поля (#2008, cian SERP). Для дашборда покрытия / matching,
# НЕ для valuation — estimator их пока НЕ читает (follow-up #2012). cian отдаёт
# их прямо в карточке; раньше лежали только в raw_payload.
mortgage_available: bool | None = None # продавец допускает ипотеку
is_apartments: bool | None = None # апартаменты (НЕ квартира — иной правовой статус)
is_rosreestr_checked: bool | None = None # объявление сверено с ЕГРН
# Avito house linking (Stage 2a — search → houses)
house_source: str | None = None # 'avito' / 'cian'
house_ext_id: str | None = None # Avito's '3171365'
house_url: str | None = None # full URL of /catalog/houses/...
listing_segment: str | None = None # 'vtorichka' / 'novostroyki'
# Newbuilding (ЖК) linking — slug-id + canonical ЖК-subdomain URL
newbuilding_id: str | None = None # e.g. 'federatsiya-ekaterinburg'
newbuilding_url: str | None = None # e.g. https://zhk-federatsiya-ekaterinburg.avito.ru
# Цена (обязательно)
price_rub: int = Field(gt=0)
price_per_m2: int | None = None
# Геокодинг
# None → точность неизвестна (координаты пришли из скрейпера напрямую).
# 'city' → геокодер вернул city-centroid (нет номера дома в адресе) —
# листинг исключается из radius-аналогов estimator'а.
geo_precision: str | None = None
# Метаданные
listing_date: date | None = None
publish_date: date | None = None
days_on_market: int | None = None
description: str | None = None
photo_urls: list[str] = Field(default_factory=list)
raw_payload: dict[str, Any] | None = None
# Cian-specific extensions (Stage 2 of CianScraper v1)
living_area_m2: float | None = None
bedrooms_count: int | None = None
balconies_count: int | None = None
loggias_count: int | None = None
description_minhash: str | None = None
cadastral_number: str | None = None
building_cadastral_number: str | None = None
phones: list[dict] = Field(default_factory=list)
is_homeowner: bool | None = None
is_pro_seller: bool | None = None
bargain_allowed: bool | None = None
sale_type: str | None = None
metro_stations: list[dict] = Field(default_factory=list)
agency_name: str | None = None
# Yandex gate-API rich fields
yandex_offer_id: str | None = None # numeric string offerId (kept as text)
predicted_price_rub: int | None = None # Yandex per-offer valuation (predictedPrice.value)
predicted_price_min: int | None = None
predicted_price_max: int | None = None
price_trend: str | None = None # INCREASED / DECREASED / UNCHANGED / ...
price_previous_rub: int | None = None # price.previous
def compute_dedup_hash(self) -> str:
"""SHA256(source + stable_key) — стабильный uniqueness key.
stable_key = source_id (Cian offer_id 330200428, Avito data-item-id) если
есть; иначе source_url БЕЗ query-string. Cian/Avito URL несут волатильный
`?context=...` токен — без strip каждый ре-скрейп давал бы новый hash.
#753: price_rub в ключ НЕ входит. Смена цены того же объявления — это UPDATE
существующей строки (`ON CONFLICT (dedup_hash) DO UPDATE`), а не новый объект.
Раньше цена была частью ключа → каждое изменение цены порождало дубль
активного listing → искажение asking_to_sold_ratio / sales-vs-listings /
активного инвентаря. Смена формулы разово даёт новый hash существующим
строкам — следующий скрейп upsert'нет их по новому ключу (всплеск разовый).
"""
h = hashlib.sha256()
h.update(self.source.encode())
h.update(b"|")
key = self.source_id if self.source_id else (self.source_url or "").split("?")[0]
h.update((key or "").encode())
return h.hexdigest()
def compute_card_hash(self) -> str:
"""SHA-256 over the volatile SERP-card fields. Used to detect whether a
listing's card content changed between scrapes (snapshot-dedup, detail-refresh gate).
Excludes last_seen/scrape timestamps and detail-only fields.
Хэшируем детерминированную нормализованную репрезентацию полей карточки:
price_rub, area_m2, floor, total_floors, rooms, address (strip+lower, None→""),
len(photo_urls). Порядок частей фиксирован (нет зависимости от порядка
итерации dict) → стабильный hash при неизменной карточке.
"""
address = (self.address or "").strip().lower()
parts = [
str(self.price_rub),
str(self.area_m2),
str(self.floor),
str(self.total_floors),
str(self.rooms),
address,
str(len(self.photo_urls)),
]
return hashlib.sha256("|".join(parts).encode()).hexdigest()
def compute_price_per_m2(self) -> int | None:
"""price_per_m2 = price_rub / area_m2 если area задана."""
if self.area_m2 and self.area_m2 > 0:
return int(self.price_rub / self.area_m2)
return None
# ── BaseScraper ─────────────────────────────────────────────────────────────
class BaseScraper(ABC):
"""Базовый класс — наследовать для каждого источника.
Subclass должен реализовать:
- `name` (class attr) — 'avito' / 'cian' / ...
- `_fetch_page(client, ...)` — async GET к источнику
- `_parse_listings(html_or_json, ...)` — html → list[ScrapedLot]
"""
name: str = "base"
base_url: str = ""
request_delay_sec: float = 5.0 # sleep между запросами (anti-ban)
def __init__(self) -> None:
self._client: httpx.AsyncClient | None = None
async def __aenter__(self) -> BaseScraper:
self._client = httpx.AsyncClient(
timeout=20.0,
follow_redirects=True,
headers={
"User-Agent": random_user_agent(),
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8",
"Accept-Encoding": "gzip, deflate, br",
"Connection": "keep-alive",
"Upgrade-Insecure-Requests": "1",
},
)
return self
async def __aexit__(self, *args: Any) -> None:
if self._client is not None:
await self._client.aclose()
@retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=2, min=2, max=30))
async def _http_get(self, url: str, **kwargs: Any) -> httpx.Response:
"""GET с retry. Subclass-ы используют это вместо raw httpx."""
assert self._client is not None, "Use scraper as async context manager"
response = await self._client.get(url, **kwargs)
# 403/429 — ретрай поможет (рандомный UA в новом контексте), 5xx тоже
if response.status_code in {403, 429, 502, 503, 504}:
response.raise_for_status()
return response
async def sleep_between_requests(self) -> None:
"""Случайный jitter в районе self.request_delay_sec — чтобы парсер
не выглядел как detstvo-bot."""
jitter = random.uniform(0.7, 1.5)
await asyncio.sleep(self.request_delay_sec * jitter)
@abstractmethod
async def fetch_around(self, lat: float, lon: float, radius_m: int = 1000) -> list[ScrapedLot]:
"""Главный метод — собрать объявления вокруг (lat, lon) в радиусе radius_m.
Returns:
Список ScrapedLot. Может быть пустым.
"""
...
# ── Запись пачки результатов в Postgres ─────────────────────────────────────
def save_listings(
db: Session,
lots: list[ScrapedLot],
*,
run_id: int | None = None,
skip_seen_today: bool = False,
) -> tuple[int, int]:
"""Пишем list[ScrapedLot] в `listings` с upsert по dedup_hash.
Дополнительно пишет point-in-time snapshot в listings_snapshots (ON CONFLICT DO UPDATE
— берётся последний за день). Ошибка записи snapshot НЕ прерывает основной INSERT.
Args:
db: SQLAlchemy session.
lots: список распарсенных объявлений.
run_id: FK scrape_runs.id текущего run'а (опционально; None при ad-hoc вызовах).
skip_seen_today: если True и листинг уже обновлялся сегодня по МСК
(last_seen_at >= начало сегодняшнего дня), то пропустить upsert и snapshot.
Используется full_load для экономии redundant upsert + price-trigger churn
при повторном прогоне в тот же день. Новые листинги (prior_row=None) всегда
вставляются. False = старое поведение (всегда upsert).
Returns:
(inserted, updated) — counters для логов.
"""
if not lots:
return 0, 0
inserted = 0
updated = 0
skipped = 0
reconciled = 0 # UPDATE by (source,source_id) при dedup_hash-дрейфе
matched = 0
match_failures = 0
today_msk = datetime.now(_MSK).date()
for lot in lots:
ppm2 = lot.price_per_m2 or lot.compute_price_per_m2()
dedup = lot.compute_dedup_hash()
card_hash = lot.compute_card_hash()
# Pre-read the existing row's card_hash and last_seen_at (keyed by
# dedup_hash) BEFORE the upsert — needed to know the *prior* card
# content and to implement skip_seen_today logic.
# The upsert's RETURNING can only expose the NEW values (EXCLUDED.*),
# never the old ones, so a lightweight SELECT is the clean way.
# None → row did not exist yet (fresh insert) → always process.
prior_row = db.execute(
text("SELECT card_hash, last_seen_at FROM listings WHERE dedup_hash = :dedup"),
{"dedup": dedup},
).fetchone()
prior_card_hash = prior_row.card_hash if prior_row is not None else None
# skip_seen_today: пропускаем листинги уже обновлённые сегодня по МСК.
# Только для existing строк (prior_row is not None); новые всегда вставляются.
if skip_seen_today and prior_row is not None and prior_row.last_seen_at is not None:
ls_msk = prior_row.last_seen_at.astimezone(_MSK).date()
if ls_msk == today_msk:
skipped += 1
continue
# ── Per-lot SAVEPOINT: изолируем listings-upsert ────────────────────
# Когда контент существующего ad дрейфит → новый dedup_hash → INSERT
# находит prior_row=None (SELECT по старому dedup_hash пуст) → INSERT
# летит в UniqueViolation по (source, source_id) при source_id IS NOT NULL
# (constraint 133_listings_uq_source_source_id). Без SAVEPOINT это роняет
# весь прогон. С SAVEPOINT ловим IntegrityError, rollback SAVEPOINT'а,
# и делаем прямой UPDATE по (source, source_id) — reconcile drifted row.
# source_id IS NULL (yandex) → constraint не фаерит, обычный путь.
_upsert_params = {
"source": lot.source,
"source_url": lot.source_url,
"source_id": lot.source_id,
"dedup": dedup,
"address": lot.address,
"lat": lot.lat,
"lon": lot.lon,
"rooms": lot.rooms,
"area_m2": lot.area_m2,
"floor": lot.floor,
"total_floors": lot.total_floors,
"year_built": lot.year_built,
"house_type": lot.house_type,
"repair_state": lot.repair_state,
"has_balcony": lot.has_balcony,
"kitchen_area_m2": lot.kitchen_area_m2,
"ceiling_height_m": lot.ceiling_height_m,
"mortgage_available": lot.mortgage_available,
"is_apartments": lot.is_apartments,
"is_rosreestr_checked": lot.is_rosreestr_checked,
"house_source": lot.house_source,
"house_ext_id": lot.house_ext_id,
"house_url": lot.house_url,
"listing_segment": lot.listing_segment,
"newbuilding_id": lot.newbuilding_id,
"newbuilding_url": lot.newbuilding_url,
"price_rub": lot.price_rub,
"ppm2": ppm2,
"listing_date": lot.listing_date,
"publish_date": lot.publish_date,
"days_on_market": lot.days_on_market,
"description": lot.description,
"photos": _to_json(lot.photo_urls),
"raw": _to_json(lot.raw_payload) if lot.raw_payload else None,
# Cian-specific columns — None for non-Cian scrapers (→ SQL NULL)
"living_area_m2": lot.living_area_m2,
"bedrooms_count": lot.bedrooms_count,
"balconies_count": lot.balconies_count,
"loggias_count": lot.loggias_count,
"description_minhash": lot.description_minhash,
"cadastral_number": lot.cadastral_number,
"building_cadastral_number": lot.building_cadastral_number,
"phones": _to_json(lot.phones) if lot.phones else None,
"is_homeowner": lot.is_homeowner,
"is_pro_seller": lot.is_pro_seller,
"bargain_allowed": lot.bargain_allowed,
"sale_type": lot.sale_type,
"metro_stations": _to_json(lot.metro_stations) if lot.metro_stations else None,
"agency_name": lot.agency_name,
"yandex_offer_id": lot.yandex_offer_id,
"predicted_price_rub": lot.predicted_price_rub,
"predicted_price_min": lot.predicted_price_min,
"predicted_price_max": lot.predicted_price_max,
"price_trend": lot.price_trend,
"price_previous_rub": lot.price_previous_rub,
"geo_precision": lot.geo_precision,
"card_hash": card_hash,
}
_upsert_sql = text(
"""
INSERT INTO listings (
source, source_url, source_id, dedup_hash,
address, lat, lon, region_code,
rooms, area_m2, floor, total_floors, year_built,
house_type, repair_state, has_balcony,
kitchen_area_m2, ceiling_height, ceiling_height_m,
mortgage_available, is_apartments, is_rosreestr_checked,
house_source, house_ext_id, house_url, listing_segment,
newbuilding_id, newbuilding_url,
price_rub, price_per_m2,
listing_date, publish_date, days_on_market, description,
photo_urls, raw_payload,
living_area_m2, bedrooms_count, balconies_count, loggias_count,
description_minhash, cadastral_number, building_cadastral_number,
phones, is_homeowner, is_pro_seller,
bargain_allowed, sale_type, metro_stations, agency_name,
yandex_offer_id, predicted_price_rub,
predicted_price_min, predicted_price_max,
price_trend, price_previous_rub,
geo_precision, card_hash,
scraped_at, last_seen_at
) VALUES (
:source, :source_url, :source_id, :dedup,
:address, :lat, :lon, 66,
:rooms, :area_m2, :floor, :total_floors, :year_built,
:house_type, :repair_state, :has_balcony,
-- ceiling: один param :ceiling_height_m пишем в ОБЕ колонки —
-- ceiling_height (019, читает coverage-дашборд + yandex_detail/cian_detail)
-- и ceiling_height_m (111, живая avito-колонка). См. #2007.
:kitchen_area_m2, :ceiling_height_m, :ceiling_height_m,
:mortgage_available, :is_apartments, :is_rosreestr_checked,
:house_source, :house_ext_id, :house_url, :listing_segment,
:newbuilding_id, :newbuilding_url,
:price_rub, :ppm2,
:listing_date, :publish_date, :days_on_market, :description,
CAST(:photos AS jsonb),
CAST(:raw AS jsonb),
:living_area_m2, :bedrooms_count, :balconies_count, :loggias_count,
:description_minhash, :cadastral_number, :building_cadastral_number,
CAST(:phones AS jsonb), :is_homeowner, :is_pro_seller,
:bargain_allowed, :sale_type, CAST(:metro_stations AS jsonb), :agency_name,
:yandex_offer_id, :predicted_price_rub,
:predicted_price_min, :predicted_price_max,
:price_trend, :price_previous_rub,
:geo_precision, :card_hash,
NOW(), NOW()
)
-- Конфликт-арбитр — dedup_hash (sha256(source|source_id)).
-- Для одного (source, source_id) формула даёт ОДИН dedup_hash,
-- поэтому этот апсерт уже не вставит второй раз → согласован с
-- частичным UNIQUE (source, source_id) WHERE source_id IS NOT NULL
-- (133_listings_uq_source_source_id.sql). Менять арбитр на
-- (source, source_id) НЕЛЬЗЯ: yandex/url-only дают source_id=NULL,
-- а NULL не годится как conflict-target → дубли вернулись бы (#1773).
ON CONFLICT (dedup_hash) DO UPDATE
SET last_seen_at = NOW(),
-- #2206: ре-подтверждение живым = свежий скрейп; без bump'а
-- эстиматор (scraped_at > NOW()-14d) терял ~55% живого инвентаря.
scraped_at = NOW(),
is_active = true,
-- если цена изменилась — обновляем
price_rub = EXCLUDED.price_rub,
price_per_m2 = EXCLUDED.price_per_m2,
-- Cian-specific: обновляем при каждом re-scrape
living_area_m2 = EXCLUDED.living_area_m2,
bedrooms_count = EXCLUDED.bedrooms_count,
balconies_count = EXCLUDED.balconies_count,
loggias_count = EXCLUDED.loggias_count,
description_minhash = EXCLUDED.description_minhash,
cadastral_number = EXCLUDED.cadastral_number,
building_cadastral_number = EXCLUDED.building_cadastral_number,
phones = EXCLUDED.phones,
is_homeowner = EXCLUDED.is_homeowner,
is_pro_seller = EXCLUDED.is_pro_seller,
bargain_allowed = EXCLUDED.bargain_allowed,
sale_type = EXCLUDED.sale_type,
metro_stations = EXCLUDED.metro_stations,
listing_date = COALESCE(EXCLUDED.listing_date, listings.listing_date),
area_m2 = COALESCE(EXCLUDED.area_m2, listings.area_m2),
-- kitchen/ceiling (#2007): COALESCE — SERP re-scrape источника без
-- этих полей (avito SERP → NULL) НЕ затирает detail-enriched значение
-- (avito_detail пишет ceiling_height_m отдельным UPDATE).
kitchen_area_m2 = COALESCE(
EXCLUDED.kitchen_area_m2, listings.kitchen_area_m2
),
ceiling_height = COALESCE(
EXCLUDED.ceiling_height, listings.ceiling_height
),
ceiling_height_m = COALESCE(
EXCLUDED.ceiling_height_m, listings.ceiling_height_m
),
-- Class B promote (#2008, cian SERP): COALESCE — re-scrape
-- источника без поля (avito/yandex SERP → NULL) НЕ затирает
-- значение, ранее записанное cian-ом / detail-ом.
mortgage_available = COALESCE(
EXCLUDED.mortgage_available, listings.mortgage_available
),
is_apartments = COALESCE(
EXCLUDED.is_apartments, listings.is_apartments
),
is_rosreestr_checked = COALESCE(
EXCLUDED.is_rosreestr_checked, listings.is_rosreestr_checked
),
-- Yandex rich fields (+ shared description/agency/publish_date):
-- COALESCE so a source that does not provide them never wipes
-- a value previously written by another source / scrape.
publish_date = COALESCE(EXCLUDED.publish_date, listings.publish_date),
days_on_market = COALESCE(
EXCLUDED.days_on_market, listings.days_on_market
),
description = COALESCE(EXCLUDED.description, listings.description),
agency_name = COALESCE(EXCLUDED.agency_name, listings.agency_name),
yandex_offer_id = COALESCE(
EXCLUDED.yandex_offer_id, listings.yandex_offer_id
),
predicted_price_rub = COALESCE(
EXCLUDED.predicted_price_rub, listings.predicted_price_rub
),
predicted_price_min = COALESCE(
EXCLUDED.predicted_price_min, listings.predicted_price_min
),
predicted_price_max = COALESCE(
EXCLUDED.predicted_price_max, listings.predicted_price_max
),
price_trend = COALESCE(EXCLUDED.price_trend, listings.price_trend),
price_previous_rub = COALESCE(
EXCLUDED.price_previous_rub, listings.price_previous_rub
),
newbuilding_id = COALESCE(
EXCLUDED.newbuilding_id, listings.newbuilding_id
),
newbuilding_url = COALESCE(
EXCLUDED.newbuilding_url, listings.newbuilding_url
),
card_hash = EXCLUDED.card_hash
RETURNING id, (xmax = 0) AS inserted
"""
)
# ── Per-lot SAVEPOINT вокруг listings-upsert ────────────────────────
# Когда контент существующего ad дрейфит → новый dedup_hash → INSERT
# не находит existing row по dedup_hash → пробует fresh INSERT → летит
# UniqueViolation по (source, source_id) WHERE source_id IS NOT NULL
# (constraint listings_source_source_id_uq). Без SAVEPOINT это переводит
# весь Session в InFailedSqlTransaction → весь прогон падает.
# С SAVEPOINT ловим IntegrityError/UniqueViolation, делаем rollback,
# и применяем UPDATE by (source, source_id) — reconcile drifted row.
# source_id IS NULL (yandex) → constraint не фаерит, обычный upsert путь.
result = None
listing_id: int | None = None
try:
with db.begin_nested():
result = db.execute(_upsert_sql, _upsert_params).fetchone()
except IntegrityError as ie:
# Проверяем что это UniqueViolation именно по (source, source_id).
if lot.source_id is not None and isinstance(ie.orig, psycopg.errors.UniqueViolation):
logger.warning(
"save_listings:dedup_hash_drift source=%s source_id=%s "
"— reconcile UPDATE by (source, source_id)",
lot.source,
lot.source_id,
)
# SAVEPOINT уже откачен; session чистая — применяем прямой UPDATE.
rec_row = db.execute(
text(
"""
UPDATE listings
SET dedup_hash = :dedup,
last_seen_at = NOW(),
-- #2206: ре-подтверждение живым = свежий скрейп;
-- без bump'а эстиматор терял ~55% живого инвентаря.
scraped_at = NOW(),
is_active = true,
price_rub = :price_rub,
price_per_m2 = :ppm2,
living_area_m2 = :living_area_m2,
bedrooms_count = :bedrooms_count,
balconies_count = :balconies_count,
loggias_count = :loggias_count,
description_minhash = :description_minhash,
cadastral_number = :cadastral_number,
building_cadastral_number = :building_cadastral_number,
phones = CAST(:phones AS jsonb),
is_homeowner = :is_homeowner,
is_pro_seller = :is_pro_seller,
bargain_allowed = :bargain_allowed,
sale_type = :sale_type,
metro_stations = CAST(:metro_stations AS jsonb),
listing_date = COALESCE(:listing_date, listing_date),
area_m2 = COALESCE(:area_m2, area_m2),
kitchen_area_m2 = COALESCE(:kitchen_area_m2, kitchen_area_m2),
ceiling_height = COALESCE(:ceiling_height_m, ceiling_height),
ceiling_height_m = COALESCE(:ceiling_height_m, ceiling_height_m),
mortgage_available = COALESCE(
:mortgage_available, mortgage_available
),
is_apartments = COALESCE(:is_apartments, is_apartments),
is_rosreestr_checked = COALESCE(
:is_rosreestr_checked, is_rosreestr_checked
),
publish_date = COALESCE(:publish_date, publish_date),
days_on_market = COALESCE(:days_on_market, days_on_market),
description = COALESCE(:description, description),
agency_name = COALESCE(:agency_name, agency_name),
yandex_offer_id = COALESCE(:yandex_offer_id, yandex_offer_id),
predicted_price_rub = COALESCE(
:predicted_price_rub, predicted_price_rub
),
predicted_price_min = COALESCE(
:predicted_price_min, predicted_price_min
),
predicted_price_max = COALESCE(
:predicted_price_max, predicted_price_max
),
price_trend = COALESCE(:price_trend, price_trend),
price_previous_rub = COALESCE(
:price_previous_rub, price_previous_rub
),
newbuilding_id = COALESCE(:newbuilding_id, newbuilding_id),
newbuilding_url = COALESCE(:newbuilding_url, newbuilding_url),
card_hash = :card_hash
WHERE source = :source
AND source_id = :source_id
RETURNING id
"""
),
_upsert_params,
).fetchone()
if rec_row is not None:
listing_id = int(rec_row.id)
reconciled += 1
else:
logger.error(
"save_listings:reconcile_failed source=%s source_id=%s "
"— UPDATE returned no row (unexpected)",
lot.source,
lot.source_id,
)
else:
logger.error(
"save_listings:integrity_error source=%s source_id=%s: %s",
lot.source,
lot.source_id,
ie,
)
raise
else:
if result is not None:
listing_id = int(result.id)
if result.inserted:
inserted += 1
else:
updated += 1
# ── Snapshot: point-in-time observation in listings_snapshots ───
# Fault-tolerant: failure here MUST NOT abort the listings batch.
# ON CONFLICT DO UPDATE — самый последний snapshot за день.
#
# Snapshot-dedup: пропускаем запись только когда МЫ УВЕРЕНЫ что карточка
# не изменилась (prior_card_hash == card_hash, row существовал) И snapshot
# за сегодня уже есть. Принцип "when in doubt — write": лишний snapshot
# неизменной карточки безвреден, ПРОПУЩЕННЫЙ snapshot реального изменения —
# недопустим. Поэтому equality-проверки мало (первый скрейп за день мог
# ещё не записать сегодняшний snapshot) — дополнительно подтверждаем
# наличие сегодняшней строки лёгким EXISTS.
if listing_id is not None:
skip_snapshot = False
if prior_card_hash is not None and prior_card_hash == card_hash:
today_row = db.execute(
text(
"SELECT 1 FROM listings_snapshots "
"WHERE listing_id = CAST(:lid AS bigint) "
" AND snapshot_date = CURRENT_DATE"
),
{"lid": listing_id},
).fetchone()
skip_snapshot = today_row is not None
if skip_snapshot:
logger.debug(
"save_listings:snapshot_skipped (card unchanged) " "source=%s listing_id=%s",
lot.source,
listing_id,
)
if listing_id is not None and not skip_snapshot:
try:
with db.begin_nested():
upsert_listing_snapshot(
db,
listing_id=listing_id,
price_rub=lot.price_rub,
price_per_m2=ppm2,
run_id=run_id,
status="active",
)
except Exception as e:
logger.warning(
"save_listings:snapshot_failed source=%s listing_id=%s: %s",
lot.source,
listing_id,
e,
)
# ── Hook: link listing → house via matching service ─────────────
# Idempotent (UPSERT on listing_sources UNIQUE(ext_source, ext_id))
# and fault-tolerant: failure here MUST NOT abort the listings batch.
# Wrapped in SAVEPOINT so a failed match only rolls back its own work,
# not the surrounding INSERT (see .claude/rules/backend.md SAVEPOINT).
if listing_id is not None:
try:
with db.begin_nested():
_link_listing_to_house(db, listing_id, lot)
matched += 1
except Exception as e:
# Best-effort hook: log and continue so the listings batch isn't aborted.
match_failures += 1
logger.warning(
"save_listings:match_failed source=%s source_id=%s listing_id=%s: %s",
lot.source,
lot.source_id,
listing_id,
e,
)
db.commit()
logger.info(
"save_listings: source=%s inserted=%d updated=%d reconciled=%d "
"skipped_seen_today=%d matched=%d match_failures=%d (total %d)",
lots[0].source if lots else "?",
inserted,
updated,
reconciled,
skipped,
matched,
match_failures,
len(lots),
)
return inserted, updated
def _to_json(value: Any) -> str:
"""JSON serialization helper — для jsonb колонок."""
import json
return json.dumps(value, ensure_ascii=False, default=str)
def _link_listing_to_house(db: Session, listing_id: int, lot: ScrapedLot) -> None:
"""Hook scraped listing into matching service: resolve house, upsert listing_sources.
Steps:
1. match_or_create_house() — find or create canonical houses row for this
listing's address/coords. Uses Tier 0-3 matching (cadastr → source_exact →
fingerprint → geo_proximity → new).
2. upsert_listing_source() — register (ext_source, ext_id) → listing_id in
listing_sources. Idempotent via UNIQUE (ext_source, ext_id).
The matching method recorded is 'source_link': scraper already deduped by
`dedup_hash` (INSERT … ON CONFLICT DO UPDATE), so this is a direct registration
rather than a fuzzy listing-level match.
ext_id source: lot.source_id if present, else dedup_hash (Yandex without
stable source_id falls back to URL-based dedup_hash — same hash on re-scrape).
Skips silently if:
- lot has no source_id AND no address/lat/lon (cannot match house anyway)
Raises on DB errors — caller wraps in try/except + SAVEPOINT.
"""
ext_id = lot.source_id or lot.compute_dedup_hash()
# House resolution: needs at least address or (lat, lon). Skip otherwise —
# listing_sources requires listing_id but not house linkage, so still upsert.
house_id: int | None = None
if lot.address or (lot.lat is not None and lot.lon is not None):
# Use house_source/house_ext_id when scraper extracted them (Avito Houses
# catalog link, Cian newbuilding). Falls back to per-listing identity
# so each unique listing creates its own row if no canonical house exists.
h_src = lot.house_source or lot.source
h_ext = lot.house_ext_id or ext_id
# No inner begin_nested here — the caller (save_listings) already wraps this
# entire function in a per-row SAVEPOINT (backend.md SAVEPOINT pattern).
# A second nested SAVEPOINT inside would be redundant: if match_or_create_house
# raises a DB error, that error propagates out of _link_listing_to_house,
# the caller's SAVEPOINT rolls back only the hook work (not the listings INSERT),
# and the caller logs a match_failed warning. One SAVEPOINT level is sufficient.
house_id, _conf, _method = match_or_create_house(
db,
ext_source=h_src,
ext_id=h_ext,
address=lot.address,
lat=lot.lat,
lon=lot.lon,
year_built=lot.year_built,
building_cadastral_number=lot.building_cadastral_number,
cadastral_number=lot.cadastral_number or lot.kadastr_num,
source_url=lot.house_url or lot.source_url,
)
# Mirror the resolved house into listings.house_id_fk so direct
# `JOIN houses ON house_id_fk = id` keeps working on freshly scraped /
# re-scraped active rows. Without this the FK is populated only by the
# offline backfill (063_*.sql / backfill_listing_sources.py), leaving
# new active listings unlinked (avito 0/444, cian ~7%). Mirrors the
# backfill script's column/param style. IS DISTINCT FROM avoids a no-op
# write when the linkage is already correct.
if house_id is not None:
db.execute(
text(
"UPDATE listings "
" SET house_id_fk = CAST(:hid AS bigint) "
" WHERE id = CAST(:lid AS bigint) "
" AND house_id_fk IS DISTINCT FROM CAST(:hid AS bigint)"
),
{"hid": house_id, "lid": listing_id},
)
upsert_listing_source(
db,
listing_id=listing_id,
ext_source=lot.source,
ext_id=str(ext_id),
method="source_link",
confidence=1.0,
price_rub=lot.price_rub,
area_m2=lot.area_m2,
floor=lot.floor,
rooms_count=lot.rooms,
source_url=lot.source_url,
source_data={"house_id": house_id} if house_id else None,
)

262
tradein-mvp/backend/app/services/scrapers/browser_fetcher.py
View file

@ -1,262 +0,0 @@
"""browser_fetcher.py — HTTP-клиент к tradein-browser сервису (#884/#905).
Тонкий клиент над tradein-browser контейнером, который запускает AsyncCamoufox
локально и экспонирует HTTP API (POST /fetch).
Публичный интерфейс не изменился:
async with BrowserFetcher() as fetcher:
html = await fetcher.fetch("https://example.com")
Внутреннее устройство: httpx.AsyncClient + POST к settings.browser_http_endpoint.
Recycle, crash-recovery и управление браузером живут на стороне сервера (server.py).
При HTTPError / ConnectError делает одну повторную попытку после короткой паузы,
затем пробрасывает исключение.
Архитектура выбрана потому, что playwright WS-сервер (launch_server) несовместим
с playwright >=1.45: ``browserServerImpl.js`` отсутствует → MODULE_NOT_FOUND.
Локальный AsyncCamoufox + HTTP — работающая альтернатива.
"""
from __future__ import annotations
import asyncio
import logging
import httpx
logger = logging.getLogger(__name__)
_RETRY_SLEEP_S: float = 1.0
_HTTP_TIMEOUT_S: float = 120.0 # навигация медленная → щедрый таймаут
class BrowserFetcher:
"""Async context manager: HTTP-клиент к tradein-browser HTTP-сервису.
Использование::
async with BrowserFetcher() as fetcher:
html = await fetcher.fetch("https://example.com")
"""
def __init__(self, source: str = "avito", fetch_timeout_s: float = _HTTP_TIMEOUT_S) -> None:
# source — логический источник ("avito"/"cian"/"yandex"/"domclick"). Сервер
# роутит /fetch по нему на отдельный браузер+прокси, когда включён
# FEATURE_BROWSER_POOL_ENABLED (Phase 1). При выключенном флаге source
# игнорируется — поведение не меняется.
# fetch_timeout_s — таймаут httpx-клиента для POST /fetch. Yandex-путь передаёт
# 30s чтобы один проблемный combo занимал ≤30s×retries вместо 120s×retries.
self._source = source
self._fetch_timeout_s = fetch_timeout_s
self._client: httpx.AsyncClient | None = None
self._endpoint: str | None = None
# ── lifecycle ──────────────────────────────────────────────────────────────
async def __aenter__(self) -> BrowserFetcher:
from app.core.config import settings
self._endpoint = settings.browser_http_endpoint
self._client = httpx.AsyncClient(timeout=self._fetch_timeout_s)
logger.info("BrowserFetcher: клиент создан, endpoint=%s", self._endpoint)
return self
async def __aexit__(self, *_: object) -> None:
if self._client is not None:
await self._client.aclose()
self._client = None
logger.debug("BrowserFetcher: клиент закрыт")
# ── public API ─────────────────────────────────────────────────────────────
async def fetch(self, url: str) -> str:
"""Запрашивает HTML страницы через tradein-browser HTTP-сервис.
При HTTPError или ConnectError делает одну повторную попытку после
короткой паузы. Остальные исключения всплывают к вызывающему коду.
Returns:
Полный HTML-контент страницы.
"""
assert self._client is not None, "BrowserFetcher: используй как async context manager"
assert self._endpoint is not None, "BrowserFetcher: endpoint не задан"
try:
return await self._post_fetch(url)
except (httpx.HTTPError, httpx.TransportError) as exc:
logger.warning(
"BrowserFetcher: ошибка запроса (%s), retry через %.1fs: %s",
type(exc).__name__,
_RETRY_SLEEP_S,
url,
)
await asyncio.sleep(_RETRY_SLEEP_S)
return await self._post_fetch(url)
async def fetch_json(
self,
url: str,
*,
method: str = "GET",
headers: dict[str, str] | None = None,
body: str | None = None,
origin: str | None = None,
) -> dict:
"""In-page fetch() через sidecar /fetch-json. Возвращает {"status": int, "body": str}.
body — уже сериализованная строка (caller делает json.dumps для POST).
origin — same-origin страница, на которую перейдёт камуфокс перед fetch.
При HTTPError / TransportError делает одну повторную попытку после короткой
паузы. Остальные исключения всплывают к вызывающему коду.
"""
assert self._client is not None, "BrowserFetcher: используй как async context manager"
assert self._endpoint is not None, "BrowserFetcher: endpoint не задан"
try:
return await self._post_fetch_json(url, method, headers, body, origin)
except (httpx.HTTPError, httpx.TransportError) as exc:
logger.warning(
"BrowserFetcher: ошибка fetch-json запроса (%s), retry через %.1fs: %s",
type(exc).__name__,
_RETRY_SLEEP_S,
url,
)
await asyncio.sleep(_RETRY_SLEEP_S)
return await self._post_fetch_json(url, method, headers, body, origin)
async def login(
self,
*,
url: str,
email: str,
password: str,
email_selector: str,
password_selector: str,
submit_selector: str,
success_cookie: str,
pre_click_selectors: list[str] | None = None,
wait_ms: int | None = None,
) -> dict[str, str]:
"""Логинится через tradein-browser /login и возвращает cookies как dict name→value.
Использует увеличенный таймаут (60s) — логин медленнее обычного fetch.
При HTTPError / TransportError делает одну повторную попытку.
Returns:
Плоский словарь {cookie_name: cookie_value}.
"""
assert self._client is not None, "BrowserFetcher: используй как async context manager"
assert self._endpoint is not None, "BrowserFetcher: endpoint не задан"
body: dict[str, object] = {
"url": url,
"email": email,
"password": password,
"email_selector": email_selector,
"password_selector": password_selector,
"submit_selector": submit_selector,
"success_cookie": success_cookie,
"pre_click_selectors": pre_click_selectors or [],
}
if wait_ms is not None:
body["wait_ms"] = wait_ms
try:
return await self._post_login(body)
except (httpx.HTTPError, httpx.TransportError) as exc:
logger.warning(
"BrowserFetcher: ошибка login-запроса (%s), retry через %.1fs",
type(exc).__name__,
_RETRY_SLEEP_S,
)
await asyncio.sleep(_RETRY_SLEEP_S)
return await self._post_login(body)
# ── internal ───────────────────────────────────────────────────────────────
async def _post_fetch(self, url: str) -> str:
"""Один HTTP POST к /fetch эндпоинту сервиса."""
assert self._client is not None
assert self._endpoint is not None
resp = await self._client.post(
f"{self._endpoint}/fetch",
json={"url": url, "source": self._source},
)
resp.raise_for_status()
data: dict[str, str] = resp.json()
html = data["html"]
logger.debug("BrowserFetcher: fetch OK url=%r html_len=%d", url, len(html))
return html
async def _post_fetch_json(
self,
url: str,
method: str,
headers: dict[str, str] | None,
body: str | None,
origin: str | None,
) -> dict:
"""Один HTTP POST к /fetch-json эндпоинту сервиса."""
assert self._client is not None
assert self._endpoint is not None
resp = await self._client.post(
f"{self._endpoint}/fetch-json",
json={
"url": url,
"source": self._source,
"method": method,
"headers": headers or {},
"body": body,
"origin": origin,
},
)
resp.raise_for_status()
data: dict = resp.json()
# Defensive: контракт сервера — {"status": int, "body": str} (зеркалит как
# _post_fetch читает data["html"]). Если ключи пропали (несовместимый сервер /
# прокинутый error-payload) — падаем с понятной ошибкой, а не KeyError ниже по
# стеку у адаптера, который ждёт r["status"]/r["body"].
if "status" not in data or "body" not in data:
raise RuntimeError(
f"BrowserFetcher: /fetch-json вернул некорректный ответ "
f"(нет 'status'/'body'): keys={sorted(data)} url={url!r}"
)
logger.debug(
"BrowserFetcher: fetch-json OK url=%r status=%s body_len=%d",
url,
data.get("status"),
len(data.get("body") or ""),
)
return data
async def _post_login(self, body: dict[str, object]) -> dict[str, str]:
"""Один HTTP POST к /login эндпоинту сервиса."""
assert self._client is not None
assert self._endpoint is not None
resp = await self._client.post(
f"{self._endpoint}/login",
json=body,
timeout=httpx.Timeout(60.0),
)
if resp.status_code == 502:
data = resp.json()
has_screenshot = bool(data.get("screenshot_b64"))
logger.debug(
"BrowserFetcher: login 502 — has_screenshot=%s page_url=%r",
has_screenshot,
data.get("page_url"),
)
raise RuntimeError(
f"browser login failed: {data.get('error')} url={data.get('page_url')}"
)
resp.raise_for_status()
data = resp.json()
raw: list[dict[str, object]] = data["cookies"]
result = {str(c["name"]): str(c["value"]) for c in raw}
logger.info("BrowserFetcher: login OK cookie_count=%d", len(result))
return result

788
tradein-mvp/backend/app/services/scrapers/cian_newbuilding.py
View file

@ -1,788 +0,0 @@
"""Cian.ru newbuilding (ЖК) catalog page scraper.
URL: https://zhk-<slug>-<city>-i.cian.ru/
MFE: 'newbuilding-card-desktop-frontend', key: 'initialState'
Sister state containers extracted from same MFE initialState top-level keys:
- realtyValuation: 7-month price chart (data.priceDynamics.chart.data.{labels,values})
→ houses_price_dynamics
- reliabilityState.reliability: наш.дом.рф checkStatus + details[] → house_reliability_checks
- offers: grouped by roomType (fromDeveloperRooms[].layouts[])
- reviews: house reviews list
- housesByTurn: корпуса по очередям (stored as jsonb on houses)
Stage 6 of CianScraper v1.
"""
from __future__ import annotations
import json
import logging
import re
from dataclasses import dataclass, field
from typing import Any
from curl_cffi.requests import AsyncSession # type: ignore[import-untyped]
from app.core.config import settings
from app.services.scrapers.browser_fetcher import BrowserFetcher
from app.services.scrapers.cian_state_parser import extract_all_states, extract_state
logger = logging.getLogger(__name__)
# Canonical ЖК-slug subdomain URL embedded in any Cian SERP/page that references a
# newbuilding (e.g. https://zhk-parkovyy-kvartal-ekb-i.cian.ru). The cat.php SERP for a
# single newobject[] id puts this slug in its title block — extracting it is the WORKING
# nb_id → ЖК-url resolution (#972; see resolve_cian_zhk_url_via_search).
_ZHK_SLUG_URL_RE = re.compile(r"https://zhk-[a-z0-9-]+\.cian\.ru")
# Title-anchored extraction (#972 hardening). The cat.php newbuilding-SERP renders the
# canonical ЖК-slug of the QUERIED newobject as the <a> href inside its page <h1>:
#
# <h1 data-name="Title" ...>Купить квартиру в
# <a href="https://zhk-<slug>.cian.ru/" ...>ЖК «...»</a></h1>
#
# (verified on the live ~2.8 MB prod SERP for nb 48853 — the header sits inside a
# `NewbuildingHeaderSeo` block; there is NO <link rel=canonical> / og:url on this page
# type, so the H1 anchor is the only reliable canonical carrier). Anchoring on it avoids
# the WRONG-ЖК hazard of a naive first-match: cat.php pages also carry banner / "похожие
# ЖК" / recommendation blocks that can embed OTHER zhk-*.cian.ru urls, and those blocks
# may render BEFORE the title — a plain first-match could then resolve a foreign ЖК and
# silently enrich the house with another building's data.
#
# Matches the FIRST zhk-slug url that occurs inside the page's <h1 data-name="Title"> …
# </h1>. `[^>]*` keeps the attr scan within the opening tag; `.*?` (DOTALL) spans the
# inner markup up to the slug href. Falls back to naive first-match only when this finds
# nothing (markup drift), preserving the previous behaviour as a safety net.
_ZHK_TITLE_ANCHOR_RE = re.compile(
r'<h1[^>]*data-name="Title"[^>]*>.*?(https://zhk-[a-z0-9-]+\.cian\.ru)',
re.DOTALL,
)
# cat.php newbuilding-SERP template: one row per `newobject[0]=<nb_id>`. ekb. host is the
# Свердловская-обл. mirror tradein targets; the slug URL it returns is host-agnostic
# (the canonical zhk-*.cian.ru subdomain), so this resolves ЕКБ newbuildings correctly.
_CATPH_NEWBUILDING_SERP = (
"https://ekb.cian.ru/cat.php?deal_type=sale&engine_version=2"
"&offer_type=flat&object_type[0]=2&newobject[0]={nb_id}"
)
@dataclass
class NewbuildingEnrichment:
"""Result of fetch_newbuilding() — houses row + sister containers."""
# Core house metadata (from state.newbuilding)
cian_internal_house_id: int | None = None
# ZHK page URL — canonical slug form stored after fetch / redirect resolution
cian_zhk_url: str | None = None
name: str | None = None
address: str | None = None
year_built: int | None = None
deadline_year: int | None = None
floors_count_min: int | None = None
floors_count_max: int | None = None
building_class: str | None = None # 'comfort' / 'business' / 'premium'
# Management
management_company: dict[str, Any] | None = None # name, phones[], email, ...
# Aggregated metrics
transport_accessibility_rate: int | None = None
advantages: list[dict[str, Any]] = field(default_factory=list)
# Time-series price chart (Stage 6 main deliverable)
# Each entry: {'month_date': str, 'room_count': str, 'prices_type': str, 'period': str,
# 'price_per_sqm': float | None}
realty_valuation_chart: list[dict[str, Any]] = field(default_factory=list)
# Reliability (наш.дом.рф) — overall status + checks detail array
# {check_status: str, check_name: str, details: list[dict]}
reliability_checks: list[dict[str, Any]] = field(default_factory=list)
# Builders / banks (related orgs)
builders: list[dict[str, Any]] = field(default_factory=list)
banks: list[dict[str, Any]] = field(default_factory=list)
# Corpuses (houses_by_turn — корпуса по очередям)
houses_by_turn: list[dict[str, Any]] = field(default_factory=list)
# Reviews
reviews: list[dict[str, Any]] = field(default_factory=list)
# Mini-SERP offers (grouped by roomType, layouts)
nested_offers: list[dict[str, Any]] = field(default_factory=list)
# Raw payloads для debugging
raw_state: dict[str, Any] | None = None
raw_sister_states: dict[str, Any] = field(default_factory=dict)
async def fetch_newbuilding(
zhk_url: str,
*,
session: AsyncSession | None = None, # kept for backward-compat; unused for page fetch
) -> NewbuildingEnrichment | None:
"""Fetch ЖК catalog page и extract все containers.
HTML страница ЖК теперь тянется через BrowserFetcher (camoufox), потому что
curl_cffi получает страницу без initialState под анти-ботом Cian (#972).
tradein-browser верифицирован живым: 1.17 MB, initialState присутствует.
Args:
zhk_url: e.g. 'https://zhk-ekaterininskiy-park-ekb-i.cian.ru/'
session: зарезервирован для обратной совместимости с вызывающими;
больше не используется для fetch страницы ЖК. Resolve-функции
(resolve_cian_zhk_url_via_search) по-прежнему используют curl_cffi.
Returns: NewbuildingEnrichment, or None если fetch / parse failed.
"""
async with BrowserFetcher(source="cian") as browser:
html = await browser.fetch(zhk_url)
# Cian ЖК-карточка: MFE 'newbuilding-card-desktop-frontend', key 'initialState'
# (verified live 2026-06-15).
mfe = "newbuilding-card-desktop-frontend"
nb_state = extract_state(html, mfe=mfe, key="initialState")
if nb_state is None:
logger.warning("Cian newbuilding %s: initialState extraction failed", zhk_url)
return None
# Primary newbuilding object: state.newbuilding (60 fields per schema sec 15.4)
nb = nb_state.get("newbuilding") or {}
if not isinstance(nb, dict):
nb = {}
result = NewbuildingEnrichment(
cian_internal_house_id=nb.get("id"),
cian_zhk_url=zhk_url,
name=nb.get("name") or nb.get("displayName"),
address=_extract_address(nb),
year_built=nb.get("yearBuilt") or nb.get("buildYear"),
deadline_year=_extract_deadline_year(nb),
floors_count_min=nb.get("floorsCountMin"),
floors_count_max=nb.get("floorsCountMax"),
building_class=nb.get("newbuildingClassName") or nb.get("buildingClass"),
management_company=nb.get("managementCompany"),
transport_accessibility_rate=_extract_transport_rate(nb),
advantages=_extract_advantages(nb),
builders=_safe_list(nb.get("builders")),
banks=_safe_list(nb.get("banks")),
houses_by_turn=_safe_list(nb.get("housesByTurn")),
raw_state=nb_state,
)
# Sister containers — all live under the same MFE initialState top-level keys
# (not separate push() entries) per schema sec 15.3
all_states = extract_all_states(html)
mfe_states = all_states.get(mfe, {})
# initialState top-level keys contain sister data: realtyValuation, reliability, offers
top = nb_state
# realtyValuation 7-month chart (Stage 6 main deliverable)
# Located at initialState.realtyValuation (sec 15.5)
rv_state = top.get("realtyValuation") or mfe_states.get("realtyValuation")
if rv_state and isinstance(rv_state, dict):
result.realty_valuation_chart = _extract_chart(rv_state)
result.raw_sister_states["realtyValuation"] = rv_state
# reliability (наш.дом.рф) — initialState.reliabilityState.reliability
# ({checkStatus, details[], banner, actions}, verified live 2026-06-15).
rel_state = (top.get("reliabilityState") or {}).get("reliability")
if rel_state and isinstance(rel_state, dict):
result.reliability_checks = _extract_reliability_checks(rel_state)
result.raw_sister_states["reliability"] = rel_state
# reviews — located at initialState or separate state
reviews_state = top.get("reviews") or mfe_states.get("reviews")
if reviews_state and isinstance(reviews_state, dict):
items = reviews_state.get("items") or reviews_state.get("data") or []
result.reviews = items if isinstance(items, list) else []
result.raw_sister_states["reviews"] = reviews_state
# offers (grouped by roomType) — located at initialState.offers (sec 15.7)
offers_state = top.get("offers") or mfe_states.get("offers") or mfe_states.get("miniSerp")
if offers_state and isinstance(offers_state, dict):
result.nested_offers = _extract_nested_offers(offers_state)
result.raw_sister_states["offers"] = offers_state
logger.info(
"Cian newbuilding %s parsed: id=%s chart=%d reliability=%d offers=%d",
zhk_url,
result.cian_internal_house_id,
len(result.realty_valuation_chart),
len(result.reliability_checks),
len(result.nested_offers),
)
return result
# ---- private extractors ----
def _safe_list(v: Any) -> list[dict[str, Any]]:
"""Return v as list[dict] or empty list."""
if not isinstance(v, list):
return []
return [item for item in v if isinstance(item, dict)]
def _extract_address(nb: dict[str, Any]) -> str | None:
"""Extract display address string from newbuilding object."""
addr = nb.get("fullAddress") or nb.get("address")
if isinstance(addr, str):
return addr
if isinstance(addr, dict):
return addr.get("full") or addr.get("display") or addr.get("name")
return None
def _extract_deadline_year(nb: dict[str, Any]) -> int | None:
"""Extract planned completion year from newbuilding.deadline or similar field."""
dl = nb.get("deadline") or nb.get("buildingDeadline")
if isinstance(dl, dict):
return dl.get("year")
if isinstance(dl, int):
return dl
return None
def _extract_transport_rate(nb: dict[str, Any]) -> int | None:
"""Extract transport accessibility rate as int|None.
Cian payload drift (#972): `transportAccessibilityRate` is now a nested dict
({'transportAccessibility': <int|null>}) rather than a bare int. Older pages
returned a plain int. Handle both — returning a dict here would violate the
NewbuildingEnrichment.transport_accessibility_rate: int|None contract and break
the CAST(:tar AS ...) bind in save_newbuilding_enrichment (psycopg cannot adapt
a dict). Anything non-int collapses to None.
"""
raw = nb.get("transportAccessibilityRate")
if isinstance(raw, dict):
raw = raw.get("transportAccessibility") or raw.get("rate") or raw.get("value")
if isinstance(raw, bool): # bool is an int subclass — exclude explicitly
return None
if isinstance(raw, int):
return raw
if isinstance(raw, float):
return int(raw)
return None
def _extract_advantages(nb: dict[str, Any]) -> list[dict[str, Any]]:
"""Extract typed advantages list from newbuilding.advantages."""
raw = nb.get("advantages") or nb.get("advantagesTyped") or []
if not isinstance(raw, list):
return []
return [a for a in raw if isinstance(a, dict)]
def _extract_chart(rv_state: dict[str, Any]) -> list[dict[str, Any]]:
"""Extract realtyValuation time-series chart points.
Actual Cian shape (sec 15.5):
rv_state.data.priceDynamics.chart.data = {labels: [date...], values: [int...]}
Falls back to alternative shapes if structure varies.
Each output point:
{month_date: str, room_count: str, prices_type: str, period: str, price_per_sqm: float|None}
Note: The initial page state provides aggregate (all room types) chart only.
Per-room-type data requires XHR requests (not done here).
"""
points: list[dict[str, Any]] = []
# Primary shape: data.priceDynamics.chart.data.{labels, values}
data_block = rv_state.get("data")
if isinstance(data_block, dict):
price_dynamics = data_block.get("priceDynamics") or {}
if isinstance(price_dynamics, dict):
chart = price_dynamics.get("chart") or {}
if isinstance(chart, dict):
chart_data = chart.get("data") or {}
if isinstance(chart_data, dict):
labels = chart_data.get("labels") or []
values = chart_data.get("values") or []
if isinstance(labels, list) and isinstance(values, list):
for label, value in zip(labels, values, strict=False):
if label is None or value is None:
continue
points.append(
{
"month_date": str(label)[:10], # YYYY-MM-DD
"room_count": "all", # aggregate (initial state)
"prices_type": "price", # total price (not priceSqm)
"period": "halfYear", # default period shown
"price_per_sqm": None, # chart shows total price, not sqm
}
)
# Store raw value as price_per_sqm placeholder for DB
# (actual sqm price would require area context)
# Use the value as-is; caller / downstream can enrich
points[-1]["price_per_sqm"] = (
float(value) if isinstance(value, int | float) else None
)
if points:
return points
# Fallback shape A: rv_state.chart (flat list with {month/date, price, roomType})
chart_flat = rv_state.get("chart") or rv_state.get("prices") or rv_state.get("dynamics")
if isinstance(chart_flat, list):
for entry in chart_flat:
if not isinstance(entry, dict):
continue
month = entry.get("month") or entry.get("date") or entry.get("monthDate")
price = entry.get("price") or entry.get("value") or entry.get("pricePerSqm")
if month is None or price is None:
continue
price_val: float | None = None
try:
price_val = float(price)
except (TypeError, ValueError):
pass
points.append(
{
"month_date": str(month)[:10],
"room_count": str(entry.get("roomType") or entry.get("roomCount") or "all"),
"prices_type": str(
entry.get("pricesType") or entry.get("priceType") or "price"
),
"period": str(entry.get("period") or "halfYear"),
"price_per_sqm": price_val,
}
)
return points
return []
def _extract_reliability_checks(rel_state: dict[str, Any]) -> list[dict[str, Any]]:
"""Extract reliability check summary from наш.дом.рф state.
Actual Cian shape (sec 15.6):
rel_state = {
checkStatus: {status: 'reliable', title: '...', date: '...'},
details: [{title, iconType, type}, ...],
...
}
Returns a list with ONE entry representing the overall check (per DB schema):
[{check_status, check_name, details}]
The 'details' array is stored as jsonb (full check list) in house_reliability_checks.
"""
check_status_block = rel_state.get("checkStatus")
if not isinstance(check_status_block, dict):
# Alternative: flat list shape
raw = rel_state.get("checks") or rel_state.get("items") or []
if isinstance(raw, list) and raw:
return [
{
"check_name": entry.get("name") or entry.get("title") or entry.get("checkName"),
"check_status": entry.get("status") or entry.get("checkStatus"),
"details": entry.get("details") or entry,
}
for entry in raw
if isinstance(entry, dict)
]
return []
status = check_status_block.get("status")
name = check_status_block.get("title") or check_status_block.get("mobileTitle")
details = rel_state.get("details") or []
return [
{
"check_name": name,
"check_status": status,
"details": details if isinstance(details, list) else [details],
}
]
def _extract_nested_offers(offers_state: dict[str, Any]) -> list[dict[str, Any]]:
"""Extract mini-SERP offers from offers container.
Actual Cian shape (sec 15.7):
offers_state.data.total.fromDeveloperRooms[].{roomType, layouts[]}
Returns flat list of layout offers with room_count injected.
"""
result: list[dict[str, Any]] = []
# Shape A: miniSerp / items list
if "items" in offers_state:
items = offers_state["items"]
return items if isinstance(items, list) else []
# Shape B: data.total.fromDeveloperRooms[].layouts[]
data = offers_state.get("data")
if isinstance(data, dict):
total = data.get("total") or {}
if isinstance(total, dict):
rooms = total.get("fromDeveloperRooms") or []
else:
rooms = []
else:
rooms = []
if not isinstance(rooms, list):
return result
for room in rooms:
if not isinstance(room, dict):
continue
room_type = room.get("roomType") or room.get("roomTypeDisplay")
layouts = room.get("layouts") or []
if not isinstance(layouts, list):
continue
for layout in layouts:
if isinstance(layout, dict):
result.append(
{
"room_count": room_type,
**layout,
}
)
return result
# ---- save helpers ----
def save_newbuilding_enrichment(
db: Any,
house_id: int,
enrichment: NewbuildingEnrichment,
) -> None:
"""Persist NewbuildingEnrichment to DB.
Steps:
1. UPSERT management_companies (if present) → get mc_id
2. UPDATE houses with Cian metadata (incl. cian_zhk_url if present)
3. INSERT INTO houses_price_dynamics (chart points, ON CONFLICT DO UPDATE)
4. INSERT INTO house_reliability_checks (overall + details)
"""
from sqlalchemy import text
# 1. UPSERT management_company
mc_id: int | None = None
if enrichment.management_company and isinstance(enrichment.management_company, dict):
mc = enrichment.management_company
mc_name = mc.get("name") or "Unknown"
mc_ext_id = mc.get("id")
mc_result = db.execute(
text("""
INSERT INTO management_companies (
name, phones, email, opening_hours, chief_name, ext_source, ext_id
) VALUES (
:name,
CAST(:phones AS text[]),
:email,
:oh,
:chief,
'cian',
CAST(:ext_id AS bigint)
)
ON CONFLICT (ext_source, name, ext_id) DO UPDATE SET
phones = EXCLUDED.phones,
email = COALESCE(EXCLUDED.email, management_companies.email)
RETURNING id
"""),
{
"name": mc_name,
"phones": mc.get("phones") or [],
"email": mc.get("email"),
"oh": mc.get("openingHours"),
"chief": mc.get("chiefName"),
"ext_id": mc_ext_id,
},
)
row = mc_result.fetchone()
mc_id = row[0] if row else None
# 2. UPDATE houses
db.execute(
text("""
UPDATE houses SET
cian_internal_house_id = COALESCE(
CAST(:cihi AS bigint), cian_internal_house_id
),
cian_zhk_url = COALESCE(:zhk_url, cian_zhk_url),
year_built = COALESCE(:yb, year_built),
management_company_id = COALESCE(CAST(:mcid AS bigint), management_company_id),
transport_accessibility_rate = COALESCE(
:tar, transport_accessibility_rate
),
advantages = CAST(:adv AS jsonb),
banks = CAST(:bk AS jsonb),
builders = CAST(:bld AS jsonb),
houses_by_turn = CAST(:hbt AS jsonb)
WHERE id = CAST(:hid AS bigint)
"""),
{
"hid": house_id,
"cihi": enrichment.cian_internal_house_id,
"zhk_url": enrichment.cian_zhk_url,
"yb": enrichment.year_built,
"mcid": mc_id,
"tar": enrichment.transport_accessibility_rate,
"adv": json.dumps(enrichment.advantages, ensure_ascii=False),
"bk": json.dumps(enrichment.banks, ensure_ascii=False),
"bld": json.dumps(enrichment.builders, ensure_ascii=False),
"hbt": json.dumps(enrichment.houses_by_turn, ensure_ascii=False),
},
)
# 3. INSERT houses_price_dynamics
# UNIQUE constraint: houses_price_dynamics_dim_key
# (house_id, source, room_count, prices_type, period, month_date) — per migration 029
chart_saved = 0
for point in enrichment.realty_valuation_chart:
if point.get("price_per_sqm") is None:
continue
room_count = point.get("room_count") or "all"
prices_type = point.get("prices_type") or "price"
period = point.get("period") or "halfYear"
db.execute(
text("""
INSERT INTO houses_price_dynamics (
house_id, month_date, source,
room_count, prices_type, period,
price_per_sqm, recorded_at
) VALUES (
CAST(:hid AS bigint),
CAST(:md AS date),
'cian_realty_valuation',
:rc, :pt, :pd,
CAST(:pps AS numeric),
NOW()
)
ON CONFLICT ON CONSTRAINT houses_price_dynamics_dim_key DO UPDATE SET
price_per_sqm = EXCLUDED.price_per_sqm,
recorded_at = NOW()
"""),
{
"hid": house_id,
"md": point["month_date"],
"rc": room_count,
"pt": prices_type,
"pd": period,
"pps": point["price_per_sqm"],
},
)
chart_saved += 1
# 4. INSERT house_reliability_checks (stores overall check + details array)
# Schema (025): (house_id, check_status, check_name, details jsonb, source, recorded_at)
# No UNIQUE constraint — caller should manage duplicates if needed
for check in enrichment.reliability_checks:
if not check.get("check_name") and not check.get("check_status"):
continue
db.execute(
text("""
INSERT INTO house_reliability_checks (
house_id, check_status, check_name, details, source, recorded_at
) VALUES (
CAST(:hid AS bigint),
:cs,
:cn,
CAST(:det AS jsonb),
'cian_nashdom',
NOW()
)
"""),
{
"hid": house_id,
"cs": check.get("check_status"),
"cn": check.get("check_name"),
"det": json.dumps(check.get("details") or [], ensure_ascii=False),
},
)
db.commit()
logger.info(
"Cian newbuilding saved house_id=%s (chart=%d points, reliability=%d checks, mc_id=%s)",
house_id,
chart_saved,
len(enrichment.reliability_checks),
mc_id,
)
async def resolve_cian_zhk_url(
cian_internal_house_id: int,
*,
session: AsyncSession | None = None,
) -> str | None:
"""Resolve canonical ZHK slug URL from a Cian internal house ID (LEGACY — BROKEN).
.. deprecated::
The redirect path this relies on — ``https://cian.ru/zhk/<id>/`` → canonical
slug — NO LONGER EXISTS. Cian now serves HTTP **404** for ``/zhk/<id>/``
(verified 8/8 on prod, #972), so this returns None for every real id. Use
:func:`resolve_cian_zhk_url_via_search` instead: it fetches the cat.php
newbuilding-SERP and extracts the canonical ``zhk-*.cian.ru`` slug from its
markup (the WORKING path). Kept only for backward-compat / reference; do not
wire new callers to it.
Args:
cian_internal_house_id: Cian's numeric ЖК identifier.
session: optional shared curl_cffi AsyncSession (caller owns lifecycle).
Returns:
Canonical ZHK URL string, or None if request failed / redirect not followed.
Note:
This function makes a real HTTP request — do NOT call it without rate limiting.
Caller must enforce per-request sleep matching scraper_settings 'cian' delay.
"""
close_session = False
if session is None:
# Mobile proxy wiring (#806 follow-up): resolve ЖК URL через мобильный прокси.
_proxy_url = settings.cian_proxy_url
_proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None
session = AsyncSession(
impersonate="chrome120",
timeout=15.0,
proxies=_proxies,
headers={
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8",
},
)
close_session = True
fallback_url = f"https://cian.ru/zhk/{cian_internal_house_id}/"
try:
resp = await session.get(fallback_url, allow_redirects=True)
final_url = str(resp.url)
if final_url and final_url != fallback_url:
logger.debug("resolve_cian_zhk_url id=%s → %s", cian_internal_house_id, final_url)
return final_url
# Redirect not followed or same URL — return fallback as canonical
if resp.status_code == 200:
return fallback_url
logger.warning(
"resolve_cian_zhk_url id=%s: HTTP %d, no usable URL",
cian_internal_house_id,
resp.status_code,
)
return None
except Exception as exc:
logger.warning("resolve_cian_zhk_url id=%s failed: %s", cian_internal_house_id, exc)
return None
finally:
if close_session:
await session.close()
def _extract_zhk_url_from_serp(html: str) -> str | None:
"""Extract the canonical ЖК-slug URL of the QUERIED newobject from a cat.php SERP.
Pure helper (no I/O) so the extraction can be unit-tested against a saved fixture.
The cat.php SERP for a single ``newobject[0]=<nb_id>`` renders that ЖК's canonical
``https://zhk-<slug>.cian.ru`` URL as the ``<a>`` href inside its page heading
``<h1 data-name="Title">`` (the ``NewbuildingHeaderSeo`` header). We anchor on that
heading rather than taking the first ``zhk-*.cian.ru`` match in the document, because
cat.php pages also carry banner / "похожие ЖК" / recommendation blocks that can embed
OTHER ЖК slug URLs — and some of those blocks render BEFORE the title. A naive
first-match could pick a foreign ЖК and enrich the house with the wrong building's
price_dynamics / reliability (and the backfill's idempotency would then lock the
corruption in). Anchoring on the title guarantees we resolve the queried ЖК.
Falls back to a naive first ``zhk-*.cian.ru`` match ONLY when the title anchor finds
nothing (markup drift / unexpected page shape), preserving the previous behaviour as
a last-resort safety net.
Returns the slug URL string, or None when the SERP carries no such anchor at all
(empty / no-results page, or markup drift).
"""
anchored = _ZHK_TITLE_ANCHOR_RE.search(html)
if anchored:
return anchored.group(1)
# Fallback: no title-anchored slug found — degrade to naive first-match so a markup
# drift that moves/renames the header still resolves *something* rather than failing.
match = _ZHK_SLUG_URL_RE.search(html)
return match.group(0) if match else None
async def resolve_cian_zhk_url_via_search(
nb_id: int,
*,
session: AsyncSession | None = None,
) -> str | None:
"""Resolve the canonical ЖК-slug URL for a Cian newbuilding id (the WORKING path).
Fetches the cat.php newbuilding-SERP for a single ``newobject[0]=<nb_id>`` and
extracts the canonical ``https://zhk-<slug>.cian.ru`` URL from its markup. This
replaces the legacy :func:`resolve_cian_zhk_url`, whose ``/zhk/<id>/`` redirect path
now 404s (verified on prod, #972). The returned slug URL is exactly what
:func:`fetch_newbuilding` parses, so the enrichment chain is
``nb_id → cat.php SERP → zhk-slug-url → fetch_newbuilding → enrich``.
Verified live (HTTP 200, slug extracted):
nb 48853 → https://zhk-parkovyy-kvartal-ekb-i.cian.ru (ЖК «Парковый квартал»)
nb 102791 → https://zhk-izumrudnyy-bor-ekb-i.cian.ru
nb 24991 → https://zhk-baltym-park-ekb-i.cian.ru
Args:
nb_id: Cian newbuilding id (``house_sources.ext_id`` for cian houses).
session: optional shared curl_cffi AsyncSession (caller owns lifecycle). When
None, an own session is created using the same mobile-proxy wiring as the
rest of the Cian scrapers (``settings.cian_proxy_url`` when set, else direct).
Returns:
The canonical ЖК-slug URL string, or None on non-200 / empty SERP / no match /
request failure (each logs a warning).
Note:
Makes ONE real HTTP request and does NOT sleep — the CALLER enforces the
anti-bot delay (matching scraper_settings 'cian'). At scale this needs the
mobile proxy; low-volume direct fetches work for the bounded proof.
"""
close_session = False
if session is None:
# Mobile proxy wiring (#806): Cian блокирует datacenter-IP. proxy=None → прямое
# подключение (dev / proxy-down fallback — single fetches survive direct).
_proxy_url = settings.cian_proxy_url
_proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None
session = AsyncSession(
impersonate="chrome120",
timeout=30.0,
proxies=_proxies,
headers={
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8",
},
)
close_session = True
serp_url = _CATPH_NEWBUILDING_SERP.format(nb_id=nb_id)
try:
resp = await session.get(serp_url, allow_redirects=True)
if resp.status_code != 200:
logger.warning(
"resolve_cian_zhk_url_via_search nb_id=%s: cat.php → HTTP %d",
nb_id,
resp.status_code,
)
return None
zhk_url = _extract_zhk_url_from_serp(resp.text)
if zhk_url is None:
logger.warning(
"resolve_cian_zhk_url_via_search nb_id=%s: no zhk-slug URL in SERP "
"(empty results / markup drift?)",
nb_id,
)
return None
logger.info("resolve_cian_zhk_url_via_search nb_id=%s → %s", nb_id, zhk_url)
return zhk_url
except Exception as exc:
logger.warning("resolve_cian_zhk_url_via_search nb_id=%s failed: %s", nb_id, exc)
return None
finally:
if close_session:
await session.close()

160
tradein-mvp/backend/app/services/scrapers/cian_state_parser.py
View file

@ -1,160 +0,0 @@
"""Shared utility для извлечения Cian Redux state из _cianConfig.
Pattern: window._cianConfig['<mfe>'].push({key:..., value:..., priority:..., filter:...})
Used by SERP / detail / newbuilding / valuation scrapers.
"""
import json
import logging
import re
from typing import Any
logger = logging.getLogger(__name__)
# Match both patterns Cian uses:
# 1. window._cianConfig['mfe'].push({key:..., value:..., priority:..., filter:...})
# 2. window._cianConfig['mfe'] = window._cianConfig['mfe'] || [];
# window._cianConfig['mfe'].push({...})
_RE_CIAN_PUSH = re.compile(
r"window\._cianConfig\['(?P<mfe>[\w-]+)'\]"
r"(?:\.push\(\s*|"
r"\s*=\s*window\._cianConfig\['[\w-]+'\]\s*\|\|\s*\[\];\s*"
r"window\._cianConfig\['[\w-]+'\]\.push\(\s*)"
r"\{\s*key:\s*['\"](?P<key>\w+)['\"]\s*,\s*"
r"value:\s*(?P<value>.+?)\s*,\s*"
r"priority:",
re.DOTALL,
)
# 2026-05 drift (#639): Cian сменил формат на
# window._cianConfig['mfe'] = (window._cianConfig['mfe'] || []).concat([{"key":..,"value":..},..])
# вместо .push({key:..}). Ключи теперь в JSON-кавычках, аргумент concat([...]) — валидный
# JSON-массив → парсим его целиком (надёжнее regexp по одному value). Старый .push формат
# (_RE_CIAN_PUSH) оставлен как fallback для legacy-фикстур.
_RE_CIAN_CONCAT = re.compile(
r"window\._cianConfig\['(?P<mfe>[\w-]+)'\]\s*=\s*"
r"\(\s*window\._cianConfig\['[\w-]+'\]\s*\|\|\s*\[\]\s*\)\.concat\(\s*"
)
def _balanced_array(s: str, start: int) -> str | None:
"""Срез s[start:end] — сбалансированный ``[...]`` от индекса '[', с учётом JSON-строк."""
depth = 0
in_str = False
esc = False
for i in range(start, len(s)):
c = s[i]
if in_str:
if esc:
esc = False
elif c == "\\":
esc = True
elif c == '"':
in_str = False
continue
if c == '"':
in_str = True
elif c == "[":
depth += 1
elif c == "]":
depth -= 1
if depth == 0:
return s[start : i + 1]
return None
def _extract_from_concat(html: str, mfe: str, key: str) -> Any | None:
"""Новый формат: .concat([{"key":..,"value":..},..]) — массив-аргумент = валидный JSON."""
for m in _RE_CIAN_CONCAT.finditer(html):
if m.group("mfe") != mfe:
continue
start = m.end()
if start >= len(html) or html[start] != "[":
start = html.find("[", m.end())
if start < 0:
continue
arr = _balanced_array(html, start)
if arr is None:
continue
try:
items = json.loads(arr)
except json.JSONDecodeError as exc:
logger.debug("concat array parse failed for mfe=%s key=%s: %s", mfe, key, exc)
continue
for it in items:
if isinstance(it, dict) and it.get("key") == key:
return it.get("value")
return None
def extract_state(html: str, mfe: str, key: str = "initialState") -> dict[str, Any] | None:
"""Extract state.value where _cianConfig[<mfe>] entry.key matches.
Поддерживает оба формата Cian: новый ``.concat([{"key":..,"value":..}])`` (2026-05,
#639) и legacy ``.push({key:..,value:..})``. Returns parsed JSON dict, or None.
"""
# Новый формат (concat) — приоритетный.
concat_val = _extract_from_concat(html, mfe, key)
if concat_val is not None:
return concat_val
for match in _RE_CIAN_PUSH.finditer(html):
if match.group("mfe") != mfe or match.group("key") != key:
continue
value_str = match.group("value").strip()
# Strategy 1: direct JSON parse (value уже JSON-like)
try:
return json.loads(value_str)
except json.JSONDecodeError:
pass
# Strategy 2: JSON.parse("escaped string")
# Use json.loads on the outer quoted string to unescape atomically — avoids
# order-dependent manual .replace() chains that break when descriptions contain
# backslashes (e.g. '\\' in listing text corrupts the rest of the parse).
m_jp = re.match(r'JSON\.parse\(\s*("(?:[^"\\]|\\.)*")\s*\)\s*$', value_str, re.DOTALL)
if m_jp:
outer_json_str = m_jp.group(1) # the full "..." including surrounding quotes
try:
inner = json.loads(outer_json_str) # inner is now a Python str
except json.JSONDecodeError as exc:
logger.debug(
"JSON.parse() outer unescape failed for mfe=%s key=%s: %s", mfe, key, exc
)
continue
try:
return json.loads(inner)
except json.JSONDecodeError as exc:
logger.debug("JSON.parse() inner parse failed for mfe=%s key=%s: %s", mfe, key, exc)
continue
# Strategy 3: raw JS object literal — try demjson3 (optional dep)
try:
import demjson3 # type: ignore[import-untyped]
return demjson3.decode(value_str)
except ImportError:
logger.debug(
"demjson3 not installed; cannot parse raw JS object for mfe=%s key=%s", mfe, key
)
except Exception as exc:
logger.debug("demjson3 parse failed for mfe=%s key=%s: %s", mfe, key, exc)
return None
def extract_all_states(html: str) -> dict[str, dict[str, Any]]:
"""Returns {mfe_name: {key: parsed_value}} for all push() entries.
Useful for discovery / debugging during scraper development.
"""
result: dict[str, dict[str, Any]] = {}
for match in _RE_CIAN_PUSH.finditer(html):
mfe = match.group("mfe")
key = match.group("key")
parsed = extract_state(html, mfe, key)
if parsed is not None:
result.setdefault(mfe, {})[key] = parsed
return result

553
tradein-mvp/backend/app/services/scrapers/cian_valuation.py
View file

@ -1,553 +0,0 @@
"""Cian.ru Valuation Calculator scraper — auth required.
URL: https://www.cian.ru/kalkulator-nedvizhimosti/?address=...&totalArea=...
MFE: 'valuation-for-agent-frontend'
Key: 'initialState'
Auth: loads Cian session cookies from cian_session_cookies table (Stage 4).
Cache: 24h TTL по sha256(address + params) in external_valuations table.
State shape (sec 24.3–24.6 Schema_Cian_SERP_Inventory):
estimation.sale.data.{price, accuracy, priceFrom, priceTo, priceSqm}
estimation.rent.data.{price, accuracy, priceFrom, priceTo, taxPrice}
estimationChart.data.{title.{change, changeValue}, chartData.data[{date, price}]}
houseInfo.data.{items[{title, value}], isRenovation, isEmergency, isCulturalHeritage}
managementCompany.data.{name, address, phones, email, openingHours, chiefName}
filters.houseId — Cian's internal house_id
Used as 6th evaluation source в estimator.py (Stage 9).
"""
from __future__ import annotations
import hashlib
import json
import logging
from dataclasses import dataclass, field
from typing import Any
from urllib.parse import urlencode
from curl_cffi.requests import AsyncSession
from sqlalchemy import text
from sqlalchemy.orm import Session
from app.core.config import settings
from app.services.cian_session import load_session, mark_session_invalid
from app.services.scrapers.cian_state_parser import extract_state
logger = logging.getLogger(__name__)
VALUATION_BASE_URL = "https://www.cian.ru/kalkulator-nedvizhimosti/"
VALUATION_MFE = "valuation-for-agent-frontend"
# Cian floor enum: floor 1 → floorOne, floor 2 → floorTwo,
# last floor → floorLast, все остальные → floorOther.
_FLOOR_ENUM_SPECIAL = {1: "floorOne", 2: "floorTwo"}
# Cian repair type enum
_REPAIR_TYPE_MAP: dict[str, str] = {
"without": "repairTypeWithout",
"no": "repairTypeWithout",
"cosmetic": "repairTypeCosmetic",
"euro": "repairTypeEuro",
"design": "repairTypeDesign",
}
# Cian chart change direction → normalized storage values (per DDL: increase/decrease/neutral)
_CHANGE_DIR_MAP: dict[str, str] = {
"increase": "increase",
"decrease": "decrease",
"neutral": "neutral",
"up": "increase",
"down": "decrease",
}
def _cian_floor_enum(floor: int, total_floors: int | None) -> str:
if floor in _FLOOR_ENUM_SPECIAL:
return _FLOOR_ENUM_SPECIAL[floor]
if total_floors is not None and floor == total_floors:
return "floorLast"
return "floorOther"
@dataclass
class CianValuationResult:
"""Cian Valuation Calculator response parsed."""
# Sale estimate
sale_price_rub: float | None = None
sale_accuracy: float | None = None
sale_price_from: float | None = None
sale_price_to: float | None = None
sale_price_sqm: float | None = None
# Rent estimate (Cian-specific — both sale + rent per 1 request)
rent_price_rub: float | None = None
rent_accuracy: float | None = None
rent_price_from: float | None = None
rent_price_to: float | None = None
rent_tax_price: float | None = None
# Chart (7 monthly points for this specific apartment)
chart: list[dict[str, Any]] = field(default_factory=list)
chart_change_pct: float | None = None
chart_change_direction: str | None = None # increase/decrease/neutral
# House info (15 items: year, type, floors, etc.)
house_info: list[dict[str, Any]] = field(default_factory=list)
external_house_id: int | None = None # filters.houseId
filters_hash: str | None = None # estimation.sale.data.filtersHash
# Management company (unique to Cian Valuation)
management_company: dict[str, Any] | None = None
# Authentication state
is_authenticated: bool = False
user_id: int | None = None
# Raw payload (full state for raw_payload column)
raw_state: dict[str, Any] | None = None
def compute_cache_key(
address: str,
total_area: float,
rooms_count: int,
floor: int,
repair_type: str,
deal_type: str,
) -> str:
"""sha256 of normalized params for 24h cache lookup."""
norm = (
f"{address.strip().lower()}|{total_area}|{rooms_count}|{floor}"
f"|{repair_type}|{deal_type}"
)
return hashlib.sha256(norm.encode("utf-8")).hexdigest()
async def estimate_via_cian_valuation(
db: Session,
*,
address: str,
total_area: float,
rooms_count: int,
floor: int,
total_floors: int | None = None,
repair_type: str = "cosmetic",
deal_type: str = "sale",
use_cache: bool = True,
house_id: int | None = None,
listing_id: int | None = None,
) -> CianValuationResult | None:
"""Estimate value via Cian's authenticated Valuation Calculator.
Выполняет 1 GET на cian.ru/kalkulator-nedvizhimosti с cookies из DB,
парсит state['estimation'] + estimationChart + houseInfo + managementCompany.
Returns CianValuationResult, or None если cookies expired/invalid/fetch failed.
Graceful: не кидает исключений — caller может продолжить без этого source.
"""
cache_key = compute_cache_key(address, total_area, rooms_count, floor, repair_type, deal_type)
# 1. Cache lookup (только если use_cache=True)
if use_cache:
cached = _load_from_cache(db, cache_key)
if cached is not None:
logger.info("Cian valuation cache HIT for cache_key=%s...", cache_key[:12])
return cached
# 2. Load auth cookies из cian_session_cookies
cookies = load_session(db)
if cookies is None:
logger.warning("Cian valuation: no auth session in DB — skipping (graceful)")
return None
# 3. Build URL с корректными enum values
params: dict[str, str] = {
"address": address,
"totalArea": str(total_area),
"roomsCount": str(rooms_count),
"valuationType": deal_type,
"floor[0]": _cian_floor_enum(floor, total_floors),
"repairType[0]": _REPAIR_TYPE_MAP.get(repair_type, "repairTypeCosmetic"),
}
url = f"{VALUATION_BASE_URL}?{urlencode(params, safe='[]')}"
# 4. Fetch с TLS fingerprint (curl_cffi, impersonate chrome120)
# Mobile proxy wiring (#806 follow-up): Cian валюация — такой же datacenter-бан риск
# как SERP. Используем cian_proxy_url. proxy=None → прямое подключение (dev).
_proxy_url = settings.cian_proxy_url
_proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None
try:
async with AsyncSession(
impersonate="chrome120",
cookies=cookies,
timeout=25.0,
proxies=_proxies,
headers={"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"},
) as session:
resp = await session.get(url, allow_redirects=True)
if resp.status_code != 200:
logger.warning("Cian valuation fetch → HTTP %d for %s", resp.status_code, url)
return None
html = resp.text
except Exception as exc:
logger.warning("Cian valuation fetch failed: %s", exc)
return None
# 5. Parse state из _cianConfig['valuation-for-agent-frontend']
state = extract_state(html, mfe=VALUATION_MFE, key="initialState")
if state is None:
logger.warning("Cian valuation: state extraction failed (mfe=%s)", VALUATION_MFE)
return None
# 6. Verify auth — если false → session expired
user = state.get("user", {}) or {}
if not user.get("isAuthenticated"):
user_id = user.get("userId")
logger.warning(
"Cian valuation: session not authenticated (userId=%s) — marking invalid", user_id
)
if user_id:
mark_session_invalid(db, user_id)
return None
# 7. Extract structured result
result = _parse_valuation_state(state)
# 7a. Sanity-check price bounds (Mera-audit fix-1).
# Битый API-ответ (999_999 < min или 9_999_999_999 > max) → не кэшируем, возвращаем None.
# Аналогично проверяем low/high bound: low>high или отрицательные → drop.
if not _is_price_sane(result):
return None
# 8. Persist to cache (24h TTL)
_save_to_cache(
db,
cache_key=cache_key,
address=address,
total_area=total_area,
rooms_count=rooms_count,
floor=floor,
total_floors=total_floors,
repair_type=repair_type,
deal_type=deal_type,
result=result,
house_id=house_id,
listing_id=listing_id,
)
logger.info(
"Cian valuation extracted: sale=%s (acc=%s%%) rent=%s",
result.sale_price_rub,
result.sale_accuracy,
result.rent_price_rub,
)
return result
def _parse_valuation_state(state: dict[str, Any]) -> CianValuationResult:
"""Extract CianValuationResult from Cian valuation initialState.
Mapping (per Schema_Cian_SERP_Inventory sec 24.4–24.6):
estimation.sale.data.{price, accuracy, priceFrom, priceTo, priceSqm, filtersHash}
estimation.rent.data.{price, accuracy, priceFrom, priceTo, taxPrice}
estimationChart.data.{title.{change, changeValue}, chartData.data[]}
houseInfo.data.{items[], ...}
filters.houseId
managementCompany.data
"""
result = CianValuationResult(raw_state=state)
user = state.get("user") or {}
result.is_authenticated = bool(user.get("isAuthenticated"))
result.user_id = user.get("userId")
# --- estimation: sale ---
estimation = state.get("estimation") or {}
sale_wrapper = estimation.get("sale") or {}
sale_data = sale_wrapper.get("data") or {}
result.sale_price_rub = _parse_num(sale_data.get("price"))
result.sale_accuracy = _parse_num(sale_data.get("accuracy"))
result.sale_price_from = _parse_num(sale_data.get("priceFrom"))
result.sale_price_to = _parse_num(sale_data.get("priceTo"))
result.sale_price_sqm = _parse_num(sale_data.get("priceSqm"))
result.filters_hash = sale_data.get("filtersHash")
# --- estimation: rent ---
rent_wrapper = estimation.get("rent") or {}
rent_data = rent_wrapper.get("data") or {}
result.rent_price_rub = _parse_num(rent_data.get("price"))
result.rent_accuracy = _parse_num(rent_data.get("accuracy"))
result.rent_price_from = _parse_num(rent_data.get("priceFrom"))
result.rent_price_to = _parse_num(rent_data.get("priceTo"))
result.rent_tax_price = _parse_num(rent_data.get("taxPrice"))
# --- estimationChart: 7 monthly time-series points ---
est_chart = state.get("estimationChart") or {}
chart_outer = est_chart.get("data") or {}
chart_title = chart_outer.get("title") or {}
# changeValue: "8,3%" → parse out float
change_value_str = chart_title.get("changeValue") or ""
result.chart_change_pct = _parse_change_pct(change_value_str)
# change direction: increase / decrease / neutral
change_dir_raw = chart_title.get("change")
if change_dir_raw:
result.chart_change_direction = _CHANGE_DIR_MAP.get(change_dir_raw, change_dir_raw)
chart_data_wrapper = chart_outer.get("chartData") or {}
chart_points = chart_data_wrapper.get("data") or []
if isinstance(chart_points, list):
for entry in chart_points:
if not isinstance(entry, dict):
continue
# date: unix_ms → ISO string "YYYY-MM-DD"
date_ms = entry.get("date")
date_str: str | None = None
if isinstance(date_ms, int | float) and date_ms > 0:
import datetime
date_str = datetime.datetime.fromtimestamp(
date_ms / 1000, tz=datetime.UTC
).strftime("%Y-%m-%d")
result.chart.append(
{
"month_date": date_str or "",
"price": _parse_num(entry.get("price")),
"price_formatted": entry.get("priceFormatted"),
}
)
# --- houseInfo: 15 items о доме ---
house_info_wrapper = state.get("houseInfo") or {}
house_info_data = house_info_wrapper.get("data") or {}
items = house_info_data.get("items")
if isinstance(items, list):
result.house_info = [i for i in items if isinstance(i, dict)]
# --- filters.houseId: Cian internal house_id ---
filters = state.get("filters") or {}
result.external_house_id = filters.get("houseId")
# --- managementCompany ---
mc_wrapper = state.get("managementCompany") or {}
mc_data = mc_wrapper.get("data")
if isinstance(mc_data, dict) and mc_data:
result.management_company = mc_data
return result
def _parse_num(value: Any) -> float | None:
if value is None:
return None
try:
return float(value)
except (ValueError, TypeError):
return None
def _parse_change_pct(value_str: str) -> float | None:
"""Parse '8,3%' → 8.3, '-2,5%' → -2.5, '' → None."""
if not value_str:
return None
cleaned = value_str.replace(",", ".").replace("%", "").strip()
try:
return float(cleaned)
except ValueError:
return None
def _is_price_sane(result: CianValuationResult) -> bool:
"""Mera-audit fix-1: проверка разумного диапазона цены Cian Valuation.
Возвращает False (→ caller вернёт None, не кэшируем) если:
- sale_price_rub задан и вне [settings.cian_valuation_min_rub, cian_valuation_max_rub]
- sale_price_from или sale_price_to отрицательные
- sale_price_from > sale_price_to (инвертированный диапазон)
None-поля не судим: Cian может не вернуть диапазон — это не ошибка.
"""
price = result.sale_price_rub
if price is not None:
min_rub = settings.cian_valuation_min_rub
max_rub = settings.cian_valuation_max_rub
if price < min_rub or price > max_rub:
logger.warning(
"Cian valuation sanity: sale_price_rub=%.0f вне [%.0f, %.0f] → drop",
price,
min_rub,
max_rub,
)
return False
low = result.sale_price_from
high = result.sale_price_to
if low is not None and low < 0:
logger.warning("Cian valuation sanity: sale_price_from=%.0f < 0 → drop", low)
return False
if high is not None and high < 0:
logger.warning("Cian valuation sanity: sale_price_to=%.0f < 0 → drop", high)
return False
if low is not None and high is not None and low > high:
logger.warning(
"Cian valuation sanity: low=%.0f > high=%.0f (инвертированный диапазон) → drop",
low,
high,
)
return False
return True
def _load_from_cache(db: Session, cache_key: str) -> CianValuationResult | None:
"""SELECT from external_valuations where cache_key + source='cian_valuation' + not expired."""
row = (
db.execute(
text("""
SELECT
raw_payload,
sale_price_rub,
sale_accuracy,
rent_price_rub,
rent_accuracy,
chart,
chart_change_pct,
chart_change_direction,
external_house_id,
filters_hash,
low_price,
high_price
FROM external_valuations
WHERE source = 'cian_valuation'
AND cache_key = :ck
AND expires_at > NOW()
ORDER BY fetched_at DESC
LIMIT 1
"""),
{"ck": cache_key},
)
.mappings()
.first()
)
if row is None:
return None
return CianValuationResult(
sale_price_rub=_parse_num(row["sale_price_rub"]),
sale_accuracy=_parse_num(row["sale_accuracy"]),
rent_price_rub=_parse_num(row["rent_price_rub"]),
rent_accuracy=_parse_num(row["rent_accuracy"]),
chart=row["chart"] if isinstance(row["chart"], list) else [],
chart_change_pct=_parse_num(row["chart_change_pct"]),
chart_change_direction=row["chart_change_direction"],
external_house_id=row["external_house_id"],
filters_hash=row["filters_hash"],
sale_price_from=_parse_num(row["low_price"]),
sale_price_to=_parse_num(row["high_price"]),
raw_state=row["raw_payload"] if isinstance(row["raw_payload"], dict) else None,
is_authenticated=True, # cache только authenticated results
)
def _save_to_cache(
db: Session,
*,
cache_key: str,
address: str,
total_area: float,
rooms_count: int,
floor: int,
total_floors: int | None,
repair_type: str,
deal_type: str,
result: CianValuationResult,
house_id: int | None = None,
listing_id: int | None = None,
) -> None:
"""INSERT result into external_valuations (source='cian_valuation') с 24h expiry.
Использует ON CONFLICT (source, cache_key) DO UPDATE для идемпотентности.
Все числовые поля через CAST(:x AS numeric) — psycopg v3 safe, без ::type.
"""
try:
db.execute(
text("""
INSERT INTO external_valuations (
source, cache_key, address,
total_area, rooms_count, floor, total_floors,
repair_type, deal_type,
price_rub, accuracy,
low_price, high_price,
house_id, listing_id,
sale_price_rub, sale_accuracy,
rent_price_rub, rent_accuracy,
chart, chart_change_pct, chart_change_direction,
external_house_id, filters_hash,
raw_payload, fetched_at, expires_at
) VALUES (
'cian_valuation', :ck, :addr,
CAST(:ta AS numeric), :rc, :fl, :tf,
:rt, :dt,
CAST(:sp AS numeric), CAST(:sa AS numeric),
CAST(:lp AS numeric), CAST(:hp AS numeric),
CAST(:hid AS bigint), CAST(:lid AS bigint),
CAST(:sp AS numeric), CAST(:sa AS numeric),
CAST(:rp AS numeric), CAST(:ra AS numeric),
CAST(:ch AS jsonb), CAST(:ccp AS numeric), :ccd,
:ehi, :fh,
CAST(:raw AS jsonb), NOW(), NOW() + INTERVAL '24 hours'
)
ON CONFLICT (source, cache_key) DO UPDATE SET
price_rub = EXCLUDED.price_rub,
accuracy = EXCLUDED.accuracy,
low_price = EXCLUDED.low_price,
high_price = EXCLUDED.high_price,
house_id = COALESCE(EXCLUDED.house_id, external_valuations.house_id),
listing_id = COALESCE(EXCLUDED.listing_id, external_valuations.listing_id),
sale_price_rub = EXCLUDED.sale_price_rub,
sale_accuracy = EXCLUDED.sale_accuracy,
rent_price_rub = EXCLUDED.rent_price_rub,
rent_accuracy = EXCLUDED.rent_accuracy,
chart = EXCLUDED.chart,
chart_change_pct = EXCLUDED.chart_change_pct,
chart_change_direction = EXCLUDED.chart_change_direction,
external_house_id = EXCLUDED.external_house_id,
filters_hash = EXCLUDED.filters_hash,
raw_payload = EXCLUDED.raw_payload,
fetched_at = NOW(),
expires_at = NOW() + INTERVAL '24 hours'
"""),
{
"ck": cache_key,
"addr": address,
"ta": total_area,
"rc": rooms_count,
"fl": floor,
"tf": total_floors,
"rt": repair_type,
"dt": deal_type,
"sp": result.sale_price_rub,
"sa": result.sale_accuracy,
"lp": result.sale_price_from,
"hp": result.sale_price_to,
"hid": house_id,
"lid": listing_id,
"rp": result.rent_price_rub,
"ra": result.rent_accuracy,
"ch": json.dumps(result.chart),
"ccp": result.chart_change_pct,
"ccd": result.chart_change_direction,
"ehi": result.external_house_id,
"fh": result.filters_hash,
"raw": json.dumps(result.raw_state) if result.raw_state else None,
},
)
db.commit()
except Exception as exc:
logger.error("Cian valuation cache save failed: %s", exc, exc_info=True)
raise

99
tradein-mvp/backend/app/services/scrapers/house_type_normalizer.py
View file

@ -1,99 +0,0 @@
"""Нормализация house_type → каноничный enum для listings.house_type.
Целевой канон (сверено с estimator._IMV_HOUSE_TYPE_MAP, estimator.py:146):
panel / brick / monolith / monolith_brick / block / wood.
Зачем (#2007): estimator применяет soft-penalty по house_type — аналог с
house_type != target штрафуется. Yandex SERP отдаёт SCREAMING-значения
(MONOLIT / BRICK / PANEL / ...), которые НИКОГДА не равны каноничным
(monolith / brick / panel / ...) → ~70% yandex-аналогов получали ложный штраф
и фактически выпадали из скоринга. Нормализация на ингесте чинит это.
Важно про None: неизвестное / 'other' / '' → None, НЕ 'other'. В estimator
`house_type IS NULL` нейтрально (без штрафа), а любое не-канон значение всегда
!= target → ложный штраф. Поэтому unknown лучше схлопнуть в NULL.
Источники raw-значений:
- yandex SERP: building.buildingType — SCREAMING_SNAKE (MONOLIT, MONOLIT_BRICK, ...)
- cian SERP: building.materialType — camelCase (monolith, monolithBrick,
gasSilicateBlock, ...) — переиспользуется в #2008 (cian.py:815).
- avito detail page: «Тип дома» — русские лейблы (монолитный, панельный, ...) —
переиспользуется в #2009 (avito_detail.py) вместо локального дубля-мэппинга.
"""
from __future__ import annotations
import logging
logger = logging.getLogger(__name__)
# Канон listings.house_type — выровнен по estimator._IMV_HOUSE_TYPE_MAP.
_CANON: frozenset[str] = frozenset(
{"panel", "brick", "monolith", "monolith_brick", "block", "wood"}
)
# raw-токен → канон. Ключи — точные значения вокабуляров источников; сравнение
# регистронезависимое (см. _LOOKUP ниже), но строго по полному токену енума,
# без fuzzy-матчинга. Неизвестные токены сюда НЕ попадают → normalize вернёт None.
_RAW_TO_CANON: dict[str, str] = {
# ── yandex SERP (SCREAMING_SNAKE buildingType) ───────────────────────────
"MONOLIT": "monolith",
"BRICK": "brick",
"PANEL": "panel",
"MONOLIT_BRICK": "monolith_brick",
"BLOCK": "block",
"WOOD": "wood",
# ── cian SERP (camelCase materialType) — reuse в #2008 ───────────────────
"monolith": "monolith",
"brick": "brick",
"panel": "panel",
"block": "block",
"wood": "wood",
"monolithBrick": "monolith_brick",
"gasSilicateBlock": "block",
"aerocreteBlock": "block",
"foamConcreteBlock": "block",
"stalin": "brick", # «сталинка» — кирпич
# ── avito detail page («Тип дома», русские лейблы) — reuse в #2009 ───────
"монолитный": "monolith",
"панельный": "panel",
"кирпичный": "brick",
"монолитно-кирпичный": "monolith_brick",
"блочный": "block",
"деревянный": "wood",
}
# Регистронезависимый lookup. Лоуэркейс-ключи не коллизят между вокабулярами:
# 'MONOLIT'→'monolit' и 'monolith'→'monolith' — разные ключи.
_LOOKUP: dict[str, str] = {k.lower(): v for k, v in _RAW_TO_CANON.items()}
def normalize_house_type(raw: str | None) -> str | None:
"""Преобразовать raw house_type в каноничный enum.
Уже-каноничное значение возвращается as-is (идемпотентность — нужна при
ре-обработке и для backfill-миграции). Неизвестное / 'other' / пустое /
None → None (НЕ 'other': см. docstring модуля про estimator soft-penalty).
Args:
raw: сырое значение из парсера (e.g. «MONOLIT», «monolithBrick») или None.
Returns:
Одно из panel / brick / monolith / monolith_brick / block / wood, либо None.
"""
if raw is None:
return None
stripped = raw.strip()
if not stripped:
return None
# Pass-through: уже каноничное значение (idempotency — в т.ч. 'monolith_brick',
# которого нет среди raw-ключей карты).
if stripped in _CANON:
return stripped
# Регистронезависимый exact-token lookup по обоим вокабулярам.
canon = _LOOKUP.get(stripped) or _LOOKUP.get(stripped.lower())
if canon is None:
# Неизвестное / 'other' — нормально (NULL нейтрально для estimator).
# debug, не warning: 'other' встречается массово, warning засорил бы лог.
logger.debug("house_type_normalizer: unmapped raw value %r — stored as NULL", raw)
return canon

32
tradein-mvp/backend/app/services/scrapers/price_brackets.py
View file

@ -1,32 +0,0 @@
"""Общие seed price-brackets для exhaustive-скрапперов (avito/cian/yandex).
EKB-настроенные (анализ прод-распределения вторички ЕКБ). Структура city-dict —
готова под per-city (multi-city: Москва пик 15-40М, Тюмень — иные; multi-city —
отдельная фича, см. vault decision). Сейчас единственный город — ekaterinburg.
"""
# (lo, hi) рубли; hi=None — открытый верхний брекет (без потолка). Закрытые
# брекеты соприкасаются — вызывающий применяет hi-1 (эксклюзивный верх), дедуп
# по source_id страхует на стыках.
_EKB_SECONDARY: list[tuple[int, int | None]] = [
(0, 4_000_000),
(4_000_000, 5_000_000),
(5_000_000, 6_000_000),
(6_000_000, 7_000_000),
(7_000_000, 8_000_000),
(8_000_000, 10_000_000),
(10_000_000, 13_000_000),
(13_000_000, 17_000_000),
(17_000_000, 25_000_000),
(25_000_000, 250_000_000),
(250_000_000, None), # 250М+ открытый catch-all (без потолка)
]
CITY_PRICE_SEED_BRACKETS: dict[str, list[tuple[int, int | None]]] = {
"ekaterinburg": _EKB_SECONDARY,
}
def get_price_seed_brackets(city: str = "ekaterinburg") -> list[tuple[int, int | None]]:
"""Seed price-brackets для города (fallback на ekaterinburg)."""
return CITY_PRICE_SEED_BRACKETS.get(city, _EKB_SECONDARY)

162
tradein-mvp/backend/app/services/scrapers/repair_state_normalizer.py
View file

@ -1,162 +0,0 @@
"""Нормализация repair_state → каноничный enum для listings.repair_state.
Целевой enum: needs_repair / standard / good / excellent.
Используется во всех инgest-путях: avito_detail, cian SERP, cian_detail.
Источники raw-значений:
- Avito HTML: «Ремонт: евро» → avito_detail._REPAIR_MAP → «euro» → good
- Cian SERP JSON: offer.decoration → «euro» / «cosmetic» / «without» / ...
- Cian Detail JSON: offer.repairType → «cosmetic» / «design» / «no» / «euro» / ...
Маппинг основан на _IMV_REPAIR_MAP из estimator.py (обратная форма):
needs_repair→«required», standard→«cosmetic», good→«euro», excellent→«designer».
Связь с estimator._REPAIR_COEF:
needs_repair=0.94 / standard=1.00 / good=1.05 / excellent=1.10
"""
from __future__ import annotations
import logging
import re
logger = logging.getLogger(__name__)
# Все известные raw-значения с источником в комментарии
_RAW_TO_ENUM: dict[str, str] = {
# ── needs_repair ────────────────────────────────────────────────────────────
"required": "needs_repair", # Avito: «требуется» / «без ремонта» (через _REPAIR_MAP)
"without": "needs_repair", # Cian: без ремонта
"no": "needs_repair", # Cian: нет ремонта (синоним without)
"rough": "needs_repair", # Cian/общий: черновая отделка
"prefine": "needs_repair", # Cian detail: предчистовая/whitebox (offer.decoration, #1791)
# ── standard ────────────────────────────────────────────────────────────────
"cosmetic": "standard", # Avito/Cian: косметический
# ── good ────────────────────────────────────────────────────────────────────
"euro": "good", # Avito/Cian: евро
"fine": "good", # Cian: хорошая отделка (предчистовая / whitebox)
# ── excellent ───────────────────────────────────────────────────────────────
"designer": "excellent", # Avito: дизайнерский
"design": "excellent", # Cian: дизайнерский (camelCase вариант)
}
# Допустимые значения целевого enum — для pass-through защиты
_VALID_ENUM: frozenset[str] = frozenset({"needs_repair", "standard", "good", "excellent"})
def normalize_repair_state(raw: str | None) -> str | None:
"""Преобразовать raw repair-значение в каноничный enum.
Если raw уже является корректным enum-значением — вернуть as-is (идемпотентно).
Если raw не распознан — вернуть None + залогировать WARNING один раз.
Args:
raw: сырое значение из парсера (e.g. «euro», «without», «design»)
или None.
Returns:
Одно из needs_repair / standard / good / excellent, либо None.
"""
if raw is None:
return None
# Pass-through: уже нормализованный enum (idempotency — нужно при ре-обработке)
if raw in _VALID_ENUM:
return raw
# Cian отдаёт camelCase ('preFine', 'whiteBox') — ключи карты в нижнем регистре,
# сравниваем case-insensitive, чтобы не плодить дубль-ключи на каждый регистр.
result = _RAW_TO_ENUM.get(raw) or _RAW_TO_ENUM.get(raw.lower())
if result is None:
logger.warning("repair_state_normalizer: unknown raw value %r — stored as NULL", raw)
return result
# ── Inference из текста описания (#622) ──────────────────────────────────────
# Когда структурное поле repair отсутствует (avito ~1% / cian ~5% / yandex 0%),
# извлекаем состояние из текста описания регэкспами по русским фразам.
#
# Порядок ВАЖЕН: проверяем от самого сильного сигнала к слабому
# excellent → good → needs_repair → standard,
# чтобы «после черновой сделали дизайнерский ремонт» дал excellent, а не
# needs_repair. Первое совпадение по порядку выигрывает.
#
# Целевой enum выровнен по normalize_repair_state() — те же 4 значения.
_TEXT_PATTERNS: tuple[tuple[re.Pattern[str], str], ...] = (
# ── excellent: дизайнерский / премиум / эксклюзивный ремонт ──────────────
(
re.compile(
r"дизайнерск(?:ий|ого|ом|ая)\s+ремонт"
r"|дизайнерск(?:ая|ой)\s+отделк"
r"|премиальн\w*\s+ремонт"
r"|эксклюзивн\w*\s+ремонт"
r"|авторск\w*\s+ремонт",
re.IGNORECASE,
),
"excellent",
),
# ── good: евроремонт / отличный / свежий / современный ремонт ────────────
(
re.compile(
r"евроремонт"
r"|евро[\s-]?ремонт"
r"|отличн\w*\s+ремонт"
r"|свеж\w*\s+ремонт"
r"|современн\w*\s+ремонт"
r"|качественн\w*\s+ремонт"
r"|ремонт\s+в\s+отличн",
re.IGNORECASE,
),
"good",
),
# ── needs_repair: без отделки / черновая / требует ремонта / под ремонт ──
(
re.compile(
r"без\s+отделк"
r"|без\s+ремонт"
r"|черновая\s+отделк"
r"|черновой\s+ремонт"
r"|требует(?:ся)?\s+ремонт"
r"|требует(?:ся)?\s+космет"
r"|нужен\s+ремонт"
r"|под\s+ремонт"
r"|под\s+чистов\w+\s+отделк"
r"|предчистов",
re.IGNORECASE,
),
"needs_repair",
),
# ── standard: косметический / обычный / жилое состояние ─────────────────
(
re.compile(
r"косметическ\w*\s+ремонт"
r"|косметическ\w*\s+отделк"
r"|обычн\w*\s+ремонт"
r"|жил(?:ое|ом)\s+состоян"
r"|хорош\w*\s+состоян",
re.IGNORECASE,
),
"standard",
),
)
def infer_repair_state_from_text(description: str | None) -> str | None:
"""Извлечь repair_state из текста описания объявления (#622).
Fallback-путь для повышения покрытия listings.repair_state, когда
структурное поле источника отсутствует. Проверяет паттерны в порядке
убывания силы сигнала (excellent → good → needs_repair → standard);
первое совпадение выигрывает.
Args:
description: текст описания объявления (avito/cian/yandex) или None.
Returns:
Одно из needs_repair / standard / good / excellent, либо None если
ни один паттерн не сработал (НЕ фабрикуем — unknown остаётся NULL).
"""
if not description:
return None
for pattern, enum_value in _TEXT_PATTERNS:
if pattern.search(description):
return enum_value
return None

92
tradein-mvp/backend/app/services/scrapers/snapshot_writer.py
View file

@ -1,92 +0,0 @@
"""listings_snapshots writer — point-in-time observation per listing per day.
Таблица listings_snapshots (016_listings_snapshots.sql):
PRIMARY KEY (listing_id, snapshot_date) — максимум 1 snapshot в сутки.
ON CONFLICT DO UPDATE — берём последний за день (перезаписываем при повторном run'е).
Используется двумя путями:
1. SERP scrape (save_listings в base.py) — записывает цену + позицию в выдаче
каждый раз когда listing появляется в поиске.
2. Detail backfill (save_detail_enrichment в cian_detail.py) — записывает цену
из detail-страницы для listings которые раньше не имели snapshot'а.
"""
from __future__ import annotations
import logging
from datetime import date
from sqlalchemy import text
from sqlalchemy.orm import Session
logger = logging.getLogger(__name__)
def upsert_listing_snapshot(
db: Session,
*,
listing_id: int,
price_rub: int,
price_per_m2: int | None = None,
run_id: int | None = None,
snapshot_date: date | None = None,
position_in_serp: int | None = None,
status: str | None = "active",
) -> None:
"""Записать / обновить snapshot для listing за текущий (или указанный) день.
Идемпотентно: повторный вызов с теми же listing_id + snapshot_date перезаписывает
строку (берём последние данные за день — ON CONFLICT DO UPDATE).
Args:
db: открытая SQLAlchemy session. Commit делает CALLER.
listing_id: PK из таблицы listings.
price_rub: текущая цена в рублях.
price_per_m2: цена за кв.м (вычисляется в caller'е, None если нет).
run_id: FK scrape_runs.id — с каким run'ом связан snapshot (None если backfill
запускается вне run'а).
snapshot_date: дата наблюдения; если None — используется CURRENT_DATE (по БД).
position_in_serp: позиция в SERP (1..60+), только при SERP-scrape.
status: 'active' / 'closed' / None. Default 'active' при обычном scrape.
"""
db.execute(
text("""
INSERT INTO listings_snapshots
(listing_id, snapshot_date, run_id,
price_rub, price_per_m2, position_in_serp, status, observed_at)
VALUES (
CAST(:lid AS bigint),
COALESCE(CAST(:snap_date AS date), CURRENT_DATE),
CAST(:run_id AS bigint),
CAST(:price AS bigint),
CAST(:ppm2 AS int),
CAST(:pos AS int),
:status,
NOW()
)
ON CONFLICT (listing_id, snapshot_date) DO UPDATE SET
run_id = COALESCE(EXCLUDED.run_id, listings_snapshots.run_id),
price_rub = EXCLUDED.price_rub,
price_per_m2 = COALESCE(EXCLUDED.price_per_m2, listings_snapshots.price_per_m2),
position_in_serp = COALESCE(
EXCLUDED.position_in_serp, listings_snapshots.position_in_serp
),
status = COALESCE(EXCLUDED.status, listings_snapshots.status),
observed_at = EXCLUDED.observed_at
"""),
{
"lid": listing_id,
"snap_date": snapshot_date,
"run_id": run_id,
"price": price_rub,
"ppm2": price_per_m2,
"pos": position_in_serp,
"status": status,
},
)
logger.debug(
"upsert_listing_snapshot: listing_id=%s date=%s price=%s run_id=%s",
listing_id,
snapshot_date or "CURRENT_DATE",
price_rub,
run_id,
)

355
tradein-mvp/backend/app/services/scrapers/yandex_helpers.py
View file

@ -1,355 +0,0 @@
"""Yandex Realty parsing helpers — JSON-LD + regex + DOM utilities for SERP, detail, newbuilding,
and valuation parsers."""
from __future__ import annotations
import json
import re
from datetime import date, timedelta
from typing import Any
from selectolax.parser import HTMLParser
__all__ = [
"RE_AGENCY_FOUNDED",
"RE_AGENCY_OBJECTS",
"RE_FLOOR",
"RE_JK_ID",
"RE_METRO_WALK",
"RE_OFFER_ID",
"RE_PPM2",
"RE_PRICE",
"RE_STREET_ID",
"RE_TITLE_AREA",
"RE_TITLE_ROOMS",
"RE_VIEWS",
"RE_YEAR",
"RU_MONTHS",
"RU_MONTH_NOMINATIVE",
"extract_json_ld",
"find_ld_by_type",
"parse_dmy",
"parse_house_class",
"parse_house_type",
"parse_listing_date",
"parse_ru_date",
"parse_rub",
]
# ---------------------------------------------------------------------------
# Section 1 — JSON-LD extraction
# ---------------------------------------------------------------------------
def extract_json_ld(html: str) -> list[dict[str, Any]]:
"""Extract all <script type='application/ld+json'> blocks from HTML.
Returns a list of parsed JSON-LD dicts. Skips blocks that fail to parse.
"""
tree = HTMLParser(html)
results: list[dict[str, Any]] = []
for script in tree.css('script[type="application/ld+json"]'):
text = script.text() or ""
if not text.strip():
continue
try:
parsed = json.loads(text)
except json.JSONDecodeError:
continue
if isinstance(parsed, dict):
results.append(parsed)
elif isinstance(parsed, list):
# Some pages wrap multiple LDs in a JSON array
results.extend(item for item in parsed if isinstance(item, dict))
return results
def find_ld_by_type(html: str, type_name: str) -> dict[str, Any] | None:
"""Return the first JSON-LD dict whose @type equals type_name.
If @type is a list, returns the dict if type_name is contained in it.
"""
for ld in extract_json_ld(html):
t = ld.get("@type")
if t == type_name:
return ld
if isinstance(t, list) and type_name in t:
return ld
return None
# ---------------------------------------------------------------------------
# Section 2 — Regex extractors (module-level compiled)
# ---------------------------------------------------------------------------
# URL pattern extractors
RE_OFFER_ID = re.compile(r"/offer/(\d+)/?")
RE_JK_ID = re.compile(r"/novostrojka/([\w-]+?)-(\d+)/?") # group(1)=slug, group(2)=id
RE_STREET_ID = re.compile(r"/kupit/kvartira/st-([\w-]+?)-(\d+)/?")
# Card / title text extractors
RE_TITLE_AREA = re.compile(r"(\d+[.,]?\d*)\s*м²")
RE_TITLE_ROOMS = re.compile(r"(\d+)\s*-?\s*комнатн|(студи[яюй])", re.IGNORECASE)
RE_FLOOR = re.compile(r"(\d+)\s+этаж\s+из\s+(\d+)", re.IGNORECASE)
RE_PRICE = re.compile(r"(\d[\d\s]+\d)\s*₽")
RE_PPM2 = re.compile(r"(\d[\d\s]+)\s*₽\s*за\s*м²", re.IGNORECASE)
RE_VIEWS = re.compile(r"(\d+)\s+просмотр", re.IGNORECASE)
# Detail / NLP extractors
RE_METRO_WALK = re.compile(
r"(\d+)\s+минут\s+(?:неспешной\s+прогулки|ходьбы|пешком)", re.IGNORECASE
)
RE_YEAR = re.compile(r"\b(19\d{2}|20[0-2]\d)\b")
# Agency block ("Год основания 1995", "150 объектов")
RE_AGENCY_FOUNDED = re.compile(r"Год\s+основания\s+(\d{4})", re.IGNORECASE)
RE_AGENCY_OBJECTS = re.compile(r"(\d+)\s+объект", re.IGNORECASE)
# ---------------------------------------------------------------------------
# Section 3 — Russian date parsing
# ---------------------------------------------------------------------------
RU_MONTHS: dict[str, int] = {
"января": 1, "февраля": 2, "марта": 3, "апреля": 4, "мая": 5, "июня": 6,
"июля": 7, "августа": 8, "сентября": 9, "октября": 10, "ноября": 11, "декабря": 12,
}
RU_MONTH_NOMINATIVE: dict[str, int] = {
"январь": 1, "февраль": 2, "март": 3, "апрель": 4, "май": 5, "июнь": 6,
"июль": 7, "август": 8, "сентябрь": 9, "октябрь": 10, "ноябрь": 11, "декабрь": 12,
}
RE_RU_DATE = re.compile(r"(\d{1,2})\s+(\w+)\s+(\d{4})")
RE_DMY = re.compile(r"(\d{2})\.(\d{2})\.(\d{4})")
def parse_ru_date(text: str | None) -> date | None:
"""Parse Russian date string like '9 мая 2026' → date(2026, 5, 9). Genitive month form only."""
if not text:
return None
m = RE_RU_DATE.search(text)
if not m:
return None
day_s, month_ru, year_s = m.groups()
month = RU_MONTHS.get(month_ru.lower())
if not month:
return None
try:
return date(int(year_s), month, int(day_s))
except (ValueError, TypeError):
return None
def parse_dmy(text: str | None) -> date | None:
"""Parse DD.MM.YYYY format (used by Yandex Valuation tool history) → date."""
if not text:
return None
m = RE_DMY.search(text)
if not m:
return None
try:
return date(int(m.group(3)), int(m.group(2)), int(m.group(1)))
except (ValueError, TypeError):
return None
# Relative date parsing (Yandex SERP shows "N дней назад" / "N часов назад")
_RE_REL_DATE = re.compile(
r"(?P<n>\d+)\s+(?P<unit>"
r"секунд[ауы]?|"
r"минут[ауы]?|"
r"час(?:а|ов)?|"
r"день|дн(?:я|ей?)|"
r"недел[июяь]+|"
r"месяц(?:а|ев)?|"
r"год(?:а|ов)?"
r")\s+назад",
flags=re.I,
)
_REL_UNIT_DAYS: dict[str, int] = {
"секунд": 0, "минут": 0, "час": 0,
"день": 1, "дн": 1,
"недел": 7,
"месяц": 30,
"год": 365,
}
def _parse_relative_date_yandex(s: str | None) -> date | None:
"""Parse 'N дней/недель/месяцев назад' → date. Returns None if not matched."""
if not s:
return None
m = _RE_REL_DATE.search(s)
if not m:
return None
n = int(m["n"])
unit_raw = m["unit"].lower()
for prefix, days in _REL_UNIT_DAYS.items():
if unit_raw.startswith(prefix):
return date.today() - timedelta(days=n * days)
return None
RE_RU_DATE_NO_YEAR = re.compile(
r"(\d{1,2})\s+(январ\w*|феврал\w*|март\w*|апрел\w*|ма[яй]|"
r"июн\w*|июл\w*|август\w*|сентябр\w*|октябр\w*|ноябр\w*|декабр\w*)"
r"(?!\s+\d{4})",
flags=re.I,
)
_RU_MONTH_PREFIXES: dict[str, int] = {
"январ": 1, "феврал": 2, "март": 3, "апрел": 4,
"ма": 5, "июн": 6, "июл": 7, "август": 8,
"сентябр": 9, "октябр": 10, "ноябр": 11, "декабр": 12,
}
def parse_listing_date(text: str | None) -> date | None:
"""Parse listing date from Yandex card text.
Tries in order:
1. Keyword shortcut: 'сегодня' / 'вчера' / 'позавчера' → relative day
2. Absolute Russian date with year: '9 мая 2026' → date(2026, 5, 9)
3. Absolute without year: '18 мая' → current year (rollback to prev
year if month >30 дней в будущем)
4. Relative date: 'N дней назад' → date.today() - N days
Returns None if none match.
"""
if not text:
return None
text_lower = text.lower()
# 1. Keyword shortcuts (common on Yandex SERP for recent cards).
if "позавчера" in text_lower:
return date.today() - timedelta(days=2)
if "вчера" in text_lower:
return date.today() - timedelta(days=1)
if "сегодня" in text_lower or "только что" in text_lower:
return date.today()
# 2. Absolute with year (existing).
result = parse_ru_date(text)
if result is not None:
return result
# 3. Absolute without year — infer current, rollback to prev if future.
m = RE_RU_DATE_NO_YEAR.search(text)
if m:
day = int(m.group(1))
month_raw = m.group(2).lower()
month: int | None = None
for prefix, num in _RU_MONTH_PREFIXES.items():
if month_raw.startswith(prefix):
month = num
break
if month is not None and 1 <= day <= 31:
today = date.today()
try:
parsed = date(today.year, month, day)
except ValueError:
parsed = None
if parsed and (parsed - today).days > 30:
try:
parsed = date(today.year - 1, month, day)
except ValueError:
parsed = None
if parsed:
return parsed
# 4. Relative numeric.
return _parse_relative_date_yandex(text)
# ---------------------------------------------------------------------------
# Section 4 — House type / class NLP from description text
# ---------------------------------------------------------------------------
def parse_house_type(text: str | None) -> str | None:
"""Extract house material type from free-form description.
Returns: 'panel' | 'monolith_brick' | 'monolith' | 'brick' | 'block' | 'wood' | None.
Order matters: 'monolith_brick' is checked before 'monolith' and 'brick' alone.
"""
if not text:
return None
t = text.lower()
if "панельн" in t:
return "panel"
if "монолит" in t and "кирпич" in t:
return "monolith_brick"
if "монолит" in t:
return "monolith"
if "кирпичн" in t:
return "brick"
if "блочн" in t:
return "block"
if "деревянн" in t:
return "wood"
return None
def parse_house_class(text: str | None) -> str | None:
"""Extract ЖК class from description.
Returns: 'elite' | 'premium' | 'business' | 'comfort_plus' | 'comfort' | 'economy' | None.
"""
if not text:
return None
t = text.lower()
if re.search(r"класса?\s+элит|элитный\s+класс", t):
return "elite"
if re.search(r"класса?\s+премиум|премиум.{0,5}класс", t):
return "premium"
if re.search(r"класса?\s+бизнес", t):
return "business"
if re.search(r"класса?\s+комфорт\s*\+", t):
return "comfort_plus"
if re.search(r"класса?\s+комфорт(?!\s*\+)", t):
return "comfort"
if re.search(r"класса?\s+эконом", t):
return "economy"
return None
# ---------------------------------------------------------------------------
# Section 5 — Money parsing
# ---------------------------------------------------------------------------
RE_RUB_MLN = re.compile(r"([\d,.]+)\s*млн\s*₽", re.IGNORECASE)
RE_RUB_RAW = re.compile(r"(\d[\d\s]*\d|\d)")
def parse_rub(text: str | None) -> int | None:
"""Parse Russian price string → integer rubles.
Supports:
'4 399 000 ₽' → 4_399_000
'4,4 млн ₽' → 4_400_000
'4.4 млн ₽' → 4_400_000
'117 500' → 117_500 (raw digits without currency, e.g. ppm2)
None / 'без цены' / '' → None
"""
if not text:
return None
text = text.strip()
if not text:
return None
m = RE_RUB_MLN.search(text)
if m:
try:
# `\s` strips also NBSP (\xa0), thin space, и т.д. — Yandex SERP
# форматирует «9\xa0800\xa0000» через NBSP, не regular space.
cleaned = re.sub(r"\s", "", m.group(1)).replace(",", ".")
return int(float(cleaned) * 1_000_000)
except (ValueError, TypeError):
pass
m = RE_RUB_RAW.search(text)
if m:
# Same NBSP issue: `.replace(" ", "")` оставляет \xa0 → int() raises.
cleaned = re.sub(r"\s", "", m.group(1))
try:
return int(cleaned)
except ValueError:
return None
return None

432
tradein-mvp/backend/app/services/scrapers/yandex_newbuilding.py
View file

@ -1,432 +0,0 @@
"""Yandex Realty ЖК landing page parser.
URL pattern: /{city}/kupit/novostrojka/<slug>-<id>/
Reference target: ЖК Татлин (id=1592987, slug=tatlin) — comfort+, June 2023,
PRINZIP, rating 4.3, 1505 ratings, 353 text reviews, coords (56.855312, 60.576668).
Fetch strategy (#974):
- Все network-запросы (ЖК-лендинг + SERP slug-resolve) идут через BrowserFetcher
(tradein-browser camoufox), а НЕ через httpx/_http_get.
- Yandex Realty — JS-heavy / anti-bot; curl_cffi / httpx не получают данные.
- BrowserFetcher.fetch(url) → str (полный HTML); вызывающий код парсит через parse().
"""
from __future__ import annotations
import logging
import re
from typing import Any
from pydantic import BaseModel, Field
from selectolax.parser import HTMLParser, Node
from app.services.scraper_settings import get_scraper_delay
from app.services.scrapers.base import BaseScraper
from app.services.scrapers.yandex_helpers import (
RE_JK_ID,
parse_house_class,
parse_house_type,
)
logger = logging.getLogger(__name__)
# ── Models ───────────────────────────────────────────────────────────────────
class JKMetroStation(BaseModel):
name: str
walk_min: int | None = None
class YandexNewbuildingInfo(BaseModel):
"""ЖК landing scrape payload."""
ext_id: str # '1592987'
ext_slug: str # 'tatlin'
source_url: str
# Identification
name: str | None = None # 'ЖК «Татлин»'
address: str | None = None # 'Екатеринбург, ул. Черепанова / ул. Готвальда'
# Coords (inline in HTML — Yandex unique vs Avito/Cian)
lat: float | None = None
lon: float | None = None
# Classification + commission
house_class: str | None = None # 'comfort_plus' etc.
commission_year: int | None = None
commission_month: str | None = None # 'июнь' (RU)
# Footprint
total_floors: int | None = None # 35
corpus_count: int | None = None # 3
total_area_ha: float | None = None # 1.5
# Developer
developer_name: str | None = None # 'PRINZIP'
developer_url: str | None = None
developer_other_jk: list[str] = Field(default_factory=list)
# Reviews
rating: float | None = None # 4.3
ratings_count: int | None = None # 1505 — оценок
text_reviews_count: int | None = None # 353 — текстовых отзывов
# NLP / description
description: str | None = None
# Metro
metro_stations: list[JKMetroStation] = Field(default_factory=list)
# House type (best-effort NLP from description)
house_type: str | None = None
raw_payload: dict[str, Any] | None = None
# ── Regex constants ───────────────────────────────────────────────────────────
RE_FLOORS_TOWERS = re.compile(r"(\d+)-этажн\w+\s+башн", re.IGNORECASE)
RE_AREA_HA = re.compile(r"участке\s+([\d,]+)\s*га", re.IGNORECASE)
RE_COMMISSION = re.compile(
r"(?:введ[её]н|сдан)\s+в\s+эксплуатацию\s+в\s+(\w+)\s+(\d{4})", re.IGNORECASE
)
# Yandex drift (#974 follow-up): рейтинг рендерится без пробела перед «из»
# ("4.3из 5" / "4,5из 5", nbsp внутри) → \s* (а не \s+) перед «из», иначе rating=NULL
# при заполненных ratings_count/text_reviews (verified live 2026-06-15, ЖК Татлин/Рио).
RE_RATING = re.compile(r"(\d[.,]\d)\s*из\s+5")
RE_RATINGS_COUNT = re.compile(r"(\d+)\s+оценок", re.IGNORECASE)
RE_TEXT_REVIEWS = re.compile(r"Смотреть\s+все\s+(\d+)\s+отзыв", re.IGNORECASE)
RE_CORPUS_COUNT_WORD = re.compile(
r"(одн[ау]|две|три|четыре|пять|шесть|семь|восемь|\d+)\s+(?:[\d\s-]*)\s*-?\s*этажн",
re.IGNORECASE,
)
RE_METRO_INLINE = re.compile(
r"([А-ЯЁ][А-Яа-яё\s-]{2,30}?)\s+(\d+)\s*мин",
)
# Slug из href: /{city}/kupit/novostrojka/<slug>-<id>/
# Используется в resolve_yandex_jk_slug для извлечения slug из SERP href.
_JK_SLUG_RE = re.compile(
r"/(?:ekaterinburg|moskva|spb|[a-z-]+)/kupit/novostrojka/([a-z0-9-]+)-(\d+)/"
)
# Ekb-specific coord ranges (extend per-city later)
LAT_RANGE = (55.5, 57.5)
LON_RANGE = (59.5, 61.5)
# Word → number map
_WORD_NUM: dict[str, int] = {
"одна": 1,
"одну": 1,
"одной": 1,
"две": 2,
"три": 3,
"четыре": 4,
"пять": 5,
"шесть": 6,
"семь": 7,
"восемь": 8,
}
# ── Scraper class ─────────────────────────────────────────────────────────────
class YandexNewbuildingScraper(BaseScraper):
name = "yandex_realty_nb"
base_url = "https://realty.yandex.ru"
request_delay_sec = 5.0 # class default; instance value loaded from scraper_settings
def __init__(self) -> None:
super().__init__()
self.request_delay_sec = get_scraper_delay(self.name)
async def fetch_around(self, lat: float, lon: float, radius_m: int = 1000) -> list: # type: ignore[override]
raise NotImplementedError(
"YandexNewbuildingScraper is JK-slug-based; use fetch_jk(slug, id) instead."
)
async def fetch_jk(
self, jk_slug: str, jk_id: str, city: str = "ekaterinburg"
) -> YandexNewbuildingInfo | None:
"""Загрузить ЖК-лендинг через BrowserFetcher и распарсить.
Yandex Realty — JS/anti-bot: curl/httpx не получают данные. Запрос идёт
через tradein-browser (camoufox) контейнер — единственный рабочий путь (#974).
"""
from app.services.scrapers.browser_fetcher import BrowserFetcher
url = f"{self.base_url}/{city}/kupit/novostrojka/{jk_slug}-{jk_id}/"
try:
async with BrowserFetcher(source="yandex") as fetcher:
html = await fetcher.fetch(url)
except Exception:
logger.exception("yandex nb browser fetch failed: %s", url)
return None
if not html or len(html) < 500:
logger.warning(
"yandex nb browser returned empty/tiny HTML (%d bytes): %s",
len(html) if html else 0,
url,
)
return None
result = self.parse(html, jk_slug=jk_slug, jk_id=jk_id, source_url=url)
return result
def parse(self, html: str, jk_slug: str, jk_id: str, source_url: str) -> YandexNewbuildingInfo:
tree = HTMLParser(html)
body = tree.body
body_text = body.text(strip=True) if body else ""
# Header
h1 = tree.css_first("h1")
name = h1.text(strip=True) if h1 else None
# Sections (heading-based extraction)
location_text = _find_section_text(tree, "Расположение") or ""
description_section = (
_find_section_text(tree, "О комплексе")
or _find_section_text(tree, "Расположение, транспортная доступность")
or location_text
)
# Coords — inline scan within plausible range
lat, lon = _extract_coords(html)
# Class
house_class = parse_house_class(description_section) or parse_house_class(body_text)
# Commission
commission_year: int | None = None
commission_month: str | None = None
c_m = RE_COMMISSION.search(description_section) or RE_COMMISSION.search(body_text)
if c_m:
commission_month = c_m.group(1).lower()
try:
commission_year = int(c_m.group(2))
except ValueError:
pass
# Floors + corpus + area
floors_m = RE_FLOORS_TOWERS.search(description_section) or RE_FLOORS_TOWERS.search(
body_text
)
total_floors = int(floors_m.group(1)) if floors_m else None
corpus_count = _parse_corpus_count(description_section or body_text)
area_m = RE_AREA_HA.search(description_section) or RE_AREA_HA.search(body_text)
total_area_ha = float(area_m.group(1).replace(",", ".")) if area_m else None
# Developer
dev_link = tree.css_first('[data-test="CARD_DEV_BADGE_DEVELOPER_LINK"]')
developer_name: str | None = None
developer_url: str | None = None
if dev_link is not None:
developer_name = dev_link.text(strip=True) or None
href = dev_link.attributes.get("href", "")
if href:
developer_url = (
href if href.startswith("http") else f"https://realty.yandex.ru{href}"
)
# Developer's other JKs
other_jk_block = tree.css_first('[data-test="CardDevSites"]')
developer_other_jk: list[str] = []
if other_jk_block is not None:
for jk_a in other_jk_block.css("a"):
t = jk_a.text(strip=True)
if t and t not in developer_other_jk:
developer_other_jk.append(t)
# Reviews
rating_m = RE_RATING.search(body_text)
rating = float(rating_m.group(1).replace(",", ".")) if rating_m else None
rcount_m = RE_RATINGS_COUNT.search(body_text)
ratings_count = int(rcount_m.group(1)) if rcount_m else None
treviews_m = RE_TEXT_REVIEWS.search(body_text)
text_reviews_count = int(treviews_m.group(1)) if treviews_m else None
# Address (header area before lat/lon block)
address = _extract_jk_address(body_text, name)
# Metro stations
metro_stations = _parse_metro(body_text)
# NLP house_type fallback
house_type = parse_house_type(description_section) or parse_house_type(body_text)
return YandexNewbuildingInfo(
ext_id=jk_id,
ext_slug=jk_slug,
source_url=source_url,
name=name,
address=address,
lat=lat,
lon=lon,
house_class=house_class,
commission_year=commission_year,
commission_month=commission_month,
total_floors=total_floors,
corpus_count=corpus_count,
total_area_ha=total_area_ha,
developer_name=developer_name,
developer_url=developer_url,
developer_other_jk=developer_other_jk[:10],
rating=rating,
ratings_count=ratings_count,
text_reviews_count=text_reviews_count,
description=description_section or None,
metro_stations=metro_stations,
house_type=house_type,
raw_payload={
"description_len": len(description_section or ""),
"body_len": len(body_text),
},
)
# ── Slug resolution via SERP ──────────────────────────────────────────────────
async def resolve_yandex_jk_slug(
jk_id: str,
city: str = "ekaterinburg",
) -> str | None:
"""Найти Yandex Realty slug для ЖК по его ext_id (jk_id) через SERP.
Стратегия (#974 — зеркало resolve_cian_zhk_url_via_search):
1. Запросить поисковую страницу Yandex Realty через BrowserFetcher.
URL: /ekaterinburg/kupit/novostrojka/?siteId=<jk_id>
2. В HTML найти первую ссылку вида /<city>/kupit/novostrojka/<slug>-<jk_id>/
через regex _JK_SLUG_RE.
3. Вернуть slug или None при любой ошибке.
Caller несёт ответственность за anti-bot sleep (зеркало cian_newbuilding.py).
BrowserFetcher обязателен — Yandex Realty JS/anti-bot, httpx/curl не работают.
Returns:
slug (str без id-суффикса), или None при ошибке / не найден.
"""
from app.services.scrapers.browser_fetcher import BrowserFetcher
# Yandex Realty SERP: фильтр по siteId → первый результат = нужный ЖК.
# Альтернативный путь через Яндекс Поиск (web SERP) менее надёжен из-за
# вариативности разметки. Прямой realty.yandex.ru SERP — стабильнее.
serp_url = f"https://realty.yandex.ru/{city}/kupit/novostrojka/?siteId={jk_id}"
try:
async with BrowserFetcher(source="yandex") as fetcher:
html = await fetcher.fetch(serp_url)
except Exception as exc:
logger.warning("resolve_yandex_jk_slug jk_id=%s browser fetch failed: %s", jk_id, exc)
return None
if not html:
logger.warning("resolve_yandex_jk_slug jk_id=%s: empty HTML from browser", jk_id)
return None
# Ищем ссылку вида /{city}/kupit/novostrojka/<slug>-<id>/
# Ограничиваем: id в ссылке должен совпадать с искомым jk_id.
for m in _JK_SLUG_RE.finditer(html):
if m.group(2) == str(jk_id):
slug = m.group(1)
logger.info("resolve_yandex_jk_slug jk_id=%s → slug=%s", jk_id, slug)
return slug
logger.warning(
"resolve_yandex_jk_slug jk_id=%s: no matching slug in SERP HTML (markup drift?)",
jk_id,
)
return None
# ── helpers ───────────────────────────────────────────────────────────────────
def _extract_coords(html: str) -> tuple[float | None, float | None]:
"""Scan HTML for any `<lat>.\\d+` and `<lon>.\\d+` matching plausible city range."""
lats = [
float(m)
for m in re.findall(r"\b(\d{2}\.\d{4,8})\b", html)
if LAT_RANGE[0] <= float(m) <= LAT_RANGE[1]
]
lons = [
float(m)
for m in re.findall(r"\b(\d{2}\.\d{4,8})\b", html)
if LON_RANGE[0] <= float(m) <= LON_RANGE[1]
]
lat = lats[0] if lats else None
lon = lons[0] if lons else None
return lat, lon
def _find_section_text(tree: HTMLParser, heading: str) -> str | None:
for h in tree.css("h2, h3"):
if heading.lower() in (h.text(strip=True) or "").lower():
parts: list[str] = []
node: Node | None = h.next
while node is not None:
tag = (node.tag or "").lower()
if tag in {"h2", "h3"}:
break
txt = node.text(strip=True) if hasattr(node, "text") else ""
if txt:
parts.append(txt)
node = node.next
return " ".join(parts).strip() or None
return None
def _parse_corpus_count(text: str) -> int | None:
m = RE_CORPUS_COUNT_WORD.search(text)
if not m:
return None
token = m.group(1).lower()
if token.isdigit():
return int(token)
return _WORD_NUM.get(token)
def _parse_metro(text: str) -> list[JKMetroStation]:
stations: list[JKMetroStation] = []
for m in RE_METRO_INLINE.finditer(text):
name = m.group(1).strip()
# filter common noise
if not 2 <= len(name) <= 40 or "ходьб" in name.lower() or "минут" in name.lower():
continue
try:
stations.append(JKMetroStation(name=name, walk_min=int(m.group(2))))
except ValueError:
continue
if len(stations) >= 5:
break
return stations
def _extract_jk_address(body_text: str, name: str | None) -> str | None:
"""Find address line — typically right after JK name, before metro entries."""
if name and name in body_text:
after_name = body_text.split(name, 1)[1]
# Match 'Город, ул. X / ул. Y' until first metro mention
m = re.match(
r"\s*([А-ЯЁ][^•]{5,150}?)(?:\s+[А-ЯЁ][а-яё]+\s+\d+\s*мин|\d+\s+оценок|$)",
after_name,
)
if m:
addr = m.group(1).strip().rstrip(",").strip()
if 10 < len(addr) < 200:
return addr
return None
# Expose RE_JK_ID at module level for external use (e.g. SERP link → newbuilding detail)
__all__ = [
"RE_JK_ID",
"JKMetroStation",
"YandexNewbuildingInfo",
"YandexNewbuildingScraper",
"resolve_yandex_jk_slug",
]

8
tradein-mvp/backend/app/services/yandex_price_history.py
View file

@ -29,10 +29,10 @@ import logging
from datetime import UTC, datetime, timedelta
# #2306: ScrapedLot migrated to scraper_kit (byte-identical field/method surface to
# legacy app.services.scrapers.base, см. tests/test_scraper_kit_pricehistory_session_parity.py).
# record_yandex_price_history reads lot.* via duck-typing only — no isinstance checks —
# так что kit- и legacy-скрапперы (оба конструируют ScrapedLot по (source, source_id,
# price_rub, price_previous_rub, price_trend)) остаются взаимозаменяемы для этой функции.
# legacy base-module ScrapedLot, deleted #2397 финальный шаг E — kit единственный
# живой путь). record_yandex_price_history reads lot.* via duck-typing only — no
# isinstance checks — так что любой ScrapedLot-совместимый объект (source, source_id,
# price_rub, price_previous_rub, price_trend) остаётся взаимозаменяем для этой функции.
from scraper_kit.base import ScrapedLot
from sqlalchemy import text
from sqlalchemy.orm import Session

78
tradein-mvp/backend/tests/scrapers/test_admin_domclick_ingest_kit_parity.py
View file

@ -1,78 +0,0 @@
"""Parity-proof: `scripts/ingest_domclick_jsonl.py` (#2305, Group A) — legacy vs scraper_kit.
`scripts/ingest_domclick_jsonl.py` теперь строит `ScrapedLot` из
`scraper_kit.base` (вместо `app.services.scrapers.base`) и вызывает
`scraper_kit.base.save_listings` (matcher/region_code инжектируются — см.
`app/services/scraper_adapters.RealMatcherAdapter` + `DEFAULT_REGION_CODE`).
`DomClickDetailEnrichment`/`save_detail_enrichment` изначально планировались
остаться на legacy: на момент начала миграции admin.py `domclick_detail.py` не
имел kit-эквивалента (см. audit `Scraper_Kit_Legacy_Dependency_Audit_0703`).
Issue #2307 (Group D, PR #2318) портировал `domclick_detail.py` в kit ПАРАЛЛЕЛЬНО
с этой задачей и явно оставил `scripts/ingest_domclick_jsonl.py` untouched
("the only other domclick_detail caller is scripts/ingest_domclick_jsonl.py,
also untouched") — поэтому переключение этого caller'а сделано ЗДЕСЬ, в рамках
#2305, теперь когда kit-эквивалент существует.
Легаси `app.services.scrapers.domclick_detail` удалён (#2277 финальный шаг
scraper_kit-миграции, 0 runtime importers, `scripts/ingest_domclick_jsonl.py` уже
100% на kit-стороне для `DomClickDetailEnrichment`) — legacy-vs-kit parity для
`DomClickDetailEnrichment`-конструирования убрана вместе с ним; `ScrapedLot`
parity (`app.services.scrapers.base`, всё ещё легаси/живой модуль) остаётся.
"""
from __future__ import annotations
import os
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
from scraper_kit.base import ScrapedLot as KitScrapedLot
from app.services.scrapers.base import ScrapedLot as LegacyScrapedLot
from tests.support.parity import assert_parity
# Аналог одной ok-записи JSONL-раннера (см. ingest_domclick_jsonl._build_lot).
_LOT_KWARGS = {
"source": "domklik",
"source_url": "https://ekaterinburg.domclick.ru/card/sale__flat__2075729321",
"source_id": "2075729321",
"address": "Екатеринбург, ул. Тестовая, 1",
"lat": 56.83,
"lon": 60.6,
"rooms": 2,
"area_m2": 54.3,
"floor": 5,
"total_floors": 9,
"year_built": 2015,
"price_rub": 8500000,
"listing_segment": "vtorichka",
"repair_state": "euro",
"raw_payload": {"is_sber_collateral": False, "square_price": 156538, "avm": {}},
}
def _build_legacy_lot(rec: dict[str, object]) -> LegacyScrapedLot:
return LegacyScrapedLot(**rec) # type: ignore[arg-type]
def _build_kit_lot(rec: dict[str, object]) -> KitScrapedLot:
return KitScrapedLot(**rec) # type: ignore[arg-type]
def test_scraped_lot_are_class_distinct() -> None:
"""Sanity: legacy/kit ScrapedLot — разные классы (pydantic), обычный `==` даёт False."""
legacy_result = _build_legacy_lot(_LOT_KWARGS)
kit_result = _build_kit_lot(_LOT_KWARGS)
assert type(legacy_result) is not type(kit_result)
assert legacy_result.model_dump() == kit_result.model_dump()
def test_parity_ingest_domclick_scraped_lot() -> None:
"""`ingest_domclick_jsonl._build_lot()` kwargs → ScrapedLot(legacy) vs ScrapedLot(kit)."""
assert_parity(
legacy_fn=_build_legacy_lot,
kit_fn=_build_kit_lot,
fixtures=[_LOT_KWARGS],
)

220
tradein-mvp/backend/tests/scrapers/test_avito_403_backconnect_reconnect.py
View file

@ -1,220 +0,0 @@
"""curl_cffi 403/firewall reconnect-retry под backconnect-прокси.
Под backconnect rotating-прокси (mproxy.site, нет changeip-URL) одиночный HTTP 403
или firewall-интерстициал ронял весь avito_full_load sweep через AvitoBlockedError →
mark_banned. Каждое новое соединение через backconnect даёт свежий exit-IP, поэтому
ретраим пересозданием curl_cffi-сессии до _AVITO_403_MAX_RETRIES раз.
curl_cffi-сессия всегда egress через scraper_proxy_url (backconnect mproxy), поэтому
reconnect-retry гейтим ТОЛЬКО на наличие proxy — независимо от того, задан ли changeip-URL
(тот относится к ДРУГОМУ, browser/auv-прокси). Включаем через scraper_proxy_url_env
(scraper_proxy_url — read-only property, его не monkeypatch'ишь).
"""
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from app.services.scrapers import avito as avito_mod
from app.services.scrapers.avito import AvitoScraper
from app.services.scrapers.avito_exceptions import AvitoBlockedError
# Валидная карточка SERP (одна) — successful page после reconnect.
_CARD_HTML = (
"<html><body>"
'<div data-marker="item" data-item-id="123456789">'
'<a data-marker="item-title" href="/ekaterinburg/kvartiry/test-123456789">'
"2-к. квартира, 54,3 м², 5/9 эт."
"</a>"
'<meta itemprop="price" content="4500000"/>'
"</div>"
"</body></html>"
)
# Firewall-интерстициал (HTTP 200, не 403).
_FIREWALL_HTML = (
"<html><head><title>Доступ ограничен: проблема с IP</title></head>"
'<body><div class="firewall-container">...</div></body></html>'
)
def _enable_backconnect(monkeypatch: pytest.MonkeyPatch) -> None:
"""scraper_proxy_url задан (через env-поле), changeip-URL — нет → backconnect."""
monkeypatch.setattr(avito_mod.settings, "scraper_proxy_url_env", "http://u:p@mproxy.site:14619")
monkeypatch.setattr(avito_mod.settings, "avito_proxy_url", None)
monkeypatch.setattr(avito_mod.settings, "avito_proxy_rotate_url", None)
@pytest.mark.asyncio
async def test_backconnect_403_403_200_reconnect_succeeds(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""backconnect: 403 → 403 → 200 → reconnect-retry возвращает HTML, _reset_cffi дёрнут 2×."""
_enable_backconnect(monkeypatch)
scraper = AvitoScraper()
r403 = MagicMock()
r403.status_code = 403
r403.text = ""
ok = MagicMock()
ok.status_code = 200
ok.text = _CARD_HTML
mock_session = AsyncMock()
mock_session.get = AsyncMock(side_effect=[r403, r403, ok])
scraper._cffi = mock_session
# _reset_cffi не должен реально пересоздавать сессию (иначе .get-side_effect
# потеряется) — патчим на no-op счётчик, проверяем что вызвался.
reset = AsyncMock()
with (
patch("app.services.scrapers.avito.asyncio.sleep", AsyncMock()),
patch.object(AvitoScraper, "_reset_cffi", reset),
patch.object(AvitoScraper, "_rotate_ip", AsyncMock(return_value=True)) as rot,
):
html = await scraper._fetch_serp_html("https://www.avito.ru/x", page=1)
assert html == _CARD_HTML
assert reset.await_count == 2 # 403, 403 → два reconnect перед 200
rot.assert_not_awaited() # backconnect не дёргает changeip-ротацию
assert mock_session.get.await_count == 3
@pytest.mark.asyncio
async def test_backconnect_firewall_reconnect_succeeds(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""backconnect: firewall-200 → 200 → reconnect-retry (firewall тоже триггерит)."""
_enable_backconnect(monkeypatch)
scraper = AvitoScraper()
fw = MagicMock()
fw.status_code = 200
fw.text = _FIREWALL_HTML
ok = MagicMock()
ok.status_code = 200
ok.text = _CARD_HTML
mock_session = AsyncMock()
mock_session.get = AsyncMock(side_effect=[fw, ok])
scraper._cffi = mock_session
reset = AsyncMock()
with (
patch("app.services.scrapers.avito.asyncio.sleep", AsyncMock()),
patch.object(AvitoScraper, "_reset_cffi", reset),
):
html = await scraper._fetch_serp_html("https://www.avito.ru/x", page=1)
assert html == _CARD_HTML
assert reset.await_count == 1
@pytest.mark.asyncio
async def test_backconnect_403_exhausted_raises_blocked(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""backconnect: 403 _AVITO_403_MAX_RETRIES+1 раз → всё равно AvitoBlockedError."""
_enable_backconnect(monkeypatch)
scraper = AvitoScraper()
r403 = MagicMock()
r403.status_code = 403
r403.text = ""
mock_session = AsyncMock()
mock_session.get = AsyncMock(return_value=r403)
scraper._cffi = mock_session
reset = AsyncMock()
with (
patch("app.services.scrapers.avito.asyncio.sleep", AsyncMock()),
patch.object(AvitoScraper, "_reset_cffi", reset),
):
with pytest.raises(AvitoBlockedError, match="IP banned"):
await scraper._fetch_serp_html("https://www.avito.ru/x", page=1)
# ровно _AVITO_403_MAX_RETRIES reconnect, потом raise
assert reset.await_count == avito_mod._AVITO_403_MAX_RETRIES
# get: 1 первичный + _AVITO_403_MAX_RETRIES повторов
assert mock_session.get.await_count == avito_mod._AVITO_403_MAX_RETRIES + 1
@pytest.mark.asyncio
async def test_reconnect_fires_even_when_rotate_url_set(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Прод-конфиг run-212 регрессия: scraper_proxy_url + leftover avito_proxy_rotate_url +
max_rotations=0 → reconnect-retry ВСЁ РАВНО срабатывает (gate только на наличие proxy).
changeip-URL относится к ДРУГОМУ (auv/browser) прокси; curl_cffi egress через
scraper_proxy_url, поэтому пересоздание сессии меняет exit-IP независимо от changeip.
С max_rotations=0 changeip-путь — no-op, fallback-reconnect снимает 403.
"""
monkeypatch.setattr(
avito_mod.settings, "avito_proxy_rotate_url", "https://changeip.example/?proxy_key=x"
)
monkeypatch.setattr(avito_mod.settings, "avito_proxy_max_rotations", 0)
monkeypatch.setattr(avito_mod.settings, "scraper_proxy_url_env", "http://u:p@mproxy.site:14619")
scraper = AvitoScraper()
r403 = MagicMock()
r403.status_code = 403
r403.text = ""
ok = MagicMock()
ok.status_code = 200
ok.text = _CARD_HTML
mock_session = AsyncMock()
mock_session.get = AsyncMock(side_effect=[r403, r403, ok])
scraper._cffi = mock_session
reset = AsyncMock()
with (
patch("app.services.scrapers.avito.asyncio.sleep", AsyncMock()),
patch.object(AvitoScraper, "_reset_cffi", reset),
patch.object(AvitoScraper, "_rotate_ip", AsyncMock(return_value=True)) as rot,
):
html = await scraper._fetch_serp_html("https://www.avito.ru/x", page=1)
assert html == _CARD_HTML # 403→403→200 снят reconnect'ом, нет raise
assert reset.await_count == 2 # reconnect сработал несмотря на rotate_url
rot.assert_not_awaited() # max_rotations=0 → changeip-путь не дёргается
assert mock_session.get.await_count == 3
@pytest.mark.asyncio
async def test_changeip_path_tried_first_when_rotations_available(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""changeip precedence: scraper_proxy_url + rotate_url + max_rotations=1 → на 403
changeip-ротация пробуется ПЕРЕД reconnect-fallback.
Гарантирует, что новый gate не сломал приоритет legacy changeip-пути.
"""
monkeypatch.setattr(
avito_mod.settings, "avito_proxy_rotate_url", "https://changeip.example/?proxy_key=x"
)
monkeypatch.setattr(avito_mod.settings, "avito_proxy_max_rotations", 1)
monkeypatch.setattr(avito_mod.settings, "scraper_proxy_url_env", "http://u:p@mproxy.site:14619")
scraper = AvitoScraper()
r403 = MagicMock()
r403.status_code = 403
r403.text = ""
ok = MagicMock()
ok.status_code = 200
ok.text = _CARD_HTML
mock_session = AsyncMock()
mock_session.get = AsyncMock(side_effect=[r403, ok])
scraper._cffi = mock_session
reset = AsyncMock()
rot = AsyncMock(return_value=True)
with (
patch("app.services.scrapers.avito.asyncio.sleep", AsyncMock()),
patch.object(AvitoScraper, "_reset_cffi", reset),
patch.object(AvitoScraper, "_rotate_ip", rot),
):
html = await scraper._fetch_serp_html("https://www.avito.ru/x", page=1)
assert html == _CARD_HTML
rot.assert_awaited_once() # changeip-путь использован первым на 403
reset.assert_not_awaited() # до reconnect-fallback не дошло
assert mock_session.get.await_count == 2

242
tradein-mvp/backend/tests/scrapers/test_avito_429_bucket_skip.py
View file

@ -1,242 +0,0 @@
"""429-hardening: widened short-retry + reconnect-fallback в _fetch_serp_html и
per-bucket skip в fetch_all_secondary.
Run 216 (прод): merged 403-reconnect-retry восстановил два 403, но прогон всё равно
забанился на HTTP 429 на page 36 (deep-pagination dense-бакета). 429 под backconnect =
transient conn-limit (лимит 5 соединений), не IP-ban.
Две защиты:
A. _fetch_serp_html: short-retry того же session расширен до _AVITO_429_MAX_RETRIES=8;
на исчерпании под backconnect — ещё _AVITO_429_RECONNECT_RETRIES попыток через
_reset_cffi (освобождает conn-slot) перед AvitoRateLimitedError.
B. fetch_all_secondary: одиночный заблокированный бакет скипается (continue), sweep
продолжается; только _AVITO_SWEEP_MAX_CONSECUTIVE_BLOCKED ПОДРЯД блоков = hard ban
→ re-raise (сохраняет real-ban detection).
"""
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from app.services.scrapers import avito as avito_mod
from app.services.scrapers.avito import AvitoScraper
from app.services.scrapers.avito_exceptions import AvitoBlockedError, AvitoRateLimitedError
# Валидная карточка SERP (одна) — successful page.
_CARD_HTML = (
"<html><body>"
'<div data-marker="item" data-item-id="123456789">'
'<a data-marker="item-title" href="/ekaterinburg/kvartiry/test-123456789">'
"2-к. квартира, 54,3 м², 5/9 эт."
"</a>"
'<meta itemprop="price" content="4500000"/>'
"</div>"
"</body></html>"
)
def _enable_backconnect(monkeypatch: pytest.MonkeyPatch) -> None:
"""scraper_proxy_url задан (через env-поле), changeip-URL — нет → backconnect."""
monkeypatch.setattr(avito_mod.settings, "scraper_proxy_url_env", "http://u:p@mproxy.site:14619")
monkeypatch.setattr(avito_mod.settings, "avito_proxy_url", None)
monkeypatch.setattr(avito_mod.settings, "avito_proxy_rotate_url", None)
def _resp(status: int, text: str = "") -> MagicMock:
r = MagicMock()
r.status_code = status
r.text = text
return r
# --- A. _fetch_serp_html 429 short-retry + reconnect-fallback -------------------
@pytest.mark.asyncio
async def test_429_short_retries_then_200_within_widened_budget(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""429×6 (< _AVITO_429_MAX_RETRIES=8) → 200: short-retry того же session снимает,
_reset_cffi НЕ дёргается (reconnect-fallback не нужен)."""
_enable_backconnect(monkeypatch)
n_429 = 6
assert n_429 < avito_mod._AVITO_429_MAX_RETRIES
scraper = AvitoScraper()
seq = [_resp(429) for _ in range(n_429)] + [_resp(200, _CARD_HTML)]
mock_session = AsyncMock()
mock_session.get = AsyncMock(side_effect=seq)
scraper._cffi = mock_session
reset = AsyncMock()
with (
patch("app.services.scrapers.avito.asyncio.sleep", AsyncMock()),
patch.object(AvitoScraper, "_reset_cffi", reset),
):
html = await scraper._fetch_serp_html("https://www.avito.ru/x", page=36)
assert html == _CARD_HTML
reset.assert_not_awaited() # short-retry хватило, reconnect не понадобился
assert mock_session.get.await_count == n_429 + 1
@pytest.mark.asyncio
async def test_429_exhausted_then_reconnect_then_200(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""short-retry исчерпан (429×_AVITO_429_MAX_RETRIES) → reconnect-fallback (_reset_cffi)
→ 200 успех. Подтверждает: одна reconnect-фаза освобождает conn-slot."""
_enable_backconnect(monkeypatch)
n_short = avito_mod._AVITO_429_MAX_RETRIES
scraper = AvitoScraper()
# _AVITO_429_MAX_RETRIES коротких retry (все 429), ещё один 429 проваливается в
# reconnect-block, после _reset_cffi следующий ответ 200.
seq = [_resp(429) for _ in range(n_short + 1)] + [_resp(200, _CARD_HTML)]
mock_session = AsyncMock()
mock_session.get = AsyncMock(side_effect=seq)
scraper._cffi = mock_session
reset = AsyncMock()
with (
patch("app.services.scrapers.avito.asyncio.sleep", AsyncMock()),
patch.object(AvitoScraper, "_reset_cffi", reset),
):
html = await scraper._fetch_serp_html("https://www.avito.ru/x", page=36)
assert html == _CARD_HTML
reset.assert_awaited() # reconnect-fallback сработал
assert reset.await_count == 1
@pytest.mark.asyncio
async def test_429_fully_exhausted_raises_rate_limited(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""429 на каждый запрос: short-retry + все reconnect-fallback исчерпаны →
AvitoRateLimitedError. _reset_cffi дёрнут ровно _AVITO_429_RECONNECT_RETRIES раз."""
_enable_backconnect(monkeypatch)
scraper = AvitoScraper()
mock_session = AsyncMock()
mock_session.get = AsyncMock(return_value=_resp(429))
scraper._cffi = mock_session
reset = AsyncMock()
with (
patch("app.services.scrapers.avito.asyncio.sleep", AsyncMock()),
patch.object(AvitoScraper, "_reset_cffi", reset),
):
with pytest.raises(AvitoRateLimitedError, match="rate limited"):
await scraper._fetch_serp_html("https://www.avito.ru/x", page=36)
assert reset.await_count == avito_mod._AVITO_429_RECONNECT_RETRIES
# get: 1 первичный + _AVITO_429_MAX_RETRIES short-retry + _AVITO_429_RECONNECT_RETRIES
assert (
mock_session.get.await_count
== 1 + avito_mod._AVITO_429_MAX_RETRIES + avito_mod._AVITO_429_RECONNECT_RETRIES
)
# --- B. fetch_all_secondary per-bucket skip -----------------------------------
def _single_room() -> list[tuple[str, str]]:
"""Один room-bucket → число брекетов = len(_AVITO_PRICE_SEED_BRACKETS)."""
return [("2-комнатные", "2-komnatnye")]
@pytest.mark.asyncio
async def test_one_blocked_bucket_skipped_sweep_completes(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""_walk_price_range кидает AvitoRateLimitedError на 1-м брекете, успех на остальных →
sweep завершается без raise, заблокированный бакет посчитан."""
n_brackets = len(avito_mod._AVITO_PRICE_SEED_BRACKETS)
assert n_brackets >= 2
scraper = AvitoScraper()
calls = {"n": 0}
async def fake_walk(self: object, **kwargs: object) -> None:
calls["n"] += 1
if calls["n"] == 1:
raise AvitoRateLimitedError("429 deep pagination")
# успех: ничего не добавляем в seen (не нужно для теста контракта)
with patch.object(AvitoScraper, "_walk_price_range", fake_walk):
result = await scraper.fetch_all_secondary(rooms_buckets=_single_room())
assert result == [] # seen пуст, но без raise
assert calls["n"] == n_brackets # все брекеты пройдены (1 скипнут, остальные ок)
@pytest.mark.asyncio
async def test_consecutive_blocked_threshold_reraises(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""_walk_price_range кидает _AVITO_SWEEP_MAX_CONSECUTIVE_BLOCKED раз ПОДРЯД →
последнее исключение пробрасывается (real hard-ban detection сохранена)."""
thr = avito_mod._AVITO_SWEEP_MAX_CONSECUTIVE_BLOCKED
assert len(avito_mod._AVITO_PRICE_SEED_BRACKETS) >= thr
scraper = AvitoScraper()
async def always_block(self: object, **kwargs: object) -> None:
raise AvitoBlockedError("IP banned")
with patch.object(AvitoScraper, "_walk_price_range", always_block):
with pytest.raises(AvitoBlockedError, match="IP banned"):
await scraper.fetch_all_secondary(rooms_buckets=_single_room())
@pytest.mark.asyncio
async def test_success_between_blocks_resets_counter(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Чередование block/success НЕ срабатывает порог: успех между блоками сбрасывает
consecutive_blocked. Паттерн block,success,block,success,... → sweep завершается."""
thr = avito_mod._AVITO_SWEEP_MAX_CONSECUTIVE_BLOCKED
n_brackets = len(avito_mod._AVITO_PRICE_SEED_BRACKETS)
# Нужно достаточно брекетов чтобы суммарно было > thr блоков, но не подряд.
assert n_brackets > thr
scraper = AvitoScraper()
calls = {"n": 0}
async def alternate(self: object, **kwargs: object) -> None:
calls["n"] += 1
# нечётные вызовы блокируются, чётные — успех → никогда не thr подряд
if calls["n"] % 2 == 1:
raise AvitoRateLimitedError("429 transient")
with patch.object(AvitoScraper, "_walk_price_range", alternate):
result = await scraper.fetch_all_secondary(rooms_buckets=_single_room())
assert result == [] # без raise — порог не пробит
assert calls["n"] == n_brackets
@pytest.mark.asyncio
async def test_incremental_path_also_skips_blocked_bucket(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Per-bucket skip применяется и к incremental-пути (_paginate_incremental_bracket)."""
from datetime import date
n_brackets = len(avito_mod._AVITO_PRICE_SEED_BRACKETS)
scraper = AvitoScraper()
calls = {"n": 0}
async def fake_incr(self: object, **kwargs: object) -> None:
calls["n"] += 1
if calls["n"] == 1:
raise AvitoRateLimitedError("429")
with patch.object(AvitoScraper, "_paginate_incremental_bracket", fake_incr):
result = await scraper.fetch_all_secondary(
rooms_buckets=_single_room(), since=date(2026, 1, 1)
)
assert result == []
assert calls["n"] == n_brackets

425
tradein-mvp/backend/tests/scrapers/test_avito_anti_bot.py
View file

@ -1,425 +0,0 @@
"""Test anti-bot hardening: blocked exceptions + early abort."""
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from app.services.scrapers.avito_exceptions import (
AvitoBlockedError,
AvitoContentBlockedError,
AvitoRateLimitedError,
)
# ── fetch_detail tests ──────────────────────────────────────────
@pytest.mark.asyncio
async def test_fetch_detail_raises_blocked_on_403() -> None:
"""fetch_detail должен raise AvitoBlockedError на HTTP 403."""
from app.services.scrapers.avito_detail import fetch_detail
mock_session = AsyncMock()
mock_response = MagicMock()
mock_response.status_code = 403
mock_session.get = AsyncMock(return_value=mock_response)
with pytest.raises(AvitoBlockedError, match="403"):
await fetch_detail("/ekaterinburg/kvartiry/test-123", cffi_session=mock_session)
@pytest.mark.asyncio
async def test_fetch_detail_raises_rate_limited_on_429() -> None:
"""fetch_detail должен raise AvitoRateLimitedError на HTTP 429."""
from app.services.scrapers.avito_detail import fetch_detail
mock_session = AsyncMock()
mock_response = MagicMock()
mock_response.status_code = 429
mock_session.get = AsyncMock(return_value=mock_response)
with pytest.raises(AvitoRateLimitedError, match="429"):
await fetch_detail("/ekaterinburg/kvartiry/test-123", cffi_session=mock_session)
@pytest.mark.asyncio
async def test_fetch_detail_raises_value_error_on_other_non_200() -> None:
"""fetch_detail должен raise ValueError на прочие non-200 (не 403/429)."""
from app.services.scrapers.avito_detail import fetch_detail
mock_session = AsyncMock()
mock_response = MagicMock()
mock_response.status_code = 503
mock_session.get = AsyncMock(return_value=mock_response)
with pytest.raises(ValueError, match="503"):
await fetch_detail("/ekaterinburg/kvartiry/test-123", cffi_session=mock_session)
# ── SERP (avito.py) tests ───────────────────────────────────────
@pytest.mark.asyncio
async def test_fetch_around_raises_content_blocked_on_page1_empty_html() -> None:
"""fetch_around должен raise AvitoContentBlockedError на HTTP 200 + 0 карточек на странице 1."""
from app.services.scrapers.avito import AvitoScraper
scraper = AvitoScraper()
mock_session = AsyncMock()
mock_response = MagicMock()
mock_response.status_code = 200
# Garbage HTML: нет data-marker="item" → _parse_html вернёт []
mock_response.text = "<html><body><p>captcha challenge</p></body></html>"
mock_session.get = AsyncMock(return_value=mock_response)
scraper._cffi = mock_session
with pytest.raises(AvitoContentBlockedError, match="content-block"):
await scraper.fetch_around(56.84, 60.60, 1000, pages=1)
@pytest.mark.asyncio
async def test_fetch_around_content_blocked_is_subtype_of_blocked() -> None:
"""AvitoContentBlockedError должен быть поймать как AvitoBlockedError (наследование)."""
from app.services.scrapers.avito import AvitoScraper
scraper = AvitoScraper()
mock_session = AsyncMock()
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.text = "<html><body></body></html>"
mock_session.get = AsyncMock(return_value=mock_response)
scraper._cffi = mock_session
with pytest.raises(AvitoBlockedError):
await scraper.fetch_around(56.84, 60.60, 1000, pages=1)
@pytest.mark.asyncio
async def test_fetch_around_page_gt1_empty_returns_gracefully() -> None:
"""fetch_around на page>1 с 0 карточек должен тихо завершить пагинацию (не raise)."""
from app.services.scrapers.avito import AvitoScraper
scraper = AvitoScraper()
mock_session = AsyncMock()
# Страница 1: нормальный HTML с одной карточкой
valid_html = (
"<html><body>"
'<div data-marker="item" data-item-id="123456789">'
'<a data-marker="item-title" href="/ekaterinburg/kvartiry/test-123456789">'
"2-к. квартира, 54,3 м², 5/9 эт."
"</a>"
'<meta itemprop="price" content="4500000"/>'
"</div>"
"</body></html>"
)
# Страница 2: пустой HTML → end of pagination
empty_html = "<html><body><p>no listings</p></body></html>"
page1_resp = MagicMock()
page1_resp.status_code = 200
page1_resp.text = valid_html
page2_resp = MagicMock()
page2_resp.status_code = 200
page2_resp.text = empty_html
mock_session.get = AsyncMock(side_effect=[page1_resp, page2_resp])
scraper._cffi = mock_session
with patch("app.services.scrapers.avito.AvitoScraper.sleep_between_requests", AsyncMock()):
result = await scraper.fetch_around(56.84, 60.60, 1000, pages=2)
# Должен вернуть лоты со страницы 1, не raise на странице 2
assert len(result) >= 1
@pytest.mark.asyncio
async def test_fetch_around_raises_blocked_on_403() -> None:
"""AvitoScraper.fetch_around должен raise AvitoBlockedError на HTTP 403."""
from app.services.scrapers.avito import AvitoScraper
scraper = AvitoScraper()
mock_session = AsyncMock()
mock_response = MagicMock()
mock_response.status_code = 403
mock_session.get = AsyncMock(return_value=mock_response)
scraper._cffi = mock_session
with pytest.raises(AvitoBlockedError):
await scraper.fetch_around(56.84, 60.60, 1000, pages=1)
@pytest.mark.asyncio
async def test_fetch_around_raises_rate_limited_on_429() -> None:
"""AvitoScraper.fetch_around должен raise AvitoRateLimitedError на persistent HTTP 429.
429 (backconnect conn-limit) сначала проходит короткие retry; при отсутствии
rotate-url после исчерпания retry сохраняется старый fallback — AvitoRateLimitedError.
"""
from app.services.scrapers.avito import AvitoScraper
scraper = AvitoScraper()
mock_session = AsyncMock()
mock_response = MagicMock()
mock_response.status_code = 429
mock_session.get = AsyncMock(return_value=mock_response)
scraper._cffi = mock_session
with patch("app.services.scrapers.avito.asyncio.sleep", AsyncMock()):
with pytest.raises(AvitoRateLimitedError):
await scraper.fetch_around(56.84, 60.60, 1000, pages=1)
# ── #623 firewall-детект + ротация IP ────────────────────────────
#
# Legacy `run_avito_pipeline` / `run_avito_city_sweep` orchestration-level tests
# (pipeline consecutive-block abort, inter-detail sleep, city-sweep mark_banned)
# удалены вместе с `app.services.scrape_pipeline` (#2397 Part E1) — эквивалентная
# regression-coverage теперь в `test_scraper_kit_pipeline_parity.py` (kit
# `run_avito_city_sweep`: ban/rotation state-machine, detail-block propagation).
_CARD_HTML = (
"<html><body>"
'<div data-marker="item" data-item-id="123456789">'
'<a data-marker="item-title" href="/ekaterinburg/kvartiry/test-123456789">'
"2-к. квартира, 54,3 м², 5/9 эт."
"</a>"
'<meta itemprop="price" content="4500000"/>'
"</div>"
"</body></html>"
)
# Firewall-интерстициал Avito (HTTP 200, не 403) — IP-блок datacenter.
_FIREWALL_HTML = (
"<html><head><title>Доступ ограничен: проблема с IP</title></head>"
'<body><div class="firewall-container">...</div></body></html>'
)
@pytest.mark.asyncio
async def test_fetch_around_raises_blocked_on_firewall_html_200() -> None:
"""HTTP 200 + firewall-страница «Доступ ограничен» → AvitoBlockedError (#623).
Avito банит datacenter-IP отдачей firewall-интерстициала с кодом 200 (не 403) —
раньше это молча проходило как «0 cards / content-block», теперь явный IP-блок.
"""
from app.services.scrapers.avito import AvitoScraper
scraper = AvitoScraper()
mock_session = AsyncMock()
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.text = _FIREWALL_HTML
mock_session.get = AsyncMock(return_value=mock_response)
scraper._cffi = mock_session
with pytest.raises(AvitoBlockedError, match="firewall"):
await scraper.fetch_around(56.84, 60.60, 1000, pages=1)
@pytest.mark.asyncio
async def test_fetch_around_rotates_ip_then_succeeds(monkeypatch: pytest.MonkeyPatch) -> None:
"""При firewall + заданной changeip-ссылке: ротация IP и повтор страницы (#623)."""
from app.services.scrapers import avito as avito_mod
from app.services.scrapers.avito import AvitoScraper
monkeypatch.setattr(
avito_mod.settings, "avito_proxy_rotate_url", "https://changeip.example/?proxy_key=x"
)
monkeypatch.setattr(avito_mod.settings, "avito_proxy_max_rotations", 1)
scraper = AvitoScraper()
fw = MagicMock()
fw.status_code = 200
fw.text = _FIREWALL_HTML
ok = MagicMock()
ok.status_code = 200
ok.text = _CARD_HTML
mock_session = AsyncMock()
mock_session.get = AsyncMock(side_effect=[fw, ok])
scraper._cffi = mock_session
with (
patch.object(AvitoScraper, "_rotate_ip", AsyncMock(return_value=True)) as rot,
patch.object(AvitoScraper, "sleep_between_requests", AsyncMock()),
):
result = await scraper.fetch_around(56.84, 60.60, 1000, pages=1)
rot.assert_awaited_once()
assert len(result) == 1
assert mock_session.get.await_count == 2 # firewall → rotate → retry OK
@pytest.mark.asyncio
async def test_fetch_around_blocked_when_rotation_exhausted(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Если ротация не сняла бан (max_rotations исчерпан) → AvitoBlockedError (#623)."""
from app.services.scrapers import avito as avito_mod
from app.services.scrapers.avito import AvitoScraper
monkeypatch.setattr(
avito_mod.settings, "avito_proxy_rotate_url", "https://changeip.example/?proxy_key=x"
)
monkeypatch.setattr(avito_mod.settings, "avito_proxy_max_rotations", 1)
scraper = AvitoScraper()
fw = MagicMock()
fw.status_code = 200
fw.text = _FIREWALL_HTML
mock_session = AsyncMock()
mock_session.get = AsyncMock(return_value=fw) # всегда firewall
scraper._cffi = mock_session
with patch.object(AvitoScraper, "_rotate_ip", AsyncMock(return_value=True)) as rot:
with pytest.raises(AvitoBlockedError, match="firewall"):
await scraper.fetch_around(56.84, 60.60, 1000, pages=1)
assert rot.await_count == 1 # одна попытка ротации, потом сдаёмся
# ── #623 sweep-pipeline proxy (SERP+detail через mobile proxy) ───
def test_avito_proxies_helper() -> None:
"""Kit _avito_proxies(config): None без proxy, dict {http,https} при заданном URL.
Регрессия: sweep создаёт shared-сессию, через неё идут и SERP, и detail —
без прокси detail летит с datacenter-IP → 429 → run banned (#623 followup).
"""
from types import SimpleNamespace
from scraper_kit.orchestration.pipeline import _avito_proxies
assert _avito_proxies(SimpleNamespace(scraper_proxy_url=None)) is None
url = "http://u:p@mproxy.site:16928"
assert _avito_proxies(SimpleNamespace(scraper_proxy_url=url)) == {"http": url, "https": url}
# ── curl_cffi 429 short-retry (backconnect conn-limit transient) ──────────────
@pytest.mark.asyncio
async def test_fetch_around_429_short_retry_then_succeeds() -> None:
"""429 дважды → 200: короткий retry снимает transient conn-limit БЕЗ ротации IP.
backconnect-прокси (mproxy.site, лимит 5 соединений) отдаёт 429 при параллельной
пагинации — это transient, проходит на коротком retry. _rotate_ip НЕ должен дёргаться.
"""
from app.services.scrapers.avito import AvitoScraper
scraper = AvitoScraper()
r429 = MagicMock()
r429.status_code = 429
ok = MagicMock()
ok.status_code = 200
ok.text = _CARD_HTML
mock_session = AsyncMock()
mock_session.get = AsyncMock(side_effect=[r429, r429, ok])
scraper._cffi = mock_session
with (
patch("app.services.scrapers.avito.asyncio.sleep", AsyncMock()) as slp,
patch.object(AvitoScraper, "_rotate_ip", AsyncMock(return_value=True)) as rot,
patch.object(AvitoScraper, "sleep_between_requests", AsyncMock()),
):
result = await scraper.fetch_around(56.84, 60.60, 1000, pages=1)
assert len(result) == 1
assert mock_session.get.await_count == 3 # 429, 429, 200
rot.assert_not_awaited() # короткий retry, не ротация
# короткий backoff (_AVITO_429_BACKOFF_SEC ~1.5s), не 9с-settle ротации
assert slp.await_count >= 2
for c in slp.await_args_list:
assert c.args[0] < 5.0
@pytest.mark.asyncio
async def test_fetch_around_429_exhausted_no_rotate_url_raises() -> None:
"""429 постоянно + НЕТ rotate-url → после исчерпания коротких retry: старый fallback."""
from app.services.scrapers import avito as avito_mod
from app.services.scrapers.avito import AvitoScraper
scraper = AvitoScraper()
r429 = MagicMock()
r429.status_code = 429
mock_session = AsyncMock()
mock_session.get = AsyncMock(return_value=r429)
scraper._cffi = mock_session
with (
patch("app.services.scrapers.avito.asyncio.sleep", AsyncMock()) as slp,
patch.object(AvitoScraper, "_rotate_ip", AsyncMock(return_value=True)) as rot,
):
with pytest.raises(AvitoRateLimitedError):
await scraper.fetch_around(56.84, 60.60, 1000, pages=1)
rot.assert_not_awaited() # нет rotate-url → ротация не дёргается
# ровно _AVITO_429_MAX_RETRIES коротких retry, потом fallback
assert slp.await_count == avito_mod._AVITO_429_MAX_RETRIES
@pytest.mark.asyncio
async def test_fetch_around_429_exhausted_with_rotate_url_rotates(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""429 постоянно + ЕСТЬ rotate-url → короткие retry, затем ротация (старый путь)."""
from app.services.scrapers import avito as avito_mod
from app.services.scrapers.avito import AvitoScraper
monkeypatch.setattr(
avito_mod.settings, "avito_proxy_rotate_url", "https://changeip.example/?proxy_key=x"
)
monkeypatch.setattr(avito_mod.settings, "avito_proxy_max_rotations", 1)
scraper = AvitoScraper()
r429 = MagicMock()
r429.status_code = 429
ok = MagicMock()
ok.status_code = 200
ok.text = _CARD_HTML
mock_session = AsyncMock()
# 429 на всех коротких retry (1 первичный + _AVITO_429_MAX_RETRIES), после
# исчерпания short-retry — ротация → 200. backconnect-reconnect здесь НЕ
# срабатывает (scraper_proxy_url не задан), поэтому сразу rotate-путь.
n_429 = 1 + avito_mod._AVITO_429_MAX_RETRIES
mock_session.get = AsyncMock(side_effect=[r429] * n_429 + [ok])
scraper._cffi = mock_session
with (
patch("app.services.scrapers.avito.asyncio.sleep", AsyncMock()),
patch.object(AvitoScraper, "_rotate_ip", AsyncMock(return_value=True)) as rot,
patch.object(AvitoScraper, "sleep_between_requests", AsyncMock()),
):
result = await scraper.fetch_around(56.84, 60.60, 1000, pages=1)
rot.assert_awaited_once() # после исчерпания коротких retry — ротация
assert len(result) == 1
@pytest.mark.asyncio
async def test_fetch_around_403_no_429_retry_path(monkeypatch: pytest.MonkeyPatch) -> None:
"""403 (реальный IP-ban) НЕ проходит через 429-retry — поведение без изменений."""
from app.services.scrapers import avito as avito_mod
from app.services.scrapers.avito import AvitoScraper
monkeypatch.setattr(avito_mod.settings, "avito_proxy_rotate_url", None)
scraper = AvitoScraper()
r403 = MagicMock()
r403.status_code = 403
mock_session = AsyncMock()
mock_session.get = AsyncMock(return_value=r403)
scraper._cffi = mock_session
with (
patch("app.services.scrapers.avito.asyncio.sleep", AsyncMock()) as slp,
patch.object(AvitoScraper, "_rotate_ip", AsyncMock(return_value=True)),
):
with pytest.raises(AvitoBlockedError):
await scraper.fetch_around(56.84, 60.60, 1000, pages=1)
# 403 без rotate-url → немедленный raise, ни одного 429-backoff sleep
slp.assert_not_awaited()
assert mock_session.get.await_count == 1

191
tradein-mvp/backend/tests/scrapers/test_avito_detail_403_reconnect.py
View file

@ -1,191 +0,0 @@
"""curl_cffi 403/firewall reconnect-retry на detail-странице под backconnect-прокси.
fetch_detail ронял весь detail-backfill на первом 403/429 через AvitoBlockedError /
AvitoRateLimitedError (run 191 обогатил 48 → zombie). Зеркало SERP-фикса (#1765): под
backconnect rotating-прокси (mproxy.site) каждое НОВОЕ соединение даёт свежий exit-IP,
поэтому 403/firewall ретраим пересозданием эфемерной curl_cffi-сессии до
_AVITO_DETAIL_403_MAX_RETRIES раз. 429 — короткий retry той же сессии без reconnect.
Ownership-правило: caller-provided cffi_session НИКОГДА не закрывается/не пересоздаётся —
retries идут через свою эфемерную retry_session (закрываем в finally).
Включаем backconnect через scraper_proxy_url_env (scraper_proxy_url — read-only property,
его не monkeypatch'ишь). Зеркало fixture-стиля test_avito_403_backconnect_reconnect.py.
"""
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from app.core.config import settings
from app.services.scrapers import avito_detail as detail_mod
from app.services.scrapers.avito_detail import fetch_detail
from app.services.scrapers.avito_exceptions import AvitoBlockedError
_SENTINEL = object()
_FIREWALL_HTML = (
"<html><head><title>Доступ ограничен: проблема с IP</title></head>"
'<body><div class="firewall-container">...</div></body></html>'
)
def _resp(status_code: int, text: str = "") -> MagicMock:
r = MagicMock()
r.status_code = status_code
r.text = text
return r
def _enable_backconnect(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(settings, "scraper_proxy_url_env", "http://u:p@mproxy.site:14619")
def _disable_backconnect(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(settings, "scraper_proxy_url_env", None)
@pytest.fixture(autouse=True)
def _patch_common() -> object:
"""asyncio.sleep → no-op; parse_detail_html → sentinel (нет нужды в реальном HTML)."""
with (
patch("app.services.scrapers.avito_detail.asyncio.sleep", AsyncMock()),
patch(
"app.services.scrapers.avito_detail.parse_detail_html", return_value=_SENTINEL
) as parse,
):
yield parse
@pytest.mark.asyncio
async def test_403_403_200_caller_session_reconnect_succeeds(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""caller-provided session: 403 → 403 → 200 → reconnect-retry возвращает enrichment.
_build_detail_session дёрнут для retries; caller-session НЕ закрыта.
"""
_enable_backconnect(monkeypatch)
caller = AsyncMock()
caller.get = AsyncMock(return_value=_resp(403))
# Каждый retry → свежая эфемерная сессия. Первая отдаёт 403 (для 2-го reconnect),
# вторая — 200.
retry1 = AsyncMock()
retry1.get = AsyncMock(return_value=_resp(403))
retry2 = AsyncMock()
retry2.get = AsyncMock(return_value=_resp(200, "<html/>"))
build = MagicMock(side_effect=[retry1, retry2])
with patch.object(detail_mod, "_build_detail_session", build):
result = await fetch_detail("/ekaterinburg/kvartiry/test-123", cffi_session=caller)
assert result is _SENTINEL
assert build.call_count == 2 # 403, 403 → два reconnect перед 200
caller.close.assert_not_called() # caller-provided session НЕ закрывается
assert caller.get.await_count == 1 # первый attempt
retry1.close.assert_awaited() # промежуточная retry-session закрыта
retry2.close.assert_awaited() # последняя retry-session закрыта в finally
@pytest.mark.asyncio
async def test_firewall_200_reconnect_succeeds(monkeypatch: pytest.MonkeyPatch) -> None:
"""firewall(HTTP 200) → 200 → новый HTTP-200 firewall-detect триггерит reconnect."""
_enable_backconnect(monkeypatch)
caller = AsyncMock()
caller.get = AsyncMock(return_value=_resp(200, _FIREWALL_HTML))
retry1 = AsyncMock()
retry1.get = AsyncMock(return_value=_resp(200, "<html/>"))
build = MagicMock(side_effect=[retry1])
with patch.object(detail_mod, "_build_detail_session", build):
result = await fetch_detail("/x", cffi_session=caller)
assert result is _SENTINEL
assert build.call_count == 1
caller.close.assert_not_called()
@pytest.mark.asyncio
async def test_403_exhausted_raises_blocked(monkeypatch: pytest.MonkeyPatch) -> None:
"""403 _AVITO_DETAIL_403_MAX_RETRIES+1 раз → AvitoBlockedError."""
_enable_backconnect(monkeypatch)
caller = AsyncMock()
caller.get = AsyncMock(return_value=_resp(403))
def _make_retry() -> AsyncMock:
s = AsyncMock()
s.get = AsyncMock(return_value=_resp(403))
return s
build = MagicMock(side_effect=lambda: _make_retry())
with patch.object(detail_mod, "_build_detail_session", build):
with pytest.raises(AvitoBlockedError, match="HTTP 403"):
await fetch_detail("/x", cffi_session=caller)
assert build.call_count == detail_mod._AVITO_DETAIL_403_MAX_RETRIES
caller.close.assert_not_called()
@pytest.mark.asyncio
async def test_429_short_retry_then_200_same_session(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""429 → 429 → 200: short-retry той же сессии, без reconnect."""
_enable_backconnect(monkeypatch)
caller = AsyncMock()
caller.get = AsyncMock(side_effect=[_resp(429), _resp(429), _resp(200, "<html/>")])
build = MagicMock()
with patch.object(detail_mod, "_build_detail_session", build):
result = await fetch_detail("/x", cffi_session=caller)
assert result is _SENTINEL
build.assert_not_called() # 429 не дёргает reconnect
assert caller.get.await_count == 3 # та же сессия переиспользована
caller.close.assert_not_called()
@pytest.mark.asyncio
async def test_no_proxy_single_403_raises_immediately(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""scraper_proxy_url unset → нет reconnect-пути → первый 403 сразу raise."""
_disable_backconnect(monkeypatch)
caller = AsyncMock()
caller.get = AsyncMock(return_value=_resp(403))
build = MagicMock()
with patch.object(detail_mod, "_build_detail_session", build):
with pytest.raises(AvitoBlockedError, match="HTTP 403"):
await fetch_detail("/x", cffi_session=caller)
build.assert_not_called()
assert caller.get.await_count == 1
@pytest.mark.asyncio
async def test_own_session_403_200_closes_both(monkeypatch: pytest.MonkeyPatch) -> None:
"""own-session path (cffi_session=None): 403 → 200; own session + retry session
обе закрыты в finally."""
_enable_backconnect(monkeypatch)
own = AsyncMock()
own.get = AsyncMock(return_value=_resp(403))
retry1 = AsyncMock()
retry1.get = AsyncMock(return_value=_resp(200, "<html/>"))
# Первый _build_detail_session → own (own_session-ветка), второй → retry1 (reconnect).
build = MagicMock(side_effect=[own, retry1])
with patch.object(detail_mod, "_build_detail_session", build):
result = await fetch_detail("/x", cffi_session=None)
assert result is _SENTINEL
own.close.assert_awaited() # own session закрыта (own_session=True)
retry1.close.assert_awaited() # retry session закрыта в finally
assert build.call_count == 2

153
tradein-mvp/backend/tests/scrapers/test_avito_detail_429_reconnect.py
View file

@ -1,153 +0,0 @@
"""curl_cffi 429 reconnect-retry на detail-странице под backconnect-прокси.
fetch_detail имел только short-retry той же сессии на 429 (_AVITO_DETAIL_429_MAX_RETRIES),
затем raise AvitoRateLimitedError. Но keep-alive detail-fetch переиспользует ОДИН
backconnect exit-IP на последовательные запросы → avito лочит этот IP, и short-retry той
же сессии не помогает (run 217: 5 подряд 429, abort на 19 enriched). Зеркало SERP-фикса
(#1769): при исчерпании short-retry под backconnect делаем ещё
_AVITO_DETAIL_429_RECONNECT_RETRIES попыток через эфемерную сессию (свежий exit-IP),
переиспользуя ту же retry_session-машинерию что и 403-путь.
Ownership-правило: caller-provided cffi_session НИКОГДА не закрывается/не пересоздаётся —
reconnect идёт через свою эфемерную retry_session (закрываем в finally).
Включаем backconnect через scraper_proxy_url_env (scraper_proxy_url — read-only property).
Зеркало fixture-стиля test_avito_detail_403_reconnect.py.
"""
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from app.core.config import settings
from app.services.scrapers import avito_detail as detail_mod
from app.services.scrapers.avito_detail import fetch_detail
from app.services.scrapers.avito_exceptions import AvitoRateLimitedError
_SENTINEL = object()
def _resp(status_code: int, text: str = "") -> MagicMock:
r = MagicMock()
r.status_code = status_code
r.text = text
return r
def _enable_backconnect(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(settings, "scraper_proxy_url_env", "http://u:p@mproxy.site:14619")
def _disable_backconnect(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(settings, "scraper_proxy_url_env", None)
@pytest.fixture(autouse=True)
def _patch_common() -> object:
"""asyncio.sleep → no-op; parse_detail_html → sentinel (нет нужды в реальном HTML)."""
with (
patch("app.services.scrapers.avito_detail.asyncio.sleep", AsyncMock()),
patch(
"app.services.scrapers.avito_detail.parse_detail_html", return_value=_SENTINEL
) as parse,
):
yield parse
@pytest.mark.asyncio
async def test_429_short_retry_exhausted_then_reconnect_200(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""429×(short-retry budget) → 429 → reconnect (свежая сессия) → 200.
После _AVITO_DETAIL_429_MAX_RETRIES short-retry той же сессии ещё один 429 исчерпывает
short-budget → reconnect через _build_detail_session → 200. Caller-session НЕ закрыта.
"""
_enable_backconnect(monkeypatch)
# short-budget short-retries (каждый получает 429 и continue) + ещё один 429, который
# исчерпывает budget и триггерит reconnect. Итого budget+1 ответов 429 на caller-сессии.
n429 = detail_mod._AVITO_DETAIL_429_MAX_RETRIES + 1
caller = AsyncMock()
caller.get = AsyncMock(side_effect=[_resp(429)] * n429)
retry1 = AsyncMock()
retry1.get = AsyncMock(return_value=_resp(200, "<html/>"))
build = MagicMock(side_effect=[retry1])
with patch.object(detail_mod, "_build_detail_session", build):
result = await fetch_detail("/ekaterinburg/kvartiry/test-123", cffi_session=caller)
assert result is _SENTINEL
assert build.call_count == 1 # один reconnect перед 200
assert caller.get.await_count == n429 # short-budget + исчерпавший 429
caller.close.assert_not_called() # caller-provided session НЕ закрывается
retry1.close.assert_awaited() # эфемерная retry-session закрыта в finally
@pytest.mark.asyncio
async def test_429_fully_exhausted_raises_rate_limited(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""429 на short-retry budget + всех reconnect-попытках → AvitoRateLimitedError."""
_enable_backconnect(monkeypatch)
# short-budget+1 (исчерпание short) на caller; затем каждая reconnect-сессия тоже 429.
n429 = detail_mod._AVITO_DETAIL_429_MAX_RETRIES + 1
caller = AsyncMock()
caller.get = AsyncMock(side_effect=[_resp(429)] * n429)
def _make_retry() -> AsyncMock:
s = AsyncMock()
s.get = AsyncMock(return_value=_resp(429))
return s
build = MagicMock(side_effect=lambda: _make_retry())
with patch.object(detail_mod, "_build_detail_session", build):
with pytest.raises(AvitoRateLimitedError, match="HTTP 429"):
await fetch_detail("/x", cffi_session=caller)
# Все reconnect-попытки исчерпаны.
assert build.call_count == detail_mod._AVITO_DETAIL_429_RECONNECT_RETRIES
caller.close.assert_not_called()
@pytest.mark.asyncio
async def test_429_short_retry_then_200_no_reconnect(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""429 → 200 в рамках short-retry budget: та же сессия, БЕЗ reconnect (regression guard)."""
_enable_backconnect(monkeypatch)
caller = AsyncMock()
caller.get = AsyncMock(side_effect=[_resp(429), _resp(200, "<html/>")])
build = MagicMock()
with patch.object(detail_mod, "_build_detail_session", build):
result = await fetch_detail("/x", cffi_session=caller)
assert result is _SENTINEL
build.assert_not_called() # short-retry успешен → reconnect не дёрнут
assert caller.get.await_count == 2 # та же сессия переиспользована
caller.close.assert_not_called()
@pytest.mark.asyncio
async def test_no_proxy_429_short_exhausted_raises_immediately(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""scraper_proxy_url unset → нет reconnect-пути → исчерпание short-retry сразу raise."""
_disable_backconnect(monkeypatch)
n429 = detail_mod._AVITO_DETAIL_429_MAX_RETRIES + 1
caller = AsyncMock()
caller.get = AsyncMock(side_effect=[_resp(429)] * n429)
build = MagicMock()
with patch.object(detail_mod, "_build_detail_session", build):
with pytest.raises(AvitoRateLimitedError, match="HTTP 429"):
await fetch_detail("/x", cffi_session=caller)
build.assert_not_called() # no proxy → нет reconnect
assert caller.get.await_count == n429

109
tradein-mvp/backend/tests/scrapers/test_avito_detail_browser_only.py
View file

@ -1,109 +0,0 @@
"""Browser-only режим fetch_detail (#1814).
Когда передан browser_fetcher, detail-fetch идёт СТРОГО через camoufox-сервис.
Никакой curl_cffi-сессии (AsyncSession) и никакого curl-fallback — avito-антибот
банит curl-фингерпринт под нагрузкой (429 CONNECT tunnel failed). Любой
firewall/ошибка браузера → AvitoBlockedError; caller (backfill) метит failed и идёт
дальше, НЕ создавая curl-сессию.
Тесты проверяют именно отсутствие инстанцирования AsyncSession и поднятие
AvitoBlockedError на browser-firewall/ошибке.
"""
from unittest.mock import AsyncMock, MagicMock, patch
import httpx
import pytest
from app.services.scrapers.avito_detail import fetch_detail
from app.services.scrapers.avito_exceptions import AvitoBlockedError
_SENTINEL = object()
_FIREWALL_HTML = (
"<html><head><title>Доступ ограничен: проблема с IP</title></head>"
'<body><div class="firewall-container">...</div></body></html>'
)
_OK_HTML = "<html><body>ok</body></html>"
# AsyncSession (curl_cffi) в модуле avito_detail — НЕ должна инстанцироваться в browser-mode.
_ASYNC_SESSION = "app.services.scrapers.avito_detail.AsyncSession"
_BUILD_SESSION = "app.services.scrapers.avito_detail._build_detail_session"
_PARSE = "app.services.scrapers.avito_detail.parse_detail_html"
def _browser(*, fetch_side: object) -> MagicMock:
bf = MagicMock()
if isinstance(fetch_side, BaseException):
bf.fetch = AsyncMock(side_effect=fetch_side)
else:
bf.fetch = AsyncMock(return_value=fetch_side)
return bf
@pytest.mark.asyncio
async def test_browser_mode_success_no_curl_session() -> None:
"""browser_fetcher OK → parse, AsyncSession НЕ создаётся."""
bf = _browser(fetch_side=_OK_HTML)
with (
patch(_ASYNC_SESSION) as async_session,
patch(_BUILD_SESSION) as build_session,
patch(_PARSE, return_value=_SENTINEL) as parse,
):
result = await fetch_detail("/items/123", browser_fetcher=bf)
assert result is _SENTINEL
bf.fetch.assert_awaited_once()
parse.assert_called_once()
async_session.assert_not_called()
build_session.assert_not_called()
@pytest.mark.asyncio
async def test_browser_firewall_raises_blocked_no_curl_session() -> None:
"""browser вернул firewall-HTML (status 200) → AvitoBlockedError, без curl."""
bf = _browser(fetch_side=_FIREWALL_HTML)
with (
patch(_ASYNC_SESSION) as async_session,
patch(_BUILD_SESSION) as build_session,
patch(_PARSE, return_value=_SENTINEL),
):
with pytest.raises(AvitoBlockedError):
await fetch_detail("/items/123", browser_fetcher=bf)
async_session.assert_not_called()
build_session.assert_not_called()
@pytest.mark.asyncio
async def test_browser_http_error_raises_blocked_no_curl_session() -> None:
"""browser_fetcher.fetch кидает HTTPError (502/firewall сервиса) → AvitoBlockedError,
AsyncSession НЕ инстанцируется (НЕ откатываемся в curl-фингерпринт → 429-бан)."""
bf = _browser(fetch_side=httpx.HTTPError("browser service 502"))
with (
patch(_ASYNC_SESSION) as async_session,
patch(_BUILD_SESSION) as build_session,
patch(_PARSE, return_value=_SENTINEL),
):
with pytest.raises(AvitoBlockedError):
await fetch_detail("/items/123", browser_fetcher=bf)
async_session.assert_not_called()
build_session.assert_not_called()
@pytest.mark.asyncio
async def test_browser_blocked_error_propagates_unchanged() -> None:
"""Если browser_fetcher сам кинул AvitoBlockedError — пробрасываем как есть, без curl."""
original = AvitoBlockedError("already blocked")
bf = _browser(fetch_side=original)
with (
patch(_ASYNC_SESSION) as async_session,
patch(_BUILD_SESSION) as build_session,
patch(_PARSE, return_value=_SENTINEL),
):
with pytest.raises(AvitoBlockedError) as exc_info:
await fetch_detail("/items/123", browser_fetcher=bf)
assert exc_info.value is original
async_session.assert_not_called()
build_session.assert_not_called()

129
tradein-mvp/backend/tests/scrapers/test_avito_detail_kit_parity.py
View file

@ -1,129 +0,0 @@
"""Smoke-proof для parity-harness'а (tests/support/parity.py) — issue #2304.
Доказывает, что harness реально ловит эквивалентность legacy ↔ scraper_kit на
РЕАЛЬНОЙ паре функций, а не на синтетике: `parse_detail_html` — parsing-ядро,
которое `app/api/v1/admin.py::scrape_avito_detail` (debug-роут ручного запуска
парсера) вызывает косвенно через `fetch_detail()` (fetch_detail = HTTP GET +
`parse_detail_html(html, url)`; сам parse — чистая функция, без сети/БД, что и
позволяет сравнить её offline на фиксированной HTML-фикстуре).
`app/services/scrapers/avito_detail.py::parse_detail_html` (legacy) и
`scraper_kit/providers/avito/detail.py::parse_detail_html` (kit) на момент
написания теста ИДЕНТИЧНЫ по телу функции (см. audit
`Scraper_Kit_Legacy_Dependency_Audit_0703` в vault) — различаются только
import-пути модулей, из которых берутся зависимости (`app.services.scrapers.*`
vs `scraper_kit.*`) и Strangler-инжекция `ScraperConfig` в соседних функциях
(`fetch_detail`, `_build_detail_session`), которую сам `parse_detail_html` не
использует. Это НЕ миграция admin.py (issue #2305+) — только доказательство
инструмента.
HTML-фикстура — тот же minimal-marker набор, что и в
`tests/test_avito_detail_parse.py::test_minimal_parse` (независимая копия
литерала, чтобы этот тест не зависел от другого test-модуля).
"""
# ruff: noqa: E501 # HTML-фикстура содержит длинные строки разметки
from __future__ import annotations
import dataclasses
import os
import pytest
# app.services.scrapers.avito_detail импортирует app.core.config.settings=Settings(),
# которому нужен DATABASE_URL. Офлайн-парсинг БД не трогает — фиктивный DSN достаточен
# (mirror tests/test_scraper_kit_avito_golden_parity.py — тест должен жить и без
# CI-уровневого DATABASE_URL из ci-tradein.yml/deploy-tradein.yml).
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
from scraper_kit.providers.avito.detail import (
parse_detail_html as kit_parse_detail_html,
)
from app.services.scrapers.avito_detail import (
parse_detail_html as legacy_parse_detail_html,
)
from tests.support.parity import ParityMismatchError, assert_parity
_DETAIL_HTML = """
<html><body>
<div data-marker="item-view/item-id">№ 7986882804 · 18 мая в 13:40 · 1899 просмотров (+ 12 сегодня)</div>
<span itemprop="price" content="11990000">11 990 000 ₽</span>
<div data-marker="item-map-wrapper" data-map-lat="56.797713" data-map-lon="60.609135" data-location-id="654070"></div>
<div data-marker="item-view/item-params">
<ul>
<li>Количество комнат: 3</li>
<li>Общая площадь: 75 м²</li>
<li>Площадь кухни: 15 м²</li>
<li>Этаж: 20 из 26</li>
<li>Балкон или лоджия: лоджия</li>
<li>Ремонт: евро</li>
<li>Способ продажи: свободная</li>
<li>Условия продажи: возможна ипотека</li>
</ul>
<ul>
<li>Тип дома: монолитный</li>
<li>Этажей в доме: 26</li>
</ul>
</div>
<a data-marker="nd-jk-details-button" href="/catalog/houses/ekaterinburg/ul_postovskogo_17a/3171365">Дом</a>
<div data-marker="domoteka-entry-block">
<p data-marker="TeaserData.item">1 собственник</p>
<p data-marker="TeaserData.item">Последняя смена собственника 5 мая 2020</p>
<p data-marker="TeaserData.item">Не найдены ограничения и обременения</p>
<p data-marker="TeaserData.item">Совпадают площадь, адрес и этаж</p>
</div>
<div data-marker="item-view/item-description">Отличная квартира в новом ЖК. Чкаловская 11-15 мин пешком.</div>
</body></html>
"""
_SOURCE_URL = "https://www.avito.ru/ekaterinburg/kvartiry/test_7986882804"
def test_legacy_and_kit_parse_detail_html_are_class_distinct() -> None:
"""Sanity-check ДО harness'а: подтверждает, зачем harness вообще нужен.
legacy DetailEnrichment и kit DetailEnrichment — РАЗНЫЕ классы (разные
модули) → обычный `==` даёт False, хотя данные идентичны. Именно это
harness должен обходить через структурное сравнение (dataclasses.fields).
"""
legacy_result = legacy_parse_detail_html(_DETAIL_HTML, _SOURCE_URL)
kit_result = kit_parse_detail_html(_DETAIL_HTML, _SOURCE_URL)
assert type(legacy_result) is not type(kit_result)
assert legacy_result != kit_result # dataclass __eq__ проверяет class identity первым
assert dataclasses.asdict(legacy_result) == dataclasses.asdict(kit_result)
def test_parity_harness_confirms_avito_detail_parse_equivalence() -> None:
"""Реальное proof-of-concept использование harness'а (issues #2305-#2310 template)."""
assert_parity(
legacy_fn=legacy_parse_detail_html,
kit_fn=kit_parse_detail_html,
fixtures=[(_DETAIL_HTML, _SOURCE_URL)],
)
def test_parity_harness_catches_real_field_divergence() -> None:
"""Доказывает, что harness реально ЛОВИТ расхождение на этой же реальной
30+-полевой dataclass-форме — а не только сообщает "нет разницы" на двух
genuinely-identical инстансах, как предыдущий тест.
Мутируем один field (``price_rub``) в kit-результате через
``dataclasses.replace`` и убеждаемся, что harness падает с
``ParityMismatchError``, называющим именно этот field.
"""
legacy_result = legacy_parse_detail_html(_DETAIL_HTML, _SOURCE_URL)
kit_result = kit_parse_detail_html(_DETAIL_HTML, _SOURCE_URL)
assert kit_result.price_rub is not None # sanity: поле реально распарсилось из фикстуры
mutated_kit_result = dataclasses.replace(kit_result, price_rub=kit_result.price_rub + 1)
with pytest.raises(ParityMismatchError) as exc_info:
assert_parity(
legacy_fn=lambda: legacy_result,
kit_fn=lambda: mutated_kit_result,
fixtures=[()],
)
assert "price_rub" in str(exc_info.value)

82
tradein-mvp/backend/tests/scrapers/test_avito_detail_not_found.py
View file

@ -1,82 +0,0 @@
"""404 / removed-listing detection на detail-странице (#2034).
Lat-null pending-очередь backfill в основном содержит МЁРТВЫЕ листинги (404 — потому
они и стали координатными дырами). В browser-mode 404-страница Avito («Ошибка 404.
Страница не найдена», тело тоже содержит generic «Объявления на сайте Авито») не имеет
item-view → раньше ловилась как soft-block → AvitoBlockedError → backfill крутил
consecutive-block breaker и абортил run ДО live-листингов (run 458: attempted=5
enriched=0 blocked=5 → abort).
_is_detail_not_found детектит её ОТДЕЛЬНО → fetch_detail кидает AvitoListingGoneError
(НЕ AvitoBlockedError), backfill метит листинг is_active=FALSE и продолжает без
ротации/ретрая.
Зеркало fixture-стиля test_avito_detail_soft_block.py / test_avito_detail_browser_only.py.
"""
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from app.services.scrapers import avito_detail as detail_mod
from app.services.scrapers.avito_detail import fetch_detail
from app.services.scrapers.avito_exceptions import AvitoBlockedError, AvitoListingGoneError
_SENTINEL = object()
# 404-страница Avito: специфичный <title>; тело тоже содержит generic-витринный текст,
# который сам по себе тригерит soft-block — поэтому not-found проверяется ПЕРВЫМ.
_NOT_FOUND_HTML = (
'<!DOCTYPE html><html lang="ru"><head>'
"<title>Ошибка 404. Страница не найдена</title>"
"</head><body><div>Объявления на сайте Авито</div></body></html>"
)
# Soft-block deflect: generic-витрина БЕЗ 404-текста — это блок, НЕ gone.
_SOFT_BLOCK_HTML = (
'<!DOCTYPE html><html lang="ru"><head>'
"<title>Авито — Объявления на сайте Авито</title>"
"</head><body><div>...</div></body></html>"
)
_ITEM_VIEW_HTML = "<html><body><span data-marker='item-view/item-id'>№ 123</span></body></html>"
_ASYNC_SESSION = "app.services.scrapers.avito_detail.AsyncSession"
_BUILD_SESSION = "app.services.scrapers.avito_detail._build_detail_session"
_PARSE = "app.services.scrapers.avito_detail.parse_detail_html"
def test_is_detail_not_found_unit() -> None:
"""Юнит _is_detail_not_found: 404 → True; soft-block deflect / item-view / пусто → False."""
assert detail_mod._is_detail_not_found(_NOT_FOUND_HTML) is True
# soft-block deflect (generic-витрина без 404-текста) НЕ должен ловиться как gone
assert detail_mod._is_detail_not_found(_SOFT_BLOCK_HTML) is False
# реальное объявление с item-view → False
assert detail_mod._is_detail_not_found(_ITEM_VIEW_HTML) is False
assert detail_mod._is_detail_not_found("") is False
def _browser_returning(html: str) -> MagicMock:
bf = MagicMock()
bf.fetch = AsyncMock(return_value=html)
return bf
@pytest.mark.asyncio
async def test_browser_mode_404_raises_listing_gone_not_blocked() -> None:
"""browser_fetcher вернул 404-HTML → AvitoListingGoneError (НЕ AvitoBlockedError),
без curl-сессии."""
bf = _browser_returning(_NOT_FOUND_HTML)
with (
patch(_ASYNC_SESSION) as async_session,
patch(_BUILD_SESSION) as build_session,
patch(_PARSE, return_value=_SENTINEL) as parse,
):
with pytest.raises(AvitoListingGoneError) as exc_info:
await fetch_detail("/items/123", browser_fetcher=bf)
# Тип строго AvitoListingGoneError и НЕ подкласс AvitoBlockedError —
# иначе backfill-ловушка `except (AvitoBlockedError, ...)` перехватила бы его.
assert not isinstance(exc_info.value, AvitoBlockedError)
bf.fetch.assert_awaited_once()
parse.assert_not_called()
async_session.assert_not_called()
build_session.assert_not_called()

101
tradein-mvp/backend/tests/scrapers/test_avito_detail_soft_block.py
View file

@ -1,101 +0,0 @@
"""Soft-block (HTTP 200 generic-витрина) detection на detail-странице (#1967 / #1950).
Под нагрузкой Avito иногда отдаёт HTTP 200 с ОБЩЕЙ витриной
(<title>Авито — Объявления на сайте Авито</title>) вместо запрошенного объявления —
мягкий анти-бот deflect. У такой страницы нет ни firewall-маркеров, ни item-view →
parse_detail_html ронял ValueError → generic failed, IP НЕ ротировался, enriched
каскадно падал к 0. _is_detail_soft_block детектит её как блок → существующая
403/firewall reconnect-машинерия пересоздаёт эфемерную сессию (свежий exit-IP).
Зеркало fixture-стиля test_avito_detail_403_reconnect.py.
"""
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from app.core.config import settings
from app.services.scrapers import avito_detail as detail_mod
from app.services.scrapers.avito_detail import fetch_detail
from app.services.scrapers.avito_exceptions import AvitoBlockedError
_SENTINEL = object()
_SOFT_BLOCK_HTML = (
'<!DOCTYPE html><html lang="ru"><head>'
"<title>Авито — Объявления на сайте Авито</title>"
"</head><body><div>...</div></body></html>"
)
def _resp(status_code: int, text: str = "") -> MagicMock:
r = MagicMock()
r.status_code = status_code
r.text = text
return r
def _enable_backconnect(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(settings, "scraper_proxy_url_env", "http://u:p@mproxy.site:14619")
@pytest.fixture(autouse=True)
def _patch_common() -> object:
"""asyncio.sleep → no-op; parse_detail_html → sentinel (нет нужды в реальном HTML)."""
with (
patch("app.services.scrapers.avito_detail.asyncio.sleep", AsyncMock()),
patch(
"app.services.scrapers.avito_detail.parse_detail_html", return_value=_SENTINEL
) as parse,
):
yield parse
@pytest.mark.asyncio
async def test_soft_block_200_reconnect_succeeds(monkeypatch: pytest.MonkeyPatch) -> None:
"""soft-block(HTTP 200) → 200: generic-витрина детектится как блок → reconnect."""
_enable_backconnect(monkeypatch)
caller = AsyncMock()
caller.get = AsyncMock(return_value=_resp(200, _SOFT_BLOCK_HTML))
retry1 = AsyncMock()
retry1.get = AsyncMock(return_value=_resp(200, "<html/>"))
build = MagicMock(side_effect=[retry1])
with patch.object(detail_mod, "_build_detail_session", build):
result = await fetch_detail("/x", cffi_session=caller)
assert result is _SENTINEL
assert build.call_count == 1
caller.close.assert_not_called()
@pytest.mark.asyncio
async def test_soft_block_exhausted_raises_blocked(monkeypatch: pytest.MonkeyPatch) -> None:
"""soft-block на каждом attempt _AVITO_DETAIL_403_MAX_RETRIES+1 раз → AvitoBlockedError."""
_enable_backconnect(monkeypatch)
caller = AsyncMock()
caller.get = AsyncMock(return_value=_resp(200, _SOFT_BLOCK_HTML))
def _make_retry() -> AsyncMock:
s = AsyncMock()
s.get = AsyncMock(return_value=_resp(200, _SOFT_BLOCK_HTML))
return s
build = MagicMock(side_effect=lambda: _make_retry())
with patch.object(detail_mod, "_build_detail_session", build):
with pytest.raises(AvitoBlockedError):
await fetch_detail("/x", cffi_session=caller)
assert build.call_count == detail_mod._AVITO_DETAIL_403_MAX_RETRIES
caller.close.assert_not_called()
def test_is_detail_soft_block_unit() -> None:
"""Юнит _is_detail_soft_block: generic-витрина → True; item-view / пусто → False."""
assert detail_mod._is_detail_soft_block(_SOFT_BLOCK_HTML) is True
item_view = "<html><body><span data-marker='item-view/item-id'>№ 123</span></body></html>"
assert detail_mod._is_detail_soft_block(item_view) is False
assert detail_mod._is_detail_soft_block("<html/>") is False
assert detail_mod._is_detail_soft_block("") is False

140
tradein-mvp/backend/tests/scrapers/test_avito_detail_warmup.py
View file

@ -1,140 +0,0 @@
"""Живой прогрев сессии + батч-на-сессию (#1551).
warm_up_session: GET Yandex SERP (best-effort) -> GET Avito EKB search-страница с
Referer=yandex сеет антибот-куки. Прогретая сессия потом держит батч detail-GET'ов
без 403 (доказано пробами на проде: 22/22 карточек подряд, 0 блоков).
fetch_detail(referer=..., reconnect_on_block=False): шлёт Referer detail-GET'а и при
403/firewall поднимает AvitoBlockedError СРАЗУ — без холодных per-card reconnect'ов
(их делает outer-loop через rebuild+rewarm прогретой сессии в avito_detail_backfill).
Стиль фикстур/моков зеркалит test_avito_detail_403_reconnect.py.
"""
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from app.core.config import settings
from app.services.scrapers import avito_detail as detail_mod
from app.services.scrapers.avito_detail import (
fetch_detail,
research_in_session,
warm_up_session,
)
from app.services.scrapers.avito_exceptions import AvitoBlockedError
def _resp(status_code: int, text: str = "") -> MagicMock:
r = MagicMock()
r.status_code = status_code
r.text = text
return r
def _enable_backconnect(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(settings, "scraper_proxy_url_env", "http://u:p@mproxy.site:14619")
# ── warm_up_session ───────────────────────────────────────────────────────────
@pytest.mark.asyncio
async def test_warm_up_session_search_with_yandex_referer_true_on_200() -> None:
"""warm_up_session шлёт GET на _AVITO_WARM_SEARCH_URL с Referer=yandex и
возвращает True на 200/non-firewall."""
session = AsyncMock()
session.get = AsyncMock(return_value=_resp(200, "<html>ok</html>"))
with (
patch("app.services.scrapers.avito_detail.asyncio.sleep", AsyncMock()),
patch("app.services.scrapers.avito_detail._is_firewall_page", return_value=False),
):
ok = await warm_up_session(session)
assert ok is True
# search-GET (на avito-search URL) ушёл ровно один раз и с Referer=yandex.
search_calls = [
c
for c in session.get.await_args_list
if c.args and c.args[0] == detail_mod._AVITO_WARM_SEARCH_URL
]
assert len(search_calls) == 1
assert search_calls[0].kwargs["headers"]["Referer"] == detail_mod._AVITO_WARM_YANDEX_REFERER
# Yandex SERP — органический referer-источник — тоже фетчился (best-effort, до search).
assert any(
c.args and c.args[0] == detail_mod._AVITO_WARM_YANDEX_URL
for c in session.get.await_args_list
)
@pytest.mark.asyncio
async def test_warm_up_session_false_on_403() -> None:
"""search-страница вернула 403 → warm_up_session возвращает False."""
session = AsyncMock()
session.get = AsyncMock(return_value=_resp(403))
with (
patch("app.services.scrapers.avito_detail.asyncio.sleep", AsyncMock()),
patch("app.services.scrapers.avito_detail._is_firewall_page", return_value=False),
):
ok = await warm_up_session(session)
assert ok is False
# ── fetch_detail referer + reconnect_on_block=False ───────────────────────────
@pytest.mark.asyncio
async def test_fetch_detail_referer_forwarded_and_block_raises_immediately(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""fetch_detail(referer="X", reconnect_on_block=False): Referer="X" уходит в
detail-GET, а 403 поднимает AvitoBlockedError СРАЗУ — без холодного reconnect
(даже когда backconnect-proxy включён → гейт = reconnect_on_block, не наличие proxy).
"""
_enable_backconnect(monkeypatch)
caller = AsyncMock()
caller.get = AsyncMock(return_value=_resp(403))
build = MagicMock() # _build_detail_session НЕ должна дёргаться (нет reconnect'а)
with patch.object(detail_mod, "_build_detail_session", build):
with pytest.raises(AvitoBlockedError, match="HTTP 403"):
await fetch_detail(
"/ekaterinburg/kvartiry/test-123",
cffi_session=caller,
referer="X",
reconnect_on_block=False,
)
build.assert_not_called() # никаких холодных reconnect'ов
assert caller.get.await_count == 1 # один detail-GET → сразу raise
assert caller.get.await_args.kwargs["headers"] == {"Referer": "X"}
# ── research_in_session (in-session перепоиск, #1551 follow-up) ────────────────
@pytest.mark.asyncio
async def test_research_in_session_re_get_search_with_yandex_referer_true_on_200() -> None:
"""research_in_session re-GET'ит avito-search на ТОЙ ЖЕ сессии с Referer=yandex
и возвращает True на 200/non-firewall (свежесть куки без смены exit-IP)."""
session = AsyncMock()
session.get = AsyncMock(return_value=_resp(200, "<html>ok</html>"))
with patch("app.services.scrapers.avito_detail._is_firewall_page", return_value=False):
ok = await research_in_session(session)
assert ok is True
assert session.get.await_count == 1 # ровно один re-GET, без yandex-фазы (in-session)
assert session.get.await_args.args[0] == detail_mod._AVITO_WARM_SEARCH_URL
sent_referer = session.get.await_args.kwargs["headers"]["Referer"]
assert sent_referer == detail_mod._AVITO_WARM_YANDEX_REFERER
@pytest.mark.asyncio
async def test_research_in_session_false_on_403() -> None:
"""search вернул 403 → research_in_session возвращает False (best-effort, не fatal)."""
session = AsyncMock()
session.get = AsyncMock(return_value=_resp(403))
with patch("app.services.scrapers.avito_detail._is_firewall_page", return_value=False):
ok = await research_in_session(session)
assert ok is False

921
tradein-mvp/backend/tests/scrapers/test_avito_exhaustive.py
View file

@ -1,921 +0,0 @@
"""Тесты exhaustive-загрузки Avito: fetch_all_secondary + _walk_price_range.
Зеркало tests/scrapers/test_cian_exhaustive.py. Мокаем сетевые вызовы
(_fetch_rooms_page_html, _extract_total_count, _parse_html) для детерминированного
контроля бисекции без реального HTTP.
Дополнительно: _build_rooms_url с pmin/pmax (backward-compat) и _extract_total_count
на реальном span-фрагменте `data-marker="page-title/count"`.
"""
from __future__ import annotations
import os
from unittest.mock import MagicMock, patch
# Settings requires DATABASE_URL at import time — set dummy DSN before any app import.
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
import httpx
import pytest
from app.services.scrapers.avito import (
_AVITO_MAX_PRICE,
_AVITO_MIN_BRACKET,
_AVITO_OFFERS_PER_PAGE,
_AVITO_PRICE_SEED_BRACKETS,
AvitoScraper,
)
from app.services.scrapers.avito_exceptions import AvitoBlockedError
from app.services.scrapers.base import ScrapedLot
def _make_lot(source_id: str, price: int = 3_000_000, segment: str = "vtorichka") -> ScrapedLot:
"""Минимальный ScrapedLot для теста."""
return ScrapedLot(
source="avito",
source_url=f"https://www.avito.ru/ekaterinburg/kvartiry/{source_id}",
source_id=source_id,
address="Екатеринбург",
price_rub=price,
listing_segment=segment,
)
@pytest.fixture
def scraper() -> AvitoScraper:
"""AvitoScraper без реального __aenter__ (без HTTP/IP-rotation)."""
s = AvitoScraper.__new__(AvitoScraper)
s.name = "avito"
s.base_url = "https://www.avito.ru"
s.request_delay_sec = 0.0
s._cffi = MagicMock() # не None
s._browser = None
s.parse_failures = 0
return s
# ── _build_rooms_url с pmin/pmax ──────────────────────────────────────────────
def test_build_rooms_url_both_prices(scraper: AvitoScraper) -> None:
"""min_price + max_price → pmin/pmax оба в query."""
url = scraper._build_rooms_url(
"1-komnatnye-X", page=2, min_price=3_000_000, max_price=4_000_000
)
assert "pmin=3000000" in url
assert "pmax=4000000" in url
assert "p=2" in url
assert "s=104" in url
assert "/kvartiry/prodam/1-komnatnye-X?" in url
def test_build_rooms_url_min_only(scraper: AvitoScraper) -> None:
"""Только min_price → pmin в query, pmax отсутствует."""
url = scraper._build_rooms_url("1-komnatnye-X", min_price=3_000_000)
assert "pmin=3000000" in url
assert "pmax" not in url
def test_build_rooms_url_no_prices_backward_compat(scraper: AvitoScraper) -> None:
"""Без цен URL не меняется (backward-compat для fetch_by_rooms)."""
url = scraper._build_rooms_url("1-komnatnye-X", page=1)
assert "pmin" not in url
assert "pmax" not in url
assert url.endswith("/kvartiry/prodam/1-komnatnye-X?s=104&p=1")
# ── _extract_total_count ──────────────────────────────────────────────────────
# Реальный span-фрагмент из живого Avito SERP (NBSP-разделитель разрядов).
_REAL_COUNT_SPAN = '<span class="some-css" data-marker="page-title/count">1\xa0178</span>'
def test_extract_total_count_real_span(scraper: AvitoScraper) -> None:
"""Реальный span с NBSP `1\\xa0178` → 1178."""
html = f"<div><h1>Квартиры</h1>{_REAL_COUNT_SPAN}</div>"
assert scraper._extract_total_count(html) == 1178
def test_extract_total_count_nbsp_entity(scraper: AvitoScraper) -> None:
"""Вариант с `&nbsp;`-entity → корректно очищается."""
html = '<span data-marker="page-title/count">12&nbsp;345</span>'
assert scraper._extract_total_count(html) == 12345
def test_extract_total_count_zero(scraper: AvitoScraper) -> None:
"""0 результатов → 0 (не None)."""
html = '<span data-marker="page-title/count">0</span>'
assert scraper._extract_total_count(html) == 0
def test_extract_total_count_missing_returns_none(scraper: AvitoScraper) -> None:
"""Captcha/firewall (нет span, нет no-results-маркера) → None."""
html = "<html><head><title>Доступ ограничен</title></head></html>"
assert scraper._extract_total_count(html) is None
def test_extract_total_count_no_results_marker_returns_zero(scraper: AvitoScraper) -> None:
"""Валидная пустая выборка: нет `page-title/count`, но есть «ничего не найдено» → 0.
Даже если на странице есть fallback-«похожие» объявления (item-id'ы) — это
не подходящие под фильтр листинги, total всё равно 0. Это предотвращает
дробление пустого брекета + ротацию IP (прод-инцидент run 209).
"""
html = (
'<div data-marker="item" data-item-id="1234567"></div>'
'<div data-marker="item" data-item-id="7654321"></div>'
"<p>По вашему запросу ничего не найдено</p>"
)
assert scraper._extract_total_count(html) == 0
def test_extract_total_count_no_results_marker_case_insensitive(scraper: AvitoScraper) -> None:
"""Маркер регистронезависим: «Ничего не найдено» с заглавной → тоже 0."""
html = "<div><h2>Ничего не найдено</h2><p>Попробуйте изменить фильтры</p></div>"
assert scraper._extract_total_count(html) == 0
def test_extract_total_count_count_wins_over_no_results(scraper: AvitoScraper) -> None:
"""Непустая выборка с count имеет приоритет, даже если где-то ниже есть маркер."""
html = (
f"<div><h1>Квартиры</h1>{_REAL_COUNT_SPAN}</div>"
"<footer>Если ничего не найдено, измените запрос</footer>"
)
assert scraper._extract_total_count(html) == 1178
# ── Бисекция: split / paginate / dedup ────────────────────────────────────────
@pytest.mark.asyncio
async def test_fetch_all_secondary_splits_on_cap(scraper: AvitoScraper) -> None:
"""Внутри seed-брекета total > cap → бакет делится пополам; суб-бакеты пагинируются."""
# Первый закрытый seed-брекет (0, 4_000_000) → walk_hi=3_999_999.
br_lo, br_hi_raw = _AVITO_PRICE_SEED_BRACKETS[0]
assert br_hi_raw is not None
br_hi = br_hi_raw - 1 # эксклюзивный верх
mid = (br_lo + br_hi) // 2
call_log: list[tuple] = []
async def fake_fetch(
room_slug: str, page: int, min_price: int | None, max_price: int | None
) -> str:
call_log.append((page, min_price, max_price))
return f"<html>page={page} min={min_price} max={max_price}</html>"
def fake_total(html: str) -> int | None:
import re
min_m = re.search(r"min=(\w+)", html)
max_m = re.search(r"max=(\w+)", html)
min_p = int(min_m.group(1)) if min_m and min_m.group(1) != "None" else 0
max_p = int(max_m.group(1)) if max_m and max_m.group(1) != "None" else None
# Внутри первого брекета [0, 3_999_999]: весь брекет > cap → split;
# половинки < cap → пагинируются. Прочие seed-брекеты — пусто (0).
if min_p == 0 and max_p == br_hi:
return 3000 # весь первый брекет — split
if min_p == 0 and max_p == mid:
return 800 # левая половина — пагинировать
if min_p == mid + 1 and max_p == br_hi:
return 700 # правая половина — пагинировать
return 0 # остальные seed-брекеты пусты
def fake_parse(html: str, source_url_base: str) -> list[ScrapedLot]:
import re
page_m = re.search(r"page=(\d+)", html)
page = int(page_m.group(1)) if page_m else 1
min_m = re.search(r"min=(\w+)", html)
max_m = re.search(r"max=(\w+)", html)
min_p = min_m.group(1) if min_m else "0"
max_p = max_m.group(1) if max_m else "MAX"
if page == 1:
return [
_make_lot(f"lot_{min_p}_{max_p}_1"),
_make_lot(f"lot_{min_p}_{max_p}_2"),
]
return []
with (
patch.object(scraper, "_fetch_rooms_page_html", side_effect=fake_fetch),
patch.object(scraper, "_extract_total_count", side_effect=fake_total),
patch.object(scraper, "_parse_html", side_effect=fake_parse),
patch.object(scraper, "_rotate_ip", return_value=False),
):
lots = await scraper.fetch_all_secondary(
rooms_buckets=[("1-комн.", "1-komnatnye-X")],
price_cap_per_bucket=1400,
)
assert len(lots) > 0
# Должны быть пробы суб-бакетов (после split весь брекет не пагинируется).
probed_max = [mx for (p, _mn, mx) in call_log if p == 1]
assert any(mx == mid for mx in probed_max), f"Ожидался probe левой половины, got {probed_max}"
@pytest.mark.asyncio
async def test_fetch_all_secondary_paginates_when_under_cap(scraper: AvitoScraper) -> None:
"""total <= cap → пагинирует бакет полностью."""
pages_fetched: list[int] = []
async def fake_fetch(
room_slug: str, page: int, min_price: int | None, max_price: int | None
) -> str:
pages_fetched.append(page)
return f"<html>page={page}</html>"
def fake_total(html: str) -> int | None:
return 100 # ceil(100/50) = 2 страницы
def fake_parse(html: str, source_url_base: str) -> list[ScrapedLot]:
import re
page_m = re.search(r"page=(\d+)", html)
page = int(page_m.group(1)) if page_m else 1
if page <= 2:
return [_make_lot(f"lot_p{page}_{i}") for i in range(_AVITO_OFFERS_PER_PAGE)]
return []
with (
patch.object(scraper, "_fetch_rooms_page_html", side_effect=fake_fetch),
patch.object(scraper, "_extract_total_count", side_effect=fake_total),
patch.object(scraper, "_parse_html", side_effect=fake_parse),
patch.object(scraper, "_rotate_ip", return_value=False),
):
lots = await scraper.fetch_all_secondary(
rooms_buckets=[("2-комн.", "2-komnatnye-X")],
price_cap_per_bucket=1400,
)
assert len(lots) == 100, f"Ожидалось 100 уникальных лотов, получено {len(lots)}"
assert 2 in pages_fetched, "Должна быть запрошена страница 2"
@pytest.mark.asyncio
async def test_fetch_all_secondary_min_bracket_guard(scraper: AvitoScraper) -> None:
"""hi - lo < MIN_BRACKET → пагинируем как есть (нет бесконечной рекурсии)."""
call_count = 0
async def fake_fetch(
room_slug: str, page: int, min_price: int | None, max_price: int | None
) -> str:
nonlocal call_count
call_count += 1
if call_count > 200:
raise RuntimeError("Бесконечная рекурсия — MIN_BRACKET guard не сработал")
return f"<html>page={page}</html>"
def fake_total(html: str) -> int | None:
return 5000 # всегда > cap → без guard делилось бы бесконечно
def fake_parse(html: str, source_url_base: str) -> list[ScrapedLot]:
return []
with (
patch.object(scraper, "_fetch_rooms_page_html", side_effect=fake_fetch),
patch.object(scraper, "_extract_total_count", side_effect=fake_total),
patch.object(scraper, "_parse_html", side_effect=fake_parse),
patch.object(scraper, "_rotate_ip", return_value=False),
):
await scraper._walk_price_range(
room_slug="1-komnatnye-X",
room_label="room_1_komn",
lo=1_000_000,
hi=1_000_000 + _AVITO_MIN_BRACKET - 1,
seen={},
price_cap_per_bucket=1400,
max_pages_per_bucket=100,
)
# Guard сработал: рекурсии нет, только один проход пагинации (probe + до cap страниц).
assert call_count <= 101, f"Слишком много вызовов ({call_count}) — guard не сработал"
@pytest.mark.asyncio
async def test_fetch_all_secondary_concurrent_pages_deduped(scraper: AvitoScraper) -> None:
"""Параллельная пагинация не ломает дедуп по source_id."""
async def fake_fetch(
room_slug: str, page: int, min_price: int | None, max_price: int | None
) -> str:
return f"<html>page={page}</html>"
def fake_total(html: str) -> int | None:
return 150 # ceil(150/50) = 3 страницы
def fake_parse(html: str, source_url_base: str) -> list[ScrapedLot]:
import re
page_m = re.search(r"page=(\d+)", html)
page = int(page_m.group(1)) if page_m else 1
if page == 3:
return [_make_lot(f"lot_p1_{i}") for i in range(10)] # дубликаты стр 1
return [_make_lot(f"lot_p{page}_{i}") for i in range(10)]
with (
patch.object(scraper, "_fetch_rooms_page_html", side_effect=fake_fetch),
patch.object(scraper, "_extract_total_count", side_effect=fake_total),
patch.object(scraper, "_parse_html", side_effect=fake_parse),
patch.object(scraper, "_rotate_ip", return_value=False),
):
lots = await scraper.fetch_all_secondary(
rooms_buckets=[("1-комн.", "1-komnatnye-X")],
price_cap_per_bucket=1400,
concurrency=5,
)
# Страницы 1+2 = 20 уникальных, страница 3 дублирует → 20
assert len(lots) == 20, f"Ожидалось 20 уникальных лотов (дедуп), получено {len(lots)}"
@pytest.mark.asyncio
async def test_on_bucket_receives_key_and_lots(scraper: AvitoScraper) -> None:
"""on_bucket получает (bucket_key, lots) на каждый seed-брекет.
bucket_key закрытого брекета = "room_label:lo:hi" (hi = br_hi-1, эксклюзивный);
открытого (последнего) = "room_label:lo:open". Одна on_bucket-нотификация
на каждый непустой seed-брекет (здесь все непустые → len(brackets) вызовов).
"""
received: list[tuple[str, int]] = []
def fake_on_bucket(bucket_key: str, lots: list[ScrapedLot]) -> None:
received.append((bucket_key, len(lots)))
# Уникальные source_id на каждый брекет, чтобы on_bucket получал непустые
# списки во ВСЕХ брекетах (дедуп в seen не должен опустошать поздние брекеты).
async def fake_fetch(
room_slug: str, page: int, min_price: int | None, max_price: int | None
) -> str:
return f"<html>page={page} min={min_price} max={max_price}</html>"
def fake_total(html: str) -> int | None:
return 5 # 1 страница
def fake_parse(html: str, source_url_base: str) -> list[ScrapedLot]:
import re
min_m = re.search(r"min=(\w+)", html)
max_m = re.search(r"max=(\w+)", html)
min_p = min_m.group(1) if min_m else "0"
max_p = max_m.group(1) if max_m else "open"
page_m = re.search(r"page=(\d+)", html)
page = int(page_m.group(1)) if page_m else 1
return [_make_lot(f"lot_{min_p}_{max_p}_p{page}_{i}") for i in range(5)]
with (
patch.object(scraper, "_fetch_rooms_page_html", side_effect=fake_fetch),
patch.object(scraper, "_extract_total_count", side_effect=fake_total),
patch.object(scraper, "_parse_html", side_effect=fake_parse),
patch.object(scraper, "_rotate_ip", return_value=False),
):
await scraper.fetch_all_secondary(
rooms_buckets=[("2-комн.", "2-komnatnye-X")],
price_cap_per_bucket=1400,
on_bucket=fake_on_bucket,
)
# Один on_bucket на каждый seed-брекет (все непустые).
assert len(received) == len(
_AVITO_PRICE_SEED_BRACKETS
), f"Ожидалось {len(_AVITO_PRICE_SEED_BRACKETS)} вызовов on_bucket, получено {len(received)}"
keys = [k for (k, _c) in received]
# Закрытые брекеты: hi = br_hi - 1. Открытый (последний): ':open'.
first_lo, first_hi = _AVITO_PRICE_SEED_BRACKETS[0]
assert first_hi is not None
assert keys[0] == f"room_2_komn:{first_lo}:{first_hi - 1}", f"Неверный ключ: {keys[0]!r}"
last_lo, last_hi = _AVITO_PRICE_SEED_BRACKETS[-1]
assert last_hi is None
assert keys[-1] == f"room_2_komn:{last_lo}:open", f"Открытый ключ неверен: {keys[-1]!r}"
assert all(count == 5 for (_k, count) in received)
@pytest.mark.asyncio
async def test_skip_buckets_skips_pagination(scraper: AvitoScraper) -> None:
"""skip_buckets: для skip-бакета probe выполняется, пагинация и on_bucket — нет."""
fetch_calls: list[tuple[int, int | None, int | None]] = []
on_bucket_calls: list[str] = []
async def fake_fetch(
room_slug: str, page: int, min_price: int | None, max_price: int | None
) -> str:
fetch_calls.append((page, min_price, max_price))
return f"<html>page={page} min={min_price} max={max_price}</html>"
def fake_total(html: str) -> int | None:
return 5
def fake_parse(html: str, source_url_base: str) -> list[ScrapedLot]:
import re
min_m = re.search(r"min=(\w+)", html)
max_m = re.search(r"max=(\w+)", html)
return [_make_lot(f"lot_{min_m.group(1) if min_m else 0}_{max_m.group(1) if max_m else 0}")]
def fake_on_bucket(bucket_key: str, lots: list[ScrapedLot]) -> None:
on_bucket_calls.append(bucket_key)
# Skip ВСЕ брекеты → ни одной on_bucket-нотификации, но каждый брекет
# всё равно делает probe (page=1).
first_lo, first_hi = _AVITO_PRICE_SEED_BRACKETS[0]
assert first_hi is not None
first_key = f"room_1_komn:{first_lo}:{first_hi - 1}"
skip_all: set[str] = set()
for br_lo, br_hi in _AVITO_PRICE_SEED_BRACKETS:
hi_repr = "open" if br_hi is None else str(br_hi - 1)
skip_all.add(f"room_1_komn:{br_lo}:{hi_repr}")
with (
patch.object(scraper, "_fetch_rooms_page_html", side_effect=fake_fetch),
patch.object(scraper, "_extract_total_count", side_effect=fake_total),
patch.object(scraper, "_parse_html", side_effect=fake_parse),
patch.object(scraper, "_rotate_ip", return_value=False),
):
lots = await scraper.fetch_all_secondary(
rooms_buckets=[("1-комн.", "1-komnatnye-X")],
price_cap_per_bucket=1400,
on_bucket=fake_on_bucket,
skip_buckets=skip_all,
)
# Ровно один probe (page=1) на каждый seed-брекет, без доп. страниц.
assert first_key in skip_all
assert all(
p == 1 for (p, _mn, _mx) in fetch_calls
), f"Ожидались только probe-вызовы: {fetch_calls}"
assert len(fetch_calls) == len(
_AVITO_PRICE_SEED_BRACKETS
), f"Ожидался 1 probe на брекет ({len(_AVITO_PRICE_SEED_BRACKETS)}), got {len(fetch_calls)}"
assert on_bucket_calls == [], "on_bucket не должен вызываться для skip-бакетов"
assert lots == []
@pytest.mark.asyncio
async def test_secondary_only_drops_novostroyki(scraper: AvitoScraper) -> None:
"""secondary_only=True: новостройки (listing_segment=='novostroyki') отброшены."""
mixed = [
_make_lot("nb_1", segment="novostroyki"),
_make_lot("vt_1", segment="vtorichka"),
_make_lot("vt_2", segment="vtorichka"),
]
async def fake_fetch(
room_slug: str, page: int, min_price: int | None, max_price: int | None
) -> str:
return f"<html>page={page}</html>"
def fake_total(html: str) -> int | None:
return 3
def fake_parse(html: str, source_url_base: str) -> list[ScrapedLot]:
import re
page_m = re.search(r"page=(\d+)", html)
page = int(page_m.group(1)) if page_m else 1
return mixed if page == 1 else []
with (
patch.object(scraper, "_fetch_rooms_page_html", side_effect=fake_fetch),
patch.object(scraper, "_extract_total_count", side_effect=fake_total),
patch.object(scraper, "_parse_html", side_effect=fake_parse),
patch.object(scraper, "_rotate_ip", return_value=False),
):
lots = await scraper.fetch_all_secondary(
rooms_buckets=[("1-комн.", "1-komnatnye-X")],
price_cap_per_bucket=1400,
secondary_only=True,
)
ids = {lot.source_id for lot in lots}
assert "nb_1" not in ids
assert ids == {"vt_1", "vt_2"}
@pytest.mark.asyncio
async def test_on_bucket_cancel_stops_run(scraper: AvitoScraper) -> None:
"""on_bucket кидает RuntimeError('cancelled') → прогон прерывается."""
def cancel_on_bucket(bucket_key: str, lots: list[ScrapedLot]) -> None:
raise RuntimeError("cancelled")
async def fake_fetch(
room_slug: str, page: int, min_price: int | None, max_price: int | None
) -> str:
return f"<html>page={page}</html>"
def fake_total(html: str) -> int | None:
return 5
def fake_parse(html: str, source_url_base: str) -> list[ScrapedLot]:
return [_make_lot("lot_1")]
with (
patch.object(scraper, "_fetch_rooms_page_html", side_effect=fake_fetch),
patch.object(scraper, "_extract_total_count", side_effect=fake_total),
patch.object(scraper, "_parse_html", side_effect=fake_parse),
patch.object(scraper, "_rotate_ip", return_value=False),
):
with pytest.raises(RuntimeError, match="cancelled"):
await scraper.fetch_all_secondary(
rooms_buckets=[("1-комн.", "1-komnatnye-X"), ("2-комн.", "2-komnatnye-X")],
price_cap_per_bucket=1400,
on_bucket=cancel_on_bucket,
)
# ── Probe-устойчивость к browser-ошибкам (HTTP 500 / timeout) ─────────────────
def _http_500_error() -> httpx.HTTPStatusError:
"""Сборка httpx.HTTPStatusError(500) как из browser_fetcher.raise_for_status()."""
request = httpx.Request("POST", "http://tradein-browser:3000/fetch")
response = httpx.Response(500, request=request)
return httpx.HTTPStatusError("Server error '500'", request=request, response=response)
@pytest.mark.asyncio
async def test_probe_browser_500_on_wide_bucket_splits_not_crash(
scraper: AvitoScraper,
) -> None:
"""Probe бросает HTTP 500 на ШИРОКОМ bucket → НЕ крах, происходит split.
Воспроизводит прод-инцидент run 199: студии pmax=200M таймаутят на goto →
browser HTTP 500. Раньше это роняло весь full-load с 0 uniq. Теперь широкий
bucket принудительно делится пополам; под-бакеты пагинируются нормально.
"""
# Первый закрытый seed-брекет (0, 4_000_000) → walk_hi=3_999_999.
_br_lo, br_hi_raw = _AVITO_PRICE_SEED_BRACKETS[0]
assert br_hi_raw is not None
br_hi = br_hi_raw - 1
probe_calls: list[tuple[int | None, int | None]] = []
async def fake_fetch(
room_slug: str, page: int, min_price: int | None, max_price: int | None
) -> str:
# Probe верхней границы ШИРОКОГО первого брекета падает (имитация HTTP 500
# от tradein-browser на тяжёлой выдаче) → должен произойти split.
if page == 1 and (min_price in (None, 0)) and max_price == br_hi:
probe_calls.append((min_price, max_price))
raise _http_500_error()
return f"<html>page={page} min={min_price} max={max_price}</html>"
def fake_total(html: str) -> int | None:
return 50
def fake_parse(html: str, source_url_base: str) -> list[ScrapedLot]:
import re
page_m = re.search(r"page=(\d+)", html)
page = int(page_m.group(1)) if page_m else 1
min_m = re.search(r"min=(\w+)", html)
max_m = re.search(r"max=(\w+)", html)
min_p = min_m.group(1) if min_m else "0"
max_p = max_m.group(1) if max_m else "MAX"
if page == 1:
return [_make_lot(f"lot_{min_p}_{max_p}_{i}") for i in range(3)]
return []
with (
patch.object(scraper, "_fetch_rooms_page_html", side_effect=fake_fetch),
patch.object(scraper, "_extract_total_count", side_effect=fake_total),
patch.object(scraper, "_parse_html", side_effect=fake_parse),
patch.object(scraper, "_rotate_ip", return_value=False),
):
lots = await scraper.fetch_all_secondary(
rooms_buckets=[("студии", "studii-X")],
price_cap_per_bucket=1400,
)
assert probe_calls, "Probe первого (широкого) брекета должен был вызваться и упасть"
assert len(lots) > 0, f"Ожидались лоты из под-бакетов после split, got {len(lots)}"
@pytest.mark.asyncio
async def test_probe_timeout_on_wide_bucket_splits_not_crash(
scraper: AvitoScraper,
) -> None:
"""httpx.TimeoutException на широком bucket обрабатывается как HTTP 500 → split."""
_br_lo, br_hi_raw = _AVITO_PRICE_SEED_BRACKETS[0]
assert br_hi_raw is not None
br_hi = br_hi_raw - 1
probe_failed = False
async def fake_fetch(
room_slug: str, page: int, min_price: int | None, max_price: int | None
) -> str:
nonlocal probe_failed
if page == 1 and (min_price in (None, 0)) and max_price == br_hi:
probe_failed = True
raise httpx.TimeoutException("Page.goto timeout")
return f"<html>page={page} min={min_price} max={max_price}</html>"
def fake_total(html: str) -> int | None:
return 40
def fake_parse(html: str, source_url_base: str) -> list[ScrapedLot]:
import re
page_m = re.search(r"page=(\d+)", html)
page = int(page_m.group(1)) if page_m else 1
min_m = re.search(r"min=(\w+)", html)
max_m = re.search(r"max=(\w+)", html)
min_p = min_m.group(1) if min_m else "0"
max_p = max_m.group(1) if max_m else "MAX"
return [_make_lot(f"lot_{min_p}_{max_p}_{i}") for i in range(2)] if page == 1 else []
with (
patch.object(scraper, "_fetch_rooms_page_html", side_effect=fake_fetch),
patch.object(scraper, "_extract_total_count", side_effect=fake_total),
patch.object(scraper, "_parse_html", side_effect=fake_parse),
patch.object(scraper, "_rotate_ip", return_value=False),
):
lots = await scraper.fetch_all_secondary(
rooms_buckets=[("студии", "studii-X")],
price_cap_per_bucket=1400,
)
assert probe_failed, "Probe должен был таймаутнуть"
assert len(lots) > 0, "После split под-бакеты должны собрать лоты"
@pytest.mark.asyncio
async def test_probe_browser_500_on_narrow_bucket_skips_not_crash(
scraper: AvitoScraper,
) -> None:
"""Probe бросает HTTP 500 на УЗКОМ bucket (< MIN_BRACKET) → skip+return, без краха.
Узкий bucket делить бесполезно — хвост теряется осознанно. Прогон не падает,
остальные комнатности продолжаются.
"""
second_room_paginated = False
async def fake_fetch(
room_slug: str, page: int, min_price: int | None, max_price: int | None
) -> str:
nonlocal second_room_paginated
if room_slug == "narrow-X":
raise _http_500_error()
if room_slug == "ok-X" and page == 1:
second_room_paginated = True
return f"<html>page={page}</html>"
def fake_total(html: str) -> int | None:
return 5
def fake_parse(html: str, source_url_base: str) -> list[ScrapedLot]:
return [_make_lot("ok_lot")] if "page=1" in html else []
narrow_lo = 1_000_000
narrow_hi = narrow_lo + _AVITO_MIN_BRACKET - 1
seen: dict[str, ScrapedLot] = {}
with (
patch.object(scraper, "_fetch_rooms_page_html", side_effect=fake_fetch),
patch.object(scraper, "_extract_total_count", side_effect=fake_total),
patch.object(scraper, "_parse_html", side_effect=fake_parse),
patch.object(scraper, "_rotate_ip", return_value=False),
):
await scraper._walk_price_range(
room_slug="narrow-X",
room_label="room_narrow",
lo=narrow_lo,
hi=narrow_hi,
seen=seen,
price_cap_per_bucket=1400,
max_pages_per_bucket=100,
)
assert seen == {}, "Узкий bucket с упавшим probe не должен дать лотов"
await scraper._walk_price_range(
room_slug="ok-X",
room_label="room_ok",
lo=0,
hi=_AVITO_MAX_PRICE,
seen=seen,
price_cap_per_bucket=1400,
max_pages_per_bucket=100,
)
assert second_room_paginated, "Вторая комнатность должна пагинироваться после skip"
assert "ok_lot" in seen
@pytest.mark.asyncio
async def test_probe_avito_blocked_propagates_not_swallowed(
scraper: AvitoScraper,
) -> None:
"""AvitoBlockedError из probe ВСПЛЫВАЕТ (hard-block), а не глушится как total=None."""
async def fake_fetch(
room_slug: str, page: int, min_price: int | None, max_price: int | None
) -> str:
raise AvitoBlockedError("Avito SERP firewall (browser-mode) — IP banned")
def fake_total(html: str) -> int | None:
return 100
with (
patch.object(scraper, "_fetch_rooms_page_html", side_effect=fake_fetch),
patch.object(scraper, "_extract_total_count", side_effect=fake_total),
patch.object(scraper, "_parse_html", side_effect=lambda h, source_url_base: []),
patch.object(scraper, "_rotate_ip", return_value=False),
):
with pytest.raises(AvitoBlockedError):
await scraper.fetch_all_secondary(
rooms_buckets=[("1-комн.", "1-komnatnye-X")],
price_cap_per_bucket=1400,
)
# ── Seed price-brackets entry point (perf: open top bucket) ───────────────────
@pytest.mark.asyncio
async def test_fetch_all_secondary_walks_seed_brackets_per_room(
scraper: AvitoScraper,
) -> None:
"""fetch_all_secondary зовёт _walk_price_range РОВНО len(seed) раз на комнатность.
Аргументы lo/hi совпадают с seed-брекетами: закрытые передаются с hi=br_hi-1
(эксклюзивный верх, без overlap на стыках), последний (открытый) — hi=None.
"""
walk_calls: list[tuple[int, int | None]] = []
async def spy_walk(*args: object, **kwargs: object) -> None:
walk_calls.append((kwargs["lo"], kwargs["hi"])) # type: ignore[arg-type]
# Не уходим в реальную рекурсию — фиксируем только верхнеуровневые входы.
return None
with patch.object(scraper, "_walk_price_range", side_effect=spy_walk):
await scraper.fetch_all_secondary(
rooms_buckets=[("1-комн.", "1-komnatnye-X")],
price_cap_per_bucket=2000,
)
expected = [
(br_lo, (br_hi - 1) if br_hi is not None else None)
for (br_lo, br_hi) in _AVITO_PRICE_SEED_BRACKETS
]
assert walk_calls == expected, f"Неверные входы _walk_price_range: {walk_calls}"
# Последний — открытый (hi=None).
assert walk_calls[-1][1] is None, "Последний seed-брекет должен быть открытым (hi=None)"
# Ровно один вызов на seed-брекет (1 комнатность).
assert len(walk_calls) == len(_AVITO_PRICE_SEED_BRACKETS)
@pytest.mark.asyncio
async def test_open_bracket_probes_pmax_none_and_paginates_without_split(
scraper: AvitoScraper,
) -> None:
"""Открытый брекет (hi=None): probe с pmax=None, пагинация БЕЗ попытки split.
Даже если total > cap (мокнуто) — НЕ должно быть рекурсивного деления:
при hi=None mid=(lo+hi)//2 невозможен, поэтому leaf пагинируется напрямую.
Проверяем: (1) probe идёт с max_price=None (pmax не ставится), (2) НЕТ
под-вызовов с пересчитанным mid (никаких max_price != None в fetch'ах).
"""
open_lo = _AVITO_PRICE_SEED_BRACKETS[-1][0]
assert _AVITO_PRICE_SEED_BRACKETS[-1][1] is None
fetch_args: list[tuple[int, int | None, int | None]] = []
async def fake_fetch(
room_slug: str, page: int, min_price: int | None, max_price: int | None
) -> str:
fetch_args.append((page, min_price, max_price))
return f"<html>page={page} min={min_price} max={max_price}</html>"
# total > cap для ОТКРЫТОГО брекета — провоцируем split-соблазн. Закрытые
# брекеты возвращают 0 (не пагинируются), чтобы изолировать открытый.
def fake_total(html: str) -> int | None:
import re
max_m = re.search(r"max=(\w+)", html)
max_raw = max_m.group(1) if max_m else "None"
if max_raw == "None": # открытый брекет (pmax отсутствует)
return 5000 # > cap=2000 — но split НЕ должен случиться
return 0
def fake_parse(html: str, source_url_base: str) -> list[ScrapedLot]:
import re
page_m = re.search(r"page=(\d+)", html)
page = int(page_m.group(1)) if page_m else 1
# Только открытый брекет (max=None) отдаёт лоты.
if "max=None" not in html:
return []
return [_make_lot(f"open_lot_p{page}_{i}") for i in range(3)]
with (
patch.object(scraper, "_fetch_rooms_page_html", side_effect=fake_fetch),
patch.object(scraper, "_extract_total_count", side_effect=fake_total),
patch.object(scraper, "_parse_html", side_effect=fake_parse),
patch.object(scraper, "_rotate_ip", return_value=False),
):
lots = await scraper.fetch_all_secondary(
rooms_buckets=[("1-комн.", "1-komnatnye-X")],
price_cap_per_bucket=2000,
max_pages_per_bucket=100,
)
# Probe открытого брекета: min_price=open_lo, max_price=None.
open_probes = [
(mn, mx) for (p, mn, mx) in fetch_args if p == 1 and mn == open_lo and mx is None
]
assert open_probes, f"Ожидался probe открытого брекета с pmax=None: {fetch_args}"
# Все fetch'и открытого брекета (min=open_lo) идут с max_price=None — НЕТ
# под-брекетов с вычисленным mid (что означало бы split).
open_fetches = [(mn, mx) for (p, mn, mx) in fetch_args if mn == open_lo]
assert all(
mx is None for (_mn, mx) in open_fetches
), f"Открытый брекет НЕ должен порождать closed-под-брекеты (split): {open_fetches}"
# Открытый брекет всё же собрал лоты (пагинация прошла, total=5000 → cap pages).
assert len(lots) > 0, "Открытый брекет должен пагинироваться и собрать лоты"
@pytest.mark.asyncio
async def test_closed_bracket_under_cap_single_probe_then_paginate(
scraper: AvitoScraper,
) -> None:
"""Закрытый брекет total<cap → 1 probe + прямая пагинация (без split)."""
# Изолируем второй закрытый брекет (4_000_000, 5_000_000) → walk_hi=4_999_999.
target_lo, target_hi_raw = _AVITO_PRICE_SEED_BRACKETS[1]
assert target_hi_raw is not None
target_hi = target_hi_raw - 1
fetch_args: list[tuple[int, int | None, int | None]] = []
async def fake_fetch(
room_slug: str, page: int, min_price: int | None, max_price: int | None
) -> str:
fetch_args.append((page, min_price, max_price))
return f"<html>page={page} min={min_price} max={max_price}</html>"
def fake_total(html: str) -> int | None:
import re
min_m = re.search(r"min=(\w+)", html)
max_m = re.search(r"max=(\w+)", html)
min_p = min_m.group(1) if min_m else "0"
max_p = max_m.group(1) if max_m else "None"
if min_p == str(target_lo) and max_p == str(target_hi):
return 60 # ceil(60/50)=2 страницы, < cap → без split
return 0
def fake_parse(html: str, source_url_base: str) -> list[ScrapedLot]:
import re
if f"min={target_lo} max={target_hi}" not in html:
return []
page_m = re.search(r"page=(\d+)", html)
page = int(page_m.group(1)) if page_m else 1
if page <= 2:
return [_make_lot(f"t_lot_p{page}_{i}") for i in range(30)]
return []
with (
patch.object(scraper, "_fetch_rooms_page_html", side_effect=fake_fetch),
patch.object(scraper, "_extract_total_count", side_effect=fake_total),
patch.object(scraper, "_parse_html", side_effect=fake_parse),
patch.object(scraper, "_rotate_ip", return_value=False),
):
lots = await scraper.fetch_all_secondary(
rooms_buckets=[("1-комн.", "1-komnatnye-X")],
price_cap_per_bucket=2000,
)
# Целевой брекет: probe (page=1) + страница 2, без под-брекетов (split).
target_pages = sorted(p for (p, mn, mx) in fetch_args if mn == target_lo and mx == target_hi)
assert target_pages == [1, 2], f"Ожидались probe+page2 без split: {target_pages}"
assert len(lots) == 60, f"Ожидалось 60 лотов (2 страницы × 30), got {len(lots)}"
@pytest.mark.asyncio
async def test_dedup_source_id_across_brackets(scraper: AvitoScraper) -> None:
"""Дедуп по source_id работает между разными seed-брекетами (общий seen)."""
async def fake_fetch(
room_slug: str, page: int, min_price: int | None, max_price: int | None
) -> str:
return f"<html>page={page} min={min_price} max={max_price}</html>"
def fake_total(html: str) -> int | None:
return 3 # 1 страница на каждый брекет
def fake_parse(html: str, source_url_base: str) -> list[ScrapedLot]:
# ВСЕ брекеты отдают ОДИН и тот же source_id → должен остаться 1 уникальный.
return [_make_lot("shared_dup_id")]
with (
patch.object(scraper, "_fetch_rooms_page_html", side_effect=fake_fetch),
patch.object(scraper, "_extract_total_count", side_effect=fake_total),
patch.object(scraper, "_parse_html", side_effect=fake_parse),
patch.object(scraper, "_rotate_ip", return_value=False),
):
lots = await scraper.fetch_all_secondary(
rooms_buckets=[("1-комн.", "1-komnatnye-X")],
price_cap_per_bucket=2000,
)
assert len(lots) == 1, f"Дедуп между брекетами должен дать 1 уникальный лот, got {len(lots)}"
assert lots[0].source_id == "shared_dup_id"

315
tradein-mvp/backend/tests/scrapers/test_avito_incremental_pagination.py
View file

@ -1,315 +0,0 @@
"""INCREMENTAL SERP-пагинация Avito с date early-stop (param-gated, default off).
`fetch_all_secondary(since=date)` на каждый (комнатность × seed-брекет) идёт
ПОСЛЕДОВАТЕЛЬНО newest-first и останавливается, как только страница уходит ниже
watermark `since` (SERP s=104 newest-first). `since=None` сохраняет точный
exhaustive bisection-обход (_walk_price_range) — без изменений.
Тестируем через monkeypatch page-fetch seam (`_fetch_rooms_page_html`) +
`_parse_html`, отдавая контролируемые страницы ScrapedLot с заданными listing_date.
asyncio.sleep мокается в no-op для скорости (паттерн из
test_avito_403_backconnect_reconnect.py).
"""
from datetime import date
from unittest.mock import AsyncMock, patch
import pytest
from app.services.scrapers import avito as avito_mod
from app.services.scrapers.avito import AvitoScraper
from app.services.scrapers.avito_exceptions import AvitoBlockedError
from app.services.scrapers.base import ScrapedLot
SINCE = date(2026, 6, 10)
def _lot(item_id: str, listing_date: date | None, segment: str = "vtorichka") -> ScrapedLot:
"""Минимальный валидный ScrapedLot с заданной датой/сегментом."""
return ScrapedLot(
source="avito",
source_url=f"https://www.avito.ru/ekaterinburg/kvartiry/{item_id}",
source_id=item_id,
rooms=2,
price_rub=4_500_000,
listing_date=listing_date,
listing_segment=segment,
)
def _make_scraper() -> AvitoScraper:
scraper = AvitoScraper()
scraper.request_delay_sec = 0.0
return scraper
def _install_pages(
scraper: AvitoScraper,
monkeypatch: pytest.MonkeyPatch,
pages: list[list[ScrapedLot] | None],
) -> dict[str, int]:
"""Подменяет page-fetch seam: страница p (1-based) → pages[p-1].
pages[i] is None → _fetch_rooms_page_html вернёт None (end-of-pages).
pages[i] == [] → HTML непустой, но _parse_html вернёт [] (end-of-pages).
Возвращает счётчик {"fetched": N} — сколько раз _fetch_rooms_page_html вызван.
"""
counter = {"fetched": 0}
async def fake_fetch(room_slug: str, page: int, lo: int | None, hi: int | None) -> str | None:
counter["fetched"] += 1
if page > len(pages):
return None
canned = pages[page - 1]
if canned is None:
return None
# Непустой HTML-маркер; реальный парс подменён ниже.
return f"<html data-page='{page}'></html>"
def fake_parse(html: str, source_url_base: str) -> list[ScrapedLot]:
# Достаём номер страницы из маркера, отдаём соответствующие лоты.
marker = html.split("data-page='")[1].split("'")[0]
page = int(marker)
canned = pages[page - 1]
return list(canned) if canned else []
monkeypatch.setattr(scraper, "_fetch_rooms_page_html", fake_fetch)
monkeypatch.setattr(scraper, "_parse_html", fake_parse)
return counter
@pytest.mark.asyncio
async def test_early_stop_on_all_older_page(monkeypatch: pytest.MonkeyPatch) -> None:
"""page1 fresh, page2 mixed (fresh+old), page3 all older → стоп на page3, page4 НЕ фетчится."""
scraper = _make_scraper()
pages: list[list[ScrapedLot] | None] = [
[_lot("p1a", date(2026, 6, 15)), _lot("p1b", date(2026, 6, 14))], # все свежие
[_lot("p2a", date(2026, 6, 11)), _lot("p2b", date(2026, 6, 5))], # mixed (один >= since)
[_lot("p3a", date(2026, 6, 3)), _lot("p3b", date(2026, 6, 1))], # все старше since
[_lot("p4a", date(2026, 5, 20))], # НЕ должна фетчиться
]
counter = _install_pages(scraper, monkeypatch, pages)
seen: dict[str, ScrapedLot] = {}
captured: list[tuple[str, list[ScrapedLot]]] = []
with patch.object(avito_mod.asyncio, "sleep", AsyncMock()):
await scraper._paginate_incremental_bracket(
room_slug="rooms-slug",
room_label="2к",
lo=0,
hi=None,
since=SINCE,
seen=seen,
max_pages_per_bucket=100,
secondary_only=True,
on_bucket=lambda k, lots: captured.append((k, lots)),
)
# page3 обработана (она old → break ПОСЛЕ неё), page4 не фетчилась.
assert counter["fetched"] == 3
# Собраны лоты страниц 1-3 (включая boundary-страницу 3), но не 4.
assert set(seen.keys()) == {"p1a", "p1b", "p2a", "p2b", "p3a", "p3b"}
assert "p4a" not in seen
assert captured and captured[0][0] == "2к:0:open"
@pytest.mark.asyncio
async def test_no_early_stop_when_all_fresh(monkeypatch: pytest.MonkeyPatch) -> None:
"""Все страницы свежее since до естественного конца (page возвращает None) → walk до конца."""
scraper = _make_scraper()
pages: list[list[ScrapedLot] | None] = [
[_lot("p1", date(2026, 6, 18))],
[_lot("p2", date(2026, 6, 16))],
[_lot("p3", date(2026, 6, 12))],
None, # end-of-pages
]
counter = _install_pages(scraper, monkeypatch, pages)
seen: dict[str, ScrapedLot] = {}
with patch.object(avito_mod.asyncio, "sleep", AsyncMock()):
await scraper._paginate_incremental_bracket(
room_slug="rooms-slug",
room_label="2к",
lo=0,
hi=None,
since=SINCE,
seen=seen,
max_pages_per_bucket=100,
secondary_only=True,
on_bucket=None,
)
# 3 страницы с лотами + 4-я (None) → 4 фетча, затем стоп.
assert counter["fetched"] == 4
assert set(seen.keys()) == {"p1", "p2", "p3"}
@pytest.mark.asyncio
async def test_all_none_dates_grace_stop(monkeypatch: pytest.MonkeyPatch) -> None:
"""Страницы all-None дат: page1 не стопит сразу, но 2 подряд undated → grace-стоп на page2."""
scraper = _make_scraper()
pages: list[list[ScrapedLot] | None] = [
[_lot("p1", None)], # undated → не стоп, streak=1
[_lot("p2", None)], # undated → streak=2 → grace-стоп ПОСЛЕ page2
[_lot("p3", None)], # НЕ должна фетчиться
]
counter = _install_pages(scraper, monkeypatch, pages)
seen: dict[str, ScrapedLot] = {}
with patch.object(avito_mod.asyncio, "sleep", AsyncMock()):
await scraper._paginate_incremental_bracket(
room_slug="rooms-slug",
room_label="2к",
lo=0,
hi=None,
since=SINCE,
seen=seen,
max_pages_per_bucket=100,
secondary_only=True,
on_bucket=None,
)
assert counter["fetched"] == 2 # page3 не фетчилась (grace bound)
assert set(seen.keys()) == {"p1", "p2"}
@pytest.mark.asyncio
async def test_none_date_does_not_stop_then_fresh_continues(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Одна undated-страница между свежими не триггерит стоп (streak сбрасывается)."""
scraper = _make_scraper()
pages: list[list[ScrapedLot] | None] = [
[_lot("p1", date(2026, 6, 15))], # fresh
[_lot("p2", None)], # undated → streak=1, продолжаем
[_lot("p3", date(2026, 6, 12))], # fresh → streak сброшен
[_lot("p4", date(2026, 6, 2))], # all older → стоп после page4
[_lot("p5", date(2026, 5, 1))], # НЕ фетчится
]
counter = _install_pages(scraper, monkeypatch, pages)
seen: dict[str, ScrapedLot] = {}
with patch.object(avito_mod.asyncio, "sleep", AsyncMock()):
await scraper._paginate_incremental_bracket(
room_slug="rooms-slug",
room_label="2к",
lo=0,
hi=None,
since=SINCE,
seen=seen,
max_pages_per_bucket=100,
secondary_only=True,
on_bucket=None,
)
assert counter["fetched"] == 4
assert set(seen.keys()) == {"p1", "p2", "p3", "p4"}
@pytest.mark.asyncio
async def test_secondary_only_filters_novostroyki(monkeypatch: pytest.MonkeyPatch) -> None:
"""secondary_only=True отбрасывает novostroyki до дедупа в seen (как leaf-bucket)."""
scraper = _make_scraper()
pages: list[list[ScrapedLot] | None] = [
[
_lot("vt1", date(2026, 6, 15), segment="vtorichka"),
_lot("nb1", date(2026, 6, 15), segment="novostroyki"),
],
None,
]
_install_pages(scraper, monkeypatch, pages)
seen: dict[str, ScrapedLot] = {}
with patch.object(avito_mod.asyncio, "sleep", AsyncMock()):
await scraper._paginate_incremental_bracket(
room_slug="rooms-slug",
room_label="2к",
lo=0,
hi=None,
since=SINCE,
seen=seen,
max_pages_per_bucket=100,
secondary_only=True,
on_bucket=None,
)
assert set(seen.keys()) == {"vt1"} # novostroyki отфильтрована
@pytest.mark.asyncio
async def test_blocked_error_propagates(monkeypatch: pytest.MonkeyPatch) -> None:
"""AvitoBlockedError из page-фетча пробрасывается наверх (не глушится)."""
scraper = _make_scraper()
async def fake_fetch(room_slug: str, page: int, lo: int | None, hi: int | None) -> str | None:
raise AvitoBlockedError("hard block")
monkeypatch.setattr(scraper, "_fetch_rooms_page_html", fake_fetch)
seen: dict[str, ScrapedLot] = {}
with patch.object(avito_mod.asyncio, "sleep", AsyncMock()):
with pytest.raises(AvitoBlockedError):
await scraper._paginate_incremental_bracket(
room_slug="rooms-slug",
room_label="2к",
lo=0,
hi=None,
since=SINCE,
seen=seen,
max_pages_per_bucket=100,
secondary_only=True,
on_bucket=None,
)
@pytest.mark.asyncio
async def test_since_none_uses_exhaustive_path(monkeypatch: pytest.MonkeyPatch) -> None:
"""since=None → НЕ зовёт _paginate_incremental_bracket; идёт бисекция (_walk_price_range)."""
scraper = _make_scraper()
walk_calls = {"n": 0}
incr_calls = {"n": 0}
async def fake_walk(**kwargs: object) -> None:
walk_calls["n"] += 1
async def fake_incr(**kwargs: object) -> None:
incr_calls["n"] += 1
monkeypatch.setattr(scraper, "_walk_price_range", fake_walk)
monkeypatch.setattr(scraper, "_paginate_incremental_bracket", fake_incr)
await scraper.fetch_all_secondary(
rooms_buckets=[("2-комнатные", "rooms-slug")],
since=None,
)
# Один room × N seed-брекетов → _walk_price_range зовётся, incremental — нет.
assert walk_calls["n"] == len(avito_mod._AVITO_PRICE_SEED_BRACKETS)
assert incr_calls["n"] == 0
@pytest.mark.asyncio
async def test_since_date_uses_incremental_path(monkeypatch: pytest.MonkeyPatch) -> None:
"""since=date → зовёт _paginate_incremental_bracket вместо _walk_price_range."""
scraper = _make_scraper()
walk_calls = {"n": 0}
incr_calls = {"n": 0}
async def fake_walk(**kwargs: object) -> None:
walk_calls["n"] += 1
async def fake_incr(**kwargs: object) -> None:
incr_calls["n"] += 1
monkeypatch.setattr(scraper, "_walk_price_range", fake_walk)
monkeypatch.setattr(scraper, "_paginate_incremental_bracket", fake_incr)
await scraper.fetch_all_secondary(
rooms_buckets=[("2-комнатные", "rooms-slug")],
since=SINCE,
)
assert incr_calls["n"] == len(avito_mod._AVITO_PRICE_SEED_BRACKETS)
assert walk_calls["n"] == 0

225
tradein-mvp/backend/tests/scrapers/test_avito_sweep_retry.py
View file

@ -1,225 +0,0 @@
"""P2 resilience — page-level retry для avito sweep (citywide / byrooms).
Покрывает _fetch_serp_html_with_retry и интеграцию в fetch_city_wide:
один транзиентный сбой страницы больше НЕ обрывает весь прогон.
Тесты:
1. transient на 1-й попытке → успех на 2-й → возвращает html (1 retry).
2. все попытки кидают generic → re-raise после исчерпания.
3. AvitoBlockedError на 1-й попытке → немедленный re-raise (НЕ retry, call_count==1).
4. None (non-200) → возвращается без retry.
5. integration fetch_city_wide: page=2 транзиентный 1 раз потом ок → обе страницы.
6. integration fetch_city_wide: page=3 persistent-fail → break после retry,
страницы 1-2 сохранены.
Без сети, без БД, без curl_cffi.
"""
from __future__ import annotations
import os
from unittest.mock import AsyncMock, patch
import pytest
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
from app.services.scrapers.avito import AvitoScraper
from app.services.scrapers.avito_exceptions import AvitoBlockedError, AvitoRateLimitedError
from app.services.scrapers.base import ScrapedLot
def _make_lot(source_id: str) -> ScrapedLot:
return ScrapedLot(
source="avito",
source_url=f"https://www.avito.ru/ekaterinburg/kvartiry/{source_id}",
source_id=source_id,
price_rub=5_000_000,
)
# ---------------------------------------------------------------------------
# 1. transient → retry → success
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_retry_recovers_after_one_transient() -> None:
s = AvitoScraper()
calls = 0
async def flaky(url: str, page: int) -> str:
nonlocal calls
calls += 1
if calls == 1:
raise RuntimeError("transient network blip")
return "<html>ok</html>"
with patch.object(s, "_fetch_serp_html", side_effect=flaky):
with patch("app.services.scrapers.avito.asyncio.sleep", new=AsyncMock()):
html = await s._fetch_serp_html_with_retry("u", 1, max_retries=2)
assert html == "<html>ok</html>"
assert calls == 2 # 1 fail + 1 success
# ---------------------------------------------------------------------------
# 2. all attempts fail → re-raise
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_retry_reraises_after_exhaustion() -> None:
s = AvitoScraper()
calls = 0
async def always_fail(url: str, page: int) -> str:
nonlocal calls
calls += 1
raise RuntimeError("persistent failure")
with patch.object(s, "_fetch_serp_html", side_effect=always_fail):
with patch("app.services.scrapers.avito.asyncio.sleep", new=AsyncMock()):
with pytest.raises(RuntimeError, match="persistent failure"):
await s._fetch_serp_html_with_retry("u", 1, max_retries=2)
assert calls == 3 # max_retries=2 → 3 attempts total
# ---------------------------------------------------------------------------
# 3. hard-block → immediate re-raise, no retry
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_blocked_error_not_retried() -> None:
s = AvitoScraper()
calls = 0
async def blocked(url: str, page: int) -> str:
nonlocal calls
calls += 1
raise AvitoBlockedError("IP banned")
with patch.object(s, "_fetch_serp_html", side_effect=blocked):
with patch("app.services.scrapers.avito.asyncio.sleep", new=AsyncMock()):
with pytest.raises(AvitoBlockedError):
await s._fetch_serp_html_with_retry("u", 1, max_retries=2)
assert calls == 1 # hard-block: no retry
@pytest.mark.asyncio
async def test_rate_limited_error_not_retried() -> None:
s = AvitoScraper()
calls = 0
async def rl(url: str, page: int) -> str:
nonlocal calls
calls += 1
raise AvitoRateLimitedError("429")
with patch.object(s, "_fetch_serp_html", side_effect=rl):
with patch("app.services.scrapers.avito.asyncio.sleep", new=AsyncMock()):
with pytest.raises(AvitoRateLimitedError):
await s._fetch_serp_html_with_retry("u", 1, max_retries=2)
assert calls == 1
# ---------------------------------------------------------------------------
# 4. None (non-200) → returned without retry
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_none_returned_without_retry() -> None:
s = AvitoScraper()
calls = 0
async def returns_none(url: str, page: int) -> None:
nonlocal calls
calls += 1
return None
with patch.object(s, "_fetch_serp_html", side_effect=returns_none):
with patch("app.services.scrapers.avito.asyncio.sleep", new=AsyncMock()):
result = await s._fetch_serp_html_with_retry("u", 1, max_retries=2)
assert result is None
assert calls == 1 # None is end-of-pagination, not an error
# ---------------------------------------------------------------------------
# 5. integration: transient page recovers, run does NOT abort
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_fetch_city_wide_transient_page_does_not_abort() -> None:
"""page=2 кидает transient 1 раз, потом ок → собираются обе страницы."""
s = AvitoScraper()
page_fail_budget = {2: 1} # page=2 фейлит ровно один раз
async def flaky_html(url: str, page: int) -> str:
if page_fail_budget.get(page, 0) > 0:
page_fail_budget[page] -= 1
raise RuntimeError(f"transient on page {page}")
return f"<html>page{page}</html>"
page_lots: dict[int, list[ScrapedLot]] = {
1: [_make_lot("P1a"), _make_lot("P1b")],
2: [_make_lot("P2a")],
3: [], # end of pagination
}
def mock_parse(html: str, source_url_base: str) -> list[ScrapedLot]:
for p, lots in page_lots.items():
if f"page{p}" in html:
return lots
return []
with patch.object(s, "_fetch_serp_html", side_effect=flaky_html):
with patch.object(s, "_parse_html", side_effect=mock_parse):
with patch.object(s, "sleep_between_requests", new=AsyncMock()):
with patch("app.services.scrapers.avito.asyncio.sleep", new=AsyncMock()):
result = await s.fetch_city_wide(pages=10, delay_override_sec=0)
ids = {lot.source_id for lot in result}
assert ids == {"P1a", "P1b", "P2a"} # page=2 survived the transient blip
# ---------------------------------------------------------------------------
# 6. integration: persistent page failure → break after retry, head kept
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_fetch_city_wide_persistent_page_breaks_keeping_head() -> None:
"""page=3 фейлит всегда → break после исчерпания retry; страницы 1-2 сохранены."""
s = AvitoScraper()
async def html(url: str, page: int) -> str:
if page >= 3:
raise RuntimeError(f"persistent on page {page}")
return f"<html>page{page}</html>"
page_lots: dict[int, list[ScrapedLot]] = {
1: [_make_lot("P1a")],
2: [_make_lot("P2a")],
}
def mock_parse(html_str: str, source_url_base: str) -> list[ScrapedLot]:
for p, lots in page_lots.items():
if f"page{p}" in html_str:
return lots
return []
with patch.object(s, "_fetch_serp_html", side_effect=html):
with patch.object(s, "_parse_html", side_effect=mock_parse):
with patch.object(s, "sleep_between_requests", new=AsyncMock()):
with patch("app.services.scrapers.avito.asyncio.sleep", new=AsyncMock()):
result = await s.fetch_city_wide(pages=10, delay_override_sec=0)
ids = {lot.source_id for lot in result}
assert ids == {"P1a", "P2a"} # pages 1-2 kept; page=3 persistent → graceful break

56
tradein-mvp/backend/tests/scrapers/test_avito_unix_date_tz_consistency.py
View file

@ -3,25 +3,21 @@
До фикса было 3 независимые копии с разными timezone:
- avito.py (inline в `_build_sort_timestamp_map`) — MSK (документировано #726)
- avito_houses.py (легаси, удалён #2397 Part E2) — UTC
- avito_imv.py (легаси, удалён #2277) — naive/system-local
(`datetime.fromtimestamp(ts)` без tz)
- avito_houses.py — UTC
- avito_imv.py — naive/system-local (`datetime.fromtimestamp(ts)` без tz)
Один и тот же epoch мог давать РАЗНЫЕ `date` в разных путях (пример из живой fixture
avito_imv_getdata.json: removedDate=1756077885 → UTC 2025-08-24, MSK 2025-08-25).
Офлайн: без сети/curl_cffi/БД, только импорт + чистые функции.
Легаси `app.services.scrapers.avito_imv` удалён (#2277 финальный шаг scraper_kit-
миграции, 0 runtime importers) — IMV-ветка consolidation-проверки переведена на
kit-эквивалент `scraper_kit.providers.avito.imv`, который использует ТОТ ЖЕ паттерн
(`from scraper_kit.providers.avito.shared import _unix_to_date`) — регрессия
"3 независимые копии с разными TZ" остаётся под наблюдением и на kit-стороне.
Легаси `app.services.scrapers.avito_houses` удалён (#2397 Part E2, 0 runtime
importers) — houses-ветка consolidation-проверки переведена на kit-эквивалент
`scraper_kit.providers.avito.houses`, который использует ТОТ ЖЕ паттерн
(`from scraper_kit.providers.avito.shared import RUS_MONTHS, _unix_to_date`).
Легаси `app.services.scrapers.avito` / `avito_shared` / `avito_detail` / `avito_imv` /
`avito_houses` удалены (#2397 финальный шаг E, 0 runtime importers) — вся
consolidation-проверка переведена на kit-эквиваленты (`scraper_kit.providers.avito.
serp/shared/detail/houses/imv`), которые используют ТОТ ЖЕ единый источник правды
(`scraper_kit.providers.avito.shared._unix_to_date` / `RUS_MONTHS`) — регрессия
"3 независимые копии с разными TZ" остаётся под наблюдением на kit-стороне (теперь
единственной живой).
"""
from __future__ import annotations
@ -31,12 +27,12 @@ from datetime import date
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
from scraper_kit.providers.avito import detail as kit_avito_detail
from scraper_kit.providers.avito import houses as kit_avito_houses
from scraper_kit.providers.avito import imv as kit_avito_imv
from scraper_kit.providers.avito import serp as kit_avito_serp
from scraper_kit.providers.avito import shared as kit_avito_shared
from app.services.scrapers import avito, avito_shared
# Живой пример из fixtures/avito_imv_getdata.json (placementHistory.items[0].removedDate) —
# именно тот timestamp, на котором UTC и MSK расходились в date() ДО фикса
# (UTC → 2025-08-24, MSK → 2025-08-25).
@ -45,54 +41,50 @@ _EXPECTED_MSK_DATE = date(2025, 8, 25)
def test_shared_unix_to_date_uses_msk() -> None:
"""Канонический источник правды — MSK (обоснование в avito_shared.py)."""
assert avito_shared._unix_to_date(_EPOCH_SECONDS) == _EXPECTED_MSK_DATE
"""Канонический источник правды — MSK (обоснование в shared.py)."""
assert kit_avito_shared._unix_to_date(_EPOCH_SECONDS) == _EXPECTED_MSK_DATE
def test_avito_houses_reexports_shared_unix_to_date() -> None:
"""kit houses.py не держит свою копию — импортирует общую функцию (тот же паттерн,
что и легаси avito_houses.py до удаления, #2397 Part E2)."""
"""kit houses.py не держит свою копию — импортирует общую функцию."""
assert kit_avito_houses._unix_to_date is kit_avito_shared._unix_to_date
assert kit_avito_houses._unix_to_date(_EPOCH_SECONDS) == _EXPECTED_MSK_DATE
def test_kit_avito_imv_reexports_shared_unix_to_date() -> None:
"""kit imv.py не держит свою копию — импортирует общую функцию (тот же паттерн,
что и kit houses.py/avito_shared.py после фикса #2129)."""
что и kit houses.py/shared.py после фикса #2129)."""
assert kit_avito_imv._unix_to_date is kit_avito_shared._unix_to_date
assert kit_avito_imv._unix_to_date(_EPOCH_SECONDS) == _EXPECTED_MSK_DATE
def test_avito_serp_sort_timestamp_matches_shared_date() -> None:
"""avito.py путь (SERP `sortTimeStamp`, epoch-МС) для того же момента времени
"""serp.py путь (SERP `sortTimeStamp`, epoch-МС) для того же момента времени
даёт ТУ ЖЕ дату, что и houses/IMV путь (epoch-секунды) — единственная разница
в единицах измерения epoch, TZ теперь общая (MSK) во всех путях (легаси и kit)."""
в единицах измерения epoch, TZ общая (MSK) во всех путях."""
html = (
f'<div>{{"id":8043936560,"sortTimeStamp":{_EPOCH_SECONDS * 1000},'
f'"addressDetailed":{{"foo":"bar"}}}}</div>'
)
ts_map = avito._build_sort_timestamp_map(html)
ts_map = kit_avito_serp._build_sort_timestamp_map(html)
assert ts_map["8043936560"] == _EXPECTED_MSK_DATE
# Легаси avito/avito_shared + kit avito_houses/avito_imv все согласованы на одном
# моменте времени:
# avito/shared + avito_houses/avito_imv все согласованы на одном моменте времени:
assert (
ts_map["8043936560"]
== kit_avito_houses._unix_to_date(_EPOCH_SECONDS)
== kit_avito_imv._unix_to_date(_EPOCH_SECONDS)
== avito_shared._unix_to_date(_EPOCH_SECONDS)
== kit_avito_shared._unix_to_date(_EPOCH_SECONDS)
)
def test_rus_months_single_source() -> None:
"""RUS_MONTHS (тоже была продублирована ×3) — общий словарь, не 3 копии."""
from app.services.scrapers import avito_detail
assert avito_detail.RUS_MONTHS is avito_shared.RUS_MONTHS
assert kit_avito_detail.RUS_MONTHS is kit_avito_shared.RUS_MONTHS
assert kit_avito_houses.RUS_MONTHS is kit_avito_shared.RUS_MONTHS
assert avito_shared.RUS_MONTHS["декабря"] == 12
assert kit_avito_shared.RUS_MONTHS["декабря"] == 12
def test_unix_to_date_none_and_zero() -> None:
"""None и 0 — sentinel «нет даты» → None (сохранено поведение старых копий)."""
assert avito_shared._unix_to_date(None) is None
assert avito_shared._unix_to_date(0) is None
assert kit_avito_shared._unix_to_date(None) is None
assert kit_avito_shared._unix_to_date(0) is None

375
tradein-mvp/backend/tests/scrapers/test_browser_fetcher.py
View file

@ -1,375 +0,0 @@
"""Tests for BrowserFetcher (#884/#905, HTTP-клиент).
Полностью замоканы через monkeypatch на httpx.AsyncClient.post:
не требуют установленного Firefox, camoufox или реального HTTP-сервера.
Покрывают:
- fetch(): POST к правильному endpoint + разбор {"html": "..."} из ответа
- HTTPError триггерит один retry, затем поднимает исключение
- URL строится из settings.browser_http_endpoint
- импорт модуля не ломается без playwright/camoufox
"""
from __future__ import annotations
import os
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, patch
import httpx
import pytest
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
# ── helpers ────────────────────────────────────────────────────────────────────
def _mock_settings(
browser_http_endpoint: str = "http://fake-browser:3000",
) -> SimpleNamespace:
return SimpleNamespace(browser_http_endpoint=browser_http_endpoint)
def _make_ok_response(html: str = "<html>ok</html>") -> MagicMock:
"""Создаёт мок httpx.Response с JSON {"html": html} и статусом 200."""
resp = MagicMock(spec=httpx.Response)
resp.status_code = 200
resp.json = MagicMock(return_value={"html": html})
resp.raise_for_status = MagicMock() # нет исключения
return resp
def _make_ok_json_response(status: int = 200, body: str = "{}") -> MagicMock:
"""Создаёт мок httpx.Response с JSON {"status": ..., "body": ...} (для /fetch-json)."""
resp = MagicMock(spec=httpx.Response)
resp.status_code = 200
resp.json = MagicMock(return_value={"status": status, "body": body})
resp.raise_for_status = MagicMock() # нет исключения
return resp
def _make_error_response(status: int = 500) -> MagicMock:
"""Создаёт мок httpx.Response который кидает HTTPStatusError при raise_for_status."""
resp = MagicMock(spec=httpx.Response)
resp.status_code = status
resp.raise_for_status = MagicMock(
side_effect=httpx.HTTPStatusError(
f"Server error {status}",
request=MagicMock(),
response=resp,
)
)
return resp
# ── импорт не ломается ─────────────────────────────────────────────────────────
def test_import_browser_fetcher() -> None:
"""import app.services.scrapers.browser_fetcher не поднимает исключений."""
import app.services.scrapers.browser_fetcher # noqa: F401
# ── fetch(): возвращает HTML из JSON-ответа ────────────────────────────────────
@pytest.mark.asyncio
async def test_fetch_returns_html_from_json_response() -> None:
"""fetch() разбирает {'html': ...} из 200-ответа и возвращает строку."""
from app.services.scrapers.browser_fetcher import BrowserFetcher
expected_html = "<html><body>avito</body></html>"
ok_resp = _make_ok_response(html=expected_html)
ms = _mock_settings()
with patch("app.core.config.settings", ms):
fetcher = BrowserFetcher()
async with fetcher:
assert fetcher._client is not None
fetcher._client.post = AsyncMock(return_value=ok_resp) # type: ignore[method-assign]
result = await fetcher.fetch("https://avito.ru/test")
assert result == expected_html
assert isinstance(result, str)
@pytest.mark.asyncio
async def test_fetch_posts_to_correct_endpoint() -> None:
"""fetch() делает POST к {browser_http_endpoint}/fetch с {"url", "source"}.
Дефолтный source="avito" — обратная совместимость со старым поведением, когда
сервер игнорирует source при выключенном FEATURE_BROWSER_POOL_ENABLED.
"""
from app.services.scrapers.browser_fetcher import BrowserFetcher
endpoint = "http://fake-browser:3000"
ok_resp = _make_ok_response()
ms = _mock_settings(browser_http_endpoint=endpoint)
with patch("app.core.config.settings", ms):
fetcher = BrowserFetcher()
async with fetcher:
assert fetcher._client is not None
post_mock = AsyncMock(return_value=ok_resp)
fetcher._client.post = post_mock # type: ignore[method-assign]
await fetcher.fetch("https://example.com/page")
post_mock.assert_called_once_with(
f"{endpoint}/fetch",
json={"url": "https://example.com/page", "source": "avito"},
)
@pytest.mark.asyncio
async def test_fetch_posts_source_field_when_set() -> None:
"""BrowserFetcher(source="yandex")._post_fetch шлёт body с "source": "yandex"."""
from app.services.scrapers.browser_fetcher import BrowserFetcher
endpoint = "http://fake-browser:3000"
ok_resp = _make_ok_response()
ms = _mock_settings(browser_http_endpoint=endpoint)
with patch("app.core.config.settings", ms):
fetcher = BrowserFetcher(source="yandex")
async with fetcher:
assert fetcher._client is not None
post_mock = AsyncMock(return_value=ok_resp)
fetcher._client.post = post_mock # type: ignore[method-assign]
await fetcher.fetch("https://realty.yandex.ru/ekb/")
post_mock.assert_called_once_with(
f"{endpoint}/fetch",
json={"url": "https://realty.yandex.ru/ekb/", "source": "yandex"},
)
# ── fetch_json(): in-page fetch через /fetch-json (#915 Stage 2) ───────────────
@pytest.mark.asyncio
async def test_fetch_json_returns_dict_from_json_response() -> None:
"""fetch_json() возвращает {"status","body"} dict из 200-ответа /fetch-json."""
from app.services.scrapers.browser_fetcher import BrowserFetcher
ok_resp = _make_ok_json_response(status=200, body='{"items": []}')
ms = _mock_settings()
with patch("app.core.config.settings", ms):
fetcher = BrowserFetcher()
async with fetcher:
assert fetcher._client is not None
fetcher._client.post = AsyncMock(return_value=ok_resp) # type: ignore[method-assign]
result = await fetcher.fetch_json("https://avito.ru/api/x")
assert result == {"status": 200, "body": '{"items": []}'}
@pytest.mark.asyncio
async def test_fetch_json_posts_full_payload() -> None:
"""fetch_json() POST'ит url/source/method/headers/body/origin к /fetch-json."""
from app.services.scrapers.browser_fetcher import BrowserFetcher
endpoint = "http://fake-browser:3000"
ok_resp = _make_ok_json_response()
ms = _mock_settings(browser_http_endpoint=endpoint)
with patch("app.core.config.settings", ms):
fetcher = BrowserFetcher(source="avito")
async with fetcher:
assert fetcher._client is not None
post_mock = AsyncMock(return_value=ok_resp)
fetcher._client.post = post_mock # type: ignore[method-assign]
await fetcher.fetch_json(
"https://avito.ru/api/listings",
method="POST",
headers={"content-type": "application/json"},
body='{"q": 1}',
origin="https://avito.ru/moskva/kvartiry",
)
post_mock.assert_called_once_with(
f"{endpoint}/fetch-json",
json={
"url": "https://avito.ru/api/listings",
"source": "avito",
"method": "POST",
"headers": {"content-type": "application/json"},
"body": '{"q": 1}',
"origin": "https://avito.ru/moskva/kvartiry",
},
)
@pytest.mark.asyncio
async def test_fetch_json_defaults_get_empty_headers_none_body() -> None:
"""Дефолты: method=GET, headers={}, body=None, origin=None."""
from app.services.scrapers.browser_fetcher import BrowserFetcher
endpoint = "http://fake-browser:3000"
ok_resp = _make_ok_json_response()
ms = _mock_settings(browser_http_endpoint=endpoint)
with patch("app.core.config.settings", ms):
fetcher = BrowserFetcher(source="cian")
async with fetcher:
assert fetcher._client is not None
post_mock = AsyncMock(return_value=ok_resp)
fetcher._client.post = post_mock # type: ignore[method-assign]
await fetcher.fetch_json("https://cian.ru/api/x")
post_mock.assert_called_once_with(
f"{endpoint}/fetch-json",
json={
"url": "https://cian.ru/api/x",
"source": "cian",
"method": "GET",
"headers": {},
"body": None,
"origin": None,
},
)
@pytest.mark.asyncio
async def test_fetch_json_retries_once_on_transport_error() -> None:
"""TransportError первый раз → один retry → успешный второй вызов."""
from app.services.scrapers.browser_fetcher import BrowserFetcher
ok_resp = _make_ok_json_response(status=200, body="retry-ok")
connect_err = httpx.ConnectError("Connection refused")
ms = _mock_settings()
with patch("app.core.config.settings", ms):
fetcher = BrowserFetcher()
async with fetcher:
assert fetcher._client is not None
post_mock = AsyncMock(side_effect=[connect_err, ok_resp])
fetcher._client.post = post_mock # type: ignore[method-assign]
with patch("app.services.scrapers.browser_fetcher.asyncio.sleep", AsyncMock()):
result = await fetcher.fetch_json("https://avito.ru/api/retry")
assert result == {"status": 200, "body": "retry-ok"}
assert post_mock.await_count == 2
@pytest.mark.asyncio
async def test_fetch_json_raises_after_two_http_errors() -> None:
"""Оба вызова /fetch-json кидают HTTPError → исключение пробрасывается."""
from app.services.scrapers.browser_fetcher import BrowserFetcher
error_resp_1 = _make_error_response(500)
error_resp_2 = _make_error_response(500)
ms = _mock_settings()
with patch("app.core.config.settings", ms):
fetcher = BrowserFetcher()
async with fetcher:
assert fetcher._client is not None
post_mock = AsyncMock(side_effect=[error_resp_1, error_resp_2])
fetcher._client.post = post_mock # type: ignore[method-assign]
with patch("app.services.scrapers.browser_fetcher.asyncio.sleep", AsyncMock()):
with pytest.raises(httpx.HTTPStatusError):
await fetcher.fetch_json("https://avito.ru/api/fail")
assert post_mock.await_count == 2
# ── retry при HTTPError ────────────────────────────────────────────────────────
@pytest.mark.asyncio
async def test_fetch_retries_once_on_http_error() -> None:
"""При HTTPError первый вызов → retry → успешный второй вызов."""
from app.services.scrapers.browser_fetcher import BrowserFetcher
html = "<html>retry-ok</html>"
ok_resp = _make_ok_response(html=html)
error_resp = _make_error_response(500)
ms = _mock_settings()
with patch("app.core.config.settings", ms):
fetcher = BrowserFetcher()
async with fetcher:
assert fetcher._client is not None
# Первый вызов кидает HTTPStatusError, второй возвращает OK
post_mock = AsyncMock(side_effect=[error_resp, ok_resp])
fetcher._client.post = post_mock # type: ignore[method-assign]
with patch("app.services.scrapers.browser_fetcher.asyncio.sleep", AsyncMock()):
result = await fetcher.fetch("https://avito.ru/retry")
assert result == html
assert post_mock.await_count == 2
@pytest.mark.asyncio
async def test_fetch_raises_after_two_http_errors() -> None:
"""Если оба вызова кидают HTTPError — исключение пробрасывается."""
from app.services.scrapers.browser_fetcher import BrowserFetcher
error_resp_1 = _make_error_response(500)
error_resp_2 = _make_error_response(500)
ms = _mock_settings()
with patch("app.core.config.settings", ms):
fetcher = BrowserFetcher()
async with fetcher:
assert fetcher._client is not None
post_mock = AsyncMock(side_effect=[error_resp_1, error_resp_2])
fetcher._client.post = post_mock # type: ignore[method-assign]
with patch("app.services.scrapers.browser_fetcher.asyncio.sleep", AsyncMock()):
with pytest.raises(httpx.HTTPStatusError):
await fetcher.fetch("https://avito.ru/fail")
assert post_mock.await_count == 2
@pytest.mark.asyncio
async def test_fetch_retries_on_connect_error() -> None:
"""ConnectError (подкласс TransportError) тоже триггерит retry."""
from app.services.scrapers.browser_fetcher import BrowserFetcher
html = "<html>connect-retry-ok</html>"
ok_resp = _make_ok_response(html=html)
connect_err = httpx.ConnectError("Connection refused")
ms = _mock_settings()
with patch("app.core.config.settings", ms):
fetcher = BrowserFetcher()
async with fetcher:
assert fetcher._client is not None
post_mock = AsyncMock(side_effect=[connect_err, ok_resp])
fetcher._client.post = post_mock # type: ignore[method-assign]
with patch("app.services.scrapers.browser_fetcher.asyncio.sleep", AsyncMock()):
result = await fetcher.fetch("https://avito.ru/connect")
assert result == html
assert post_mock.await_count == 2
# ── non-http exception пробрасывается без перехвата ───────────────────────────
@pytest.mark.asyncio
async def test_fetch_propagates_unexpected_exception() -> None:
"""Не-HTTP исключение (ValueError и т.п.) не подавляется и не вызывает retry."""
from app.services.scrapers.browser_fetcher import BrowserFetcher
ms = _mock_settings()
with patch("app.core.config.settings", ms):
fetcher = BrowserFetcher()
async with fetcher:
assert fetcher._client is not None
post_mock = AsyncMock(side_effect=ValueError("unexpected"))
fetcher._client.post = post_mock # type: ignore[method-assign]
with pytest.raises(ValueError, match="unexpected"):
await fetcher.fetch("https://avito.ru/bad")
assert post_mock.await_count == 1

48
tradein-mvp/backend/tests/scrapers/test_price_brackets.py
View file

@ -1,48 +0,0 @@
"""Тесты общего seed price-brackets констант (avito/cian/yandex)."""
from itertools import pairwise
from app.services.scrapers.price_brackets import (
_EKB_SECONDARY,
CITY_PRICE_SEED_BRACKETS,
get_price_seed_brackets,
)
def test_default_returns_ekb():
"""get_price_seed_brackets() без аргумента возвращает EKB-сетку."""
assert get_price_seed_brackets() == _EKB_SECONDARY
assert get_price_seed_brackets("ekaterinburg") == _EKB_SECONDARY
def test_unknown_city_falls_back_to_ekb():
"""Неизвестный город → fallback на EKB (не KeyError)."""
assert get_price_seed_brackets("atlantis") == _EKB_SECONDARY
def test_last_bracket_is_open():
"""Последний брекет ОТКРЫТ (hi=None) — catch-all хвоста люкса без потолка."""
brackets = get_price_seed_brackets()
assert brackets[-1][1] is None
# Только последний брекет открыт — остальные закрыты.
for lo, hi in brackets[:-1]:
assert hi is not None
assert hi > lo
def test_brackets_are_contiguous():
"""Закрытые брекеты соприкасаются: hi[i] == lo[i+1] (без дыр/перекрытий)."""
brackets = get_price_seed_brackets()
for (lo, hi), (next_lo, _next_hi) in pairwise(brackets):
assert hi == next_lo, f"разрыв между [{lo},{hi}] и [{next_lo},...]"
def test_first_bracket_starts_at_zero():
"""Первый брекет стартует с 0 (вся вторичка с нижнего края)."""
assert get_price_seed_brackets()[0][0] == 0
def test_city_dict_contains_ekaterinburg():
"""City-dict готов под multi-city — сейчас единственный ключ ekaterinburg."""
assert "ekaterinburg" in CITY_PRICE_SEED_BRACKETS
assert CITY_PRICE_SEED_BRACKETS["ekaterinburg"] is _EKB_SECONDARY

18
tradein-mvp/backend/tests/services/scrapers/test_cian_resolve_zhk_url.py
View file

@ -4,7 +4,12 @@ The legacy resolve_cian_zhk_url hit https://cian.ru/zhk/<id>/ which now 404s; th
search-based resolver fetches the cat.php newbuilding-SERP and extracts the canonical
zhk-<slug>.cian.ru URL from its markup. No real network: the curl_cffi session is mocked;
the regex is exercised against a trimmed real-prod SERP fixture.
Легаси `app.services.scrapers.cian_newbuilding` удалён (#2397 финальный шаг E, 0
runtime-импортёров) — переведено на kit-эквивалент `scraper_kit.providers.cian.
newbuilding` (byte-identical strangler-копия).
"""
from __future__ import annotations
import os
@ -12,21 +17,18 @@ import sys
from pathlib import Path
from unittest.mock import AsyncMock, MagicMock
# DATABASE_URL required by config before any app import (cian_newbuilding → app.core.config).
# DATABASE_URL required by config before any app import.
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
# WeasyPrint stub — not installed in CI without GTK; pulled in transitively by app imports.
sys.modules.setdefault("weasyprint", MagicMock())
import pytest # noqa: E402
from app.services.scrapers.cian_newbuilding import ( # noqa: E402
from scraper_kit.providers.cian.newbuilding import ( # noqa: E402
_extract_zhk_url_from_serp,
resolve_cian_zhk_url_via_search,
)
_FIXTURE = (
Path(__file__).resolve().parents[2] / "fixtures" / "cian_catph_newbuilding_serp.html"
)
_FIXTURE = Path(__file__).resolve().parents[2] / "fixtures" / "cian_catph_newbuilding_serp.html"
def _serp_fixture() -> str:
@ -72,7 +74,7 @@ def test_extract_zhk_url_anchors_on_title_not_recommendation() -> None:
# Banner / recommendation block rendered BEFORE the title, carrying a foreign slug.
'<div data-name="Recommendations">'
'<a href="https://zhk-other-ad-ekb-i.cian.ru/">похожий ЖК (реклама)</a>'
'and a bare one too https://zhk-another-promo-ekb-i.cian.ru in text</div>'
"and a bare one too https://zhk-another-promo-ekb-i.cian.ru in text</div>"
# The canonical title block — its anchor is the queried ЖК.
'<div data-name="NewbuildingHeaderSeo">'
'<h1 data-name="Title" class="x31de4314--_7735e">Купить квартиру в '
@ -93,7 +95,7 @@ def test_extract_zhk_url_fallback_first_match_when_no_title() -> None:
"""
html = (
'noise <a href="https://zhk-first-ekb-i.cian.ru/">A</a> '
'and later https://zhk-second-ekb-i.cian.ru more'
"and later https://zhk-second-ekb-i.cian.ru more"
)
assert _extract_zhk_url_from_serp(html) == "https://zhk-first-ekb-i.cian.ru"

22
tradein-mvp/backend/tests/support/README.md
View file

@ -2,16 +2,23 @@
## parity.py — legacy → scraper_kit parity harness (issue #2304)
Инструмент для issues #2305-#2310 (миграция 12 неймигрированных importers
Инструмент для issues #2305-#2310 (миграция неймигрированных importers
`app/services/scrapers/*` → `scraper_kit` эквиваленты, см. audit
`Scraper_Kit_Legacy_Dependency_Audit_0703` в vault). Каждая такая миграция
должна доказать, что kit-путь даёт ТОТ ЖЕ результат, что и legacy-путь на
одном и том же входе — для этого используйте `assert_parity`.
должна была доказать, что kit-путь даёт ТОТ ЖЕ результат, что и legacy-путь на
одном и том же входе — для этого использовался `assert_parity`.
### Быстрый старт
Легаси `app/services/scrapers/` каталог полностью удалён (#2397 финальный шаг
E, миграция завершена) — все golden-parity/legacy-vs-kit тесты, построенные на
`assert_parity`, удалены вместе с ним (kit — единственный живой путь). Harness
(`parity.py`) остаётся в дереве на случай будущих strangler-миграций
(например legacy-кода за пределами `app/services/scrapers/`).
### Быстрый старт (иллюстративный пример; `legacy_fn` — гипотетическая функция
### из будущего strangler-миграции модуля, не из уже удалённого `app/services/scrapers/`)
```python
from app.services.scrapers.cian_valuation import evaluate_via_cian as legacy_fn
from some_legacy_module import evaluate_via_cian as legacy_fn
from scraper_kit.providers.cian.valuation import evaluate_via_cian as kit_fn
from tests.support.parity import assert_parity
@ -37,8 +44,9 @@ def test_cian_valuation_parity() -> None:
- любое другое значение (str, dict, ...) → передаётся как единственный
позиционный аргумент: `legacy_fn(fixture)`.
Начните с 1-2 hardcoded HTML-фикстур/dict'ов (см.
`tests/scrapers/test_avito_detail_kit_parity.py` — референсный пример).
Начните с 1-2 hardcoded HTML-фикстур/dict'ов (git-история
`tests/scrapers/test_avito_detail_kit_parity.py`, удалён вместе с legacy
`app/services/scrapers/` в #2397, содержит референсный пример).
DB-фикстуры НЕ нужны для чистых parse/compute-функций — используйте их
только если сама legacy/kit-функция реально требует Session.

16
tradein-mvp/backend/tests/test_781_quality_gate.py
View file

@ -219,7 +219,7 @@ def _make_scraped_lot(
source_id: str | None = None,
price_rub: int = 5_000_000,
) -> Any:
from app.services.scrapers.base import ScrapedLot
from scraper_kit.base import ScrapedLot
return ScrapedLot(
source=source,
@ -399,10 +399,12 @@ async def test_754_block_page_http200_page1_raises() -> None:
Before #754 this scenario caused a silent empty return ([]) masking a content block.
Refs: PR #779 / commit 489666c.
"""
from app.services.scrapers.avito import AvitoScraper
from app.services.scrapers.avito_exceptions import AvitoContentBlockedError
from scraper_kit.avito_exceptions import AvitoContentBlockedError
from scraper_kit.providers.avito.serp import AvitoScraper
scraper = AvitoScraper()
from app.services.scraper_adapters import RealScraperConfig
scraper = AvitoScraper(RealScraperConfig())
with patch.object(scraper, "_fetch_serp_html", new=AsyncMock(return_value=_BLOCKPAGE_HTML)):
with pytest.raises(AvitoContentBlockedError):
await scraper.fetch_around(56.838, 60.605, radius_m=1000, pages=1)
@ -416,9 +418,11 @@ async def test_754_block_page_http200_page_gt1_graceful_return() -> None:
end-of-pagination break). The call must NOT raise.
Refs: PR #779 / commit 489666c.
"""
from app.services.scrapers.avito import AvitoScraper
from scraper_kit.providers.avito.serp import AvitoScraper
scraper = AvitoScraper()
from app.services.scraper_adapters import RealScraperConfig
scraper = AvitoScraper(RealScraperConfig())
async def _serp_html_side_effect(url: str, page: int) -> str:
return _SERP_HTML if page == 1 else _BLOCKPAGE_HTML

337
tradein-mvp/backend/tests/test_849_matching_savepoint.py
View file

@ -1,337 +0,0 @@
"""Tests for issue #849 Parts 2 and 3.
Part 2: same normalized_address → exactly 1 house and 1 alias key.
Part 3: batch save_listings with one failing lot → siblings persist,
no orphan SAVEPOINT / no whole-batch rollback.
All tests use mock DB sessions — no real Postgres required.
"""
from __future__ import annotations
import logging
from contextlib import contextmanager
from unittest.mock import MagicMock, patch
import pytest
from app.services.matching.houses import _insert_alias, match_or_create_house
from app.services.matching.normalize import address_fingerprint, normalize_address
from app.services.scrapers.base import ScrapedLot, save_listings
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _make_db(rows: list[dict | None]) -> MagicMock:
"""Build a Session mock returning rows sequentially for each execute() call."""
db = MagicMock()
side_effects = []
for row in rows:
result = MagicMock()
mappings = MagicMock()
if row is None:
mappings.first.return_value = None
mappings.one.return_value = None
else:
mappings.first.return_value = row
mappings.one.return_value = row
result.mappings.return_value = mappings
side_effects.append(result)
db.execute.side_effect = side_effects
return db
def _mock_save_db(inserted: bool = True, listing_id: int = 42) -> MagicMock:
"""Mock Session for save_listings: INSERT RETURNING + SAVEPOINTs as no-op."""
row = MagicMock()
row.id = listing_id
row.inserted = inserted
result = MagicMock()
result.fetchone.return_value = row
db = MagicMock()
db.execute.return_value = result
@contextmanager
def _nested():
yield MagicMock()
db.begin_nested.side_effect = _nested
return db
def _base_lot(**overrides) -> ScrapedLot:
defaults = {
"source": "avito",
"source_url": "https://www.avito.ru/ekaterinburg/kvartiry/1-k_11111",
"source_id": "11111",
"price_rub": 4_000_000,
}
defaults.update(overrides)
return ScrapedLot(**defaults)
# ---------------------------------------------------------------------------
# Part 2: fingerprint↔alias sync
# ---------------------------------------------------------------------------
class TestFingerprintAliasSync:
"""Two listings with the same address → exactly 1 house and 1 alias key.
Scenario: two scrapers submit the same street address but with coords that
differ beyond the 4-decimal rounding boundary — they produce different
fingerprints. The matcher must still resolve to the same canonical house
and NOT create a second alias for the same normalized_address.
"""
def test_same_normalized_address_resolves_to_same_house_via_tier2b(self):
"""First call misses all tiers → creates house + alias (fp=fp1).
Second call with same address but different coords:
- Tier 2a: fp2 ≠ fp1 → miss
- Tier 2b: normalized_address hit → returns existing house_id
Result: same house_id, NOT a new house row.
"""
address = "Екатеринбург, ул. Ленина, 5"
lat1, lon1 = 56.838000, 60.594600 # yields fp1
lat2, lon2 = 56.839100, 60.594600 # 4dp rounds to 56.8391 ≠ 56.8380 → fp2 ≠ fp1
fp1 = address_fingerprint(address, lat1, lon1)
fp2 = address_fingerprint(address, lat2, lon2)
assert fp1 != fp2, "precondition: coords produce different fingerprints"
# ── First call: all tiers miss → INSERT new house ───────────────────
db1 = _make_db(
[
None, # pg_advisory_xact_lock
None, # house_sources miss (Tier 1)
None, # fingerprint miss (Tier 2a)
None, # normalized_address miss (Tier 2b) — first time
None, # geo miss (Tier 3; no lat/lon provided here)
{"id": 7}, # INSERT RETURNING id
None, # _upsert_house_source
None, # _insert_alias
]
)
house_id_1, _, method_1 = match_or_create_house(
db1,
"avito",
"ext-001",
address=address,
lat=lat1,
lon=lon1,
)
assert method_1 == "new"
assert house_id_1 == 7
# ── Second call (same address, different coords) ────────────────────
# Tier 2a misses (fp2 ≠ stored fp1), Tier 2b hits on normalized_address.
db2 = _make_db(
[
None, # pg_advisory_xact_lock
None, # house_sources miss (Tier 1)
None, # fingerprint miss (Tier 2a — fp2 not registered yet)
{"house_id": 7}, # normalized_address hit (Tier 2b)
None, # _upsert_house_source
None, # _insert_alias (register fp2)
]
)
house_id_2, _, method_2 = match_or_create_house(
db2,
"cian",
"ext-002",
address=address,
lat=lat2,
lon=lon2,
)
assert (
method_2 == "fingerprint"
), f"Expected 'fingerprint' (Tier 2b match), got {method_2!r}"
assert (
house_id_2 == 7
), f"Expected same house_id=7, got {house_id_2} — second scraper created a duplicate"
def test_same_normalized_address_tier2a_hits_directly_when_fp_matches(self):
"""When the fingerprint IS already registered (same address + coords within 11 m),
Tier 2a hits directly and normalized_address fallback is not reached."""
address = "ул Мира 10"
lat, lon = 56.84, 60.59
db = _make_db(
[
None, # pg_advisory_xact_lock
None, # house_sources miss
{"house_id": 99}, # fingerprint hit (Tier 2a)
None, # _upsert_house_source
None, # _insert_alias
]
)
house_id, conf, method = match_or_create_house(
db,
"yandex",
"ext-yz",
address=address,
lat=lat,
lon=lon,
)
assert method == "fingerprint"
assert house_id == 99
assert conf == 0.9
def test_insert_alias_upsert_keeps_fingerprint_in_sync(self):
"""_insert_alias with ON CONFLICT DO UPDATE ensures the stored fingerprint
is refreshed, not left stale from the first writer.
Verify the emitted SQL contains 'DO UPDATE SET' (not 'DO NOTHING').
"""
db = MagicMock()
result = MagicMock()
db.execute.return_value = result
_insert_alias(
db,
house_id=5,
address="ул. Ленина 5",
fp="aabbccdd" * 4,
source="avito",
)
assert db.execute.called
sql_str = str(db.execute.call_args[0][0])
assert (
"DO UPDATE SET" in sql_str
), "_insert_alias must use ON CONFLICT DO UPDATE SET fingerprint, not DO NOTHING"
assert "fingerprint" in sql_str
def test_two_listings_same_address_exactly_one_alias_key(self):
"""Integration check: normalize_address produces identical key for two scrapers
submitting the same address in different formats, so at most one alias row can
exist (UNIQUE normalized_address constraint holds).
This test verifies the normalization layer — the DB constraint is structural.
addr_a uses the abbreviated form 'ул.' which expands to 'улица';
addr_b already uses the full form. Both should normalize identically.
"""
addr_a = "ул. Ленина 5" # abbreviated 'ул.' → expands to 'улица'
addr_b = "улица ленина 5" # already expanded, lowercase
assert normalize_address(addr_a) == normalize_address(addr_b), (
"normalize_address must produce identical key for both address forms "
"so the UNIQUE(normalized_address) constraint prevents duplicate aliases"
)
# ---------------------------------------------------------------------------
# Part 3: single SAVEPOINT level — batch isolation
# ---------------------------------------------------------------------------
class TestSingleSavepointBatchIsolation:
"""save_listings: one failing lot → siblings persist; no orphan SAVEPOINT."""
@pytest.fixture(autouse=True)
def _patch_matching(self):
"""Stub matching so save_listings tests don't hit real DB queries."""
with (
patch("app.services.scrapers.base.match_or_create_house") as m_house,
patch("app.services.scrapers.base.upsert_listing_source") as m_link,
):
m_house.return_value = (101, 1.0, "new")
m_link.return_value = None
self.m_house = m_house
self.m_link = m_link
yield
def test_failing_lot_does_not_abort_batch(self, caplog):
"""Lot whose match_or_create_house raises → that lot's hook is rolled back
(outer SAVEPOINT), but other lots in the batch are fully inserted.
No whole-batch rollback, no orphan SAVEPOINTs.
"""
# First lot: house match succeeds; second lot: house match raises
self.m_house.side_effect = [
(10, 1.0, "new"),
RuntimeError("simulated house-match DB error"),
(12, 1.0, "new"),
]
db = _mock_save_db(inserted=True, listing_id=50)
lots = [
_base_lot(
source_id="ok-1",
address="Екатеринбург, Ленина 5",
lat=56.84,
lon=60.59,
),
_base_lot(
source_id="fail-1",
address="Екатеринбург, Мира 10",
lat=56.85,
lon=60.60,
),
_base_lot(
source_id="ok-2",
address="Екатеринбург, Куйбышева 3",
lat=56.83,
lon=60.58,
),
]
with caplog.at_level(logging.WARNING, logger="app.services.scrapers.base"):
inserted, _updated = save_listings(db, lots)
# All 3 listings INSERT'd (listing INSERT is outside any SAVEPOINT)
assert inserted == 3
# House match attempted on all 3 (all have address+coords)
assert self.m_house.call_count == 3
# upsert_listing_source: lots ok-1 and ok-2 succeed; fail-1's hook was aborted
# by the outer SAVEPOINT so link not called for it.
assert self.m_link.call_count == 2
# Warning logged for the failing lot
warns = [r for r in caplog.records if "match_failed" in r.getMessage()]
assert len(warns) == 1
def test_no_begin_nested_inside_link_listing_to_house(self):
"""Verify that _link_listing_to_house does NOT open its own begin_nested.
The outer per-row SAVEPOINT (one call per lot in save_listings) is the only
SAVEPOINT level allowed. If begin_nested is called more than N lots times,
that indicates a double-nesting.
"""
db = _mock_save_db(inserted=True, listing_id=77)
lots = [_base_lot(source_id=str(i)) for i in range(3)]
save_listings(db, lots)
# save_listings calls begin_nested THREE times per lot:
# 1) upsert SAVEPOINT — ловит IntegrityError на UNIQUE(source,source_id)
# (migration 133) при dedup_hash-дрейфе → rollback + прямой UPDATE (#1921);
# 2) snapshot SAVEPOINT;
# 3) hook SAVEPOINT (_link_listing_to_house).
# If _link_listing_to_house opened an INNER begin_nested too,
# the count would be > 3 * len(lots). The intent of this test (no double-nesting
# INSIDE the hook) holds — base.py keeps exactly one SAVEPOINT level in the hook.
assert db.begin_nested.call_count == 3 * len(lots), (
f"Expected exactly 3 begin_nested per lot (upsert + snapshot + hook outer), "
f"got {db.begin_nested.call_count} — inner SAVEPOINT may still be present"
)
def test_whole_batch_not_rolled_back_on_single_hook_failure(self, caplog):
"""When one lot's hook fails, db.commit() is still called (batch persisted)."""
self.m_house.side_effect = RuntimeError("boom")
db = _mock_save_db(inserted=True, listing_id=10)
lots = [_base_lot(source_id="only-one", address="x", lat=56.8, lon=60.5)]
with caplog.at_level(logging.WARNING, logger="app.services.scrapers.base"):
save_listings(db, lots)
# db.commit() called despite the hook failure — no whole-batch rollback
db.commit.assert_called_once()
db.rollback.assert_not_called()

82
tradein-mvp/backend/tests/test_avito_address_clean.py
View file

@ -1,82 +0,0 @@
import pytest
from app.services.scrapers.avito import _clean_address
def test_strips_css_noise():
raw = (
"ул. Токарей, 56к1Площадь 1905 года.css-39hgr0{fill:currentColor;"
"height:1em;box-sizing:border-box;}.css-39hgr0:focus{outline:none;}"
"от 31 мин.Геологическая.css-39hgr0{...}"
)
assert _clean_address(raw) == "ул. Токарей, 56к1"
def test_clean_address_passthrough():
assert _clean_address("ул. Малышева, 1") == "ул. Малышева, 1"
def test_clean_handles_none():
assert _clean_address(None) is None
assert _clean_address("") is None
assert _clean_address(" ") is None
def test_clean_strips_trailing_punctuation():
assert _clean_address("ул. Ленина, 5,") == "ул. Ленина, 5"
# ── #1773: де-слип номера дома и маркера района/нас.пункта ────────────────────
@pytest.mark.parametrize(
"raw,expected",
[
# реальный кейс с прода
(
"Свердловская обл., Екатеринбург, ул. Евгения Савкова, 29р-н Академический",
"Свердловская обл., Екатеринбург, ул. Евгения Савкова, 29, р-н Академический",
),
# литера дома слиплась с маркером
("ул. X, 17Ар-н Пионерский", "ул. X, 17А, р-н Пионерский"),
("ул. X, 44Бр-н Запад", "ул. X, 44Б, р-н Запад"),
# корпус через дробь
("ул. X, 35/2р-н Юг", "ул. X, 35/2, р-н Юг"),
# корпус через «сN»
("ул. X, 33с1р-н Центр", "ул. X, 33с1, р-н Центр"),
# прочие маркеры
("ул. X, 12мкр Солнечный", "ул. X, 12, мкр Солнечный"),
("ул. X, 5район Кировский", "ул. X, 5, район Кировский"),
("ул. X, 100к2р-н Чкаловский", "ул. X, 100к2, р-н Чкаловский"),
# маркеры населённых пунктов
("ул. X, 12с. Кашино", "ул. X, 12, с. Кашино"),
("ул. X, 10д. Малое", "ул. X, 10, д. Малое"),
("ул. X, 25г. Берёзовский", "ул. X, 25, г. Берёзовский"),
],
)
def test_deglue_house_marker(raw, expected):
assert _clean_address(raw) == expected
@pytest.mark.parametrize(
"raw",
[
# уже расклеено — идемпотентность
"ул. Евгения Савкова, 29, р-н Академический",
# нормальные адреса без склейки — разделять НЕ надо
"ул. Малышева, 1",
"ул. Токарей, 56к1",
"ул. Ленина, 5",
"ул. Серова, 45",
"г. Екатеринбург, ул. Ленина, 10",
"снт Дружба, уч. 12",
"ул. Декабристов, 16",
],
)
def test_deglue_negative_no_change(raw):
# для этих строк де-слип не должен ничего менять (кроме штатного strip пунктуации)
assert _clean_address(raw) == raw.strip(" ,.\n\t")
def test_deglue_idempotent():
once = _clean_address("ул. Евгения Савкова, 29р-н Академический")
twice = _clean_address(once)
assert once == twice == "ул. Евгения Савкова, 29, р-н Академический"

125
tradein-mvp/backend/tests/test_avito_blockpage.py
View file

@ -1,125 +0,0 @@
"""#823 — детект HTTP-200 block/captcha страницы и parse_failures счётчик.
Тест 1: _is_firewall_page возвращает True на fixtures/avito_blockpage_sample.html.
Тест 2: _parse_html на блок-странице возвращает 0 лотов (нет карточек).
Тест 3: fetch_around бросает AvitoContentBlockedError когда page=1 даёт 0 лотов
(гарантия что silent 0-done НЕ происходит — основной DoD #823).
Тест 4: _dom_card_to_lot инкрементирует parse_failures при сломанной карточке.
Без сети, без БД, без curl_cffi.
"""
import os
from pathlib import Path
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from selectolax.parser import HTMLParser
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
from app.services.scrapers.avito import AvitoScraper, _is_firewall_page
from app.services.scrapers.avito_exceptions import AvitoContentBlockedError
FIXTURES = Path(__file__).parent / "fixtures"
BLOCKPAGE_HTML = (FIXTURES / "avito_blockpage_sample.html").read_text(encoding="utf-8")
SERP_HTML = (FIXTURES / "avito_serp_sample.html").read_text(encoding="utf-8")
# ---------------------------------------------------------------------------
# 1. _is_firewall_page детектит блок-маркер в HTML
# ---------------------------------------------------------------------------
def test_is_firewall_page_detects_block_fixture() -> None:
"""Блок-страница с 'доступ ограничен' и 'firewall-container' → True."""
assert _is_firewall_page(BLOCKPAGE_HTML) is True
def test_is_firewall_page_false_for_normal_serp() -> None:
"""Нормальная SERP страница не является блок-страницей."""
assert _is_firewall_page(SERP_HTML) is False
def test_is_firewall_page_false_for_empty_html() -> None:
assert _is_firewall_page("") is False
# ---------------------------------------------------------------------------
# 2. _parse_html на блок-странице даёт 0 лотов (нет data-marker="item")
# ---------------------------------------------------------------------------
def test_parse_html_block_page_returns_empty_list() -> None:
"""Блок-страница не содержит карточек — парсер возвращает []."""
scraper = AvitoScraper()
lots = scraper._parse_html(BLOCKPAGE_HTML, "https://www.avito.ru/x")
assert lots == []
# ---------------------------------------------------------------------------
# 3. fetch_around raises AvitoContentBlockedError при 0 лотах на page=1
# (гарантия что silent 0-done НЕ происходит — основной DoD #823)
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_fetch_around_raises_on_zero_lots_page1() -> None:
"""HTTP-200 + пустой SERP (блок/captcha) на page=1 → AvitoContentBlockedError.
Мокаем _fetch_serp_html чтобы вернуть блок-страницу без сети.
"""
scraper = AvitoScraper()
with patch.object(
scraper,
"_fetch_serp_html",
new=AsyncMock(return_value=BLOCKPAGE_HTML),
):
with pytest.raises(AvitoContentBlockedError):
await scraper.fetch_around(56.838, 60.605, radius_m=1000, pages=1)
# ---------------------------------------------------------------------------
# 4. parse_failures счётчик инкрементируется при сломанной карточке
# ---------------------------------------------------------------------------
def test_parse_failures_starts_at_zero() -> None:
scraper = AvitoScraper()
assert scraper.parse_failures == 0
def test_parse_failures_not_incremented_on_link_missing() -> None:
"""Карточка без item-title link → ранний return None, НЕ через except."""
scraper = AvitoScraper()
broken_html = '<div data-marker="item" data-item-id="BROKEN"><span>no link</span></div>'
tree = HTMLParser(broken_html)
cards = tree.css('[data-marker="item"]')
result = scraper._dom_card_to_lot(cards[0], "https://www.avito.ru/x")
assert result is None
assert scraper.parse_failures == 0 # ранний return, не exception path
def test_parse_failures_incremented_on_exception() -> None:
"""css_first бросает TypeError → except path → parse_failures++."""
scraper = AvitoScraper()
bad_card = MagicMock()
bad_card.css_first.side_effect = TypeError("simulated DOM error")
bad_card.attributes = {}
result = scraper._dom_card_to_lot(bad_card, "https://www.avito.ru/x")
assert result is None
assert scraper.parse_failures == 1 # #823: счётчик вырос
def test_parse_failures_accumulates_across_cards() -> None:
"""Несколько сломанных карточек аккумулируются в одном счётчике."""
scraper = AvitoScraper()
for _ in range(3):
bad_card = MagicMock()
bad_card.css_first.side_effect = RuntimeError("bad card")
bad_card.attributes = {}
scraper._dom_card_to_lot(bad_card, "https://www.avito.ru/x")
assert scraper.parse_failures == 3

520
tradein-mvp/backend/tests/test_avito_browser_mode.py
View file

@ -1,520 +0,0 @@
"""#901/#1805 — avito SERP fetch via BrowserFetcher (camoufox), curl_cffi fallback.
Tests:
1. browser branch default (#1805): __aenter__ → _browser set AND _cffi fallback armed.
2. curl branch unchanged: _browser is None → _cffi session used directly.
3. firewall-detect in browser-mode: firewall HTML + no rotate URL + no cffi → AvitoBlockedError.
3b. #1805 curl_cffi fallback: browser firewall → curl retry returns HTML / re-raises if also banned.
4. pagination terminates via empty-parse: browser HTML with 0 lots → caller loop breaks.
Без сети, без БД, без curl_cffi.
"""
from __future__ import annotations
import os
from unittest.mock import AsyncMock, MagicMock, patch
import httpx
import pytest
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
from app.services.scrapers.avito import _FIREWALL_MARKERS, AvitoScraper
from app.services.scrapers.avito_exceptions import AvitoBlockedError, AvitoRateLimitedError
from app.services.scrapers.base import ScrapedLot
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
_NORMAL_HTML = "<html><body>normal avito SERP</body></html>"
_FIREWALL_HTML = f"<html><head><title>{_FIREWALL_MARKERS[0]}</title></head></html>"
def _sidecar_error(status: int, error: str) -> httpx.HTTPStatusError:
"""Build a tradein-browser sidecar HTTPStatusError with a real JSON body.
Mirrors what BrowserFetcher.fetch re-raises: an httpx.HTTPStatusError whose
`.response` is intact so `.response.json()["error"]` and `.response.status_code`
work the same as in production.
"""
request = httpx.Request("POST", "http://tradein-browser:3000/fetch")
response = httpx.Response(status, json={"error": error}, request=request)
return httpx.HTTPStatusError(f"{status} error", request=request, response=response)
def _make_mock_browser(html: str = _NORMAL_HTML) -> MagicMock:
"""Return a mock BrowserFetcher context manager that yields html from fetch()."""
mock = MagicMock()
mock.__aenter__ = AsyncMock(return_value=mock)
mock.__aexit__ = AsyncMock(return_value=None)
mock.fetch = AsyncMock(return_value=html)
return mock
# ---------------------------------------------------------------------------
# 1. browser branch chosen
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_aenter_browser_mode_arms_cffi_fallback() -> None:
"""#1805: mode=browser → __aenter__ поднимает BrowserFetcher И curl_cffi-fallback.
Раньше browser-mode оставлял _cffi=None; теперь curl_cffi-сессия строится тут
же как fallback на firewall/пусто (см. _fetch_serp_html_browser).
"""
mock_browser = _make_mock_browser()
scraper = AvitoScraper()
with patch("app.services.scrapers.avito.settings") as mock_settings:
mock_settings.scraper_fetch_mode = "browser"
mock_settings.scraper_proxy_url = None
mock_settings.avito_proxy_rotate_url = None
mock_settings.avito_proxy_max_rotations = 2
with patch(
"app.services.scrapers.avito.BrowserFetcher",
return_value=mock_browser,
):
result = await scraper.__aenter__()
assert result is scraper
assert scraper._browser is mock_browser
# curl_cffi fallback armed (real AsyncSession built, no network/proxy).
assert scraper._cffi is not None
await scraper._cffi.close()
@pytest.mark.asyncio
async def test_aenter_proactive_rotate_when_rotate_url_set() -> None:
"""#1731: rotate_url configured → __aenter__ calls _rotate_ip exactly once at start.
_rotate_ip is mocked (no real changeip HTTP / no 9s sleep). Browser mode used
to keep BaseScraper setup out of the picture.
"""
mock_browser = _make_mock_browser()
scraper = AvitoScraper()
mock_rotate = AsyncMock(return_value=True)
with patch("app.services.scrapers.avito.settings") as mock_settings:
mock_settings.scraper_fetch_mode = "browser"
mock_settings.scraper_proxy_url = None
mock_settings.avito_proxy_rotate_url = "http://proxy.example.com/changeip"
mock_settings.avito_proxy_max_rotations = 4
with patch(
"app.services.scrapers.avito.BrowserFetcher",
return_value=mock_browser,
):
with patch.object(scraper, "_rotate_ip", new=mock_rotate):
result = await scraper.__aenter__()
assert result is scraper
mock_rotate.assert_awaited_once()
@pytest.mark.asyncio
async def test_aenter_no_proactive_rotate_when_rotate_url_none() -> None:
"""#1731: rotate_url is None → __aenter__ must NOT call _rotate_ip."""
mock_browser = _make_mock_browser()
scraper = AvitoScraper()
mock_rotate = AsyncMock(return_value=True)
with patch("app.services.scrapers.avito.settings") as mock_settings:
mock_settings.scraper_fetch_mode = "browser"
mock_settings.scraper_proxy_url = None
mock_settings.avito_proxy_rotate_url = None
mock_settings.avito_proxy_max_rotations = 4
with patch(
"app.services.scrapers.avito.BrowserFetcher",
return_value=mock_browser,
):
with patch.object(scraper, "_rotate_ip", new=mock_rotate):
await scraper.__aenter__()
mock_rotate.assert_not_awaited()
@pytest.mark.asyncio
async def test_aenter_proactive_rotate_failure_does_not_abort() -> None:
"""#1731: a failed proactive rotate (returns False / raises) must NOT abort __aenter__."""
mock_browser = _make_mock_browser()
scraper = AvitoScraper()
with patch("app.services.scrapers.avito.settings") as mock_settings:
mock_settings.scraper_fetch_mode = "browser"
mock_settings.scraper_proxy_url = None
mock_settings.avito_proxy_rotate_url = "http://proxy.example.com/changeip"
mock_settings.avito_proxy_max_rotations = 4
with patch(
"app.services.scrapers.avito.BrowserFetcher",
return_value=mock_browser,
):
# _rotate_ip raises — sweep must proceed (browser still attached)
with patch.object(
scraper, "_rotate_ip", new=AsyncMock(side_effect=RuntimeError("changeip down"))
):
result = await scraper.__aenter__()
assert result is scraper
assert scraper._browser is mock_browser
@pytest.mark.asyncio
async def test_fetch_serp_html_browser_mode_returns_html() -> None:
"""_fetch_serp_html with _browser set returns HTML string, never touches _cffi."""
scraper = AvitoScraper()
mock_browser = _make_mock_browser(_NORMAL_HTML)
scraper._browser = mock_browser
result = await scraper._fetch_serp_html("https://www.avito.ru/test", page=1)
assert result == _NORMAL_HTML
mock_browser.fetch.assert_awaited_once_with("https://www.avito.ru/test")
# _cffi not armed (direct _browser assignment bypasses __aenter__, which would
# otherwise build the curl_cffi fallback session); the browser path never touches it.
assert scraper._cffi is None
# ---------------------------------------------------------------------------
# 2. curl branch unchanged
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_fetch_serp_html_curl_mode_uses_cffi() -> None:
"""Default mode (_browser is None) → _fetch_serp_html goes through curl path."""
scraper = AvitoScraper()
# Mock the curl session returning HTTP 200 with normal HTML
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.text = _NORMAL_HTML
mock_cffi = MagicMock()
mock_cffi.get = AsyncMock(return_value=mock_response)
scraper._cffi = mock_cffi
# _browser must be None (default curl path)
assert scraper._browser is None
result = await scraper._fetch_serp_html("https://www.avito.ru/test", page=1)
assert result == _NORMAL_HTML
mock_cffi.get.assert_awaited_once_with("https://www.avito.ru/test")
# ---------------------------------------------------------------------------
# 3. firewall-detect in browser-mode → AvitoBlockedError (no rotate URL)
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_browser_mode_firewall_raises_blocked_error() -> None:
"""Browser firewall + no rotate URL + no curl fallback (_cffi=None) → AvitoBlockedError."""
scraper = AvitoScraper()
mock_browser = _make_mock_browser(_FIREWALL_HTML)
scraper._browser = mock_browser
scraper._cffi = None # no curl_cffi fallback available (pipeline shared-browser path)
with patch("app.services.scrapers.avito.settings") as mock_settings:
mock_settings.avito_proxy_rotate_url = None # no rotation configured
mock_settings.avito_proxy_max_rotations = 2
with pytest.raises(AvitoBlockedError, match="browser-mode"):
await scraper._fetch_serp_html("https://www.avito.ru/test", page=1)
# ---------------------------------------------------------------------------
# 3b. #1805 — curl_cffi fallback when browser returns firewall
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_browser_firewall_falls_back_to_cffi_returns_html() -> None:
"""#1805: browser firewall + curl_cffi fallback armed → curl returns clean HTML."""
scraper = AvitoScraper()
scraper._browser = _make_mock_browser(_FIREWALL_HTML)
# curl_cffi fallback: HTTP 200 with clean (non-firewall) HTML.
mock_resp = MagicMock()
mock_resp.status_code = 200
mock_resp.text = _NORMAL_HTML
mock_cffi = MagicMock()
mock_cffi.get = AsyncMock(return_value=mock_resp)
scraper._cffi = mock_cffi
with patch("app.services.scrapers.avito.settings") as mock_settings:
mock_settings.avito_proxy_rotate_url = None
mock_settings.avito_proxy_max_rotations = 0
mock_settings.scraper_proxy_url = None
result = await scraper._fetch_serp_html("https://www.avito.ru/test", page=1)
assert result == _NORMAL_HTML
mock_cffi.get.assert_awaited_once_with("https://www.avito.ru/test")
@pytest.mark.asyncio
async def test_browser_firewall_cffi_fallback_also_firewall_raises() -> None:
"""#1805: browser firewall AND curl_cffi fallback also firewall → AvitoBlockedError."""
scraper = AvitoScraper()
scraper._browser = _make_mock_browser(_FIREWALL_HTML)
# curl_cffi fallback: HTTP 200 but ALSO firewall HTML → no escape.
mock_resp = MagicMock()
mock_resp.status_code = 200
mock_resp.text = _FIREWALL_HTML
mock_cffi = MagicMock()
mock_cffi.get = AsyncMock(return_value=mock_resp)
scraper._cffi = mock_cffi
with patch("app.services.scrapers.avito.settings") as mock_settings:
mock_settings.avito_proxy_rotate_url = None
mock_settings.avito_proxy_max_rotations = 0
mock_settings.scraper_proxy_url = None # no backconnect reconnect retries
# curl fallback raises its own AvitoBlockedError (firewall на HTTP 200);
# browser branch re-raises it as a hard-block.
with pytest.raises(AvitoBlockedError):
await scraper._fetch_serp_html("https://www.avito.ru/test", page=1)
@pytest.mark.asyncio
async def test_browser_mode_firewall_rotates_and_retries() -> None:
"""Browser returns firewall on first call, normal HTML after rotation → returns HTML."""
scraper = AvitoScraper()
# First call → firewall; second call → normal HTML
mock_browser = MagicMock()
mock_browser.fetch = AsyncMock(side_effect=[_FIREWALL_HTML, _NORMAL_HTML])
scraper._browser = mock_browser
with patch("app.services.scrapers.avito.settings") as mock_settings:
mock_settings.avito_proxy_rotate_url = "http://proxy.example.com/rotate"
mock_settings.avito_proxy_max_rotations = 1
with patch.object(scraper, "_rotate_ip", new=AsyncMock(return_value=True)):
result = await scraper._fetch_serp_html("https://www.avito.ru/test", page=2)
assert result == _NORMAL_HTML
assert mock_browser.fetch.await_count == 2
# ---------------------------------------------------------------------------
# 4. pagination terminates via empty-parse
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_fetch_city_wide_browser_mode_stops_on_empty_parse() -> None:
"""fetch_city_wide with browser mode: page=1 returns lots, page=2 returns [] → stops.
Verifies the documented invariant: callers break on `not lots` regardless of
fetch mode. _fetch_serp_html is mocked at method level (browser vs curl is
already covered by unit tests above); here we confirm the caller loop behaviour.
"""
scraper = AvitoScraper()
html_p1 = "<html>page1</html>"
html_p2 = "<html>page2</html>"
lot = ScrapedLot(
source="avito",
source_url="https://www.avito.ru/ekaterinburg/kvartiry/test",
source_id="LOT_1",
price_rub=5_000_000,
)
call_count = 0
async def mock_fetch_html(url: str, page: int) -> str:
return html_p1 if page == 1 else html_p2
def mock_parse(html: str, source_url_base: str) -> list[ScrapedLot]:
nonlocal call_count
call_count += 1
return [lot] if "page1" in html else []
with patch.object(scraper, "_fetch_serp_html", side_effect=mock_fetch_html):
with patch.object(scraper, "_parse_html", side_effect=mock_parse):
with patch.object(scraper, "sleep_between_requests", new=AsyncMock()):
result = await scraper.fetch_city_wide(pages=10, delay_override_sec=0)
assert len(result) == 1
assert result[0].source_id == "LOT_1"
# page1 parsed (returned lot) + page2 parsed (returned []) → stopped; page3-10 never called
assert call_count == 2
# ---------------------------------------------------------------------------
# 5. rotation returns False → immediate AvitoBlockedError
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_browser_mode_rotation_returns_false_raises_blocked() -> None:
"""rotate_url set but _rotate_ip() returns False → must immediately raise AvitoBlockedError.
Covers the branch: firewall detected + rotate_url configured + _rotate_ip() == False
(e.g. rotation API timeout/failure) → we cannot recover → blocked.
"""
scraper = AvitoScraper()
scraper._browser = _make_mock_browser(_FIREWALL_HTML)
with patch("app.services.scrapers.avito.settings") as mock_settings:
mock_settings.avito_proxy_rotate_url = "http://proxy/rotate"
mock_settings.avito_proxy_max_rotations = 2
with patch.object(scraper, "_rotate_ip", new=AsyncMock(return_value=False)):
with pytest.raises(AvitoBlockedError, match="browser-mode"):
await scraper._fetch_serp_html("https://avito.ru/test", page=1)
# ---------------------------------------------------------------------------
# 6. #356 — sidecar HTTPStatusError (500/503) → graceful AvitoRateLimitedError
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_sidecar_proxy_too_many_requests_rotates_then_succeeds() -> None:
"""#356: sidecar 500 NS_ERROR_PROXY_TOO_MANY_REQUESTS, rotation available →
after IP rotation a retry succeeds and the clean HTML is returned."""
scraper = AvitoScraper()
mock_browser = MagicMock()
# First fetch raises the soft-ban 500; after rotation the second fetch is clean.
mock_browser.fetch = AsyncMock(
side_effect=[
_sidecar_error(500, "Error: Page.goto: NS_ERROR_PROXY_TOO_MANY_REQUESTS"),
_NORMAL_HTML,
]
)
scraper._browser = mock_browser
mock_rotate = AsyncMock(return_value=True)
with patch("app.services.scrapers.avito.settings") as mock_settings:
mock_settings.avito_proxy_rotate_url = "http://proxy/rotate"
mock_settings.avito_proxy_max_rotations = 2
with patch.object(scraper, "_rotate_ip", new=mock_rotate):
result = await scraper._fetch_serp_html("https://www.avito.ru/test", page=3)
assert result == _NORMAL_HTML
mock_rotate.assert_awaited_once()
assert mock_browser.fetch.await_count == 2
@pytest.mark.asyncio
async def test_sidecar_proxy_too_many_requests_exhausted_raises_rate_limited() -> None:
"""#356: sidecar 500 proxy-too-many with rotation budget exhausted →
AvitoRateLimitedError (NOT a bare HTTPStatusError, NOT AvitoBlockedError)."""
scraper = AvitoScraper()
mock_browser = MagicMock()
mock_browser.fetch = AsyncMock(
side_effect=_sidecar_error(500, "Error: Page.goto: NS_ERROR_PROXY_TOO_MANY_REQUESTS")
)
scraper._browser = mock_browser
mock_rotate = AsyncMock(return_value=True)
with patch("app.services.scrapers.avito.settings") as mock_settings:
mock_settings.avito_proxy_rotate_url = "http://proxy/rotate"
mock_settings.avito_proxy_max_rotations = 2
with patch.object(scraper, "_rotate_ip", new=mock_rotate):
with pytest.raises(AvitoRateLimitedError, match="browser-sidecar"):
await scraper._fetch_serp_html("https://www.avito.ru/test", page=3)
# 2 rotations consumed + 1 final fetch that exhausts the budget.
assert mock_rotate.await_count == 2
assert mock_browser.fetch.await_count == 3
@pytest.mark.asyncio
async def test_sidecar_transient_timeout_exhausts_budget_no_infinite_loop() -> None:
"""#356: sidecar 500 transient (Timeout) repeated past the transient budget with
NO rotations → AvitoRateLimitedError after bounded retries (no infinite loop)."""
scraper = AvitoScraper()
mock_browser = MagicMock()
mock_browser.fetch = AsyncMock(
side_effect=_sidecar_error(500, "TimeoutError: Page.goto: Timeout 60000ms exceeded")
)
scraper._browser = mock_browser
mock_rotate = AsyncMock(return_value=True)
with patch("app.services.scrapers.avito.settings") as mock_settings:
mock_settings.avito_proxy_rotate_url = None # no rotation → transient budget only
mock_settings.avito_proxy_max_rotations = 0
# Patch sleep so the bounded backoff doesn't slow the test.
with patch("app.services.scrapers.avito.asyncio.sleep", new=AsyncMock()):
with patch.object(scraper, "_rotate_ip", new=mock_rotate):
with pytest.raises(AvitoRateLimitedError, match="browser-sidecar"):
await scraper._fetch_serp_html("https://www.avito.ru/test", page=5)
# Transient is NOT a soft-ban → never rotates.
mock_rotate.assert_not_awaited()
# Bounded: _AVITO_SIDECAR_TRANSIENT_RETRIES (2) retries + final attempt = 3.
assert mock_browser.fetch.await_count == 3
@pytest.mark.asyncio
async def test_sidecar_503_browser_unavailable_exhausted_raises_rate_limited() -> None:
"""#356: sidecar 503 'browser unavailable (proxy may be down)' with rotation
exhausted → AvitoRateLimitedError (soft-ban classification via status==503)."""
scraper = AvitoScraper()
mock_browser = MagicMock()
mock_browser.fetch = AsyncMock(
side_effect=_sidecar_error(503, "browser unavailable (proxy may be down)")
)
scraper._browser = mock_browser
mock_rotate = AsyncMock(return_value=True)
with patch("app.services.scrapers.avito.settings") as mock_settings:
mock_settings.avito_proxy_rotate_url = "http://proxy/rotate"
mock_settings.avito_proxy_max_rotations = 1
with patch.object(scraper, "_rotate_ip", new=mock_rotate):
with pytest.raises(AvitoRateLimitedError, match="browser-sidecar"):
await scraper._fetch_serp_html("https://www.avito.ru/test", page=7)
# 1 rotation consumed then exhausted.
assert mock_rotate.await_count == 1
assert mock_browser.fetch.await_count == 2
@pytest.mark.asyncio
async def test_sidecar_transient_then_soft_ban_independent_budgets() -> None:
"""#356: смешанная последовательность transient → soft-ban пинит, что бюджеты
независимы и корректно чередуются в одном while-цикле.
attempt 1: transient Timeout → тратит transient_left (1 из 2), backoff, continue.
attempt 2: too_many_requests → soft-ban, тратит ротацию (1 из 1), continue.
attempt 3: too_many_requests, ротация исчерпана → AvitoRateLimitedError.
"""
scraper = AvitoScraper()
mock_browser = MagicMock()
mock_browser.fetch = AsyncMock(
side_effect=[
_sidecar_error(500, "TimeoutError: Page.goto: Timeout 60000ms exceeded"),
_sidecar_error(500, "Error: Page.goto: NS_ERROR_PROXY_TOO_MANY_REQUESTS"),
_sidecar_error(500, "Error: Page.goto: NS_ERROR_PROXY_TOO_MANY_REQUESTS"),
]
)
scraper._browser = mock_browser
mock_rotate = AsyncMock(return_value=True)
with patch("app.services.scrapers.avito.settings") as mock_settings:
mock_settings.avito_proxy_rotate_url = "http://proxy/rotate"
mock_settings.avito_proxy_max_rotations = 1
with patch("app.services.scrapers.avito.asyncio.sleep", new=AsyncMock()):
with patch.object(scraper, "_rotate_ip", new=mock_rotate):
with pytest.raises(AvitoRateLimitedError, match="browser-sidecar"):
await scraper._fetch_serp_html("https://www.avito.ru/test", page=9)
# 1 transient retry + 1 rotation, оба бюджета тронуты до исчерпания.
assert mock_rotate.await_count == 1
assert mock_browser.fetch.await_count == 3

288
tradein-mvp/backend/tests/test_avito_citywide.py
View file

@ -1,288 +0,0 @@
"""T6 — avito citywide + byrooms sweep mode.
Тесты:
1. _build_citywide_url — без geoCoords/radius, sort=104, page param.
2. _build_rooms_url — room-slug в пути, sort=104, page param.
3. ROOM_SLUGS — 7 категорий, структура кортежей.
4. fetch_city_wide break-on-empty — останавливается когда _parse_html вернул [].
5. fetch_city_wide dedup — дублирующийся source_id не дублируется в результате.
6. fetch_by_rooms break-on-empty per-category — каждая категория прерывается.
7. fetch_by_rooms global dedup — cross-category дедуп по source_id.
8. fetch_around не затронут — сигнатура и URL-режим не изменились.
Без сети, без БД, без curl_cffi.
"""
from __future__ import annotations
import os
from unittest.mock import AsyncMock, patch
import pytest
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
from app.services.scrapers.avito import ROOM_SLUGS, AvitoScraper
from app.services.scrapers.base import ScrapedLot
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _make_lot(source_id: str) -> ScrapedLot:
return ScrapedLot(
source="avito",
source_url=f"https://www.avito.ru/ekaterinburg/kvartiry/{source_id}",
source_id=source_id,
price_rub=5_000_000,
)
# ---------------------------------------------------------------------------
# 1. _build_citywide_url — без geo, s=104
# ---------------------------------------------------------------------------
def test_build_citywide_url_no_geocoords() -> None:
s = AvitoScraper()
url = s._build_citywide_url(page=1)
assert "geoCoords" not in url
assert "radius" not in url
assert "s=104" in url
assert "p=1" in url
assert "ekaterinburg/kvartiry/prodam-ASgBAgICAUSSA8YQ" in url
def test_build_citywide_url_page_param() -> None:
s = AvitoScraper()
assert "p=3" in s._build_citywide_url(page=3)
assert "p=10" in s._build_citywide_url(page=10)
def test_build_citywide_url_default_page() -> None:
s = AvitoScraper()
assert "p=1" in s._build_citywide_url()
# ---------------------------------------------------------------------------
# 2. _build_rooms_url — slug в пути, без geo
# ---------------------------------------------------------------------------
def test_build_rooms_url_contains_slug() -> None:
s = AvitoScraper()
slug = "1-komnatnye-ASgBAgICAkSSA8YQygiAWQ"
url = s._build_rooms_url(slug, page=1)
assert slug in url
assert "geoCoords" not in url
assert "radius" not in url
assert "s=104" in url
assert "p=1" in url
assert "ekaterinburg/kvartiry/prodam/" in url
def test_build_rooms_url_page_param() -> None:
s = AvitoScraper()
slug = "studii-ASgBAgICAkSSA8YQygj~WA"
assert "p=5" in s._build_rooms_url(slug, page=5)
# ---------------------------------------------------------------------------
# 3. ROOM_SLUGS structure
# ---------------------------------------------------------------------------
def test_room_slugs_count() -> None:
assert len(ROOM_SLUGS) == 7
def test_room_slugs_are_pairs() -> None:
for name, slug in ROOM_SLUGS:
assert isinstance(name, str) and name
assert isinstance(slug, str) and slug
# Каждый slug содержит ASgB-кодировку
assert "ASgB" in slug
def test_room_slugs_names_unique() -> None:
names = [name for name, _ in ROOM_SLUGS]
assert len(names) == len(set(names))
# ---------------------------------------------------------------------------
# 4. fetch_city_wide — break-on-empty
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_fetch_city_wide_break_on_empty() -> None:
"""Если _parse_html вернул [] — цикл прерывается, не идёт на следующие страницы."""
s = AvitoScraper()
# page=1 возвращает 2 лота, page=2 возвращает [] → стоп
page_lots: dict[int, list[ScrapedLot]] = {
1: [_make_lot("AAA"), _make_lot("BBB")],
2: [],
}
call_count = 0
async def mock_fetch_html(url: str, page: int) -> str:
return f"<html>page{page}</html>"
def mock_parse(html: str, source_url_base: str) -> list[ScrapedLot]:
nonlocal call_count
call_count += 1
# Derive page number from html string
for p, lots in page_lots.items():
if f"page{p}" in html:
return lots
return []
with patch.object(s, "_fetch_serp_html", side_effect=mock_fetch_html):
with patch.object(s, "_parse_html", side_effect=mock_parse):
with patch.object(s, "sleep_between_requests", new=AsyncMock()):
result = await s.fetch_city_wide(pages=10, delay_override_sec=0)
assert len(result) == 2
assert call_count == 2 # page1 + page2(empty) → stop; page3..10 не вызваны
# ---------------------------------------------------------------------------
# 5. fetch_city_wide — dedup по source_id
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_fetch_city_wide_dedup_by_source_id() -> None:
"""Лот с повторяющимся source_id добавляется только один раз."""
s = AvitoScraper()
dup_lot = _make_lot("SAME_ID")
# pages 1 и 2 возвращают один и тот же lot
pages_data: list[list[ScrapedLot]] = [[dup_lot], [dup_lot], []]
async def mock_html(url: str, page: int) -> str:
return f"<html>{page}</html>"
call_n = 0
def mock_parse(html: str, source_url_base: str) -> list[ScrapedLot]:
nonlocal call_n
idx = call_n
call_n += 1
return pages_data[idx] if idx < len(pages_data) else []
with patch.object(s, "_fetch_serp_html", side_effect=mock_html):
with patch.object(s, "_parse_html", side_effect=mock_parse):
with patch.object(s, "sleep_between_requests", new=AsyncMock()):
result = await s.fetch_city_wide(pages=10, delay_override_sec=0)
assert len(result) == 1
assert result[0].source_id == "SAME_ID"
# ---------------------------------------------------------------------------
# 6. fetch_by_rooms — break-on-empty per category
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_fetch_by_rooms_break_on_empty_per_category() -> None:
"""Каждая категория прерывается на пустой странице, следующая категория продолжается."""
s = AvitoScraper()
slugs = [("1-комн.", "slug-1k"), ("2-комн.", "slug-2k")]
# category 1: page1=2 lots, page2=[] → стоп на этой категории
# category 2: page1=3 lots, page2=[] → стоп
parse_responses: list[list[ScrapedLot]] = [
[_make_lot("C1P1a"), _make_lot("C1P1b")], # cat1 p1
[], # cat1 p2 → break
[_make_lot("C2P1a"), _make_lot("C2P1b"), _make_lot("C2P1c")], # cat2 p1
[], # cat2 p2 → break
]
parse_call = 0
async def mock_html(url: str, page: int) -> str:
return f"<html>{url}{page}</html>"
def mock_parse(html: str, source_url_base: str) -> list[ScrapedLot]:
nonlocal parse_call
idx = parse_call
parse_call += 1
return parse_responses[idx] if idx < len(parse_responses) else []
with patch.object(s, "_fetch_serp_html", side_effect=mock_html):
with patch.object(s, "_parse_html", side_effect=mock_parse):
with patch.object(s, "sleep_between_requests", new=AsyncMock()):
result = await s.fetch_by_rooms(pages=10, delay_override_sec=0, room_slugs=slugs)
assert len(result) == 5 # 2 + 3 (нет дублей)
assert parse_call == 4 # 2 pages per category × 2 categories
# ---------------------------------------------------------------------------
# 7. fetch_by_rooms — global dedup cross-category
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_fetch_by_rooms_global_dedup() -> None:
"""Лот присутствующий в двух категориях добавляется только один раз."""
s = AvitoScraper()
slugs = [("A", "slug-a"), ("B", "slug-b")]
shared = _make_lot("SHARED")
unique_a = _make_lot("UNIQUE_A")
unique_b = _make_lot("UNIQUE_B")
parse_data: list[list[ScrapedLot]] = [
[shared, unique_a], # cat A p1
[], # cat A p2 → break
[shared, unique_b], # cat B p1 (shared — дубль)
[], # cat B p2 → break
]
call_idx = 0
async def mock_html(url: str, page: int) -> str:
return "<html/>"
def mock_parse(html: str, source_url_base: str) -> list[ScrapedLot]:
nonlocal call_idx
i = call_idx
call_idx += 1
return parse_data[i] if i < len(parse_data) else []
with patch.object(s, "_fetch_serp_html", side_effect=mock_html):
with patch.object(s, "_parse_html", side_effect=mock_parse):
with patch.object(s, "sleep_between_requests", new=AsyncMock()):
result = await s.fetch_by_rooms(pages=10, delay_override_sec=0, room_slugs=slugs)
ids = [lot.source_id for lot in result]
assert ids.count("SHARED") == 1
assert "UNIQUE_A" in ids
assert "UNIQUE_B" in ids
assert len(result) == 3
# ---------------------------------------------------------------------------
# 8. fetch_around не затронут — URL содержит geoCoords
# ---------------------------------------------------------------------------
def test_build_web_url_still_has_geocoords() -> None:
"""Существующий _build_web_url сохраняет geoCoords+radius — geo-режим не сломан."""
s = AvitoScraper()
url = s._build_web_url(56.838, 60.605, radius_km=2, page=1)
assert "geoCoords=56.838%2C60.605" in url or "geoCoords=56.838,60.605" in url
assert "radius=2" in url
assert "s=104" in url
assert "p=1" in url
def test_citywide_url_differs_from_geo_url() -> None:
"""citywide URL не содержит geoCoords — это принципиально разные URL."""
s = AvitoScraper()
geo_url = s._build_web_url(56.838, 60.605, radius_km=1, page=1)
cw_url = s._build_citywide_url(page=1)
assert geo_url != cw_url
assert "geoCoords" not in cw_url
assert "geoCoords" in geo_url

400
tradein-mvp/backend/tests/test_avito_detail_parse.py
View file

@ -1,400 +0,0 @@
"""Smoke тесты для avito_detail parser.
Загружает minimal HTML fixture и проверяет selector + conversion logic
без сетевых запросов.
"""
# ruff: noqa: E501 # HTML-фикстуры содержат длинные строки разметки
import pytest
from app.services.scrapers.avito_detail import (
DetailEnrichment,
_parse_height_m,
parse_detail_html,
)
MINIMAL_HTML = """
<html><body>
<div data-marker="item-view/item-id">№ 7986882804 · 18 мая в 13:40 · 1899 просмотров (+ 12 сегодня)</div>
<span itemprop="price" content="11990000">11 990 000 ₽</span>
<div data-marker="item-map-wrapper" data-map-lat="56.797713" data-map-lon="60.609135" data-location-id="654070"></div>
<div data-marker="item-view/item-params">
<ul>
<li>Количество комнат: 3</li>
<li>Общая площадь: 75 м²</li>
<li>Площадь кухни: 15 м²</li>
<li>Этаж: 20 из 26</li>
<li>Балкон или лоджия: лоджия</li>
<li>Ремонт: евро</li>
<li>Способ продажи: свободная</li>
<li>Условия продажи: возможна ипотека</li>
</ul>
<ul>
<li>Тип дома: монолитный</li>
<li>Этажей в доме: 26</li>
</ul>
</div>
<a data-marker="nd-jk-details-button" href="/catalog/houses/ekaterinburg/ul_postovskogo_17a/3171365">Дом</a>
<div data-marker="domoteka-entry-block">
<p data-marker="TeaserData.item">1 собственник</p>
<p data-marker="TeaserData.item">Последняя смена собственника 5 мая 2020</p>
<p data-marker="TeaserData.item">Не найдены ограничения и обременения</p>
<p data-marker="TeaserData.item">Совпадают площадь, адрес и этаж</p>
</div>
<div data-marker="item-view/item-description">Отличная квартира в новом ЖК. Чкаловская 11-15 мин пешком.</div>
</body></html>
"""
SOURCE_URL = "https://www.avito.ru/ekaterinburg/kvartiry/test_7986882804"
def test_minimal_parse() -> None:
result = parse_detail_html(MINIMAL_HTML, SOURCE_URL)
assert isinstance(result, DetailEnrichment)
assert result.item_id == "7986882804"
assert result.price_rub == 11990000
assert result.rooms == 3
assert result.area_m2 == 75.0
assert result.kitchen_area_m2 == 15.0
assert result.floor == 20
assert result.total_floors == 26
assert result.balcony_loggia == "loggia"
assert result.repair_state == "good" # «евро» → good (normalized enum, #621)
assert result.sale_type == "free"
assert result.mortgage_available is True
assert result.house_type == "monolith"
assert result.total_floors_house == 26
assert result.lat == 56.797713
assert result.lon == 60.609135
assert result.avito_location_id == 654070
assert result.house_catalog_url is not None and "3171365" in result.house_catalog_url
assert result.owners_count == 1
assert result.owners_at_least is False
assert result.last_owner_change_date is not None and result.last_owner_change_date.year == 2020
assert result.encumbrances_clean is True
assert result.registry_match is True
assert result.views_total == 1899
assert result.views_today == 12
def test_sale_type_assignment_pereustupka() -> None:
"""«Способ продажи: переуступка» → sale_type='assignment' (ДДУ переуступка,
сигнал новостройки). До фикса map не содержал ключ → None."""
html = """
<html><body>
<div data-marker="item-view/item-id">№ 8043003287</div>
<span itemprop="price" content="9500000">9 500 000 ₽</span>
<div data-marker="item-view/item-params">
<ul>
<li>Количество комнат: 2</li>
<li>Способ продажи: переуступка</li>
</ul>
</div>
</body></html>
"""
result = parse_detail_html(html, "https://www.avito.ru/test_8043003287")
assert result.sale_type == "assignment"
def _html_with_coords(item_id: str, lat: str, lon: str) -> str:
return f"""
<html><body>
<div data-marker="item-view/item-id">№ {item_id}</div>
<span itemprop="price" content="9500000">9 500 000 ₽</span>
<div data-marker="item-map-wrapper" data-map-lat="{lat}" data-map-lon="{lon}" data-location-id="654070"></div>
</body></html>
"""
def test_coord_guard_keeps_ekb_coords() -> None:
"""ЕКБ-координаты (внутри wide-bbox) пишутся как есть → попадут в UPDATE (#1871)."""
html = _html_with_coords("8043000001", "56.797713", "60.609135")
result = parse_detail_html(html, "https://www.avito.ru/test_8043000001")
assert result.lat == 56.797713
assert result.lon == 60.609135
assert result.avito_location_id == 654070
def test_coord_guard_drops_piter_coords() -> None:
"""Питерские координаты (вне wide-bbox) → lat/lon сбрасываются в None (#1871).
Это значение уходит в save_detail_enrichment UPDATE COALESCE(:lat, lat) → NULL,
то есть существующее значение в БД не перетирается мусором, листинг уходит
в geocode-cron путь.
"""
html = _html_with_coords("8043000002", "59.9386", "30.3141") # СПб
result = parse_detail_html(html, "https://www.avito.ru/test_8043000002")
assert result.lat is None
assert result.lon is None
# avito_location_id НЕ трогаем — фикс только на координаты
assert result.avito_location_id == 654070
def test_coord_guard_no_map_element() -> None:
"""Без map-элемента координат нет — guard не падает, lat/lon=None."""
html = """
<html><body>
<div data-marker="item-view/item-id">№ 8043000003</div>
<span itemprop="price" content="9500000">9 500 000 ₽</span>
</body></html>
"""
result = parse_detail_html(html, "https://www.avito.ru/test_8043000003")
assert result.lat is None
assert result.lon is None
def test_metro_extraction() -> None:
result = parse_detail_html(MINIMAL_HTML, SOURCE_URL)
# "Чкаловская 11-15 мин пешком" → metro_stations
assert len(result.metro_stations) >= 1
station = result.metro_stations[0]
assert station["name"] == "Чкаловская"
assert station["min_from"] == 11
assert station["min_to"] == 15
assert station["mode"] == "walk"
def test_owners_at_least_false() -> None:
"""Один собственник (без 'или больше') → owners_at_least=False."""
result = parse_detail_html(MINIMAL_HTML, SOURCE_URL)
assert result.owners_count == 1
assert result.owners_at_least is False
def test_owners_at_least_true() -> None:
"""'3 собственника или больше' → owners_at_least=True."""
html = """
<html><body>
<div data-marker="item-view/item-id">№ 1234567890</div>
<span itemprop="price" content="5000000">5 000 000 ₽</span>
<div data-marker="domoteka-entry-block">
<p data-marker="TeaserData.item">3 собственника или больше</p>
</div>
</body></html>
"""
result = parse_detail_html(html, "https://www.avito.ru/test_1234567890")
assert result.owners_count == 3
assert result.owners_at_least is True
def test_missing_domoteka_all_none() -> None:
"""Если нет domoteka-блока — все 5 полей None."""
html = """
<html><body>
<div data-marker="item-view/item-id">№ 9999999999</div>
<span itemprop="price" content="3000000">3 000 000 ₽</span>
</body></html>
"""
result = parse_detail_html(html, "https://www.avito.ru/test_9999999999")
assert result.owners_count is None
assert result.owners_at_least is None
assert result.last_owner_change_date is None
assert result.encumbrances_clean is None
assert result.registry_match is None
def test_invalid_html_raises_value_error() -> None:
"""HTML без item_id → ValueError."""
html = "<html><body><p>Нет айди</p></body></html>"
with pytest.raises(ValueError, match="Cannot extract item_id"):
parse_detail_html(html, "https://www.avito.ru/test")
def test_house_catalog_url_full() -> None:
"""house_catalog_url должен быть absolute URL."""
result = parse_detail_html(MINIMAL_HTML, SOURCE_URL)
assert result.house_catalog_url is not None
assert result.house_catalog_url.startswith("https://www.avito.ru")
def test_house_catalog_url_secondary_marker_no_nd_jk() -> None:
"""#871: вторичка без nd-jk-details-button → fallback на /catalog/houses/ якорь;
?context-трекинг отброшен (verified live: house-link есть в detail SSR)."""
html = """
<html><body>
<div data-marker="item-view/item-id">№ 7683141312</div>
<span itemprop="price" content="4000000">4 000 000 ₽</span>
<a href="/catalog/houses/ekaterinburg/ul_gotvalda_6k1/142792?context=H4sIabc">
Узнать больше о доме</a>
</body></html>
"""
result = parse_detail_html(html, "https://www.avito.ru/test_7683141312")
assert (
result.house_catalog_url
== "https://www.avito.ru/catalog/houses/ekaterinburg/ul_gotvalda_6k1/142792"
)
def test_bathroom_type_combined() -> None:
html = """
<html><body>
<div data-marker="item-view/item-id">№ 1111111111</div>
<span itemprop="price" content="4000000">4 000 000 ₽</span>
<div data-marker="item-view/item-params">
<ul>
<li>Санузел: совмещенный</li>
<li>Окна: во двор и на улицу</li>
<li>Тип комнат: изолированные</li>
</ul>
</div>
</body></html>
"""
result = parse_detail_html(html, "https://www.avito.ru/test_1111111111")
assert result.bathroom_type == "combined"
assert result.windows_view == "both"
assert result.room_layout == "isolated"
def test_photo_urls_extracted() -> None:
html = """
<html><body>
<div data-marker="item-view/item-id">№ 2222222222</div>
<span itemprop="price" content="7000000">7 000 000 ₽</span>
<div data-marker="image-preview/item"><img src="https://img.avito.ru/img1.jpg"/></div>
<div data-marker="image-preview/item"><img src="https://img.avito.ru/img2.jpg"/></div>
</body></html>
"""
result = parse_detail_html(html, "https://www.avito.ru/test_2222222222")
assert len(result.photo_urls) == 2
assert result.photo_urls[0] == "https://img.avito.ru/img1.jpg"
def test_source_url_preserved() -> None:
result = parse_detail_html(MINIMAL_HTML, SOURCE_URL)
assert result.source_url == SOURCE_URL
# ── repair_state fallback из описания (#622) ──────────────────────────────────
def test_repair_state_structured_wins_over_description() -> None:
"""Структурное «Ремонт: евро» (good) в приоритете даже при сигнале в тексте."""
html = """
<html><body>
<div data-marker="item-view/item-id">№ 3333333333</div>
<span itemprop="price" content="6000000">6 000 000 ₽</span>
<div data-marker="item-view/item-params">
<ul><li>Ремонт: евро</li></ul>
</div>
<div data-marker="item-view/item-description">Квартира без отделки, под чистовую</div>
</body></html>
"""
result = parse_detail_html(html, "https://www.avito.ru/test_3333333333")
assert result.repair_state == "good" # структурное поле, не needs_repair из текста
def test_repair_state_inferred_from_description_when_structured_absent() -> None:
"""Нет блока «Ремонт» → инференс из описания (#622)."""
html = """
<html><body>
<div data-marker="item-view/item-id">№ 4444444444</div>
<span itemprop="price" content="8000000">8 000 000 ₽</span>
<div data-marker="item-view/item-params">
<ul><li>Количество комнат: 2</li></ul>
</div>
<div data-marker="item-view/item-description">Сделан свежий евроремонт, заезжай и живи</div>
</body></html>
"""
result = parse_detail_html(html, "https://www.avito.ru/test_4444444444")
assert result.repair_state == "good"
def test_repair_state_none_when_no_signal() -> None:
"""Нет структурного поля и нет сигнала в тексте → None (не фабрикуем)."""
html = """
<html><body>
<div data-marker="item-view/item-id">№ 5555555555</div>
<span itemprop="price" content="9000000">9 000 000 ₽</span>
<div data-marker="item-view/item-description">Светлая квартира рядом с метро</div>
</body></html>
"""
result = parse_detail_html(html, "https://www.avito.ru/test_5555555555")
assert result.repair_state is None
# ── Новые поля: living_area_m2, ceiling_height_m, finishing ──────────────────
_NEW_FIELDS_HTML = """
<html><body>
<div data-marker="item-view/item-id">№ 6666666666</div>
<span itemprop="price" content="8500000">8 500 000 ₽</span>
<div data-marker="item-view/item-params">
<ul>
<li>Количество комнат: 1</li>
<li>Общая площадь: 43.7\xa0м²</li>
<li>Площадь кухни: 17.1\xa0м²</li>
<li>Жилая площадь: 12.6\xa0м²</li>
<li>Высота потолков: 2.7\xa0м</li>
<li>Отделка: чистовая</li>
<li>Этаж: 6 из 16</li>
</ul>
</div>
</body></html>
"""
def test_new_apartment_fields_populated() -> None:
"""living_area_m2 / ceiling_height_m / finishing парсятся из params-блока."""
result = parse_detail_html(_NEW_FIELDS_HTML, "https://www.avito.ru/test_6666666666")
assert result.living_area_m2 == 12.6
assert result.ceiling_height_m == 2.7
assert result.finishing == "чистовая"
def test_new_fields_absent_when_not_in_params() -> None:
"""Если полей нет в params — None, не дефолт."""
result = parse_detail_html(
"""
<html><body>
<div data-marker="item-view/item-id">№ 7777777777</div>
<span itemprop="price" content="4000000">4 000 000 ₽</span>
<div data-marker="item-view/item-params">
<ul><li>Количество комнат: 2</li></ul>
</div>
</body></html>
""",
"https://www.avito.ru/test_7777777777",
)
assert result.living_area_m2 is None
assert result.ceiling_height_m is None
assert result.finishing is None
def test_finishing_stored_lowercase_stripped() -> None:
"""finishing → lowercase trimmed (' Чистовая ' → 'чистовая')."""
html = """
<html><body>
<div data-marker="item-view/item-id">№ 8888888888</div>
<span itemprop="price" content="5000000">5 000 000 ₽</span>
<div data-marker="item-view/item-params">
<ul><li>Отделка: Чистовая </li></ul>
</div>
</body></html>
"""
result = parse_detail_html(html, "https://www.avito.ru/test_8888888888")
assert result.finishing == "чистовая"
# ── _parse_height_m unit tests ────────────────────────────────────────────────
def test_parse_height_m_dot() -> None:
assert _parse_height_m("2.7\xa0м") == 2.7
def test_parse_height_m_comma() -> None:
assert _parse_height_m("2,7\xa0м") == 2.7
def test_parse_height_m_integer() -> None:
assert _parse_height_m("3 м") == 3.0
def test_parse_height_m_junk() -> None:
assert _parse_height_m("нет данных") is None
def test_parse_height_m_empty() -> None:
assert _parse_height_m("") is None

154
tradein-mvp/backend/tests/test_avito_ekb_geo_filter.py
View file

@ -1,154 +0,0 @@
"""Тест per-card ЕКБ гео-фильтра в AvitoScraper._parse_html.
Avito при редких комбо (4+ комн., дорогой сегмент) добивает выдачу карточками
«по всей России» (Москва, Уфа, Челябинск и т.д.). Каждая карточка несёт
СВОЙ href с city-slug (/ekaterinburg/, /moskva/, ...). Фильтр (за флагом
settings.avito_serp_ekb_only) отбрасывает non-EKB карточки.
Тесты:
1. EKB + non-EKB в SERP → только ЕКБ остаётся при ekb_only=True.
2. ekb_only=False → все карточки проходят (старое поведение).
3. Карточка без распознанного city-slug → пропускается консервативно (не дропается).
4. Только EKB карточки → ничего не дропается (нет лишних логов).
"""
from __future__ import annotations
import os
import pytest
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
def _make_card_html(
item_id: str,
city_slug: str,
price: int = 5_000_000,
title: str = "2-к. квартира, 50 м², 3/9 эт.",
) -> str:
"""Минимальная Avito SERP-карточка с указанным city-slug в href."""
href = f"/{city_slug}/kvartiry/{item_id}-12345"
return f"""
<div data-marker="item" data-item-id="{item_id}">
<a data-marker="item-title" href="{href}">{title}</a>
<meta itemprop="price" content="{price}"/>
<div data-marker="item-location"><p>ул. Тестовая, 1</p></div>
</div>
"""
def _build_serp_html(*cards: str) -> str:
"""Обернуть карточки в минимальный SERP-HTML."""
inner = "".join(cards)
return f"<html><body>{inner}</body></html>"
# ---------------------------------------------------------------------------
# 1. EKB + non-EKB → только ЕКБ при ekb_only=True
# ---------------------------------------------------------------------------
def test_geo_filter_drops_non_ekb_cards(monkeypatch: pytest.MonkeyPatch) -> None:
"""При avito_serp_ekb_only=True карточки других городов отсеиваются."""
from app.core.config import settings
from app.services.scrapers.avito import AvitoScraper
monkeypatch.setattr(settings, "avito_serp_ekb_only", True)
ekb_card = _make_card_html("EKB1", "ekaterinburg")
msk_card = _make_card_html("MSK1", "moskva")
ufa_card = _make_card_html("UFA1", "ufa")
chelya_card = _make_card_html("CHE1", "chelyabinsk")
html = _build_serp_html(ekb_card, msk_card, ufa_card, chelya_card)
scraper = AvitoScraper()
lots = scraper._parse_html(html, source_url_base="https://www.avito.ru/ekaterinburg/kvartiry/")
source_ids = {lot.source_id for lot in lots}
assert "EKB1" in source_ids, "ЕКБ-карточка должна пройти фильтр"
assert "MSK1" not in source_ids, "Московская карточка должна быть отброшена"
assert "UFA1" not in source_ids, "Уфимская карточка должна быть отброшена"
assert "CHE1" not in source_ids, "Челябинская карточка должна быть отброшена"
assert len(lots) == 1
# ---------------------------------------------------------------------------
# 2. ekb_only=False → все карточки проходят
# ---------------------------------------------------------------------------
def test_geo_filter_disabled_passes_all(monkeypatch: pytest.MonkeyPatch) -> None:
"""При avito_serp_ekb_only=False все карточки проходят (старое поведение)."""
from app.core.config import settings
from app.services.scrapers.avito import AvitoScraper
monkeypatch.setattr(settings, "avito_serp_ekb_only", False)
ekb_card = _make_card_html("EKB1", "ekaterinburg")
msk_card = _make_card_html("MSK1", "moskva")
html = _build_serp_html(ekb_card, msk_card)
scraper = AvitoScraper()
lots = scraper._parse_html(html, source_url_base="https://www.avito.ru/ekaterinburg/kvartiry/")
source_ids = {lot.source_id for lot in lots}
assert "EKB1" in source_ids
assert "MSK1" in source_ids
assert len(lots) == 2
# ---------------------------------------------------------------------------
# 3. Карточка без city-slug → консервативно пропускается (не дропается)
# ---------------------------------------------------------------------------
def test_geo_filter_keeps_card_without_city_slug(monkeypatch: pytest.MonkeyPatch) -> None:
"""Карточка с href без city-slug (/kvartiry/...) пропускается консервативно."""
from app.core.config import settings
from app.services.scrapers.avito import AvitoScraper
monkeypatch.setattr(settings, "avito_serp_ekb_only", True)
# href без city-slug: первый сегмент = "kvartiry" → non-city, консервативно берём
no_city_card = """
<div data-marker="item" data-item-id="NOCITY">
<a data-marker="item-title" href="/kvartiry/test-99999">2-к. квартира, 50 м², 3/9 эт.</a>
<meta itemprop="price" content="5000000"/>
<div data-marker="item-location"><p>ул. Тестовая, 1</p></div>
</div>
"""
msk_card = _make_card_html("MSK1", "moskva")
html = _build_serp_html(no_city_card, msk_card)
scraper = AvitoScraper()
lots = scraper._parse_html(html, source_url_base="https://www.avito.ru/ekaterinburg/kvartiry/")
source_ids = {lot.source_id for lot in lots}
assert "NOCITY" in source_ids, "Карточка без city-slug должна быть сохранена консервативно"
assert "MSK1" not in source_ids, "Московская карточка должна быть отброшена"
# ---------------------------------------------------------------------------
# 4. Только EKB карточки → всё проходит
# ---------------------------------------------------------------------------
def test_geo_filter_all_ekb_no_drops(monkeypatch: pytest.MonkeyPatch) -> None:
"""Если все карточки из ЕКБ — dropped=0, все лоты сохраняются."""
from app.core.config import settings
from app.services.scrapers.avito import AvitoScraper
monkeypatch.setattr(settings, "avito_serp_ekb_only", True)
cards = [_make_card_html(f"EKB{i}", "ekaterinburg") for i in range(3)]
html = _build_serp_html(*cards)
scraper = AvitoScraper()
lots = scraper._parse_html(html, source_url_base="https://www.avito.ru/ekaterinburg/kvartiry/")
assert len(lots) == 3
assert all("EKB" in (lot.source_id or "") for lot in lots)

25
tradein-mvp/backend/tests/test_avito_newbuilding_sweep.py
View file

@ -12,6 +12,11 @@ save_listings + mark_done) удалён вместе с `app.services.scrape_pip
Part E1) — эквивалентная regression-coverage теперь в
`test_scraper_kit_pipeline_parity2.py::test_avito_newbuilding_sweep` (kit
`run_avito_newbuilding_sweep`).
Легаси `app.services.scrapers.avito`/`base` удалены (#2397 финальный шаг E, 0
runtime-импортёров) — AvitoScraper/ScrapedLot переведены на kit-эквиваленты
`scraper_kit.providers.avito.serp`/`scraper_kit.base` (byte-identical
strangler-копия).
"""
from __future__ import annotations
@ -24,8 +29,10 @@ import pytest
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
from app.services.scrapers.avito import NOVOSTROYKA_SLUG, AvitoScraper
from app.services.scrapers.base import ScrapedLot
from scraper_kit.base import ScrapedLot
from scraper_kit.providers.avito.serp import NOVOSTROYKA_SLUG, AvitoScraper
from app.services.scraper_adapters import RealScraperConfig
FIXTURE = Path(__file__).parent / "fixtures" / "avito_serp_novostroyka.html"
@ -44,7 +51,7 @@ def _make_lot(source_id: str) -> ScrapedLot:
def test_build_newbuilding_url_contains_slug_and_sort() -> None:
s = AvitoScraper()
s = AvitoScraper(RealScraperConfig())
url = s._build_newbuilding_url(page=1)
assert NOVOSTROYKA_SLUG in url
assert "novostroyka-" in url
@ -56,14 +63,14 @@ def test_build_newbuilding_url_contains_slug_and_sort() -> None:
def test_build_newbuilding_url_page_param() -> None:
s = AvitoScraper()
s = AvitoScraper(RealScraperConfig())
url = s._build_newbuilding_url(page=2)
assert "novostroyka-" in url
assert "p=2" in url
def test_build_newbuilding_url_differs_from_citywide() -> None:
s = AvitoScraper()
s = AvitoScraper(RealScraperConfig())
assert s._build_newbuilding_url(page=1) != s._build_citywide_url(page=1)
@ -72,7 +79,7 @@ def test_build_newbuilding_url_differs_from_citywide() -> None:
def test_parse_novostroyka_fixture_segments_and_newbuilding_id() -> None:
html = FIXTURE.read_text(encoding="utf-8")
s = AvitoScraper()
s = AvitoScraper(RealScraperConfig())
lots = s._parse_html(html, source_url_base=s._build_newbuilding_url(page=1))
assert len(lots) == 2
@ -91,7 +98,7 @@ def test_parse_novostroyka_fixture_segments_and_newbuilding_id() -> None:
@pytest.mark.asyncio
async def test_fetch_newbuildings_break_on_empty() -> None:
s = AvitoScraper()
s = AvitoScraper(RealScraperConfig())
pages_data: list[list[ScrapedLot]] = [[_make_lot("A"), _make_lot("B")], []]
call_n = 0
@ -116,7 +123,7 @@ async def test_fetch_newbuildings_break_on_empty() -> None:
@pytest.mark.asyncio
async def test_fetch_newbuildings_dedup_by_source_id() -> None:
s = AvitoScraper()
s = AvitoScraper(RealScraperConfig())
dup = _make_lot("SAME")
pages_data: list[list[ScrapedLot]] = [[dup], [dup], []]
call_n = 0
@ -141,7 +148,7 @@ async def test_fetch_newbuildings_dedup_by_source_id() -> None:
def test_fetch_city_wide_url_unchanged() -> None:
"""fetch_city_wide остаётся citywide-URL — рефактор не сменил его поведение."""
s = AvitoScraper()
s = AvitoScraper(RealScraperConfig())
assert "novostroyka-" not in s._build_citywide_url(page=1)
assert "prodam-ASgBAgICAUSSA8YQ" in s._build_citywide_url(page=1)

69
tradein-mvp/backend/tests/test_avito_relative_date.py
View file

@ -1,69 +0,0 @@
from datetime import date, timedelta
from app.services.scrapers.avito import _parse_relative_date
def test_days_ago():
assert _parse_relative_date("5 дней назад") == date.today() - timedelta(days=5)
def test_week_ago():
assert _parse_relative_date("2 недели назад") == date.today() - timedelta(days=14)
def test_hour_ago_is_today():
assert _parse_relative_date("3 часа назад") == date.today()
def test_none_passthrough():
assert _parse_relative_date(None) is None
assert _parse_relative_date("") is None
assert _parse_relative_date("random text") is None
# Keyword shortcuts — без \d+ (Avito SERP формат для свежих cards)
def test_today_keyword():
assert _parse_relative_date("сегодня в 12:30") == date.today()
def test_yesterday_keyword():
assert _parse_relative_date("вчера в 18:45") == date.today() - timedelta(days=1)
def test_day_before_yesterday():
assert _parse_relative_date("позавчера") == date.today() - timedelta(days=2)
def test_just_now():
assert _parse_relative_date("только что") == date.today()
# Absolute dates — «18 мая», «5 июня в 12:30»
def test_absolute_date_recent():
today = date.today()
# Take a day-month combo near current (last month) — gradening для CI stability
if today.month >= 2:
target = date(today.year, today.month - 1, 15)
else:
target = date(today.year - 1, 12, 15)
ru_months = {
1: "января", 2: "февраля", 3: "марта", 4: "апреля",
5: "мая", 6: "июня", 7: "июля", 8: "августа",
9: "сентября", 10: "октября", 11: "ноября", 12: "декабря",
}
text = f"{target.day} {ru_months[target.month]} в 12:30"
assert _parse_relative_date(text) == target
def test_absolute_date_future_month_rolls_back():
"""Если today январь, а текст «5 декабря» — это прошлый год."""
today = date.today()
if today.month == 1:
# Today already January, «декабря» → previous year December
result = _parse_relative_date("5 декабря")
assert result == date(today.year - 1, 12, 5)
else:
# Skip when not January — would change semantics
result = _parse_relative_date("5 января")
# Either matches current year January (если уже прошло) или previous
assert result is not None and result.month == 1 and result.day == 5

135
tradein-mvp/backend/tests/test_avito_serp_date.py
View file

@ -1,135 +0,0 @@
"""#726 — avito SERP listing_date extraction (sortTimeStamp JSON + DOM fallback).
Offline: парсит фикстуру через AvitoScraper._parse_html, без сети/curl_cffi/БД.
Регрессия на дрейф разметки: дата у lazy-карточек (без DOM item-date) берётся из
per-item JSON sortTimeStamp; у верхних карточек — из DOM data-marker="item-date".
"""
import os
from datetime import date, datetime, timedelta, timezone
from pathlib import Path
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
from app.services.scrapers.avito import (
AvitoScraper,
_build_sort_timestamp_map,
)
FIXTURE = Path(__file__).parent / "fixtures" / "avito_serp_sample.html"
# Из фикстуры (sortTimeStamp, epoch-ms).
_TS_LAZY = 1700000000000
_TS_TOP = 1700086400000
# #726: дата берётся в MSK (UTC+3), чтобы совпадать с тем, что Avito показывает на
# сайте (sortTimeStamp — UTC epoch, но отображается по Москве).
_MSK = timezone(timedelta(hours=3))
def _expected(ms: int) -> date:
return datetime.fromtimestamp(ms / 1000, tz=_MSK).date()
def test_sort_timestamp_map_built_from_json() -> None:
html = FIXTURE.read_text(encoding="utf-8")
ts_map = _build_sort_timestamp_map(html)
assert ts_map["8043936560"] == _expected(_TS_LAZY)
assert ts_map["8163084615"] == _expected(_TS_TOP)
def test_parse_html_sets_listing_date_for_all_cards() -> None:
"""Обе карточки получают listing_date != None (DoD #726)."""
html = FIXTURE.read_text(encoding="utf-8")
lots = AvitoScraper()._parse_html(html, "https://www.avito.ru/ekaterinburg/kvartiry")
assert len(lots) == 2
by_url = {lot.source_url: lot for lot in lots}
assert all(lot.listing_date is not None for lot in lots), "every card must carry a date"
# Lazy card (нет DOM item-date) → дата из JSON sortTimeStamp.
lazy = next(lot for lot in lots if "8043936560" in (lot.source_url or ""))
assert lazy.listing_date == _expected(_TS_LAZY)
assert by_url # touch to keep linter calm if structure changes
def test_lazy_card_date_comes_from_json_not_dom() -> None:
"""Карточка без data-marker='item-date' всё равно датируется (JSON primary)."""
html = FIXTURE.read_text(encoding="utf-8")
assert 'data-item-id="8043936560"' in html
# у этой карточки в фикстуре НЕТ DOM item-date — проверяем, что дата не потерялась
lots = AvitoScraper()._parse_html(html, "https://www.avito.ru/x")
lazy = next(lot for lot in lots if "8043936560" in (lot.source_url or ""))
assert lazy.listing_date == _expected(_TS_LAZY)
# ── listing_segment по DOM-маркеру item-development-name ───────────────────────
# Каждый avito-URL квартиры содержит /kvartiry/ (вкл. новостройки), поэтому
# сегмент определяется не по URL, а по наличию data-marker="item-development-name"
# (название ЖК/застройщика рендерится только у карточек новостроек).
def _serp_card(item_id: str, *, with_dev_name: bool, with_zhk_anchor: bool = False) -> str:
dev = '<div data-marker="item-development-name">ЖК «Федерация»</div>' if with_dev_name else ""
zhk = (
'<a href="https://zhk-fed-ekaterinburg.avito.ru?context=H4sIAAAA">ЖК Федерация</a>'
if with_zhk_anchor
else ""
)
return f"""
<div data-marker="item" data-item-id="{item_id}">
<a data-marker="item-title" href="/ekaterinburg/kvartiry/2-k_kvartira_{item_id}">
2-к. квартира, 50 м², 5/20 эт.</a>
<meta itemprop="price" content="9500000"/>
{dev}
{zhk}
</div>
"""
def test_segment_novostroyki_when_development_name_present() -> None:
"""Карточка с data-marker='item-development-name' → listing_segment='novostroyki'."""
html = f"<html><body>{_serp_card('8043003287', with_dev_name=True)}</body></html>"
lots = AvitoScraper()._parse_html(html, "https://www.avito.ru/ekaterinburg/kvartiry")
assert len(lots) == 1
assert lots[0].listing_segment == "novostroyki"
def test_segment_vtorichka_when_no_development_name() -> None:
"""Карточка без маркера ЖК → listing_segment='vtorichka' (даже с /kvartiry/ в URL)."""
html = f"<html><body>{_serp_card('8043936560', with_dev_name=False)}</body></html>"
lots = AvitoScraper()._parse_html(html, "https://www.avito.ru/ekaterinburg/kvartiry")
assert len(lots) == 1
assert lots[0].listing_segment == "vtorichka"
# ── newbuilding (ЖК) id/url из zhk-сабдомена ──────────────────────────────────
def test_newbuilding_captured_from_zhk_anchor() -> None:
"""Новостройка с якорем zhk-*.avito.ru → newbuilding_url без ?context, id = slug."""
card = _serp_card("8043003287", with_dev_name=True, with_zhk_anchor=True)
html = f"<html><body>{card}</body></html>"
lots = AvitoScraper()._parse_html(html, "https://www.avito.ru/ekaterinburg/kvartiry")
assert len(lots) == 1
assert lots[0].newbuilding_url == "https://zhk-fed-ekaterinburg.avito.ru"
assert lots[0].newbuilding_id == "fed-ekaterinburg"
def test_newbuilding_none_when_novostroyka_without_anchor() -> None:
"""Новостройка без zhk-якоря → оба поля None (сегмент по dev-name остаётся)."""
card = _serp_card("8043003287", with_dev_name=True, with_zhk_anchor=False)
html = f"<html><body>{card}</body></html>"
lots = AvitoScraper()._parse_html(html, "https://www.avito.ru/ekaterinburg/kvartiry")
assert len(lots) == 1
assert lots[0].listing_segment == "novostroyki"
assert lots[0].newbuilding_id is None
assert lots[0].newbuilding_url is None
def test_newbuilding_none_for_vtorichka() -> None:
"""Вторичка → newbuilding_id/url оба None."""
html = f"<html><body>{_serp_card('8043936560', with_dev_name=False)}</body></html>"
lots = AvitoScraper()._parse_html(html, "https://www.avito.ru/ekaterinburg/kvartiry")
assert len(lots) == 1
assert lots[0].newbuilding_id is None
assert lots[0].newbuilding_url is None

1357
tradein-mvp/backend/tests/test_base_save_listings.py
View file

File diff suppressed because it is too large Load diff

382
tradein-mvp/backend/tests/test_cian_newbuilding.py
View file

@ -1,382 +0,0 @@
"""Tests for cian_newbuilding scraper (Stage 6)."""
from __future__ import annotations
import pytest
from app.services.scrapers.cian_newbuilding import (
NewbuildingEnrichment,
_extract_address,
_extract_advantages,
_extract_chart,
_extract_deadline_year,
_extract_nested_offers,
_extract_reliability_checks,
fetch_newbuilding,
)
# ---- unit tests for extractors ----
def test_extract_chart_primary_shape():
"""realtyValuation primary shape: data.priceDynamics.chart.data.{labels, values}."""
rv_state = {
"data": {
"priceDynamics": {
"chart": {
"data": {
"labels": ["2025-11-01", "2025-12-01", "2026-01-01"],
"values": [13800000, 14100000, 14200000],
}
}
}
}
}
points = _extract_chart(rv_state)
assert len(points) == 3
assert points[0]["month_date"] == "2025-11-01"
assert points[0]["price_per_sqm"] == 13800000.0
assert points[0]["room_count"] == "all"
assert points[0]["prices_type"] == "price"
assert points[0]["period"] == "halfYear"
assert points[2]["month_date"] == "2026-01-01"
assert points[2]["price_per_sqm"] == 14200000.0
def test_extract_chart_normalizes_points():
"""Fallback shape: rv_state.chart[] with {month, price, roomType, period}."""
rv_state = {
"chart": [
{"month": "2025-12-01", "price": 150000, "roomType": "1", "period": "monthly"},
{"month": "2026-01-01", "price": 152000, "roomType": "1", "period": "monthly"},
]
}
points = _extract_chart(rv_state)
assert len(points) == 2
assert points[0]["month_date"] == "2025-12-01"
assert points[0]["price_per_sqm"] == 150000.0
assert points[0]["room_count"] == "1"
assert points[1]["month_date"] == "2026-01-01"
def test_extract_chart_handles_empty():
assert _extract_chart({}) == []
assert _extract_chart({"chart": None}) == []
assert _extract_chart({"data": None}) == []
assert _extract_chart({"data": {"priceDynamics": None}}) == []
def test_extract_chart_handles_none_values():
"""Entries with None month or value are skipped."""
rv_state = {
"chart": [
{"month": "2025-12-01", "price": 150000},
{"month": None, "price": 152000},
{"month": "2026-01-01", "price": None},
]
}
points = _extract_chart(rv_state)
assert len(points) == 1
assert points[0]["month_date"] == "2025-12-01"
def test_extract_chart_truncates_date():
"""Only first 10 chars of date string used (YYYY-MM-DD prefix)."""
rv_state = {
"chart": [
{"month": "2025-12-01T00:00:00Z", "price": 100000},
]
}
points = _extract_chart(rv_state)
assert points[0]["month_date"] == "2025-12-01"
def test_extract_reliability_primary_shape():
"""Primary shape: {checkStatus: {status, title}, details: [{type, title, iconType}]}."""
rel = {
"checkStatus": {
"status": "reliable",
"title": "Проверено",
"mobileTitle": "Проверено нашдом.рф",
"date": "Обновлено 5 февраля 2026",
},
"details": [
{
"title": "Нет в реестре проблемных объектов",
"iconType": "success",
"type": "problemHousesRegistry",
},
{
"title": "Застройщик не банкрот",
"iconType": "success",
"type": "developerBankruptStatus",
},
],
"banner": {"title": "ЖК проверен нашдом.рф"},
}
result = _extract_reliability_checks(rel)
assert len(result) == 1
assert result[0]["check_status"] == "reliable"
assert result[0]["check_name"] == "Проверено"
assert isinstance(result[0]["details"], list)
assert len(result[0]["details"]) == 2
assert result[0]["details"][0]["type"] == "problemHousesRegistry"
def test_extract_reliability_handles_empty():
assert _extract_reliability_checks({}) == []
assert _extract_reliability_checks({"checkStatus": None}) == []
def test_extract_reliability_fallback_shape():
"""Fallback: {checks: [{name, status, details}]}."""
rel = {
"checks": [
{"name": "License", "status": "ok", "details": {"doc": "lic-123"}},
{"name": "Permission", "status": "expired", "details": {}},
]
}
result = _extract_reliability_checks(rel)
assert len(result) == 2
assert result[0]["check_name"] == "License"
assert result[0]["check_status"] == "ok"
assert result[1]["check_name"] == "Permission"
assert result[1]["check_status"] == "expired"
def test_extract_address_handles_string_and_dict():
assert _extract_address({"fullAddress": "Екатеринбург"}) == "Екатеринбург"
assert _extract_address({"address": "ул. Ленина"}) == "ул. Ленина"
assert _extract_address({"address": {"full": "ЕКБ, ул. Ленина 1"}}) == "ЕКБ, ул. Ленина 1"
assert _extract_address({"address": {"display": "ЕКБ"}}) == "ЕКБ"
assert _extract_address({}) is None
def test_extract_address_prefers_full_address():
"""fullAddress wins over address."""
nb = {"fullAddress": "Полный адрес", "address": "Краткий"}
assert _extract_address(nb) == "Полный адрес"
def test_extract_deadline_year_dict_or_int():
assert _extract_deadline_year({"deadline": {"year": 2027}}) == 2027
assert _extract_deadline_year({"buildingDeadline": {"year": 2028}}) == 2028
assert _extract_deadline_year({"deadline": 2029}) == 2029
assert _extract_deadline_year({}) is None
def test_extract_advantages_returns_dicts_only():
nb = {
"advantages": [
{"title": "Отделка", "advantageType": "Finish"},
"not_a_dict",
None,
{"title": "Парковка", "advantageType": "Parking"},
]
}
result = _extract_advantages(nb)
assert len(result) == 2
assert result[0]["advantageType"] == "Finish"
assert result[1]["advantageType"] == "Parking"
def test_extract_advantages_handles_empty():
assert _extract_advantages({}) == []
assert _extract_advantages({"advantages": None}) == []
assert _extract_advantages({"advantages": "invalid"}) == []
def test_extract_nested_offers_developer_rooms_shape():
"""data.total.fromDeveloperRooms[].layouts[] → flat list with room_count."""
offers_state = {
"data": {
"total": {
"fromDeveloperRooms": [
{
"roomType": "oneRoom",
"layouts": [
{"offerId": 100, "price": 10000000},
{"offerId": 101, "price": 10500000},
],
},
{
"roomType": "twoRoom",
"layouts": [
{"offerId": 200, "price": 15000000},
],
},
]
}
}
}
result = _extract_nested_offers(offers_state)
assert len(result) == 3
assert result[0]["room_count"] == "oneRoom"
assert result[0]["offerId"] == 100
assert result[2]["room_count"] == "twoRoom"
assert result[2]["offerId"] == 200
def test_extract_nested_offers_items_shape():
"""items[] fast path."""
offers_state = {"items": [{"offerId": 1}, {"offerId": 2}]}
result = _extract_nested_offers(offers_state)
assert len(result) == 2
assert result[0]["offerId"] == 1
def test_extract_nested_offers_handles_empty():
assert _extract_nested_offers({}) == []
assert _extract_nested_offers({"data": {}}) == []
assert _extract_nested_offers({"data": {"total": {}}}) == []
def test_dataclass_defaults():
"""NewbuildingEnrichment initialises with safe defaults."""
nb = NewbuildingEnrichment()
assert nb.cian_internal_house_id is None
assert nb.realty_valuation_chart == []
assert nb.reliability_checks == []
assert nb.advantages == []
assert nb.builders == []
assert nb.banks == []
assert nb.houses_by_turn == []
assert nb.reviews == []
assert nb.nested_offers == []
assert nb.raw_state is None
assert nb.raw_sister_states == {}
# ---- async integration-style tests ----
# Transport: fetch_newbuilding теперь использует BrowserFetcher (camoufox) вместо
# curl_cffi AsyncSession (#972). Мокируем BrowserFetcher.fetch через monkeypatch.
def _make_browser_fetcher_mock(html: str, monkeypatch: pytest.MonkeyPatch) -> None:
"""Мокирует BrowserFetcher так, чтобы fetch() возвращал заданный html."""
from unittest.mock import AsyncMock, MagicMock
mock_fetcher = MagicMock()
mock_fetcher.fetch = AsyncMock(return_value=html)
mock_fetcher.__aenter__ = AsyncMock(return_value=mock_fetcher)
mock_fetcher.__aexit__ = AsyncMock(return_value=None)
monkeypatch.setattr(
"app.services.scrapers.cian_newbuilding.BrowserFetcher",
lambda *a, **k: mock_fetcher,
)
@pytest.mark.asyncio
async def test_fetch_newbuilding_returns_none_on_no_state(monkeypatch):
"""Если state extraction fails → return None gracefully."""
_make_browser_fetcher_mock("<html></html>", monkeypatch)
monkeypatch.setattr(
"app.services.scrapers.cian_newbuilding.extract_state",
lambda html, mfe, key: None,
)
result = await fetch_newbuilding("https://zhk-test-ekb-i.cian.ru/")
assert result is None
@pytest.mark.asyncio
async def test_fetch_newbuilding_browser_error_propagates(monkeypatch):
"""BrowserFetcher.fetch raises → fetch_newbuilding propagates the exception."""
from unittest.mock import AsyncMock, MagicMock
mock_fetcher = MagicMock()
mock_fetcher.fetch = AsyncMock(side_effect=RuntimeError("browser timeout"))
mock_fetcher.__aenter__ = AsyncMock(return_value=mock_fetcher)
mock_fetcher.__aexit__ = AsyncMock(return_value=None)
monkeypatch.setattr(
"app.services.scrapers.cian_newbuilding.BrowserFetcher",
lambda *a, **k: mock_fetcher,
)
with pytest.raises(RuntimeError, match="browser timeout"):
await fetch_newbuilding("https://zhk-test-ekb-i.cian.ru/")
@pytest.mark.asyncio
async def test_fetch_newbuilding_session_param_ignored(monkeypatch):
"""session= param принимается для backward-compat, но не используется для page-fetch."""
_make_browser_fetcher_mock("<html></html>", monkeypatch)
monkeypatch.setattr(
"app.services.scrapers.cian_newbuilding.extract_state",
lambda html, mfe, key: None,
)
from unittest.mock import MagicMock
sentinel_session = MagicMock(name="should_not_be_called")
# Передаём session= — должно работать без ошибок, session.get НЕ вызывается
result = await fetch_newbuilding("https://zhk-test-ekb-i.cian.ru/", session=sentinel_session)
assert result is None
sentinel_session.get.assert_not_called()
@pytest.mark.asyncio
async def test_fetch_newbuilding_parses_state(monkeypatch):
"""Happy-path: valid state → NewbuildingEnrichment populated."""
fake_state = {
"newbuilding": {
"id": 54016,
"name": "ЖК Тест",
"fullAddress": "Екатеринбург, ул. Теста",
"newbuildingClassName": "comfort",
"transportAccessibilityRate": 7,
"advantages": [{"title": "Парковка", "advantageType": "Parking"}],
"builders": [{"name": "Застройщик Тест"}],
"banks": [{"name": "Банк Тест"}],
"housesByTurn": [{"turn": 1, "houses": []}],
},
"realtyValuation": {
"data": {
"priceDynamics": {
"chart": {
"data": {
"labels": ["2025-11-01", "2025-12-01"],
"values": [14000000, 14500000],
}
}
}
}
},
"reliabilityState": {
"reliability": {
"checkStatus": {"status": "reliable", "title": "Проверено"},
"details": [{"type": "problemHousesRegistry", "iconType": "success"}],
},
},
}
_make_browser_fetcher_mock("<html>fake</html>", monkeypatch)
monkeypatch.setattr(
"app.services.scrapers.cian_newbuilding.extract_state",
lambda html, mfe, key: fake_state,
)
monkeypatch.setattr(
"app.services.scrapers.cian_newbuilding.extract_all_states",
lambda html: {},
)
result = await fetch_newbuilding("https://zhk-test-ekb-i.cian.ru/")
assert result is not None
assert result.cian_internal_house_id == 54016
assert result.name == "ЖК Тест"
assert result.address == "Екатеринбург, ул. Теста"
assert result.building_class == "comfort"
assert result.transport_accessibility_rate == 7
assert len(result.advantages) == 1
assert len(result.builders) == 1
assert len(result.banks) == 1
assert len(result.houses_by_turn) == 1
# realtyValuation chart
assert len(result.realty_valuation_chart) == 2
assert result.realty_valuation_chart[0]["month_date"] == "2025-11-01"
assert result.realty_valuation_chart[0]["price_per_sqm"] == 14000000.0
# reliability
assert len(result.reliability_checks) == 1
assert result.reliability_checks[0]["check_status"] == "reliable"

122
tradein-mvp/backend/tests/test_cian_state_parser.py
View file

@ -1,122 +0,0 @@
"""Tests for cian_state_parser."""
from app.services.scrapers.cian_state_parser import extract_state
def test_extract_state_simple_json():
html = """
<html><body>
<script>
window._cianConfig['frontend-serp'] = window._cianConfig['frontend-serp'] || [];
window._cianConfig['frontend-serp'].push({
key: 'initialState',
value: {"results": {"offers": [{"id": 1}]}},
priority: 1
});
</script>
</body></html>
"""
state = extract_state(html, mfe="frontend-serp", key="initialState")
assert state is not None
assert state["results"]["offers"][0]["id"] == 1
def test_extract_state_json_parse_form():
html = r"""
window._cianConfig['frontend-offer-card'].push({
key: 'defaultState',
value: JSON.parse("{\"offerData\":{\"offer\":{\"cianId\":123}}}"),
priority: 1
});
"""
state = extract_state(html, mfe="frontend-offer-card", key="defaultState")
assert state is not None
assert state["offerData"]["offer"]["cianId"] == 123
def test_extract_state_missing_returns_none():
html = "<html></html>"
assert extract_state(html, mfe="frontend-serp") is None
def test_extract_state_wrong_mfe_returns_none():
html = """
window._cianConfig['other-mfe'].push({key: 'initialState', value: {}, priority: 1});
"""
assert extract_state(html, mfe="frontend-serp", key="initialState") is None
def test_extract_state_backslash_in_description() -> None:
"""Backslash in a listing description must not break page parsing (Part D fix).
The hand-rolled .replace() chain was order-dependent: '\\\\' → '\\' after
'\\"' → '"' left stray backslashes that broke json.loads on the inner payload.
The fix uses json.loads on the outer quoted string atomically.
"""
# Build a JSON payload where the description contains a literal backslash.
# In the wire format (JSON.parse("...")), the inner JSON is double-escaped:
# inner JSON: {"desc": "с \\ слешем"}
# outer string: "{\"desc\": \"с \\\\ слешем\"}"
import json
inner_payload = {"offerData": {"offer": {"cianId": 42, "description": r"описание с \ слешем"}}}
inner_json_str = json.dumps(inner_payload, ensure_ascii=False)
# Simulate what Cian emits: JSON.parse("<double-escaped inner>")
outer_escaped = json.dumps(inner_json_str) # produces "\"...\"" with double escaping
html = f"""
window._cianConfig['frontend-offer-card'].push({{
key: 'defaultState',
value: JSON.parse({outer_escaped}),
priority: 1
}});
"""
state = extract_state(html, mfe="frontend-offer-card", key="defaultState")
assert state is not None, "extract_state returned None — page parse failed"
assert state["offerData"]["offer"]["cianId"] == 42
assert (
"\\" in state["offerData"]["offer"]["description"]
), "Backslash in description was lost during unescape"
# ── #639: новый формат Cian (2026-05) — .concat([{"key":..,"value":..}]) ──
def test_extract_state_concat_format() -> None:
"""Новый формат: .concat([{...}]) с JSON-кавычками вместо .push({key:..})."""
html = (
"<script>window._cianConfig = window._cianConfig || {};"
"window._cianConfig['frontend-serp'] = "
"(window._cianConfig['frontend-serp'] || []).concat("
'[{"key":"projectName","value":"frontend-serp","priority":1000,"filter":{}},'
'{"key":"initialState","value":{"results":{"offers":[{"id":7}]}},"priority":1,"filter":{}}]'
");</script>"
)
state = extract_state(html, mfe="frontend-serp", key="initialState")
assert state is not None
assert state["results"]["offers"][0]["id"] == 7
def test_extract_state_concat_bracket_in_string() -> None:
"""Балансировщик массива не ломается на ']'/'[' внутри JSON-строк."""
html = (
"window._cianConfig['header-frontend'] = "
"(window._cianConfig['header-frontend'] || []).concat("
'[{"key":"initialState","value":{"user":{"isAuthenticated":true,"userId":42,'
'"note":"text with ] and [ inside"}},"priority":1}]'
");"
)
state = extract_state(html, mfe="header-frontend", key="initialState")
assert state is not None
assert state["user"]["isAuthenticated"] is True
assert state["user"]["userId"] == 42
def test_extract_state_legacy_push_still_works() -> None:
"""Старый .push формат остаётся рабочим (fallback)."""
html = (
"window._cianConfig['frontend-serp'].push({"
'key: \'initialState\', value: {"results":{"offers":[{"id":3}]}}, priority: 1});'
)
state = extract_state(html, mfe="frontend-serp", key="initialState")
assert state is not None
assert state["results"]["offers"][0]["id"] == 3

477
tradein-mvp/backend/tests/test_cian_valuation.py
View file

@ -1,477 +0,0 @@
"""Tests for cian_valuation.py — Stage 7 Cian Valuation Calculator scraper."""
import os
# Settings требует DATABASE_URL при инициализации (fail-fast).
# Для offline unit-тестов задаём dummy DSN до любого импорта из app.
os.environ.setdefault("DATABASE_URL", "postgresql://test:test@localhost/test_db")
os.environ.setdefault("COOKIE_ENCRYPTION_KEY", "test-key-for-tests")
from unittest.mock import MagicMock
import pytest
from app.services.scrapers.cian_valuation import (
CianValuationResult,
_is_price_sane,
_parse_change_pct,
_parse_num,
_parse_valuation_state,
compute_cache_key,
estimate_via_cian_valuation,
)
# ---------------------------------------------------------------------------
# compute_cache_key
# ---------------------------------------------------------------------------
def test_cache_key_stable():
"""Нормализация address: lower + strip дают одинаковый ключ."""
k1 = compute_cache_key("ЕКБ ул. Ленина 1", 50.0, 2, 5, "cosmetic", "sale")
k2 = compute_cache_key("екб ул. ленина 1 ", 50.0, 2, 5, "cosmetic", "sale")
assert k1 == k2
def test_cache_key_distinct_on_deal_type():
k1 = compute_cache_key("addr", 50.0, 2, 5, "cosmetic", "sale")
k2 = compute_cache_key("addr", 50.0, 2, 5, "cosmetic", "rent")
assert k1 != k2
def test_cache_key_distinct_on_area():
k1 = compute_cache_key("addr", 50.0, 2, 5, "cosmetic", "sale")
k2 = compute_cache_key("addr", 51.0, 2, 5, "cosmetic", "sale")
assert k1 != k2
def test_cache_key_is_hex_string():
k = compute_cache_key("addr", 50.0, 2, 5, "cosmetic", "sale")
assert len(k) == 64
assert all(c in "0123456789abcdef" for c in k)
# ---------------------------------------------------------------------------
# _parse_num
# ---------------------------------------------------------------------------
def test_parse_num_float():
assert _parse_num(5.5) == 5.5
def test_parse_num_str():
assert _parse_num("100") == 100.0
def test_parse_num_none():
assert _parse_num(None) is None
def test_parse_num_bad_str():
assert _parse_num("bad") is None
def test_parse_num_int():
assert _parse_num(0) == 0.0
# ---------------------------------------------------------------------------
# _parse_change_pct
# ---------------------------------------------------------------------------
def test_parse_change_pct_positive():
assert _parse_change_pct("8,3%") == pytest.approx(8.3)
def test_parse_change_pct_negative():
assert _parse_change_pct("-2,5%") == pytest.approx(-2.5)
def test_parse_change_pct_empty():
assert _parse_change_pct("") is None
def test_parse_change_pct_dot_format():
assert _parse_change_pct("1.5%") == pytest.approx(1.5)
# ---------------------------------------------------------------------------
# _parse_valuation_state — основные сценарии
# ---------------------------------------------------------------------------
def _make_state(
is_authenticated: bool = True,
user_id: int = 12345,
sale_price: int = 7500000,
sale_accuracy: int = 94,
rent_price: int = 35000,
rent_accuracy: int = 93,
chart_change: str = "increase",
chart_change_value: str = "8,3%",
house_id: int = 54016,
) -> dict:
return {
"user": {"isAuthenticated": is_authenticated, "userId": user_id},
"estimation": {
"sale": {
"data": {
"price": sale_price,
"accuracy": sale_accuracy,
"priceFrom": 6900000,
"priceTo": 8100000,
"priceSqm": 150000,
"dealType": "sale",
"filtersHash": "abc123",
},
"isError": False,
"isFetching": False,
},
"rent": {
"data": {
"price": rent_price,
"accuracy": rent_accuracy,
"priceFrom": 32000,
"priceTo": 38000,
"taxPrice": 1400,
"dealType": "rent",
},
},
},
"estimationChart": {
"data": {
"title": {
"title": "Стоимость этой квартиры увеличилась на {{change}} за полгода",
"changeValue": chart_change_value,
"change": chart_change,
},
"chartData": {
"maxXValue": 1777593600000,
"minXValue": 1761955200000,
"data": [
{"date": 1761955200000, "price": 6033438, "priceFormatted": "6,0 млн ₽"},
{"date": 1764547200000, "price": 6013573, "priceFormatted": "6,0 млн ₽"},
{"date": 1767225600000, "price": 6116587, "priceFormatted": "6,1 млн ₽"},
{"date": 1769904000000, "price": 6236983, "priceFormatted": "6,2 млн ₽"},
{"date": 1772323200000, "price": 6232172, "priceFormatted": "6,2 млн ₽"},
{"date": 1775001600000, "price": 6331112, "priceFormatted": "6,3 млн ₽"},
{"date": 1777593600000, "price": 6532872, "priceFormatted": "6,5 млн ₽"},
],
},
},
},
"houseInfo": {
"data": {
"items": [
{"title": "Год постройки", "value": 2005},
{"title": "Тип дома", "value": "Кирпичный"},
],
"isRenovation": False,
"isEmergency": False,
},
},
"filters": {"houseId": house_id, "roomsCount": 2},
"managementCompany": {
"data": {
"name": "ТСЖ «Учителей, 18»",
"address": "ул. Учителей, 18",
"phones": ["+7 (343) 000-00-00"],
"email": "tsj@example.ru",
"chiefName": "Иванов И.И.",
"openingHours": [{"Пн–Пт": ["9:00", "18:00"]}],
},
},
"page": {"pageType": "valuationPage"},
}
def test_parse_valuation_state_extracts_sale():
state = _make_state()
result = _parse_valuation_state(state)
assert result.is_authenticated is True
assert result.user_id == 12345
assert result.sale_price_rub == 7500000.0
assert result.sale_accuracy == 94.0
assert result.sale_price_from == 6900000.0
assert result.sale_price_to == 8100000.0
assert result.sale_price_sqm == 150000.0
assert result.filters_hash == "abc123"
def test_parse_valuation_state_extracts_rent():
state = _make_state()
result = _parse_valuation_state(state)
assert result.rent_price_rub == 35000.0
assert result.rent_accuracy == 93.0
assert result.rent_price_from == 32000.0
assert result.rent_price_to == 38000.0
assert result.rent_tax_price == 1400.0
def test_parse_valuation_state_chart_7_points():
state = _make_state()
result = _parse_valuation_state(state)
assert len(result.chart) == 7
# First point: unix ms 1761955200000 → 2025-11-01 (UTC)
assert result.chart[0]["month_date"] == "2025-11-01"
assert result.chart[0]["price"] == 6033438.0
def test_parse_valuation_state_chart_change_increase():
state = _make_state(chart_change="increase", chart_change_value="8,3%")
result = _parse_valuation_state(state)
assert result.chart_change_pct == pytest.approx(8.3)
assert result.chart_change_direction == "increase"
def test_parse_valuation_state_chart_change_decrease():
state = _make_state(chart_change="decrease", chart_change_value="-2,5%")
result = _parse_valuation_state(state)
assert result.chart_change_pct == pytest.approx(-2.5)
assert result.chart_change_direction == "decrease"
def test_parse_valuation_state_house_info():
state = _make_state()
result = _parse_valuation_state(state)
assert len(result.house_info) == 2
assert result.house_info[0]["title"] == "Год постройки"
assert result.house_info[0]["value"] == 2005
def test_parse_valuation_state_external_house_id():
state = _make_state(house_id=54016)
result = _parse_valuation_state(state)
assert result.external_house_id == 54016
def test_parse_valuation_state_management_company():
state = _make_state()
result = _parse_valuation_state(state)
assert result.management_company is not None
assert result.management_company["name"] == "ТСЖ «Учителей, 18»"
assert result.management_company["chiefName"] == "Иванов И.И."
def test_parse_valuation_state_no_auth():
state = _make_state(is_authenticated=False)
result = _parse_valuation_state(state)
assert result.is_authenticated is False
# Values still parsed — auth check is done by caller, not parser
assert result.sale_price_rub == 7500000.0
def test_parse_valuation_state_empty_estimation():
state = {"user": {"isAuthenticated": True, "userId": 1}, "estimation": {}}
result = _parse_valuation_state(state)
assert result.sale_price_rub is None
assert result.rent_price_rub is None
assert result.chart == []
def test_parse_valuation_state_missing_management_company():
state = _make_state()
del state["managementCompany"]
result = _parse_valuation_state(state)
assert result.management_company is None
# ---------------------------------------------------------------------------
# CianValuationResult dataclass defaults
# ---------------------------------------------------------------------------
def test_dataclass_defaults():
r = CianValuationResult()
assert r.chart == []
assert r.house_info == []
assert r.is_authenticated is False
assert r.sale_price_rub is None
assert r.management_company is None
# ---------------------------------------------------------------------------
# estimate_via_cian_valuation — integration (mocked)
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_estimate_returns_none_when_no_cookies(monkeypatch):
"""Если load_session returns None → graceful return None без crash."""
monkeypatch.setattr(
"app.services.scrapers.cian_valuation.load_session",
lambda db: None,
)
db = MagicMock()
# cache miss
db.execute.return_value.mappings.return_value.first.return_value = None
result = await estimate_via_cian_valuation(
db,
address="ЕКБ",
total_area=50.0,
rooms_count=2,
floor=5,
use_cache=True,
)
assert result is None
@pytest.mark.asyncio
async def test_cache_hit_returns_cached(monkeypatch):
"""Если cache hit → return без fetching (load_session не вызывается)."""
db = MagicMock()
cached_row = {
"raw_payload": {"cached": True},
"sale_price_rub": 7000000,
"sale_accuracy": 80,
"rent_price_rub": 30000,
"rent_accuracy": 65,
"chart": [{"month_date": "2026-05-01", "price": 145000}],
"chart_change_pct": 0.5,
"chart_change_direction": "increase",
"external_house_id": 999,
"filters_hash": "xyz789",
"low_price": 6500000, # required by _load_from_cache → sale_price_from
"high_price": 7500000, # required by _load_from_cache → sale_price_to
}
db.execute.return_value.mappings.return_value.first.return_value = cached_row
# load_session должен НЕ вызываться при cache hit
monkeypatch.setattr(
"app.services.scrapers.cian_valuation.load_session",
lambda db: pytest.fail("load_session should not be called on cache hit"),
)
result = await estimate_via_cian_valuation(
db,
address="ЕКБ",
total_area=50.0,
rooms_count=2,
floor=5,
use_cache=True,
)
assert result is not None
assert result.sale_price_rub == 7000000.0
assert result.rent_price_rub == 30000.0
assert result.external_house_id == 999
assert result.filters_hash == "xyz789"
assert result.is_authenticated is True # cache implies authenticated
assert len(result.chart) == 1
# ---------------------------------------------------------------------------
# Mera-audit fix-1: _is_price_sane — bounds validation
# ---------------------------------------------------------------------------
def test_price_sane_valid_price() -> None:
"""Обычная цена ЕКБ-квартиры — проходит санитарную проверку."""
r = CianValuationResult(
sale_price_rub=7_500_000, sale_price_from=6_900_000, sale_price_to=8_100_000
)
assert _is_price_sane(r) is True
def test_price_sane_none_price() -> None:
"""sale_price_rub=None — проверка цены не судит None (graceful)."""
r = CianValuationResult(sale_price_rub=None)
assert _is_price_sane(r) is True
def test_price_sane_below_min_returns_false() -> None:
"""Цена 999_999 < дефолтный min (500_000... нет, 999_999 > 500_000).
Тест: цена ниже min=500_000 — задаём 100_000 → False."""
r = CianValuationResult(sale_price_rub=100_000)
assert _is_price_sane(r) is False
def test_price_sane_above_max_returns_false() -> None:
"""Цена 9_999_999_999 (> 500М) → False."""
r = CianValuationResult(sale_price_rub=9_999_999_999)
assert _is_price_sane(r) is False
def test_price_sane_at_min_boundary() -> None:
"""Цена ровно на нижней границе дефолта (500_000) → True."""
r = CianValuationResult(sale_price_rub=500_000)
assert _is_price_sane(r) is True
def test_price_sane_at_max_boundary() -> None:
"""Цена ровно на верхней границе дефолта (500_000_000) → True."""
r = CianValuationResult(sale_price_rub=500_000_000)
assert _is_price_sane(r) is True
def test_price_sane_inverted_range_returns_false() -> None:
"""low > high (инвертированный диапазон) → False."""
r = CianValuationResult(
sale_price_rub=7_000_000,
sale_price_from=8_000_000, # from > to
sale_price_to=6_000_000,
)
assert _is_price_sane(r) is False
def test_price_sane_negative_low_returns_false() -> None:
"""sale_price_from < 0 → False."""
r = CianValuationResult(sale_price_rub=7_000_000, sale_price_from=-100)
assert _is_price_sane(r) is False
def test_price_sane_negative_high_returns_false() -> None:
"""sale_price_to < 0 → False."""
r = CianValuationResult(sale_price_rub=7_000_000, sale_price_to=-1)
assert _is_price_sane(r) is False
def test_price_sane_none_bounds_ok() -> None:
"""sale_price_from/to=None при валидной цене → True (bounds не обязательны)."""
r = CianValuationResult(sale_price_rub=7_000_000, sale_price_from=None, sale_price_to=None)
assert _is_price_sane(r) is True
@pytest.mark.asyncio
async def test_estimate_use_cache_false_skips_cache(monkeypatch):
"""use_cache=False → cache lookup не вызывается, идём сразу в load_session."""
load_session_called = []
def fake_load_session(db: object) -> None:
load_session_called.append(True)
return None # simulate no cookies → None result
monkeypatch.setattr(
"app.services.scrapers.cian_valuation.load_session",
fake_load_session,
)
db = MagicMock()
result = await estimate_via_cian_valuation(
db,
address="ЕКБ",
total_area=50.0,
rooms_count=2,
floor=5,
use_cache=False,
)
assert result is None
assert load_session_called # load_session был вызван (cache bypass worked)
# db.execute НЕ должен быть вызван для cache lookup
db.execute.assert_not_called()

23
tradein-mvp/backend/tests/test_estimator_cian_integration.py
View file

@ -1,4 +1,11 @@
"""Tests for Cian Valuation integration in estimator.py (Stage 9)."""
"""Tests for Cian Valuation integration in estimator.py (Stage 9).
Легаси `app.services.scrapers.cian_valuation` удалён (#2397 финальный шаг E, 0
runtime-импортёров) — переведено на kit-эквивалент `scraper_kit.providers.cian.
valuation`, тот же тип/функции, что `estimator.py` реально использует.
`estimate_via_cian_valuation` требует mandatory `config=` (kit strangler DI,
#2337) — добавлен `RealScraperConfig()` там, где легаси-сигнатура его не требовала.
"""
import os
@ -8,8 +15,7 @@ os.environ.setdefault("DATABASE_URL", "postgresql://test:test@localhost/test_db"
from unittest.mock import AsyncMock, MagicMock, patch
import anyio
from app.services.scrapers.cian_valuation import CianValuationResult
from scraper_kit.providers.cian.valuation import CianValuationResult
def _fake_cian_result(**kwargs) -> CianValuationResult:
@ -31,7 +37,7 @@ def _fake_cian_result(**kwargs) -> CianValuationResult:
def test_cian_valuation_cache_key_deterministic() -> None:
"""compute_cache_key produces same hash for identical inputs."""
from app.services.scrapers.cian_valuation import compute_cache_key
from scraper_kit.providers.cian.valuation import compute_cache_key
k1 = compute_cache_key("ЕКБ, ул. Учителей, 18", 38.8, 1, 4, "cosmetic", "sale")
k2 = compute_cache_key("ЕКБ, ул. Учителей, 18", 38.8, 1, 4, "cosmetic", "sale")
@ -40,7 +46,7 @@ def test_cian_valuation_cache_key_deterministic() -> None:
def test_cian_valuation_cache_key_differs_for_different_inputs() -> None:
from app.services.scrapers.cian_valuation import compute_cache_key
from scraper_kit.providers.cian.valuation import compute_cache_key
a = compute_cache_key("addr A", 38.8, 1, 4, "cosmetic", "sale")
b = compute_cache_key("addr B", 38.8, 1, 4, "cosmetic", "sale")
@ -54,19 +60,22 @@ def test_cian_valuation_cache_key_differs_for_different_inputs() -> None:
def test_estimate_via_cian_returns_none_when_no_cookies() -> None:
"""estimate_via_cian_valuation returns None when no cookies in DB (graceful)."""
from app.services.scraper_adapters import RealScraperConfig
db = MagicMock()
# cache miss
db.execute.return_value.mappings.return_value.first.return_value = None
async def _run() -> None:
with patch(
"app.services.scrapers.cian_valuation.load_session",
"scraper_kit.providers.cian.valuation.load_session",
return_value=None, # no cookies stored
):
from app.services.scrapers.cian_valuation import estimate_via_cian_valuation
from scraper_kit.providers.cian.valuation import estimate_via_cian_valuation
result = await estimate_via_cian_valuation(
db,
config=RealScraperConfig(),
address="ЕКБ, ул. Учителей, 18",
total_area=38.8,
rooms_count=1,

12
tradein-mvp/backend/tests/test_extval_house_id_write_path.py
View file

@ -7,11 +7,11 @@ Best-effort: house_id=None → колонка NULL, сохранение НЕ п
DB — MagicMock, записываем параметры db.execute (никакого реального Postgres,
как в test_yandex_valuation_save / test_backfill_house_coords).
Легаси `app.services.scrapers.yandex_valuation` удалён (#2277 финальный шаг
scraper_kit-миграции) — `_yandex_result()` переведён на kit-эквивалент
`scraper_kit.providers.yandex.valuation` (тот же тип, что `estimator.py` реально
использует через `YandexValuationScraper`, который здесь пропатчен). `cian_valuation`
остаётся легаси (не мигрирован) — его импорт не тронут.
Легаси `app.services.scrapers.yandex_valuation` / `cian_valuation` удалены (#2277 /
#2397 финальный шаг E, 0 runtime-импортёров) — `_yandex_result()` и cian-импорты
переведены на kit-эквиваленты `scraper_kit.providers.yandex.valuation` /
`scraper_kit.providers.cian.valuation` (те же типы, что `estimator.py` реально
использует).
"""
from __future__ import annotations
@ -23,7 +23,7 @@ os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/
from unittest.mock import AsyncMock, MagicMock, patch
from app.services.scrapers.cian_valuation import CianValuationResult, _save_to_cache
from scraper_kit.providers.cian.valuation import CianValuationResult, _save_to_cache
# ---------------------------------------------------------------------------
# Cian valuation — _save_to_cache прокидывает house_id в INSERT params

4
tradein-mvp/backend/tests/test_geo_precision.py
View file

@ -177,7 +177,7 @@ def test_fetch_analogs_tier_h_excludes_city_precision() -> None:
def test_scraped_lot_has_geo_precision_field() -> None:
"""ScrapedLot has geo_precision field with default None."""
from app.services.scrapers.base import ScrapedLot
from scraper_kit.base import ScrapedLot
lot = ScrapedLot(
source="cian",
@ -190,7 +190,7 @@ def test_scraped_lot_has_geo_precision_field() -> None:
def test_scraped_lot_accepts_city_geo_precision() -> None:
"""ScrapedLot accepts geo_precision='city'."""
from app.services.scrapers.base import ScrapedLot
from scraper_kit.base import ScrapedLot
lot = ScrapedLot(
source="n1",

127
tradein-mvp/backend/tests/test_house_type_normalizer.py
View file

@ -1,127 +0,0 @@
"""Unit-тесты для house_type_normalizer (#2007, #2009).
Покрывают:
- normalize_house_type(): yandex SCREAMING + cian camelCase + avito русские
лейблы вокабуляры → канон,
- идемпотентность (normalize(normalize(x)) == normalize(x)),
- None / '' / whitespace / 'other' / unknown → None.
"""
import pytest
from app.services.scrapers.house_type_normalizer import (
_CANON,
normalize_house_type,
)
# ── yandex SERP (SCREAMING_SNAKE buildingType) ───────────────────────────────
@pytest.mark.parametrize(
("raw", "expected"),
[
("MONOLIT", "monolith"),
("BRICK", "brick"),
("PANEL", "panel"),
("MONOLIT_BRICK", "monolith_brick"),
("BLOCK", "block"),
("WOOD", "wood"),
],
)
def test_yandex_screaming_vocab(raw: str, expected: str) -> None:
assert normalize_house_type(raw) == expected
# ── cian SERP (camelCase materialType) — reuse #2008 ─────────────────────────
@pytest.mark.parametrize(
("raw", "expected"),
[
("monolith", "monolith"),
("brick", "brick"),
("panel", "panel"),
("block", "block"),
("wood", "wood"),
("monolithBrick", "monolith_brick"),
("gasSilicateBlock", "block"),
("aerocreteBlock", "block"),
("foamConcreteBlock", "block"),
("stalin", "brick"),
],
)
def test_cian_camelcase_vocab(raw: str, expected: str) -> None:
assert normalize_house_type(raw) == expected
# ── avito detail page («Тип дома», русские лейблы) — reuse #2009 ────────────
@pytest.mark.parametrize(
("raw", "expected"),
[
("монолитный", "monolith"),
("панельный", "panel"),
("кирпичный", "brick"),
("монолитно-кирпичный", "monolith_brick"),
("блочный", "block"),
("деревянный", "wood"),
],
)
def test_avito_russian_vocab(raw: str, expected: str) -> None:
assert normalize_house_type(raw) == expected
# ── регистронезависимость ────────────────────────────────────────────────────
@pytest.mark.parametrize(
("raw", "expected"),
[
("monolit", "monolith"), # lower yandex-token
("Brick", "brick"),
("MONOLITHBRICK", "monolith_brick"),
(" PANEL ", "panel"), # окружающие пробелы
],
)
def test_case_insensitive_and_strip(raw: str, expected: str) -> None:
assert normalize_house_type(raw) == expected
# ── идемпотентность ──────────────────────────────────────────────────────────
@pytest.mark.parametrize("canon", sorted(_CANON))
def test_canonical_passthrough(canon: str) -> None:
"""Уже каноничное значение возвращается as-is (в т.ч. monolith_brick)."""
assert normalize_house_type(canon) == canon
@pytest.mark.parametrize(
"raw",
[
"MONOLIT",
"MONOLIT_BRICK",
"monolithBrick",
"gasSilicateBlock",
"stalin",
"monolith_brick",
"panel",
"other",
None,
"",
],
)
def test_idempotency(raw: str | None) -> None:
"""normalize(normalize(x)) == normalize(x) для любого входа."""
once = normalize_house_type(raw)
assert normalize_house_type(once) == once
# ── None / unknown → None ────────────────────────────────────────────────────
@pytest.mark.parametrize("raw", [None, "", " ", "other", "OTHER", "unknown", " keramzit "])
def test_unknown_returns_none(raw: str | None) -> None:
"""Неизвестное / 'other' / пустое → None (НЕ 'other'): NULL нейтрально для estimator."""
assert normalize_house_type(raw) is None

22
tradein-mvp/backend/tests/test_pipeline_browser_routing.py
View file

@ -16,6 +16,10 @@ Legacy `run_avito_pipeline` browser/curl-mode integration tests (было ран
`fetch_house_catalog` ретаргетирован на `scraper_kit.providers.avito.houses` (#2397
Part E2) — legacy `app.services.scrapers.avito_houses` удалён (0 runtime-импортёров,
kit — единственный живой путь).
`fetch_detail`/`AvitoScraper`/`AvitoBlockedError` ретаргетированы на
`scraper_kit.providers.avito.detail`/`serp`/`avito_exceptions` (#2397 финальный шаг
E) — legacy `app.services.scrapers.avito*` удалён (0 runtime-импортёров).
"""
from __future__ import annotations
@ -30,12 +34,10 @@ import pytest
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
from scraper_kit.avito_exceptions import AvitoBlockedError as KitAvitoBlockedError
from scraper_kit.avito_exceptions import AvitoBlockedError
from scraper_kit.providers.avito.detail import fetch_detail
from scraper_kit.providers.avito.houses import fetch_house_catalog
from app.services.scrapers.avito_detail import fetch_detail
from app.services.scrapers.avito_exceptions import AvitoBlockedError
FIXTURES = Path(__file__).parent / "fixtures"
# Минимальный HTML для fetch_detail — достаточен для parse_detail_html
@ -214,7 +216,7 @@ async def test_fetch_house_catalog_browser_firewall_raises() -> None:
"""Firewall HTML через browser path → AvitoBlockedError (kit)."""
mock_fetcher = make_mock_fetcher(FIREWALL_HTML)
with pytest.raises(KitAvitoBlockedError, match="browser-mode"):
with pytest.raises(AvitoBlockedError, match="browser-mode"):
await fetch_house_catalog(
"/catalog/houses/ekaterinburg/ul_test/9999",
browser_fetcher=mock_fetcher,
@ -238,8 +240,10 @@ async def test_fetch_detail_curl_mode_no_browser_fetcher() -> None:
mock_session.get = AsyncMock(return_value=mock_resp)
mock_session.close = AsyncMock()
# Kit _build_detail_session delegates to scraper_kit.providers._base.build_document_session
# (shared helper), which constructs AsyncSession there — not in the detail module itself.
with _patch(
"app.services.scrapers.avito_detail.AsyncSession",
"scraper_kit.providers._base.AsyncSession",
return_value=mock_session,
):
result = await fetch_detail("/ekaterinburg/kvartiry/test_99887766-1234")
@ -259,9 +263,11 @@ async def test_fetch_serp_html_routes_through_browser() -> None:
SERP routing on top of #901's browser branch."""
from unittest.mock import AsyncMock
from app.services.scrapers.avito import AvitoScraper
from scraper_kit.providers.avito.serp import AvitoScraper
scraper = AvitoScraper()
from app.services.scraper_adapters import RealScraperConfig
scraper = AvitoScraper(RealScraperConfig())
assert scraper._cffi is None
scraper._browser = AsyncMock()
scraper._browser.fetch = AsyncMock(return_value="<html><body>real serp listing</body></html>")

188
tradein-mvp/backend/tests/test_repair_state_normalizer.py
View file

@ -1,188 +0,0 @@
"""Unit-тесты для repair_state_normalizer (#621, #622).
Покрывают:
- normalize_repair_state(): все known raw → enum, idempotency, None-guard (#621).
- infer_repair_state_from_text(): русские фразы → enum, приоритет сигналов,
unknown → None, None-guard (#622).
"""
import logging
import pytest
from app.services.scrapers.repair_state_normalizer import (
infer_repair_state_from_text,
normalize_repair_state,
)
# ── needs_repair ─────────────────────────────────────────────────────────────
@pytest.mark.parametrize("raw", ["required", "without", "no", "rough", "prefine"])
def test_needs_repair_values(raw: str) -> None:
assert normalize_repair_state(raw) == "needs_repair"
def test_cian_camelcase_decoration_is_case_insensitive() -> None:
"""Cian detail отдаёт offer.decoration='preFine' (camelCase) — #1791."""
assert normalize_repair_state("preFine") == "needs_repair"
# ── standard ─────────────────────────────────────────────────────────────────
def test_cosmetic_maps_to_standard() -> None:
assert normalize_repair_state("cosmetic") == "standard"
# ── good ─────────────────────────────────────────────────────────────────────
@pytest.mark.parametrize("raw", ["euro", "fine"])
def test_good_values(raw: str) -> None:
assert normalize_repair_state(raw) == "good"
# ── excellent ────────────────────────────────────────────────────────────────
@pytest.mark.parametrize("raw", ["designer", "design"])
def test_excellent_values(raw: str) -> None:
assert normalize_repair_state(raw) == "excellent"
# ── None guard ───────────────────────────────────────────────────────────────
def test_none_returns_none() -> None:
assert normalize_repair_state(None) is None
# ── idempotency ──────────────────────────────────────────────────────────────
@pytest.mark.parametrize("enum_val", ["needs_repair", "standard", "good", "excellent"])
def test_already_normalized_is_passthrough(enum_val: str) -> None:
"""Уже нормализованное значение возвращается as-is (idempotent при ре-обработке)."""
assert normalize_repair_state(enum_val) == enum_val
# ── unknown value ────────────────────────────────────────────────────────────
def test_unknown_raw_returns_none(caplog: pytest.LogCaptureFixture) -> None:
"""Нераспознанное значение → None + WARNING в лог."""
with caplog.at_level(
logging.WARNING,
logger="app.services.scrapers.repair_state_normalizer",
):
result = normalize_repair_state("whitebox")
assert result is None
assert "whitebox" in caplog.text
# ═══════════════════════════════════════════════════════════════════════════════
# infer_repair_state_from_text() — извлечение из описания (#622)
# ═══════════════════════════════════════════════════════════════════════════════
# ── excellent ────────────────────────────────────────────────────────────────
@pytest.mark.parametrize(
"text",
[
"Сделан дизайнерский ремонт с авторской мебелью",
"Квартира с дизайнерской отделкой премиум-класса",
"Премиальный ремонт от застройщика",
"Эксклюзивный ремонт по индивидуальному проекту",
"Авторский ремонт, всё новое",
],
)
def test_infer_excellent(text: str) -> None:
assert infer_repair_state_from_text(text) == "excellent"
# ── good ─────────────────────────────────────────────────────────────────────
@pytest.mark.parametrize(
"text",
[
"Свежий евроремонт, заезжай и живи",
"Сделан евро-ремонт в 2024 году",
"Современный ремонт, всё в отличном состоянии — ремонт в отличном",
"Отличный ремонт, продаётся с мебелью",
"Качественный ремонт от собственника",
],
)
def test_infer_good(text: str) -> None:
assert infer_repair_state_from_text(text) == "good"
# ── needs_repair ─────────────────────────────────────────────────────────────
@pytest.mark.parametrize(
"text",
[
"Квартира без отделки, под чистовую отделку",
"Сдаётся без ремонта, черновая отделка",
"Черновой ремонт, нужен ремонт",
"Требует ремонта, торг уместен",
"Требуется косметический ремонт",
"Квартира под ремонт, цена снижена",
"Предчистовая отделка от застройщика",
],
)
def test_infer_needs_repair(text: str) -> None:
assert infer_repair_state_from_text(text) == "needs_repair"
# ── standard ─────────────────────────────────────────────────────────────────
@pytest.mark.parametrize(
"text",
[
"Косметический ремонт, чисто и уютно",
"Сделана косметическая отделка",
"Обычный ремонт, всё работает",
"Квартира в жилом состоянии",
"Хорошее состояние, заезжай и живи",
],
)
def test_infer_standard(text: str) -> None:
assert infer_repair_state_from_text(text) == "standard"
# ── приоритет сигналов (порядок excellent > good > needs_repair > standard) ───
def test_infer_excellent_beats_needs_repair() -> None:
"""Дизайнерский ремонт после черновой → excellent, не needs_repair."""
text = "Была черновая отделка, теперь сделан дизайнерский ремонт"
assert infer_repair_state_from_text(text) == "excellent"
def test_infer_good_beats_standard() -> None:
"""Евроремонт + косметический → good (сильнее)."""
text = "Косметический ремонт обновлён до евроремонта"
assert infer_repair_state_from_text(text) == "good"
# ── None / unknown guard ─────────────────────────────────────────────────────
def test_infer_none_returns_none() -> None:
assert infer_repair_state_from_text(None) is None
def test_infer_empty_returns_none() -> None:
assert infer_repair_state_from_text("") is None
def test_infer_unknown_text_returns_none() -> None:
"""Текст без repair-сигналов → None (не фабрикуем)."""
text = "Светлая квартира рядом с метро, развитая инфраструктура, школа во дворе"
assert infer_repair_state_from_text(text) is None

327
tradein-mvp/backend/tests/test_save_listings_upsert_reconcile.py
View file

@ -1,327 +0,0 @@
"""Тесты Fix 1: save_listings — SAVEPOINT вокруг INSERT + reconcile по (source,source_id).
Корень бага: при dedup_hash-дрейфе (контент ad изменился, новый source_id не меняется)
INSERT не находит existing row по dedup_hash → пробует свежий INSERT → UniqueViolation
по listings_source_source_id_uq (source, source_id) → весь прогон падает.
Фикс: per-lot SAVEPOINT + IntegrityError catch + UPDATE by (source, source_id).
"""
from __future__ import annotations
import os
from contextlib import contextmanager
from unittest.mock import MagicMock, patch
import psycopg.errors
import pytest
from sqlalchemy.exc import IntegrityError
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
from app.services.scrapers.base import ScrapedLot, save_listings
# ── Fixtures ─────────────────────────────────────────────────────────────────
@pytest.fixture(autouse=True)
def _patch_matching():
"""Stub matching + snapshot so tests focus on the upsert/reconcile logic."""
with (
patch("app.services.scrapers.base.match_or_create_house") as m_house,
patch("app.services.scrapers.base.upsert_listing_source") as m_link,
patch("app.services.scrapers.base.upsert_listing_snapshot") as m_snap,
):
m_house.return_value = (101, 1.0, "new")
m_link.return_value = None
m_snap.return_value = None
yield {"house": m_house, "link": m_link, "snap": m_snap}
def _base_lot(**overrides) -> ScrapedLot:
defaults = {
"source": "avito",
"source_url": "https://www.avito.ru/ekaterinburg/test/1",
"source_id": "7960764619",
"price_rub": 5_000_000,
}
defaults.update(overrides)
return ScrapedLot(**defaults)
@contextmanager
def _nested_ctx():
yield MagicMock()
def _mock_db_normal(inserted: bool = True, listing_id: int = 42) -> MagicMock:
"""DB mock для нормального пути (нет UniqueViolation)."""
insert_row = MagicMock()
insert_row.id = listing_id
insert_row.inserted = inserted
db = MagicMock()
def _execute(sql, params=None):
s = str(sql)
res = MagicMock()
if "SELECT card_hash" in s and "WHERE dedup_hash" in s:
res.fetchone.return_value = None # нет prior row → fresh insert
elif "FROM listings_snapshots" in s:
res.fetchone.return_value = None
elif "INSERT INTO listings (" in s:
res.fetchone.return_value = insert_row
else:
res.fetchone.return_value = None
return res
db.execute.side_effect = _execute
db.begin_nested.side_effect = _nested_ctx
return db
def _mock_db_with_unique_violation(reconcile_id: int = 77) -> MagicMock:
"""DB mock: первый INSERT вызывает UniqueViolation (dedup_hash drift).
Симулирует кейс: INSERT по новому dedup_hash → коллизия по (source, source_id).
Reconcile UPDATE by (source, source_id) → возвращает id=reconcile_id.
"""
uv_orig = psycopg.errors.UniqueViolation()
integrity_err = IntegrityError(
statement="INSERT INTO listings ...",
params={},
orig=uv_orig,
)
rec_row = MagicMock()
rec_row.id = reconcile_id
db = MagicMock()
def _execute(sql, params=None):
s = str(sql)
res = MagicMock()
if "SELECT card_hash" in s and "WHERE dedup_hash" in s:
# prior_row=None → INSERT будет сделан (no prior dedup_hash match)
res.fetchone.return_value = None
elif "FROM listings_snapshots" in s:
res.fetchone.return_value = None
elif "INSERT INTO listings (" in s:
# Имитируем исключение внутри begin_nested() context
raise integrity_err
elif "UPDATE listings" in s and "SET dedup_hash" in s:
# reconcile UPDATE by (source, source_id)
res.fetchone.return_value = rec_row
elif "UPDATE listings" in s and "house_id_fk" in s:
res.fetchone.return_value = None
else:
res.fetchone.return_value = None
return res
db.execute.side_effect = _execute
# begin_nested() нужен для per-lot SAVEPOINT + snapshot + matching hook
# Используем реальный contextmanager который НЕ перехватывает IntegrityError
# (savepoint rollback симулируется тем, что db.execute после rollback'а
# уже вызывает reconcile UPDATE, а не снова INSERT).
savepoint_entered = [False]
@contextmanager
def _nested():
savepoint_entered[0] = True
try:
yield MagicMock()
except IntegrityError:
# SAVEPOINT rollback — пробрасываем наверх (SQLAlchemy делает rollback)
raise
db.begin_nested.side_effect = _nested
return db
# ── Tests ─────────────────────────────────────────────────────────────────────
def test_normal_insert_path_unchanged():
"""Нормальный путь (нет UniqueViolation) → inserted=1, updated=0."""
db = _mock_db_normal(inserted=True, listing_id=10)
lot = _base_lot()
inserted, updated = save_listings(db, [lot])
assert inserted == 1
assert updated == 0
def test_normal_update_path_unchanged():
"""Нормальный update path (ON CONFLICT DO UPDATE) → inserted=0, updated=1."""
insert_row = MagicMock()
insert_row.id = 42
insert_row.inserted = False # xmax != 0 → update
db = MagicMock()
def _execute(sql, params=None):
s = str(sql)
res = MagicMock()
if "SELECT card_hash" in s and "WHERE dedup_hash" in s:
res.fetchone.return_value = None
elif "FROM listings_snapshots" in s:
res.fetchone.return_value = None
elif "INSERT INTO listings (" in s:
res.fetchone.return_value = insert_row
else:
res.fetchone.return_value = None
return res
db.execute.side_effect = _execute
db.begin_nested.side_effect = _nested_ctx
lot = _base_lot(source_id="upd_test")
inserted, updated = save_listings(db, [lot])
assert inserted == 0
assert updated == 1
def test_null_source_id_no_savepoint_needed():
"""source_id=None (yandex) → обычный INSERT, UniqueViolation не может возникнуть.
Проверяем, что нормальный путь не изменился при source_id=None.
"""
db = _mock_db_normal(inserted=True, listing_id=99)
lot = _base_lot(
source="yandex",
source_url="https://realty.yandex.ru/offer/123/",
source_id=None,
)
inserted, updated = save_listings(db, [lot])
assert inserted == 1
assert updated == 0
def test_unique_violation_triggers_reconcile_update(caplog, _patch_matching):
"""UniqueViolation по (source, source_id) → SAVEPOINT rollback → UPDATE by (s,sid).
После reconcile UPDATE: listing_id определён, matching hook вызывается.
inserted=0, updated=0 (reconciled не входит в возврат, но лог есть).
"""
import logging
db = _mock_db_with_unique_violation(reconcile_id=77)
lot = _base_lot(source="avito", source_id="7960764619")
with caplog.at_level(logging.WARNING, logger="app.services.scrapers.base"):
inserted, updated = save_listings(db, [lot])
# Основные counters: reconcile не добавляет к inserted/updated
assert inserted == 0
assert updated == 0
# Лог предупреждения о reconcile
warns = [r for r in caplog.records if "dedup_hash_drift" in r.getMessage()]
assert len(warns) == 1
assert "7960764619" in warns[0].getMessage()
# Matching hook должен вызваться (listing_id=77 из reconcile UPDATE)
assert _patch_matching["link"].call_count == 1
def test_unique_violation_reconcile_uses_update_by_source_source_id():
"""Reconcile UPDATE содержит WHERE source = :source AND source_id = :source_id."""
db = _mock_db_with_unique_violation(reconcile_id=55)
lot = _base_lot(source="cian", source_id="330200428")
save_listings(db, [lot])
# Ищем вызов reconcile UPDATE (содержит SET dedup_hash)
reconcile_calls = [
c
for c in db.execute.call_args_list
if "UPDATE listings" in str(c.args[0]) and "SET dedup_hash" in str(c.args[0])
]
assert len(reconcile_calls) == 1
_sql, params = str(reconcile_calls[0].args[0]), reconcile_calls[0].args[1]
assert "WHERE source = :source" in _sql
assert "AND source_id = :source_id" in _sql
assert params["source"] == "cian"
assert params["source_id"] == "330200428"
def test_reconcile_includes_new_dedup_hash():
"""Reconcile UPDATE обновляет dedup_hash на новый (дрейфнувший) хэш."""
db = _mock_db_with_unique_violation(reconcile_id=66)
lot = _base_lot(source="avito", source_id="7960764619", price_rub=5_500_000)
expected_dedup = lot.compute_dedup_hash()
save_listings(db, [lot])
reconcile_calls = [
c
for c in db.execute.call_args_list
if "UPDATE listings" in str(c.args[0]) and "SET dedup_hash" in str(c.args[0])
]
assert len(reconcile_calls) == 1
params = reconcile_calls[0].args[1]
assert params["dedup"] == expected_dedup
def test_reconcile_run_does_not_crash_full_batch(_patch_matching):
"""Один лот с UniqueViolation в batch из 3 → остальные 2 успешно обрабатываются.
Ключевая проверка что один битый lot НЕ роняет прогон.
"""
rec_row = MagicMock()
rec_row.id = 77
ok_insert_row = MagicMock()
ok_insert_row.id = 88
ok_insert_row.inserted = True
uv_orig = psycopg.errors.UniqueViolation()
integrity_err = IntegrityError("INSERT", {}, uv_orig)
def _execute(sql, params=None):
s = str(sql)
res = MagicMock()
if "SELECT card_hash" in s and "WHERE dedup_hash" in s:
res.fetchone.return_value = None
elif "FROM listings_snapshots" in s:
res.fetchone.return_value = None
elif "INSERT INTO listings (" in s:
if params and params.get("source_id") == "CONFLICT_ID":
raise integrity_err
res.fetchone.return_value = ok_insert_row
elif "UPDATE listings" in s and "SET dedup_hash" in s:
res.fetchone.return_value = rec_row
elif "UPDATE listings" in s and "house_id_fk" in s:
res.fetchone.return_value = None
else:
res.fetchone.return_value = None
return res
db = MagicMock()
db.execute.side_effect = _execute
@contextmanager
def _nested():
try:
yield MagicMock()
except IntegrityError:
raise
db.begin_nested.side_effect = _nested
lot_conflict = _base_lot(source_id="CONFLICT_ID")
lot_ok1 = _base_lot(source_id="OK_1", source_url="https://www.avito.ru/ok1")
lot_ok2 = _base_lot(source_id="OK_2", source_url="https://www.avito.ru/ok2")
inserted, updated = save_listings(db, [lot_conflict, lot_ok1, lot_ok2])
# 2 нормальных INSERT + 1 reconcile (не считается в inserted/updated)
assert inserted == 2
assert updated == 0
# Все 3 лота дошли до matching hook
assert _patch_matching["link"].call_count == 3

64
tradein-mvp/backend/tests/test_scraped_at_bump_2206.py
View file

@ -7,8 +7,9 @@
эстиматору было видно лишь ~45.6% активного инвентаря (CIAN — 5.9%).
Фикс: и ON CONFLICT DO UPDATE, и dedup-drift reconcile UPDATE теперь выставляют
scraped_at = NOW() рядом с last_seen_at = NOW(). Проверяем оба пути в ОБОИХ
зеркальных модулях (app.services.scrapers.base + scraper_kit.base) и свойства
scraped_at = NOW() рядом с last_seen_at = NOW(). Проверяем оба пути в
`scraper_kit.base` (единственный живой модуль — `app.services.scrapers.base`
удалён #2397 финальный шаг E, mirror-тесты через legacy убраны) и свойства
ретро-бэкфилл-миграции 161.
"""
@ -22,7 +23,6 @@ from typing import Any
from unittest.mock import MagicMock, patch
import psycopg.errors
import pytest
from sqlalchemy.exc import IntegrityError
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
@ -30,9 +30,6 @@ os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/
from scraper_kit.base import ScrapedLot as KitLot
from scraper_kit.base import save_listings as kit_save_listings
from app.services.scrapers.base import ScrapedLot as AppLot
from app.services.scrapers.base import save_listings as app_save_listings
@contextmanager
def _nested_ctx() -> Any:
@ -111,59 +108,8 @@ def _find_sql(db: MagicMock, needle: str) -> str:
raise AssertionError(f"SQL containing {needle!r} not found")
# ── app.services.scrapers.base ────────────────────────────────────────────────
@pytest.fixture
def _patch_app_matching() -> Any:
with (
patch("app.services.scrapers.base.match_or_create_house") as m_house,
patch("app.services.scrapers.base.upsert_listing_source") as m_link,
patch("app.services.scrapers.base.upsert_listing_snapshot") as m_snap,
):
m_house.return_value = (101, 1.0, "new")
m_link.return_value = None
m_snap.return_value = None
yield
def test_app_on_conflict_bumps_scraped_at(_patch_app_matching: Any) -> None:
"""ON CONFLICT DO UPDATE (ре-подтверждение) двигает scraped_at = NOW()."""
db = _mock_db_update_path(inserted=False)
lot = AppLot(
source="cian",
source_url="https://ekb.cian.ru/sale/flat/1/",
source_id="1",
price_rub=5_000_000,
)
inserted, updated = app_save_listings(db, [lot])
assert (inserted, updated) == (0, 1)
sql = _find_sql(db, "INSERT INTO listings (")
# scraped_at должен обновляться в DO UPDATE SET рядом с last_seen_at
assert "scraped_at = NOW()" in sql
assert "last_seen_at = NOW()" in sql
def test_app_reconcile_bumps_scraped_at(_patch_app_matching: Any) -> None:
"""dedup-drift reconcile UPDATE by (source, source_id) двигает scraped_at = NOW()."""
db = _mock_db_reconcile(reconcile_id=88)
lot = AppLot(
source="avito",
source_url="https://www.avito.ru/ekaterinburg/kvartiry/1-k_1",
source_id="7960764619",
price_rub=5_000_000,
)
app_save_listings(db, [lot])
sql = _find_sql(db, "SET dedup_hash")
assert "scraped_at = NOW()" in sql
assert "last_seen_at = NOW()" in sql
# ── scraper_kit.base (БОЕВОЙ путь: USE_KIT_SCHEDULER=true) ─────────────────────
# ── scraper_kit.base (единственный живой путь — legacy app.services.scrapers.base
# удалён #2397 финальный шаг E) ─────────────────────────────────────────────────
def _kit_matcher() -> MagicMock:

206
tradein-mvp/backend/tests/test_scraper_kit_avito_golden_parity.py
View file

@ -1,206 +0,0 @@
"""Golden-parity: `scraper_kit.providers.avito.*` парсинг ≡ `app.services.scrapers.avito*`.
Strangler-инвариант (#2133): новая scraper_kit-копия avito-провайдера должна давать
БАЙТ-ИДЕНТИЧНЫЙ результат парсинга старому боевому коду на одинаковом входе.
Развязка (settings→ScraperConfig, get_scraper_delay→delay_provider,
geocoder→scraper_kit.geo, matcher→HouseMatcher) НЕ меняет распарсенные данные.
Гоняем ОБА модуля на одних фикстурах и сравниваем результат:
- SERP `_parse_html` (DOM-карточки → list[ScrapedLot]) на avito_serp_sample.html,
при ekb_only=False и ekb_only=True (фильтр padding-карточек);
- `_extract_total_count` (счётчик выдачи);
- `_build_sort_timestamp_map` + `_unix_to_date` (sortTimeStamp → date, MSK);
- detail `parse_detail_html` (item-view HTML → DetailEnrichment);
- чистые title/address-хелперы (_clean_address, _extract_rooms/area/floor_from_title).
Идентичность результата = kit-парсер верно скопирован (развязка не задела логику).
"""
from __future__ import annotations
import dataclasses
import os
from pathlib import Path
from types import SimpleNamespace
from typing import Any
from unittest.mock import patch
# Старый app.services.scrapers.avito импортирует app.core.config.settings=Settings(),
# которому нужен DATABASE_URL. Офлайн-парсинг БД не трогает — фиктивный DSN достаточен.
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
# НОВЫЙ (scraper_kit) и СТАРЫЙ (app.services.scrapers) avito-парсеры — оба в одном
# тесте; ruff сортирует scraper_kit (first-party) выше app-импортов.
from scraper_kit.providers.avito.detail import parse_detail_html as new_parse_detail
from scraper_kit.providers.avito.serp import AvitoScraper as NewAvitoScraper
from scraper_kit.providers.avito.serp import (
_build_sort_timestamp_map as new_build_ts_map,
)
from scraper_kit.providers.avito.serp import _clean_address as new_clean_address
from scraper_kit.providers.avito.serp import _extract_area_from_title as new_area
from scraper_kit.providers.avito.serp import _extract_floor_from_title as new_floor
from scraper_kit.providers.avito.serp import _extract_rooms_from_title as new_rooms
from scraper_kit.providers.avito.shared import _unix_to_date as new_unix_to_date
from app.services.scrapers.avito import AvitoScraper as OldAvitoScraper
from app.services.scrapers.avito import (
_build_sort_timestamp_map as old_build_ts_map,
)
from app.services.scrapers.avito import _clean_address as old_clean_address
from app.services.scrapers.avito import _extract_area_from_title as old_area
from app.services.scrapers.avito import _extract_floor_from_title as old_floor
from app.services.scrapers.avito import _extract_rooms_from_title as old_rooms
from app.services.scrapers.avito import _unix_to_date as old_unix_to_date
from app.services.scrapers.avito_detail import parse_detail_html as old_parse_detail
_FIXTURES = Path(__file__).parent / "fixtures"
_SERP_HTML = (_FIXTURES / "avito_serp_sample.html").read_text(encoding="utf-8")
# Минимальный detail item-view HTML (зеркало test_avito_detail_parse.MINIMAL_HTML).
_DETAIL_HTML = """
<html><body>
<div data-marker="item-view/item-id">№ 7986882804 · 18 мая в 13:40 · 1899 просмотров (+ 12 сегодня)</div>
<span itemprop="price" content="11990000">11 990 000 ₽</span>
<div data-marker="item-map-wrapper" data-map-lat="56.797713" data-map-lon="60.609135" data-location-id="654070"></div>
<div data-marker="item-view/item-params">
<ul>
<li>Количество комнат: 3</li>
<li>Общая площадь: 75 м²</li>
<li>Площадь кухни: 15 м²</li>
<li>Этаж: 20 из 26</li>
<li>Балкон или лоджия: лоджия</li>
<li>Ремонт: евро</li>
</ul>
<ul>
<li>Тип дома: монолитный</li>
<li>Этажей в доме: 26</li>
</ul>
</div>
<a data-marker="nd-jk-details-button" href="/catalog/houses/ekaterinburg/ul_postovskogo_17a/3171365">Дом</a>
<div data-marker="item-view/item-description">Отличная квартира в новом ЖК.</div>
</body></html>
""" # noqa: E501
_SOURCE_URL = "https://www.avito.ru/ekaterinburg/kvartiry/test_7986882804"
def _make_old(ekb_only: bool) -> tuple[OldAvitoScraper, SimpleNamespace]:
"""Старый AvitoScraper без DB: patch get_scraper_delay + settings.avito_serp_ekb_only."""
with patch("app.services.scrapers.avito.get_scraper_delay", return_value=7.0):
scraper = OldAvitoScraper()
# _parse_html читает settings.avito_serp_ekb_only — подменяем на детерминированное.
scraper_settings = SimpleNamespace(avito_serp_ekb_only=ekb_only)
return scraper, scraper_settings
def _make_new(ekb_only: bool) -> NewAvitoScraper:
"""Новый AvitoScraper с инжектируемым ScraperConfig (duck-typed namespace)."""
config = SimpleNamespace(avito_serp_ekb_only=ekb_only)
return NewAvitoScraper(config=config)
def _dump_lots(lots: list[Any]) -> list[dict[str, Any]]:
return [lot.model_dump() for lot in lots]
# ── SERP _parse_html parity ──────────────────────────────────────────────────
def _run_old_parse_html(html: str, base: str, ekb_only: bool) -> list[Any]:
old, old_settings = _make_old(ekb_only)
with patch("app.services.scrapers.avito.settings", old_settings):
return old._parse_html(html, base)
def test_serp_parse_html_parity_ekb_only_false() -> None:
"""DOM-карточки → ScrapedLot идентичны (фильтр padding выключен)."""
base = "https://www.avito.ru/ekaterinburg/kvartiry"
old_lots = _run_old_parse_html(_SERP_HTML, base, ekb_only=False)
new_lots = _make_new(ekb_only=False)._parse_html(_SERP_HTML, base)
assert len(old_lots) == len(new_lots)
assert len(new_lots) > 0, "фикстура должна содержать хотя бы одну карточку"
assert _dump_lots(old_lots) == _dump_lots(new_lots)
def test_serp_parse_html_parity_ekb_only_true() -> None:
"""С включённым ekb_only-фильтром padding-дроп идентичен."""
base = "https://www.avito.ru/ekaterinburg/kvartiry"
old_lots = _run_old_parse_html(_SERP_HTML, base, ekb_only=True)
new_lots = _make_new(ekb_only=True)._parse_html(_SERP_HTML, base)
assert len(old_lots) == len(new_lots)
assert _dump_lots(old_lots) == _dump_lots(new_lots)
def test_serp_extract_total_count_parity() -> None:
"""_extract_total_count на SERP-фикстуре идентичен."""
old, _ = _make_old(ekb_only=False)
new = _make_new(ekb_only=False)
assert old._extract_total_count(_SERP_HTML) == new._extract_total_count(_SERP_HTML)
def test_build_sort_timestamp_map_parity() -> None:
"""_build_sort_timestamp_map (sortTimeStamp → date, MSK) идентичен."""
assert old_build_ts_map(_SERP_HTML) == new_build_ts_map(_SERP_HTML)
def test_unix_to_date_parity() -> None:
"""_unix_to_date (MSK) идентичен на репрезентативных epoch-значениях."""
for ts in [None, 0, 1756077885, 1735689600, 1700000000.0, 1_600_000_000]:
assert old_unix_to_date(ts) == new_unix_to_date(ts)
# ── detail parse_detail_html parity ──────────────────────────────────────────
def test_parse_detail_html_parity() -> None:
"""parse_detail_html → DetailEnrichment идентичен (все поля)."""
old_res = old_parse_detail(_DETAIL_HTML, _SOURCE_URL)
new_res = new_parse_detail(_DETAIL_HTML, _SOURCE_URL)
assert dataclasses.asdict(old_res) == dataclasses.asdict(new_res)
# ── чистые title/address-хелперы ──────────────────────────────────────────────
def test_pure_title_helpers_parity() -> None:
titles = [
"3-к. квартира, 75 м², 20/26 эт.",
"Квартира-студия, 24,5 м², 12/25 эт.",
"1-к. квартира, 38 м², 5/9 эт.",
"2-к. квартира, 52,3 м², 7/16 эт.",
"Комната, 18 м², 3/5 эт.",
]
for t in titles:
assert old_rooms(t) == new_rooms(t)
assert old_area(t) == new_area(t)
assert old_floor(t) == new_floor(t)
def test_clean_address_parity() -> None:
addrs = [
None,
"Екатеринбург, ул. Постовского, 17А",
"Свердловская область, Екатеринбург, Малышева, 51",
" Екатеринбург, Ленина 1 ",
]
for a in addrs:
assert old_clean_address(a) == new_clean_address(a)
# ── strangler-guard: kit-провайдер не импортирует app.* ──────────────────────
def test_kit_avito_has_no_app_imports() -> None:
"""Ни один модуль scraper_kit.providers.avito.* не импортирует app.* (развязка)."""
import inspect
from scraper_kit.providers.avito import detail, houses, imv, serp, shared
for mod in (serp, detail, houses, imv, shared):
source = inspect.getsource(mod)
for line in source.splitlines():
stripped = line.strip()
assert not stripped.startswith("from app"), f"{mod.__name__}: {line!r}"
assert not stripped.startswith("import app"), f"{mod.__name__}: {line!r}"

363
tradein-mvp/backend/tests/test_scraper_kit_base_golden_parity.py
View file

@ -1,363 +0,0 @@
"""Golden-parity: `scraper_kit.base.save_listings` ≡ `app.services.scrapers.base.save_listings`.
Strangler-инвариант (#2132): новая scraper_kit-копия базового слоя должна давать
БАЙТ-ИДЕНТИЧНЫЙ эффект старому боевому коду на одинаковом входе. Единственное
намеренное расхождение — развязка `region_code`: старый хардкодит литерал `66` в
INSERT SQL, новый получает его параметром `:region_code` (kit не знает про регион).
При region_code=66 итоговый INSERT идентичен — что этот тест и доказывает,
нормализуя `:region_code` → `66` на новой стороне перед сравнением.
Метод: гоняем ОБА модуля на одном списке `ScrapedLot`, перехватывая через
recording-mock всю последовательность `db.execute(sql, params)` + аргументы hook'а
матчинга. Сравниваем:
- последовательность (SQL, params) вызовов execute (INSERT / SELECT / reconcile
UPDATE / house_id_fk UPDATE) — после нормализации region_code;
- kwargs каждого вызова matcher.match_or_create_house / upsert_listing_source;
- compute_dedup_hash / compute_card_hash / compute_price_per_m2 старого и нового
класса ScrapedLot на идентичном входе.
Фикстуры покрывают: avito (без адреса), cian (адрес+coords → house resolve → FK
UPDATE), студию rooms=0, yandex без source_id (ext_id := dedup_hash fallback) и
dedup-коллизию (INSERT → UniqueViolation → reconcile UPDATE by source,source_id).
"""
from __future__ import annotations
import contextlib
from typing import Any
from unittest.mock import MagicMock, patch
import psycopg.errors
from scraper_kit.base import ScrapedLot as NewLot
from scraper_kit.base import save_listings as new_save_listings
from sqlalchemy.exc import IntegrityError
# СТАРЫЙ (боевой) и НОВЫЙ (scraper_kit) модули — импортируем оба в одном тесте.
from app.services.scrapers.base import ScrapedLot as OldLot
from app.services.scrapers.base import save_listings as old_save_listings
_REGION_CODE = 66
# ── recording session ────────────────────────────────────────────────────────
class _Row:
def __init__(self, **kw: Any) -> None:
self.__dict__.update(kw)
class _Result:
def __init__(self, row: _Row | None) -> None:
self._row = row
def fetchone(self) -> _Row | None:
return self._row
@contextlib.contextmanager
def _nested_cm() -> Any:
yield MagicMock()
class RecordingSession:
"""Мок Session: пишет (sql, params) всех execute, детерминированно отвечает.
- SELECT card_hash … WHERE dedup_hash → None (prior_row отсутствует, fresh)
- INSERT INTO listings … → row(id, inserted); если source_id в
drift_ids → raise IntegrityError(UniqueViolation) (dedup-коллизия)
- reconcile UPDATE listings SET dedup_hash … → row(id=reconcile_id)
- UPDATE listings … house_id_fk / snapshot EXISTS → None
"""
def __init__(
self,
*,
drift_ids: frozenset[str] = frozenset(),
update_ids: frozenset[str] = frozenset(),
insert_id: int = 42,
reconcile_id: int = 99,
) -> None:
self.calls: list[tuple[str, dict[str, Any]]] = []
self._drift = drift_ids
self._update = update_ids
self._insert_id = insert_id
self._reconcile_id = reconcile_id
def execute(self, sql: Any, params: dict[str, Any] | None = None) -> _Result:
s = str(sql)
p = dict(params) if params else {}
self.calls.append((s, p))
if "INSERT INTO listings (" in s:
sid = p.get("source_id")
if sid in self._drift:
raise IntegrityError("INSERT", p, psycopg.errors.UniqueViolation("dup"))
inserted = sid not in self._update
return _Result(_Row(id=self._insert_id, inserted=inserted))
if "UPDATE listings" in s and "SET dedup_hash" in s:
return _Result(_Row(id=self._reconcile_id))
return _Result(None)
def begin_nested(self) -> Any:
return _nested_cm()
def commit(self) -> None:
pass
# ── fixtures ─────────────────────────────────────────────────────────────────
def _fixture_data() -> list[dict[str, Any]]:
"""Реалистичные входные dict'ы (общие для old/new ScrapedLot)."""
return [
# 1. avito без адреса → house resolve пропускается (нет FK UPDATE)
{
"source": "avito",
"source_url": "https://www.avito.ru/ekaterinburg/kvartiry/1-k_987654?context=AB",
"source_id": "987654",
"price_rub": 4_500_000,
"area_m2": 38.0,
"rooms": 1,
"floor": 5,
"total_floors": 9,
},
# 2. cian full: адрес + coords → house resolve → house_id_fk UPDATE
{
"source": "cian",
"source_url": "https://ekb.cian.ru/sale/flat/123456/",
"source_id": "123456",
"price_rub": 6_200_000,
"address": "Екатеринбург, улица Постовского, 17А",
"lat": 56.8385,
"lon": 60.5940,
"rooms": 2,
"area_m2": 52.0,
"floor": 7,
"total_floors": 16,
"living_area_m2": 35.0,
"bedrooms_count": 2,
"cadastral_number": "66:41:0204016:1234",
"building_cadastral_number": "66:41:0204016:100",
"phones": [{"countryCode": "7", "number": "9012345678"}],
"is_homeowner": True,
"metro_stations": [{"name": "Геологическая", "time": 10}],
"house_source": "cian",
"listing_segment": "vtorichka",
},
# 3. студия rooms=0 (avito) с адресом → house resolve
{
"source": "avito",
"source_url": "https://www.avito.ru/ekaterinburg/kvartiry/studiya_555",
"source_id": "555",
"price_rub": 3_100_000,
"address": "Екатеринбург, Малышева 51",
"lat": 56.84,
"lon": 60.60,
"rooms": 0,
"area_m2": 24.5,
"floor": 12,
"total_floors": 25,
},
# 4. yandex без source_id → ext_id := dedup_hash fallback
{
"source": "yandex",
"source_url": "https://realty.yandex.ru/offer/5500000000000000001/?from=serp",
"source_id": None,
"price_rub": 7_200_000,
"address": "Екатеринбург, улица Мира, 10",
"lat": 56.83,
"lon": 60.65,
"rooms": 2,
"area_m2": 48.0,
"yandex_offer_id": "5500000000000000001",
"predicted_price_rub": 7_554_000,
"price_trend": "DECREASED",
"price_previous_rub": 7_500_000,
},
# 5. dedup-коллизия: INSERT → UniqueViolation → reconcile UPDATE
{
"source": "cian",
"source_url": "https://ekb.cian.ru/sale/flat/DRIFT-1/",
"source_id": "DRIFT-1",
"price_rub": 5_900_000,
"address": "Екатеринбург, Ленина 1",
"lat": 56.838,
"lon": 60.597,
"rooms": 3,
"area_m2": 72.0,
},
]
_DRIFT_IDS = frozenset({"DRIFT-1"})
# ── run helpers ──────────────────────────────────────────────────────────────
def _run_old(lots: list[Any]) -> tuple[RecordingSession, MagicMock, MagicMock]:
db = RecordingSession(drift_ids=_DRIFT_IDS)
with (
patch("app.services.scrapers.base.match_or_create_house") as m_house,
patch("app.services.scrapers.base.upsert_listing_source") as m_link,
patch("app.services.scrapers.base.upsert_listing_snapshot"),
):
m_house.return_value = (101, 1.0, "new")
m_link.return_value = None
old_save_listings(db, lots)
return db, m_house, m_link
def _run_new(lots: list[Any]) -> tuple[RecordingSession, MagicMock]:
db = RecordingSession(drift_ids=_DRIFT_IDS)
matcher = MagicMock()
matcher.match_or_create_house.return_value = (101, 1.0, "new")
matcher.upsert_listing_source.return_value = None
with patch("scraper_kit.base.upsert_listing_snapshot"):
new_save_listings(db, lots, matcher=matcher, region_code=_REGION_CODE)
return db, matcher
def _normalize_new_calls(
calls: list[tuple[str, dict[str, Any]]],
) -> list[tuple[str, dict[str, Any]]]:
"""Нормализуем расхождение region_code: `:region_code` → `66`, убираем ключ.
Это единственное намеренное отличие новой копии; после нормализации при
region_code=66 SQL и params должны быть байт-идентичны старому боевому коду.
"""
out: list[tuple[str, dict[str, Any]]] = []
for sql, params in calls:
norm_sql = sql.replace(":region_code", str(_REGION_CODE))
norm_params = {k: v for k, v in params.items() if k != "region_code"}
out.append((norm_sql, norm_params))
return out
def _kwargs_list(mock_or_method: Any) -> list[dict[str, Any]]:
return [c.kwargs for c in mock_or_method.call_args_list]
# ── tests ────────────────────────────────────────────────────────────────────
def test_save_listings_execute_sequence_parity() -> None:
"""Последовательность (SQL, params) execute идентична (после region_code-нормализации)."""
lots = [NewLot(**d) for d in _fixture_data()]
old_db, _oh, _ol = _run_old(lots)
new_db, _nm = _run_new(lots)
old_calls = old_db.calls
new_calls = _normalize_new_calls(new_db.calls)
assert len(new_calls) == len(
old_calls
), f"разное число execute: old={len(old_calls)} new={len(new_calls)}"
for i, (old_c, new_c) in enumerate(zip(old_calls, new_calls, strict=True)):
assert old_c[0] == new_c[0], f"SQL расходится на вызове #{i}"
assert old_c[1] == new_c[1], f"params расходятся на вызове #{i}"
def test_insert_sql_region_code_injection() -> None:
"""Старый хардкодит `66` в VALUES; новый — параметр `:region_code`, params[region_code]=66."""
lots = [NewLot(**_fixture_data()[0])]
old_db, _oh, _ol = _run_old(lots)
new_db, _nm = _run_new(lots)
old_insert = next(s for s, _ in old_db.calls if "INSERT INTO listings (" in s)
new_insert, new_params = next((s, p) for s, p in new_db.calls if "INSERT INTO listings (" in s)
assert ", 66," in old_insert and ":region_code" not in old_insert
assert ":region_code," in new_insert and ", 66," not in new_insert
assert new_params["region_code"] == 66
# После нормализации → идентичны
assert new_insert.replace(":region_code", "66") == old_insert
def test_matching_hook_kwargs_parity() -> None:
"""kwargs match_or_create_house / upsert_listing_source идентичны old ↔ new."""
lots = [NewLot(**d) for d in _fixture_data()]
_old_db, m_house, m_link = _run_old(lots)
_new_db, matcher = _run_new(lots)
assert _kwargs_list(m_house) == _kwargs_list(matcher.match_or_create_house)
assert _kwargs_list(m_link) == _kwargs_list(matcher.upsert_listing_source)
# sanity: hook действительно сработал (не пустой прогон)
assert m_link.call_count == len(lots)
# house resolve только когда есть address/coords (4 из 5 фикстур)
assert m_house.call_count == 4
assert matcher.match_or_create_house.call_count == 4
def test_dedup_collision_triggers_reconcile_update_in_both() -> None:
"""Drift-лот → INSERT UniqueViolation → reconcile UPDATE by (source, source_id) в обоих."""
lots = [NewLot(**_fixture_data()[4])] # DRIFT-1
old_db, _oh, _ol = _run_old(lots)
new_db, _nm = _run_new(lots)
old_reconcile = [s for s, _ in old_db.calls if "UPDATE listings" in s and "SET dedup_hash" in s]
new_reconcile = [s for s, _ in new_db.calls if "UPDATE listings" in s and "SET dedup_hash" in s]
assert len(old_reconcile) == 1
assert len(new_reconcile) == 1
assert old_reconcile[0] == new_reconcile[0]
def test_return_counts_parity() -> None:
"""(inserted, updated) идентичны old ↔ new на одном входе."""
lots = [NewLot(**d) for d in _fixture_data()]
old_db = RecordingSession(drift_ids=_DRIFT_IDS)
with (
patch("app.services.scrapers.base.match_or_create_house", return_value=(101, 1.0, "new")),
patch("app.services.scrapers.base.upsert_listing_source", return_value=None),
patch("app.services.scrapers.base.upsert_listing_snapshot"),
):
old_result = old_save_listings(old_db, lots)
new_db = RecordingSession(drift_ids=_DRIFT_IDS)
matcher = MagicMock()
matcher.match_or_create_house.return_value = (101, 1.0, "new")
matcher.upsert_listing_source.return_value = None
with patch("scraper_kit.base.upsert_listing_snapshot"):
new_result = new_save_listings(new_db, lots, matcher=matcher, region_code=_REGION_CODE)
assert old_result == new_result
def test_hash_parity_old_vs_new_scraped_lot() -> None:
"""compute_dedup_hash / compute_card_hash / compute_price_per_m2 идентичны у обоих классов."""
for data in _fixture_data():
old_lot = OldLot(**data)
new_lot = NewLot(**data)
assert old_lot.compute_dedup_hash() == new_lot.compute_dedup_hash()
assert old_lot.compute_card_hash() == new_lot.compute_card_hash()
assert old_lot.compute_price_per_m2() == new_lot.compute_price_per_m2()
def test_kit_base_public_api_importable() -> None:
"""Acceptance: from scraper_kit.base import ScrapedLot, save_listings, BaseScraper."""
from scraper_kit.base import BaseScraper, ScrapedLot, save_listings
assert ScrapedLot is not None
assert callable(save_listings)
assert BaseScraper is not None
def test_kit_base_has_no_app_imports() -> None:
"""scraper_kit.base не должен импортировать app.* (strangler-развязка)."""
import inspect
import scraper_kit.base as kit_base
source = inspect.getsource(kit_base)
# Игнорируем docstring-упоминания: проверяем именно import-стейтменты.
for line in source.splitlines():
stripped = line.strip()
assert not stripped.startswith("from app"), f"app import найден: {line!r}"
assert not stripped.startswith("import app"), f"app import найден: {line!r}"

225
tradein-mvp/backend/tests/test_scraper_kit_bisection_parity.py
View file

@ -1,225 +0,0 @@
"""Golden-parity бисекции: старый боевой обход `app.services.scrapers.avito`
≡ новый единый движок `scraper_kit.pricing.walk_price_range` (#2134).
Четыре независимые копии адаптивной бисекции «комнатность × цена» были
консолидированы в один движок (`scraper_kit/pricing/bisection.py`). Провайдеры
cian/domclick/yandex_realty удалены (#2397 Part E2, 0 runtime-импортёров) —
их legacy-vs-kit секции обхода убраны отсюда вместе с ними (легаси-эталон
исчез). `avito` остаётся легаси (не мигрирован) — его parity-проверка ниже
доказывает, что unification НЕ изменил ОБХОД: при одинаковом входе
(детерминированный мок счётчика по диапазону) старая боевая реализация и
новая (движок) посещают ИДЕНТИЧНУЮ последовательность бакетов — probe-запросы,
leaf-пагинации, деления, ранние возвраты на нуле, probe-fail-ветки.
Механика: на ОБЕ реализации (старую и новую) ставятся ОДИНАКОВЫЕ моки на
низкоуровневые методы (probe-fetch + leaf-пагинатор), которые (а) возвращают
счётчик из общего оракула `count(lo, hi)` и (б) пишут событие в общий список.
Совпадение списков событий = идентичный обход.
"""
from __future__ import annotations
import os
from collections.abc import Callable
from types import SimpleNamespace
from typing import Any
from unittest.mock import AsyncMock, patch
import pytest
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
from scraper_kit.price_brackets import get_price_seed_brackets
from scraper_kit.providers.avito.serp import AvitoScraper as NewAvito
from app.services.scrapers.avito import AvitoScraper as OldAvito
# ── Оракул счётчиков ─────────────────────────────────────────────────────────
Oracle = Callable[[Any, Any], int | None]
def make_oracle(
*,
divisor: int,
open_ceiling: int,
zeros: frozenset[tuple[Any, Any]] = frozenset(),
nones: frozenset[tuple[Any, Any]] = frozenset(),
) -> Oracle:
"""Детерминированный счётчик от (lo, hi): count = span // divisor.
span считается по клампнутым границам (None-lo → 0, None-hi → open_ceiling),
чтобы широкие брекеты делились, а узкие уходили в leaf. `zeros` форсит 0 (ранний
возврат), `nones` форсит None (probe-fail).
"""
def oracle(lo: Any, hi: Any) -> int | None:
if (lo, hi) in nones:
return None
if (lo, hi) in zeros:
return 0
eff_lo = lo if lo is not None else 0
eff_hi = hi if hi is not None else open_ceiling
span = eff_hi - eff_lo
return max(0, span // divisor)
return oracle
# ── avito «сборщик событий» ──────────────────────────────────────────────────
#
# Ставит одинаковые моки на старый/новый экземпляр и возвращает список событий
# обхода. `("probe", lo, hi)` — запрос счётчика бакета; `("leaf", ...)` —
# пагинация leaf-бакета.
def _patch_avito(scraper: Any, oracle: Oracle, events: list[Any]) -> None:
def probe(room_slug: str, min_price: Any, max_price: Any) -> tuple[str | None, int | None]:
events.append(("probe", min_price, max_price))
return "H", oracle(min_price, max_price)
def leaf(**kw: Any) -> None:
events.append(("leaf", kw["lo"], kw["hi"], kw["max_pages"]))
scraper._probe_total = AsyncMock(side_effect=probe)
scraper._paginate_leaf_bucket = AsyncMock(side_effect=leaf)
scraper._rotate_ip = AsyncMock(return_value=False)
# ── Конструкторы старых/новых экземпляров (offline, без DB) ───────────────────
_KIT_CONFIG = SimpleNamespace(
scraper_proxy_url=None,
yandex_proxy_rotate_url=None,
avito_proxy_rotate_url=None,
avito_proxy_max_rotations=0,
scraper_fetch_mode="browser",
browser_http_endpoint="http://tradein-browser:9000",
glitchtip_dsn=None,
avito_serp_ekb_only=False,
)
def _old_avito() -> Any:
with patch("app.services.scrapers.avito.get_scraper_delay", return_value=7.0):
return OldAvito()
# ── avito: probe → split → leaf ───────────────────────────────────────────────
_AVITO_WALK_EXTRA = {"room_slug": "room1", "room_label": "1к"}
@pytest.mark.asyncio
async def test_bisection_parity_avito() -> None:
"""avito: split-дерево + leaf + ранний возврат на нуле — обход идентичен.
Оракул: divisor=1000, cap=1400 → брекет 8М делится 3 уровня (8М→4М→2М→1М-leaf);
один под-брекет форсим в 0 (ранний возврат без leaf).
"""
# После первого split (0,7_999_999): mid=3_999_999 → дети (None,3_999_999) и
# (4_000_000, 7_999_999). Форсим 0 на (4_000_000, 5_999_999) — под-брекет после
# второго split правой ветки.
oracle = make_oracle(
divisor=1000,
open_ceiling=30_000_000,
zeros=frozenset({(4_000_000, 5_999_999)}),
)
brackets: list[tuple[int, int | None]] = [(0, 7_999_999), (25_000_000, None)]
async def run(scraper: Any) -> list[Any]:
events: list[Any] = []
_patch_avito(scraper, oracle, events)
for lo, hi in brackets:
await scraper._walk_price_range(
lo=lo,
hi=hi,
seen={},
price_cap_per_bucket=1400,
max_pages_per_bucket=100,
concurrency=5,
secondary_only=True,
on_bucket=None,
skip_buckets=None,
**_AVITO_WALK_EXTRA,
)
return events
old_events = await run(_old_avito())
new_events = await run(NewAvito(config=_KIT_CONFIG))
assert old_events == new_events
# Санити: обход реально нетривиален (несколько probe + несколько leaf).
assert sum(1 for e in old_events if e[0] == "leaf") >= 4
assert sum(1 for e in old_events if e[0] == "probe") >= 6
@pytest.mark.asyncio
async def test_bisection_parity_avito_probe_fail() -> None:
"""avito probe-fail (total=None): SPLIT_OR_SKIP — принудительный split делимого
закрытого брекета, best-effort leaf открытого — идентичен старому."""
# (0, 7_999_999): probe None → делим (bracket>=MIN). Дети (None,4M),(4000001,...):
# им дадим реальные счётчики. Открытый (25M, None) probe None → best-effort leaf.
oracle = make_oracle(
divisor=1000,
open_ceiling=30_000_000,
nones=frozenset({(None, 7_999_999), (25_000_000, None)}),
)
brackets: list[tuple[int, int | None]] = [(0, 7_999_999), (25_000_000, None)]
async def run(scraper: Any) -> list[Any]:
events: list[Any] = []
_patch_avito(scraper, oracle, events)
for lo, hi in brackets:
await scraper._walk_price_range(
room_slug="room1",
room_label="1к",
lo=lo,
hi=hi,
seen={},
price_cap_per_bucket=1400,
max_pages_per_bucket=100,
concurrency=5,
secondary_only=True,
on_bucket=None,
skip_buckets=None,
)
return events
old_events = await run(_old_avito())
new_events = await run(NewAvito(config=_KIT_CONFIG))
assert old_events == new_events
# Открытый брекет с probe-fail → ровно один best-effort leaf на (25M, None).
assert ("leaf", 25_000_000, None, 100) in old_events
# ── Full seed-grid обход (все seed-брекеты одной комнатности) ─────────────────
@pytest.mark.asyncio
async def test_bisection_parity_full_seed_grid() -> None:
"""Полный обход всех seed-брекетов (как в fetch_all_secondary) — включая
открытый catch-all — идентичен между старым и новым для avito."""
oracle = make_oracle(divisor=1500, open_ceiling=300_000_000)
async def run(scraper: Any) -> list[Any]:
events: list[Any] = []
_patch_avito(scraper, oracle, events)
for br_lo, br_hi in get_price_seed_brackets():
walk_hi = br_hi - 1 if br_hi is not None else None
await scraper._walk_price_range(
lo=br_lo,
hi=walk_hi,
seen={},
price_cap_per_bucket=1400,
max_pages_per_bucket=100,
concurrency=5,
secondary_only=True,
on_bucket=None,
skip_buckets=None,
**_AVITO_WALK_EXTRA,
)
return events
old_events = await run(_old_avito())
new_events = await run(NewAvito(config=_KIT_CONFIG))
assert old_events == new_events

369
tradein-mvp/backend/tests/test_scraper_kit_cian_golden_parity.py
View file

@ -1,369 +0,0 @@
"""Golden-parity: `scraper_kit.providers.cian.*` парсинг ≡ ещё живым `app.services.scrapers.cian*`.
Strangler-инвариант (#2133 cian): новая scraper_kit-копия cian-провайдера должна давать
БАЙТ-ИДЕНТИЧНЫЙ результат парсинга старому боевому коду на одинаковом входе. Развязка
(settings→ScraperConfig, get_scraper_delay→delay_provider, cian_session→локальная копия)
НЕ меняет распарсенные данные.
`app.services.scrapers.cian` / `cian_detail` удалены (#2397 Part E2, 0 runtime-
импортёров — admin.py/cian_history_backfill.py уже на kit). Parity-тесты SERP
`_parse_serp_html`/`_offer_to_lot`/`_extract_total_offers`/`_format_address` и
detail `fetch_detail` удалены вместе с ними (легаси-эталон исчез).
Оставшееся здесь сравнивает kit ≡ ЕЩЁ ЖИВЫЕ легаси-модули:
- `extract_state` — `app.services.scrapers.cian_state_parser` (живой, общий парсер state);
- valuation `_parse_valuation_state` / `estimate_via_cian_valuation` —
`app.services.scrapers.cian_valuation` (живой);
- newbuilding pure-экстракторы — `app.services.scrapers.cian_newbuilding` (живой);
- kit mandatory-config guards (`estimate_via_cian_valuation`, `cian.session`) — kit-only;
- strangler-guard: kit-провайдер не импортирует `app.*`.
Идентичность результата = kit-парсер верно скопирован (развязка не задела логику).
"""
from __future__ import annotations
import asyncio
import dataclasses
import json
import os
from types import SimpleNamespace
from typing import Any
from unittest.mock import MagicMock
import pytest
# Старый app.services.scrapers.cian_state_parser/cian_valuation/cian_newbuilding
# импортируют app.core.config.settings=Settings(), которому нужен DATABASE_URL.
# Офлайн-парсинг БД не трогает — фиктивный DSN достаточен.
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
# НОВЫЙ (scraper_kit) cian-провайдер.
from scraper_kit.cian_state_parser import extract_state as new_extract_state
from scraper_kit.providers.cian.newbuilding import _extract_chart as new_extract_chart
from scraper_kit.providers.cian.newbuilding import (
_extract_nested_offers as new_extract_nested_offers,
)
from scraper_kit.providers.cian.newbuilding import (
_extract_reliability_checks as new_extract_reliability,
)
from scraper_kit.providers.cian.newbuilding import (
_extract_zhk_url_from_serp as new_extract_zhk_url,
)
from scraper_kit.providers.cian.valuation import (
_parse_valuation_state as new_parse_valuation,
)
from scraper_kit.providers.cian.valuation import (
estimate_via_cian_valuation as new_estimate_via_cian_valuation,
)
# #2335 (Group E2): estimate_via_cian_valuation full-pipeline parity + mandatory-config
# regression guard. RealScraperConfig — тот же read-only адаптер над settings, что и
# #2306's cian_price_history.py (см. app/services/scraper_adapters.py).
from app.services.scraper_adapters import RealScraperConfig
# СТАРЫЙ (app.services.scrapers) — ещё живые cian-модули (cian.py/cian_detail.py удалены).
from app.services.scrapers.cian_newbuilding import _extract_chart as old_extract_chart
from app.services.scrapers.cian_newbuilding import (
_extract_nested_offers as old_extract_nested_offers,
)
from app.services.scrapers.cian_newbuilding import (
_extract_reliability_checks as old_extract_reliability,
)
from app.services.scrapers.cian_newbuilding import (
_extract_zhk_url_from_serp as old_extract_zhk_url,
)
from app.services.scrapers.cian_state_parser import extract_state as old_extract_state
from app.services.scrapers.cian_valuation import (
_parse_valuation_state as old_parse_valuation,
)
from app.services.scrapers.cian_valuation import (
estimate_via_cian_valuation as old_estimate_via_cian_valuation,
)
from tests.support.parity import assert_parity
# ── extract_state parity (state-парсер, живой в cian_state_parser.py) ────────
def _serp_html_for_state(state: dict[str, Any]) -> str:
"""Обернуть произвольный state в валидный _cianConfig['frontend-serp'].push() HTML."""
state_json = json.dumps(state, ensure_ascii=False)
return (
"<html><body><script>\n"
"window._cianConfig['frontend-serp'] = window._cianConfig['frontend-serp'] || [];\n"
"window._cianConfig['frontend-serp'].push({"
f"key: 'initialState', value: {state_json}, priority: 10, filter: null"
"});\n</script></body></html>"
)
def test_extract_state_parity() -> None:
"""app.extract_state ≡ scraper_kit.extract_state на SERP-фикстуре."""
html = _serp_html_for_state({"results": {"totalOffers": 1, "offers": []}})
old_state = old_extract_state(html, mfe="frontend-serp", key="initialState")
new_state = new_extract_state(html, mfe="frontend-serp", key="initialState")
assert old_state == new_state
assert new_state is not None
# ── valuation _parse_valuation_state parity (живой cian_valuation.py) ────────
def _valuation_state() -> dict[str, Any]:
return {
"user": {"isAuthenticated": True, "userId": 42},
"estimation": {
"sale": {
"data": {
"price": 6800000,
"accuracy": 87,
"priceFrom": 6400000,
"priceTo": 7200000,
"priceSqm": 130000,
"filtersHash": "hash-abc",
}
},
"rent": {
"data": {
"price": 42000,
"accuracy": 80,
"priceFrom": 39000,
"priceTo": 45000,
"taxPrice": 5460,
}
},
},
"estimationChart": {
"data": {
"title": {"change": "increase", "changeValue": "8,3%"},
"chartData": {
"data": [
{"date": 1714521600000, "price": 6700000, "priceFormatted": "6,7 млн"},
{"date": 1717200000000, "price": 6800000, "priceFormatted": "6,8 млн"},
]
},
}
},
"houseInfo": {
"data": {
"items": [
{"title": "Год постройки", "value": "2010"},
{"title": "Тип дома", "value": "монолит"},
]
}
},
"filters": {"houseId": 998877},
"managementCompany": {"data": {"name": "УК Ромашка", "phones": ["+73430000000"]}},
}
def test_valuation_parse_state_parity() -> None:
"""_parse_valuation_state → CianValuationResult идентичен (все поля)."""
state = _valuation_state()
old_res = old_parse_valuation(state)
new_res = new_parse_valuation(state)
assert dataclasses.asdict(old_res) == dataclasses.asdict(new_res)
# ── estimate_via_cian_valuation full-pipeline parity (#2335, Group E2) ───────
#
# test_valuation_parse_state_parity above only covers the pure parsing helper.
# These tests exercise the FULL async call path (cache lookup → load_session →
# curl_cffi fetch → extract_state → auth-check → parse → sanity-check → cache
# write), with config=RealScraperConfig() wired per the #2306 convention
# (app/services/cian_price_history.py), proving the kit path is safe to switch
# to whenever #2337 wires estimator.py's call site.
def _valuation_mfe_html(state: dict[str, Any]) -> str:
"""Wrap a valuation state dict as a valid _cianConfig push() HTML page."""
state_json = json.dumps(state, ensure_ascii=False)
return (
"<html><body><script>\n"
"window._cianConfig['valuation-for-agent-frontend'] = "
"window._cianConfig['valuation-for-agent-frontend'] || [];\n"
"window._cianConfig['valuation-for-agent-frontend'].push({"
f"key: 'initialState', value: {state_json}, priority: 10, filter: null"
"});\n</script></body></html>"
)
def _make_fake_async_session_cls(html: str) -> type:
"""Stand-in for curl_cffi.requests.AsyncSession — returns canned HTML on .get()."""
class _FakeAsyncSession:
def __init__(self, *args: Any, **kwargs: Any) -> None:
pass
async def __aenter__(self) -> _FakeAsyncSession:
return self
async def __aexit__(self, *exc: object) -> bool:
return False
async def get(self, url: str, allow_redirects: bool = True) -> SimpleNamespace:
return SimpleNamespace(status_code=200, text=html)
return _FakeAsyncSession
def test_estimate_via_cian_valuation_full_pipeline_parity(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Full estimate_via_cian_valuation() pipeline parity — not just the pure parser.
Mocks curl_cffi.AsyncSession (canned MFE HTML) + load_session (fixed cookies) on
BOTH legacy and kit modules, cache-miss on a MagicMock db, and drives both async
call paths with identical inputs (kit wired with config=RealScraperConfig()).
"""
html = _valuation_mfe_html(_valuation_state())
fake_session_cls = _make_fake_async_session_cls(html)
def _run_legacy() -> Any:
monkeypatch.setattr("app.services.scrapers.cian_valuation.AsyncSession", fake_session_cls)
monkeypatch.setattr(
"app.services.scrapers.cian_valuation.load_session",
lambda db: {"DMIR_AUTH": "token"},
)
db = MagicMock()
db.execute.return_value.mappings.return_value.first.return_value = None # cache miss
return asyncio.run(
old_estimate_via_cian_valuation(
db,
address="Свердловская обл, Екатеринбург",
total_area=50.0,
rooms_count=2,
floor=5,
total_floors=16,
use_cache=False,
)
)
def _run_kit() -> Any:
monkeypatch.setattr("scraper_kit.providers.cian.valuation.AsyncSession", fake_session_cls)
monkeypatch.setattr(
"scraper_kit.providers.cian.valuation.load_session",
lambda db, *, config: {"DMIR_AUTH": "token"},
)
db = MagicMock()
db.execute.return_value.mappings.return_value.first.return_value = None # cache miss
return asyncio.run(
new_estimate_via_cian_valuation(
db,
config=RealScraperConfig(),
address="Свердловская обл, Екатеринбург",
total_area=50.0,
rooms_count=2,
floor=5,
total_floors=16,
use_cache=False,
)
)
assert_parity(
legacy_fn=_run_legacy,
kit_fn=_run_kit,
fixtures=[()],
)
def test_kit_estimate_via_cian_valuation_requires_config() -> None:
"""Kit's estimate_via_cian_valuation MUST require config= (no default) — mandatory.
Regression guard (#2335): config: ScraperConfig has NO default in the kit
signature, so calling it without config= raises TypeError immediately (at
argument-binding time, before any coroutine runs). This is intentional and
must stay this way: estimator.py's call site (#2337) sits inside a blanket
`except Exception` used for graceful multi-source degradation — a future
change that gave config= a default would let a wiring mistake silently
degrade into "source unavailable" instead of raising loudly at call-time.
See #2306 for the precedent this guards against (silent-degradation footgun
from an optional dependency parameter).
"""
db = MagicMock()
with pytest.raises(TypeError, match="config"):
new_estimate_via_cian_valuation( # not awaited — TypeError raises at call time
db,
address="Свердловская обл, Екатеринбург",
total_area=50.0,
rooms_count=2,
floor=5,
)
def test_kit_cian_session_config_requirement_asymmetry() -> None:
"""scraper_kit.providers.cian.session: load_session requires config=, but
mark_session_invalid does NOT.
Confirmed by direct inspection for #2335 — the two helpers estimate_via_cian_valuation
calls internally are NOT symmetric re: config, unlike the assumption in the original
task brief ("the kit version of those also requires config mandatorily"). Documented
here as a regression guard so a future refactor that changes either signature is a
deliberate, visible decision rather than an accidental drift.
"""
from scraper_kit.providers.cian.session import load_session, mark_session_invalid
db = MagicMock()
with pytest.raises(TypeError, match="config"):
load_session(db) # missing mandatory config=
db2 = MagicMock()
mark_session_invalid(db2, 42) # no config parameter at all — must NOT raise
db2.execute.assert_called_once()
# ── newbuilding pure-экстракторы parity (живой cian_newbuilding.py) ──────────
def test_newbuilding_extractors_parity() -> None:
rv_state = {
"data": {
"priceDynamics": {
"chart": {
"data": {"labels": ["2026-01-01", "2026-02-01"], "values": [120000, 125000]}
}
}
}
}
assert old_extract_chart(rv_state) == new_extract_chart(rv_state)
rel_state = {
"checkStatus": {"status": "reliable", "title": "Надёжный застройщик"},
"details": [{"title": "Проектная декларация", "type": "doc"}],
}
assert old_extract_reliability(rel_state) == new_extract_reliability(rel_state)
offers_state = {
"data": {
"total": {
"fromDeveloperRooms": [
{"roomType": "1", "layouts": [{"id": 1, "area": 38}]},
{"roomType": "2", "layouts": [{"id": 2, "area": 55}]},
]
}
}
}
assert old_extract_nested_offers(offers_state) == new_extract_nested_offers(offers_state)
serp_html = (
'<h1 data-name="Title">Купить квартиру в '
'<a href="https://zhk-parkovyy-kvartal-ekb-i.cian.ru/">ЖК «Парковый квартал»</a></h1>'
)
assert old_extract_zhk_url(serp_html) == new_extract_zhk_url(serp_html)
# ── strangler-guard: kit-провайдер не импортирует app.* ──────────────────────
def test_kit_cian_has_no_app_imports() -> None:
"""Ни один модуль scraper_kit.providers.cian.* не импортирует app.* (развязка)."""
import inspect
from scraper_kit.providers.cian import detail, newbuilding, serp, session, valuation
for mod in (serp, detail, newbuilding, valuation, session):
source = inspect.getsource(mod)
for line in source.splitlines():
stripped = line.strip()
assert not stripped.startswith("from app"), f"{mod.__name__}: {line!r}"
assert not stripped.startswith("import app"), f"{mod.__name__}: {line!r}"

42
tradein-mvp/backend/tests/test_scraper_kit_group_c_backfill_kit_parity.py
View file

@ -43,18 +43,21 @@ cian_detail.fetch_detail` vs kit, browser_fetcher call shape) removed (#2397 Par
— `app.services.scrapers.cian_detail` deleted, 0 runtime importers (cian_history_backfill.py
already calls the kit `fetch_detail` exclusively, proven by
`test_cian_history_backfill_browser_fetcher_uses_settings_endpoint` below).
`test_avito_fetch_detail_via_browser_fetcher_parity` (legacy `app.services.scrapers.
avito_detail.fetch_detail` vs kit, browser_fetcher call shape) removed (#2397 финальный
шаг E) — `app.services.scrapers.avito_detail` deleted, 0 runtime importers. Kit-only
avito_detail_backfill.py coverage of the browser_fetcher branch remains via
`test_pipeline_browser_routing.py::test_fetch_detail_browser_returns_enrichment`.
"""
from __future__ import annotations
import asyncio
import dataclasses
import os
from typing import Any
from unittest.mock import AsyncMock, MagicMock, patch
# Legacy app.services.scrapers.* / app.core.config.settings require DATABASE_URL at
# import time (same pattern as tests/scrapers/test_avito_detail_kit_parity.py).
# app.core.config.settings requires DATABASE_URL at import time.
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
from scraper_kit.avito_exceptions import AvitoBlockedError as KitAvitoBlockedError
@ -63,8 +66,6 @@ from scraper_kit.providers.avito.serp import AvitoScraper as KitAvitoScraper
from app.core.config import settings
from app.services.scraper_adapters import RealScraperConfig
from app.services.scrapers.avito_detail import fetch_detail as legacy_avito_fetch_detail
from tests.support.parity import assert_parity
# ── avito fetch_detail: backconnect config-gate (avito_detail_backfill.py delta) ──
@ -149,35 +150,6 @@ def test_avito_scraper_rotate_ip_reads_injected_config() -> None:
assert scraper._config.avito_proxy_rotate_url == settings.avito_proxy_rotate_url
# ── avito fetch_detail via browser_fetcher (avito_detail_backfill.py browser_mode) ──
def _fake_browser_fetcher(html: str) -> Any:
bf = MagicMock()
bf.fetch = AsyncMock(return_value=html)
return bf
def test_avito_fetch_detail_via_browser_fetcher_parity() -> None:
"""browser_mode branch of avito_detail_backfill.py calls fetch_detail(url,
browser_fetcher=browser_fetcher) — config is never read on this path (early
return before the config-gated backconnect logic). Confirms legacy/kit agree."""
html = _detail_html()
url = "https://www.avito.ru/ekaterinburg/kvartiry/test_1"
def _run_legacy() -> Any:
return dataclasses.asdict(
asyncio.run(legacy_avito_fetch_detail(url, browser_fetcher=_fake_browser_fetcher(html)))
)
def _run_kit() -> Any:
return dataclasses.asdict(
asyncio.run(kit_avito_fetch_detail(url, browser_fetcher=_fake_browser_fetcher(html)))
)
assert_parity(legacy_fn=_run_legacy, kit_fn=_run_kit, fixtures=[()])
# ── BrowserFetcher(endpoint=) wiring — house_imv_backfill.py / cian_history_backfill.py ──

109
tradein-mvp/backend/tests/test_scraper_kit_pricehistory_session_parity.py
View file

@ -1,109 +0,0 @@
"""Parity tests for issue #2306 (Group B of scraper_kit migration epic #2277).
Migrates the INTERNAL legacy-scraper imports of three product-side files to their
`scraper_kit` equivalents (public API of the three files is unchanged — no caller
elsewhere in the codebase needed updating, see vault audit
`Scraper_Kit_Legacy_Dependency_Audit_0703`):
- app/services/cian_price_history.py → fetch_detail/save_detail_enrichment now from
`scraper_kit.providers.cian.detail` (was `app.services.scrapers.cian_detail`).
- app/services/cian_session.py → extract_state now from
`scraper_kit.cian_state_parser` (was `app.services.scrapers.cian_state_parser`).
- app/services/yandex_price_history.py → ScrapedLot now from `scraper_kit.base`
(was `app.services.scrapers.base`).
This file proves the DELTA introduced by the migration specifically — general
cian-provider / ScrapedLot golden-parity is already covered by
`tests/test_scraper_kit_cian_golden_parity.py` and
`tests/test_scraper_kit_base_golden_parity.py`. Uses `tests/support/parity.assert_parity`
per the epic's parity-gate convention (issue #2304).
`test_fetch_detail_with_config_injection_parity` / `test_fetch_detail_own_session_proxy_
wiring_parity` (legacy `app.services.scrapers.cian_detail.fetch_detail` vs kit) removed
(#2397 Part E2) — `cian_detail` deleted, 0 runtime importers. `extract_state`
(`cian_state_parser`, still legacy-live) and `ScrapedLot` (`base`, still legacy-live)
parity below is unaffected.
"""
from __future__ import annotations
import json
import os
from typing import Any
# Старый app.services.scrapers.cian_state_parser/base / app.core.config.settings
# требует DATABASE_URL при импорте (тот же паттерн, что в
# test_scraper_kit_cian_golden_parity.py).
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
# Kit — то, на что теперь ссылаются мигрированные файлы (#2306).
from scraper_kit.base import ScrapedLot as KitScrapedLot
from scraper_kit.cian_state_parser import extract_state as kit_extract_state
# Legacy — эталон сравнения (эти модули НЕ меняются в #2306, остаются live).
from app.services.scrapers.base import ScrapedLot as LegacyScrapedLot
from app.services.scrapers.cian_state_parser import extract_state as legacy_extract_state
from tests.support.parity import assert_parity
# ── extract_state parity (cian_session.py dependency) ─────────────────────────
def _auth_state_html() -> str:
state = {"user": {"isAuthenticated": True, "userId": 42}}
state_json = json.dumps(state, ensure_ascii=False)
return (
"<html><body><script>\n"
"window._cianConfig['header-frontend'] = window._cianConfig['header-frontend'] || [];\n"
"window._cianConfig['header-frontend'].push({"
f"key: 'initialState', value: {state_json}, priority: 10, filter: null"
"});\n</script></body></html>"
)
def test_extract_state_parity_on_match() -> None:
"""cian_session.py._classify_verify_response reads this exact (mfe, key) pair."""
html = _auth_state_html()
assert_parity(
legacy_fn=legacy_extract_state,
kit_fn=kit_extract_state,
fixtures=[(html, "header-frontend", "initialState")],
)
def test_extract_state_parity_on_miss() -> None:
"""Missing mfe/key → both return None identically (not just on the happy path)."""
html = _auth_state_html()
assert_parity(
legacy_fn=legacy_extract_state,
kit_fn=kit_extract_state,
fixtures=[(html, "frontend-offer-card", "defaultState")],
)
# ── ScrapedLot parity (yandex_price_history.py dependency) ────────────────────
def _lot_kwargs() -> dict[str, Any]:
return {
"source": "yandex",
"source_url": "https://realty.yandex.ru/offer/123/",
"source_id": "123",
"price_rub": 5_000_000,
"price_previous_rub": 5_200_000,
"price_trend": "DECREASED",
}
def test_scraped_lot_model_dump_parity() -> None:
"""record_yandex_price_history duck-types lot.source/.source_id/.price_rub/...
Legacy- и kit-ScrapedLot должны давать идентичный model_dump() на одном входе —
доказывает, что смена типа импорта в yandex_price_history.py не меняет runtime-
поведение (функция принимает оба класса взаимозаменяемо).
"""
kwargs = _lot_kwargs()
assert_parity(
legacy_fn=lambda kw: LegacyScrapedLot(**kw).model_dump(),
kit_fn=lambda kw: KitScrapedLot(**kw).model_dump(),
fixtures=[kwargs],
)

123
tradein-mvp/backend/tests/test_scraper_kit_yandex_golden_parity.py
View file

@ -1,123 +0,0 @@
"""Golden-parity: `scraper_kit.providers.yandex.*` парсинг ≡ ещё живым
`app.services.scrapers.yandex_*`.
Strangler-инвариант (#2133): новая scraper_kit-копия yandex-провайдера должна давать
БАЙТ-ИДЕНТИЧНЫЙ результат парсинга старому боевому коду на одинаковом входе.
Развязка (settings→ScraperConfig, get_scraper_delay→delay_provider,
house_type_normalizer→providers.yandex.shared, base/browser_fetcher/helpers→scraper_kit.*)
НЕ меняет распарсенные данные.
`app.services.scrapers.yandex_realty` удалён (#2397 Part E2, 0 runtime-импортёров).
SERP `_entity_to_lot`/`_parse_gate_json`/`_build_url` parity-тесты (сравнение с
`YandexRealtyScraper`) удалены вместе с ним (легаси-эталон исчез).
Оставшееся сравнивает kit ≡ ЕЩЁ ЖИВЫЕ легаси-модули:
- `normalize_house_type` — `app.services.scrapers.house_type_normalizer` (живой);
- newbuilding `parse` (ЖК-лендинг → YandexNewbuildingInfo) —
`app.services.scrapers.yandex_newbuilding` (живой);
- strangler-guard: kit-провайдер не импортирует `app.*`.
Легаси `app.services.scrapers.yandex_detail` / `yandex_valuation` удалены (#2277
финальный шаг scraper_kit-миграции, 0 runtime importers) — detail/valuation
parity-проверки убраны отсюда ранее. Kit-side coverage для detail/valuation
`.parse()` сохранена без изменений в `tests/test_yandex_detail.py` /
`tests/test_yandex_detail_structural.py` / `tests/test_yandex_valuation.py` /
`tests/test_yandex_valuation_save.py` (переведены на kit-эквиваленты тем же PR).
Идентичность результата = kit-парсер верно скопирован (развязка не задела логику).
"""
from __future__ import annotations
import inspect
import os
# Старый app.services.scrapers.house_type_normalizer/yandex_newbuilding импортирует
# app.core.config.settings=Settings(), которому нужен DATABASE_URL. Офлайн-парсинг
# БД не трогает — фиктивный DSN достаточен.
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
# НОВЫЙ (scraper_kit) и СТАРЫЙ (app.services.scrapers) yandex-парсеры — оба в одном
# тесте; ruff сортирует scraper_kit (first-party) выше app-импортов.
from scraper_kit.providers.yandex.newbuilding import (
YandexNewbuildingScraper as NewNewbuildingScraper,
)
from scraper_kit.providers.yandex.shared import normalize_house_type as new_normalize_house_type
from app.services.scrapers.house_type_normalizer import (
normalize_house_type as old_normalize_house_type,
)
from app.services.scrapers.yandex_newbuilding import (
YandexNewbuildingScraper as OldNewbuildingScraper,
)
# ── normalize_house_type parity ──────────────────────────────────────────────
def test_normalize_house_type_parity() -> None:
"""SCREAMING / camelCase / канон / мусор → идентичный результат."""
raws = [
None,
"",
"MONOLIT",
"BRICK",
"PANEL",
"MONOLIT_BRICK",
"monolithBrick",
"gasSilicateBlock",
"monolith_brick",
"monolith",
"stalin",
"OTHER",
" MONOLIT ",
]
for raw in raws:
assert old_normalize_house_type(raw) == new_normalize_house_type(raw)
# ── newbuilding parse parity (синтетика) ─────────────────────────────────────
_NEWBUILDING_HTML = """
<html><body>
<h1>ЖК «Татлин»</h1>
Екатеринбург, ул. Черепанова 4.3 из 5 1505 оценок Смотреть все 353 отзыва
56.855312 60.576668
<h2>О комплексе</h2>
<div>Дом комфорт-класса, монолитный. Введён в эксплуатацию в июнь 2023.
35-этажные башни на участке 1,5 га. Три корпуса.</div>
</body></html>
"""
def test_newbuilding_parse_parity() -> None:
"""ЖК-лендинг → YandexNewbuildingInfo идентичен."""
old_scraper = OldNewbuildingScraper()
new_scraper = NewNewbuildingScraper()
old_res = old_scraper.parse(
_NEWBUILDING_HTML,
jk_slug="tatlin",
jk_id="1592987",
source_url="https://realty.yandex.ru/ekaterinburg/kupit/novostrojka/tatlin-1592987/",
)
new_res = new_scraper.parse(
_NEWBUILDING_HTML,
jk_slug="tatlin",
jk_id="1592987",
source_url="https://realty.yandex.ru/ekaterinburg/kupit/novostrojka/tatlin-1592987/",
)
assert old_res.model_dump() == new_res.model_dump()
# ── strangler-guard: kit-провайдер не импортирует app.* ──────────────────────
def test_kit_yandex_has_no_app_imports() -> None:
"""Ни один модуль scraper_kit.providers.yandex.* не импортирует app.* (развязка)."""
from scraper_kit.providers.yandex import detail, newbuilding, serp, shared, valuation
for mod in (serp, detail, valuation, newbuilding, shared):
source = inspect.getsource(mod)
for line in source.splitlines():
stripped = line.strip()
assert not stripped.startswith("from app"), f"{mod.__name__}: {line!r}"
assert not stripped.startswith("import app"), f"{mod.__name__}: {line!r}"

80
tradein-mvp/backend/tests/test_scraper_proxy.py
View file

@ -16,6 +16,15 @@ Covers:
эквивалентная и более точная kit-регрессия (config footgun: с/без `config=`) уже
покрыта в `tests/scrapers/test_avito_imv_kit_parity.py::
test_kit_evaluate_via_imv_{with,without}_config_{uses,drops}_proxy`.
Легаси `app.services.scrapers.cian_newbuilding` / `avito_detail` удалены (#2397
финальный шаг E, 0 runtime-импортёров) — CianNewbuilding/AvitoDetail proxy-wiring
тесты переведены на kit-эквиваленты (`scraper_kit.providers.cian.newbuilding` /
`scraper_kit.providers.avito.detail`). Kit `cian_newbuilding` больше не импортирует
`AsyncSession` на уровне модуля рантайма (только `TYPE_CHECKING` — страница ЖК тянется
исключительно через BrowserFetcher), поэтому patch.object(..., "AsyncSession", ...)
заменён на `hasattr`-проверку отсутствия атрибута (та же гарантия: page-fetch НЕ
использует curl_cffi AsyncSession).
"""
from __future__ import annotations
@ -243,13 +252,13 @@ async def test_cian_newbuilding_own_session_receives_proxies():
Проверяем:
- BrowserFetcher.fetch(url) вызывается с правильным URL
- curl_cffi AsyncSession НЕ создаётся для page-fetch (прокси — browser-level)
- curl_cffi AsyncSession НЕ создаётся для page-fetch (прокси — browser-level;
kit-модуль вообще не импортирует AsyncSession на уровне рантайма)
"""
from app.services.scrapers import cian_newbuilding
from scraper_kit.providers.cian import newbuilding as cian_newbuilding
zhk_url = "https://zhk-test.cian.ru/"
fetch_calls: list[str] = []
session_ctor_calls: list[dict] = []
mock_fetcher = MagicMock()
mock_fetcher.fetch = AsyncMock(
@ -258,18 +267,14 @@ async def test_cian_newbuilding_own_session_receives_proxies():
mock_fetcher.__aenter__ = AsyncMock(return_value=mock_fetcher)
mock_fetcher.__aexit__ = AsyncMock(return_value=None)
def _fake_async_session(*args, **kwargs):
session_ctor_calls.append(kwargs)
return MagicMock()
with patch.object(cian_newbuilding, "BrowserFetcher", lambda *a, **k: mock_fetcher):
with patch.object(cian_newbuilding, "AsyncSession", _fake_async_session):
await cian_newbuilding.fetch_newbuilding(zhk_url)
await cian_newbuilding.fetch_newbuilding(zhk_url)
# BrowserFetcher.fetch вызван с ЖК-url
assert fetch_calls == [zhk_url]
# curl_cffi AsyncSession НЕ создавался для page-fetch
assert session_ctor_calls == []
# curl_cffi AsyncSession НЕ создавался для page-fetch (нет такого атрибута на
# уровне рантайма — только BrowserFetcher).
assert not hasattr(cian_newbuilding, "AsyncSession")
@pytest.mark.asyncio
@ -279,11 +284,10 @@ async def test_cian_newbuilding_shared_session_not_recreated():
Даже когда передан внешний session — страница ЖК тянется через BrowserFetcher,
AsyncSession не пересоздаётся, session.get не вызывается.
"""
from app.services.scrapers import cian_newbuilding
from scraper_kit.providers.cian import newbuilding as cian_newbuilding
zhk_url = "https://zhk-test.cian.ru/"
fetch_calls: list[str] = []
session_ctor_calls: list[dict] = []
mock_fetcher = MagicMock()
mock_fetcher.fetch = AsyncMock(
@ -292,20 +296,16 @@ async def test_cian_newbuilding_shared_session_not_recreated():
mock_fetcher.__aenter__ = AsyncMock(return_value=mock_fetcher)
mock_fetcher.__aexit__ = AsyncMock(return_value=None)
def _fake_async_session(*args, **kwargs):
session_ctor_calls.append(kwargs)
return MagicMock()
sentinel_session = MagicMock(name="external_session")
with patch.object(cian_newbuilding, "BrowserFetcher", lambda *a, **k: mock_fetcher):
with patch.object(cian_newbuilding, "AsyncSession", _fake_async_session):
await cian_newbuilding.fetch_newbuilding(zhk_url, session=sentinel_session)
await cian_newbuilding.fetch_newbuilding(zhk_url, session=sentinel_session)
# BrowserFetcher.fetch вызван — именно через него тянется страница
assert fetch_calls == [zhk_url]
# Ни нового AsyncSession, ни вызова переданного session.get — proxy browser-level
assert session_ctor_calls == []
# Ни нового AsyncSession (атрибута вообще нет на рантайме), ни вызова переданного
# session.get — proxy browser-level
assert not hasattr(cian_newbuilding, "AsyncSession")
sentinel_session.get.assert_not_called()
@ -315,7 +315,7 @@ async def test_cian_newbuilding_shared_session_not_recreated():
@pytest.mark.asyncio
async def test_avito_detail_own_session_receives_proxies():
"""fetch_detail creates own session with proxies= when no cffi_session passed."""
from app.services.scrapers import avito_detail
from scraper_kit.providers.avito import detail as avito_detail
mock_session = MagicMock()
mock_resp = MagicMock()
@ -331,12 +331,15 @@ async def test_avito_detail_own_session_receives_proxies():
captured_kwargs.update(kwargs)
return mock_session
with patch.object(avito_detail, "AsyncSession", _fake_session):
with patch("app.core.config.settings", _mock_settings(_PROXY_URL)):
try:
await avito_detail.fetch_detail("/ekaterinburg/kvartiry/test-1234")
except ValueError:
pass # parse_detail_html raises ValueError (no item_id) — that's ok
# Kit _build_detail_session delegates to scraper_kit.providers._base.build_document_session
# (shared helper), which constructs AsyncSession there — not in the detail module itself.
# config= (strangler DI, #2133) replaces the old app.core.config.settings patch.
cfg = SimpleNamespace(scraper_proxy_url=_PROXY_URL)
with patch("scraper_kit.providers._base.AsyncSession", _fake_session):
try:
await avito_detail.fetch_detail("/ekaterinburg/kvartiry/test-1234", config=cfg)
except ValueError:
pass # parse_detail_html raises ValueError (no item_id) — that's ok
assert captured_kwargs.get("proxies") == _EXPECTED_PROXIES
@ -344,7 +347,7 @@ async def test_avito_detail_own_session_receives_proxies():
@pytest.mark.asyncio
async def test_avito_detail_shared_session_not_patched():
"""fetch_detail does NOT build own session when cffi_session is provided."""
from app.services.scrapers import avito_detail
from scraper_kit.providers.avito import detail as avito_detail
mock_session = MagicMock()
mock_resp = MagicMock()
@ -359,15 +362,16 @@ async def test_avito_detail_shared_session_not_patched():
ctor_calls.append(kwargs)
return MagicMock()
with patch.object(avito_detail, "AsyncSession", _fake_session):
with patch("app.core.config.settings", _mock_settings(_PROXY_URL)):
try:
await avito_detail.fetch_detail(
"/ekaterinburg/kvartiry/test-1234",
cffi_session=mock_session,
)
except ValueError:
pass
cfg = SimpleNamespace(scraper_proxy_url=_PROXY_URL)
with patch("scraper_kit.providers._base.AsyncSession", _fake_session):
try:
await avito_detail.fetch_detail(
"/ekaterinburg/kvartiry/test-1234",
cffi_session=mock_session,
config=cfg,
)
except ValueError:
pass
# No own session created — shared session was used as-is
assert ctor_calls == []

69
tradein-mvp/backend/tests/test_snapshot_writer.py
View file

@ -12,10 +12,15 @@
`app.services.scrapers.cian_detail` удалён (#2397 Part E2, 0 runtime-импортёров) —
`DetailEnrichment`/`save_detail_enrichment` ретаргетированы на
`scraper_kit.providers.cian.detail` (byte-identical strangler-копия; `snapshot_writer`
тестируется по-прежнему через живой `app.services.scrapers.snapshot_writer` —
kit и app-версии этого модуля идентичны, kit's `save_detail_enrichment` импортирует
`upsert_listing_snapshot` из `scraper_kit.snapshot_writer`).
`scraper_kit.providers.cian.detail` (byte-identical strangler-копия).
`app.services.scrapers.base`/`snapshot_writer` удалены (#2397 финальный шаг E, 0
runtime-импортёров) — `upsert_listing_snapshot`/`ScrapedLot`/`save_listings`
переведены на `scraper_kit.snapshot_writer`/`scraper_kit.base`. Kit `save_listings`
принимает инжектируемый `matcher: HouseMatcher` (mandatory kwarg, strangler DI
#2133) вместо прямого импорта `match_or_create_house`/`upsert_listing_source` —
тесты используют `MagicMock()` matcher (тот же паттерн, что
`test_scraped_at_bump_2206.py::_kit_matcher`), плюс mandatory `region_code=66`.
"""
from __future__ import annotations
@ -24,9 +29,18 @@ from contextlib import contextmanager
from datetime import date
from unittest.mock import MagicMock, patch
from scraper_kit.base import ScrapedLot, save_listings
from scraper_kit.providers.cian.detail import DetailEnrichment, save_detail_enrichment
from scraper_kit.snapshot_writer import upsert_listing_snapshot
def _mock_matcher() -> MagicMock:
"""MagicMock HouseMatcher — mirror test_scraped_at_bump_2206.py::_kit_matcher."""
matcher = MagicMock()
matcher.match_or_create_house.return_value = (1, 1.0, "new")
matcher.upsert_listing_source.return_value = None
return matcher
from app.services.scrapers.snapshot_writer import upsert_listing_snapshot
# ── Helpers ───────────────────────────────────────────────────────────────────
@ -321,7 +335,6 @@ def _mock_db_for_save_listings(
def test_save_listings_writes_snapshot_per_lot():
"""save_listings вызывает upsert_listing_snapshot для каждого сохранённого lot."""
from app.services.scrapers.base import ScrapedLot, save_listings
db = _mock_db_for_save_listings(inserted=True, listing_id=77)
lots = [
@ -339,12 +352,8 @@ def test_save_listings_writes_snapshot_per_lot():
),
]
with (
patch("app.services.scrapers.base.match_or_create_house", return_value=(1, 1.0, "new")),
patch("app.services.scrapers.base.upsert_listing_source"),
patch("app.services.scrapers.base.upsert_listing_snapshot") as mock_snap,
):
save_listings(db, lots, run_id=10)
with patch("scraper_kit.base.upsert_listing_snapshot") as mock_snap:
save_listings(db, lots, matcher=_mock_matcher(), region_code=66, run_id=10)
assert mock_snap.call_count == 2, "snapshot должен вызваться для каждого lot"
# Первый вызов
@ -357,7 +366,6 @@ def test_save_listings_writes_snapshot_per_lot():
def test_save_listings_snapshot_receives_run_id():
"""Если передан run_id — он пробрасывается в upsert_listing_snapshot."""
from app.services.scrapers.base import ScrapedLot, save_listings
db = _mock_db_for_save_listings(listing_id=88)
lot = ScrapedLot(
@ -367,12 +375,8 @@ def test_save_listings_snapshot_receives_run_id():
price_rub=3_000_000,
)
with (
patch("app.services.scrapers.base.match_or_create_house", return_value=(1, 1.0, "new")),
patch("app.services.scrapers.base.upsert_listing_source"),
patch("app.services.scrapers.base.upsert_listing_snapshot") as mock_snap,
):
save_listings(db, [lot], run_id=55)
with patch("scraper_kit.base.upsert_listing_snapshot") as mock_snap:
save_listings(db, [lot], matcher=_mock_matcher(), region_code=66, run_id=55)
kwargs = mock_snap.call_args.kwargs
assert kwargs["run_id"] == 55
@ -380,7 +384,6 @@ def test_save_listings_snapshot_receives_run_id():
def test_save_listings_snapshot_none_run_id_by_default():
"""Без run_id — upsert_listing_snapshot получает run_id=None."""
from app.services.scrapers.base import ScrapedLot, save_listings
db = _mock_db_for_save_listings(listing_id=9)
lot = ScrapedLot(
@ -390,12 +393,8 @@ def test_save_listings_snapshot_none_run_id_by_default():
price_rub=2_000_000,
)
with (
patch("app.services.scrapers.base.match_or_create_house", return_value=(1, 1.0, "new")),
patch("app.services.scrapers.base.upsert_listing_source"),
patch("app.services.scrapers.base.upsert_listing_snapshot") as mock_snap,
):
save_listings(db, [lot]) # no run_id
with patch("scraper_kit.base.upsert_listing_snapshot") as mock_snap:
save_listings(db, [lot], matcher=_mock_matcher(), region_code=66) # no run_id
kwargs = mock_snap.call_args.kwargs
assert kwargs["run_id"] is None
@ -403,7 +402,6 @@ def test_save_listings_snapshot_none_run_id_by_default():
def test_save_listings_snapshot_failure_does_not_abort_batch():
"""Ошибка snapshot не прерывает основной INSERT; все lots сохраняются."""
from app.services.scrapers.base import ScrapedLot, save_listings
db = _mock_db_for_save_listings(inserted=True, listing_id=5)
lots = [
@ -416,15 +414,11 @@ def test_save_listings_snapshot_failure_does_not_abort_batch():
for i in range(1, 4)
]
with (
patch("app.services.scrapers.base.match_or_create_house", return_value=(1, 1.0, "new")),
patch("app.services.scrapers.base.upsert_listing_source"),
patch(
"app.services.scrapers.base.upsert_listing_snapshot",
side_effect=RuntimeError("snapshot fail"),
),
with patch(
"scraper_kit.base.upsert_listing_snapshot",
side_effect=RuntimeError("snapshot fail"),
):
inserted, _updated = save_listings(db, lots)
inserted, _updated = save_listings(db, lots, matcher=_mock_matcher(), region_code=66)
# INSERT listings всё равно прошёл для всех 3 lots
assert inserted == 3
@ -432,12 +426,11 @@ def test_save_listings_snapshot_failure_does_not_abort_batch():
def test_save_listings_empty_no_snapshot_call():
"""Пустой список → upsert_listing_snapshot не вызывается."""
from app.services.scrapers.base import save_listings
db = _mock_db_for_save_listings()
with patch("app.services.scrapers.base.upsert_listing_snapshot") as mock_snap:
save_listings(db, [])
with patch("scraper_kit.base.upsert_listing_snapshot") as mock_snap:
save_listings(db, [], matcher=_mock_matcher(), region_code=66)
mock_snap.assert_not_called()

271
tradein-mvp/backend/tests/test_yandex_helpers.py
View file

@ -1,271 +0,0 @@
"""Unit tests for Yandex Realty parsing helpers."""
from datetime import date
from app.services.scrapers.yandex_helpers import (
RE_FLOOR,
RE_JK_ID,
RE_OFFER_ID,
RE_TITLE_AREA,
RE_TITLE_ROOMS,
extract_json_ld,
find_ld_by_type,
parse_dmy,
parse_house_class,
parse_house_type,
parse_ru_date,
parse_rub,
)
# --- house_type ---
def test_parse_house_type_panel():
assert parse_house_type("в тёплом, панельном доме 1981 года") == "panel"
def test_parse_house_type_monolith_brick():
assert parse_house_type("кирпично-монолитный дом") == "monolith_brick"
def test_parse_house_type_monolith_only():
assert parse_house_type("монолитное здание") == "monolith"
def test_parse_house_type_brick_only():
assert parse_house_type("кирпичный дом 1995") == "brick"
def test_parse_house_type_unknown():
assert parse_house_type("из железобетона") is None
assert parse_house_type("") is None
assert parse_house_type(None) is None
# --- house_class ---
def test_parse_house_class_comfort_plus():
assert parse_house_class("дома класса комфорт+") == "comfort_plus"
def test_parse_house_class_comfort():
assert parse_house_class("ЖК класса комфорт в центре") == "comfort"
def test_parse_house_class_business():
assert parse_house_class("проект класса бизнес") == "business"
def test_parse_house_class_premium():
assert parse_house_class("ЖК премиум-класса") == "premium"
def test_parse_house_class_elite():
assert parse_house_class("элитный класс жилья") == "elite"
def test_parse_house_class_none():
assert parse_house_class("обычный жилой комплекс") is None
assert parse_house_class(None) is None
# --- parse_rub ---
def test_parse_rub_full_price():
assert parse_rub("4 399 000 ₽") == 4_399_000
def test_parse_rub_mln_comma():
assert parse_rub("4,4 млн ₽") == 4_400_000
def test_parse_rub_mln_dot():
assert parse_rub("4.4 млн ₽") == 4_400_000
def test_parse_rub_ppm2():
assert parse_rub("117 500 ₽ за м²") == 117_500
def test_parse_rub_raw_digits():
assert parse_rub("117500") == 117_500
def test_parse_rub_empty():
assert parse_rub("") is None
assert parse_rub(None) is None
assert parse_rub("без цены") is None
def test_parse_rub_nbsp_separator():
"""Yandex SERP форматирует «9\\xa0800\\xa0000» через NBSP, не regular space.
До fix'а `.replace(" ", "")` оставлял \\xa0 → int() падал ValueError →
parse_rub() возвращал None → весь scraper не парсил lots (0/23 на live HTML).
"""
assert parse_rub("9\xa0800\xa0000\xa0₽") == 9_800_000
assert parse_rub("4\xa0399\xa0000") == 4_399_000
def test_parse_rub_mln_with_nbsp():
"""«4,4\\xa0млн\\xa0₽» — NBSP вокруг unit-keyword тоже встречается."""
assert parse_rub("4,4\xa0млн\xa0₽") == 4_400_000
# --- parse_ru_date ---
def test_parse_ru_date_basic():
assert parse_ru_date("9 мая 2026") == date(2026, 5, 9)
def test_parse_ru_date_in_sentence():
assert parse_ru_date("опубликовано 17 февраля 2024") == date(2024, 2, 17)
def test_parse_ru_date_invalid_month():
assert parse_ru_date("9 invalidmonth 2026") is None
def test_parse_ru_date_dmy_not_supported():
# parse_ru_date doesn't handle DD.MM.YYYY (that's parse_dmy's job)
assert parse_ru_date("01.02.2026") is None
def test_parse_ru_date_empty():
assert parse_ru_date("") is None
assert parse_ru_date(None) is None
# --- parse_listing_date (extended formats) ---
def test_listing_date_today_keyword():
from datetime import timedelta
from app.services.scrapers.yandex_helpers import parse_listing_date
assert parse_listing_date("сегодня в 12:30") == date.today()
assert parse_listing_date("вчера в 18:45") == date.today() - timedelta(days=1)
assert parse_listing_date("позавчера") == date.today() - timedelta(days=2)
assert parse_listing_date("только что") == date.today()
def test_listing_date_absolute_no_year():
"""Yandex SERP формат «18 мая» без года — infer current year."""
from app.services.scrapers.yandex_helpers import parse_listing_date
today = date.today()
# Pick a date в прошлом этого года, чтобы тест стабильный
if today.month >= 2:
target = date(today.year, today.month - 1, 15)
else:
target = date(today.year - 1, 12, 15)
ru_months = {
1: "января", 2: "февраля", 3: "марта", 4: "апреля",
5: "мая", 6: "июня", 7: "июля", 8: "августа",
9: "сентября", 10: "октября", 11: "ноября", 12: "декабря",
}
text = f"{target.day} {ru_months[target.month]}"
assert parse_listing_date(text) == target
def test_listing_date_relative_still_works():
from datetime import timedelta
from app.services.scrapers.yandex_helpers import parse_listing_date
assert parse_listing_date("5 дней назад") == date.today() - timedelta(days=5)
# --- parse_dmy ---
def test_parse_dmy_basic():
assert parse_dmy("17.02.2026") == date(2026, 2, 17)
def test_parse_dmy_in_sentence():
assert parse_dmy("Дата 31.12.2025 — конец года") == date(2025, 12, 31)
def test_parse_dmy_invalid():
assert parse_dmy("99.99.9999") is None
assert parse_dmy("") is None
assert parse_dmy(None) is None
# --- JSON-LD ---
def test_extract_json_ld_single():
html = (
'<html><script type="application/ld+json">{"@type": "Product", "name": "test"}'
"</script></html>"
)
ld = extract_json_ld(html)
assert len(ld) == 1
assert ld[0]["@type"] == "Product"
def test_extract_json_ld_multiple():
html = (
'<script type="application/ld+json">{"@type": "Product"}</script>'
'<script type="application/ld+json">{"@type": "Organization"}</script>'
)
ld = extract_json_ld(html)
assert len(ld) == 2
def test_extract_json_ld_array_wrapper():
html = '<script type="application/ld+json">[{"@type": "A"}, {"@type": "B"}]</script>'
ld = extract_json_ld(html)
assert len(ld) == 2
def test_extract_json_ld_invalid_skipped():
html = (
'<script type="application/ld+json">not valid json</script>'
'<script type="application/ld+json">{"@type": "Valid"}</script>'
)
ld = extract_json_ld(html)
assert len(ld) == 1
assert ld[0]["@type"] == "Valid"
def test_find_ld_by_type_simple():
html = (
'<script type="application/ld+json">{"@type": "Product", "price": 100}</script>'
'<script type="application/ld+json">{"@type": "BreadcrumbList"}</script>'
)
product = find_ld_by_type(html, "Product")
assert product is not None
assert product["price"] == 100
def test_find_ld_by_type_list_form():
# @type can be a list
html = '<script type="application/ld+json">{"@type": ["Product", "Offer"], "x": 1}</script>'
assert find_ld_by_type(html, "Offer") is not None
def test_find_ld_by_type_missing():
html = '<script type="application/ld+json">{"@type": "Other"}</script>'
assert find_ld_by_type(html, "Product") is None
# --- regex sanity ---
def test_offer_id():
m = RE_OFFER_ID.search("https://realty.yandex.ru/offer/7567094292504417257/")
assert m and m.group(1) == "7567094292504417257"
def test_jk_id():
m = RE_JK_ID.search("/ekaterinburg/kupit/novostrojka/tatlin-1592987/")
assert m
assert m.group(1) == "tatlin"
assert m.group(2) == "1592987"
def test_title_parsing():
title = "39 м² · 1-комнатная квартира · 6 этаж из 15"
area = RE_TITLE_AREA.search(title)
rooms = RE_TITLE_ROOMS.search(title)
floor = RE_FLOOR.search(title)
assert area and area.group(1) == "39"
assert rooms and rooms.group(1) == "1"
assert floor and floor.group(1) == "6" and floor.group(2) == "15"
def test_title_studio():
rooms = RE_TITLE_ROOMS.search("Студия 28 м²")
# group(2) captures студия/студию/студий when no number-room pattern present
assert rooms is not None
matched = rooms.group(1) or rooms.group(2)
assert matched is not None and matched.lower().startswith("студи")

441
tradein-mvp/backend/tests/test_yandex_newbuilding.py
View file

@ -1,441 +0,0 @@
"""Unit tests for yandex_newbuilding.py — ЖК landing parser (Worker B, Wave 4).
Reference target: ЖК Татлин (slug=tatlin, id=1592987) — comfort+, June 2023,
PRINZIP developer, rating 4.3, 1505 ratings, 353 text reviews,
coords (56.855312, 60.576668).
All tests are offline — no network calls. Parser receives hand-crafted fixture HTML.
"""
from __future__ import annotations
import pytest
from app.services.scrapers.yandex_newbuilding import (
JKMetroStation,
YandexNewbuildingInfo,
YandexNewbuildingScraper,
_extract_coords,
_find_section_text,
_parse_corpus_count,
_parse_metro,
)
# ── Fixture HTML ──────────────────────────────────────────────────────────────
TATLIN_FIXTURE_HTML = """<!DOCTYPE html>
<html>
<head><title>ЖК «Татлин»</title></head>
<body>
<h1>ЖК «Татлин»</h1>
<div>Екатеринбург, ул. Черепанова / ул. Готвальда</div>
<a data-test="CARD_DEV_BADGE_DEVELOPER_LINK" href="/developer/prinzip">PRINZIP</a>
<div data-test="CardDevSites">
<a href="/zhk/1">ЖК Кислород</a>
<a href="/zhk/2">ЖК Нова</a>
</div>
<div>56.855312, 60.576668</div>
<div>Уральская 11 мин. Динамо 16 мин.</div>
<div>4.3 из 5 1505 оценок Смотреть все 353 отзыва</div>
<h2>О комплексе</h2>
<p>ЖК «Татлин» — три 35-этажные башни в Заречном микрорайоне Екатеринбурга.
Комплекс класса комфорт+. Введён в эксплуатацию в июне 2023 года.
Расположен на участке 1,5 га. Монолитный тип дома.</p>
<h2>Расположение</h2>
<p>В 5 минутах от центра</p>
</body>
</html>"""
# Minimal fixture — missing optional sections
MINIMAL_FIXTURE_HTML = """<!DOCTYPE html>
<html><body>
<h1>ЖК Минимальный</h1>
</body></html>"""
# ── Full happy-path test ──────────────────────────────────────────────────────
def test_parse_jk_full_fixture():
"""Happy path: all major fields extracted from Татлин-like HTML."""
scraper = YandexNewbuildingScraper()
info = scraper.parse(
TATLIN_FIXTURE_HTML,
jk_slug="tatlin",
jk_id="1592987",
source_url="https://realty.yandex.ru/ekaterinburg/kupit/novostrojka/tatlin-1592987/",
)
assert isinstance(info, YandexNewbuildingInfo)
assert info.ext_id == "1592987"
assert info.ext_slug == "tatlin"
assert "Татлин" in (info.name or "")
# coords
assert info.lat == pytest.approx(56.855312, abs=1e-5)
assert info.lon == pytest.approx(60.576668, abs=1e-5)
# class + commission
assert info.house_class == "comfort_plus"
assert info.commission_year == 2023
assert info.commission_month == "июне"
# footprint
assert info.total_floors == 35
assert info.corpus_count == 3
assert info.total_area_ha == pytest.approx(1.5, abs=0.01)
# developer
assert info.developer_name == "PRINZIP"
assert info.developer_url is not None and "prinzip" in info.developer_url
assert "ЖК Кислород" in info.developer_other_jk
assert "ЖК Нова" in info.developer_other_jk
# reviews
assert info.rating == pytest.approx(4.3, abs=0.01)
assert info.ratings_count == 1505
assert info.text_reviews_count == 353
# description present
assert info.description is not None
assert len(info.description) > 10
# metro
assert len(info.metro_stations) >= 1
station_names = [s.name for s in info.metro_stations]
assert any("Уральская" in n for n in station_names)
# house type from "монолитный"
assert info.house_type == "monolith"
# raw_payload
assert info.raw_payload is not None
assert "body_len" in info.raw_payload
# ── Coord extraction tests ────────────────────────────────────────────────────
def test_extract_coords_within_ekb_range():
html = "<div>56.855312, 60.576668</div>"
lat, lon = _extract_coords(html)
assert lat == pytest.approx(56.855312, abs=1e-5)
assert lon == pytest.approx(60.576668, abs=1e-5)
def test_extract_coords_outside_range_returns_none():
# Moscow coords — outside EKB range
html = "<div>55.7558, 37.6176</div>"
_lat, lon = _extract_coords(html)
# 55.75 is in LAT_RANGE but 37.61 is NOT in LON_RANGE
assert lon is None
def test_extract_coords_no_coords_returns_none():
html = "<div>Текст без координат</div>"
lat, lon = _extract_coords(html)
assert lat is None
assert lon is None
def test_extract_coords_multiple_takes_first():
"""When multiple valid coords present, first is returned."""
html = "<div>56.855312, 60.576668</div><div>56.900000, 60.600000</div>"
lat, lon = _extract_coords(html)
assert lat == pytest.approx(56.855312, abs=1e-5)
assert lon == pytest.approx(60.576668, abs=1e-5)
# ── Corpus count tests ────────────────────────────────────────────────────────
def test_parse_corpus_count_word_three():
text = "три 35-этажные башни в центре"
result = _parse_corpus_count(text)
assert result == 3
def test_parse_corpus_count_digit():
text = "5 35-этажных корпусов"
result = _parse_corpus_count(text)
assert result == 5
def test_parse_corpus_count_two():
text = "две 20-этажные башни"
result = _parse_corpus_count(text)
assert result == 2
def test_parse_corpus_count_none_when_no_match():
text = "Обычный текст без этажей"
result = _parse_corpus_count(text)
assert result is None
# ── House class test ──────────────────────────────────────────────────────────
def test_parse_house_class_comfort_plus_from_description():
from app.services.scrapers.yandex_helpers import parse_house_class
text = "Жилой комплекс класса комфорт+ в центре города"
result = parse_house_class(text)
assert result == "comfort_plus"
def test_parse_house_class_business():
from app.services.scrapers.yandex_helpers import parse_house_class
text = "ЖК класса бизнес"
result = parse_house_class(text)
assert result == "business"
# ── Commission year/month test ────────────────────────────────────────────────
def test_parse_commission_year_month():
"""Full pipeline: parse() correctly extracts commission_year and commission_month."""
scraper = YandexNewbuildingScraper()
info = scraper.parse(
TATLIN_FIXTURE_HTML,
jk_slug="tatlin",
jk_id="1592987",
source_url="https://realty.yandex.ru/ekaterinburg/kupit/novostrojka/tatlin-1592987/",
)
assert info.commission_year == 2023
assert info.commission_month == "июне"
def test_parse_commission_sdan_variant():
"""Regex supports 'сдан в эксплуатацию' phrasing as well."""
html = """<html><body>
<h1>ЖК Тест</h1>
<h2>О комплексе</h2>
<p>Сдан в эксплуатацию в марте 2025 года.</p>
</body></html>"""
scraper = YandexNewbuildingScraper()
info = scraper.parse(html, jk_slug="test", jk_id="999", source_url="http://x")
assert info.commission_year == 2025
assert info.commission_month == "марте"
# ── Rating + counts test ──────────────────────────────────────────────────────
def test_parse_rating_and_counts():
scraper = YandexNewbuildingScraper()
info = scraper.parse(
TATLIN_FIXTURE_HTML,
jk_slug="tatlin",
jk_id="1592987",
source_url="https://realty.yandex.ru/ekaterinburg/kupit/novostrojka/tatlin-1592987/",
)
assert info.rating == pytest.approx(4.3, abs=0.01)
assert info.ratings_count == 1505
assert info.text_reviews_count == 353
def test_parse_rating_comma_separator():
"""Rating '4,3 из 5' with comma handled correctly."""
html = "<html><body><div>4,3 из 5 200 оценок</div></body></html>"
scraper = YandexNewbuildingScraper()
info = scraper.parse(html, jk_slug="x", jk_id="1", source_url="http://x")
assert info.rating == pytest.approx(4.3, abs=0.01)
assert info.ratings_count == 200
def test_parse_rating_no_space_before_iz():
"""Yandex drift (#974): '4.3из 5' / '4,5из\xa05' — без пробела перед «из», nbsp внутри.
Регрессия 2026-06-15: live-страницы рендерили рейтинг как «4.3из 5» (без
разделителя), из-за чего \\s+ перед «из» не матчил → rating=NULL при
заполненных ratings_count. \\s* должен извлечь рейтинг в обеих формах.
"""
scraper = YandexNewbuildingScraper()
info = scraper.parse(
"<html><body><div>Отзывы4.3из 5Всего 1508 оценок</div></body></html>",
jk_slug="x",
jk_id="1",
source_url="http://x",
)
assert info.rating == pytest.approx(4.3, abs=0.01)
assert info.ratings_count == 1508
# nbsp-вариант (как в реальном body_text)
info2 = scraper.parse(
"<html><body><div>4,5из\xa05Всего 850 оценок</div></body></html>",
jk_slug="y",
jk_id="2",
source_url="http://y",
)
assert info2.rating == pytest.approx(4.5, abs=0.01)
# ── Developer link test ───────────────────────────────────────────────────────
def test_developer_name_extracted_from_data_test_link():
scraper = YandexNewbuildingScraper()
info = scraper.parse(
TATLIN_FIXTURE_HTML,
jk_slug="tatlin",
jk_id="1592987",
source_url="https://realty.yandex.ru/ekaterinburg/kupit/novostrojka/tatlin-1592987/",
)
assert info.developer_name == "PRINZIP"
assert info.developer_url == "https://realty.yandex.ru/developer/prinzip"
def test_developer_absolute_url_kept_as_is():
html = """<html><body>
<a data-test="CARD_DEV_BADGE_DEVELOPER_LINK"
href="https://realty.yandex.ru/developer/abc">АБС Групп</a>
</body></html>"""
scraper = YandexNewbuildingScraper()
info = scraper.parse(html, jk_slug="x", jk_id="1", source_url="http://x")
assert info.developer_name == "АБС Групп"
assert info.developer_url == "https://realty.yandex.ru/developer/abc"
# ── Metro stations test ───────────────────────────────────────────────────────
def test_metro_stations_parsed():
scraper = YandexNewbuildingScraper()
info = scraper.parse(
TATLIN_FIXTURE_HTML,
jk_slug="tatlin",
jk_id="1592987",
source_url="https://realty.yandex.ru/ekaterinburg/kupit/novostrojka/tatlin-1592987/",
)
assert len(info.metro_stations) >= 1
uralskaya = next((s for s in info.metro_stations if "Уральская" in s.name), None)
assert uralskaya is not None
assert uralskaya.walk_min == 11
def test_parse_metro_direct():
text = "Уральская 11 мин. Динамо 16 мин."
stations = _parse_metro(text)
assert len(stations) == 2
assert stations[0].name == "Уральская"
assert stations[0].walk_min == 11
assert stations[1].name == "Динамо"
assert stations[1].walk_min == 16
def test_parse_metro_caps_at_five():
text = "Альфа 5 мин. Бета 6 мин. Гамма 7 мин. " "Дельта 8 мин. Эпсилон 9 мин. Зета 10 мин."
stations = _parse_metro(text)
assert len(stations) == 5
# ── No description section test ───────────────────────────────────────────────
def test_no_description_section_returns_none():
"""When no О комплексе / Расположение section, description is None."""
scraper = YandexNewbuildingScraper()
info = scraper.parse(
MINIMAL_FIXTURE_HTML,
jk_slug="minimal",
jk_id="42",
source_url="http://example.com/",
)
assert info.description is None
assert info.house_class is None
assert info.commission_year is None
assert info.total_floors is None
# ── fetch_around raises NotImplementedError ───────────────────────────────────
@pytest.mark.asyncio
async def test_fetch_around_raises_not_implemented():
scraper = YandexNewbuildingScraper()
with pytest.raises(NotImplementedError, match="JK-slug-based"):
await scraper.fetch_around(56.855, 60.576)
# ── _find_section_text helper ─────────────────────────────────────────────────
def test_find_section_text_returns_content_after_heading():
from selectolax.parser import HTMLParser
html = """<html><body>
<h2>О комплексе</h2>
<p>Описание комплекса здесь.</p>
<p>Ещё абзац.</p>
<h2>Другой раздел</h2>
<p>Не должен попасть.</p>
</body></html>"""
tree = HTMLParser(html)
result = _find_section_text(tree, "О комплексе")
assert result is not None
assert "Описание комплекса" in result
assert "Не должен" not in result
def test_find_section_text_returns_none_when_missing():
from selectolax.parser import HTMLParser
html = "<html><body><p>Текст</p></body></html>"
tree = HTMLParser(html)
result = _find_section_text(tree, "О комплексе")
assert result is None
# ── Developer other JK dedup ──────────────────────────────────────────────────
def test_developer_other_jk_dedup():
"""Duplicate JK names are not added twice."""
html = """<html><body>
<div data-test="CardDevSites">
<a href="/1">ЖК Один</a>
<a href="/2">ЖК Один</a>
<a href="/3">ЖК Два</a>
</div>
</body></html>"""
scraper = YandexNewbuildingScraper()
info = scraper.parse(html, jk_slug="x", jk_id="1", source_url="http://x")
assert info.developer_other_jk.count("ЖК Один") == 1
assert "ЖК Два" in info.developer_other_jk
def test_developer_other_jk_capped_at_10():
"""Only first 10 other JK entries are kept."""
links = "\n".join(f'<a href="/{i}">ЖК {i}</a>' for i in range(15))
html = f"""<html><body>
<div data-test="CardDevSites">{links}</div>
</body></html>"""
scraper = YandexNewbuildingScraper()
info = scraper.parse(html, jk_slug="x", jk_id="1", source_url="http://x")
assert len(info.developer_other_jk) == 10
# ── model defaults ────────────────────────────────────────────────────────────
def test_yandex_newbuilding_info_defaults():
"""Model initialises with safe defaults for all optional fields."""
info = YandexNewbuildingInfo(ext_id="1", ext_slug="x", source_url="http://x")
assert info.name is None
assert info.lat is None
assert info.lon is None
assert info.house_class is None
assert info.developer_other_jk == []
assert info.metro_stations == []
assert info.raw_payload is None
def test_jk_metro_station_model():
station = JKMetroStation(name="Уральская", walk_min=11)
assert station.name == "Уральская"
assert station.walk_min == 11
station_no_time = JKMetroStation(name="Геологическая")
assert station_no_time.walk_min is None

2
tradein-mvp/backend/tests/test_yandex_price_history.py
View file

@ -18,11 +18,11 @@ import re
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
import pytest
from scraper_kit.base import ScrapedLot
from sqlalchemy import create_engine, text
from sqlalchemy.orm import Session, sessionmaker
from app.services import yandex_price_history as yph
from app.services.scrapers.base import ScrapedLot
_SERVICE_SRC = inspect.getsource(yph)
# Concatenated text of every actual SQL statement the service emits (not the docstring,

20
tradein-mvp/backend/tests/test_yandex_scrapers_delay_wiring.py
View file

@ -1,23 +1,23 @@
"""Smoke: each Yandex scraper instance reads request_delay_sec from scraper_settings.
Легаси `app.services.scrapers.yandex_detail` / `yandex_valuation` удалены (#2277
финальный шаг scraper_kit-миграции). `yandex_newbuilding` пока остаётся легаси (не
мигрирован) — его delay-wiring тест не тронут. `yandex_realty` удалён (#2397 Part
E2, 0 runtime-импортёров) — его delay-wiring тест убран (kit `YandexRealtyScraper`
delay-provider wiring покрыт `test_kit_serp_proxy_pool.py`).
финальный шаг scraper_kit-миграции). `yandex_realty` удалён (#2397 Part E2, 0
runtime-импортёров) — его delay-wiring тест убран (kit `YandexRealtyScraper`
delay-provider wiring покрыт `test_kit_serp_proxy_pool.py`). `yandex_newbuilding`
удалён (#2397 финальный шаг E) — переведён на kit-эквивалент, который тоже
использует инжектируемый `delay_provider=` (strangler DI, #2133) вместо прямого
импорта `get_scraper_delay`.
`yandex_detail`/`yandex_valuation` переведены на kit-эквиваленты, которые используют
инжектируемый `delay_provider=` (strangler DI, #2133) вместо прямого импорта
`get_scraper_delay` — тест адаптирован под реальный контракт kit-класса.
"""
from types import SimpleNamespace
from unittest.mock import patch
from scraper_kit.providers.yandex.detail import YandexDetailScraper
from scraper_kit.providers.yandex.newbuilding import YandexNewbuildingScraper
from scraper_kit.providers.yandex.valuation import YandexValuationScraper
from app.services.scrapers.yandex_newbuilding import YandexNewbuildingScraper
_KIT_CONFIG = SimpleNamespace(
scraper_proxy_url=None,
yandex_proxy_rotate_url=None,
@ -32,9 +32,9 @@ def test_yandex_detail_picks_up_delay():
def test_yandex_newbuilding_picks_up_delay():
with patch("app.services.scrapers.yandex_newbuilding.get_scraper_delay", return_value=9.5):
s = YandexNewbuildingScraper()
assert s.request_delay_sec == 9.5
"""Kit YandexNewbuildingScraper: delay_provider callable вызывается с self.name."""
s = YandexNewbuildingScraper(delay_provider=lambda _name: 9.5)
assert s.request_delay_sec == 9.5
def test_yandex_valuation_picks_up_delay():

371
tradein-mvp/scripts/avito-playwright-sweep.py
View file

@ -1,371 +0,0 @@
"""Playwright-based Avito sweep — обходит Qrator JS challenge через real headless Chromium.
Use case: curl_cffi заблочен Qrator anti-DDoS (429 + captcha challenge). Real
browser proходит JS challenge автоматически.
Setup:
cd tradein-mvp/backend
uv pip install playwright
uv run playwright install chromium
Usage:
export DATABASE_URL='postgresql+psycopg://tradein:<PASS>@localhost:15433/tradein'
uv run --project tradein-mvp/backend python tradein-mvp/scripts/avito-playwright-sweep.py
# options: --pages N --delay S --mode {citywide|byrooms} --headless
"""
from __future__ import annotations
# Banner FIRST — даже до imports чтобы видно что скрипт стартанул:
import sys as _sys
print("=" * 60, flush=True)
print("AVITO PLAYWRIGHT SWEEP — starting...", flush=True)
print("=" * 60, flush=True)
_sys.stdout.flush()
import argparse
import asyncio
import json
import os
import random
import sys
import time
import types
from pathlib import Path
from urllib.parse import urlencode
# Force unbuffered output
os.environ["PYTHONUNBUFFERED"] = "1"
_BACKEND = Path(__file__).resolve().parent.parent / "backend"
if str(_BACKEND) not in sys.path:
sys.path.insert(0, str(_BACKEND))
# WeasyPrint stub (Windows без GTK)
for _m in ("weasyprint", "weasyprint.text", "weasyprint.text.ffi", "weasyprint.css", "weasyprint.css.targets"):
sys.modules[_m] = types.ModuleType(_m)
sys.modules["weasyprint"].HTML = type("HTML", (), {"__init__": lambda self, *a, **kw: None})
sys.modules["weasyprint"].CSS = type("CSS", (), {"__init__": lambda self, *a, **kw: None})
from playwright.async_api import Browser, BrowserContext, async_playwright # noqa: E402
try:
from playwright_stealth import Stealth # noqa: E402
_STEALTH_AVAILABLE = True
except ImportError:
_STEALTH_AVAILABLE = False
print("[warn] playwright-stealth not installed — без stealth, может triggered больше CAPTCHA", flush=True)
from app.core.db import SessionLocal # noqa: E402
from app.services.scrapers.avito import AvitoScraper # noqa: E402
from app.services.scrapers.base import save_listings # noqa: E402
COOKIES_FILE = Path(__file__).resolve().parent / ".avito-cookies.json"
AVITO_BASE = "https://www.avito.ru"
UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36"
# Reuse AvitoScraper's parser (DOM cards extraction)
_parser_instance = AvitoScraper()
_parser_instance._anchor_lat = 0.0 # noqa: SLF001
_parser_instance._anchor_lon = 0.0 # noqa: SLF001
_parser_instance._anchor_radius_m = 1000 # noqa: SLF001
def load_cookies() -> list[dict]:
"""Load DevTools-exported cookies → Playwright format."""
if not COOKIES_FILE.exists():
print(f"WARN: cookies file not found at {COOKIES_FILE} — будем без cookies")
return []
raw = json.loads(COOKIES_FILE.read_text(encoding="utf-8"))
out = []
for c in raw:
ck = {
"name": c["name"],
"value": c["value"],
"domain": c["domain"],
"path": c.get("path", "/"),
"secure": c.get("secure", True),
"httpOnly": c.get("httpOnly", False),
}
if c.get("expirationDate"):
ck["expires"] = int(c["expirationDate"])
ss = c.get("sameSite")
if ss == "no_restriction":
ck["sameSite"] = "None"
elif ss == "lax":
ck["sameSite"] = "Lax"
elif ss == "strict":
ck["sameSite"] = "Strict"
out.append(ck)
return out
def build_citywide_url(page: int) -> str:
return f"{AVITO_BASE}/ekaterinburg/kvartiry/prodam-ASgBAgICAUSSA8YQ?{urlencode({'s':104,'p':page})}"
ROOM_SLUGS: list[tuple[str, str]] = [
("студии", "studii-ASgBAgICAkSSA8YQygj~WA"),
("1-комн.", "1-komnatnye-ASgBAgICAkSSA8YQygiAWQ"),
("2-комн.", "2-komnatnye-ASgBAgICAkSSA8YQygiCWQ"),
("3-комн.", "3-komnatnye-ASgBAgICAkSSA8YQygiEWQ"),
("4-комн.", "4-komnatnye-ASgBAgICAkSSA8YQygiGWQ"),
("5+комн.", "5-komnatnye-ASgBAgICAkSSA8YQygiIWQ"),
("своб.планир.", "svobodnaya_planirovka-ASgBAgICAkSSA8YQygj8zzI"),
]
def build_rooms_url(slug: str, page: int) -> str:
return f"{AVITO_BASE}/ekaterinburg/kvartiry/prodam/{slug}?{urlencode({'s':104,'p':page})}"
async def fetch_page(context: BrowserContext, url: str) -> tuple[int, str | None]:
"""Open page in Playwright, wait for content, return (status, html or None).
Strategy:
1. goto с wait_until='domcontentloaded' (быстро, не ждём всех scripts)
2. Если status 200 → wait_for_selector data-marker=item (Qrator challenge должен resolve пока)
3. Иначе если 429/403 — challenge не прошёл, дать ещё 5s + retry content (maybe challenge resolves async)
"""
page = await context.new_page()
try:
try:
response = await page.goto(url, wait_until="domcontentloaded", timeout=45000)
except Exception as e: # noqa: BLE001
print(f" goto FAIL: {type(e).__name__}: {str(e)[:80]}")
return 0, None
status = response.status if response else 0
if status in (429, 403):
# Qrator challenge — wait для JS auto-resolve, потом reload
await asyncio.sleep(5)
try:
response = await page.reload(wait_until="domcontentloaded", timeout=30000)
status = response.status if response else status
except Exception: # noqa: BLE001
pass
if status != 200:
return status, None
# Wait for item cards
try:
await page.wait_for_selector('[data-marker="item"]', timeout=15000)
except Exception: # noqa: BLE001
pass
html = await page.content()
return status, html
finally:
await page.close()
async def sweep_urls(
urls_with_label: list[tuple[str, str]],
delay: float,
headless: bool,
) -> int:
"""Generic sweep: list of (label, url) → fetch + parse + save per URL."""
cookies = load_cookies()
print(f"loaded {len(cookies)} cookies")
total_fetched = 0
total_ins = 0
total_upd = 0
challenge_hits = 0
print("=== Launching Playwright browser ===", flush=True)
async with async_playwright() as p:
print(f" headless={headless}", flush=True)
browser: Browser = await p.chromium.launch(
headless=headless,
args=[
"--disable-blink-features=AutomationControlled",
"--no-sandbox",
],
)
print(" browser launched, creating context...", flush=True)
context: BrowserContext = await browser.new_context(
user_agent=UA,
locale="ru-RU",
timezone_id="Asia/Yekaterinburg",
viewport={"width": 1366, "height": 900},
)
# Stealth — optional, skip on error
if _STEALTH_AVAILABLE:
try:
stealth = Stealth()
await stealth.apply_stealth_async(context)
print(" stealth applied", flush=True)
except Exception as e: # noqa: BLE001
print(f" stealth skipped (err): {type(e).__name__}: {e}", flush=True)
if cookies:
await context.add_cookies(cookies)
print(f" cookies added ({len(cookies)})", flush=True)
# Initial warmup: open main Avito + wait для user pass CAPTCHA если нужно
print("\n=== WARMUP: opening Avito initial page ===", flush=True)
print(" creating new page tab...", flush=True)
warmup_page = await context.new_page()
print(" navigating to Avito (timeout 30s)...", flush=True)
try:
await warmup_page.goto(
"https://www.avito.ru/ekaterinburg/kvartiry/prodam-ASgBAgICAUSSA8YQ?s=104&p=1",
wait_until="domcontentloaded",
timeout=30000,
)
print(" goto done, reading content...", flush=True)
except Exception as e: # noqa: BLE001
print(f" goto FAIL: {type(e).__name__}: {e}", flush=True)
# Check if challenge page — reliable detector: NO listing cards в HTML.
# Слово "captcha" может быть в JS embed страницы normally (false positive).
warmup_html = await warmup_page.content()
print(f" content read: {len(warmup_html)} bytes", flush=True)
try:
await warmup_page.wait_for_selector('[data-marker="item"]', timeout=5000)
has_items = True
except Exception: # noqa: BLE001
has_items = False
is_qrator = "qrator" in warmup_html.lower() and len(warmup_html) < 100000
challenge_present = (not has_items) and (is_qrator or len(warmup_html) < 50000)
print(f" has_items={has_items} qrator={is_qrator} size={len(warmup_html)} challenge={challenge_present}", flush=True)
if challenge_present:
print("\n🚨 CAPTCHA detected на initial page!")
print("👉 Окно браузера открыто — пройди капчу руками,")
print("👉 затем нажми Enter здесь чтобы продолжить sweep...")
try:
input(">>> Press ENTER when ready: ")
except (EOFError, KeyboardInterrupt):
print("aborted")
await browser.close()
return 0
# Verify challenge passed
await warmup_page.reload(wait_until="domcontentloaded", timeout=30000)
new_html = await warmup_page.content()
if any(m in new_html.lower() for m in ("captcha", "qrator")):
print("⚠️ всё ещё CAPTCHA — повторно пройди или Ctrl+C")
input(">>> Retry ENTER: ")
else:
print("✅ CAPTCHA пройдена, продолжаю sweep")
else:
print("✅ Initial page OK (no challenge)")
await warmup_page.close()
try:
# Per-category skip flag: 0 lots на странице категории = end of pagination
# ДЛЯ ТЕКУЩЕЙ КАТЕГОРИИ, а не для всего sweep'а (был bug: break ломал всё).
current_cat: str | None = None
skip_until_next_cat = False
for i, (label, url) in enumerate(urls_with_label, 1):
# Категория = всё до " page=" в label ("1-комн. page=5/60" → "1-комн.")
cat = label.split(" page=")[0] if " page=" in label else label
if cat != current_cat:
current_cat = cat
skip_until_next_cat = False
challenge_hits = 0 # reset 429-counter на границе категорий
if skip_until_next_cat:
continue
print(f"[{i}/{len(urls_with_label)}] {label}", flush=True)
status, html = await fetch_page(context, url)
if status != 200 or not html:
print(f" HTTP {status} → skip", flush=True)
if status in (429, 403):
challenge_hits += 1
if challenge_hits >= 3:
print(f" ⚠️ {challenge_hits}x challenge hits — sleep 60s extra", flush=True)
print(" если CAPTCHA в окне — пройди и нажми ENTER:", flush=True)
try:
input(" >>> ")
except (EOFError, KeyboardInterrupt):
break
challenge_hits = 0
await asyncio.sleep(delay)
continue
# Parse + save
lots = _parser_instance._parse_html(html, source_url_base=url) # noqa: SLF001
if not lots:
print(f" 0 lots — end of pagination for {cat!r}", flush=True)
if " page=" in label:
# 0 lots на пагинированной странице = end of category, не sweep
skip_until_next_cat = True
continue
db = SessionLocal()
try:
ins, upd = save_listings(db, lots)
total_fetched += len(lots)
total_ins += ins
total_upd += upd
print(
f" {len(lots)} lots → ins={ins} upd={upd} "
f"[cum: tot={total_fetched} new={total_ins}]",
flush=True,
)
except Exception as e: # noqa: BLE001
print(f" save FAIL: {type(e).__name__}: {e}", flush=True)
finally:
db.close()
# Jittered delay
if i < len(urls_with_label):
j = delay * random.uniform(0.7, 1.3)
await asyncio.sleep(j)
finally:
await context.close()
await browser.close()
print(
f"\n=== TOTAL: fetched={total_fetched} ins={total_ins} upd={total_upd} ===",
flush=True,
)
return total_fetched
async def main() -> None:
# Diagnostic — show env vars used:
db_url = os.environ.get("DATABASE_URL", "<NOT SET>")
# Mask password for logging
if "://" in db_url and "@" in db_url:
before_at, after_at = db_url.rsplit("@", 1)
scheme_user = before_at.split("://", 1)
if len(scheme_user) == 2 and ":" in scheme_user[1]:
user = scheme_user[1].split(":", 1)[0]
masked = f"{scheme_user[0]}://{user}:***@{after_at}"
else:
masked = db_url[:50] + "..."
else:
masked = db_url
print(f"DATABASE_URL: {masked}", flush=True)
print(f"ENVIRONMENT: {os.environ.get('ENVIRONMENT', '<default>')}", flush=True)
p = argparse.ArgumentParser()
p.add_argument("--mode", choices=["citywide", "byrooms"], default="citywide")
p.add_argument("--pages", type=int, default=100, help="pages per query")
p.add_argument("--delay", type=float, default=7, help="sec между requests")
p.add_argument("--headless", action="store_true", default=True)
p.add_argument("--no-headless", dest="headless", action="store_false")
p.add_argument(
"--start-cat",
type=int,
default=1,
help="byrooms: с какой категории начать (1-based). 1=студии, 2=1-комн, "
"3=2-комн, 4=3-комн, 5=4-комн, 6=5+комн, 7=своб.планир.",
)
args = p.parse_args()
urls: list[tuple[str, str]] = []
if args.mode == "citywide":
for page in range(1, args.pages + 1):
urls.append((f"citywide page={page}/{args.pages}", build_citywide_url(page)))
else: # byrooms
cats = ROOM_SLUGS[args.start_cat - 1:]
if not cats:
print(f"ERROR: --start-cat={args.start_cat} > {len(ROOM_SLUGS)} cats", flush=True)
return
print(f"byrooms: starting from cat #{args.start_cat} ({cats[0][0]!r}), "
f"{len(cats)} cats total", flush=True)
for cat_name, slug in cats:
for page in range(1, args.pages + 1):
urls.append((f"{cat_name} page={page}/{args.pages}", build_rooms_url(slug, page)))
print(f"Mode: {args.mode}, pages: {args.pages}, delay: {args.delay}s, total URLs: {len(urls)}")
print(f"Estimated time: {len(urls) * args.delay / 60:.0f} min (без challenge)")
t0 = time.perf_counter()
await sweep_urls(urls, args.delay, args.headless)
print(f"\nDONE in {(time.perf_counter() - t0) / 60:.1f} min")
if __name__ == "__main__":
asyncio.run(main())

467
tradein-mvp/scripts/backfill-house-imv.py
View file

@ -1,467 +0,0 @@
"""Phase C: backfill Avito IMV per house.
Iterates `houses` with imv_status='pending' AND lat/lon NOT NULL.
For each: pick median lot-params from linked listings -> call evaluate_via_imv()
-> save house_imv_evaluations + house_placement_history + house_suggestions.
Resumable: marks houses.imv_status = 'ok' / 'not_found' / 'transient_error' / 'error'.
Usage:
# Locally with SSH tunnel (LocalForward 35432 -> tradein-postgres):
DATABASE_URL=postgresql+psycopg://tradein:<pwd>@localhost:35432/tradein \\
python scripts/backfill-house-imv.py --batch 50 --delay 5
# Inside container:
docker exec tradein-backend python /app/scripts/backfill-house-imv.py --batch 100 --delay 3
Args:
--batch N How many houses to process this run (default 50)
--delay SEC Sleep between houses (default 5 to avoid Avito captcha)
--limit-total Stop after N successful evaluations (debug; default no limit)
--only-status Process only houses with this imv_status (default 'pending';
also accepts 'transient_error')
"""
from __future__ import annotations
import argparse
import asyncio
import json
import logging
import os
import sys
from pathlib import Path
# Ensure repo backend in path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "backend"))
from sqlalchemy import create_engine, text
from sqlalchemy.orm import Session
from app.services.scrapers.avito_imv import (
IMVAddressNotFoundError,
IMVAuthError,
IMVEvaluation,
IMVTransientError,
evaluate_via_imv,
)
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s %(levelname)s %(message)s",
)
logger = logging.getLogger("phase-c")
# Avito raw log — pipe everything from app.services.scrapers.avito_imv to a file
# so we keep full request/response history for diagnosing contract changes.
_AVITO_LOG_PATH = os.environ.get("AVITO_LOG_PATH", "avito-raw.log")
_avito_fh = logging.FileHandler(_AVITO_LOG_PATH, encoding="utf-8")
_avito_fh.setFormatter(logging.Formatter("%(asctime)s %(levelname)s [%(name)s] %(message)s"))
_avito_fh.setLevel(logging.DEBUG)
_avito_logger = logging.getLogger("app.services.scrapers.avito_imv")
_avito_logger.addHandler(_avito_fh)
_avito_logger.setLevel(logging.DEBUG)
# Also pipe phase-c progress lines to the file
logger.addHandler(_avito_fh)
logger.info("avito raw log writing to %s", os.path.abspath(_AVITO_LOG_PATH))
# Avito IMV API accepts only these house_type values (verified 2026-05-24).
# Our `listings.house_type` may contain pre-2026 values like 'monolith' / 'monolith_brick'.
_HOUSE_TYPE_TO_IMV = {
"panel": "panel",
"brick": "brick",
"monolith": "monolithic", # API renamed; raw value rejected with 400
"monolithic": "monolithic",
"monolith_brick": "monolithic", # API doesn't accept hybrid → fall back to monolithic
"block": "block",
"wood": "wood",
}
_HOUSE_TYPE_DEFAULT = "panel" # most common in EKB; safe default for unknown
def _map_house_type(raw: str | None) -> str:
if not raw:
return _HOUSE_TYPE_DEFAULT
return _HOUSE_TYPE_TO_IMV.get(raw.lower().strip(), _HOUSE_TYPE_DEFAULT)
# Bounding boxes for known regions in our DB.
# When house.lat/lon falls inside one, we prepend the region name to address
# before sending to Avito (resolves ambiguity — e.g. "Орджоникидзевский район" exists
# in EKB AND in Khakasia republic).
_REGION_BBOX = [
# (region_name, lat_min, lat_max, lon_min, lon_max)
("Свердловская область, Екатеринбург", 56.5, 57.1, 59.9, 61.0),
("Свердловская область", 56.0, 61.0, 58.0, 65.0), # rest of region
("Кировская область, Киров", 58.4, 58.8, 49.4, 49.9),
("Ставропольский край, Ставрополь", 44.9, 45.2, 41.7, 42.2),
("Тюменская область, Тюмень", 57.0, 57.3, 65.3, 65.8),
("Челябинская область, Челябинск", 55.0, 55.4, 61.2, 61.7),
]
_REGION_DEFAULT = "Свердловская область, Екатеринбург" # bulk of our data is EKB
def _detect_region_prefix(lat: float | None, lon: float | None) -> str:
if lat is None or lon is None:
return _REGION_DEFAULT
for name, lat_min, lat_max, lon_min, lon_max in _REGION_BBOX:
if lat_min <= lat <= lat_max and lon_min <= lon <= lon_max:
return name
return _REGION_DEFAULT
def _enrich_address_for_imv(raw: str, lat: float | None, lon: float | None) -> str:
"""Prepend region prefix if address lacks one (ambiguity fix for Avito geocoder)."""
addr = (raw or "").strip()
# Already has region marker? skip
addr_lower = addr.lower()
if any(
m in addr_lower
for m in ("область,", " обл.,", " обл,", "край,", "респ.,", "республика,")
):
return addr
prefix = _detect_region_prefix(lat, lon)
return f"{prefix}, {addr}"
DATABASE_URL = os.environ.get("DATABASE_URL")
if not DATABASE_URL:
raise SystemExit(
"Set DATABASE_URL env"
" (e.g. postgresql+psycopg://tradein:...@localhost:35432/tradein)"
)
def pick_lot_params(db: Session, house_id: int) -> dict:
"""Pick representative lot-params from linked listings — median values."""
row = db.execute(
text("""
SELECT
CAST(percentile_cont(0.5) WITHIN GROUP (ORDER BY rooms) AS integer) AS rooms,
CAST(percentile_cont(0.5) WITHIN GROUP (ORDER BY area_m2) AS numeric(8,2))
AS area_m2,
CAST(percentile_cont(0.5) WITHIN GROUP (ORDER BY floor) AS integer) AS floor,
CAST(percentile_cont(0.5) WITHIN GROUP (ORDER BY total_floors) AS integer)
AS total_floors,
mode() WITHIN GROUP (ORDER BY house_type) AS house_type
FROM listings
WHERE house_id_fk = :hid
AND rooms IS NOT NULL
AND area_m2 IS NOT NULL
"""),
{"hid": house_id},
).mappings().first()
if row is None or row["rooms"] is None:
return {} # no listings with required fields, can't pick params
# Fall back to house's own type/floors if listings don't have them
house = db.execute(
text("SELECT house_type, total_floors FROM houses WHERE id = :hid"),
{"hid": house_id},
).mappings().first()
# Avito IMV: rooms=0 (студия) rejected with HTTP 400. Clamp to 1.
rooms_raw = int(row["rooms"])
return {
"rooms": max(rooms_raw, 1),
"area_m2": float(row["area_m2"]),
"floor": int(row["floor"] or 1),
"floor_at_home": int(row["total_floors"] or (house and house["total_floors"]) or 9),
"house_type": _map_house_type(row["house_type"] or (house and house["house_type"])),
"renovation_type": "cosmetic",
"has_balcony": True,
"has_loggia": False,
}
def save_imv_result(db: Session, house_id: int, params: dict, result: IMVEvaluation) -> None:
"""Save 3 things: house_imv_evaluations + history items + suggestions.
Uses already-parsed dataclass fields from IMVEvaluation (placement_history,
suggestions) instead of re-parsing raw_response to avoid duplication and
ensure consistency with _parse_placement_item / _parse_suggestion logic.
"""
# 1. House-level IMV evaluation (UPSERT on house_id)
db.execute(
text("""
INSERT INTO house_imv_evaluations (
house_id, cache_key, recommended_price, lower_price, higher_price,
market_count, raw_response, fetched_at,
rooms, area_m2, floor, floor_at_home, house_type,
renovation_type, has_balcony, has_loggia
) VALUES (
:hid, :ck, :rec, :lo, :hi,
:mc, CAST(:raw AS jsonb), NOW(),
:rooms, :area, :floor, :fah, :htype,
:rtype, :balcony, :loggia
)
ON CONFLICT (house_id) DO UPDATE SET
cache_key = EXCLUDED.cache_key,
recommended_price = EXCLUDED.recommended_price,
lower_price = EXCLUDED.lower_price,
higher_price = EXCLUDED.higher_price,
market_count = EXCLUDED.market_count,
raw_response = EXCLUDED.raw_response,
fetched_at = NOW(),
rooms = EXCLUDED.rooms,
area_m2 = EXCLUDED.area_m2,
floor = EXCLUDED.floor,
floor_at_home = EXCLUDED.floor_at_home,
house_type = EXCLUDED.house_type,
renovation_type = EXCLUDED.renovation_type,
has_balcony = EXCLUDED.has_balcony,
has_loggia = EXCLUDED.has_loggia
"""),
{
"hid": house_id,
"ck": result.cache_key,
"rec": result.recommended_price,
"lo": result.lower_price,
"hi": result.higher_price,
"mc": result.market_count,
"raw": (
json.dumps(result.raw_response, ensure_ascii=False)
if result.raw_response
else None
),
"rooms": params["rooms"],
"area": params["area_m2"],
"floor": params["floor"],
"fah": params["floor_at_home"],
"htype": params["house_type"],
"rtype": params["renovation_type"],
"balcony": params["has_balcony"],
"loggia": params["has_loggia"],
},
)
# 2. Placement history items (already parsed into IMVPlacementHistoryItem dataclasses)
for item in result.placement_history:
db.execute(
text("""
INSERT INTO house_placement_history (
source, house_id, ext_item_id, title, rooms, area_m2,
floor, total_floors, start_price, start_price_date,
last_price, last_price_date, removed_date, exposure_days,
raw_payload, scraped_at
) VALUES (
'avito_imv', :hid, :ext, :title, :rooms, :area,
:floor, :total_floors, :sp, :spd, :lp, :lpd, :rmd, :exp,
CAST(:raw AS jsonb), NOW()
)
ON CONFLICT (source, ext_item_id) DO UPDATE SET
house_id = EXCLUDED.house_id,
title = EXCLUDED.title,
rooms = EXCLUDED.rooms,
area_m2 = EXCLUDED.area_m2,
floor = EXCLUDED.floor,
total_floors = EXCLUDED.total_floors,
start_price = EXCLUDED.start_price,
start_price_date = EXCLUDED.start_price_date,
last_price = EXCLUDED.last_price,
last_price_date = EXCLUDED.last_price_date,
removed_date = EXCLUDED.removed_date,
exposure_days = EXCLUDED.exposure_days,
raw_payload = EXCLUDED.raw_payload,
scraped_at = NOW()
"""),
{
"hid": house_id,
"ext": item.ext_item_id,
"title": item.title,
"rooms": item.rooms,
"area": item.area_m2,
"floor": item.floor,
"total_floors": item.total_floors,
"sp": item.start_price,
"spd": item.start_price_date,
"lp": item.last_price,
"lpd": item.last_price_date,
"rmd": item.removed_date,
"exp": item.exposure_days,
"raw": (
json.dumps(item.raw_payload, ensure_ascii=False)
if item.raw_payload
else None
),
},
)
# 3. Suggestions (already parsed into IMVSuggestion dataclasses)
for sug in result.suggestions:
db.execute(
text("""
INSERT INTO house_suggestions (
house_id, ext_item_id, title, address, price_rub,
exposure_days, publish_date,
item_link, metro_name, metro_distance,
has_good_price_badge, raw_payload, fetched_at
) VALUES (
:hid, :ext, :title, :addr, :price,
:exp, :pdate,
:link, :mname, :mdist,
:gpb, CAST(:raw AS jsonb), NOW()
)
ON CONFLICT (house_id, ext_item_id) DO UPDATE SET
title = EXCLUDED.title,
price_rub = EXCLUDED.price_rub,
exposure_days = EXCLUDED.exposure_days,
publish_date = EXCLUDED.publish_date,
item_link = EXCLUDED.item_link,
metro_name = EXCLUDED.metro_name,
metro_distance = EXCLUDED.metro_distance,
has_good_price_badge = EXCLUDED.has_good_price_badge,
raw_payload = EXCLUDED.raw_payload,
fetched_at = NOW()
"""),
{
"hid": house_id,
"ext": sug.ext_item_id,
"title": sug.title,
"addr": sug.address,
"price": sug.price_rub,
"exp": sug.exposure_days,
"pdate": sug.publish_date,
"link": sug.item_url,
"mname": sug.metro_name,
"mdist": sug.metro_distance,
"gpb": sug.has_good_price_badge,
"raw": (
json.dumps(sug.raw_payload, ensure_ascii=False)
if sug.raw_payload
else None
),
},
)
# 4. Mark house success
db.execute(
text("""
UPDATE houses
SET imv_status = 'ok', last_imv_attempt_at = NOW(), imv_error_reason = NULL
WHERE id = :hid
"""),
{"hid": house_id},
)
def mark_status(db: Session, house_id: int, status: str, reason: str | None = None) -> None:
db.execute(
text("""
UPDATE houses
SET imv_status = :s, last_imv_attempt_at = NOW(), imv_error_reason = :r
WHERE id = :hid
"""),
{"hid": house_id, "s": status, "r": reason},
)
async def process_one(db: Session, house: dict) -> str:
"""Process a single house: geocode + IMV call + save.
Returns final status string:
'ok' / 'no_params' / 'no_address' / 'not_found' / 'auth_error' / 'transient' / 'error'
"""
params = pick_lot_params(db, house["id"])
if not params:
mark_status(db, house["id"], "no_params", "no listings with rooms+area")
return "no_params"
address = house["address"] or house["full_address"]
if not address:
mark_status(db, house["id"], "no_address", "house.address is NULL")
return "no_address"
# Enrich with region prefix to prevent Avito geocoder mis-resolving ambiguous
# street names to wrong regions (e.g. "улица Победы" exists in EKB and Khakasia).
enriched = _enrich_address_for_imv(address, house.get("lat"), house.get("lon"))
if enriched != address:
logger.debug("enriched address house=%d %r -> %r", house["id"], address, enriched)
try:
result = await evaluate_via_imv(address=enriched, **params)
except IMVAddressNotFoundError as e:
mark_status(db, house["id"], "not_found", str(e)[:200])
return "not_found"
except IMVAuthError as e:
mark_status(db, house["id"], "transient_error", f"auth: {e!s}"[:200])
logger.warning("AUTH ERROR house=%d — pausing 60s", house["id"])
await asyncio.sleep(60) # long back-off on auth block
return "auth_error"
except IMVTransientError as e:
mark_status(db, house["id"], "transient_error", str(e)[:200])
return "transient"
except Exception as e:
mark_status(db, house["id"], "error", repr(e)[:200])
logger.error("Unexpected error for house=%d: %r", house["id"], e)
return "error"
save_imv_result(db, house["id"], params, result)
return "ok"
async def main() -> None:
parser = argparse.ArgumentParser(description="Phase C: backfill Avito IMV per house")
parser.add_argument("--batch", type=int, default=50, help="Houses to process this run")
parser.add_argument("--delay", type=float, default=5.0, help="Sleep between houses (s)")
parser.add_argument(
"--limit-total", type=int, default=0,
help="Stop after N successful evaluations (0 = no limit)"
)
parser.add_argument(
"--only-status", default="pending",
help="Process only houses with this imv_status (default: pending)"
)
args = parser.parse_args()
engine = create_engine(DATABASE_URL, future=True)
with Session(engine) as db:
rows = db.execute(
text("""
SELECT id, address, full_address, lat, lon
FROM houses
WHERE imv_status = :status
AND lat IS NOT NULL
AND lon IS NOT NULL
AND address IS NOT NULL
ORDER BY last_imv_attempt_at NULLS FIRST, id
LIMIT :batch
"""),
{"status": args.only_status, "batch": args.batch},
).mappings().all()
if not rows:
logger.info("nothing to process — all done or no houses match filter")
return
logger.info(
"processing %d houses (status=%r delay=%.1fs)",
len(rows), args.only_status, args.delay
)
counters: dict[str, int] = {
"ok": 0, "not_found": 0, "no_params": 0, "no_address": 0,
"transient": 0, "auth_error": 0, "error": 0,
}
for i, house in enumerate(rows, 1):
status = await process_one(db, dict(house))
counters[status] = counters.get(status, 0) + 1
db.commit()
logger.info(
"[%d/%d] house=%d status=%s addr=%r",
i, len(rows), house["id"], status, (house["address"] or "")[:50],
)
if args.limit_total and counters["ok"] >= args.limit_total:
logger.info("hit --limit-total=%d; stopping", args.limit_total)
break
if i < len(rows):
await asyncio.sleep(args.delay)
logger.info("done: %s", counters)
if __name__ == "__main__":
asyncio.run(main())

122
tradein-mvp/scripts/backfill_cian_zhk_url.py
View file

@ -1,122 +0,0 @@
"""Backfill houses.cian_zhk_url for rows that have cian_internal_house_id but no URL.
Requires migration 071_houses_cian_zhk_url.sql to be applied first.
What it does:
SELECT houses WHERE cian_internal_house_id IS NOT NULL AND cian_zhk_url IS NULL
For each row: resolve https://cian.ru/zhk/<id>/ → follow redirect → store final URL.
Cian redirects the canonical /zhk/<id>/ path to the slug URL
(e.g. https://zhk-park-ekb-i.cian.ru/). We capture the final redirect destination.
Usage:
cd tradein-mvp/backend
DATABASE_URL=postgresql://... uv run python ../scripts/backfill_cian_zhk_url.py
Env vars:
DATABASE_URL required — psycopg v3 DSN
DELAY_SEC default 5.0 — seconds between requests (Cian rate limit)
BATCH_LIMIT default 500 — max houses to process per run (safety cap)
Idempotent — skips houses where cian_zhk_url IS NOT NULL. Safe to re-run.
"""
from __future__ import annotations
import asyncio
import logging
import os
import sys
import time
import psycopg
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "backend"))
from app.services.scrapers.cian_newbuilding import resolve_cian_zhk_url
logging.basicConfig(
format="%(asctime)s [%(levelname)s] %(message)s",
level=logging.INFO,
)
logger = logging.getLogger("backfill_cian_zhk_url")
DATABASE_URL = os.environ.get("DATABASE_URL", "")
DELAY_SEC = float(os.environ.get("DELAY_SEC", "5.0"))
BATCH_LIMIT = int(os.environ.get("BATCH_LIMIT", "500"))
async def main() -> None:
if not DATABASE_URL:
logger.error("DATABASE_URL env var not set")
sys.exit(1)
conn = await psycopg.AsyncConnection.connect(DATABASE_URL)
async with conn:
rows = await (
await conn.execute(
"""
SELECT id, cian_internal_house_id
FROM houses
WHERE cian_internal_house_id IS NOT NULL
AND cian_zhk_url IS NULL
ORDER BY id
LIMIT %(lim)s
""",
{"lim": BATCH_LIMIT},
)
).fetchall()
total = len(rows)
logger.info(
"Found %d houses with cian_internal_house_id and no cian_zhk_url (limit=%d)",
total,
BATCH_LIMIT,
)
if total == 0:
logger.info("Nothing to backfill.")
return
succeeded = 0
failed = 0
t0 = time.time()
conn = await psycopg.AsyncConnection.connect(DATABASE_URL)
async with conn:
for i, (house_id, cian_id) in enumerate(rows, start=1):
url = await resolve_cian_zhk_url(cian_id)
if url:
await conn.execute(
"""
UPDATE houses
SET cian_zhk_url = %(url)s
WHERE id = %(hid)s
AND cian_zhk_url IS NULL
""",
{"url": url, "hid": house_id},
)
await conn.commit()
succeeded += 1
logger.info("[%d/%d] house_id=%s id=%s → %s", i, total, house_id, cian_id, url)
else:
failed += 1
logger.warning(
"[%d/%d] house_id=%s id=%s — could not resolve URL",
i, total, house_id, cian_id,
)
if i < total:
await asyncio.sleep(DELAY_SEC)
elapsed = time.time() - t0
logger.info(
"Backfill complete: %d succeeded, %d failed, %.1fs total (%.1fs/req avg)",
succeeded,
failed,
elapsed,
elapsed / max(succeeded + failed, 1),
)
if __name__ == "__main__":
asyncio.run(main())

103
tradein-mvp/scripts/probe-avito-1anchor.py
View file

@ -1,103 +0,0 @@
"""Avito probe: 1 anchor × 1 page с настраиваемой задержкой.
Минимальный dry-run чтобы проверить bans / detection после anti-bot
hardening (PR #487) + global delay setting (PR #485).
Setup:
ssh -f -N -L 15433:172.20.0.2:5432 gendesign
cd tradein-mvp/backend
$env:DATABASE_URL = 'postgresql+psycopg://tradein:<PASS>@localhost:15433/tradein'
Usage:
uv run python ../scripts/probe-avito-1anchor.py
uv run python ../scripts/probe-avito-1anchor.py --delay 5 --radius 1000
"""
from __future__ import annotations
import argparse
import asyncio
import sys
import time
import types
from pathlib import Path
_BACKEND_DIR = Path(__file__).resolve().parent.parent / "backend"
if str(_BACKEND_DIR) not in sys.path:
sys.path.insert(0, str(_BACKEND_DIR))
for _m in (
"weasyprint",
"weasyprint.text",
"weasyprint.text.ffi",
"weasyprint.css",
"weasyprint.css.targets",
):
sys.modules[_m] = types.ModuleType(_m)
sys.modules["weasyprint"].HTML = type("HTML", (), {"__init__": lambda self, *a, **kw: None})
sys.modules["weasyprint"].CSS = type("CSS", (), {"__init__": lambda self, *a, **kw: None})
from app.core.db import SessionLocal # noqa: E402
from app.services.scrapers.avito import AvitoScraper # noqa: E402
from app.services.scrapers.base import save_listings # noqa: E402
ANCHOR_NAME = "Центр ЕКБ"
ANCHOR_LAT = 56.8400
ANCHOR_LON = 60.6050
async def main() -> None:
p = argparse.ArgumentParser()
p.add_argument("--delay", type=float, default=3.0, help="sec между HTTP requests")
p.add_argument("--radius", type=int, default=1500, help="метры")
p.add_argument("--pages", type=int, default=1)
args = p.parse_args()
print(
f"=== Avito probe ===\n"
f" anchor: {ANCHOR_NAME} ({ANCHOR_LAT}, {ANCHOR_LON})\n"
f" radius: {args.radius}m\n"
f" pages: {args.pages}\n"
f" delay: {args.delay}s\n",
flush=True,
)
t0 = time.perf_counter()
try:
async with AvitoScraper() as s:
lots = await s.fetch_around(
ANCHOR_LAT, ANCHOR_LON, args.radius,
pages=args.pages, delay_override_sec=args.delay,
)
except Exception as e: # noqa: BLE001
print(f"FETCH FAIL: {type(e).__name__}: {e}", flush=True)
return
fetched = len(lots)
print(f"fetched: {fetched} lots in {time.perf_counter() - t0:.1f}s", flush=True)
if fetched == 0:
print("⚠️ 0 lots — вероятно 429/ban. Смотри backend logs.", flush=True)
return
db = SessionLocal()
try:
ins, upd = save_listings(db, lots)
print(f"saved: ins={ins} upd={upd}", flush=True)
except Exception as e: # noqa: BLE001
print(f"SAVE FAIL: {type(e).__name__}: {e}", flush=True)
finally:
db.close()
sample = lots[:3]
print("\nsample listings:", flush=True)
for lot in sample:
addr = (lot.address or "")[:60]
print(
f" source_id={lot.source_id} price={lot.price_rub}₽ "
f"rooms={lot.rooms} area={lot.area_m2}м² floor={lot.floor}/{lot.total_floors} "
f"addr={addr!r}",
flush=True,
)
if __name__ == "__main__":
asyncio.run(main())

83
tradein-mvp/scripts/probe-imv-by-address.py
View file

@ -1,83 +0,0 @@
"""Diagnose IMV API behavior for a specific address.
Usage:
python scripts/probe-imv-by-address.py "Свердловская область, г. Екатеринбург, Склад, ул. Заводская, д. 44-а"
Prints: raw API response, error category, cleaned-address retry result.
"""
import asyncio
import logging
import sys
logging.basicConfig(level=logging.DEBUG, format="%(asctime)s %(levelname)s %(message)s")
async def main(address: str) -> None:
from app.services.scrapers.avito_imv import (
IMVAddressNotFoundError,
IMVAuthError,
IMVTransientError,
evaluate_via_imv,
)
# Default params for ЕКБ 3-room 60м²
params = dict(
rooms=3,
area_m2=60.0,
floor=5,
floor_at_home=17,
house_type="panel",
renovation_type="cosmetic",
has_balcony=False,
has_loggia=False,
)
print(f"\n=== Attempt 1: original address ===\n{address}\n")
try:
result = await evaluate_via_imv(address=address, **params) # type: ignore[arg-type]
print(f"OK: recommended={result.recommended_price}")
return
except IMVAddressNotFoundError as e:
print(f"FAIL: address not found (IMVAddressNotFoundError): {e}")
except IMVAuthError as e:
print(f"FAIL: auth/quota error (IMVAuthError): {e}")
return
except IMVTransientError as e:
print(f"FAIL: transient/network error (IMVTransientError): {e}")
return
except Exception as e:
print(f"FAIL: {type(e).__name__}: {e}")
return
import re
cleaned = re.sub(
r"(Склад|Гараж|Подсобка|Нежилое|Помещение|Цех),\s*",
"",
address,
flags=re.IGNORECASE,
)
if cleaned != address:
print(f"\n=== Attempt 2: cleaned address ===\n{cleaned}\n")
try:
result = await evaluate_via_imv(address=cleaned, **params) # type: ignore[arg-type]
print(f"OK on retry: recommended={result.recommended_price}")
except IMVAddressNotFoundError as e:
print(f"FAIL on retry (IMVAddressNotFoundError): {e}")
except IMVAuthError as e:
print(f"FAIL on retry (IMVAuthError — quota/auth): {e}")
except IMVTransientError as e:
print(f"FAIL on retry (IMVTransientError — network/5xx): {e}")
except Exception as e:
print(f"FAIL on retry: {type(e).__name__}: {e}")
else:
print("\nNo noise prefix found — cleaned address identical, skipping retry.")
if __name__ == "__main__":
_address = (
sys.argv[1]
if len(sys.argv) > 1
else "Свердловская область, г. Екатеринбург, Склад, ул. Заводская, д. 44-а"
)
asyncio.run(main(_address))

49
tradein-mvp/scripts/probe-parser-region-filter.py
View file

@ -1,49 +0,0 @@
"""Прогнать current parser на HTML из Playwright debug-страницы (своб.планир p=10).
Проверяет что URL-filter работает: до фикса parser возвращал ~30 карточек других
городов, после — должен 0 или только ЕКБ.
"""
from __future__ import annotations
import asyncio
import sys
import types
from pathlib import Path
_BACKEND = Path(__file__).resolve().parent.parent / "backend"
if str(_BACKEND) not in sys.path:
sys.path.insert(0, str(_BACKEND))
for _m in ("weasyprint", "weasyprint.text", "weasyprint.text.ffi", "weasyprint.css", "weasyprint.css.targets"):
sys.modules[_m] = types.ModuleType(_m)
sys.modules["weasyprint"].HTML = type("HTML", (), {"__init__": lambda self, *a, **kw: None})
sys.modules["weasyprint"].CSS = type("CSS", (), {"__init__": lambda self, *a, **kw: None})
from app.services.scrapers.avito import AvitoScraper # noqa: E402
URL = "https://www.avito.ru/ekaterinburg/kvartiry/prodam/svobodnaya_planirovka-ASgBAgICAkSSA8YQygj8zzI?p=10&s=104"
async def main() -> None:
async with AvitoScraper() as s:
assert s._cffi is not None
# Use scraper's own session — может попасть в 403 (curl_cffi), это OK,
# тогда переключимся на reading from Playwright MCP page.
print(f"Fetching {URL[:80]}...")
try:
r = await s._cffi.get(URL)
print(f" HTTP {r.status_code}, html size: {len(r.text)}")
if r.status_code != 200:
print(" curl_cffi 403 — Avito не отдал. Пробни через MCP / browser save.")
return
except Exception as e: # noqa: BLE001
print(f" fetch fail: {type(e).__name__}: {e}")
return
lots = s._parse_html(r.text, source_url_base=URL)
print(f"\nParsed {len(lots)} EKB lots (после фильтра).")
for lot in lots[:5]:
print(f" {lot.source_id} {lot.source_url[:80]}")
print(f" {lot.address or '-'!r}")
if __name__ == "__main__":
asyncio.run(main())

314
tradein-mvp/scripts/sweep-yandex-valuation-ekb.py
View file

@ -1,314 +0,0 @@
"""Bulk sweep of Yandex.Недвижимость house-history for all EKB yandex addresses.
Iterates `SELECT DISTINCT address FROM listings WHERE source='yandex' AND region_code=66
AND address IS NOT NULL AND is_active=TRUE` and for each address calls the same
cached fetch + save pipeline used by estimator. Idempotent — repeat runs deduplicate
via house_placement_history UNIQUE (source, ext_item_id) and external_valuations cache.
Why this script:
- 160 orphan rows in house_placement_history have house_id=NULL because match
pipeline was wired only after PRs #526 + #531 (2026-05-24).
- 3704 yandex listings on prod with ~hundreds of unique addresses → after one
sweep we get a population of house-history rows linked to houses + with
removed_date / total_floors / source_confidence filled.
Setup (same as local-sweep-ekb-yandex.py):
ssh -f -N -L 15433:172.20.0.2:5432 gendesign # tradein-postgres tunnel
cd tradein-mvp/backend
export DATABASE_URL='postgresql+psycopg://tradein:<PASS>@localhost:15433/tradein'
export ENVIRONMENT='local'
export GLITCHTIP_DSN=''
Usage:
uv run python ../scripts/sweep-yandex-valuation-ekb.py # full
uv run python ../scripts/sweep-yandex-valuation-ekb.py --limit 10 # smoke
uv run python ../scripts/sweep-yandex-valuation-ekb.py --max-pages 3
uv run python ../scripts/sweep-yandex-valuation-ekb.py --skip-recent-days 0 # force re-scrape
uv run python ../scripts/sweep-yandex-valuation-ekb.py \
--address "Екатеринбург, улица Куйбышева, 106"
"""
from __future__ import annotations
import argparse
import asyncio
import random
import sys
import time
import types
from pathlib import Path
# Auto-inject backend/ in sys.path
_BACKEND_DIR = Path(__file__).resolve().parent.parent / "backend"
if str(_BACKEND_DIR) not in sys.path:
sys.path.insert(0, str(_BACKEND_DIR))
# Stub weasyprint (Windows GTK-less import chain compat)
for _m in (
"weasyprint",
"weasyprint.text",
"weasyprint.text.ffi",
"weasyprint.css",
"weasyprint.css.targets",
):
sys.modules[_m] = types.ModuleType(_m)
sys.modules["weasyprint"].HTML = type("HTML", (), {"__init__": lambda self, *a, **kw: None})
sys.modules["weasyprint"].CSS = type("CSS", (), {"__init__": lambda self, *a, **kw: None})
# Backend imports
from sqlalchemy import text # noqa: E402
from app.core.db import SessionLocal # noqa: E402
from app.services.estimator import ( # noqa: E402
_get_or_fetch_yandex_valuation_cached,
_save_yandex_history_items,
)
def _addresses_to_sweep(
limit: int | None,
skip_recent_days: int,
single_address: str | None,
require_house_number: bool = True,
) -> list[str]:
"""Pull unique active EKB yandex addresses, optionally skipping recently-cached ones.
require_house_number: filter to addresses containing a house number. Yandex's
valuation endpoint needs a specific building — bare street names ('Академическая
улица') return 0 items. Heuristic: address must contain a digit after a comma
OR end with a digit token. Default True.
"""
if single_address:
return [single_address.strip()]
# Postgres regex for house-number tokens — used when require_house_number=True.
# Matches ", 12" / ", 12к3" / ", 5Б" anywhere in the address.
house_num_clause = " AND address ~ ',\\s*\\d+' " if require_house_number else ""
db = SessionLocal()
try:
if skip_recent_days > 0:
sql = text(
f"""
SELECT DISTINCT l.address
FROM listings l
WHERE l.source = 'yandex'
AND l.region_code = 66
AND l.is_active = TRUE
AND l.address IS NOT NULL
{house_num_clause}
AND NOT EXISTS (
SELECT 1 FROM external_valuations ev
WHERE ev.source = 'yandex_valuation'
AND ev.address = l.address
AND ev.fetched_at > NOW() - (:days || ' days')::interval
)
ORDER BY l.address
"""
)
rows = db.execute(sql, {"days": skip_recent_days}).mappings().all()
else:
sql = text(
f"""
SELECT DISTINCT address
FROM listings
WHERE source = 'yandex'
AND region_code = 66
AND is_active = TRUE
AND address IS NOT NULL
{house_num_clause}
ORDER BY address
"""
)
rows = db.execute(sql).mappings().all()
addrs = [r["address"] for r in rows if r.get("address")]
finally:
db.close()
if limit is not None and limit > 0:
addrs = addrs[:limit]
return addrs
async def _sweep_one_address(
address: str,
max_pages: int,
delay: float,
) -> tuple[int, int]:
"""Fetch+save up to max_pages for one address.
Pagination stop conditions (whichever fires first):
1. Page returns 0 items (empty / past last page)
2. Cumulative fetched >= total_objects from page-1 header (when known)
3. max_pages reached
Returns (items_fetched_total, items_saved_total).
"""
total_fetched = 0
total_saved = 0
total_objects_expected: int | None = None
seen_keys: set[tuple] = set() # dedup within sweep (page overlap protection)
for page in range(1, max_pages + 1):
db = SessionLocal()
try:
# _get_or_fetch_yandex_valuation_cached only fetches page 1 (no `page`
# arg); cache key includes address only. Pages >= 2 go direct to scraper.
if page == 1:
result = await _get_or_fetch_yandex_valuation_cached(
db,
address=address,
)
else:
from app.services.scrapers.yandex_valuation import YandexValuationScraper
async with YandexValuationScraper() as scraper:
result = await scraper.fetch_house_history(
address=address,
offer_category="APARTMENT",
offer_type="SELL",
page=page,
)
if result is None or not result.history_items:
break
# Capture total_objects on first response (subsequent pages may not carry it)
if total_objects_expected is None and result.house.total_objects:
total_objects_expected = result.house.total_objects
# Dedup against prior pages (Yandex sometimes repeats items at page boundaries)
new_items = []
for it in result.history_items:
key = (it.publish_date, it.area_m2, it.floor, it.start_price)
if key not in seen_keys:
seen_keys.add(key)
new_items.append(it)
saved = _save_yandex_history_items(db, result)
total_fetched += len(new_items)
total_saved += saved
exp_str = f"/{total_objects_expected}" if total_objects_expected is not None else ""
print(
f" page={page}/{max_pages}: items={len(result.history_items)} "
f"new={len(new_items)} saved={saved} cum={total_fetched}{exp_str}",
flush=True,
)
# Stop if we've collected at least what the header promised
if total_objects_expected is not None and total_fetched >= total_objects_expected:
break
# Or if this page yielded zero NEW items (pure duplicates → end of feed)
if not new_items:
break
except Exception as e:
print(
f" page={page} FAIL: {type(e).__name__}: {e}",
flush=True,
)
break
finally:
db.close()
if page < max_pages:
await asyncio.sleep(delay * random.uniform(0.85, 1.3))
return total_fetched, total_saved
async def main() -> None:
p = argparse.ArgumentParser(
description="Bulk sweep Yandex.Недвижимость house-history for EKB yandex addresses"
)
p.add_argument("--limit", type=int, default=None, help="Cap number of addresses (default all)")
p.add_argument(
"--max-pages",
type=int,
default=12,
help="Max pages per address (Yandex pagination cap; default 12)",
)
p.add_argument(
"--delay",
type=float,
default=5.0,
help="Sec between page requests within one address (default 5)",
)
p.add_argument(
"--address-delay",
type=float,
default=15.0,
help="Sec between switching to next address (default 15)",
)
p.add_argument(
"--skip-recent-days",
type=int,
default=1,
help="Skip addresses cached within N days (default 1; 0 = no skip)",
)
p.add_argument(
"--address",
type=str,
default=None,
help="Single address mode — sweep only this address",
)
p.add_argument(
"--require-house-number",
dest="require_house_number",
action="store_true",
default=True,
help="Skip addresses without a house number (default: True). "
"Yandex valuation endpoint needs a specific building; street-only addresses "
"return 0 items. Use --no-require-house-number to disable.",
)
p.add_argument(
"--no-require-house-number",
dest="require_house_number",
action="store_false",
help="Include all addresses (sweep street-only entries too — most return 0).",
)
args = p.parse_args()
addresses = _addresses_to_sweep(
args.limit,
args.skip_recent_days,
args.address,
require_house_number=args.require_house_number,
)
if not addresses:
print("No addresses to sweep (all recently cached or none match query).", flush=True)
return
print(
f"=== Sweep {len(addresses)} addresses, max_pages={args.max_pages}, "
f"delay={args.delay}s, address-delay={args.address_delay}s, "
f"skip-recent={args.skip_recent_days}d ===",
flush=True,
)
t0 = time.perf_counter()
grand_fetched = 0
grand_saved = 0
for i, addr in enumerate(addresses, 1):
print(f"\n[{i}/{len(addresses)}] {addr[:90]}", flush=True)
try:
fetched, saved = await _sweep_one_address(addr, args.max_pages, args.delay)
except Exception as e:
print(f" FATAL on this address: {type(e).__name__}: {e}", flush=True)
continue
grand_fetched += fetched
grand_saved += saved
print(
f" done: fetched={fetched} saved={saved} [grand: f={grand_fetched} s={grand_saved}]",
flush=True,
)
if i < len(addresses):
await asyncio.sleep(args.address_delay * random.uniform(0.85, 1.3))
elapsed = time.perf_counter() - t0
print(
f"\n=== ALL DONE in {elapsed / 60:.1f} min. "
f"addresses={len(addresses)} fetched={grand_fetched} saved={grand_saved} ===",
flush=True,
)
if __name__ == "__main__":
asyncio.run(main())