From 27bbd579c343758179d509e3a030964d657b59a7 Mon Sep 17 00:00:00 2001 From: bot-backend Date: Sat, 4 Jul 2026 15:58:15 +0300 Subject: [PATCH] =?UTF-8?q?chore(tradein/scrapers):=20=D1=83=D0=B4=D0=B0?= =?UTF-8?q?=D0=BB=D0=B8=D1=82=D1=8C=20=D0=B2=D0=B5=D1=81=D1=8C=20legacy=20?= =?UTF-8?q?scrapers/=20=D0=BA=D0=B0=D1=82=D0=B0=D0=BB=D0=BE=D0=B3=20?= =?UTF-8?q?=E2=80=94=20final=20E=20(#2397,=20#2277)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Удаляет весь `app/services/scrapers/` (16 файлов, ~7100 строк) — Part D (D1-D5) убрал всех внешних вызывающих, 0 runtime importers подтверждено grep'ом на main. Заодно: - удалены 7 осиротевших локальных probe/sweep-скриптов (tradein-mvp/scripts/), импортировавших уже-удалённые или удаляемые сейчас legacy-модули - тест-хирургия по 65+ файлам: DELETE прямых legacy-юнит-тестов, RETARGET тестов, тестирующих ещё живую бизнес-логику (переключены на scraper_kit.* эквиваленты, включая quality-gate #781/#753/#754/#755/#773/#740), partial-delete golden-parity тестов, потерявших legacy-оракл - kit save_listings/AvitoScraper/etc. требуют инжектируемые matcher/config — ретаргетированные тесты обновлены под новую сигнатуру (RealScraperConfig(), MagicMock HouseMatcher, region_code=66) Полный pytest suite зелёный (2255 passed, 6 skipped) кроме известного флейка #2208 (test_search_cache_hit, не связан со scrapers). --- .../app/services/cian_price_history.py | 11 +- .../backend/app/services/cian_session.py | 4 +- .../backend/app/services/scrapers/__init__.py | 0 .../backend/app/services/scrapers/avito.py | 2169 ----------------- .../app/services/scrapers/avito_detail.py | 1075 -------- .../app/services/scrapers/avito_exceptions.py | 36 - .../app/services/scrapers/avito_shared.py | 79 - .../backend/app/services/scrapers/base.py | 830 ------- .../app/services/scrapers/browser_fetcher.py | 262 -- .../app/services/scrapers/cian_newbuilding.py | 788 ------ .../services/scrapers/cian_state_parser.py | 160 -- .../app/services/scrapers/cian_valuation.py | 553 ----- .../scrapers/house_type_normalizer.py | 99 - .../app/services/scrapers/price_brackets.py | 32 - .../scrapers/repair_state_normalizer.py | 162 -- .../app/services/scrapers/snapshot_writer.py | 92 - .../app/services/scrapers/yandex_helpers.py | 355 --- .../services/scrapers/yandex_newbuilding.py | 432 ---- .../app/services/yandex_price_history.py | 8 +- .../test_admin_domclick_ingest_kit_parity.py | 78 - .../test_avito_403_backconnect_reconnect.py | 220 -- .../scrapers/test_avito_429_bucket_skip.py | 242 -- .../tests/scrapers/test_avito_anti_bot.py | 425 ---- .../test_avito_detail_403_reconnect.py | 191 -- .../test_avito_detail_429_reconnect.py | 153 -- .../test_avito_detail_browser_only.py | 109 - .../scrapers/test_avito_detail_kit_parity.py | 129 - .../scrapers/test_avito_detail_not_found.py | 82 - .../scrapers/test_avito_detail_soft_block.py | 101 - .../scrapers/test_avito_detail_warmup.py | 140 -- .../tests/scrapers/test_avito_exhaustive.py | 921 ------- .../test_avito_incremental_pagination.py | 315 --- .../tests/scrapers/test_avito_sweep_retry.py | 225 -- .../test_avito_unix_date_tz_consistency.py | 56 +- .../tests/scrapers/test_browser_fetcher.py | 375 --- .../tests/scrapers/test_price_brackets.py | 48 - .../scrapers/test_cian_resolve_zhk_url.py | 18 +- tradein-mvp/backend/tests/support/README.md | 22 +- .../backend/tests/test_781_quality_gate.py | 16 +- .../tests/test_849_matching_savepoint.py | 337 --- .../backend/tests/test_avito_address_clean.py | 82 - .../backend/tests/test_avito_blockpage.py | 125 - .../backend/tests/test_avito_browser_mode.py | 520 ---- .../backend/tests/test_avito_citywide.py | 288 --- .../backend/tests/test_avito_detail_parse.py | 400 --- .../tests/test_avito_ekb_geo_filter.py | 154 -- .../tests/test_avito_newbuilding_sweep.py | 25 +- .../backend/tests/test_avito_relative_date.py | 69 - .../backend/tests/test_avito_serp_date.py | 135 - .../backend/tests/test_base_save_listings.py | 1357 ----------- .../backend/tests/test_cian_newbuilding.py | 382 --- .../backend/tests/test_cian_state_parser.py | 122 - .../backend/tests/test_cian_valuation.py | 477 ---- .../tests/test_estimator_cian_integration.py | 23 +- .../tests/test_extval_house_id_write_path.py | 12 +- .../backend/tests/test_geo_precision.py | 4 +- .../tests/test_house_type_normalizer.py | 127 - .../tests/test_pipeline_browser_routing.py | 22 +- .../tests/test_repair_state_normalizer.py | 188 -- .../test_save_listings_upsert_reconcile.py | 327 --- .../tests/test_scraped_at_bump_2206.py | 64 +- .../test_scraper_kit_avito_golden_parity.py | 206 -- .../test_scraper_kit_base_golden_parity.py | 363 --- .../test_scraper_kit_bisection_parity.py | 225 -- .../test_scraper_kit_cian_golden_parity.py | 369 --- ...scraper_kit_group_c_backfill_kit_parity.py | 42 +- ...scraper_kit_pricehistory_session_parity.py | 109 - .../test_scraper_kit_yandex_golden_parity.py | 123 - .../backend/tests/test_scraper_proxy.py | 80 +- .../backend/tests/test_snapshot_writer.py | 69 +- .../backend/tests/test_yandex_helpers.py | 271 -- .../backend/tests/test_yandex_newbuilding.py | 441 ---- .../tests/test_yandex_price_history.py | 2 +- .../test_yandex_scrapers_delay_wiring.py | 20 +- tradein-mvp/scripts/avito-playwright-sweep.py | 371 --- tradein-mvp/scripts/backfill-house-imv.py | 467 ---- tradein-mvp/scripts/backfill_cian_zhk_url.py | 122 - tradein-mvp/scripts/probe-avito-1anchor.py | 103 - tradein-mvp/scripts/probe-imv-by-address.py | 83 - .../scripts/probe-parser-region-filter.py | 49 - .../scripts/sweep-yandex-valuation-ekb.py | 314 --- 81 files changed, 221 insertions(+), 19861 deletions(-) delete mode 100644 tradein-mvp/backend/app/services/scrapers/__init__.py delete mode 100644 tradein-mvp/backend/app/services/scrapers/avito.py delete mode 100644 tradein-mvp/backend/app/services/scrapers/avito_detail.py delete mode 100644 tradein-mvp/backend/app/services/scrapers/avito_exceptions.py delete mode 100644 tradein-mvp/backend/app/services/scrapers/avito_shared.py delete mode 100644 tradein-mvp/backend/app/services/scrapers/base.py delete mode 100644 tradein-mvp/backend/app/services/scrapers/browser_fetcher.py delete mode 100644 tradein-mvp/backend/app/services/scrapers/cian_newbuilding.py delete mode 100644 tradein-mvp/backend/app/services/scrapers/cian_state_parser.py delete mode 100644 tradein-mvp/backend/app/services/scrapers/cian_valuation.py delete mode 100644 tradein-mvp/backend/app/services/scrapers/house_type_normalizer.py delete mode 100644 tradein-mvp/backend/app/services/scrapers/price_brackets.py delete mode 100644 tradein-mvp/backend/app/services/scrapers/repair_state_normalizer.py delete mode 100644 tradein-mvp/backend/app/services/scrapers/snapshot_writer.py delete mode 100644 tradein-mvp/backend/app/services/scrapers/yandex_helpers.py delete mode 100644 tradein-mvp/backend/app/services/scrapers/yandex_newbuilding.py delete mode 100644 tradein-mvp/backend/tests/scrapers/test_admin_domclick_ingest_kit_parity.py delete mode 100644 tradein-mvp/backend/tests/scrapers/test_avito_403_backconnect_reconnect.py delete mode 100644 tradein-mvp/backend/tests/scrapers/test_avito_429_bucket_skip.py delete mode 100644 tradein-mvp/backend/tests/scrapers/test_avito_anti_bot.py delete mode 100644 tradein-mvp/backend/tests/scrapers/test_avito_detail_403_reconnect.py delete mode 100644 tradein-mvp/backend/tests/scrapers/test_avito_detail_429_reconnect.py delete mode 100644 tradein-mvp/backend/tests/scrapers/test_avito_detail_browser_only.py delete mode 100644 tradein-mvp/backend/tests/scrapers/test_avito_detail_kit_parity.py delete mode 100644 tradein-mvp/backend/tests/scrapers/test_avito_detail_not_found.py delete mode 100644 tradein-mvp/backend/tests/scrapers/test_avito_detail_soft_block.py delete mode 100644 tradein-mvp/backend/tests/scrapers/test_avito_detail_warmup.py delete mode 100644 tradein-mvp/backend/tests/scrapers/test_avito_exhaustive.py delete mode 100644 tradein-mvp/backend/tests/scrapers/test_avito_incremental_pagination.py delete mode 100644 tradein-mvp/backend/tests/scrapers/test_avito_sweep_retry.py delete mode 100644 tradein-mvp/backend/tests/scrapers/test_browser_fetcher.py delete mode 100644 tradein-mvp/backend/tests/scrapers/test_price_brackets.py delete mode 100644 tradein-mvp/backend/tests/test_849_matching_savepoint.py delete mode 100644 tradein-mvp/backend/tests/test_avito_address_clean.py delete mode 100644 tradein-mvp/backend/tests/test_avito_blockpage.py delete mode 100644 tradein-mvp/backend/tests/test_avito_browser_mode.py delete mode 100644 tradein-mvp/backend/tests/test_avito_citywide.py delete mode 100644 tradein-mvp/backend/tests/test_avito_detail_parse.py delete mode 100644 tradein-mvp/backend/tests/test_avito_ekb_geo_filter.py delete mode 100644 tradein-mvp/backend/tests/test_avito_relative_date.py delete mode 100644 tradein-mvp/backend/tests/test_avito_serp_date.py delete mode 100644 tradein-mvp/backend/tests/test_base_save_listings.py delete mode 100644 tradein-mvp/backend/tests/test_cian_newbuilding.py delete mode 100644 tradein-mvp/backend/tests/test_cian_state_parser.py delete mode 100644 tradein-mvp/backend/tests/test_cian_valuation.py delete mode 100644 tradein-mvp/backend/tests/test_house_type_normalizer.py delete mode 100644 tradein-mvp/backend/tests/test_repair_state_normalizer.py delete mode 100644 tradein-mvp/backend/tests/test_save_listings_upsert_reconcile.py delete mode 100644 tradein-mvp/backend/tests/test_scraper_kit_avito_golden_parity.py delete mode 100644 tradein-mvp/backend/tests/test_scraper_kit_base_golden_parity.py delete mode 100644 tradein-mvp/backend/tests/test_scraper_kit_bisection_parity.py delete mode 100644 tradein-mvp/backend/tests/test_scraper_kit_cian_golden_parity.py delete mode 100644 tradein-mvp/backend/tests/test_scraper_kit_pricehistory_session_parity.py delete mode 100644 tradein-mvp/backend/tests/test_scraper_kit_yandex_golden_parity.py delete mode 100644 tradein-mvp/backend/tests/test_yandex_helpers.py delete mode 100644 tradein-mvp/backend/tests/test_yandex_newbuilding.py delete mode 100644 tradein-mvp/scripts/avito-playwright-sweep.py delete mode 100644 tradein-mvp/scripts/backfill-house-imv.py delete mode 100644 tradein-mvp/scripts/backfill_cian_zhk_url.py delete mode 100644 tradein-mvp/scripts/probe-avito-1anchor.py delete mode 100644 tradein-mvp/scripts/probe-imv-by-address.py delete mode 100644 tradein-mvp/scripts/probe-parser-region-filter.py delete mode 100644 tradein-mvp/scripts/sweep-yandex-valuation-ekb.py diff --git a/tradein-mvp/backend/app/services/cian_price_history.py b/tradein-mvp/backend/app/services/cian_price_history.py index 93d155e5..8fdc7ed7 100644 --- a/tradein-mvp/backend/app/services/cian_price_history.py +++ b/tradein-mvp/backend/app/services/cian_price_history.py @@ -16,11 +16,12 @@ import time from dataclasses import dataclass, field # #2306: fetch_detail/save_detail_enrichment migrated to scraper_kit (byte-identical -# golden-parity была доказана против legacy app.services.scrapers.cian_detail до его -# удаления, #2397 Part E2 — см. tests/test_scraper_kit_pricehistory_session_parity.py -# для оставшейся extract_state/ScrapedLot parity). RealScraperConfig — тот же -# read-only адаптер над settings, что и остальные kit-инжекции (#2131) — сохраняет -# proxy-поведение (config.cian_proxy_url) идентичным прежнему прямому импорту settings. +# golden-parity была доказана против legacy cian_detail-модуля до его удаления, +# #2397 Part E2; extract_state/ScrapedLot parity-тесты убраны вместе с остальным +# legacy scrapers-каталогом, #2397 финальный шаг E — kit единственный живой путь). +# RealScraperConfig — тот же read-only адаптер над settings, что и остальные +# kit-инжекции (#2131) — сохраняет proxy-поведение (config.cian_proxy_url) +# идентичным прежнему прямому импорту settings. from scraper_kit.providers.cian.detail import fetch_detail, save_detail_enrichment from sqlalchemy import text from sqlalchemy.orm import Session diff --git a/tradein-mvp/backend/app/services/cian_session.py b/tradein-mvp/backend/app/services/cian_session.py index 9768b41a..56d83a16 100644 --- a/tradein-mvp/backend/app/services/cian_session.py +++ b/tradein-mvp/backend/app/services/cian_session.py @@ -13,8 +13,8 @@ from typing import Any from curl_cffi.requests import AsyncSession # #2306: extract_state migrated to scraper_kit (byte-identical to legacy -# app.services.scrapers.cian_state_parser, см. tests/test_scraper_kit_cian_golden_parity.py -# + tests/test_scraper_kit_pricehistory_session_parity.py). +# cian_state_parser module, deleted #2397 финальный шаг E — golden-parity против +# него уже недоступна, kit — единственный живой путь). from scraper_kit.cian_state_parser import extract_state from sqlalchemy import text from sqlalchemy.orm import Session diff --git a/tradein-mvp/backend/app/services/scrapers/__init__.py b/tradein-mvp/backend/app/services/scrapers/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tradein-mvp/backend/app/services/scrapers/avito.py b/tradein-mvp/backend/app/services/scrapers/avito.py deleted file mode 100644 index 7a3b673b..00000000 --- a/tradein-mvp/backend/app/services/scrapers/avito.py +++ /dev/null @@ -1,2169 +0,0 @@ -"""Avito.ru scraper — парсинг вторички вокруг точки + citywide/byrooms sweep. - -Стратегия: HTML scrape карточек объявлений через DOM. - -URL patterns (EKB): - geo-mode (default): - https://www.avito.ru/ekaterinburg/kvartiry/prodam-ASgBAgICAUSSA8YQ - ?geoCoords=56.838,60.605 - &radius=1 # в км - &s=104 # sort by date - &p=1 # страница - - citywide-mode (без geo, весь город, T6): - https://www.avito.ru/ekaterinburg/kvartiry/prodam-ASgBAgICAUSSA8YQ - ?s=104&p=1 - - byrooms-mode (citywide + фильтр комнатности, T6): - https://www.avito.ru/ekaterinburg/kvartiry/prodam/?s=104&p=1 - -ВАЖНО: Avito банит httpx (403/429) по TLS fingerprint от server IP. -Используем curl_cffi с impersonate='chrome120' — настоящий Chrome TLS ClientHello. -""" - -from __future__ import annotations - -import asyncio -import inspect -import logging -import math -import random -import re -from collections.abc import Callable -from datetime import date, timedelta -from typing import Any -from urllib.parse import urlencode, urljoin, urlparse, urlunparse - -import httpx -from curl_cffi.requests import AsyncSession -from selectolax.parser import HTMLParser - -from app.core.config import settings -from app.services.scraper_settings import get_scraper_delay -from app.services.scrapers.avito_exceptions import ( - AvitoBlockedError, - AvitoContentBlockedError, - AvitoRateLimitedError, -) -from app.services.scrapers.avito_shared import _unix_to_date -from app.services.scrapers.base import BaseScraper, ScrapedLot -from app.services.scrapers.browser_fetcher import BrowserFetcher -from app.services.scrapers.price_brackets import get_price_seed_brackets - -logger = logging.getLogger(__name__) - -# ── Relative date parsing ──────────────────────────────────────────────────── -_REL_DATE_RE = re.compile( - r"(?P\d+)\s+(?P" - r"секунд[ауы]?|" - r"минут[ауы]?|" - r"час(?:а|ов)?|" - r"день|дн(?:я|ей?)|" - r"недел[июяь]+|" - r"месяц(?:а|ев)?|" - r"год(?:а|ов)?" - r")\s+назад", - flags=re.I, -) -_UNIT_DAYS: dict[str, int] = { - "секунд": 0, - "минут": 0, - "час": 0, - "день": 1, - "дн": 1, - "недел": 7, - "месяц": 30, - "год": 365, -} -# #1387: Avito SERP часто показывает давность в единственном числе БЕЗ ведущего -# числа — «час назад», «минуту назад», «день назад», «неделю назад», «месяц -# назад», «год назад». Числовой `_REL_DATE_RE` их не матчит (требует `\d+`), -# поэтому отдельная ветка с неявным n=1. Юниты в винительном падеже («минуту», -# «неделю») плюс именительный («час», «день»). -_REL_DATE_SINGULAR_RE = re.compile( - r"(?P" r"секунду|" r"минуту|" r"час|" r"день|" r"неделю|" r"месяц|" r"год" r")\s+назад", - flags=re.I, -) - -# Russian month names → number. Avito показывает абсолютные даты как «18 мая». -_RU_MONTHS: dict[str, int] = { - "январ": 1, - "феврал": 2, - "март": 3, - "апрел": 4, - "ма": 5, - "июн": 6, - "июл": 7, - "август": 8, - "сентябр": 9, - "октябр": 10, - "ноябр": 11, - "декабр": 12, -} -# «18 мая», «5 июня в 12:30» — day + month (year inferred current). -_ABS_DATE_RE = re.compile( - r"(?P\d{1,2})\s+(?Pянвар\w*|феврал\w*|март\w*|апрел\w*|" - r"ма[яй]|июн\w*|июл\w*|август\w*|сентябр\w*|октябр\w*|ноябр\w*|декабр\w*)", - flags=re.I, -) - - -def _parse_relative_date(s: str | None) -> date | None: - """Avito показывает дату публикации в трёх форматах: - * '5 дней назад' / 'час назад' / 'неделю назад' — relative numeric - * 'сегодня в 12:30' / 'вчера в 18:45' / 'позавчера' — relative keyword - * '18 мая' / '5 июня в 12:30' — absolute day + month (year inferred) - - PR M-followup 2026-05-27: расширено покрытие — раньше только relative numeric - matched (regex `\\d+ unit назад`), 100% Avito listings заходило с listing_date=None - т.к. Avito SERP для свежих cards использует keyword/absolute форматы. См. - тесты для каждого case. - """ - if not s: - return None - text_lower = s.lower() - - # Keyword shortcuts (no \d+). - if "позавчера" in text_lower: - return date.today() - timedelta(days=2) - if "вчера" in text_lower: - return date.today() - timedelta(days=1) - if "сегодня" in text_lower or "только что" in text_lower: - return date.today() - - # Relative numeric («5 дней назад»). - m = _REL_DATE_RE.search(s) - if m: - n = int(m["n"]) - unit_raw = m["unit"].lower() - for prefix, days in _UNIT_DAYS.items(): - if unit_raw.startswith(prefix): - return date.today() - timedelta(days=n * days) - - # #1387: Relative singular без числа («час назад», «неделю назад»). Неявное - # n=1; те же _UNIT_DAYS. Винительный падеж юнита нормализуем к ключу-префиксу. - m = _REL_DATE_SINGULAR_RE.search(s) - if m: - unit_raw = m["unit"].lower() - # «минуту»→«минут», «неделю»→«недел», «секунду»→«секунд»; остальные уже - # совпадают с префиксами _UNIT_DAYS. - unit_key = {"минуту": "минут", "неделю": "недел", "секунду": "секунд"}.get( - unit_raw, unit_raw - ) - for prefix, days in _UNIT_DAYS.items(): - if unit_key.startswith(prefix): - return date.today() - timedelta(days=days) - - # Absolute («18 мая», «5 июня в 12:30»). Year — current; если месяц - # в будущем относительно сегодня (например сейчас декабрь, увидели - # «5 января») — откатываем к прошлому году. - m = _ABS_DATE_RE.search(s) - if m: - day = int(m["day"]) - month_raw = m["month"].lower() - month: int | None = None - for prefix, num in _RU_MONTHS.items(): - if month_raw.startswith(prefix): - month = num - break - if month is not None and 1 <= day <= 31: - today = date.today() - year = today.year - try: - parsed = date(year, month, day) - except ValueError: - return None - # Если parsed в будущем больше чем на 30 дней — Avito показывает - # прошлогоднюю дату («декабрь» в феврале значит прошлый декабрь). - if (parsed - today).days > 30: - try: - parsed = date(year - 1, month, day) - except ValueError: - return None - return parsed - - return None - - -# #726: Avito SERP лениво рендерит DOM — `data-marker="item-date"` присутствует только -# у верхних (above-the-fold) карточек (~10 из ~46), у остальных его НЕТ в raw-HTML, что -# curl_cffi получает (JS-гидрация не выполняется). Но per-item JSON в той же HTML несёт -# `sortTimeStamp` (epoch-ms) почти у всех карточек — это drift-resistant источник даты. -# Маппим item_id (= DOM `data-item-id`) → дата. `{0,1500}` ограничивает поиск телом -# одного item-объекта (id и sortTimeStamp лежат в нём; внутри ещё есть короткий вложенный -# `addressDetailed{…}`, поэтому не `[^{}]`). re.S — sortTimeStamp может быть на новой строке. -# sortTimeStamp — UTC epoch-ms. Avito показывает дату в MSK (UTC+3, без DST с 2014). -# Берём .date() в MSK через общий `_unix_to_date` (avito_shared.py, #2129), чтобы -# совпадало с тем, что видно на сайте (иначе объявления, поднятые в 00:00–03:00 MSK, -# давали бы дату на день раньше). #726. -_SORT_TS_RE = re.compile(r'"id":(?P\d{8,12})\b.{0,1500}?"sortTimeStamp":(?P\d+)', re.S) - - -def _build_sort_timestamp_map(html: str) -> dict[str, date]: - """item_id → дата публикации из embedded JSON (`sortTimeStamp`, epoch-ms). - - Best-effort: при любой ошибке конкретной записи — пропускаем её (не роняем парс). - Первое вхождение id выигрывает (item-объект идёт раньше recommendations-дублей). - `_unix_to_date` ожидает СЕКУНДЫ — sortTimeStamp в МИЛЛИСЕКУНДАХ, делим на 1000. - """ - out: dict[str, date] = {} - for m in _SORT_TS_RE.finditer(html): - item_id = m.group("id") - if item_id in out: - continue - date_val = _unix_to_date(int(m.group("ts")) / 1000) - if date_val is not None: - out[item_id] = date_val - return out - - -# #623: Avito банит datacenter-IP — на HTTP 200 отдаёт firewall-интерстициал -# «Доступ ограничен: проблема с IP» (title в ), а НЕ 403. Детектим по -# маркерам в начале документа (firewall-страница объёмна, не сканируем целиком). -_FIREWALL_MARKERS = ("доступ ограничен", "проблема с ip", "firewall-container") - -# EKB-фильтр «Новостройка» (slug категории kvartiry/prodam). Кодирует -# ASgB-параметры выборки только новостроек — SERP возвращает 100% new-build -# карточки (с data-marker="item-development-name"). Извлечён из реального -# search-URL: /ekaterinburg/kvartiry/prodam/novostroyka-ASgBAgICAkSSA8YQ5geOUg -NOVOSTROYKA_SLUG = "novostroyka-ASgBAgICAkSSA8YQ5geOUg" - -# ── Exhaustive full-load (room×price bisection) ─────────────────────────────── -# Верхняя граница цены при первом рекурсивном делении (нет явного hi). -# 200 млн ₽ заведомо выше ТОПа ЕКБ — дальние бакеты дадут 0 результатов. -# Больше НЕ корень бисекции (см. _AVITO_PRICE_SEED_BRACKETS) — оставлен для -# совместимости (используется в тестах probe-устойчивости как sentinel). -_AVITO_MAX_PRICE = 200_000_000 -# Минимальный ценовой диапазон для рекурсии; при hi - lo < MIN_BRACKET прекращаем -# деление и пагинируем как есть (принимаем возможный «хвост»). -_AVITO_MIN_BRACKET = 50_000 - -# Seed price-brackets (руб) — data-driven по прод-распределению вторички ЕКБ -# (пик 4-8М, bulk 3-16М). Теперь общий констант для всех exhaustive-скрапперов -# (avito/cian/yandex) — app.services.scrapers.price_brackets.get_price_seed_brackets(). -# Локальный алиас сохранён для читаемости и тестов; значения идентичны shared EKB. -# Каждый (комнатность × брекет) дальше дробится _walk_price_range если total > cap. -# Последний брекет ОТКРЫТ (hi=None → pmax не ставится) — ловит весь хвост люкса -# без потолка; он крошечный (avito >50М ≈ 73), пагинируется напрямую без бисекции. -_AVITO_PRICE_SEED_BRACKETS: list[tuple[int, int | None]] = get_price_seed_brackets() -# Avito SERP показывает ~50 карточек на страницу. -_AVITO_OFFERS_PER_PAGE = 50 - -# HTTP 429 в curl_cffi-режиме через backconnect-прокси (mproxy.site) — НЕ IP-ban, а -# transient «слишком много одновременных соединений» (лимит 5). Проходит на коротком -# retry без ротации IP. Делаем до _AVITO_429_MAX_RETRIES коротких пауз; если они -# исчерпаны под backconnect — ещё _AVITO_429_RECONNECT_RETRIES попыток через -# пересоздание сессии (_reset_cffi): свежее соединение освобождает удерживаемый -# conn-slot и может попасть на менее загруженный путь. Только потом — -# AvitoRateLimitedError (run 216: deep-pagination 429 на page 36 ронял весь sweep). -_AVITO_429_MAX_RETRIES = 8 -_AVITO_429_BACKOFF_SEC = 1.5 -# Reconnect-fallback на исчерпанном 429 short-retry (только backconnect): дольше спим -# (_AVITO_403_BACKOFF_SEC), чтобы дать conn-slot'ам разгрузиться. -_AVITO_429_RECONNECT_RETRIES = 3 - -# HTTP 403 / firewall-интерстициал в curl_cffi-режиме через backconnect-прокси -# (mproxy.site, нет changeip-URL): провайдер выдаёт СВЕЖИЙ exit-IP на каждое НОВОЕ -# соединение (5 последовательных запросов → 5 разных IP). 403 значит залочен лишь -# ТЕКУЩИЙ exit-IP — открытие нового CONNECT-туннеля escape'ит бан. Keep-alive curl_cffi -# сессия переиспользует тот же туннель = тот же IP, поэтому retry на том же self._cffi -# не помогает: нужно пересоздать сессию (_reset_cffi). Делаем до _AVITO_403_MAX_RETRIES -# reconnect-попыток прежде чем свалиться в AvitoBlockedError (mark_banned). Старый -# single-IP changeip-путь и browser-режим не затрагиваются. -_AVITO_403_MAX_RETRIES = 5 -_AVITO_403_BACKOFF_SEC = 2.0 - -# tradein-browser sidecar (POST /fetch) отдаёт HTTP 500 на ЛЮБУЮ внутреннюю -# ошибку страницы (тело {"error": "ErrorType: message"}) и HTTP 503 когда браузер -# не поднимается (прокси упал). Транзиентные сбои (Page.goto Timeout, Target -# closed, прочие NS_ERROR_/5xx) лечатся коротким backoff-retry того же url БЕЗ -# ротации IP; soft-ban (NS_ERROR_PROXY_TOO_MANY_REQUESTS / 503 / browser -# unavailable) — ротацией exit-IP. Когда ОБА бюджета исчерпаны → AvitoRateLimitedError, -# чтобы run_avito_full_load/run_avito_city_sweep ушли в graceful mark_banned путь -# (partial preserved), а не в generic except → mark_failed (прод-инцидент run #356: -# один поздний 500 ронял весь обход после 3186 сохранённых объявлений). -_AVITO_SIDECAR_TRANSIENT_RETRIES = 2 -_AVITO_SIDECAR_TRANSIENT_BACKOFF_SEC = 2.0 - -# Per-bucket block tolerance в fetch_all_secondary: одиночный 429/блок на ОДНОМ -# (комнатность × ценовой брекет) бакете (напр. deep-pagination dense-бакета) НЕ должен -# ронять весь sweep — бакет скипается, обход продолжается. Только N ПОДРЯД заблокированных -# бакетов = реальный hard IP-ban → re-raise (run_avito_full_load → mark_banned). -_AVITO_SWEEP_MAX_CONSECUTIVE_BLOCKED = 4 - -# Счётчик результатов в шапке SERP: 1 178. -# Текст может содержать NBSP (\xa0),  -entity или обычные пробелы как разделители -# разрядов — чистятся в _extract_total_count. [^<]* нежадно ловит весь текст до . -_TOTAL_COUNT_RE = re.compile( - r'data-marker="page-title/count"[^>]*>(?P[^<]*)<', - flags=re.I, -) - -# Маркер валидной ПУСТОЙ выборки. На 0-результатной странице (напр. студии с -# pmin=17M, которых нет) Avito отдаёт HTTP 200 БЕЗ `page-title/count`, но с -# текстом «ничего не найдено» И ~50 fallback-«похожими» объявлениями (НЕ -# подходящими под ценовой фильтр). Без этого маркера _extract_total_count -# возвращал None → resilience-логика дробила пустой брекет + ротировала IP -# (прод-инцидент run 209: 14+ мин в пустых верхних брекетах, errors=1058). -# Lowercase substring «ничего не найден» ловит «найдено»/«найдены»/любой регистр. -_AVITO_NO_RESULTS_MARKERS = ("ничего не найден",) - - -def _is_firewall_page(html: str) -> bool: - """True если Avito вернул firewall-страницу IP-блока (на HTTP 200).""" - head = html[:4096].lower() - return any(marker in head for marker in _FIREWALL_MARKERS) - - -class AvitoScraper(BaseScraper): - """Avito vtorichka parser. Источник = 'avito'. - - Использует curl_cffi с impersonate=chrome120 для обхода TLS fingerprint бана. - """ - - name = "avito" - base_url = "https://www.avito.ru" - # Avito жёстко мониторит — спим долго между запросами. - # Класс-дефолт; реальное значение загружается из scraper_settings при создании экземпляра. - request_delay_sec = 7.0 - - def __init__(self) -> None: - super().__init__() - self.request_delay_sec = get_scraper_delay(self.name) - self._cffi: AsyncSession | None = None - self._browser: BrowserFetcher | None = None - # #823: счётчик карточек, которые не удалось распарсить из-за неожиданной структуры DOM. - self.parse_failures: int = 0 - - async def __aenter__(self) -> AvitoScraper: - # Проактивная ротация IP в начале sweep (#1731): apw-IP «протухает» по - # Datadome между суточными прогонами, поэтому первые якоря блокируются и - # реактивного бюджета max_rotations не хватает (3 подряд блока → - # mark_banned). Один changeip-вызов до первого SERP-fetch даёт свежий IP. - # _rotate_ip() сам ждёт ~9с (settle) и логирует/глотает ошибку, поэтому - # тут только дополнительный guard: сбой ротации НЕ должен ронять sweep. - if settings.avito_proxy_rotate_url: - try: - if await self._rotate_ip(): - logger.info("avito: proactive IP rotate at sweep start (#1731)") - else: - logger.warning("avito: proactive IP rotate at sweep start failed — proceeding") - except Exception: - logger.warning( - "avito: proactive IP rotate at sweep start raised — proceeding", exc_info=True - ) - if settings.scraper_fetch_mode == "browser": - # Default (#1805): SERP-фетч через camoufox (tradein-browser per-provider - # + ротирующий backconnect-прокси). curl_cffi поднимаем тут же как - # FALLBACK — если браузер отдаёт firewall/пусто, _fetch_serp_html делает - # один retry через curl_cffi. _build_cffi_session самодостаточен - # (отдельный AsyncSession, не зависит от BaseScraper.__aenter__). - self._browser = BrowserFetcher(source="avito") - await self._browser.__aenter__() - self._cffi = self._build_cffi_session() - logger.info( - "avito: SERP fetch via BrowserFetcher (camoufox) default, " - "curl_cffi fallback armed — #1805" - ) - return self - # Explicit opt-out: legacy curl_cffi-only путь (SCRAPER_FETCH_MODE=curl_cffi). - await super().__aenter__() - self._cffi = self._build_cffi_session() - return self - - def _build_cffi_session(self) -> AsyncSession: - """Создать curl_cffi-сессию (impersonate chrome120, через прокси если задан). - - Вынесено из __aenter__ чтобы _reset_cffi мог пересоздать сессию (новый - CONNECT-туннель) для escape 403 на залоченном backconnect exit-IP. - """ - proxies = None - if settings.scraper_proxy_url: - proxies = {"http": settings.scraper_proxy_url, "https": settings.scraper_proxy_url} - logger.info("avito: routing through mobile proxy egress (#623)") - return AsyncSession( - impersonate="chrome120", - timeout=25, - proxies=proxies, - headers={ - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", - "Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8", - "Cache-Control": "max-age=0", - "Sec-Fetch-Dest": "document", - "Sec-Fetch-Mode": "navigate", - "Sec-Fetch-Site": "none", - "Sec-Fetch-User": "?1", - "Upgrade-Insecure-Requests": "1", - }, - ) - - async def _reset_cffi(self) -> None: - """Закрыть и пересоздать curl_cffi-сессию — новый CONNECT-туннель через - backconnect-прокси = свежий exit-IP (escape 403 на залоченном IP).""" - if self._cffi is not None: - try: - await self._cffi.close() - except Exception: - pass - self._cffi = self._build_cffi_session() - - async def __aexit__(self, *args: Any) -> None: - if self._browser is not None: - await self._browser.__aexit__(*args) - if self._cffi is not None: - await self._cffi.close() - await super().__aexit__(*args) - - # ── Anti-block (#623) ───────────────────────────────────────────────────── - async def _rotate_ip(self) -> bool: - """Сменить мобильный IP через changeip-ссылку mobileproxy (#623). - - Дёргается напрямую (без прокси) — это API провайдера, не Avito. Ждём ~9с: - мобильному модему нужно время поднять новый IP. Returns True при успехе. - """ - rotate_url = settings.avito_proxy_rotate_url - if not rotate_url: - return False - sep = "&" if "?" in rotate_url else "?" - try: - async with AsyncSession(timeout=30) as rot: - await rot.get(f"{rotate_url}{sep}format=json") - await asyncio.sleep(9) - logger.info("avito proxy: IP rotated via changeip") - return True - except Exception: - logger.warning("avito proxy: IP rotation failed", exc_info=True) - return False - - async def _fetch_serp_html(self, url: str, page: int) -> str | None: - """GET SERP HTML — dispatcher между browser-default и curl_cffi (#1805). - - Если поднят BrowserFetcher (default с #1805) — фетчим через camoufox с - curl_cffi-fallback на firewall (_fetch_serp_html_browser). Иначе - (legacy/explicit curl_cffi opt-out, либо тесты с прямым _cffi) — прямой - curl_cffi-путь (_fetch_serp_html_cffi). - - Returns: - HTML при успехе, либо ``None`` при non-200 (конец пагинации). - Raises: - AvitoBlockedError / AvitoRateLimitedError — если бан не снят. - """ - if self._browser is not None: - return await self._fetch_serp_html_browser(url, page) - return await self._fetch_serp_html_cffi(url, page) - - async def _fetch_serp_html_browser(self, url: str, page: int) -> str | None: - """SERP-фетч через camoufox-браузер (#1805, default) с curl_cffi-fallback. - - Браузер не отдаёт HTTP-статус — бан виден только как rendered firewall-HTML - (camoufox исполняет JS, поэтому firewall-интерстициал приходит готовым - документом; ловим теми же текст-маркерами _is_firewall_page). - - Pipeline: - 1. browser.fetch → firewall? → ротация IP (до max_rotations) + retry. - 2. Ротация исчерпана и всё ещё firewall/пусто → ОДИН retry через curl_cffi - (другой egress/TLS-fingerprint может проскочить). curl-fallback - доступен только если _cffi поднят (standalone __aenter__); в - pipeline-режиме с shared-browser _cffi=None → fallback пропускается. - 3. curl-fallback тоже firewall/пусто (или недоступен) → AvitoBlockedError. - - Отдельная ветка — sidecar HTTP-ошибка (#356): BrowserFetcher пробрасывает - httpx.HTTPStatusError (500/503), если /fetch упал внутри (Page.goto Timeout, - NS_ERROR_PROXY_TOO_MANY_REQUESTS, браузер не поднялся). Раньше она всплывала - неперехваченной → generic except в run_avito_full_load → mark_failed + - re-raise, что роняло весь обход даже после тысяч инкрементально сохранённых - карточек. Теперь классифицируем тело ошибки и роутим в ту же - ротация+retry-машинерию: soft-ban → ротация IP, transient → bounded backoff, - оба бюджета исчерпаны → AvitoRateLimitedError (graceful mark_banned). - """ - assert self._browser is not None - max_rot = settings.avito_proxy_max_rotations if settings.avito_proxy_rotate_url else 0 - rot_done = 0 - # Независимый бюджет коротких retry на transient sidecar-ошибки (timeout/ - # crash) — НЕ тратит бюджет ротации. Каждая ветка ниже (ротация / transient) - # уменьшает свой счётчик → цикл всегда движется к условию исчерпания. - transient_left = _AVITO_SIDECAR_TRANSIENT_RETRIES - while True: - try: - html = await self._browser.fetch(url) - except httpx.HTTPStatusError as exc: - # Тело sidecar-ошибки: {"error": "ErrorType: message"}. Парсим - # устойчиво — невалидный JSON не должен ронять обработку. - status = exc.response.status_code - try: - error_text = str(exc.response.json().get("error", "")) - except Exception: - error_text = exc.response.text - lowered = error_text.lower() - - is_soft_ban = ( - "too_many_requests" in lowered - or status == 503 - or "browser unavailable" in lowered - or "proxy may be down" in lowered - ) - if is_soft_ban: - # Soft proxy rate-limit / прокси недоступен → свежий exit-IP. - if rot_done < max_rot and await self._rotate_ip(): - rot_done += 1 - logger.info( - "avito page=%d sidecar soft-ban (status=%d) — " - "retry after rotation #%d: %s", - page, - status, - rot_done, - error_text[:200], - ) - continue - else: - # Transient (Page.goto Timeout / Target closed / прочие 5xx) → - # короткий backoff-retry того же url БЕЗ ротации. - if transient_left > 0: - transient_left -= 1 - backoff = _AVITO_SIDECAR_TRANSIENT_BACKOFF_SEC + random.uniform(0, 0.5) - logger.warning( - "avito page=%d sidecar transient (status=%d) — " - "retry in %.1fs (left=%d): %s", - page, - status, - backoff, - transient_left, - error_text[:200], - ) - await asyncio.sleep(backoff) - continue - # Оба бюджета (ротация + transient) исчерпаны → graceful soft-ban, - # чтобы run_avito_full_load/city_sweep ушли в mark_banned (partial - # preserved), а НЕ в generic except → mark_failed. Намеренно - # AvitoRateLimitedError, не AvitoBlockedError и не bare exception. - logger.error( - "avito page=%d sidecar error budget exhausted (status=%d) url=%s: %s", - page, - status, - url, - error_text[:200], - ) - raise AvitoRateLimitedError( - f"avito SERP browser-sidecar error (page={page}): {error_text[:200]}" - ) from exc - # Пустой рендер ("" / None) = таймаут рендера sidecar'а, НЕ firewall: - # _is_firewall_page("") вернул бы False → пустой HTML утёк бы как - # «валидный». Возвращаем None — caller (fetch_around) трактует это как - # конец пагинации (break), а не как бан. - if not html: - return None - if not _is_firewall_page(html): - return html - if rot_done < max_rot and await self._rotate_ip(): - rot_done += 1 - logger.info( - "avito page=%d browser firewall — retry after rotation #%d", - page, - rot_done, - ) - continue - # Ротация исчерпана/недоступна. Fallback: один retry через curl_cffi. - if self._cffi is not None: - logger.warning( - "avito page=%d browser firewall — falling back to curl_cffi (#1805)", - page, - ) - try: - fb_html = await self._fetch_serp_html_cffi(url, page) - except (AvitoBlockedError, AvitoRateLimitedError): - # curl-fallback тоже забанен → пробрасываем как hard-block. - logger.error("avito page=%d curl_cffi fallback also blocked url=%s", page, url) - raise - if fb_html is None or not _is_firewall_page(fb_html): - # None = non-200/конец пагинации; иначе валидный HTML. - return fb_html - logger.error("avito page=%d curl_cffi fallback still firewall url=%s", page, url) - logger.error("avito SERP firewall in browser-mode page=%d url=%s", page, url) - raise AvitoBlockedError( - f"Avito SERP firewall (browser-mode) at page={page} — IP banned" - ) - - async def _fetch_serp_html_cffi(self, url: str, page: int) -> str | None: - """GET SERP HTML через curl_cffi (legacy/fallback путь) с anti-block-логикой. - - Avito на забаненном datacenter-IP отдаёт HTTP 200 + firewall-страницу (не - 403). 429 через backconnect-прокси — transient conn-limit (short-retry без - ротации). 403/firewall на backconnect — залочен текущий exit-IP, escape - через reconnect (_reset_cffi) либо changeip-ротацию. - - Returns: - HTML при HTTP 200, либо ``None`` при non-200 (конец пагинации). - Raises: - AvitoBlockedError / AvitoRateLimitedError — если бан не снят. - """ - assert self._cffi is not None - max_rot = settings.avito_proxy_max_rotations if settings.avito_proxy_rotate_url else 0 - # curl_cffi-сессия всегда egress через settings.scraper_proxy_url (backconnect - # mproxy). Пересоздание сессии (_reset_cffi) даёт свежий exit-IP вне зависимости - # от наличия changeip-URL (тот относится к ДРУГОМУ, browser/auv-прокси). Поэтому - # reconnect-retry гейтим ТОЛЬКО на наличие proxy, не на отсутствие rotate_url. - backconnect = bool(settings.scraper_proxy_url) - rot_done = 0 - transient_left = 2 - r429 = 0 - r429_reconnect = 0 - r403 = 0 - while True: - try: - response = await self._cffi.get(url) - except Exception: - if transient_left > 0: - transient_left -= 1 - logger.warning( - "avito page=%d transient fetch error — retry (left=%d)", - page, - transient_left, - exc_info=True, - ) - await asyncio.sleep(3) - continue - raise - - sc = response.status_code - is_firewall = sc == 200 and _is_firewall_page(response.text) - # 429 в curl_cffi через backconnect-прокси — transient conn-limit, НЕ - # IP-ban: короткий retry того же url без ротации. Только если короткие - # retry исчерпаны — проваливаемся в общую блок-логику ниже (rotation/raise). - if sc == 429 and r429 < _AVITO_429_MAX_RETRIES: - r429 += 1 - logger.info( - "avito page=%d HTTP 429 (backconnect conn-limit) — short retry %d/%d", - page, - r429, - _AVITO_429_MAX_RETRIES, - ) - jitter = (r429 % 3) * 0.1 - await asyncio.sleep(_AVITO_429_BACKOFF_SEC + jitter) - continue - if sc in (403, 429) or is_firewall: - if rot_done < max_rot and await self._rotate_ip(): - rot_done += 1 - logger.info( - "avito page=%d blocked (HTTP %d, firewall=%s) — retry after rotation #%d", - page, - sc, - is_firewall, - rot_done, - ) - continue - # backconnect: 403/firewall = только текущий exit-IP залочен. Новый - # CONNECT-туннель (пересоздание сессии) = свежий IP → reconnect-retry. - # Только для 403/firewall (НЕ 429 — у него свой short-retry выше). - if backconnect and (sc == 403 or is_firewall) and r403 < _AVITO_403_MAX_RETRIES: - r403 += 1 - logger.info( - "avito page=%d blocked (HTTP %d, firewall=%s) — " - "backconnect reconnect retry %d/%d", - page, - sc, - is_firewall, - r403, - _AVITO_403_MAX_RETRIES, - ) - await self._reset_cffi() - await asyncio.sleep(_AVITO_403_BACKOFF_SEC) - continue - if sc == 429: - # short-retry (тот же session) исчерпан. Под backconnect одна - # фаза reconnect-fallback: пересоздание сессии освобождает - # удерживаемый conn-slot и может лечь на менее загруженный путь - # (deep-pagination 429 = conn-limit, не IP-ban). Только потом raise. - if backconnect and r429_reconnect < _AVITO_429_RECONNECT_RETRIES: - r429_reconnect += 1 - logger.info( - "avito page=%d HTTP 429 short-retry exhausted — " - "backconnect reconnect retry %d/%d", - page, - r429_reconnect, - _AVITO_429_RECONNECT_RETRIES, - ) - await self._reset_cffi() - await asyncio.sleep(_AVITO_403_BACKOFF_SEC) - continue - logger.error("avito SERP HTTP 429 (rate limited) page=%d url=%s", page, url) - raise AvitoRateLimitedError( - f"Avito SERP returned 429 — rate limited at page={page}" - ) - logger.error( - "avito SERP blocked page=%d (HTTP %d, firewall=%s) url=%s", - page, - sc, - is_firewall, - url, - ) - raise AvitoBlockedError( - f"Avito SERP blocked (HTTP {sc}, firewall={is_firewall}) " - f"at page={page} — IP banned" - ) - if sc != 200: - logger.warning("avito HTML page=%d returned %d for %s", page, sc, url) - return None - return response.text - - async def _fetch_serp_html_with_retry( - self, url: str, page: int, *, max_retries: int = 2 - ) -> str | None: - """Page-level retry поверх _fetch_serp_html для sweep-обходов (P2 resilience). - - _fetch_serp_html уже ретраит транзиентные fetch-обрывы и ротирует IP, но - если после внутренних попыток он всё равно поднимает generic Exception — - внешний sweep-цикл (citywide/byrooms) делал break и терял весь хвост. - Здесь даём странице ещё ``max_retries`` попыток перед тем как пробросить. - - Поведение: - - AvitoBlockedError / AvitoRateLimitedError — hard-block, пробрасываются - немедленно (НЕ retry). - - generic Exception — если попытки остались: warning + короткий backoff + - следующая попытка; иначе re-raise последнего исключения (внешний цикл - решит break). - - None — это «non-200 / конец пагинации», НЕ ошибка → возвращается как есть - без ретраев. - """ - attempts = max_retries + 1 - backoff = max(2.0, min(self.request_delay_sec, 4.0)) - for attempt in range(1, attempts + 1): - try: - return await self._fetch_serp_html(url, page) - except (AvitoBlockedError, AvitoRateLimitedError): - raise - except Exception: - if attempt >= attempts: - raise - logger.warning( - "avito page=%d transient error — retry (attempt %d/%d)", - page, - attempt, - max_retries, - exc_info=True, - ) - await asyncio.sleep(backoff) - return None # pragma: no cover — loop either returns or raises - - # ── Public ────────────────────────────────────────────────────────────── - async def fetch_around( - self, - lat: float, - lon: float, - radius_m: int = 1000, - *, - pages: int = 1, - delay_override_sec: float | None = None, - ) -> list[ScrapedLot]: - """Найти объявления Авито вокруг (lat, lon) в radius_m метрах. - - Avito работает в км — конвертируем. Минимум 1 км. - - pages=1 (default) — backward compat, одна страница (~50 lots). - pages>1 — итерируем p=1..pages, собираем до 50*pages lots. - Между запросами sleep request_delay_sec (или delay_override_sec если передан). - - Coords НЕ заполняем из anchor (избегаем silent corruption через jitter). - Точные lat/lon приходят позже из avito_detail.py — для search-карточек - lat=lon=None, далее geocode-missing cron подтянет из address. - """ - radius_km = max(1, round(radius_m / 1000)) - if delay_override_sec is not None: - self.request_delay_sec = delay_override_sec - - all_lots: list[ScrapedLot] = [] - for page in range(1, pages + 1): - url = self._build_web_url(lat, lon, radius_km, page=page) - try: - html = await self._fetch_serp_html(url, page) - except (AvitoBlockedError, AvitoRateLimitedError): - raise - except Exception: - logger.exception("avito HTML page=%d fetch failed for %s", page, url) - break - if html is None: - break - - lots = self._parse_html(html, source_url_base=url) - if not lots: - if page == 1: - logger.error( - "avito SERP page=1 returned HTTP 200 but 0 cards — " - "likely content-block/captcha url=%s", - url, - ) - raise AvitoContentBlockedError( - "Avito SERP HTTP 200 with 0 cards on page=1 — content-block suspected" - ) - logger.info("avito page=%d: 0 lots — end of pagination", page) - break - all_lots.extend(lots) - logger.info( - "avito page=%d: %d lots (total %d) around (%.4f, %.4f)", - page, - len(lots), - len(all_lots), - lat, - lon, - ) - if page < pages: - await self.sleep_between_requests() - - if pages == 1 and all_lots: - # Backward compat: single-page path does trailing sleep (как раньше) - await self.sleep_between_requests() - elif pages > 1: - logger.info( - "avito fetch_around pages=%d total_lots=%d around (%.4f, %.4f)", - pages, - len(all_lots), - lat, - lon, - ) - - return all_lots - - # ── Strategy B: HTML scrape ───────────────────────────────────────────── - def _build_web_url(self, lat: float, lon: float, radius_km: int, page: int = 1) -> str: - params = { - "geoCoords": f"{lat},{lon}", - "radius": radius_km, - "s": 104, # sort by date - "p": page, - } - return f"{self.base_url}/ekaterinburg/kvartiry/prodam-ASgBAgICAUSSA8YQ?{urlencode(params)}" - - def _build_citywide_url(self, page: int = 1) -> str: - """T6: URL всего города ЕКБ без geo-фильтра, сортировка по дате. - - Avito отдаёт все объявления города (cap ~5000 = 100 страниц × 50 карточек). - """ - params = {"s": 104, "p": page} - return f"{self.base_url}/ekaterinburg/kvartiry/prodam-ASgBAgICAUSSA8YQ?{urlencode(params)}" - - def _build_newbuilding_url(self, page: int = 1) -> str: - """URL ЕКБ-выборки только новостроек (novostroyka-filter), сортировка по дате. - - Тот же shape параметров, что _build_citywide_url (s=104, p=page), но slug - категории NOVOSTROYKA_SLUG отдаёт исключительно new-build карточки. - """ - params = {"s": 104, "p": page} - path = f"/ekaterinburg/kvartiry/prodam/{NOVOSTROYKA_SLUG}" - return f"{self.base_url}{path}?{urlencode(params)}" - - def _build_rooms_url( - self, - room_slug: str, - page: int = 1, - min_price: int | None = None, - max_price: int | None = None, - ) -> str: - """T6: URL с фильтром по комнатности для всего ЕКБ (no geo). - - Дробление по комнатности позволяет обойти cap ~5000 на category. - room_slug — берётся из ROOM_SLUGS (кодирует ASgB-параметры выборки). - - min_price/max_price — опциональные ценовые границы (Avito `pmin`/`pmax`), - добавляются в query ТОЛЬКО если заданы (backward-compat: вызовы без цены - не меняют URL). Используются exhaustive full-load для room×price бисекции. - """ - params: dict[str, Any] = {"s": 104, "p": page} - if min_price is not None: - params["pmin"] = min_price - if max_price is not None: - params["pmax"] = max_price - return f"{self.base_url}/ekaterinburg/kvartiry/prodam/{room_slug}?{urlencode(params)}" - - def _extract_total_count(self, html: str) -> int | None: - """Извлечь общее число результатов из Avito SERP (`page-title/count`). - - Avito рендерит счётчик в 1 178. - Это аналог cian totalOffers — gate для бисекции (total > cap → split). - Парсинг: текст span → убрать NBSP (\xa0)/пробелы/` ` → int. - - Порядок: сначала `page-title/count` (его несёт ТОЛЬКО непустая выборка), - потом no-results-маркер → 0 (валидная пустая выборка: брекет скипается без - split/rotation и без скрапа ~50 fallback-«похожих»), и только при отсутствии - обоих — None (реально битая/captcha/firewall/незагруженная страница). - - Returns: - int при успехе (включая 0 на no-results-странице), либо None при - captcha/firewall/незагруженной странице (ни count, ни no-results). - """ - m = _TOTAL_COUNT_RE.search(html) - if m is not None: - raw = m.group("count") - # Убрать NBSP (\xa0),  -entity и обычные пробелы-разделители разрядов. - cleaned = raw.replace("\xa0", "").replace(" ", "").replace(" ", "") - if cleaned: - try: - return int(cleaned) - except (TypeError, ValueError): - pass - # Счётчик не найден/не распарсился. Проверяем no-results-маркер: его наличие - # = валидная пустая выборка (total=0), несмотря на ~50 fallback-объявлений. - lowered = html.lower() - if any(marker in lowered for marker in _AVITO_NO_RESULTS_MARKERS): - return 0 - return None - - async def _fetch_rooms_page_html( - self, - room_slug: str, - page: int, - min_price: int | None, - max_price: int | None, - ) -> str | None: - """GET одной room×price SERP-страницы через anti-block pipeline. - - Делегирует _fetch_serp_html (firewall-detect, IP-rotation, транзиентные - ретраи). AvitoBlockedError/AvitoRateLimitedError пробрасываются наверх. - """ - url = self._build_rooms_url(room_slug, page, min_price, max_price) - return await self._fetch_serp_html(url, page) - - async def _probe_total( - self, room_slug: str, min_price: int | None, max_price: int | None - ) -> tuple[str | None, int | None]: - """Probe page=1 бакета и извлекает total, устойчиво к browser-ошибкам. - - tradein-browser на тяжёлой выдаче (огромный диапазон, напр. студии - pmax=200M) может таймаутить на goto → HTTP 500 → ``httpx.HTTPStatusError``, - либо отдавать ``httpx.TimeoutException``/``ReadTimeout``. Раньше эти ошибки - пробрасывались из probe до самого верха и роняли весь full-load с 0 uniq. - Теперь они трактуются как «total недоступен» (``None``) — вызывающий - ``_walk_price_range`` решит: split (бакет делим) или skip (узкий бакет). - - AvitoBlockedError/AvitoRateLimitedError НЕ глушатся — это hard-block, - должен всплыть в pipeline (mark_banned). Возвращает (html, total); при - ошибке fetch'а html=None. - """ - try: - html = await self._fetch_rooms_page_html(room_slug, 1, min_price, max_price) - except (AvitoBlockedError, AvitoRateLimitedError): - # Hard-block — пробрасываем наверх (pipeline пометит run banned). - raise - except (httpx.HTTPStatusError, httpx.TimeoutException, httpx.HTTPError) as exc: - logger.warning( - "avito: probe browser error %s for %s [%s, %s] — treating as total=None", - type(exc).__name__, - room_slug, - min_price, - max_price, - ) - return None, None - except Exception: - logger.warning( - "avito: probe unexpected error for %s [%s, %s] — treating as total=None", - room_slug, - min_price, - max_price, - exc_info=True, - ) - return None, None - - total = self._extract_total_count(html) if html is not None else None - return html, total - - async def fetch_all_secondary( - self, - *, - rooms_buckets: list[tuple[str, str]] | None = None, - price_cap_per_bucket: int = 1400, - max_pages_per_bucket: int = 100, - concurrency: int = 5, - secondary_only: bool = True, - on_bucket: Callable[..., Any] | None = None, - on_progress: Callable[[int], None] | None = None, - skip_buckets: set[str] | None = None, - since: date | None = None, - ) -> list[ScrapedLot]: - """Exhaustive- ИЛИ инкрементальная загрузка Avito ЕКБ вторички (КОМНАТНОСТЬ × ЦЕНА). - - Обходит Avito SERP-cap (~5000 результатов на запрос). Вместо корень-бисекции - [0, _AVITO_MAX_PRICE] на каждую комнатность (≈5 холостых probe-спусков через - пустой верх, т.к. вторичка ЕКБ почти вся <16М) — стартуем с data-driven - seed-сетки ценовых брекетов (_AVITO_PRICE_SEED_BRACKETS) по прод-распределению. - Каждый (комнатность × seed-брекет) дальше дробится _walk_price_range если - total > price_cap_per_bucket. Последний seed-брекет ОТКРЫТ (hi=None → pmax - не ставится) — ловит весь хвост люкса без потолка. - - Страницы leaf-бакета — параллельно (Semaphore). Дедуп по source_id (dict seen), - ОБЩИЙ по всем брекетам и комнатностям. - - Параметры: - rooms_buckets: список (name, slug) комнатностей (default: все ROOM_SLUGS, - вкл. студии и своб.планировку). - price_cap_per_bucket: максимум результатов в бакете перед делением (< 5000). - max_pages_per_bucket: верхний предел страниц на leaf-бакет (default=100). - concurrency: максимум параллельных page-фетчей в leaf-бакете (default=5). - secondary_only: если True (default) — отбрасывает новостройки - (listing_segment=="novostroyki") после парсинга, до save/on_bucket. - Avito тегирует сегмент по DOM-маркеру item-development-name в _parse_html. - on_bucket: опциональный callback(bucket_key, list[ScrapedLot]) после каждого - leaf-бакета. Может быть async или sync. Исключение прерывает прогон. - on_progress: опциональный callback(unique_count) для heartbeat (per room-bucket). - skip_buckets: множество ключей «room_label:lo:hi» уже завершённых бакетов — - пагинация и on_bucket для них пропускаются. Probe-запросы выполняются. - since: если None (default) — EXHAUSTIVE bisection-обход (поведение без - изменений). Если задана date — INCREMENTAL: на каждый (комнатность × - seed-брекет) последовательная пагинация newest-first с ранней остановкой, - как только страница уходит ниже watermark `since`. Бисекция по цене НЕ - выполняется (читаем только верхние свежие страницы, до SERP-cap далеко). - - Возвращает list[ScrapedLot] уникальных лотов (дедуп по source_id/source_url). - """ - _buckets = rooms_buckets if rooms_buckets is not None else _AVITO_DEFAULT_ROOMS - seen: dict[str, ScrapedLot] = {} - # Tolerance к одиночным per-bucket блокам (deep-pagination 429 dense-бакета): - # скипаем заблокированный бакет и продолжаем; N ПОДРЯД блоков = hard ban → re-raise. - consecutive_blocked = 0 - skipped_blocked = 0 - - for name, slug in _buckets: - room_label = _avito_room_label(name) - logger.info( - "avito exhaustive: starting %s (price_cap=%d concurrency=%d secondary_only=%s)", - room_label, - price_cap_per_bucket, - concurrency, - secondary_only, - ) - before = len(seen) - # Перебираем seed-брекеты: для ЗАКРЫТЫХ передаём hi = br_hi - 1 - # (pmax эксклюзивный сверху → соседние брекеты не пересекаются), для - # ОТКРЫТОГО (br_hi is None) — hi=None (без потолка). Дедуп по source_id - # в общем `seen` страхует на стыках. - for br_lo, br_hi in _AVITO_PRICE_SEED_BRACKETS: - walk_hi = br_hi - 1 if br_hi is not None else None - try: - if since is not None: - # INCREMENTAL: последовательная newest-first пагинация с ранней - # остановкой по watermark `since`; без бисекции по цене. - await self._paginate_incremental_bracket( - room_slug=slug, - room_label=room_label, - lo=br_lo, - hi=walk_hi, - since=since, - seen=seen, - max_pages_per_bucket=max_pages_per_bucket, - secondary_only=secondary_only, - on_bucket=on_bucket, - ) - else: - await self._walk_price_range( - room_slug=slug, - room_label=room_label, - lo=br_lo, - hi=walk_hi, - seen=seen, - price_cap_per_bucket=price_cap_per_bucket, - max_pages_per_bucket=max_pages_per_bucket, - concurrency=concurrency, - secondary_only=secondary_only, - on_bucket=on_bucket, - skip_buckets=skip_buckets, - ) - except (AvitoRateLimitedError, AvitoBlockedError) as exc: - # Одиночный блок на бакете (deep-pagination 429 dense-бакета) НЕ - # роняет sweep: скипаем бакет, продолжаем. N ПОДРЯД блоков = реальный - # hard IP-ban → re-raise (run_avito_full_load → mark_banned). - consecutive_blocked += 1 - skipped_blocked += 1 - logger.warning( - "avito exhaustive: %s bracket [%s..%s] skipped — blocked (%s); " - "consecutive_blocked=%d/%d", - room_label, - br_lo, - walk_hi, - type(exc).__name__, - consecutive_blocked, - _AVITO_SWEEP_MAX_CONSECUTIVE_BLOCKED, - ) - if consecutive_blocked >= _AVITO_SWEEP_MAX_CONSECUTIVE_BLOCKED: - logger.error( - "avito exhaustive: %d consecutive blocked buckets — " - "treating as hard IP-ban, aborting sweep", - consecutive_blocked, - ) - raise - continue - # Успешный бакет — сбрасываем счётчик подряд-блоков. - consecutive_blocked = 0 - room_collected = len(seen) - before - logger.info( - "avito exhaustive: %s done — collected %d (total unique=%d)", - room_label, - room_collected, - len(seen), - ) - if on_progress is not None: - on_progress(len(seen)) - - logger.info( - "avito exhaustive: DONE — total unique=%d lots skipped_blocked=%d", - len(seen), - skipped_blocked, - ) - return list(seen.values()) - - async def _walk_price_range( - self, - *, - room_slug: str, - room_label: str, - lo: int, - hi: int | None, - seen: dict[str, ScrapedLot], - price_cap_per_bucket: int, - max_pages_per_bucket: int, - concurrency: int = 5, - secondary_only: bool = True, - on_bucket: Callable[..., Any] | None = None, - skip_buckets: set[str] | None = None, - _depth: int = 0, - ) -> None: - """Рекурсивное адаптивное бинарное партиционирование ценового диапазона [lo, hi]. - - Алгоритм для ЗАКРЫТОГО брекета (hi is not None): - 1. Запросить page=1 с pmin=lo, pmax=hi → total (page-title/count). - 2. Если total <= cap → пагинировать leaf-бакет параллельно (concurrency). - 3. Если total > cap → разбить бакет пополам (рекурсия). - Guard: hi - lo < _AVITO_MIN_BRACKET → пагинировать как есть (WARNING). - - ОТКРЫТЫЙ брекет (hi is None) — верхний seed-брекет без потолка (pmax не - ставится, avito отдаёт всё ≥lo). Делить нельзя (mid=(lo+hi)//2 невозможен) — - probe + пагинируем leaf напрямую. Хвост 250М+ крошечный (≈73), за cap - обычно не выходит; если вышел — пагинируем до max_pages + WARNING (acceptable). - - После пагинации leaf-бакета вызывает on_bucket(bucket_key, bucket_lots). - bucket_key = "room_label:lo:hi" (закрытый) либо "room_label:lo:open" (открытый). - on_bucket может быть async или sync. Исключение в on_bucket прерывает прогон. - skip_buckets: если bucket_key в skip_buckets — пагинация и on_bucket пропускаются. - secondary_only: новостройки (listing_segment=="novostroyki") отбрасываются после - сбора bucket_lots, до дедупа в seen и вызова on_bucket. - AvitoBlockedError/AvitoRateLimitedError из _fetch_serp_html пробрасываются наверх. - """ - _lo_param = lo if lo > 0 else None - _hi_repr = "open" if hi is None else str(hi) - - # ── Шаг 1: probe page 1 ──────────────────────────────────────────────── - # _probe_total ловит browser-ошибки (HTTP 500 / timeout от tradein-browser - # на тяжёлой выдаче, напр. студии pmax=200M) → total=None вместо краха. - # AvitoBlockedError/AvitoRateLimitedError всплывают наверх (mark_banned). - html, total = await self._probe_total(room_slug, _lo_param, hi) - - # Ретрай на captcha/missing-count/browser-ошибке: rotate IP + 1 retry. - if total is None: - logger.warning( - "avito: total=None for %s [%d, %s] depth=%d — rotating IP + retry", - room_label, - lo, - _hi_repr, - _depth, - ) - rotated = await self._rotate_ip() - if rotated: - html, total = await self._probe_total(room_slug, _lo_param, hi) - - if total is None: - # ОТКРЫТЫЙ брекет: делить нельзя (нет hi). Probe failed → нет total для - # ceil(); пагинируем до max_pages_per_bucket (best-effort, хвост 250М+ - # крошечный) и WARNING. НЕ роняем прогон. - if hi is None: - logger.warning( - "avito: probe failed for OPEN bucket %s [%d, open] depth=%d — " - "paginating up to max_pages (tail-loss accepted)", - room_label, - lo, - _depth, - ) - await self._paginate_leaf_bucket( - room_slug=room_slug, - room_label=room_label, - lo=lo, - hi=None, - html=None, - max_pages=max_pages_per_bucket, - seen=seen, - price_cap_per_bucket=price_cap_per_bucket, - max_pages_per_bucket=max_pages_per_bucket, - concurrency=concurrency, - secondary_only=secondary_only, - on_bucket=on_bucket, - skip_buckets=skip_buckets, - ) - return - bracket_size = hi - lo - # Probe не удался даже после ретрая. Если бакет ещё делим — НЕ теряем - # его целиком: тяжёлая страница (огромная выдача) скорее всего станет - # загружаемой в более узком ценовом диапазоне → принудительный split. - if bracket_size >= _AVITO_MIN_BRACKET: - logger.warning( - "avito: probe failed for %s [%d, %d] depth=%d — splitting (assume heavy)", - room_label, - lo, - hi, - _depth, - ) - mid = (lo + hi) // 2 - await self._walk_price_range( - room_slug=room_slug, - room_label=room_label, - lo=lo, - hi=mid, - seen=seen, - price_cap_per_bucket=price_cap_per_bucket, - max_pages_per_bucket=max_pages_per_bucket, - concurrency=concurrency, - secondary_only=secondary_only, - on_bucket=on_bucket, - skip_buckets=skip_buckets, - _depth=_depth + 1, - ) - await self._walk_price_range( - room_slug=room_slug, - room_label=room_label, - lo=mid + 1, - hi=hi, - seen=seen, - price_cap_per_bucket=price_cap_per_bucket, - max_pages_per_bucket=max_pages_per_bucket, - concurrency=concurrency, - secondary_only=secondary_only, - on_bucket=on_bucket, - skip_buckets=skip_buckets, - _depth=_depth + 1, - ) - return - # Бакет уже узкий (< MIN_BRACKET) — split не поможет, хвост теряем - # осознанно (как cian) и логируем. - logger.error( - "avito: skipping bucket %s [%d, %d] — total unavailable after retry " - "(bracket=%d < MIN_BRACKET=%d, tail loss accepted)", - room_label, - lo, - hi, - bracket_size, - _AVITO_MIN_BRACKET, - ) - return - - logger.info( - "avito: %s [%d, %s] total=%d depth=%d", - room_label, - lo, - _hi_repr, - total, - _depth, - ) - - if total == 0: - return - - # ── ОТКРЫТЫЙ брекет (hi is None): делить нельзя — пагинируем leaf напрямую ─ - # total известен из probe → max_pages = min(ceil(total/per_page), cap). - # Если total > cap (не должно для 250М+, но guard) — пагинируем как есть до - # max_pages_per_bucket с WARNING (хвост люкса крошечный, tail-loss accepted). - if hi is None: - if total > price_cap_per_bucket: - logger.warning( - "avito: OPEN bucket %s [%d, open] total=%d > cap=%d — paginating " - "without split (tail-loss accepted, lux tail tiny)", - room_label, - lo, - total, - price_cap_per_bucket, - ) - pages_needed = math.ceil(total / _AVITO_OFFERS_PER_PAGE) - max_pages = min(pages_needed, max_pages_per_bucket) - await self._paginate_leaf_bucket( - room_slug=room_slug, - room_label=room_label, - lo=lo, - hi=None, - html=html, - max_pages=max_pages, - seen=seen, - price_cap_per_bucket=price_cap_per_bucket, - max_pages_per_bucket=max_pages_per_bucket, - concurrency=concurrency, - secondary_only=secondary_only, - on_bucket=on_bucket, - skip_buckets=skip_buckets, - ) - return - - # ── Шаг 2: деление или пагинация ────────────────────────────────────── - bracket_size = hi - lo - need_split = total > price_cap_per_bucket - too_narrow = bracket_size < _AVITO_MIN_BRACKET - - if need_split and too_narrow: - logger.warning( - "avito: %s [%d, %d] total=%d > cap=%d but bracket=%d < MIN_BRACKET=%d " - "— paginating as-is (tail loss ~%d)", - room_label, - lo, - hi, - total, - price_cap_per_bucket, - bracket_size, - _AVITO_MIN_BRACKET, - max(0, total - price_cap_per_bucket), - ) - need_split = False # принудительно пагинируем - - if need_split: - mid = (lo + hi) // 2 - # [lo, mid] - await self._walk_price_range( - room_slug=room_slug, - room_label=room_label, - lo=lo, - hi=mid, - seen=seen, - price_cap_per_bucket=price_cap_per_bucket, - max_pages_per_bucket=max_pages_per_bucket, - concurrency=concurrency, - secondary_only=secondary_only, - on_bucket=on_bucket, - skip_buckets=skip_buckets, - _depth=_depth + 1, - ) - # [mid+1, hi] - await self._walk_price_range( - room_slug=room_slug, - room_label=room_label, - lo=mid + 1, - hi=hi, - seen=seen, - price_cap_per_bucket=price_cap_per_bucket, - max_pages_per_bucket=max_pages_per_bucket, - concurrency=concurrency, - secondary_only=secondary_only, - on_bucket=on_bucket, - skip_buckets=skip_buckets, - _depth=_depth + 1, - ) - return - - # ── Закрытый leaf-бакет: считаем max_pages + warning, делегируем helper'у ─ - pages_needed = math.ceil(total / _AVITO_OFFERS_PER_PAGE) - max_pages = min(pages_needed, max_pages_per_bucket) - - if pages_needed > max_pages_per_bucket: - tail_loss = total - max_pages_per_bucket * _AVITO_OFFERS_PER_PAGE - logger.warning( - "avito: %s [%d, %d] total=%d exceeds page cap " - "(max_pages=%d × %d=%d offers) — tail loss ~%d offers", - room_label, - lo, - hi, - total, - max_pages_per_bucket, - _AVITO_OFFERS_PER_PAGE, - max_pages_per_bucket * _AVITO_OFFERS_PER_PAGE, - tail_loss, - ) - - await self._paginate_leaf_bucket( - room_slug=room_slug, - room_label=room_label, - lo=lo, - hi=hi, - html=html, - max_pages=max_pages, - seen=seen, - price_cap_per_bucket=price_cap_per_bucket, - max_pages_per_bucket=max_pages_per_bucket, - concurrency=concurrency, - secondary_only=secondary_only, - on_bucket=on_bucket, - skip_buckets=skip_buckets, - ) - - async def _paginate_leaf_bucket( - self, - *, - room_slug: str, - room_label: str, - lo: int, - hi: int | None, - html: str | None, - max_pages: int, - seen: dict[str, ScrapedLot], - price_cap_per_bucket: int, - max_pages_per_bucket: int, - concurrency: int, - secondary_only: bool, - on_bucket: Callable[..., Any] | None, - skip_buckets: set[str] | None, - ) -> None: - """Параллельная пагинация одного leaf-бакета + фильтр + дедуп + on_bucket. - - Общая для ЗАКРЫТЫХ ([lo, hi]) и ОТКРЫТОГО (hi=None → pmax не ставится) - брекетов. ``html`` — HTML страницы 1 из probe (переиспользуем, не - перезапрашиваем); для открытого-без-probe передаётся None → стр.1 фетчится - наравне с остальными. ``max_pages`` уже посчитан вызывающим. - - bucket_key = "room_label:lo:hi" (закрытый) либо "room_label:lo:open" (открытый). - skip_buckets: если bucket_key в skip_buckets — пагинация и on_bucket пропускаются. - AvitoBlockedError/AvitoRateLimitedError из page-фетчей пробрасываются наверх. - """ - _lo_param = lo if lo > 0 else None - _hi_param = hi # None → _build_rooms_url не ставит pmax - _hi_repr = "open" if hi is None else str(hi) - bucket_key = f"{room_label}:{lo}:{_hi_repr}" - - # Если бакет уже завершён в предыдущем запуске — пропускаем пагинацию и on_bucket. - if skip_buckets and bucket_key in skip_buckets: - logger.info("avito: skip bucket %s — already done (resume)", bucket_key) - return - - # Страница 1 — из probe-html если он есть (закрытый/открытый с probe); - # иначе фетчим её как обычную страницу (открытый брекет без probe-html). - sem = asyncio.Semaphore(concurrency) - first_url = self._build_rooms_url(room_slug, 1, _lo_param, _hi_param) - - async def _one_page(p: int) -> list[ScrapedLot]: - if p == 1 and html is not None: - return self._parse_html(html, source_url_base=first_url) - async with sem: - page_html = await self._fetch_rooms_page_html(room_slug, p, _lo_param, _hi_param) - await asyncio.sleep(self.request_delay_sec) - if page_html is None: - logger.warning( - "avito: page_html=None %s [%d, %s] page=%d — skipping page", - room_label, - lo, - _hi_repr, - p, - ) - return [] - page_url = self._build_rooms_url(room_slug, p, _lo_param, _hi_param) - return self._parse_html(page_html, source_url_base=page_url) - - page_results = await asyncio.gather( - *[_one_page(p) for p in range(1, max_pages + 1)], - return_exceptions=True, - ) - - bucket_lots: list[ScrapedLot] = [] - for p_idx, res in enumerate(page_results, start=1): - if isinstance(res, BaseException): - # Блокировки пробрасываем наверх (mark_banned в pipeline-обёртке). - if isinstance(res, AvitoBlockedError | AvitoRateLimitedError): - raise res - logger.warning( - "avito: page exception %s [%d, %s] page=%d — %r", - room_label, - lo, - _hi_repr, - p_idx, - res, - ) - else: - bucket_lots.extend(res) - - collected_this_bucket = len(bucket_lots) - - # ── Фильтр новостроек (secondary_only) ──────────────────────────────── - dropped_nb = 0 - if secondary_only: - filtered = [lot for lot in bucket_lots if lot.listing_segment != "novostroyki"] - dropped_nb = collected_this_bucket - len(filtered) - bucket_lots = filtered - - # Дедуп в общий seen - for lot in bucket_lots: - key = lot.source_id or lot.source_url - if key: - seen[key] = lot - - logger.info( - "avito: %s [%d, %s] paginated=%d pages collected=%d dropped_nb=%d unique_total=%d", - room_label, - lo, - _hi_repr, - max_pages, - collected_this_bucket, - dropped_nb, - len(seen), - ) - - # ── on_bucket callback: инкрементальный save ────────────────────────── - if on_bucket is not None and bucket_lots: - res_cb = on_bucket(bucket_key, bucket_lots) - if inspect.isawaitable(res_cb): - await res_cb - - async def _paginate_incremental_bracket( - self, - *, - room_slug: str, - room_label: str, - lo: int, - hi: int | None, - since: date, - seen: dict[str, ScrapedLot], - max_pages_per_bucket: int, - secondary_only: bool, - on_bucket: Callable[..., Any] | None, - ) -> None: - """INCREMENTAL пагинация одного (комнатность × seed-брекет) с ранней остановкой. - - SERP отсортирован newest-first (s=104). Идём ПОСЛЕДОВАТЕЛЬНО со страницы 1; - как только страница не содержит ни одной карточки с listing_date >= since — - весь остаток бакета старше watermark → останавливаемся. Бисекция по цене НЕ - выполняется: читаем только верхние свежие страницы, до SERP-cap (~5000) далеко. - - Early-stop semantics: - - На каждой странице берём max(listing_date) среди датированных карточек. - Если ни одна карточка не имеет listing_date >= since → страница «старая». - - Карточки с listing_date is None НЕ триггерят стоп сами по себе (трактуем - как «продолжить», чтобы не потерять свежие-но-недатированные карточки). - - Но страница, целиком состоящая из None-или-старых карточек, считается - «не-свежей»; 2 ПОДРЯД не-свежие страницы → стоп (grace против runaway, - когда весь хвост недатирован). - - page_html is None или parse == [] → конец пагинации, стоп. - - bucket_key, secondary_only-фильтр и дедуп в seen — идентичны _paginate_leaf_bucket. - on_bucket вызывается один раз для собранного бакета (async/sync-aware). - AvitoBlockedError/AvitoRateLimitedError из page-фетчей пробрасываются наверх. - """ - _lo_param = lo if lo > 0 else None - _hi_param = hi # None → _build_rooms_url не ставит pmax - _hi_repr = "open" if hi is None else str(hi) - bucket_key = f"{room_label}:{lo}:{_hi_repr}" - - bucket_lots: list[ScrapedLot] = [] - pages_fetched = 0 - not_fresh_streak = 0 # подряд идущие не-свежие (all None-или-старые) страницы - stop_reason = "max-pages" # перетирается ниже на реальную причину - - for p in range(1, max_pages_per_bucket + 1): - # Последовательный фетч (НЕ asyncio.gather): early-stop требует читать - # страницы по порядку. Тот же helper, что и в _paginate_leaf_bucket. - page_html = await self._fetch_rooms_page_html(room_slug, p, _lo_param, _hi_param) - await asyncio.sleep(self.request_delay_sec) - pages_fetched += 1 - - if page_html is None: - stop_reason = "end-of-pages (page_html=None)" - break - - page_url = self._build_rooms_url(room_slug, p, _lo_param, _hi_param) - page_lots = self._parse_html(page_html, source_url_base=page_url) - if not page_lots: - stop_reason = "end-of-pages (empty parse)" - break - - bucket_lots.extend(page_lots) - - # Свежесть страницы: есть ли хоть одна карточка с датой >= since. - dated = [lot.listing_date for lot in page_lots if lot.listing_date is not None] - has_fresh = any(d >= since for d in dated) - if has_fresh: - not_fresh_streak = 0 - continue - - # Страница не содержит свежих датированных карточек. - if dated: - # Есть даты, но все < since → newest-first гарантирует, что дальше - # только старее → стоп немедленно. - stop_reason = "early-stop (page all older than since)" - break - # Все карточки undated (None) → не стопим сразу (могут быть свежие без - # даты), но копим streak; 2 подряд недатированные страницы → grace-стоп. - not_fresh_streak += 1 - if not_fresh_streak >= 2: - stop_reason = "early-stop (2 consecutive all-undated pages, grace)" - break - - collected_this_bucket = len(bucket_lots) - - # ── Фильтр новостроек (secondary_only) — идентично _paginate_leaf_bucket ── - dropped_nb = 0 - if secondary_only: - filtered = [lot for lot in bucket_lots if lot.listing_segment != "novostroyki"] - dropped_nb = collected_this_bucket - len(filtered) - bucket_lots = filtered - - # Дедуп в общий seen - for lot in bucket_lots: - key = lot.source_id or lot.source_url - if key: - seen[key] = lot - - logger.info( - "avito: %s [%d, %s] paginated=%d pages collected=%d dropped_nb=%d " - "unique_total=%d incremental since=%s stop=%s", - room_label, - lo, - _hi_repr, - pages_fetched, - collected_this_bucket, - dropped_nb, - len(seen), - since.isoformat(), - stop_reason, - ) - - # ── on_bucket callback: инкрементальный save ────────────────────────── - if on_bucket is not None and bucket_lots: - res_cb = on_bucket(bucket_key, bucket_lots) - if inspect.isawaitable(res_cb): - await res_cb - - # ── T6: citywide / byrooms fetch methods ────────────────────────────────── - - async def fetch_city_wide( - self, - pages: int = 100, - *, - delay_override_sec: float | None = None, - ) -> list[ScrapedLot]: - """T6: Обход всего ЕКБ без geo-фильтра (citywide-mode), paginated. - - Возвращает дедуплицированный список лотов (по source_id/source_url). - Break-on-empty: останавливается когда страница отдаёт 0 карточек. - Сохраняет весь anti-block pipeline (_fetch_serp_html: firewall-detect, - IP rotation, транзиентные ретраи). Не трогает fetch_around. - - Args: - pages: максимальное число страниц (default 100 = Avito cap ~5000 лотов). - delay_override_sec: если задан — переопределяет request_delay_sec для - этого вызова. - - Returns: - Список ScrapedLot (все страницы, дедуп по source_id). - """ - all_lots = await self._paginate_sweep( - pages, - self._build_citywide_url, - label="citywide", - delay_override_sec=delay_override_sec, - ) - logger.info("avito fetch_city_wide pages=%d total_lots=%d", pages, len(all_lots)) - return all_lots - - async def _paginate_sweep( - self, - pages: int, - url_builder: Callable[[int], str], - *, - label: str, - delay_override_sec: float | None = None, - ) -> list[ScrapedLot]: - """Общий paginated-обход ЕКБ (citywide / novostroyka) с break-on-empty. - - url_builder(page) — функция построения URL страницы (citywide или - novostroyka). Сохраняет anti-block pipeline (_fetch_serp_html: firewall, - IP rotation, ретраи), дедуп по source_id, break-on-empty. label — только - для логов. Не меняет наблюдаемое поведение fetch_city_wide. - """ - if delay_override_sec is not None: - self.request_delay_sec = delay_override_sec - - all_lots: list[ScrapedLot] = [] - seen_ids: set[str] = set() - - for page in range(1, pages + 1): - url = url_builder(page) - try: - html = await self._fetch_serp_html_with_retry(url, page) - except (AvitoBlockedError, AvitoRateLimitedError): - raise - except Exception: - logger.exception( - "avito %s page=%d fetch failed after retries url=%s", label, page, url - ) - break - if html is None: - logger.info("avito %s page=%d: non-200 — end of pagination", label, page) - break - - lots = self._parse_html(html, source_url_base=url) - if not lots: - logger.info("avito %s page=%d: 0 lots — end of pagination", label, page) - break - - new_lots = [lot for lot in lots if lot.source_id not in seen_ids] - for lot in new_lots: - if lot.source_id: - seen_ids.add(lot.source_id) - all_lots.extend(new_lots) - logger.info( - "avito %s page=%d: %d lots (%d new, %d dedup'd, total=%d)", - label, - page, - len(lots), - len(new_lots), - len(lots) - len(new_lots), - len(all_lots), - ) - if page < pages: - await self.sleep_between_requests() - return all_lots - - async def fetch_newbuildings( - self, - pages: int = 30, - *, - delay_override_sec: float | None = None, - ) -> list[ScrapedLot]: - """Обход ЕКБ-выборки только новостроек (novostroyka-filter), paginated. - - Avito отдаёт dedicated SERP с фильтром «Новостройка» — 100% new-build - карточек (с маркером застройщика → listing_segment="novostroyki", - newbuilding_id/newbuilding_url заполнены). Citywide (без geo/anchor). - Возвращает дедуплицированный список (по source_id), break-on-empty. - Сохраняет весь anti-block pipeline. Не трогает fetch_city_wide/fetch_around. - - Args: - pages: максимальное число страниц (default 30). - delay_override_sec: если задан — переопределяет request_delay_sec для - этого вызова. - - Returns: - Список ScrapedLot новостроек (все страницы, дедуп по source_id). - """ - all_lots = await self._paginate_sweep( - pages, - self._build_newbuilding_url, - label="newbuilding", - delay_override_sec=delay_override_sec, - ) - logger.info("avito fetch_newbuildings pages=%d total_lots=%d", pages, len(all_lots)) - return all_lots - - async def fetch_by_rooms( - self, - pages: int = 100, - *, - delay_override_sec: float | None = None, - room_slugs: list[tuple[str, str]] | None = None, - ) -> list[ScrapedLot]: - """T6: Обход ЕКБ по категориям комнатности (byrooms-mode). - - Дробление по 7 категориям комнатности позволяет обойти Avito cap ~5000 - результатов на поиск: каждый сегмент < 5000. Итого до 7 × 5000 = 35k лотов. - Break-on-empty на каждой категории. Глобальный dedup по source_id. - - Args: - pages: страниц на категорию (default 100). - delay_override_sec: переопределяет request_delay_sec. - room_slugs: список (name, slug) для обхода; по умолчанию ROOM_SLUGS - (все 7 категорий). Передай подмножество для частичного обхода. - - Returns: - Список ScrapedLot (все категории, глобальный дедуп по source_id). - """ - if delay_override_sec is not None: - self.request_delay_sec = delay_override_sec - - slugs = room_slugs if room_slugs is not None else ROOM_SLUGS - all_lots: list[ScrapedLot] = [] - seen_ids: set[str] = set() - - for ci, (name, slug) in enumerate(slugs, 1): - logger.info( - "avito byrooms [%d/%d] category=%s slug_prefix=%.30s", - ci, - len(slugs), - name, - slug, - ) - cat_count = 0 - for page in range(1, pages + 1): - url = self._build_rooms_url(slug, page) - try: - html = await self._fetch_serp_html_with_retry(url, page) - except (AvitoBlockedError, AvitoRateLimitedError): - raise - except Exception: - logger.exception( - "avito byrooms category=%s page=%d fetch failed after retries url=%s", - name, - page, - url, - ) - break - if html is None: - logger.info("avito byrooms category=%s page=%d: non-200 — end", name, page) - break - - lots = self._parse_html(html, source_url_base=url) - if not lots: - logger.info( - "avito byrooms category=%s page=%d: 0 lots — end of category", - name, - page, - ) - break - - new_lots = [lot for lot in lots if lot.source_id not in seen_ids] - for lot in new_lots: - if lot.source_id: - seen_ids.add(lot.source_id) - all_lots.extend(new_lots) - cat_count += len(new_lots) - logger.info( - "avito byrooms category=%s page=%d: %d lots (%d new, total=%d)", - name, - page, - len(lots), - len(new_lots), - len(all_lots), - ) - if page < pages: - await self.sleep_between_requests() - - logger.info( - "avito byrooms category=%s done: cat_new=%d grand_total=%d", - name, - cat_count, - len(all_lots), - ) - - logger.info("avito fetch_by_rooms pages_per_cat=%d total_lots=%d", pages, len(all_lots)) - return all_lots - - def _parse_html(self, html: str, source_url_base: str) -> list[ScrapedLot]: - """Парсим карточки объявлений из HTML через DOM scrape. - - Поля карточки берём из DOM (`data-marker="item-*"`). Дату публикации — из - embedded per-item JSON (`sortTimeStamp`), т.к. DOM `item-date` рендерится - лениво только у верхних карточек (#726); DOM-marker остаётся fallback. - - Если settings.avito_serp_ekb_only=True (дефолт), отбрасываем карточки, - чей per-card source_url не содержит /ekaterinburg/ — Avito добивает выдачу - «по всей России» при малом числе ЕКБ-результатов (4+ комн., дорогие сегменты). - Карточки с нераспознанным city-slug (href без ведущего /city/) пропускаются - консервативно (не дропаются по ошибке). - """ - tree = HTMLParser(html) - ts_map = _build_sort_timestamp_map(html) - cards = tree.css('[data-marker="item"]') - lots: list[ScrapedLot] = [] - ekb_only = settings.avito_serp_ekb_only - dropped = 0 - no_city = 0 - for card in cards: - lot = self._dom_card_to_lot(card, source_url_base, ts_map) - if lot is None: - continue - if ekb_only: - card_url = lot.source_url or "" - # Определяем city-slug по первому сегменту пути в абсолютном URL. - # urljoin всегда возвращает абсолютный URL (https://www.avito.ru/...). - # Avito city-slug — первый path-сегмент после netloc. - # Пример: https://www.avito.ru/ekaterinburg/... → /ekaterinburg/ присутствует. - # https://www.avito.ru/moskva/... → нет /ekaterinburg/. - # Для href без явного city (/kvartiry/... — теоретически не должно быть, - # но консервативно пропускаем). - if "/ekaterinburg/" in card_url: - # ЕКБ-карточка — берём - pass - else: - # Проверяем, есть ли вообще city-slug (хотя бы один path-сегмент - # после авито-хоста). Если нет — консервативно берём. - parsed_path = urlparse(card_url).path - # Первый непустой сегмент пути — city-slug в Avito URL. - first_seg = next((s for s in parsed_path.split("/") if s), None) - # Известные не-city сегменты Avito (/kvartiry/, /item/, ...) — - # если первый сегмент — не похоже на city-slug, пропускаем консервативно. - # Простая эвристика: city-slug содержит только строчные буквы/дефис/цифры - # и НЕ является известным non-city сегментом. - _non_city_segs = {"kvartiry", "item", "catalog", "user", "api", ""} - if first_seg is None or first_seg in _non_city_segs: - no_city += 1 - # Нет city-slug → консервативно берём карточку - lots.append(lot) - continue - # Есть city-slug, но не /ekaterinburg/ → padding по России → дропаем - dropped += 1 - continue - lots.append(lot) - if ekb_only and (dropped or no_city): - logger.info( - "avito SERP: dropped %d non-EKB cards (padding), %d cards without city-slug kept", - dropped, - no_city, - ) - return lots - - def _dom_card_to_lot( - self, card: Any, source_url_base: str, ts_map: dict[str, date] | None = None - ) -> ScrapedLot | None: - """Парсинг карточки объявления через DOM. - - Avito state давно пустой — единственный путь. Coords из карточки не отдаются, - оставляем lat=lon=None (geocode-missing cron подтянет из address). - """ - try: - link_el = card.css_first('a[data-marker="item-title"]') - if link_el is None: - return None - href = link_el.attributes.get("href", "") - url = urljoin("https://www.avito.ru", href) - title = link_el.text(strip=True) - - price_el = card.css_first('meta[itemprop="price"]') - price = int(price_el.attributes.get("content", 0)) if price_el is not None else 0 - if price <= 0: - return None - - rooms = _extract_rooms_from_title(title) - area = _extract_area_from_title(title) - floor, total = _extract_floor_from_title(title) - - # Адрес: первый

внутри data-marker="item-location" - address: str | None = None - loc_el = card.css_first('[data-marker="item-location"]') - if loc_el is not None: - street_el = loc_el.css_first("p") - if street_el is not None: - address = _clean_address(street_el.text(strip=True)) - - # Фото - photo_urls: list[str] = [] - for img in card.css('img[itemprop="image"]'): - src = img.attributes.get("src") or "" - if src.startswith("http") and src not in photo_urls: - photo_urls.append(src) - if len(photo_urls) >= 5: - break - - # Дата публикации (#726): primary — sortTimeStamp из per-item JSON по - # data-item-id (DOM item-date рендерится лениво только у верхних карточек), - # fallback — DOM marker item-date/item-date-info + relative-date parse. - listing_date: date | None = None - item_id = card.attributes.get("data-item-id") - if ts_map and item_id: - listing_date = ts_map.get(item_id) - if listing_date is None: - date_el = card.css_first('[data-marker="item-date"]') - if date_el is None: - date_el = card.css_first('[data-marker="item-date-info"]') - if date_el is not None: - listing_date = _parse_relative_date(date_el.text(strip=True)) - if listing_date is None: - # Fallback: ищем любой текст «N дней/недель/месяцев назад» в карточке - card_text = card.text(strip=True) - listing_date = _parse_relative_date(card_text) - - # House link: - # or /catalog/houses// - house_source: str | None = None - house_ext_id: str | None = None - house_url: str | None = None - house_link = card.css_first('a[href^="/catalog/houses/"]') - if house_link is not None: - h_href = house_link.attributes.get("href", "") - parts = [p for p in h_href.strip("/").split("/") if p] - # ['catalog', 'houses', , , ] - if len(parts) >= 4 and parts[-1].isdigit(): - house_source = "avito" - house_ext_id = parts[-1] - house_url = urljoin("https://www.avito.ru", h_href) - - # listing_segment по DOM-маркеру карточки: новостройки несут - # data-marker="item-development-name" (название ЖК/застройщика), - # вторичка — нет. URL-паттерн ненадёжен: каждый avito-URL квартиры - # содержит /kvartiry/ (вкл. новостройки) → ветка всегда vtorichka. - dev_name = card.css_first('[data-marker="item-development-name"]') - listing_segment = "novostroyki" if dev_name is not None else "vtorichka" - - # Newbuilding (ЖК) link: новостройки несут якорь на ЖК-сабдомен - # . - # newbuilding_url — host+path без ?context (mirror urlparse-cleaning), - # newbuilding_id — = host без 'zhk-' префикса и '.avito.ru' суффикса. - newbuilding_id: str | None = None - newbuilding_url: str | None = None - if dev_name is not None: - nb_link = card.css_first('a[href*="zhk-"][href*=".avito.ru"]') - if nb_link is not None: - nb_href = nb_link.attributes.get("href", "") or "" - parsed = urlparse(nb_href) - host = parsed.netloc - if host.startswith("zhk-") and host.endswith(".avito.ru"): - # strip query/fragment, keep scheme+host+path - newbuilding_url = urlunparse((parsed.scheme, host, parsed.path, "", "", "")) - newbuilding_id = host[len("zhk-") : -len(".avito.ru")] - - return ScrapedLot( - source="avito", - source_url=url, - source_id=card.attributes.get("data-item-id"), - address=address, # БЕЗ fallback "Екатеринбург (Avito)" — пусть None - lat=None, # C-5 fix: НЕ jitter - lon=None, # C-5 fix: НЕ jitter - rooms=rooms, - area_m2=area, - floor=floor, - total_floors=total, - price_rub=price, - photo_urls=photo_urls, - listing_date=listing_date, - raw_payload={"title": title, "address": address, "house_ext_id": house_ext_id}, - house_source=house_source, - house_ext_id=house_ext_id, - house_url=house_url, - listing_segment=listing_segment, - newbuilding_id=newbuilding_id, - newbuilding_url=newbuilding_url, - ) - except Exception: - # #823: логируем каждую сломанную карточку и считаем для observability. - # Карточки с неожиданной DOM-структурой скипаем, но не роняем весь парс. - self.parse_failures += 1 - item_id_attr = getattr(card, "attributes", {}).get("data-item-id", "?") - logger.warning( - "avito _dom_card_to_lot: failed to parse card item_id=%s (parse_failures=%d)", - item_id_attr, - self.parse_failures, - exc_info=True, - ) - return None - - -# ── Helpers: title parser ─────────────────────────────────────────────────── -_RE_ROOMS = re.compile(r"(\d)-к\.?\s*(квартира|кв\.)", re.IGNORECASE) -_RE_STUDIO = re.compile(r"студи[яиюей]", re.IGNORECASE) -_RE_AREA = re.compile(r"(\d+[.,]?\d*)\s*м[²2]", re.IGNORECASE) -_RE_FLOOR = re.compile(r"(\d+)\s*/\s*(\d+)\s*эт\.?", re.IGNORECASE) - -# ── T6: citywide / byrooms sweep ────────────────────────────────────────────── -# Avito URL slugs для room-filter (извлечены из реальных search-страниц 2026-05-23). -# Каждый slug кодирует ASgB-параметры выборки комнатности. -ROOM_SLUGS: list[tuple[str, str]] = [ - ("студии", "studii-ASgBAgICAkSSA8YQygj~WA"), - ("1-комн.", "1-komnatnye-ASgBAgICAkSSA8YQygiAWQ"), - ("2-комн.", "2-komnatnye-ASgBAgICAkSSA8YQygiCWQ"), - ("3-комн.", "3-komnatnye-ASgBAgICAkSSA8YQygiEWQ"), - ("4-комн.", "4-komnatnye-ASgBAgICAkSSA8YQygiGWQ"), - ("5+комн.", "5-komnatnye-ASgBAgICAkSSA8YQygiIWQ"), - ("своб.планир.", "svobodnaya_planirovka-ASgBAgICAkSSA8YQygj8zzI"), -] - -# Дефолтные комнатности для exhaustive full-load — все ROOM_SLUGS (вкл. студии и -# своб.планировку), т.к. каждая категория < SERP-cap после room×price бисекции. -_AVITO_DEFAULT_ROOMS: list[tuple[str, str]] = list(ROOM_SLUGS) - -_NON_SLUG_CHARS_RE = re.compile(r"[^a-z0-9]+") - - -def _avito_room_label(name: str) -> str: - """Стабильный bucket_key-префикс из человекочитаемого имени комнатности. - - «1-комн.» → «room_1_komn», «студии» → «room_studii», «5+комн.» → «room_5_komn». - Транслитерация не нужна — берём латиницу/цифры из slug-friendly формы; для - кириллицы используем порядковый индекс категории как fallback стабильности. - """ - # Цифры из имени (комнатность) + транслит ключевых маркеров для читабельности логов. - digits = "".join(ch for ch in name if ch.isdigit()) - base = name.lower() - if "студ" in base: - return "room_studii" - if "планир" in base: - return "room_svobodnaya" - if digits: - return f"room_{digits}_komn" - # Fallback: нормализуем что есть (на случай новых категорий). - norm = _NON_SLUG_CHARS_RE.sub("_", base).strip("_") - return f"room_{norm or 'unknown'}" - - -_CSS_NOISE_RE = re.compile(r"\.?css-[a-z0-9_-]+\s*\{[^}]*\}", flags=re.I) -_NOT_ADDRESS_TAIL_RE = re.compile( - r"\s*(Площадь \d|от \d+\s?мин\.|css-[a-z0-9_-]+)", - flags=re.I, -) - -# #1773: Avito DOM иногда выдаёт номер дома СЛИПШИМСЯ с маркером района/нас.пункта -# без разделителя — «…Савкова, 29р-н Академический», «17Ар-н», «35/2р-н», «33с1р-н». -# Это ломает извлечение номера дома и матчинг «того же дома» (см. issue #1773). -# Де-слип вставляет «, » между токеном номера дома и следующим маркером. -# -# Группа 1 — токен номера дома: цифры + опц. кириллическая литера (А/Б/...) + -# опц. корпус/строение (/N, сN, кN, корпус N, стр N) — напр. 29, 29А, 35/2, 33с1, 44Б. -# Группа 2 — маркер района/нас.пункта, начинающийся СРАЗУ за номером без пробела. -# -# Тонкость со слипшейся литерой: в «17Ар-н» литера «А» и маркер «р-н» — соседние -# кириллические буквы без границы. Поэтому литеру дома матчим ТОЛЬКО когда сразу -# за ней идёт известный маркер (lookahead на _GLUE_MARKER) — иначе риск съесть -# первую букву обычного слова. Корпус «сN/кN» требует цифры, поэтому не путается -# с маркерами «с.» (село) / «к» — у тех нет цифры вплотную. -_GLUE_MARKER = ( - r"(?:р-?н|мкр\.?|микрорайон|район|г\.|пос\.?|посёлок|поселок" r"|снт|кв-л|квартал|д\.|с\.|пгт)" -) -_GLUE_HOUSE_TOKEN = ( - r"\d+" # дом: цифры - # опц. одиночная литера (29А) — только если за ней сразу маркер - rf"(?:\s?[А-Яа-я](?={_GLUE_MARKER}))?" - r"(?:\s?(?:/\d+|с\d+|к\d+|корп\.?\s?\d+|стр\.?\s?\d+))?" # опц. корпус/строение -) -# Хвост `(?![а-яёa-z])` гарантирует, что маркер — отдельный токен, а не начало слова. -_DEGLUE_RE = re.compile( - rf"({_GLUE_HOUSE_TOKEN})({_GLUE_MARKER}(?![а-яёa-z]))", - flags=re.IGNORECASE, -) - - -def _deglue_house_marker(addr: str) -> str: - """Вставить «, » между слипшимся номером дома и маркером района/нас.пункта. - - «…Савкова, 29р-н Академический» -> «…Савкова, 29, р-н Академический» - «…, 17Ар-н Пионерский» -> «…, 17А, р-н Пионерский» - «…, 35/2р-н …» -> «…, 35/2, р-н …» - «…, 33с1р-н …» -> «…, 33с1, р-н …» - - Идемпотентно: на уже-расклеенной строке («29, р-н») паттерн не срабатывает, - т.к. между номером и маркером уже есть запятая/пробел. - """ - return _DEGLUE_RE.sub(r"\1, \2", addr) - - -def _clean_address(raw: str | None) -> str | None: - """Strip Emotion CSS and post-address noise that Avito DOM leaks via .text(). - - Examples: - "ул. Токарей, 56к1Площадь 1905 года.css-39hgr0{fill:..." - -> "ул. Токарей, 56к1" - "ул. Малышева, 1.css-xxx{...}" - -> "ул. Малышева, 1" - "ул. Евгения Савкова, 29р-н Академический" - -> "ул. Евгения Савкова, 29, р-н Академический" - """ - if not raw: - return None - cleaned = _CSS_NOISE_RE.sub("", raw) - cleaned = _NOT_ADDRESS_TAIL_RE.split(cleaned, maxsplit=1)[0] - cleaned = _deglue_house_marker(cleaned) - cleaned = cleaned.strip(" ,.\n\t") - return cleaned or None - - -def _extract_rooms_from_title(title: str) -> int | None: - if _RE_STUDIO.search(title): - return 0 - m = _RE_ROOMS.search(title) - if m: - return int(m.group(1)) - return None - - -def _extract_area_from_title(title: str) -> float | None: - m = _RE_AREA.search(title) - if m: - return float(m.group(1).replace(",", ".")) - return None - - -def _extract_floor_from_title(title: str) -> tuple[int | None, int | None]: - m = _RE_FLOOR.search(title) - if m: - return int(m.group(1)), int(m.group(2)) - return None, None diff --git a/tradein-mvp/backend/app/services/scrapers/avito_detail.py b/tradein-mvp/backend/app/services/scrapers/avito_detail.py deleted file mode 100644 index 074b0bb6..00000000 --- a/tradein-mvp/backend/app/services/scrapers/avito_detail.py +++ /dev/null @@ -1,1075 +0,0 @@ -"""Avito detail page parser — Stage 2b. - -Извлекает 30+ полей из страницы объявления: -- Identity (item_id, title, price, publish_date, views) -- Apartment params (rooms, area, floor, balcony_loggia, room_layout, - bathroom_type, windows_view, repair_state, sale_type, mortgage_available) -- Location (lat, lon, avito_location_id, metro_stations[], address_full) -- House params (house_type, total_floors_house, lifts, concierge, closed_yard, - house_catalog_url) — house_type + house_catalog_url сохраняются в listings - (#2009); total_floors_house/lifts/concierge/closed_yard остаются canonical - из Houses Catalog (Stage 2c, avito_houses.py → houses table) -- Description -- Domoteka 5 полей (owners_count, owners_at_least, last_owner_change_date, - encumbrances_clean, registry_match) -- Gallery (photo_urls[]) - -Точка входа для batch-обогащения: - enrichment = await fetch_detail(item_url) - saved = save_detail_enrichment(db, enrichment) -""" - -from __future__ import annotations - -import asyncio -import json -import logging -import random -import re -from dataclasses import dataclass, field -from datetime import date -from typing import TYPE_CHECKING, Any -from urllib.parse import quote, urljoin, urlparse - -from curl_cffi.requests import AsyncSession -from selectolax.parser import HTMLParser, Node -from sqlalchemy import text -from sqlalchemy.orm import Session - -from app.services.geocoder import is_within_ekb_bbox_wide -from app.services.scrapers.avito import _clean_address, _is_firewall_page -from app.services.scrapers.avito_exceptions import ( - AvitoBlockedError, - AvitoListingGoneError, - AvitoRateLimitedError, -) -from app.services.scrapers.avito_shared import RUS_MONTHS -from app.services.scrapers.house_type_normalizer import normalize_house_type -from app.services.scrapers.repair_state_normalizer import infer_repair_state_from_text - -if TYPE_CHECKING: - from app.services.scrapers.browser_fetcher import BrowserFetcher - -logger = logging.getLogger(__name__) - -AVITO_BASE = "https://www.avito.ru" - -# Живой прогрев сессии (#1551): GET Yandex SERP (органический referer-источник) -> GET -# Avito EKB search-страница (Referer=yandex) сеет антибот-куки, после чего прогретая -# curl_cffi-сессия держит батч detail-GET'ов без 403 (доказано пробами на проде: 22/22 -# карточек подряд, 0 блоков). referer на detail-GET = URL avito-search страницы. -_AVITO_WARM_SEARCH_URL = "https://www.avito.ru/ekaterinburg/kvartiry/prodam?q=" + quote( - "купить квартиру" -) -_AVITO_WARM_YANDEX_URL = "https://yandex.ru/search/?text=" + quote( - "купить квартиру екатеринбург авито" -) -_AVITO_WARM_YANDEX_REFERER = "https://yandex.ru/" - -# Reconnect-retry на detail-странице под backconnect-прокси — зеркало SERP-фикса -# (#1765/#1769, avito.py): 403/firewall = залочен лишь текущий exit-IP backconnect-прокси; -# пересоздание curl_cffi-сессии = новый CONNECT-туннель = свежий exit-IP → escape. -# 429 = transient conn-limit: сначала короткий retry на ТОЙ ЖЕ сессии. Но keep-alive -# detail-fetch переиспользует ОДИН backconnect exit-IP на последовательные запросы → -# avito лочит этот IP, и short-retry той же сессии не помогает (run 217: 5 подряд 429, -# abort на 19 enriched). Поэтому при исчерпании short-retry под backconnect делаем ещё -# _AVITO_DETAIL_429_RECONNECT_RETRIES попыток через эфемерную сессию (свежий exit-IP). -_AVITO_DETAIL_403_MAX_RETRIES = 5 -_AVITO_DETAIL_403_BACKOFF_SEC = 2.0 -_AVITO_DETAIL_429_MAX_RETRIES = 8 -_AVITO_DETAIL_429_BACKOFF_SEC = 1.5 -_AVITO_DETAIL_429_RECONNECT_RETRIES = 3 - -# ── Regex ──────────────────────────────────────────────────────────────────── -_ITEM_ID_RE = re.compile(r"№\s*(\d+)") -_VIEWS_TOTAL_RE = re.compile(r"(\d+)\s+просмотр") -_VIEWS_TODAY_RE = re.compile(r"\+\s*(\d+)\s+сегодня") -_PUBLISH_DATE_RE = re.compile( - r"(\d{1,2})\s+(января|февраля|марта|апреля|мая|июня|" - r"июля|августа|сентября|октября|ноября|декабря)\s+в", - re.IGNORECASE, -) -_FLOAT_RE = re.compile(r"[\d.,]+") -_INT_RE = re.compile(r"\d+") -_FLOOR_SPLIT_RE = re.compile(r"(\d+)\s+из\s+(\d+)") - -OWNERS_RE = re.compile(r"^(\d+)\s+собственник(?:\s+или\s+больше)?", re.IGNORECASE) -OWNER_CHANGE_RE = re.compile(r"смена собственника\s+(\d+)\s+(\w+)\s+(\d{4})", re.IGNORECASE) -ENCUMBRANCES_RE = re.compile(r"Не\s+найдены\s+ограничения", re.IGNORECASE) -REGISTRY_MATCH_RE = re.compile(r"Совпадают\s+площадь", re.IGNORECASE) - -METRO_RE = re.compile( - r"([А-ЯЁ][а-яё]+(?:ская|инская|итская|овская|енская))\s*" - r"(?:(\d+)[–\-](\d+)|от\s+(\d+))\s*мин", - re.IGNORECASE, -) - -# ── Маппинг RUS полей params-блока ─────────────────────────────────────────── -# (rus_label) -> (field_name, conversion_type) -RUS_FIELD_MAP: dict[str, tuple[str, str | None]] = { - "Количество комнат": ("rooms", "int"), - "Общая площадь": ("area_m2", "float_m2"), - "Площадь кухни": ("kitchen_area_m2", "float_m2"), - "Жилая площадь": ("living_area_m2", "float_m2"), - "Высота потолков": ("ceiling_height_m", "height_m"), - "Отделка": ("finishing", "text_lower"), - "Этаж": ("floor_split", None), - "Балкон или лоджия": ("balcony_loggia", "balcony"), - "Тип комнат": ("room_layout", "layout"), - "Санузел": ("bathroom_type", "bathroom"), - "Окна": ("windows_view", "windows"), - "Ремонт": ("repair_state", "repair"), - "Способ продажи": ("sale_type", "sale_type"), - "Условия продажи": ("sale_conditions_raw", None), - "Тип дома": ("house_type", "house_type"), - "Этажей в доме": ("total_floors_house", "int"), - "Пассажирский лифт": ("passenger_elevators", "int"), - "Грузовой лифт": ("cargo_elevators", "int"), - "В доме": ("in_house_features_raw", None), - "Двор": ("yard_features_raw", None), -} - -# ── Enum-маппинги ───────────────────────────────────────────────────────────── -_BALCONY_MAP: dict[str, str] = { - "балкон": "balcony", - "лоджия": "loggia", - "балкон, лоджия": "both", - "лоджия, балкон": "both", - "нет": "none", -} - -_LAYOUT_MAP: dict[str, str] = { - "изолированные": "isolated", - "смежные": "adjacent", - "смешанные": "mixed", -} - -_BATHROOM_MAP: dict[str, str] = { - "совмещенный": "combined", - "раздельный": "separate", - "совмещенный, раздельный": "both", - "раздельный, совмещенный": "both", -} - -_WINDOWS_MAP: dict[str, str] = { - "во двор": "yard", - "на улицу": "street", - "во двор и на улицу": "both", - "на улицу и во двор": "both", - "в парк": "park", -} - -_REPAIR_MAP: dict[str, str] = { - # Русские значения из Avito HTML → каноничный enum (needs_repair/standard/good/excellent) - # Маппинг выровнен по repair_state_normalizer.normalize_repair_state(). - "косметический": "standard", - "евро": "good", - "дизайнерский": "excellent", - "требуется": "needs_repair", - "без ремонта": "needs_repair", -} - -_SALE_TYPE_MAP: dict[str, str] = { - "свободная": "free", - "альтернатива": "alternative", - "аукцион": "auction", - "переуступка": "assignment", -} - -# house_type: канон panel/brick/monolith/monolith_brick/block/wood — через shared -# normalize_house_type() (#2009, консолидация с #2007/#2008; см. -# house_type_normalizer.py — русские лейблы avito добавлены туда же вокабуляром, -# НЕ дублируются локально). - - -# ── DetailEnrichment dataclass ──────────────────────────────────────────────── -@dataclass -class DetailEnrichment: - """Результат парсинга detail page Avito. - - Содержит 30+ полей из страницы объявления. - save_detail_enrichment() пишет подмножество в listings; house_type + house_catalog_url - сохраняются (#2009), остальные house-level поля (total_floors_house, elevators, - concierge, closed_yard) пропускаются — они canonical из Houses Catalog Stage 2c. - """ - - # Identity - item_id: str - source_url: str - title: str | None = None - price_rub: int | None = None - publish_date: date | None = None - views_total: int | None = None - views_today: int | None = None - - # Apartment params - rooms: int | None = None - area_m2: float | None = None - kitchen_area_m2: float | None = None - living_area_m2: float | None = None - ceiling_height_m: float | None = None - finishing: str | None = None - floor: int | None = None - total_floors: int | None = None - balcony_loggia: str | None = None - room_layout: str | None = None - bathroom_type: str | None = None - windows_view: str | None = None - repair_state: str | None = None - sale_type: str | None = None - mortgage_available: bool | None = None - - # House params — house_type/house_catalog_url сохраняются в listings (#2009); - # остальные пропускаются, canonical источник — Houses Catalog (Stage 2c) - house_type: str | None = None - total_floors_house: int | None = None - passenger_elevators: int | None = None - cargo_elevators: int | None = None - has_concierge: bool | None = None - closed_yard: bool | None = None - house_catalog_url: str | None = None - - # Location - address_full: str | None = None - lat: float | None = None - lon: float | None = None - avito_location_id: int | None = None - metro_stations: list[dict[str, Any]] = field(default_factory=list) - - # Description - description: str | None = None - - # Domoteka (5 полей) - owners_count: int | None = None - owners_at_least: bool | None = None - last_owner_change_date: date | None = None - encumbrances_clean: bool | None = None - registry_match: bool | None = None - - # Gallery - photo_urls: list[str] = field(default_factory=list) - - # Raw для retrofit / debug - raw_html_meta: dict[str, Any] = field(default_factory=dict) - - -def _build_detail_session() -> AsyncSession: - """Создать curl_cffi-сессию для detail-fetch (impersonate chrome120, через прокси - если задан settings.scraper_proxy_url). - - Вынесено из own-session-ветки fetch_detail чтобы reconnect-retry мог пересоздать - эфемерную сессию (новый CONNECT-туннель = свежий backconnect exit-IP) под 403. - proxy=None → прямое подключение. - """ - from app.core.config import settings - - proxy_url = settings.scraper_proxy_url - proxies = {"http": proxy_url, "https": proxy_url} if proxy_url else None - return AsyncSession( - impersonate="chrome120", - timeout=25, - proxies=proxies, - headers={ - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", - "Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8", - "Cache-Control": "max-age=0", - "Sec-Fetch-Dest": "document", - "Sec-Fetch-Mode": "navigate", - "Sec-Fetch-Site": "none", - "Sec-Fetch-User": "?1", - "Upgrade-Insecure-Requests": "1", - }, - ) - - -async def warm_up_session(session: AsyncSession) -> bool: - """Органический прогрев сессии: Yandex SERP (best-effort) -> Avito EKB search - (Referer=yandex). Сеет антибот-куки, после чего сессия держит батч detail-GET'ов - без 403. Возвращает True если search-страница загрузилась (не firewall). - """ - try: - await session.get(_AVITO_WARM_YANDEX_URL) # органический referer-источник, best-effort - except Exception as exc: - logger.debug("avito warm-up: yandex GET failed (non-fatal): %s", exc) - await asyncio.sleep(random.uniform(2.0, 3.5)) - try: - r = await session.get( - _AVITO_WARM_SEARCH_URL, headers={"Referer": _AVITO_WARM_YANDEX_REFERER} - ) - except Exception as exc: - logger.warning("avito warm-up: search fetch error: %s", exc) - return False - ok = r.status_code == 200 and not _is_firewall_page(r.text) - if not ok: - logger.warning("avito warm-up: search page blocked sc=%s", r.status_code) - return ok - - -async def research_in_session(session: AsyncSession) -> bool: - """Лёгкий in-session перепоиск: re-GET avito search-страницы (Referer=yandex) на - ТЕКУЩЕЙ сессии — освежает антибот-куки БЕЗ смены exit-IP. Best-effort (search-страница - иногда 429, это не ломает detail-фетчи). Возвращает True если страница не firewall. - """ - try: - r = await session.get( - _AVITO_WARM_SEARCH_URL, headers={"Referer": _AVITO_WARM_YANDEX_REFERER} - ) - except Exception as exc: - logger.warning("avito re-search: fetch error: %s", exc) - return False - ok = r.status_code == 200 and not _is_firewall_page(r.text) - if not ok: - logger.info("avito re-search: search page sc=%s (non-fatal)", r.status_code) - return ok - - -async def build_warmed_session() -> AsyncSession: - """Построить detail-сессию (_build_detail_session) и прогреть её. Возвращает - прогретую сессию (даже если warm-up soft-failed — caller может всё равно пробовать). - """ - session = _build_detail_session() - await warm_up_session(session) - return session - - -# ── Soft-block detection (#1967 / #1950) ────────────────────────────────────── -# Под нагрузкой Avito иногда вместо detail-страницы отдаёт HTTP 200 с ОБЩЕЙ -# витриной (Авито — Объявления на сайте Авито) вместо запрошенного -# объявления — мягкий анти-бот deflect. У такой страницы НЕТ ни firewall-маркеров -# (_is_firewall_page=False), ни item-view → parse_detail_html падал -# ValueError("Cannot extract item_id") → generic failed, IP НЕ ротировался, enriched -# каскадно падал к 0 (прод runs 431→437: 115→82→8→2→1→0 при blocked=0). Детектим -# как блок → reconnect свежей exit-IP сессией / loop-ротация. -_DETAIL_SOFT_BLOCK_TITLE_MARKERS = ("объявления на сайте авито",) - - -def _is_detail_soft_block(html: str) -> bool: - """True если detail-fetch вернул HTTP 200, но это анти-бот deflect — общая - витрина Авито вместо объявления. - - Сигнатура строго специфична deflect-витрине, чтобы НЕ задевать легитимные - не-блок ответы: - * нормальное объявление содержит item-view-маркеры → False; - * снятое объявление Avito рендерит как item-view + баннер «снято» → False; - * 404 / не-200 обрабатывается выше (ValueError → failed), сюда не попадает. - Положительный сигнал: НЕТ item-view-структуры И generic-витринный . - """ - if "item-view/item-id" in html or "item-view/title-info" in html: - return False - head = html[:16384].lower() - return any(m in head for m in _DETAIL_SOFT_BLOCK_TITLE_MARKERS) - - -def _is_detail_not_found(html: str) -> bool: - """True if the page is Avito's 404 / removed-listing page (not an anti-bot deflect). - Guard: a real listing has item-view → False. Keyed on the specific 404 title so it - does NOT catch the soft-block deflect ("объявления на сайте авито" without 404 text).""" - if "item-view/item-id" in html or "item-view/title-info" in html: - return False - head = html[:16384].lower() - return "страница не найдена" in head or "ошибка 404" in head - - -# ── fetch_detail ────────────────────────────────────────────────────────────── -async def fetch_detail( - item_url: str, - *, - cffi_session: AsyncSession | None = None, - browser_fetcher: BrowserFetcher | None = None, - referer: str | None = None, - reconnect_on_block: bool = True, -) -> DetailEnrichment: - """GET <avito>/{item_url} → parse HTML via selectolax → DetailEnrichment. - - Если browser_fetcher передан — использует браузерный fetch (browser mode). - Если cffi_session не передана — создаёт новую (impersonate='chrome120'). - referer — если задан, шлётся в Referer-заголовке detail-GET'а (warm-batch #1551: - referer = URL avito-search страницы, на которой прогрелась сессия). - reconnect_on_block — если False, 403/firewall поднимает AvitoBlockedError СРАЗУ - (без холодных reconnect'ов — ими займётся outer-loop через rebuild+rewarm прогретой - сессии); 429 после short-retry поднимает AvitoRateLimitedError. Дефолт True - сохраняет старое поведение own-session путей (напр. admin endpoint). - Raises: - httpx.HTTPError — если status != 200 (через raise_for_status-like). - ValueError — если item_id не извлечён из HTML. - """ - if browser_fetcher is not None: - # Browser-only режим (#1814): когда передан browser_fetcher, fetch идёт - # СТРОГО через camoufox-сервис. Никакой curl_cffi-сессии и никакого - # curl-fallback — avito-антибот банит curl-фингерпринт под нагрузкой - # (429 CONNECT tunnel failed), а browser проходит. Любой firewall/ошибка - # браузера → AvitoBlockedError; caller (backfill) пометит failed и пойдёт - # дальше, НЕ трогая curl. - full_url = item_url if item_url.startswith("http") else urljoin(AVITO_BASE, item_url) - try: - html = await browser_fetcher.fetch(full_url) - except (AvitoBlockedError, AvitoRateLimitedError): - raise - except Exception as exc: - # browser-сервис вернул HTTPError / ConnectError / 502 (firewall, dead - # listing, redirect). НЕ откатываемся в curl — поднимаем block, чтобы - # caller пометил listing и продолжил без curl-фингерпринта. - raise AvitoBlockedError( - f"Avito detail browser fetch failed for {full_url}: {exc}" - ) from exc - # 404/removed-listing (#2034): мёртвый листинг рендерит «Ошибка 404» без - # item-view. Это НЕ анти-бот deflect — отдельный сигнал, чтобы backfill - # пометил листинг is_active=FALSE и НЕ крутил consecutive-block breaker - # (run 458: 5 dead подряд → abort до live-листингов). Проверяем ДО - # firewall/soft-block (иначе 404-страница ловится как soft-block). - if _is_detail_not_found(html): - raise AvitoListingGoneError( - f"Avito detail 404/listing-gone (browser-mode) for {full_url}" - ) - if _is_firewall_page(html) or _is_detail_soft_block(html): - raise AvitoBlockedError( - f"Avito detail firewall/soft-block (browser-mode) for {full_url}" - ) - return parse_detail_html(html, full_url) - - from app.core.config import settings - - own_session = cffi_session is None - if cffi_session is None: - # Mobile proxy wiring (#806 follow-up): own-session path (no shared session passed, - # e.g. admin endpoint). Caller-provided sessions already have proxy from - # AvitoScraper.__aenter__ — don't override. proxy=None → прямое подключение. - session = _build_detail_session() - else: - session = cffi_session - - full_url = item_url if item_url.startswith("http") else urljoin(AVITO_BASE, item_url) - # curl_cffi egress всегда через scraper_proxy_url (backconnect mproxy). 403/firewall = - # залочен лишь текущий exit-IP → reconnect новой сессией = свежий IP. Гейтим reconnect - # ТОЛЬКО на наличие proxy (зеркало SERP-фикса #1765). Без proxy — старое поведение - # (raise на первом 403). Caller-provided сессию НЕ трогаем: reconnect использует - # эфемерную retry_session, которую сами закрываем в finally. - # reconnect_on_block=False (warm-batch #1551): не делаем холодных per-card - # reconnect'ов — блок поднимает AvitoBlockedError сразу, outer-loop пересоздаёт+ - # прогревает сессию (свежий exit-IP). reconnect_on_block=True (own-session) — - # старое поведение: backconnect-reconnect эфемерной сессией. - backconnect = bool(settings.scraper_proxy_url) and reconnect_on_block - req_headers = {"Referer": referer} if referer else None - r403 = 0 - r429 = 0 - r429_reconnect = 0 - attempt_session = session - retry_session: AsyncSession | None = None - try: - while True: - response = await attempt_session.get(full_url, headers=req_headers) - sc = response.status_code - is_firewall = sc == 200 and ( - _is_firewall_page(response.text) or _is_detail_soft_block(response.text) - ) - - # 429 через backconnect = transient conn-limit, НЕ IP-ban: короткий retry той - # же сессии без reconnect (зеркало SERP). Только при исчерпании — raise. - if sc == 429 and r429 < _AVITO_DETAIL_429_MAX_RETRIES: - r429 += 1 - logger.info( - "avito detail HTTP 429 (backconnect conn-limit) — short retry %d/%d", - r429, - _AVITO_DETAIL_429_MAX_RETRIES, - ) - await asyncio.sleep(_AVITO_DETAIL_429_BACKOFF_SEC) - continue - - if sc == 403 or is_firewall: - if backconnect and r403 < _AVITO_DETAIL_403_MAX_RETRIES: - r403 += 1 - # Эфемерная свежая сессия (новый CONNECT-туннель = свежий exit-IP). - # Caller-provided/own session НЕ пересоздаём — только retry_session. - if retry_session is not None: - try: - await retry_session.close() - except Exception: - pass - retry_session = _build_detail_session() - attempt_session = retry_session - logger.info( - "avito detail 403/firewall/soft-block %s — backconnect reconnect " - "retry %d/%d", - full_url, - r403, - _AVITO_DETAIL_403_MAX_RETRIES, - ) - await asyncio.sleep(_AVITO_DETAIL_403_BACKOFF_SEC) - continue - raise AvitoBlockedError(f"Avito detail HTTP 403 for {full_url}") - - if sc == 429: - # short-retry (та же сессия) исчерпан. Под backconnect — ещё несколько - # попыток через эфемерную сессию (свежий exit-IP): keep-alive держал ОДИН - # залоченный IP, новый CONNECT-туннель уходит на другой. Делит ту же - # retry_session-машинерию что и 403-путь (close-prev → rebuild → retry, - # cleanup в finally). Только потом raise. - if backconnect and r429_reconnect < _AVITO_DETAIL_429_RECONNECT_RETRIES: - r429_reconnect += 1 - # Caller-provided/own session НЕ пересоздаём — только retry_session. - if retry_session is not None: - try: - await retry_session.close() - except Exception: - pass - retry_session = _build_detail_session() - attempt_session = retry_session - logger.info( - "avito detail HTTP 429 short-retry exhausted %s — " - "backconnect reconnect retry %d/%d", - full_url, - r429_reconnect, - _AVITO_DETAIL_429_RECONNECT_RETRIES, - ) - await asyncio.sleep(_AVITO_DETAIL_403_BACKOFF_SEC) - continue - raise AvitoRateLimitedError(f"Avito detail HTTP 429 for {full_url}") - if sc != 200: - raise ValueError(f"avito detail HTTP {sc} for {full_url}") - return parse_detail_html(response.text, full_url) - finally: - if own_session: - await session.close() - if retry_session is not None and retry_session is not session: - try: - await retry_session.close() - except Exception: - pass - - -# ── parse_detail_html ───────────────────────────────────────────────────────── -def parse_detail_html(html: str, source_url: str) -> DetailEnrichment: - """Pure parser — отдельно для тестов на fixtures без сети. - - Raises: - ValueError — если item_id не извлечён. - """ - tree = HTMLParser(html) - - # ── Identity ──────────────────────────────────────────────────────────── - item_id = _extract_item_id(tree) - if not item_id: - raise ValueError(f"Cannot extract item_id from {source_url}") - - title = _text(tree, "[data-marker='item-view/title-info']") - price_rub = _extract_price(tree) - publish_date, views_total, views_today = _extract_meta(tree) - - # ── Location ──────────────────────────────────────────────────────────── - map_el = tree.css_first("[data-marker='item-map-wrapper']") - lat: float | None = None - lon: float | None = None - avito_location_id: int | None = None - if map_el is not None: - lat = _try_float(map_el.attributes.get("data-map-lat")) - lon = _try_float(map_el.attributes.get("data-map-lon")) - avito_location_id = _try_int(map_el.attributes.get("data-location-id")) - - # bbox-guard (#1871): detail-страница иногда отдаёт координаты вне ЕКБ - # (Питер/Тюмень/Уфа — кросс-региональные дубли объявлений). Эти координаты - # попадают в UPDATE COALESCE без bbox-валидации (в отличие от geocoder.py, - # где bbox-фильтр везде). Сбрасываем в None → листинг уходит в geocode-cron - # путь, который сам отфильтрует по адресу. SERP-ingest координат не даёт вовсе. - if lat is not None and lon is not None and not is_within_ekb_bbox_wide(lat, lon): - logger.warning( - "avito_detail #1871: dropped non-EKB coords item=%s lat=%s lon=%s", - item_id, - lat, - lon, - ) - lat = lon = None - - address_full = _clean_address(_text(tree, "[itemprop='address']")) - - # ── Description ───────────────────────────────────────────────────────── - description = _text(tree, "[data-marker='item-view/item-description']") - - # ── Metro from description ─────────────────────────────────────────────── - metro_stations: list[dict[str, Any]] = [] - if description: - metro_stations = _extract_metro(description) - - # ── House catalog link ─────────────────────────────────────────────────── - # Новостройки отдают кнопку ЖК (nd-jk-details-button); вторичка — ссылку - # «узнать больше о доме» под другим маркером. Fallback на любой /catalog/houses/ - # якорь: verified 2026-05-31 (#871) — эта ссылка есть в SSR detail-страницы, - # даже когда отсутствует в JS-рендеренной SERP-карточке (откуда мы брали 0). - house_catalog_url: str | None = None - house_link = tree.css_first("[data-marker='nd-jk-details-button']") - if house_link is None: - house_link = tree.css_first("a[href*='/catalog/houses/']") - if house_link is not None: - href = house_link.attributes.get("href", "") - if href: - # отбрасываем ?context-трекинг → чистый catalog-путь - house_catalog_url = urljoin(AVITO_BASE, urlparse(href).path) - - # ── Photo gallery ──────────────────────────────────────────────────────── - photo_urls: list[str] = [] - for img in tree.css("[data-marker='image-preview/item'] img"): - src = img.attributes.get("src") or "" - if src and src not in photo_urls: - photo_urls.append(src) - - # ── Params blocks ──────────────────────────────────────────────────────── - params_block = tree.css_first("[data-marker='item-view/item-params']") - apt_params: dict[str, str] = {} - house_params: dict[str, str] = {} - if params_block is not None: - ul_els = params_block.css("ul") - if len(ul_els) >= 1: - apt_params = _parse_params_ul(ul_els[0]) - if len(ul_els) >= 2: - house_params = _parse_params_ul(ul_els[1]) - - # Merge house_params into apt_params — некоторые страницы совмещают всё в одном ul - all_params = {**apt_params, **house_params} - - # ── Parse apartment fields ─────────────────────────────────────────────── - rooms: int | None = None - area_m2: float | None = None - kitchen_area_m2: float | None = None - living_area_m2: float | None = None - ceiling_height_m: float | None = None - finishing: str | None = None - floor: int | None = None - total_floors: int | None = None - balcony_loggia: str | None = None - room_layout: str | None = None - bathroom_type: str | None = None - windows_view: str | None = None - repair_state: str | None = None - sale_type: str | None = None - mortgage_available: bool | None = None - house_type: str | None = None - total_floors_house: int | None = None - passenger_elevators: int | None = None - cargo_elevators: int | None = None - has_concierge: bool | None = None - closed_yard: bool | None = None - - for rus_label, val in all_params.items(): - mapping = RUS_FIELD_MAP.get(rus_label) - if mapping is None: - continue - field_name, _conv = mapping - - if field_name == "rooms": - rooms = _try_int(val) - elif field_name == "area_m2": - area_m2 = _parse_float_m2(val) - elif field_name == "kitchen_area_m2": - kitchen_area_m2 = _parse_float_m2(val) - elif field_name == "living_area_m2": - living_area_m2 = _parse_float_m2(val) - elif field_name == "ceiling_height_m": - ceiling_height_m = _parse_height_m(val) - elif field_name == "finishing": - finishing = val.strip().lower() or None - elif field_name == "floor_split": - floor, total_floors = _parse_floor_split(val) - elif field_name == "balcony_loggia": - balcony_loggia = _map_lower(val, _BALCONY_MAP) - elif field_name == "room_layout": - room_layout = _map_lower(val, _LAYOUT_MAP) - elif field_name == "bathroom_type": - bathroom_type = _map_lower(val, _BATHROOM_MAP) - elif field_name == "windows_view": - windows_view = _map_lower(val, _WINDOWS_MAP) - elif field_name == "repair_state": - repair_state = _map_lower(val, _REPAIR_MAP) - elif field_name == "sale_type": - sale_type = _map_lower(val, _SALE_TYPE_MAP) - elif field_name == "sale_conditions_raw": - if "возможна ипотека" in val.lower(): - mortgage_available = True - elif field_name == "house_type": - house_type = normalize_house_type(val) - elif field_name == "total_floors_house": - total_floors_house = _try_int(val) - elif field_name == "passenger_elevators": - passenger_elevators = _try_int(val) - elif field_name == "cargo_elevators": - cargo_elevators = _try_int(val) - elif field_name == "in_house_features_raw": - if "консьерж" in val.lower(): - has_concierge = True - elif field_name == "yard_features_raw": - if "закрытая территория" in val.lower(): - closed_yard = True - - # ── Fallback: repair_state из описания, если структурное поле пустое (#622) ── - if repair_state is None and description: - repair_state = infer_repair_state_from_text(description) - - # ── Domoteka ───────────────────────────────────────────────────────────── - owners_count: int | None = None - owners_at_least: bool | None = None - last_owner_change_date: date | None = None - encumbrances_clean: bool | None = None - registry_match: bool | None = None - - domoteka_block = tree.css_first("[data-marker='domoteka-entry-block']") - if domoteka_block is not None: - badges = domoteka_block.css("p[data-marker='TeaserData.item']") - for badge in badges: - badge_text = badge.text(strip=True) - if not badge_text: - continue - - m_owners = OWNERS_RE.match(badge_text) - if m_owners: - owners_count = int(m_owners.group(1)) - owners_at_least = "или больше" in badge_text.lower() - continue - - m_change = OWNER_CHANGE_RE.search(badge_text) - if m_change: - try: - day = int(m_change.group(1)) - month_word = m_change.group(2).lower() - year = int(m_change.group(3)) - month = RUS_MONTHS.get(month_word) - if month: - last_owner_change_date = date(year, month, day) - except (ValueError, KeyError): - logger.warning("Cannot parse owner change date: %s", badge_text) - continue - - if ENCUMBRANCES_RE.search(badge_text): - encumbrances_clean = True - continue - - if REGISTRY_MATCH_RE.search(badge_text): - registry_match = True - continue - - return DetailEnrichment( - item_id=item_id, - source_url=source_url, - title=title, - price_rub=price_rub, - publish_date=publish_date, - views_total=views_total, - views_today=views_today, - rooms=rooms, - area_m2=area_m2, - kitchen_area_m2=kitchen_area_m2, - living_area_m2=living_area_m2, - ceiling_height_m=ceiling_height_m, - finishing=finishing, - floor=floor, - total_floors=total_floors, - balcony_loggia=balcony_loggia, - room_layout=room_layout, - bathroom_type=bathroom_type, - windows_view=windows_view, - repair_state=repair_state, - sale_type=sale_type, - mortgage_available=mortgage_available, - house_type=house_type, - total_floors_house=total_floors_house, - passenger_elevators=passenger_elevators, - cargo_elevators=cargo_elevators, - has_concierge=has_concierge, - closed_yard=closed_yard, - house_catalog_url=house_catalog_url, - address_full=address_full, - lat=lat, - lon=lon, - avito_location_id=avito_location_id, - metro_stations=metro_stations, - description=description, - owners_count=owners_count, - owners_at_least=owners_at_least, - last_owner_change_date=last_owner_change_date, - encumbrances_clean=encumbrances_clean, - registry_match=registry_match, - photo_urls=photo_urls, - ) - - -# ── save_detail_enrichment ──────────────────────────────────────────────────── -def save_detail_enrichment(db: Session, e: DetailEnrichment) -> bool: - """UPDATE listings SET <25+ cols> WHERE source='avito' AND source_id=:item_id. - - Returns True если строка обновлена, False если listing не найден. - Устанавливает detail_enriched_at=NOW(). - - house_type СОХРАНЯЕТСЯ (уже нормализован парсером → канон panel/brick/monolith/ - monolith_brick/block/wood) — нужен estimator soft-penalty по типу дома. - house_catalog_url СОХРАНЯЕТСЯ в listings.house_url (линковка дома); отдельной - house_catalog_url колонки нет. house_type — COALESCE existing-first (не затираем - canonical из Houses Catalog Stage 2c); house_url — new-first additive (avito-owned - поле, нет конкурирующего canonical-писателя). - Остальные house-level поля (passenger_elevators, cargo_elevators, has_concierge, - closed_yard, total_floors_house) по-прежнему опускаются — нет колонок в listings, - canonical приходит из Houses Catalog Stage 2c (отдельный follow-up). - """ - result = db.execute( - text(""" - UPDATE listings SET - kitchen_area_m2 = COALESCE(:kitchen_area_m2, kitchen_area_m2), - living_area_m2 = COALESCE(:living_area_m2, living_area_m2), - ceiling_height_m = COALESCE(:ceiling_height_m, ceiling_height_m), - finishing = COALESCE(:finishing, finishing), - balcony_loggia = COALESCE(:balcony_loggia, balcony_loggia), - room_layout = COALESCE(:room_layout, room_layout), - bathroom_type = COALESCE(:bathroom_type, bathroom_type), - windows_view = COALESCE(:windows_view, windows_view), - sale_type = COALESCE(:sale_type, sale_type), - mortgage_available = COALESCE(:mortgage_available, mortgage_available), - views_total = COALESCE(:views_total, views_total), - views_today = COALESCE(:views_today, views_today), - publish_date = COALESCE(:publish_date, publish_date), - description = COALESCE(:description, description), - owners_count = COALESCE(:owners_count, owners_count), - owners_at_least = COALESCE(:owners_at_least, owners_at_least), - last_owner_change_date = COALESCE(:last_owner_change_date, last_owner_change_date), - encumbrances_clean = COALESCE(:encumbrances_clean, encumbrances_clean), - registry_match = COALESCE(:registry_match, registry_match), - metro_stations = COALESCE(CAST(:metro_stations AS jsonb), metro_stations), - avito_location_id = COALESCE(:avito_location_id, avito_location_id), - lat = COALESCE(:lat, lat), - lon = COALESCE(:lon, lon), - address = COALESCE(:address, address), - rooms = COALESCE(:rooms, rooms), - area_m2 = COALESCE(:area_m2, area_m2), - floor = COALESCE(:floor, floor), - total_floors = COALESCE(:total_floors, total_floors), - repair_state = COALESCE(:repair_state, repair_state), - price_rub = COALESCE(:price_rub, price_rub), - house_type = COALESCE(house_type, :house_type), - house_url = COALESCE(:house_catalog_url, house_url), - detail_enriched_at = NOW() - WHERE source = 'avito' AND source_id = :item_id - """), - { - "item_id": e.item_id, - "kitchen_area_m2": e.kitchen_area_m2, - "living_area_m2": e.living_area_m2, - "ceiling_height_m": e.ceiling_height_m, - "finishing": e.finishing, - "balcony_loggia": e.balcony_loggia, - "room_layout": e.room_layout, - "bathroom_type": e.bathroom_type, - "windows_view": e.windows_view, - "sale_type": e.sale_type, - "mortgage_available": e.mortgage_available, - "views_total": e.views_total, - "views_today": e.views_today, - "publish_date": e.publish_date, - "description": e.description, - "owners_count": e.owners_count, - "owners_at_least": e.owners_at_least, - "last_owner_change_date": e.last_owner_change_date, - "encumbrances_clean": e.encumbrances_clean, - "registry_match": e.registry_match, - "metro_stations": ( - json.dumps(e.metro_stations, ensure_ascii=False) if e.metro_stations else None - ), - "avito_location_id": e.avito_location_id, - "lat": e.lat, - "lon": e.lon, - "address": e.address_full, - "rooms": e.rooms, - "area_m2": e.area_m2, - "floor": e.floor, - "total_floors": e.total_floors, - "repair_state": e.repair_state, - "price_rub": e.price_rub, - "house_type": e.house_type, - "house_catalog_url": e.house_catalog_url, - }, - ) - db.commit() - found = result.rowcount > 0 - if not found: - logger.warning("save_detail_enrichment: listing not found in DB for item_id=%s", e.item_id) - else: - logger.info("save_detail_enrichment: updated listing item_id=%s", e.item_id) - return found - - -# ── Internal helpers ───────────────────────────────────────────────────────── -def _text(tree: HTMLParser, selector: str) -> str | None: - """Вернуть stripped text первого совпавшего элемента или None.""" - el = tree.css_first(selector) - if el is None: - return None - t = el.text(strip=True) - return t or None - - -def _try_float(val: Any) -> float | None: - if val is None: - return None - try: - return float(str(val).replace(",", ".")) - except (ValueError, TypeError): - return None - - -def _try_int(val: Any) -> int | None: - if val is None: - return None - s = str(val).strip() - m = _INT_RE.search(s) - if m: - try: - return int(m.group()) - except ValueError: - return None - return None - - -def _parse_float_m2(val: str) -> float | None: - """'90.1 м²' → 90.1. '75 м²' → 75.0.""" - m = _FLOAT_RE.search(val) - if m: - try: - return float(m.group().replace(",", ".")) - except ValueError: - return None - return None - - -def _parse_height_m(val: str) -> float | None: - """'2.7 м' / '2,7\xa0м' / '3 м' → float метры. Junk → None.""" - m = _FLOAT_RE.search(val) - if m: - try: - return float(m.group().replace(",", ".")) - except ValueError: - return None - return None - - -def _parse_floor_split(val: str) -> tuple[int | None, int | None]: - """'20 из 26' → (20, 26).""" - m = _FLOOR_SPLIT_RE.search(val) - if m: - return _try_int(m.group(1)), _try_int(m.group(2)) - return None, None - - -def _map_lower(val: str, mapping: dict[str, str]) -> str | None: - """Нижний регистр + strip → lookup в mapping. None если не найдено.""" - key = val.strip().lower() - return mapping.get(key) - - -def _extract_item_id(tree: HTMLParser) -> str | None: - """Извлечь item_id из [data-marker='item-view/item-id'] text.""" - el = tree.css_first("[data-marker='item-view/item-id']") - if el is None: - return None - text_val = el.text(strip=True) - m = _ITEM_ID_RE.search(text_val) - return m.group(1) if m else None - - -def _extract_price(tree: HTMLParser) -> int | None: - """Извлечь price из [itemprop='price'] content attribute.""" - el = tree.css_first("[itemprop='price']") - if el is None: - return None - content = el.attributes.get("content") - return _try_int(content) - - -def _extract_meta(tree: HTMLParser) -> tuple[date | None, int | None, int | None]: - """Извлечь publish_date, views_total, views_today из item-id блока.""" - el = tree.css_first("[data-marker='item-view/item-id']") - if el is None: - return None, None, None - - # Поиск по всему тексту включая дочерние элементы - full_text = el.text(strip=False) - - publish_date: date | None = None - m_date = _PUBLISH_DATE_RE.search(full_text) - if m_date: - try: - day = int(m_date.group(1)) - month_word = m_date.group(2).lower() - month = RUS_MONTHS.get(month_word) - # Год — текущий (Avito не показывает год для свежих объявлений) - import datetime - - current_year = datetime.date.today().year - if month: - publish_date = date(current_year, month, day) - except (ValueError, KeyError): - pass - - views_total: int | None = None - m_total = _VIEWS_TOTAL_RE.search(full_text) - if m_total: - views_total = int(m_total.group(1)) - - views_today: int | None = None - m_today = _VIEWS_TODAY_RE.search(full_text) - if m_today: - views_today = int(m_today.group(1)) - - # Попробуем найти views в отдельных data-marker элементах (новый Avito layout) - if views_total is None: - v_el = tree.css_first("[data-marker='item-view/total-views']") - if v_el is not None: - m = _VIEWS_TOTAL_RE.search(v_el.text(strip=True)) - if m: - views_total = int(m.group(1)) - - if views_today is None: - t_el = tree.css_first("[data-marker='item-view/today-views']") - if t_el is not None: - m = _VIEWS_TODAY_RE.search(t_el.text(strip=True)) - if m: - views_today = int(m.group(1)) - - return publish_date, views_total, views_today - - -def _parse_params_ul(ul: Node) -> dict[str, str]: - """Парсим <ul> с <li>Label: Value</li> → {label: value} dict.""" - result: dict[str, str] = {} - for li in ul.css("li"): - text_val = li.text(strip=True) - if ":" in text_val: - label, _, value = text_val.partition(":") - label = label.strip() - value = value.strip() - if label and value: - result[label] = value - return result - - -def _extract_metro(text_val: str) -> list[dict[str, Any]]: - """Извлечь станции метро из текста описания. - - Примеры: 'Чкаловская 11-15 мин пешком', 'от 5 мин пешком'. - """ - stations: list[dict[str, Any]] = [] - for m in METRO_RE.finditer(text_val): - name = m.group(1) - if m.group(2) and m.group(3): - # диапазон: 11-15 - min_from = int(m.group(2)) - min_to = int(m.group(3)) - elif m.group(4): - # "от N мин" - min_from = None - min_to = int(m.group(4)) - else: - continue - - stations.append( - { - "name": name, - "min_to": min_to, - "min_from": min_from, - "mode": "walk", - } - ) - return stations diff --git a/tradein-mvp/backend/app/services/scrapers/avito_exceptions.py b/tradein-mvp/backend/app/services/scrapers/avito_exceptions.py deleted file mode 100644 index 3895df76..00000000 --- a/tradein-mvp/backend/app/services/scrapers/avito_exceptions.py +++ /dev/null @@ -1,36 +0,0 @@ -"""Avito-specific exceptions для anti-bot detection.""" - - -class AvitoError(Exception): - """Base для всех Avito scraper exceptions.""" - - -class AvitoBlockedError(AvitoError): - """HTTP 403 от Avito — IP-level block detected.""" - - -class AvitoRateLimitedError(AvitoError): - """HTTP 429 от Avito — rate limit triggered.""" - - -class AvitoListingGoneError(AvitoError): - """Объявление удалено / вернуло 404 (страница «Ошибка 404» вместо item-view). - - Отдельный сигнал от AvitoBlockedError/AvitoRateLimitedError: 404 — НЕ анти-бот - deflect, а мёртвый листинг (его незачем ротировать/ретраить). Наследует общий - AvitoError, но НЕ AvitoBlockedError — чтобы backfill-ловушка - `except (AvitoBlockedError, AvitoRateLimitedError)` его НЕ перехватывала и не крутила - consecutive-block breaker на мёртвых координатных дырах (run 458: 5 dead → abort). - """ - - -class AvitoContentBlockedError(AvitoBlockedError): - """HTTP 200 от Avito, но 0 карточек на первой странице — content-block / captcha. - - Наследует AvitoBlockedError, чтобы существующие except-блоки на AvitoBlockedError - продолжали ловить этот сигнал (mark_banned, pipeline abort и т.д.). - """ - - -class AvitoParseError(AvitoError): - """Cannot parse Avito HTML structure (selector changes).""" diff --git a/tradein-mvp/backend/app/services/scrapers/avito_shared.py b/tradein-mvp/backend/app/services/scrapers/avito_shared.py deleted file mode 100644 index c499178d..00000000 --- a/tradein-mvp/backend/app/services/scrapers/avito_shared.py +++ /dev/null @@ -1,79 +0,0 @@ -"""avito_shared.py — общие константы/хелперы для avito*.py модулей. - -Единая точка правды для: - - `RUS_MONTHS` — рус. месяц (родительный падеж) → номер, для парсинга дат вида - "9 октября 2025" (было продублировано в avito.py/avito_detail.py/avito_houses.py). - - `_unix_to_date` — unix-epoch (в СЕКУНДАХ) → `date`, в таймзоне Europe/Moscow. - -TZ-обоснование (issue #2129): было 3 независимые копии конвертации unix→date с -РАЗНЫМИ timezone — MSK (avito.py, документировано issue #726), UTC (avito_houses.py), -naive/system-local (avito_imv.py). Один и тот же epoch давал разные `date` в -SERP / houses / IMV путях для одного и того же объявления. - -Почему MSK — канонический выбор (не UTC, не local): - - Avito — российская площадка, отображает даты объявлений (`item-date` на SERP, - `publishDate`, история цен `startPriceDate`/`lastPriceDate`/`removedDate`) в - Europe/Moscow. Объявление, поднятое в 00:00–03:00 MSK, при наивной UTC-конвертации - получило бы дату на день раньше, чем видно на самом Avito (обоснование #726). - - avito_houses.py и avito_imv.py используются на одних и тех же данных - (`housePlacementHistory` widget / IMV `placementHistory` — идентичные поля - startPriceDate/lastPriceDate/removedDate, см. fixtures/avito_imv_getdata.json) — - единственный смысл держать их согласованными с уже принятым MSK-решением из #726. - - Проверено на живой fixture: startPriceDate/lastPriceDate — всегда полночь UTC - (00:00:00), поэтому MSK vs UTC не меняет date для них. `removedDate`/`publishDate` - несут реальное время суток — там UTC/naive-local и MSK ДЕЙСТВИТЕЛЬНО расходятся - (пример из fixture: removedDate=1756077885 → UTC 2025-08-24, MSK 2025-08-25). - Это единственный реальный сдвиг данных от этого фикса: `avito_imv_evaluations` / - `house_placement_history`, поля `removed_date`/`publish_date`, у ~12.5% строк - (события в 21:00–23:59 UTC/local) дата сдвинется на +1 день при следующем прогоне - скрейпера. Схему/уже сохранённые строки не трогаем — это единственно правильное - представление на будущее. - - Fixed-offset `timezone(timedelta(hours=3))`, а не `zoneinfo("Europe/Moscow")` — - не зависит от системной tzdata в контейнере; Москва не переходит на DST с 2014, - так что fixed offset корректен всегда. -""" - -from __future__ import annotations - -import logging -from datetime import date, datetime, timedelta, timezone - -logger = logging.getLogger(__name__) - -# Europe/Moscow, UTC+3, без перехода на летнее время (с 2014). -MSK = timezone(timedelta(hours=3)) - -# Русские месяцы (родительный падеж) для парсинга дат вида "9 октября 2025". -RUS_MONTHS: dict[str, int] = { - "января": 1, - "февраля": 2, - "марта": 3, - "апреля": 4, - "мая": 5, - "июня": 6, - "июля": 7, - "августа": 8, - "сентября": 9, - "октября": 10, - "ноября": 11, - "декабря": 12, -} - - -def _unix_to_date(ts: int | float | None) -> date | None: - """Конвертирует unix-epoch (в СЕКУНДАХ) в `date`, в таймзоне Europe/Moscow (MSK). - - Возвращает `None` для `None`/`0` (Avito использует 0 как sentinel «нет даты») и - для значений, которые не конвертируются в валидную дату. - - ВАЖНО: ожидает epoch в СЕКУНДАХ. Avito SERP-поле `sortTimeStamp` отдаёт epoch в - МИЛЛИСЕКУНДАХ — вызывающий код обязан поделить на 1000 до вызова (см. - `avito.py::_build_sort_timestamp_map`). - """ - if not ts: - return None - try: - return datetime.fromtimestamp(float(ts), tz=MSK).date() - except (OSError, OverflowError, ValueError): - logger.warning("Не удалось конвертировать unix timestamp: %r", ts) - return None diff --git a/tradein-mvp/backend/app/services/scrapers/base.py b/tradein-mvp/backend/app/services/scrapers/base.py deleted file mode 100644 index 93160745..00000000 --- a/tradein-mvp/backend/app/services/scrapers/base.py +++ /dev/null @@ -1,830 +0,0 @@ -"""Базовый framework для парсеров объявлений (Avito / Cian / DomKlik / ...). - -Общие задачи: -- httpx.AsyncClient с realistic browser headers + UA rotation -- tenacity retry с exp backoff -- Sleep между запросами (anti-ban) -- ScrapedLot — единая Pydantic схема всех источников -- Запись в `listings` Postgres с дедупом по dedup_hash -- После INSERT/UPDATE — hook в matching service для регистрации в - `listing_sources` и привязки к каноническому `houses` через `house_sources`. - -Каждый конкретный парсер (avito.py, cian.py, ...) наследуется от BaseScraper -и реализует `_fetch_page()` и `_parse_listings()`. -""" - -from __future__ import annotations - -import asyncio -import hashlib -import logging -import random -from abc import ABC, abstractmethod -from datetime import date, datetime -from typing import Any -from zoneinfo import ZoneInfo - -import httpx -import psycopg.errors -from pydantic import BaseModel, Field -from sqlalchemy import text -from sqlalchemy.exc import IntegrityError -from sqlalchemy.orm import Session -from tenacity import retry, stop_after_attempt, wait_exponential - -from app.services.matching import match_or_create_house, upsert_listing_source -from app.services.scrapers.snapshot_writer import upsert_listing_snapshot - -logger = logging.getLogger(__name__) - -_MSK = ZoneInfo("Europe/Moscow") - - -# ── Realistic browser User-Agents (Apr 2026 versions) ────────────────────── - -_USER_AGENTS: list[str] = [ - # Chrome on macOS - "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36", # noqa: E501 - # Chrome on Windows - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36", # noqa: E501 - # Firefox on macOS - "Mozilla/5.0 (Macintosh; Intel Mac OS X 14.5; rv:127.0) Gecko/20100101 Firefox/127.0", - # Safari on macOS - "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Safari/605.1.15", # noqa: E501 - # Edge on Windows - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36 Edg/132.0.0.0", # noqa: E501 -] - - -def random_user_agent() -> str: - return random.choice(_USER_AGENTS) - - -# ── Унифицированная схема результата ──────────────────────────────────────── -class ScrapedLot(BaseModel): - """Результат парсинга одного объявления. - - Все источники приводятся к этому виду перед записью в Postgres. - Поля optional если источник их не отдаёт. - """ - - source: str # 'avito' / 'cian' / 'domklik' / ... - source_url: str # ссылка на оригинал — кликается из UI - source_id: str | None = None # внутренний id источника (для update) - - # Локация — должна быть хотя бы address ИЛИ (lat, lon) - address: str | None = None - lat: float | None = None - lon: float | None = None - - # Параметры квартиры - rooms: int | None = None # 0 = студия - area_m2: float | None = None - floor: int | None = None - total_floors: int | None = None - year_built: int | None = None - house_type: str | None = None - repair_state: str | None = None - has_balcony: bool | None = None - kadastr_num: str | None = None - - # Detail-поля кухни/потолка (#2007). Обычно из detail-страницы (avito_detail, - # cian_detail, yandex_detail), но yandex SERP отдаёт их прямо в карточке — - # промоутим из raw_payload в колонки для дашборда покрытия и matching. - kitchen_area_m2: float | None = None - ceiling_height_m: float | None = None - - # Class B promote-поля (#2008, cian SERP). Для дашборда покрытия / matching, - # НЕ для valuation — estimator их пока НЕ читает (follow-up #2012). cian отдаёт - # их прямо в карточке; раньше лежали только в raw_payload. - mortgage_available: bool | None = None # продавец допускает ипотеку - is_apartments: bool | None = None # апартаменты (НЕ квартира — иной правовой статус) - is_rosreestr_checked: bool | None = None # объявление сверено с ЕГРН - - # Avito house linking (Stage 2a — search → houses) - house_source: str | None = None # 'avito' / 'cian' - house_ext_id: str | None = None # Avito's '3171365' - house_url: str | None = None # full URL of /catalog/houses/... - listing_segment: str | None = None # 'vtorichka' / 'novostroyki' - - # Newbuilding (ЖК) linking — slug-id + canonical ЖК-subdomain URL - newbuilding_id: str | None = None # e.g. 'federatsiya-ekaterinburg' - newbuilding_url: str | None = None # e.g. https://zhk-federatsiya-ekaterinburg.avito.ru - - # Цена (обязательно) - price_rub: int = Field(gt=0) - price_per_m2: int | None = None - - # Геокодинг - # None → точность неизвестна (координаты пришли из скрейпера напрямую). - # 'city' → геокодер вернул city-centroid (нет номера дома в адресе) — - # листинг исключается из radius-аналогов estimator'а. - geo_precision: str | None = None - - # Метаданные - listing_date: date | None = None - publish_date: date | None = None - days_on_market: int | None = None - description: str | None = None - photo_urls: list[str] = Field(default_factory=list) - raw_payload: dict[str, Any] | None = None - - # Cian-specific extensions (Stage 2 of CianScraper v1) - living_area_m2: float | None = None - bedrooms_count: int | None = None - balconies_count: int | None = None - loggias_count: int | None = None - description_minhash: str | None = None - cadastral_number: str | None = None - building_cadastral_number: str | None = None - phones: list[dict] = Field(default_factory=list) - is_homeowner: bool | None = None - is_pro_seller: bool | None = None - bargain_allowed: bool | None = None - sale_type: str | None = None - metro_stations: list[dict] = Field(default_factory=list) - agency_name: str | None = None - - # Yandex gate-API rich fields - yandex_offer_id: str | None = None # numeric string offerId (kept as text) - predicted_price_rub: int | None = None # Yandex per-offer valuation (predictedPrice.value) - predicted_price_min: int | None = None - predicted_price_max: int | None = None - price_trend: str | None = None # INCREASED / DECREASED / UNCHANGED / ... - price_previous_rub: int | None = None # price.previous - - def compute_dedup_hash(self) -> str: - """SHA256(source + stable_key) — стабильный uniqueness key. - - stable_key = source_id (Cian offer_id 330200428, Avito data-item-id) если - есть; иначе source_url БЕЗ query-string. Cian/Avito URL несут волатильный - `?context=...` токен — без strip каждый ре-скрейп давал бы новый hash. - - #753: price_rub в ключ НЕ входит. Смена цены того же объявления — это UPDATE - существующей строки (`ON CONFLICT (dedup_hash) DO UPDATE`), а не новый объект. - Раньше цена была частью ключа → каждое изменение цены порождало дубль - активного listing → искажение asking_to_sold_ratio / sales-vs-listings / - активного инвентаря. Смена формулы разово даёт новый hash существующим - строкам — следующий скрейп upsert'нет их по новому ключу (всплеск разовый). - """ - h = hashlib.sha256() - h.update(self.source.encode()) - h.update(b"|") - key = self.source_id if self.source_id else (self.source_url or "").split("?")[0] - h.update((key or "").encode()) - return h.hexdigest() - - def compute_card_hash(self) -> str: - """SHA-256 over the volatile SERP-card fields. Used to detect whether a - listing's card content changed between scrapes (snapshot-dedup, detail-refresh gate). - Excludes last_seen/scrape timestamps and detail-only fields. - - Хэшируем детерминированную нормализованную репрезентацию полей карточки: - price_rub, area_m2, floor, total_floors, rooms, address (strip+lower, None→""), - len(photo_urls). Порядок частей фиксирован (нет зависимости от порядка - итерации dict) → стабильный hash при неизменной карточке. - """ - address = (self.address or "").strip().lower() - parts = [ - str(self.price_rub), - str(self.area_m2), - str(self.floor), - str(self.total_floors), - str(self.rooms), - address, - str(len(self.photo_urls)), - ] - return hashlib.sha256("|".join(parts).encode()).hexdigest() - - def compute_price_per_m2(self) -> int | None: - """price_per_m2 = price_rub / area_m2 если area задана.""" - if self.area_m2 and self.area_m2 > 0: - return int(self.price_rub / self.area_m2) - return None - - -# ── BaseScraper ───────────────────────────────────────────────────────────── -class BaseScraper(ABC): - """Базовый класс — наследовать для каждого источника. - - Subclass должен реализовать: - - `name` (class attr) — 'avito' / 'cian' / ... - - `_fetch_page(client, ...)` — async GET к источнику - - `_parse_listings(html_or_json, ...)` — html → list[ScrapedLot] - """ - - name: str = "base" - base_url: str = "" - request_delay_sec: float = 5.0 # sleep между запросами (anti-ban) - - def __init__(self) -> None: - self._client: httpx.AsyncClient | None = None - - async def __aenter__(self) -> BaseScraper: - self._client = httpx.AsyncClient( - timeout=20.0, - follow_redirects=True, - headers={ - "User-Agent": random_user_agent(), - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", - "Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8", - "Accept-Encoding": "gzip, deflate, br", - "Connection": "keep-alive", - "Upgrade-Insecure-Requests": "1", - }, - ) - return self - - async def __aexit__(self, *args: Any) -> None: - if self._client is not None: - await self._client.aclose() - - @retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=2, min=2, max=30)) - async def _http_get(self, url: str, **kwargs: Any) -> httpx.Response: - """GET с retry. Subclass-ы используют это вместо raw httpx.""" - assert self._client is not None, "Use scraper as async context manager" - response = await self._client.get(url, **kwargs) - # 403/429 — ретрай поможет (рандомный UA в новом контексте), 5xx тоже - if response.status_code in {403, 429, 502, 503, 504}: - response.raise_for_status() - return response - - async def sleep_between_requests(self) -> None: - """Случайный jitter в районе self.request_delay_sec — чтобы парсер - не выглядел как detstvo-bot.""" - jitter = random.uniform(0.7, 1.5) - await asyncio.sleep(self.request_delay_sec * jitter) - - @abstractmethod - async def fetch_around(self, lat: float, lon: float, radius_m: int = 1000) -> list[ScrapedLot]: - """Главный метод — собрать объявления вокруг (lat, lon) в радиусе radius_m. - - Returns: - Список ScrapedLot. Может быть пустым. - """ - ... - - -# ── Запись пачки результатов в Postgres ───────────────────────────────────── -def save_listings( - db: Session, - lots: list[ScrapedLot], - *, - run_id: int | None = None, - skip_seen_today: bool = False, -) -> tuple[int, int]: - """Пишем list[ScrapedLot] в `listings` с upsert по dedup_hash. - - Дополнительно пишет point-in-time snapshot в listings_snapshots (ON CONFLICT DO UPDATE - — берётся последний за день). Ошибка записи snapshot НЕ прерывает основной INSERT. - - Args: - db: SQLAlchemy session. - lots: список распарсенных объявлений. - run_id: FK scrape_runs.id текущего run'а (опционально; None при ad-hoc вызовах). - skip_seen_today: если True и листинг уже обновлялся сегодня по МСК - (last_seen_at >= начало сегодняшнего дня), то пропустить upsert и snapshot. - Используется full_load для экономии redundant upsert + price-trigger churn - при повторном прогоне в тот же день. Новые листинги (prior_row=None) всегда - вставляются. False = старое поведение (всегда upsert). - - Returns: - (inserted, updated) — counters для логов. - """ - if not lots: - return 0, 0 - - inserted = 0 - updated = 0 - skipped = 0 - reconciled = 0 # UPDATE by (source,source_id) при dedup_hash-дрейфе - matched = 0 - match_failures = 0 - - today_msk = datetime.now(_MSK).date() - - for lot in lots: - ppm2 = lot.price_per_m2 or lot.compute_price_per_m2() - dedup = lot.compute_dedup_hash() - card_hash = lot.compute_card_hash() - - # Pre-read the existing row's card_hash and last_seen_at (keyed by - # dedup_hash) BEFORE the upsert — needed to know the *prior* card - # content and to implement skip_seen_today logic. - # The upsert's RETURNING can only expose the NEW values (EXCLUDED.*), - # never the old ones, so a lightweight SELECT is the clean way. - # None → row did not exist yet (fresh insert) → always process. - prior_row = db.execute( - text("SELECT card_hash, last_seen_at FROM listings WHERE dedup_hash = :dedup"), - {"dedup": dedup}, - ).fetchone() - prior_card_hash = prior_row.card_hash if prior_row is not None else None - - # skip_seen_today: пропускаем листинги уже обновлённые сегодня по МСК. - # Только для existing строк (prior_row is not None); новые всегда вставляются. - if skip_seen_today and prior_row is not None and prior_row.last_seen_at is not None: - ls_msk = prior_row.last_seen_at.astimezone(_MSK).date() - if ls_msk == today_msk: - skipped += 1 - continue - - # ── Per-lot SAVEPOINT: изолируем listings-upsert ──────────────────── - # Когда контент существующего ad дрейфит → новый dedup_hash → INSERT - # находит prior_row=None (SELECT по старому dedup_hash пуст) → INSERT - # летит в UniqueViolation по (source, source_id) при source_id IS NOT NULL - # (constraint 133_listings_uq_source_source_id). Без SAVEPOINT это роняет - # весь прогон. С SAVEPOINT ловим IntegrityError, rollback SAVEPOINT'а, - # и делаем прямой UPDATE по (source, source_id) — reconcile drifted row. - # source_id IS NULL (yandex) → constraint не фаерит, обычный путь. - _upsert_params = { - "source": lot.source, - "source_url": lot.source_url, - "source_id": lot.source_id, - "dedup": dedup, - "address": lot.address, - "lat": lot.lat, - "lon": lot.lon, - "rooms": lot.rooms, - "area_m2": lot.area_m2, - "floor": lot.floor, - "total_floors": lot.total_floors, - "year_built": lot.year_built, - "house_type": lot.house_type, - "repair_state": lot.repair_state, - "has_balcony": lot.has_balcony, - "kitchen_area_m2": lot.kitchen_area_m2, - "ceiling_height_m": lot.ceiling_height_m, - "mortgage_available": lot.mortgage_available, - "is_apartments": lot.is_apartments, - "is_rosreestr_checked": lot.is_rosreestr_checked, - "house_source": lot.house_source, - "house_ext_id": lot.house_ext_id, - "house_url": lot.house_url, - "listing_segment": lot.listing_segment, - "newbuilding_id": lot.newbuilding_id, - "newbuilding_url": lot.newbuilding_url, - "price_rub": lot.price_rub, - "ppm2": ppm2, - "listing_date": lot.listing_date, - "publish_date": lot.publish_date, - "days_on_market": lot.days_on_market, - "description": lot.description, - "photos": _to_json(lot.photo_urls), - "raw": _to_json(lot.raw_payload) if lot.raw_payload else None, - # Cian-specific columns — None for non-Cian scrapers (→ SQL NULL) - "living_area_m2": lot.living_area_m2, - "bedrooms_count": lot.bedrooms_count, - "balconies_count": lot.balconies_count, - "loggias_count": lot.loggias_count, - "description_minhash": lot.description_minhash, - "cadastral_number": lot.cadastral_number, - "building_cadastral_number": lot.building_cadastral_number, - "phones": _to_json(lot.phones) if lot.phones else None, - "is_homeowner": lot.is_homeowner, - "is_pro_seller": lot.is_pro_seller, - "bargain_allowed": lot.bargain_allowed, - "sale_type": lot.sale_type, - "metro_stations": _to_json(lot.metro_stations) if lot.metro_stations else None, - "agency_name": lot.agency_name, - "yandex_offer_id": lot.yandex_offer_id, - "predicted_price_rub": lot.predicted_price_rub, - "predicted_price_min": lot.predicted_price_min, - "predicted_price_max": lot.predicted_price_max, - "price_trend": lot.price_trend, - "price_previous_rub": lot.price_previous_rub, - "geo_precision": lot.geo_precision, - "card_hash": card_hash, - } - - _upsert_sql = text( - """ - INSERT INTO listings ( - source, source_url, source_id, dedup_hash, - address, lat, lon, region_code, - rooms, area_m2, floor, total_floors, year_built, - house_type, repair_state, has_balcony, - kitchen_area_m2, ceiling_height, ceiling_height_m, - mortgage_available, is_apartments, is_rosreestr_checked, - house_source, house_ext_id, house_url, listing_segment, - newbuilding_id, newbuilding_url, - price_rub, price_per_m2, - listing_date, publish_date, days_on_market, description, - photo_urls, raw_payload, - living_area_m2, bedrooms_count, balconies_count, loggias_count, - description_minhash, cadastral_number, building_cadastral_number, - phones, is_homeowner, is_pro_seller, - bargain_allowed, sale_type, metro_stations, agency_name, - yandex_offer_id, predicted_price_rub, - predicted_price_min, predicted_price_max, - price_trend, price_previous_rub, - geo_precision, card_hash, - scraped_at, last_seen_at - ) VALUES ( - :source, :source_url, :source_id, :dedup, - :address, :lat, :lon, 66, - :rooms, :area_m2, :floor, :total_floors, :year_built, - :house_type, :repair_state, :has_balcony, - -- ceiling: один param :ceiling_height_m пишем в ОБЕ колонки — - -- ceiling_height (019, читает coverage-дашборд + yandex_detail/cian_detail) - -- и ceiling_height_m (111, живая avito-колонка). См. #2007. - :kitchen_area_m2, :ceiling_height_m, :ceiling_height_m, - :mortgage_available, :is_apartments, :is_rosreestr_checked, - :house_source, :house_ext_id, :house_url, :listing_segment, - :newbuilding_id, :newbuilding_url, - :price_rub, :ppm2, - :listing_date, :publish_date, :days_on_market, :description, - CAST(:photos AS jsonb), - CAST(:raw AS jsonb), - :living_area_m2, :bedrooms_count, :balconies_count, :loggias_count, - :description_minhash, :cadastral_number, :building_cadastral_number, - CAST(:phones AS jsonb), :is_homeowner, :is_pro_seller, - :bargain_allowed, :sale_type, CAST(:metro_stations AS jsonb), :agency_name, - :yandex_offer_id, :predicted_price_rub, - :predicted_price_min, :predicted_price_max, - :price_trend, :price_previous_rub, - :geo_precision, :card_hash, - NOW(), NOW() - ) - -- Конфликт-арбитр — dedup_hash (sha256(source|source_id)). - -- Для одного (source, source_id) формула даёт ОДИН dedup_hash, - -- поэтому этот апсерт уже не вставит второй раз → согласован с - -- частичным UNIQUE (source, source_id) WHERE source_id IS NOT NULL - -- (133_listings_uq_source_source_id.sql). Менять арбитр на - -- (source, source_id) НЕЛЬЗЯ: yandex/url-only дают source_id=NULL, - -- а NULL не годится как conflict-target → дубли вернулись бы (#1773). - ON CONFLICT (dedup_hash) DO UPDATE - SET last_seen_at = NOW(), - -- #2206: ре-подтверждение живым = свежий скрейп; без bump'а - -- эстиматор (scraped_at > NOW()-14d) терял ~55% живого инвентаря. - scraped_at = NOW(), - is_active = true, - -- если цена изменилась — обновляем - price_rub = EXCLUDED.price_rub, - price_per_m2 = EXCLUDED.price_per_m2, - -- Cian-specific: обновляем при каждом re-scrape - living_area_m2 = EXCLUDED.living_area_m2, - bedrooms_count = EXCLUDED.bedrooms_count, - balconies_count = EXCLUDED.balconies_count, - loggias_count = EXCLUDED.loggias_count, - description_minhash = EXCLUDED.description_minhash, - cadastral_number = EXCLUDED.cadastral_number, - building_cadastral_number = EXCLUDED.building_cadastral_number, - phones = EXCLUDED.phones, - is_homeowner = EXCLUDED.is_homeowner, - is_pro_seller = EXCLUDED.is_pro_seller, - bargain_allowed = EXCLUDED.bargain_allowed, - sale_type = EXCLUDED.sale_type, - metro_stations = EXCLUDED.metro_stations, - listing_date = COALESCE(EXCLUDED.listing_date, listings.listing_date), - area_m2 = COALESCE(EXCLUDED.area_m2, listings.area_m2), - -- kitchen/ceiling (#2007): COALESCE — SERP re-scrape источника без - -- этих полей (avito SERP → NULL) НЕ затирает detail-enriched значение - -- (avito_detail пишет ceiling_height_m отдельным UPDATE). - kitchen_area_m2 = COALESCE( - EXCLUDED.kitchen_area_m2, listings.kitchen_area_m2 - ), - ceiling_height = COALESCE( - EXCLUDED.ceiling_height, listings.ceiling_height - ), - ceiling_height_m = COALESCE( - EXCLUDED.ceiling_height_m, listings.ceiling_height_m - ), - -- Class B promote (#2008, cian SERP): COALESCE — re-scrape - -- источника без поля (avito/yandex SERP → NULL) НЕ затирает - -- значение, ранее записанное cian-ом / detail-ом. - mortgage_available = COALESCE( - EXCLUDED.mortgage_available, listings.mortgage_available - ), - is_apartments = COALESCE( - EXCLUDED.is_apartments, listings.is_apartments - ), - is_rosreestr_checked = COALESCE( - EXCLUDED.is_rosreestr_checked, listings.is_rosreestr_checked - ), - -- Yandex rich fields (+ shared description/agency/publish_date): - -- COALESCE so a source that does not provide them never wipes - -- a value previously written by another source / scrape. - publish_date = COALESCE(EXCLUDED.publish_date, listings.publish_date), - days_on_market = COALESCE( - EXCLUDED.days_on_market, listings.days_on_market - ), - description = COALESCE(EXCLUDED.description, listings.description), - agency_name = COALESCE(EXCLUDED.agency_name, listings.agency_name), - yandex_offer_id = COALESCE( - EXCLUDED.yandex_offer_id, listings.yandex_offer_id - ), - predicted_price_rub = COALESCE( - EXCLUDED.predicted_price_rub, listings.predicted_price_rub - ), - predicted_price_min = COALESCE( - EXCLUDED.predicted_price_min, listings.predicted_price_min - ), - predicted_price_max = COALESCE( - EXCLUDED.predicted_price_max, listings.predicted_price_max - ), - price_trend = COALESCE(EXCLUDED.price_trend, listings.price_trend), - price_previous_rub = COALESCE( - EXCLUDED.price_previous_rub, listings.price_previous_rub - ), - newbuilding_id = COALESCE( - EXCLUDED.newbuilding_id, listings.newbuilding_id - ), - newbuilding_url = COALESCE( - EXCLUDED.newbuilding_url, listings.newbuilding_url - ), - card_hash = EXCLUDED.card_hash - RETURNING id, (xmax = 0) AS inserted - """ - ) - - # ── Per-lot SAVEPOINT вокруг listings-upsert ──────────────────────── - # Когда контент существующего ad дрейфит → новый dedup_hash → INSERT - # не находит existing row по dedup_hash → пробует fresh INSERT → летит - # UniqueViolation по (source, source_id) WHERE source_id IS NOT NULL - # (constraint listings_source_source_id_uq). Без SAVEPOINT это переводит - # весь Session в InFailedSqlTransaction → весь прогон падает. - # С SAVEPOINT ловим IntegrityError/UniqueViolation, делаем rollback, - # и применяем UPDATE by (source, source_id) — reconcile drifted row. - # source_id IS NULL (yandex) → constraint не фаерит, обычный upsert путь. - result = None - listing_id: int | None = None - try: - with db.begin_nested(): - result = db.execute(_upsert_sql, _upsert_params).fetchone() - except IntegrityError as ie: - # Проверяем что это UniqueViolation именно по (source, source_id). - if lot.source_id is not None and isinstance(ie.orig, psycopg.errors.UniqueViolation): - logger.warning( - "save_listings:dedup_hash_drift source=%s source_id=%s " - "— reconcile UPDATE by (source, source_id)", - lot.source, - lot.source_id, - ) - # SAVEPOINT уже откачен; session чистая — применяем прямой UPDATE. - rec_row = db.execute( - text( - """ - UPDATE listings - SET dedup_hash = :dedup, - last_seen_at = NOW(), - -- #2206: ре-подтверждение живым = свежий скрейп; - -- без bump'а эстиматор терял ~55% живого инвентаря. - scraped_at = NOW(), - is_active = true, - price_rub = :price_rub, - price_per_m2 = :ppm2, - living_area_m2 = :living_area_m2, - bedrooms_count = :bedrooms_count, - balconies_count = :balconies_count, - loggias_count = :loggias_count, - description_minhash = :description_minhash, - cadastral_number = :cadastral_number, - building_cadastral_number = :building_cadastral_number, - phones = CAST(:phones AS jsonb), - is_homeowner = :is_homeowner, - is_pro_seller = :is_pro_seller, - bargain_allowed = :bargain_allowed, - sale_type = :sale_type, - metro_stations = CAST(:metro_stations AS jsonb), - listing_date = COALESCE(:listing_date, listing_date), - area_m2 = COALESCE(:area_m2, area_m2), - kitchen_area_m2 = COALESCE(:kitchen_area_m2, kitchen_area_m2), - ceiling_height = COALESCE(:ceiling_height_m, ceiling_height), - ceiling_height_m = COALESCE(:ceiling_height_m, ceiling_height_m), - mortgage_available = COALESCE( - :mortgage_available, mortgage_available - ), - is_apartments = COALESCE(:is_apartments, is_apartments), - is_rosreestr_checked = COALESCE( - :is_rosreestr_checked, is_rosreestr_checked - ), - publish_date = COALESCE(:publish_date, publish_date), - days_on_market = COALESCE(:days_on_market, days_on_market), - description = COALESCE(:description, description), - agency_name = COALESCE(:agency_name, agency_name), - yandex_offer_id = COALESCE(:yandex_offer_id, yandex_offer_id), - predicted_price_rub = COALESCE( - :predicted_price_rub, predicted_price_rub - ), - predicted_price_min = COALESCE( - :predicted_price_min, predicted_price_min - ), - predicted_price_max = COALESCE( - :predicted_price_max, predicted_price_max - ), - price_trend = COALESCE(:price_trend, price_trend), - price_previous_rub = COALESCE( - :price_previous_rub, price_previous_rub - ), - newbuilding_id = COALESCE(:newbuilding_id, newbuilding_id), - newbuilding_url = COALESCE(:newbuilding_url, newbuilding_url), - card_hash = :card_hash - WHERE source = :source - AND source_id = :source_id - RETURNING id - """ - ), - _upsert_params, - ).fetchone() - if rec_row is not None: - listing_id = int(rec_row.id) - reconciled += 1 - else: - logger.error( - "save_listings:reconcile_failed source=%s source_id=%s " - "— UPDATE returned no row (unexpected)", - lot.source, - lot.source_id, - ) - else: - logger.error( - "save_listings:integrity_error source=%s source_id=%s: %s", - lot.source, - lot.source_id, - ie, - ) - raise - else: - if result is not None: - listing_id = int(result.id) - if result.inserted: - inserted += 1 - else: - updated += 1 - - # ── Snapshot: point-in-time observation in listings_snapshots ─── - # Fault-tolerant: failure here MUST NOT abort the listings batch. - # ON CONFLICT DO UPDATE — самый последний snapshot за день. - # - # Snapshot-dedup: пропускаем запись только когда МЫ УВЕРЕНЫ что карточка - # не изменилась (prior_card_hash == card_hash, row существовал) И snapshot - # за сегодня уже есть. Принцип "when in doubt — write": лишний snapshot - # неизменной карточки безвреден, ПРОПУЩЕННЫЙ snapshot реального изменения — - # недопустим. Поэтому equality-проверки мало (первый скрейп за день мог - # ещё не записать сегодняшний snapshot) — дополнительно подтверждаем - # наличие сегодняшней строки лёгким EXISTS. - if listing_id is not None: - skip_snapshot = False - if prior_card_hash is not None and prior_card_hash == card_hash: - today_row = db.execute( - text( - "SELECT 1 FROM listings_snapshots " - "WHERE listing_id = CAST(:lid AS bigint) " - " AND snapshot_date = CURRENT_DATE" - ), - {"lid": listing_id}, - ).fetchone() - skip_snapshot = today_row is not None - if skip_snapshot: - logger.debug( - "save_listings:snapshot_skipped (card unchanged) " "source=%s listing_id=%s", - lot.source, - listing_id, - ) - if listing_id is not None and not skip_snapshot: - try: - with db.begin_nested(): - upsert_listing_snapshot( - db, - listing_id=listing_id, - price_rub=lot.price_rub, - price_per_m2=ppm2, - run_id=run_id, - status="active", - ) - except Exception as e: - logger.warning( - "save_listings:snapshot_failed source=%s listing_id=%s: %s", - lot.source, - listing_id, - e, - ) - - # ── Hook: link listing → house via matching service ───────────── - # Idempotent (UPSERT on listing_sources UNIQUE(ext_source, ext_id)) - # and fault-tolerant: failure here MUST NOT abort the listings batch. - # Wrapped in SAVEPOINT so a failed match only rolls back its own work, - # not the surrounding INSERT (see .claude/rules/backend.md SAVEPOINT). - if listing_id is not None: - try: - with db.begin_nested(): - _link_listing_to_house(db, listing_id, lot) - matched += 1 - except Exception as e: - # Best-effort hook: log and continue so the listings batch isn't aborted. - match_failures += 1 - logger.warning( - "save_listings:match_failed source=%s source_id=%s listing_id=%s: %s", - lot.source, - lot.source_id, - listing_id, - e, - ) - - db.commit() - logger.info( - "save_listings: source=%s inserted=%d updated=%d reconciled=%d " - "skipped_seen_today=%d matched=%d match_failures=%d (total %d)", - lots[0].source if lots else "?", - inserted, - updated, - reconciled, - skipped, - matched, - match_failures, - len(lots), - ) - return inserted, updated - - -def _to_json(value: Any) -> str: - """JSON serialization helper — для jsonb колонок.""" - import json - - return json.dumps(value, ensure_ascii=False, default=str) - - -def _link_listing_to_house(db: Session, listing_id: int, lot: ScrapedLot) -> None: - """Hook scraped listing into matching service: resolve house, upsert listing_sources. - - Steps: - 1. match_or_create_house() — find or create canonical houses row for this - listing's address/coords. Uses Tier 0-3 matching (cadastr → source_exact → - fingerprint → geo_proximity → new). - 2. upsert_listing_source() — register (ext_source, ext_id) → listing_id in - listing_sources. Idempotent via UNIQUE (ext_source, ext_id). - - The matching method recorded is 'source_link': scraper already deduped by - `dedup_hash` (INSERT … ON CONFLICT DO UPDATE), so this is a direct registration - rather than a fuzzy listing-level match. - - ext_id source: lot.source_id if present, else dedup_hash (Yandex without - stable source_id falls back to URL-based dedup_hash — same hash on re-scrape). - - Skips silently if: - - lot has no source_id AND no address/lat/lon (cannot match house anyway) - - Raises on DB errors — caller wraps in try/except + SAVEPOINT. - """ - ext_id = lot.source_id or lot.compute_dedup_hash() - - # House resolution: needs at least address or (lat, lon). Skip otherwise — - # listing_sources requires listing_id but not house linkage, so still upsert. - house_id: int | None = None - if lot.address or (lot.lat is not None and lot.lon is not None): - # Use house_source/house_ext_id when scraper extracted them (Avito Houses - # catalog link, Cian newbuilding). Falls back to per-listing identity - # so each unique listing creates its own row if no canonical house exists. - h_src = lot.house_source or lot.source - h_ext = lot.house_ext_id or ext_id - # No inner begin_nested here — the caller (save_listings) already wraps this - # entire function in a per-row SAVEPOINT (backend.md SAVEPOINT pattern). - # A second nested SAVEPOINT inside would be redundant: if match_or_create_house - # raises a DB error, that error propagates out of _link_listing_to_house, - # the caller's SAVEPOINT rolls back only the hook work (not the listings INSERT), - # and the caller logs a match_failed warning. One SAVEPOINT level is sufficient. - house_id, _conf, _method = match_or_create_house( - db, - ext_source=h_src, - ext_id=h_ext, - address=lot.address, - lat=lot.lat, - lon=lot.lon, - year_built=lot.year_built, - building_cadastral_number=lot.building_cadastral_number, - cadastral_number=lot.cadastral_number or lot.kadastr_num, - source_url=lot.house_url or lot.source_url, - ) - - # Mirror the resolved house into listings.house_id_fk so direct - # `JOIN houses ON house_id_fk = id` keeps working on freshly scraped / - # re-scraped active rows. Without this the FK is populated only by the - # offline backfill (063_*.sql / backfill_listing_sources.py), leaving - # new active listings unlinked (avito 0/444, cian ~7%). Mirrors the - # backfill script's column/param style. IS DISTINCT FROM avoids a no-op - # write when the linkage is already correct. - if house_id is not None: - db.execute( - text( - "UPDATE listings " - " SET house_id_fk = CAST(:hid AS bigint) " - " WHERE id = CAST(:lid AS bigint) " - " AND house_id_fk IS DISTINCT FROM CAST(:hid AS bigint)" - ), - {"hid": house_id, "lid": listing_id}, - ) - - upsert_listing_source( - db, - listing_id=listing_id, - ext_source=lot.source, - ext_id=str(ext_id), - method="source_link", - confidence=1.0, - price_rub=lot.price_rub, - area_m2=lot.area_m2, - floor=lot.floor, - rooms_count=lot.rooms, - source_url=lot.source_url, - source_data={"house_id": house_id} if house_id else None, - ) diff --git a/tradein-mvp/backend/app/services/scrapers/browser_fetcher.py b/tradein-mvp/backend/app/services/scrapers/browser_fetcher.py deleted file mode 100644 index 77c8a5ab..00000000 --- a/tradein-mvp/backend/app/services/scrapers/browser_fetcher.py +++ /dev/null @@ -1,262 +0,0 @@ -"""browser_fetcher.py — HTTP-клиент к tradein-browser сервису (#884/#905). - -Тонкий клиент над tradein-browser контейнером, который запускает AsyncCamoufox -локально и экспонирует HTTP API (POST /fetch). - -Публичный интерфейс не изменился: - - async with BrowserFetcher() as fetcher: - html = await fetcher.fetch("https://example.com") - -Внутреннее устройство: httpx.AsyncClient + POST к settings.browser_http_endpoint. -Recycle, crash-recovery и управление браузером живут на стороне сервера (server.py). -При HTTPError / ConnectError делает одну повторную попытку после короткой паузы, -затем пробрасывает исключение. - -Архитектура выбрана потому, что playwright WS-сервер (launch_server) несовместим -с playwright >=1.45: ``browserServerImpl.js`` отсутствует → MODULE_NOT_FOUND. -Локальный AsyncCamoufox + HTTP — работающая альтернатива. -""" - -from __future__ import annotations - -import asyncio -import logging - -import httpx - -logger = logging.getLogger(__name__) - -_RETRY_SLEEP_S: float = 1.0 -_HTTP_TIMEOUT_S: float = 120.0 # навигация медленная → щедрый таймаут - - -class BrowserFetcher: - """Async context manager: HTTP-клиент к tradein-browser HTTP-сервису. - - Использование:: - - async with BrowserFetcher() as fetcher: - html = await fetcher.fetch("https://example.com") - """ - - def __init__(self, source: str = "avito", fetch_timeout_s: float = _HTTP_TIMEOUT_S) -> None: - # source — логический источник ("avito"/"cian"/"yandex"/"domclick"). Сервер - # роутит /fetch по нему на отдельный браузер+прокси, когда включён - # FEATURE_BROWSER_POOL_ENABLED (Phase 1). При выключенном флаге source - # игнорируется — поведение не меняется. - # fetch_timeout_s — таймаут httpx-клиента для POST /fetch. Yandex-путь передаёт - # 30s чтобы один проблемный combo занимал ≤30s×retries вместо 120s×retries. - self._source = source - self._fetch_timeout_s = fetch_timeout_s - self._client: httpx.AsyncClient | None = None - self._endpoint: str | None = None - - # ── lifecycle ────────────────────────────────────────────────────────────── - - async def __aenter__(self) -> BrowserFetcher: - from app.core.config import settings - - self._endpoint = settings.browser_http_endpoint - self._client = httpx.AsyncClient(timeout=self._fetch_timeout_s) - logger.info("BrowserFetcher: клиент создан, endpoint=%s", self._endpoint) - return self - - async def __aexit__(self, *_: object) -> None: - if self._client is not None: - await self._client.aclose() - self._client = None - logger.debug("BrowserFetcher: клиент закрыт") - - # ── public API ───────────────────────────────────────────────────────────── - - async def fetch(self, url: str) -> str: - """Запрашивает HTML страницы через tradein-browser HTTP-сервис. - - При HTTPError или ConnectError делает одну повторную попытку после - короткой паузы. Остальные исключения всплывают к вызывающему коду. - - Returns: - Полный HTML-контент страницы. - """ - assert self._client is not None, "BrowserFetcher: используй как async context manager" - assert self._endpoint is not None, "BrowserFetcher: endpoint не задан" - - try: - return await self._post_fetch(url) - except (httpx.HTTPError, httpx.TransportError) as exc: - logger.warning( - "BrowserFetcher: ошибка запроса (%s), retry через %.1fs: %s", - type(exc).__name__, - _RETRY_SLEEP_S, - url, - ) - await asyncio.sleep(_RETRY_SLEEP_S) - return await self._post_fetch(url) - - async def fetch_json( - self, - url: str, - *, - method: str = "GET", - headers: dict[str, str] | None = None, - body: str | None = None, - origin: str | None = None, - ) -> dict: - """In-page fetch() через sidecar /fetch-json. Возвращает {"status": int, "body": str}. - - body — уже сериализованная строка (caller делает json.dumps для POST). - origin — same-origin страница, на которую перейдёт камуфокс перед fetch. - - При HTTPError / TransportError делает одну повторную попытку после короткой - паузы. Остальные исключения всплывают к вызывающему коду. - """ - assert self._client is not None, "BrowserFetcher: используй как async context manager" - assert self._endpoint is not None, "BrowserFetcher: endpoint не задан" - - try: - return await self._post_fetch_json(url, method, headers, body, origin) - except (httpx.HTTPError, httpx.TransportError) as exc: - logger.warning( - "BrowserFetcher: ошибка fetch-json запроса (%s), retry через %.1fs: %s", - type(exc).__name__, - _RETRY_SLEEP_S, - url, - ) - await asyncio.sleep(_RETRY_SLEEP_S) - return await self._post_fetch_json(url, method, headers, body, origin) - - async def login( - self, - *, - url: str, - email: str, - password: str, - email_selector: str, - password_selector: str, - submit_selector: str, - success_cookie: str, - pre_click_selectors: list[str] | None = None, - wait_ms: int | None = None, - ) -> dict[str, str]: - """Логинится через tradein-browser /login и возвращает cookies как dict name→value. - - Использует увеличенный таймаут (60s) — логин медленнее обычного fetch. - При HTTPError / TransportError делает одну повторную попытку. - - Returns: - Плоский словарь {cookie_name: cookie_value}. - """ - assert self._client is not None, "BrowserFetcher: используй как async context manager" - assert self._endpoint is not None, "BrowserFetcher: endpoint не задан" - - body: dict[str, object] = { - "url": url, - "email": email, - "password": password, - "email_selector": email_selector, - "password_selector": password_selector, - "submit_selector": submit_selector, - "success_cookie": success_cookie, - "pre_click_selectors": pre_click_selectors or [], - } - if wait_ms is not None: - body["wait_ms"] = wait_ms - - try: - return await self._post_login(body) - except (httpx.HTTPError, httpx.TransportError) as exc: - logger.warning( - "BrowserFetcher: ошибка login-запроса (%s), retry через %.1fs", - type(exc).__name__, - _RETRY_SLEEP_S, - ) - await asyncio.sleep(_RETRY_SLEEP_S) - return await self._post_login(body) - - # ── internal ─────────────────────────────────────────────────────────────── - - async def _post_fetch(self, url: str) -> str: - """Один HTTP POST к /fetch эндпоинту сервиса.""" - assert self._client is not None - assert self._endpoint is not None - - resp = await self._client.post( - f"{self._endpoint}/fetch", - json={"url": url, "source": self._source}, - ) - resp.raise_for_status() - data: dict[str, str] = resp.json() - html = data["html"] - logger.debug("BrowserFetcher: fetch OK url=%r html_len=%d", url, len(html)) - return html - - async def _post_fetch_json( - self, - url: str, - method: str, - headers: dict[str, str] | None, - body: str | None, - origin: str | None, - ) -> dict: - """Один HTTP POST к /fetch-json эндпоинту сервиса.""" - assert self._client is not None - assert self._endpoint is not None - - resp = await self._client.post( - f"{self._endpoint}/fetch-json", - json={ - "url": url, - "source": self._source, - "method": method, - "headers": headers or {}, - "body": body, - "origin": origin, - }, - ) - resp.raise_for_status() - data: dict = resp.json() - # Defensive: контракт сервера — {"status": int, "body": str} (зеркалит как - # _post_fetch читает data["html"]). Если ключи пропали (несовместимый сервер / - # прокинутый error-payload) — падаем с понятной ошибкой, а не KeyError ниже по - # стеку у адаптера, который ждёт r["status"]/r["body"]. - if "status" not in data or "body" not in data: - raise RuntimeError( - f"BrowserFetcher: /fetch-json вернул некорректный ответ " - f"(нет 'status'/'body'): keys={sorted(data)} url={url!r}" - ) - logger.debug( - "BrowserFetcher: fetch-json OK url=%r status=%s body_len=%d", - url, - data.get("status"), - len(data.get("body") or ""), - ) - return data - - async def _post_login(self, body: dict[str, object]) -> dict[str, str]: - """Один HTTP POST к /login эндпоинту сервиса.""" - assert self._client is not None - assert self._endpoint is not None - - resp = await self._client.post( - f"{self._endpoint}/login", - json=body, - timeout=httpx.Timeout(60.0), - ) - if resp.status_code == 502: - data = resp.json() - has_screenshot = bool(data.get("screenshot_b64")) - logger.debug( - "BrowserFetcher: login 502 — has_screenshot=%s page_url=%r", - has_screenshot, - data.get("page_url"), - ) - raise RuntimeError( - f"browser login failed: {data.get('error')} url={data.get('page_url')}" - ) - resp.raise_for_status() - data = resp.json() - raw: list[dict[str, object]] = data["cookies"] - result = {str(c["name"]): str(c["value"]) for c in raw} - logger.info("BrowserFetcher: login OK cookie_count=%d", len(result)) - return result diff --git a/tradein-mvp/backend/app/services/scrapers/cian_newbuilding.py b/tradein-mvp/backend/app/services/scrapers/cian_newbuilding.py deleted file mode 100644 index 4e12f0e0..00000000 --- a/tradein-mvp/backend/app/services/scrapers/cian_newbuilding.py +++ /dev/null @@ -1,788 +0,0 @@ -"""Cian.ru newbuilding (ЖК) catalog page scraper. - -URL: https://zhk-<slug>-<city>-i.cian.ru/ -MFE: 'newbuilding-card-desktop-frontend', key: 'initialState' - -Sister state containers extracted from same MFE initialState top-level keys: -- realtyValuation: 7-month price chart (data.priceDynamics.chart.data.{labels,values}) - → houses_price_dynamics -- reliabilityState.reliability: наш.дом.рф checkStatus + details[] → house_reliability_checks -- offers: grouped by roomType (fromDeveloperRooms[].layouts[]) -- reviews: house reviews list -- housesByTurn: корпуса по очередям (stored as jsonb on houses) - -Stage 6 of CianScraper v1. -""" - -from __future__ import annotations - -import json -import logging -import re -from dataclasses import dataclass, field -from typing import Any - -from curl_cffi.requests import AsyncSession # type: ignore[import-untyped] - -from app.core.config import settings -from app.services.scrapers.browser_fetcher import BrowserFetcher -from app.services.scrapers.cian_state_parser import extract_all_states, extract_state - -logger = logging.getLogger(__name__) - -# Canonical ЖК-slug subdomain URL embedded in any Cian SERP/page that references a -# newbuilding (e.g. https://zhk-parkovyy-kvartal-ekb-i.cian.ru). The cat.php SERP for a -# single newobject[] id puts this slug in its title block — extracting it is the WORKING -# nb_id → ЖК-url resolution (#972; see resolve_cian_zhk_url_via_search). -_ZHK_SLUG_URL_RE = re.compile(r"https://zhk-[a-z0-9-]+\.cian\.ru") - -# Title-anchored extraction (#972 hardening). The cat.php newbuilding-SERP renders the -# canonical ЖК-slug of the QUERIED newobject as the <a> href inside its page <h1>: -# -# <h1 data-name="Title" ...>Купить квартиру в -# <a href="https://zhk-<slug>.cian.ru/" ...>ЖК «...»</a></h1> -# -# (verified on the live ~2.8 MB prod SERP for nb 48853 — the header sits inside a -# `NewbuildingHeaderSeo` block; there is NO <link rel=canonical> / og:url on this page -# type, so the H1 anchor is the only reliable canonical carrier). Anchoring on it avoids -# the WRONG-ЖК hazard of a naive first-match: cat.php pages also carry banner / "похожие -# ЖК" / recommendation blocks that can embed OTHER zhk-*.cian.ru urls, and those blocks -# may render BEFORE the title — a plain first-match could then resolve a foreign ЖК and -# silently enrich the house with another building's data. -# -# Matches the FIRST zhk-slug url that occurs inside the page's <h1 data-name="Title"> … -# </h1>. `[^>]*` keeps the attr scan within the opening tag; `.*?` (DOTALL) spans the -# inner markup up to the slug href. Falls back to naive first-match only when this finds -# nothing (markup drift), preserving the previous behaviour as a safety net. -_ZHK_TITLE_ANCHOR_RE = re.compile( - r'<h1[^>]*data-name="Title"[^>]*>.*?(https://zhk-[a-z0-9-]+\.cian\.ru)', - re.DOTALL, -) - -# cat.php newbuilding-SERP template: one row per `newobject[0]=<nb_id>`. ekb. host is the -# Свердловская-обл. mirror tradein targets; the slug URL it returns is host-agnostic -# (the canonical zhk-*.cian.ru subdomain), so this resolves ЕКБ newbuildings correctly. -_CATPH_NEWBUILDING_SERP = ( - "https://ekb.cian.ru/cat.php?deal_type=sale&engine_version=2" - "&offer_type=flat&object_type[0]=2&newobject[0]={nb_id}" -) - - -@dataclass -class NewbuildingEnrichment: - """Result of fetch_newbuilding() — houses row + sister containers.""" - - # Core house metadata (from state.newbuilding) - cian_internal_house_id: int | None = None - # ZHK page URL — canonical slug form stored after fetch / redirect resolution - cian_zhk_url: str | None = None - name: str | None = None - address: str | None = None - year_built: int | None = None - deadline_year: int | None = None - floors_count_min: int | None = None - floors_count_max: int | None = None - building_class: str | None = None # 'comfort' / 'business' / 'premium' - - # Management - management_company: dict[str, Any] | None = None # name, phones[], email, ... - - # Aggregated metrics - transport_accessibility_rate: int | None = None - advantages: list[dict[str, Any]] = field(default_factory=list) - - # Time-series price chart (Stage 6 main deliverable) - # Each entry: {'month_date': str, 'room_count': str, 'prices_type': str, 'period': str, - # 'price_per_sqm': float | None} - realty_valuation_chart: list[dict[str, Any]] = field(default_factory=list) - - # Reliability (наш.дом.рф) — overall status + checks detail array - # {check_status: str, check_name: str, details: list[dict]} - reliability_checks: list[dict[str, Any]] = field(default_factory=list) - - # Builders / banks (related orgs) - builders: list[dict[str, Any]] = field(default_factory=list) - banks: list[dict[str, Any]] = field(default_factory=list) - - # Corpuses (houses_by_turn — корпуса по очередям) - houses_by_turn: list[dict[str, Any]] = field(default_factory=list) - - # Reviews - reviews: list[dict[str, Any]] = field(default_factory=list) - - # Mini-SERP offers (grouped by roomType, layouts) - nested_offers: list[dict[str, Any]] = field(default_factory=list) - - # Raw payloads для debugging - raw_state: dict[str, Any] | None = None - raw_sister_states: dict[str, Any] = field(default_factory=dict) - - -async def fetch_newbuilding( - zhk_url: str, - *, - session: AsyncSession | None = None, # kept for backward-compat; unused for page fetch -) -> NewbuildingEnrichment | None: - """Fetch ЖК catalog page и extract все containers. - - HTML страница ЖК теперь тянется через BrowserFetcher (camoufox), потому что - curl_cffi получает страницу без initialState под анти-ботом Cian (#972). - tradein-browser верифицирован живым: 1.17 MB, initialState присутствует. - - Args: - zhk_url: e.g. 'https://zhk-ekaterininskiy-park-ekb-i.cian.ru/' - session: зарезервирован для обратной совместимости с вызывающими; - больше не используется для fetch страницы ЖК. Resolve-функции - (resolve_cian_zhk_url_via_search) по-прежнему используют curl_cffi. - - Returns: NewbuildingEnrichment, or None если fetch / parse failed. - """ - async with BrowserFetcher(source="cian") as browser: - html = await browser.fetch(zhk_url) - - # Cian ЖК-карточка: MFE 'newbuilding-card-desktop-frontend', key 'initialState' - # (verified live 2026-06-15). - mfe = "newbuilding-card-desktop-frontend" - - nb_state = extract_state(html, mfe=mfe, key="initialState") - if nb_state is None: - logger.warning("Cian newbuilding %s: initialState extraction failed", zhk_url) - return None - - # Primary newbuilding object: state.newbuilding (60 fields per schema sec 15.4) - nb = nb_state.get("newbuilding") or {} - if not isinstance(nb, dict): - nb = {} - - result = NewbuildingEnrichment( - cian_internal_house_id=nb.get("id"), - cian_zhk_url=zhk_url, - name=nb.get("name") or nb.get("displayName"), - address=_extract_address(nb), - year_built=nb.get("yearBuilt") or nb.get("buildYear"), - deadline_year=_extract_deadline_year(nb), - floors_count_min=nb.get("floorsCountMin"), - floors_count_max=nb.get("floorsCountMax"), - building_class=nb.get("newbuildingClassName") or nb.get("buildingClass"), - management_company=nb.get("managementCompany"), - transport_accessibility_rate=_extract_transport_rate(nb), - advantages=_extract_advantages(nb), - builders=_safe_list(nb.get("builders")), - banks=_safe_list(nb.get("banks")), - houses_by_turn=_safe_list(nb.get("housesByTurn")), - raw_state=nb_state, - ) - - # Sister containers — all live under the same MFE initialState top-level keys - # (not separate push() entries) per schema sec 15.3 - all_states = extract_all_states(html) - mfe_states = all_states.get(mfe, {}) - # initialState top-level keys contain sister data: realtyValuation, reliability, offers - top = nb_state - - # realtyValuation 7-month chart (Stage 6 main deliverable) - # Located at initialState.realtyValuation (sec 15.5) - rv_state = top.get("realtyValuation") or mfe_states.get("realtyValuation") - if rv_state and isinstance(rv_state, dict): - result.realty_valuation_chart = _extract_chart(rv_state) - result.raw_sister_states["realtyValuation"] = rv_state - - # reliability (наш.дом.рф) — initialState.reliabilityState.reliability - # ({checkStatus, details[], banner, actions}, verified live 2026-06-15). - rel_state = (top.get("reliabilityState") or {}).get("reliability") - if rel_state and isinstance(rel_state, dict): - result.reliability_checks = _extract_reliability_checks(rel_state) - result.raw_sister_states["reliability"] = rel_state - - # reviews — located at initialState or separate state - reviews_state = top.get("reviews") or mfe_states.get("reviews") - if reviews_state and isinstance(reviews_state, dict): - items = reviews_state.get("items") or reviews_state.get("data") or [] - result.reviews = items if isinstance(items, list) else [] - result.raw_sister_states["reviews"] = reviews_state - - # offers (grouped by roomType) — located at initialState.offers (sec 15.7) - offers_state = top.get("offers") or mfe_states.get("offers") or mfe_states.get("miniSerp") - if offers_state and isinstance(offers_state, dict): - result.nested_offers = _extract_nested_offers(offers_state) - result.raw_sister_states["offers"] = offers_state - - logger.info( - "Cian newbuilding %s parsed: id=%s chart=%d reliability=%d offers=%d", - zhk_url, - result.cian_internal_house_id, - len(result.realty_valuation_chart), - len(result.reliability_checks), - len(result.nested_offers), - ) - return result - - -# ---- private extractors ---- - - -def _safe_list(v: Any) -> list[dict[str, Any]]: - """Return v as list[dict] or empty list.""" - if not isinstance(v, list): - return [] - return [item for item in v if isinstance(item, dict)] - - -def _extract_address(nb: dict[str, Any]) -> str | None: - """Extract display address string from newbuilding object.""" - addr = nb.get("fullAddress") or nb.get("address") - if isinstance(addr, str): - return addr - if isinstance(addr, dict): - return addr.get("full") or addr.get("display") or addr.get("name") - return None - - -def _extract_deadline_year(nb: dict[str, Any]) -> int | None: - """Extract planned completion year from newbuilding.deadline or similar field.""" - dl = nb.get("deadline") or nb.get("buildingDeadline") - if isinstance(dl, dict): - return dl.get("year") - if isinstance(dl, int): - return dl - return None - - -def _extract_transport_rate(nb: dict[str, Any]) -> int | None: - """Extract transport accessibility rate as int|None. - - Cian payload drift (#972): `transportAccessibilityRate` is now a nested dict - ({'transportAccessibility': <int|null>}) rather than a bare int. Older pages - returned a plain int. Handle both — returning a dict here would violate the - NewbuildingEnrichment.transport_accessibility_rate: int|None contract and break - the CAST(:tar AS ...) bind in save_newbuilding_enrichment (psycopg cannot adapt - a dict). Anything non-int collapses to None. - """ - raw = nb.get("transportAccessibilityRate") - if isinstance(raw, dict): - raw = raw.get("transportAccessibility") or raw.get("rate") or raw.get("value") - if isinstance(raw, bool): # bool is an int subclass — exclude explicitly - return None - if isinstance(raw, int): - return raw - if isinstance(raw, float): - return int(raw) - return None - - -def _extract_advantages(nb: dict[str, Any]) -> list[dict[str, Any]]: - """Extract typed advantages list from newbuilding.advantages.""" - raw = nb.get("advantages") or nb.get("advantagesTyped") or [] - if not isinstance(raw, list): - return [] - return [a for a in raw if isinstance(a, dict)] - - -def _extract_chart(rv_state: dict[str, Any]) -> list[dict[str, Any]]: - """Extract realtyValuation time-series chart points. - - Actual Cian shape (sec 15.5): - rv_state.data.priceDynamics.chart.data = {labels: [date...], values: [int...]} - - Falls back to alternative shapes if structure varies. - Each output point: - {month_date: str, room_count: str, prices_type: str, period: str, price_per_sqm: float|None} - - Note: The initial page state provides aggregate (all room types) chart only. - Per-room-type data requires XHR requests (not done here). - """ - points: list[dict[str, Any]] = [] - - # Primary shape: data.priceDynamics.chart.data.{labels, values} - data_block = rv_state.get("data") - if isinstance(data_block, dict): - price_dynamics = data_block.get("priceDynamics") or {} - if isinstance(price_dynamics, dict): - chart = price_dynamics.get("chart") or {} - if isinstance(chart, dict): - chart_data = chart.get("data") or {} - if isinstance(chart_data, dict): - labels = chart_data.get("labels") or [] - values = chart_data.get("values") or [] - if isinstance(labels, list) and isinstance(values, list): - for label, value in zip(labels, values, strict=False): - if label is None or value is None: - continue - points.append( - { - "month_date": str(label)[:10], # YYYY-MM-DD - "room_count": "all", # aggregate (initial state) - "prices_type": "price", # total price (not priceSqm) - "period": "halfYear", # default period shown - "price_per_sqm": None, # chart shows total price, not sqm - } - ) - # Store raw value as price_per_sqm placeholder for DB - # (actual sqm price would require area context) - # Use the value as-is; caller / downstream can enrich - points[-1]["price_per_sqm"] = ( - float(value) if isinstance(value, int | float) else None - ) - if points: - return points - - # Fallback shape A: rv_state.chart (flat list with {month/date, price, roomType}) - chart_flat = rv_state.get("chart") or rv_state.get("prices") or rv_state.get("dynamics") - if isinstance(chart_flat, list): - for entry in chart_flat: - if not isinstance(entry, dict): - continue - month = entry.get("month") or entry.get("date") or entry.get("monthDate") - price = entry.get("price") or entry.get("value") or entry.get("pricePerSqm") - if month is None or price is None: - continue - price_val: float | None = None - try: - price_val = float(price) - except (TypeError, ValueError): - pass - points.append( - { - "month_date": str(month)[:10], - "room_count": str(entry.get("roomType") or entry.get("roomCount") or "all"), - "prices_type": str( - entry.get("pricesType") or entry.get("priceType") or "price" - ), - "period": str(entry.get("period") or "halfYear"), - "price_per_sqm": price_val, - } - ) - return points - - return [] - - -def _extract_reliability_checks(rel_state: dict[str, Any]) -> list[dict[str, Any]]: - """Extract reliability check summary from наш.дом.рф state. - - Actual Cian shape (sec 15.6): - rel_state = { - checkStatus: {status: 'reliable', title: '...', date: '...'}, - details: [{title, iconType, type}, ...], - ... - } - - Returns a list with ONE entry representing the overall check (per DB schema): - [{check_status, check_name, details}] - - The 'details' array is stored as jsonb (full check list) in house_reliability_checks. - """ - check_status_block = rel_state.get("checkStatus") - if not isinstance(check_status_block, dict): - # Alternative: flat list shape - raw = rel_state.get("checks") or rel_state.get("items") or [] - if isinstance(raw, list) and raw: - return [ - { - "check_name": entry.get("name") or entry.get("title") or entry.get("checkName"), - "check_status": entry.get("status") or entry.get("checkStatus"), - "details": entry.get("details") or entry, - } - for entry in raw - if isinstance(entry, dict) - ] - return [] - - status = check_status_block.get("status") - name = check_status_block.get("title") or check_status_block.get("mobileTitle") - details = rel_state.get("details") or [] - - return [ - { - "check_name": name, - "check_status": status, - "details": details if isinstance(details, list) else [details], - } - ] - - -def _extract_nested_offers(offers_state: dict[str, Any]) -> list[dict[str, Any]]: - """Extract mini-SERP offers from offers container. - - Actual Cian shape (sec 15.7): - offers_state.data.total.fromDeveloperRooms[].{roomType, layouts[]} - - Returns flat list of layout offers with room_count injected. - """ - result: list[dict[str, Any]] = [] - - # Shape A: miniSerp / items list - if "items" in offers_state: - items = offers_state["items"] - return items if isinstance(items, list) else [] - - # Shape B: data.total.fromDeveloperRooms[].layouts[] - data = offers_state.get("data") - if isinstance(data, dict): - total = data.get("total") or {} - if isinstance(total, dict): - rooms = total.get("fromDeveloperRooms") or [] - else: - rooms = [] - else: - rooms = [] - - if not isinstance(rooms, list): - return result - - for room in rooms: - if not isinstance(room, dict): - continue - room_type = room.get("roomType") or room.get("roomTypeDisplay") - layouts = room.get("layouts") or [] - if not isinstance(layouts, list): - continue - for layout in layouts: - if isinstance(layout, dict): - result.append( - { - "room_count": room_type, - **layout, - } - ) - - return result - - -# ---- save helpers ---- - - -def save_newbuilding_enrichment( - db: Any, - house_id: int, - enrichment: NewbuildingEnrichment, -) -> None: - """Persist NewbuildingEnrichment to DB. - - Steps: - 1. UPSERT management_companies (if present) → get mc_id - 2. UPDATE houses with Cian metadata (incl. cian_zhk_url if present) - 3. INSERT INTO houses_price_dynamics (chart points, ON CONFLICT DO UPDATE) - 4. INSERT INTO house_reliability_checks (overall + details) - """ - from sqlalchemy import text - - # 1. UPSERT management_company - mc_id: int | None = None - if enrichment.management_company and isinstance(enrichment.management_company, dict): - mc = enrichment.management_company - mc_name = mc.get("name") or "Unknown" - mc_ext_id = mc.get("id") - mc_result = db.execute( - text(""" - INSERT INTO management_companies ( - name, phones, email, opening_hours, chief_name, ext_source, ext_id - ) VALUES ( - :name, - CAST(:phones AS text[]), - :email, - :oh, - :chief, - 'cian', - CAST(:ext_id AS bigint) - ) - ON CONFLICT (ext_source, name, ext_id) DO UPDATE SET - phones = EXCLUDED.phones, - email = COALESCE(EXCLUDED.email, management_companies.email) - RETURNING id - """), - { - "name": mc_name, - "phones": mc.get("phones") or [], - "email": mc.get("email"), - "oh": mc.get("openingHours"), - "chief": mc.get("chiefName"), - "ext_id": mc_ext_id, - }, - ) - row = mc_result.fetchone() - mc_id = row[0] if row else None - - # 2. UPDATE houses - db.execute( - text(""" - UPDATE houses SET - cian_internal_house_id = COALESCE( - CAST(:cihi AS bigint), cian_internal_house_id - ), - cian_zhk_url = COALESCE(:zhk_url, cian_zhk_url), - year_built = COALESCE(:yb, year_built), - management_company_id = COALESCE(CAST(:mcid AS bigint), management_company_id), - transport_accessibility_rate = COALESCE( - :tar, transport_accessibility_rate - ), - advantages = CAST(:adv AS jsonb), - banks = CAST(:bk AS jsonb), - builders = CAST(:bld AS jsonb), - houses_by_turn = CAST(:hbt AS jsonb) - WHERE id = CAST(:hid AS bigint) - """), - { - "hid": house_id, - "cihi": enrichment.cian_internal_house_id, - "zhk_url": enrichment.cian_zhk_url, - "yb": enrichment.year_built, - "mcid": mc_id, - "tar": enrichment.transport_accessibility_rate, - "adv": json.dumps(enrichment.advantages, ensure_ascii=False), - "bk": json.dumps(enrichment.banks, ensure_ascii=False), - "bld": json.dumps(enrichment.builders, ensure_ascii=False), - "hbt": json.dumps(enrichment.houses_by_turn, ensure_ascii=False), - }, - ) - - # 3. INSERT houses_price_dynamics - # UNIQUE constraint: houses_price_dynamics_dim_key - # (house_id, source, room_count, prices_type, period, month_date) — per migration 029 - chart_saved = 0 - for point in enrichment.realty_valuation_chart: - if point.get("price_per_sqm") is None: - continue - room_count = point.get("room_count") or "all" - prices_type = point.get("prices_type") or "price" - period = point.get("period") or "halfYear" - db.execute( - text(""" - INSERT INTO houses_price_dynamics ( - house_id, month_date, source, - room_count, prices_type, period, - price_per_sqm, recorded_at - ) VALUES ( - CAST(:hid AS bigint), - CAST(:md AS date), - 'cian_realty_valuation', - :rc, :pt, :pd, - CAST(:pps AS numeric), - NOW() - ) - ON CONFLICT ON CONSTRAINT houses_price_dynamics_dim_key DO UPDATE SET - price_per_sqm = EXCLUDED.price_per_sqm, - recorded_at = NOW() - """), - { - "hid": house_id, - "md": point["month_date"], - "rc": room_count, - "pt": prices_type, - "pd": period, - "pps": point["price_per_sqm"], - }, - ) - chart_saved += 1 - - # 4. INSERT house_reliability_checks (stores overall check + details array) - # Schema (025): (house_id, check_status, check_name, details jsonb, source, recorded_at) - # No UNIQUE constraint — caller should manage duplicates if needed - for check in enrichment.reliability_checks: - if not check.get("check_name") and not check.get("check_status"): - continue - db.execute( - text(""" - INSERT INTO house_reliability_checks ( - house_id, check_status, check_name, details, source, recorded_at - ) VALUES ( - CAST(:hid AS bigint), - :cs, - :cn, - CAST(:det AS jsonb), - 'cian_nashdom', - NOW() - ) - """), - { - "hid": house_id, - "cs": check.get("check_status"), - "cn": check.get("check_name"), - "det": json.dumps(check.get("details") or [], ensure_ascii=False), - }, - ) - - db.commit() - logger.info( - "Cian newbuilding saved house_id=%s (chart=%d points, reliability=%d checks, mc_id=%s)", - house_id, - chart_saved, - len(enrichment.reliability_checks), - mc_id, - ) - - -async def resolve_cian_zhk_url( - cian_internal_house_id: int, - *, - session: AsyncSession | None = None, -) -> str | None: - """Resolve canonical ZHK slug URL from a Cian internal house ID (LEGACY — BROKEN). - - .. deprecated:: - The redirect path this relies on — ``https://cian.ru/zhk/<id>/`` → canonical - slug — NO LONGER EXISTS. Cian now serves HTTP **404** for ``/zhk/<id>/`` - (verified 8/8 on prod, #972), so this returns None for every real id. Use - :func:`resolve_cian_zhk_url_via_search` instead: it fetches the cat.php - newbuilding-SERP and extracts the canonical ``zhk-*.cian.ru`` slug from its - markup (the WORKING path). Kept only for backward-compat / reference; do not - wire new callers to it. - - Args: - cian_internal_house_id: Cian's numeric ЖК identifier. - session: optional shared curl_cffi AsyncSession (caller owns lifecycle). - - Returns: - Canonical ZHK URL string, or None if request failed / redirect not followed. - - Note: - This function makes a real HTTP request — do NOT call it without rate limiting. - Caller must enforce per-request sleep matching scraper_settings 'cian' delay. - """ - close_session = False - if session is None: - # Mobile proxy wiring (#806 follow-up): resolve ЖК URL через мобильный прокси. - _proxy_url = settings.cian_proxy_url - _proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None - session = AsyncSession( - impersonate="chrome120", - timeout=15.0, - proxies=_proxies, - headers={ - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", - "Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8", - }, - ) - close_session = True - - fallback_url = f"https://cian.ru/zhk/{cian_internal_house_id}/" - try: - resp = await session.get(fallback_url, allow_redirects=True) - final_url = str(resp.url) - if final_url and final_url != fallback_url: - logger.debug("resolve_cian_zhk_url id=%s → %s", cian_internal_house_id, final_url) - return final_url - # Redirect not followed or same URL — return fallback as canonical - if resp.status_code == 200: - return fallback_url - logger.warning( - "resolve_cian_zhk_url id=%s: HTTP %d, no usable URL", - cian_internal_house_id, - resp.status_code, - ) - return None - except Exception as exc: - logger.warning("resolve_cian_zhk_url id=%s failed: %s", cian_internal_house_id, exc) - return None - finally: - if close_session: - await session.close() - - -def _extract_zhk_url_from_serp(html: str) -> str | None: - """Extract the canonical ЖК-slug URL of the QUERIED newobject from a cat.php SERP. - - Pure helper (no I/O) so the extraction can be unit-tested against a saved fixture. - The cat.php SERP for a single ``newobject[0]=<nb_id>`` renders that ЖК's canonical - ``https://zhk-<slug>.cian.ru`` URL as the ``<a>`` href inside its page heading - ``<h1 data-name="Title">`` (the ``NewbuildingHeaderSeo`` header). We anchor on that - heading rather than taking the first ``zhk-*.cian.ru`` match in the document, because - cat.php pages also carry banner / "похожие ЖК" / recommendation blocks that can embed - OTHER ЖК slug URLs — and some of those blocks render BEFORE the title. A naive - first-match could pick a foreign ЖК and enrich the house with the wrong building's - price_dynamics / reliability (and the backfill's idempotency would then lock the - corruption in). Anchoring on the title guarantees we resolve the queried ЖК. - - Falls back to a naive first ``zhk-*.cian.ru`` match ONLY when the title anchor finds - nothing (markup drift / unexpected page shape), preserving the previous behaviour as - a last-resort safety net. - - Returns the slug URL string, or None when the SERP carries no such anchor at all - (empty / no-results page, or markup drift). - """ - anchored = _ZHK_TITLE_ANCHOR_RE.search(html) - if anchored: - return anchored.group(1) - # Fallback: no title-anchored slug found — degrade to naive first-match so a markup - # drift that moves/renames the header still resolves *something* rather than failing. - match = _ZHK_SLUG_URL_RE.search(html) - return match.group(0) if match else None - - -async def resolve_cian_zhk_url_via_search( - nb_id: int, - *, - session: AsyncSession | None = None, -) -> str | None: - """Resolve the canonical ЖК-slug URL for a Cian newbuilding id (the WORKING path). - - Fetches the cat.php newbuilding-SERP for a single ``newobject[0]=<nb_id>`` and - extracts the canonical ``https://zhk-<slug>.cian.ru`` URL from its markup. This - replaces the legacy :func:`resolve_cian_zhk_url`, whose ``/zhk/<id>/`` redirect path - now 404s (verified on prod, #972). The returned slug URL is exactly what - :func:`fetch_newbuilding` parses, so the enrichment chain is - ``nb_id → cat.php SERP → zhk-slug-url → fetch_newbuilding → enrich``. - - Verified live (HTTP 200, slug extracted): - nb 48853 → https://zhk-parkovyy-kvartal-ekb-i.cian.ru (ЖК «Парковый квартал») - nb 102791 → https://zhk-izumrudnyy-bor-ekb-i.cian.ru - nb 24991 → https://zhk-baltym-park-ekb-i.cian.ru - - Args: - nb_id: Cian newbuilding id (``house_sources.ext_id`` for cian houses). - session: optional shared curl_cffi AsyncSession (caller owns lifecycle). When - None, an own session is created using the same mobile-proxy wiring as the - rest of the Cian scrapers (``settings.cian_proxy_url`` when set, else direct). - - Returns: - The canonical ЖК-slug URL string, or None on non-200 / empty SERP / no match / - request failure (each logs a warning). - - Note: - Makes ONE real HTTP request and does NOT sleep — the CALLER enforces the - anti-bot delay (matching scraper_settings 'cian'). At scale this needs the - mobile proxy; low-volume direct fetches work for the bounded proof. - """ - close_session = False - if session is None: - # Mobile proxy wiring (#806): Cian блокирует datacenter-IP. proxy=None → прямое - # подключение (dev / proxy-down fallback — single fetches survive direct). - _proxy_url = settings.cian_proxy_url - _proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None - session = AsyncSession( - impersonate="chrome120", - timeout=30.0, - proxies=_proxies, - headers={ - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", - "Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8", - }, - ) - close_session = True - - serp_url = _CATPH_NEWBUILDING_SERP.format(nb_id=nb_id) - try: - resp = await session.get(serp_url, allow_redirects=True) - if resp.status_code != 200: - logger.warning( - "resolve_cian_zhk_url_via_search nb_id=%s: cat.php → HTTP %d", - nb_id, - resp.status_code, - ) - return None - zhk_url = _extract_zhk_url_from_serp(resp.text) - if zhk_url is None: - logger.warning( - "resolve_cian_zhk_url_via_search nb_id=%s: no zhk-slug URL in SERP " - "(empty results / markup drift?)", - nb_id, - ) - return None - logger.info("resolve_cian_zhk_url_via_search nb_id=%s → %s", nb_id, zhk_url) - return zhk_url - except Exception as exc: - logger.warning("resolve_cian_zhk_url_via_search nb_id=%s failed: %s", nb_id, exc) - return None - finally: - if close_session: - await session.close() diff --git a/tradein-mvp/backend/app/services/scrapers/cian_state_parser.py b/tradein-mvp/backend/app/services/scrapers/cian_state_parser.py deleted file mode 100644 index f6a46a7a..00000000 --- a/tradein-mvp/backend/app/services/scrapers/cian_state_parser.py +++ /dev/null @@ -1,160 +0,0 @@ -"""Shared utility для извлечения Cian Redux state из _cianConfig. - -Pattern: window._cianConfig['<mfe>'].push({key:..., value:..., priority:..., filter:...}) -Used by SERP / detail / newbuilding / valuation scrapers. -""" - -import json -import logging -import re -from typing import Any - -logger = logging.getLogger(__name__) - -# Match both patterns Cian uses: -# 1. window._cianConfig['mfe'].push({key:..., value:..., priority:..., filter:...}) -# 2. window._cianConfig['mfe'] = window._cianConfig['mfe'] || []; -# window._cianConfig['mfe'].push({...}) -_RE_CIAN_PUSH = re.compile( - r"window\._cianConfig\['(?P<mfe>[\w-]+)'\]" - r"(?:\.push\(\s*|" - r"\s*=\s*window\._cianConfig\['[\w-]+'\]\s*\|\|\s*\[\];\s*" - r"window\._cianConfig\['[\w-]+'\]\.push\(\s*)" - r"\{\s*key:\s*['\"](?P<key>\w+)['\"]\s*,\s*" - r"value:\s*(?P<value>.+?)\s*,\s*" - r"priority:", - re.DOTALL, -) - -# 2026-05 drift (#639): Cian сменил формат на -# window._cianConfig['mfe'] = (window._cianConfig['mfe'] || []).concat([{"key":..,"value":..},..]) -# вместо .push({key:..}). Ключи теперь в JSON-кавычках, аргумент concat([...]) — валидный -# JSON-массив → парсим его целиком (надёжнее regexp по одному value). Старый .push формат -# (_RE_CIAN_PUSH) оставлен как fallback для legacy-фикстур. -_RE_CIAN_CONCAT = re.compile( - r"window\._cianConfig\['(?P<mfe>[\w-]+)'\]\s*=\s*" - r"\(\s*window\._cianConfig\['[\w-]+'\]\s*\|\|\s*\[\]\s*\)\.concat\(\s*" -) - - -def _balanced_array(s: str, start: int) -> str | None: - """Срез s[start:end] — сбалансированный ``[...]`` от индекса '[', с учётом JSON-строк.""" - depth = 0 - in_str = False - esc = False - for i in range(start, len(s)): - c = s[i] - if in_str: - if esc: - esc = False - elif c == "\\": - esc = True - elif c == '"': - in_str = False - continue - if c == '"': - in_str = True - elif c == "[": - depth += 1 - elif c == "]": - depth -= 1 - if depth == 0: - return s[start : i + 1] - return None - - -def _extract_from_concat(html: str, mfe: str, key: str) -> Any | None: - """Новый формат: .concat([{"key":..,"value":..},..]) — массив-аргумент = валидный JSON.""" - for m in _RE_CIAN_CONCAT.finditer(html): - if m.group("mfe") != mfe: - continue - start = m.end() - if start >= len(html) or html[start] != "[": - start = html.find("[", m.end()) - if start < 0: - continue - arr = _balanced_array(html, start) - if arr is None: - continue - try: - items = json.loads(arr) - except json.JSONDecodeError as exc: - logger.debug("concat array parse failed for mfe=%s key=%s: %s", mfe, key, exc) - continue - for it in items: - if isinstance(it, dict) and it.get("key") == key: - return it.get("value") - return None - - -def extract_state(html: str, mfe: str, key: str = "initialState") -> dict[str, Any] | None: - """Extract state.value where _cianConfig[<mfe>] entry.key matches. - - Поддерживает оба формата Cian: новый ``.concat([{"key":..,"value":..}])`` (2026-05, - #639) и legacy ``.push({key:..,value:..})``. Returns parsed JSON dict, or None. - """ - # Новый формат (concat) — приоритетный. - concat_val = _extract_from_concat(html, mfe, key) - if concat_val is not None: - return concat_val - - for match in _RE_CIAN_PUSH.finditer(html): - if match.group("mfe") != mfe or match.group("key") != key: - continue - - value_str = match.group("value").strip() - - # Strategy 1: direct JSON parse (value уже JSON-like) - try: - return json.loads(value_str) - except json.JSONDecodeError: - pass - - # Strategy 2: JSON.parse("escaped string") - # Use json.loads on the outer quoted string to unescape atomically — avoids - # order-dependent manual .replace() chains that break when descriptions contain - # backslashes (e.g. '\\' in listing text corrupts the rest of the parse). - m_jp = re.match(r'JSON\.parse\(\s*("(?:[^"\\]|\\.)*")\s*\)\s*$', value_str, re.DOTALL) - if m_jp: - outer_json_str = m_jp.group(1) # the full "..." including surrounding quotes - try: - inner = json.loads(outer_json_str) # inner is now a Python str - except json.JSONDecodeError as exc: - logger.debug( - "JSON.parse() outer unescape failed for mfe=%s key=%s: %s", mfe, key, exc - ) - continue - try: - return json.loads(inner) - except json.JSONDecodeError as exc: - logger.debug("JSON.parse() inner parse failed for mfe=%s key=%s: %s", mfe, key, exc) - continue - - # Strategy 3: raw JS object literal — try demjson3 (optional dep) - try: - import demjson3 # type: ignore[import-untyped] - - return demjson3.decode(value_str) - except ImportError: - logger.debug( - "demjson3 not installed; cannot parse raw JS object for mfe=%s key=%s", mfe, key - ) - except Exception as exc: - logger.debug("demjson3 parse failed for mfe=%s key=%s: %s", mfe, key, exc) - - return None - - -def extract_all_states(html: str) -> dict[str, dict[str, Any]]: - """Returns {mfe_name: {key: parsed_value}} for all push() entries. - - Useful for discovery / debugging during scraper development. - """ - result: dict[str, dict[str, Any]] = {} - for match in _RE_CIAN_PUSH.finditer(html): - mfe = match.group("mfe") - key = match.group("key") - parsed = extract_state(html, mfe, key) - if parsed is not None: - result.setdefault(mfe, {})[key] = parsed - return result diff --git a/tradein-mvp/backend/app/services/scrapers/cian_valuation.py b/tradein-mvp/backend/app/services/scrapers/cian_valuation.py deleted file mode 100644 index 74486a9e..00000000 --- a/tradein-mvp/backend/app/services/scrapers/cian_valuation.py +++ /dev/null @@ -1,553 +0,0 @@ -"""Cian.ru Valuation Calculator scraper — auth required. - -URL: https://www.cian.ru/kalkulator-nedvizhimosti/?address=...&totalArea=... -MFE: 'valuation-for-agent-frontend' -Key: 'initialState' - -Auth: loads Cian session cookies from cian_session_cookies table (Stage 4). -Cache: 24h TTL по sha256(address + params) in external_valuations table. - -State shape (sec 24.3–24.6 Schema_Cian_SERP_Inventory): - estimation.sale.data.{price, accuracy, priceFrom, priceTo, priceSqm} - estimation.rent.data.{price, accuracy, priceFrom, priceTo, taxPrice} - estimationChart.data.{title.{change, changeValue}, chartData.data[{date, price}]} - houseInfo.data.{items[{title, value}], isRenovation, isEmergency, isCulturalHeritage} - managementCompany.data.{name, address, phones, email, openingHours, chiefName} - filters.houseId — Cian's internal house_id - -Used as 6th evaluation source в estimator.py (Stage 9). -""" - -from __future__ import annotations - -import hashlib -import json -import logging -from dataclasses import dataclass, field -from typing import Any -from urllib.parse import urlencode - -from curl_cffi.requests import AsyncSession -from sqlalchemy import text -from sqlalchemy.orm import Session - -from app.core.config import settings -from app.services.cian_session import load_session, mark_session_invalid -from app.services.scrapers.cian_state_parser import extract_state - -logger = logging.getLogger(__name__) - -VALUATION_BASE_URL = "https://www.cian.ru/kalkulator-nedvizhimosti/" -VALUATION_MFE = "valuation-for-agent-frontend" - -# Cian floor enum: floor 1 → floorOne, floor 2 → floorTwo, -# last floor → floorLast, все остальные → floorOther. -_FLOOR_ENUM_SPECIAL = {1: "floorOne", 2: "floorTwo"} - -# Cian repair type enum -_REPAIR_TYPE_MAP: dict[str, str] = { - "without": "repairTypeWithout", - "no": "repairTypeWithout", - "cosmetic": "repairTypeCosmetic", - "euro": "repairTypeEuro", - "design": "repairTypeDesign", -} - -# Cian chart change direction → normalized storage values (per DDL: increase/decrease/neutral) -_CHANGE_DIR_MAP: dict[str, str] = { - "increase": "increase", - "decrease": "decrease", - "neutral": "neutral", - "up": "increase", - "down": "decrease", -} - - -def _cian_floor_enum(floor: int, total_floors: int | None) -> str: - if floor in _FLOOR_ENUM_SPECIAL: - return _FLOOR_ENUM_SPECIAL[floor] - if total_floors is not None and floor == total_floors: - return "floorLast" - return "floorOther" - - -@dataclass -class CianValuationResult: - """Cian Valuation Calculator response parsed.""" - - # Sale estimate - sale_price_rub: float | None = None - sale_accuracy: float | None = None - sale_price_from: float | None = None - sale_price_to: float | None = None - sale_price_sqm: float | None = None - - # Rent estimate (Cian-specific — both sale + rent per 1 request) - rent_price_rub: float | None = None - rent_accuracy: float | None = None - rent_price_from: float | None = None - rent_price_to: float | None = None - rent_tax_price: float | None = None - - # Chart (7 monthly points for this specific apartment) - chart: list[dict[str, Any]] = field(default_factory=list) - chart_change_pct: float | None = None - chart_change_direction: str | None = None # increase/decrease/neutral - - # House info (15 items: year, type, floors, etc.) - house_info: list[dict[str, Any]] = field(default_factory=list) - external_house_id: int | None = None # filters.houseId - filters_hash: str | None = None # estimation.sale.data.filtersHash - - # Management company (unique to Cian Valuation) - management_company: dict[str, Any] | None = None - - # Authentication state - is_authenticated: bool = False - user_id: int | None = None - - # Raw payload (full state for raw_payload column) - raw_state: dict[str, Any] | None = None - - -def compute_cache_key( - address: str, - total_area: float, - rooms_count: int, - floor: int, - repair_type: str, - deal_type: str, -) -> str: - """sha256 of normalized params for 24h cache lookup.""" - norm = ( - f"{address.strip().lower()}|{total_area}|{rooms_count}|{floor}" - f"|{repair_type}|{deal_type}" - ) - return hashlib.sha256(norm.encode("utf-8")).hexdigest() - - -async def estimate_via_cian_valuation( - db: Session, - *, - address: str, - total_area: float, - rooms_count: int, - floor: int, - total_floors: int | None = None, - repair_type: str = "cosmetic", - deal_type: str = "sale", - use_cache: bool = True, - house_id: int | None = None, - listing_id: int | None = None, -) -> CianValuationResult | None: - """Estimate value via Cian's authenticated Valuation Calculator. - - Выполняет 1 GET на cian.ru/kalkulator-nedvizhimosti с cookies из DB, - парсит state['estimation'] + estimationChart + houseInfo + managementCompany. - - Returns CianValuationResult, or None если cookies expired/invalid/fetch failed. - Graceful: не кидает исключений — caller может продолжить без этого source. - """ - cache_key = compute_cache_key(address, total_area, rooms_count, floor, repair_type, deal_type) - - # 1. Cache lookup (только если use_cache=True) - if use_cache: - cached = _load_from_cache(db, cache_key) - if cached is not None: - logger.info("Cian valuation cache HIT for cache_key=%s...", cache_key[:12]) - return cached - - # 2. Load auth cookies из cian_session_cookies - cookies = load_session(db) - if cookies is None: - logger.warning("Cian valuation: no auth session in DB — skipping (graceful)") - return None - - # 3. Build URL с корректными enum values - params: dict[str, str] = { - "address": address, - "totalArea": str(total_area), - "roomsCount": str(rooms_count), - "valuationType": deal_type, - "floor[0]": _cian_floor_enum(floor, total_floors), - "repairType[0]": _REPAIR_TYPE_MAP.get(repair_type, "repairTypeCosmetic"), - } - url = f"{VALUATION_BASE_URL}?{urlencode(params, safe='[]')}" - - # 4. Fetch с TLS fingerprint (curl_cffi, impersonate chrome120) - # Mobile proxy wiring (#806 follow-up): Cian валюация — такой же datacenter-бан риск - # как SERP. Используем cian_proxy_url. proxy=None → прямое подключение (dev). - _proxy_url = settings.cian_proxy_url - _proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None - try: - async with AsyncSession( - impersonate="chrome120", - cookies=cookies, - timeout=25.0, - proxies=_proxies, - headers={"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"}, - ) as session: - resp = await session.get(url, allow_redirects=True) - if resp.status_code != 200: - logger.warning("Cian valuation fetch → HTTP %d for %s", resp.status_code, url) - return None - html = resp.text - except Exception as exc: - logger.warning("Cian valuation fetch failed: %s", exc) - return None - - # 5. Parse state из _cianConfig['valuation-for-agent-frontend'] - state = extract_state(html, mfe=VALUATION_MFE, key="initialState") - if state is None: - logger.warning("Cian valuation: state extraction failed (mfe=%s)", VALUATION_MFE) - return None - - # 6. Verify auth — если false → session expired - user = state.get("user", {}) or {} - if not user.get("isAuthenticated"): - user_id = user.get("userId") - logger.warning( - "Cian valuation: session not authenticated (userId=%s) — marking invalid", user_id - ) - if user_id: - mark_session_invalid(db, user_id) - return None - - # 7. Extract structured result - result = _parse_valuation_state(state) - - # 7a. Sanity-check price bounds (Mera-audit fix-1). - # Битый API-ответ (999_999 < min или 9_999_999_999 > max) → не кэшируем, возвращаем None. - # Аналогично проверяем low/high bound: low>high или отрицательные → drop. - if not _is_price_sane(result): - return None - - # 8. Persist to cache (24h TTL) - _save_to_cache( - db, - cache_key=cache_key, - address=address, - total_area=total_area, - rooms_count=rooms_count, - floor=floor, - total_floors=total_floors, - repair_type=repair_type, - deal_type=deal_type, - result=result, - house_id=house_id, - listing_id=listing_id, - ) - - logger.info( - "Cian valuation extracted: sale=%s (acc=%s%%) rent=%s", - result.sale_price_rub, - result.sale_accuracy, - result.rent_price_rub, - ) - return result - - -def _parse_valuation_state(state: dict[str, Any]) -> CianValuationResult: - """Extract CianValuationResult from Cian valuation initialState. - - Mapping (per Schema_Cian_SERP_Inventory sec 24.4–24.6): - estimation.sale.data.{price, accuracy, priceFrom, priceTo, priceSqm, filtersHash} - estimation.rent.data.{price, accuracy, priceFrom, priceTo, taxPrice} - estimationChart.data.{title.{change, changeValue}, chartData.data[]} - houseInfo.data.{items[], ...} - filters.houseId - managementCompany.data - """ - result = CianValuationResult(raw_state=state) - - user = state.get("user") or {} - result.is_authenticated = bool(user.get("isAuthenticated")) - result.user_id = user.get("userId") - - # --- estimation: sale --- - estimation = state.get("estimation") or {} - sale_wrapper = estimation.get("sale") or {} - sale_data = sale_wrapper.get("data") or {} - result.sale_price_rub = _parse_num(sale_data.get("price")) - result.sale_accuracy = _parse_num(sale_data.get("accuracy")) - result.sale_price_from = _parse_num(sale_data.get("priceFrom")) - result.sale_price_to = _parse_num(sale_data.get("priceTo")) - result.sale_price_sqm = _parse_num(sale_data.get("priceSqm")) - result.filters_hash = sale_data.get("filtersHash") - - # --- estimation: rent --- - rent_wrapper = estimation.get("rent") or {} - rent_data = rent_wrapper.get("data") or {} - result.rent_price_rub = _parse_num(rent_data.get("price")) - result.rent_accuracy = _parse_num(rent_data.get("accuracy")) - result.rent_price_from = _parse_num(rent_data.get("priceFrom")) - result.rent_price_to = _parse_num(rent_data.get("priceTo")) - result.rent_tax_price = _parse_num(rent_data.get("taxPrice")) - - # --- estimationChart: 7 monthly time-series points --- - est_chart = state.get("estimationChart") or {} - chart_outer = est_chart.get("data") or {} - chart_title = chart_outer.get("title") or {} - - # changeValue: "8,3%" → parse out float - change_value_str = chart_title.get("changeValue") or "" - result.chart_change_pct = _parse_change_pct(change_value_str) - - # change direction: increase / decrease / neutral - change_dir_raw = chart_title.get("change") - if change_dir_raw: - result.chart_change_direction = _CHANGE_DIR_MAP.get(change_dir_raw, change_dir_raw) - - chart_data_wrapper = chart_outer.get("chartData") or {} - chart_points = chart_data_wrapper.get("data") or [] - if isinstance(chart_points, list): - for entry in chart_points: - if not isinstance(entry, dict): - continue - # date: unix_ms → ISO string "YYYY-MM-DD" - date_ms = entry.get("date") - date_str: str | None = None - if isinstance(date_ms, int | float) and date_ms > 0: - import datetime - - date_str = datetime.datetime.fromtimestamp( - date_ms / 1000, tz=datetime.UTC - ).strftime("%Y-%m-%d") - result.chart.append( - { - "month_date": date_str or "", - "price": _parse_num(entry.get("price")), - "price_formatted": entry.get("priceFormatted"), - } - ) - - # --- houseInfo: 15 items о доме --- - house_info_wrapper = state.get("houseInfo") or {} - house_info_data = house_info_wrapper.get("data") or {} - items = house_info_data.get("items") - if isinstance(items, list): - result.house_info = [i for i in items if isinstance(i, dict)] - - # --- filters.houseId: Cian internal house_id --- - filters = state.get("filters") or {} - result.external_house_id = filters.get("houseId") - - # --- managementCompany --- - mc_wrapper = state.get("managementCompany") or {} - mc_data = mc_wrapper.get("data") - if isinstance(mc_data, dict) and mc_data: - result.management_company = mc_data - - return result - - -def _parse_num(value: Any) -> float | None: - if value is None: - return None - try: - return float(value) - except (ValueError, TypeError): - return None - - -def _parse_change_pct(value_str: str) -> float | None: - """Parse '8,3%' → 8.3, '-2,5%' → -2.5, '' → None.""" - if not value_str: - return None - cleaned = value_str.replace(",", ".").replace("%", "").strip() - try: - return float(cleaned) - except ValueError: - return None - - -def _is_price_sane(result: CianValuationResult) -> bool: - """Mera-audit fix-1: проверка разумного диапазона цены Cian Valuation. - - Возвращает False (→ caller вернёт None, не кэшируем) если: - - sale_price_rub задан и вне [settings.cian_valuation_min_rub, cian_valuation_max_rub] - - sale_price_from или sale_price_to отрицательные - - sale_price_from > sale_price_to (инвертированный диапазон) - - None-поля не судим: Cian может не вернуть диапазон — это не ошибка. - """ - price = result.sale_price_rub - if price is not None: - min_rub = settings.cian_valuation_min_rub - max_rub = settings.cian_valuation_max_rub - if price < min_rub or price > max_rub: - logger.warning( - "Cian valuation sanity: sale_price_rub=%.0f вне [%.0f, %.0f] → drop", - price, - min_rub, - max_rub, - ) - return False - - low = result.sale_price_from - high = result.sale_price_to - if low is not None and low < 0: - logger.warning("Cian valuation sanity: sale_price_from=%.0f < 0 → drop", low) - return False - if high is not None and high < 0: - logger.warning("Cian valuation sanity: sale_price_to=%.0f < 0 → drop", high) - return False - if low is not None and high is not None and low > high: - logger.warning( - "Cian valuation sanity: low=%.0f > high=%.0f (инвертированный диапазон) → drop", - low, - high, - ) - return False - - return True - - -def _load_from_cache(db: Session, cache_key: str) -> CianValuationResult | None: - """SELECT from external_valuations where cache_key + source='cian_valuation' + not expired.""" - row = ( - db.execute( - text(""" - SELECT - raw_payload, - sale_price_rub, - sale_accuracy, - rent_price_rub, - rent_accuracy, - chart, - chart_change_pct, - chart_change_direction, - external_house_id, - filters_hash, - low_price, - high_price - FROM external_valuations - WHERE source = 'cian_valuation' - AND cache_key = :ck - AND expires_at > NOW() - ORDER BY fetched_at DESC - LIMIT 1 - """), - {"ck": cache_key}, - ) - .mappings() - .first() - ) - - if row is None: - return None - - return CianValuationResult( - sale_price_rub=_parse_num(row["sale_price_rub"]), - sale_accuracy=_parse_num(row["sale_accuracy"]), - rent_price_rub=_parse_num(row["rent_price_rub"]), - rent_accuracy=_parse_num(row["rent_accuracy"]), - chart=row["chart"] if isinstance(row["chart"], list) else [], - chart_change_pct=_parse_num(row["chart_change_pct"]), - chart_change_direction=row["chart_change_direction"], - external_house_id=row["external_house_id"], - filters_hash=row["filters_hash"], - sale_price_from=_parse_num(row["low_price"]), - sale_price_to=_parse_num(row["high_price"]), - raw_state=row["raw_payload"] if isinstance(row["raw_payload"], dict) else None, - is_authenticated=True, # cache только authenticated results - ) - - -def _save_to_cache( - db: Session, - *, - cache_key: str, - address: str, - total_area: float, - rooms_count: int, - floor: int, - total_floors: int | None, - repair_type: str, - deal_type: str, - result: CianValuationResult, - house_id: int | None = None, - listing_id: int | None = None, -) -> None: - """INSERT result into external_valuations (source='cian_valuation') с 24h expiry. - - Использует ON CONFLICT (source, cache_key) DO UPDATE для идемпотентности. - Все числовые поля через CAST(:x AS numeric) — psycopg v3 safe, без ::type. - """ - try: - db.execute( - text(""" - INSERT INTO external_valuations ( - source, cache_key, address, - total_area, rooms_count, floor, total_floors, - repair_type, deal_type, - price_rub, accuracy, - low_price, high_price, - house_id, listing_id, - sale_price_rub, sale_accuracy, - rent_price_rub, rent_accuracy, - chart, chart_change_pct, chart_change_direction, - external_house_id, filters_hash, - raw_payload, fetched_at, expires_at - ) VALUES ( - 'cian_valuation', :ck, :addr, - CAST(:ta AS numeric), :rc, :fl, :tf, - :rt, :dt, - CAST(:sp AS numeric), CAST(:sa AS numeric), - CAST(:lp AS numeric), CAST(:hp AS numeric), - CAST(:hid AS bigint), CAST(:lid AS bigint), - CAST(:sp AS numeric), CAST(:sa AS numeric), - CAST(:rp AS numeric), CAST(:ra AS numeric), - CAST(:ch AS jsonb), CAST(:ccp AS numeric), :ccd, - :ehi, :fh, - CAST(:raw AS jsonb), NOW(), NOW() + INTERVAL '24 hours' - ) - ON CONFLICT (source, cache_key) DO UPDATE SET - price_rub = EXCLUDED.price_rub, - accuracy = EXCLUDED.accuracy, - low_price = EXCLUDED.low_price, - high_price = EXCLUDED.high_price, - house_id = COALESCE(EXCLUDED.house_id, external_valuations.house_id), - listing_id = COALESCE(EXCLUDED.listing_id, external_valuations.listing_id), - sale_price_rub = EXCLUDED.sale_price_rub, - sale_accuracy = EXCLUDED.sale_accuracy, - rent_price_rub = EXCLUDED.rent_price_rub, - rent_accuracy = EXCLUDED.rent_accuracy, - chart = EXCLUDED.chart, - chart_change_pct = EXCLUDED.chart_change_pct, - chart_change_direction = EXCLUDED.chart_change_direction, - external_house_id = EXCLUDED.external_house_id, - filters_hash = EXCLUDED.filters_hash, - raw_payload = EXCLUDED.raw_payload, - fetched_at = NOW(), - expires_at = NOW() + INTERVAL '24 hours' - """), - { - "ck": cache_key, - "addr": address, - "ta": total_area, - "rc": rooms_count, - "fl": floor, - "tf": total_floors, - "rt": repair_type, - "dt": deal_type, - "sp": result.sale_price_rub, - "sa": result.sale_accuracy, - "lp": result.sale_price_from, - "hp": result.sale_price_to, - "hid": house_id, - "lid": listing_id, - "rp": result.rent_price_rub, - "ra": result.rent_accuracy, - "ch": json.dumps(result.chart), - "ccp": result.chart_change_pct, - "ccd": result.chart_change_direction, - "ehi": result.external_house_id, - "fh": result.filters_hash, - "raw": json.dumps(result.raw_state) if result.raw_state else None, - }, - ) - db.commit() - except Exception as exc: - logger.error("Cian valuation cache save failed: %s", exc, exc_info=True) - raise diff --git a/tradein-mvp/backend/app/services/scrapers/house_type_normalizer.py b/tradein-mvp/backend/app/services/scrapers/house_type_normalizer.py deleted file mode 100644 index 9ac346dc..00000000 --- a/tradein-mvp/backend/app/services/scrapers/house_type_normalizer.py +++ /dev/null @@ -1,99 +0,0 @@ -"""Нормализация house_type → каноничный enum для listings.house_type. - -Целевой канон (сверено с estimator._IMV_HOUSE_TYPE_MAP, estimator.py:146): - panel / brick / monolith / monolith_brick / block / wood. - -Зачем (#2007): estimator применяет soft-penalty по house_type — аналог с -house_type != target штрафуется. Yandex SERP отдаёт SCREAMING-значения -(MONOLIT / BRICK / PANEL / ...), которые НИКОГДА не равны каноничным -(monolith / brick / panel / ...) → ~70% yandex-аналогов получали ложный штраф -и фактически выпадали из скоринга. Нормализация на ингесте чинит это. - -Важно про None: неизвестное / 'other' / '' → None, НЕ 'other'. В estimator -`house_type IS NULL` нейтрально (без штрафа), а любое не-канон значение всегда -!= target → ложный штраф. Поэтому unknown лучше схлопнуть в NULL. - -Источники raw-значений: -- yandex SERP: building.buildingType — SCREAMING_SNAKE (MONOLIT, MONOLIT_BRICK, ...) -- cian SERP: building.materialType — camelCase (monolith, monolithBrick, - gasSilicateBlock, ...) — переиспользуется в #2008 (cian.py:815). -- avito detail page: «Тип дома» — русские лейблы (монолитный, панельный, ...) — - переиспользуется в #2009 (avito_detail.py) вместо локального дубля-мэппинга. -""" - -from __future__ import annotations - -import logging - -logger = logging.getLogger(__name__) - -# Канон listings.house_type — выровнен по estimator._IMV_HOUSE_TYPE_MAP. -_CANON: frozenset[str] = frozenset( - {"panel", "brick", "monolith", "monolith_brick", "block", "wood"} -) - -# raw-токен → канон. Ключи — точные значения вокабуляров источников; сравнение -# регистронезависимое (см. _LOOKUP ниже), но строго по полному токену енума, -# без fuzzy-матчинга. Неизвестные токены сюда НЕ попадают → normalize вернёт None. -_RAW_TO_CANON: dict[str, str] = { - # ── yandex SERP (SCREAMING_SNAKE buildingType) ─────────────────────────── - "MONOLIT": "monolith", - "BRICK": "brick", - "PANEL": "panel", - "MONOLIT_BRICK": "monolith_brick", - "BLOCK": "block", - "WOOD": "wood", - # ── cian SERP (camelCase materialType) — reuse в #2008 ─────────────────── - "monolith": "monolith", - "brick": "brick", - "panel": "panel", - "block": "block", - "wood": "wood", - "monolithBrick": "monolith_brick", - "gasSilicateBlock": "block", - "aerocreteBlock": "block", - "foamConcreteBlock": "block", - "stalin": "brick", # «сталинка» — кирпич - # ── avito detail page («Тип дома», русские лейблы) — reuse в #2009 ─────── - "монолитный": "monolith", - "панельный": "panel", - "кирпичный": "brick", - "монолитно-кирпичный": "monolith_brick", - "блочный": "block", - "деревянный": "wood", -} - -# Регистронезависимый lookup. Лоуэркейс-ключи не коллизят между вокабулярами: -# 'MONOLIT'→'monolit' и 'monolith'→'monolith' — разные ключи. -_LOOKUP: dict[str, str] = {k.lower(): v for k, v in _RAW_TO_CANON.items()} - - -def normalize_house_type(raw: str | None) -> str | None: - """Преобразовать raw house_type в каноничный enum. - - Уже-каноничное значение возвращается as-is (идемпотентность — нужна при - ре-обработке и для backfill-миграции). Неизвестное / 'other' / пустое / - None → None (НЕ 'other': см. docstring модуля про estimator soft-penalty). - - Args: - raw: сырое значение из парсера (e.g. «MONOLIT», «monolithBrick») или None. - - Returns: - Одно из panel / brick / monolith / monolith_brick / block / wood, либо None. - """ - if raw is None: - return None - stripped = raw.strip() - if not stripped: - return None - # Pass-through: уже каноничное значение (idempotency — в т.ч. 'monolith_brick', - # которого нет среди raw-ключей карты). - if stripped in _CANON: - return stripped - # Регистронезависимый exact-token lookup по обоим вокабулярам. - canon = _LOOKUP.get(stripped) or _LOOKUP.get(stripped.lower()) - if canon is None: - # Неизвестное / 'other' — нормально (NULL нейтрально для estimator). - # debug, не warning: 'other' встречается массово, warning засорил бы лог. - logger.debug("house_type_normalizer: unmapped raw value %r — stored as NULL", raw) - return canon diff --git a/tradein-mvp/backend/app/services/scrapers/price_brackets.py b/tradein-mvp/backend/app/services/scrapers/price_brackets.py deleted file mode 100644 index f3c075fc..00000000 --- a/tradein-mvp/backend/app/services/scrapers/price_brackets.py +++ /dev/null @@ -1,32 +0,0 @@ -"""Общие seed price-brackets для exhaustive-скрапперов (avito/cian/yandex). - -EKB-настроенные (анализ прод-распределения вторички ЕКБ). Структура city-dict — -готова под per-city (multi-city: Москва пик 15-40М, Тюмень — иные; multi-city — -отдельная фича, см. vault decision). Сейчас единственный город — ekaterinburg. -""" - -# (lo, hi) рубли; hi=None — открытый верхний брекет (без потолка). Закрытые -# брекеты соприкасаются — вызывающий применяет hi-1 (эксклюзивный верх), дедуп -# по source_id страхует на стыках. -_EKB_SECONDARY: list[tuple[int, int | None]] = [ - (0, 4_000_000), - (4_000_000, 5_000_000), - (5_000_000, 6_000_000), - (6_000_000, 7_000_000), - (7_000_000, 8_000_000), - (8_000_000, 10_000_000), - (10_000_000, 13_000_000), - (13_000_000, 17_000_000), - (17_000_000, 25_000_000), - (25_000_000, 250_000_000), - (250_000_000, None), # 250М+ открытый catch-all (без потолка) -] - -CITY_PRICE_SEED_BRACKETS: dict[str, list[tuple[int, int | None]]] = { - "ekaterinburg": _EKB_SECONDARY, -} - - -def get_price_seed_brackets(city: str = "ekaterinburg") -> list[tuple[int, int | None]]: - """Seed price-brackets для города (fallback на ekaterinburg).""" - return CITY_PRICE_SEED_BRACKETS.get(city, _EKB_SECONDARY) diff --git a/tradein-mvp/backend/app/services/scrapers/repair_state_normalizer.py b/tradein-mvp/backend/app/services/scrapers/repair_state_normalizer.py deleted file mode 100644 index d31d70b2..00000000 --- a/tradein-mvp/backend/app/services/scrapers/repair_state_normalizer.py +++ /dev/null @@ -1,162 +0,0 @@ -"""Нормализация repair_state → каноничный enum для listings.repair_state. - -Целевой enum: needs_repair / standard / good / excellent. -Используется во всех инgest-путях: avito_detail, cian SERP, cian_detail. - -Источники raw-значений: -- Avito HTML: «Ремонт: евро» → avito_detail._REPAIR_MAP → «euro» → good -- Cian SERP JSON: offer.decoration → «euro» / «cosmetic» / «without» / ... -- Cian Detail JSON: offer.repairType → «cosmetic» / «design» / «no» / «euro» / ... - -Маппинг основан на _IMV_REPAIR_MAP из estimator.py (обратная форма): - needs_repair→«required», standard→«cosmetic», good→«euro», excellent→«designer». - -Связь с estimator._REPAIR_COEF: - needs_repair=0.94 / standard=1.00 / good=1.05 / excellent=1.10 -""" - -from __future__ import annotations - -import logging -import re - -logger = logging.getLogger(__name__) - -# Все известные raw-значения с источником в комментарии -_RAW_TO_ENUM: dict[str, str] = { - # ── needs_repair ──────────────────────────────────────────────────────────── - "required": "needs_repair", # Avito: «требуется» / «без ремонта» (через _REPAIR_MAP) - "without": "needs_repair", # Cian: без ремонта - "no": "needs_repair", # Cian: нет ремонта (синоним without) - "rough": "needs_repair", # Cian/общий: черновая отделка - "prefine": "needs_repair", # Cian detail: предчистовая/whitebox (offer.decoration, #1791) - # ── standard ──────────────────────────────────────────────────────────────── - "cosmetic": "standard", # Avito/Cian: косметический - # ── good ──────────────────────────────────────────────────────────────────── - "euro": "good", # Avito/Cian: евро - "fine": "good", # Cian: хорошая отделка (предчистовая / whitebox) - # ── excellent ─────────────────────────────────────────────────────────────── - "designer": "excellent", # Avito: дизайнерский - "design": "excellent", # Cian: дизайнерский (camelCase вариант) -} - -# Допустимые значения целевого enum — для pass-through защиты -_VALID_ENUM: frozenset[str] = frozenset({"needs_repair", "standard", "good", "excellent"}) - - -def normalize_repair_state(raw: str | None) -> str | None: - """Преобразовать raw repair-значение в каноничный enum. - - Если raw уже является корректным enum-значением — вернуть as-is (идемпотентно). - Если raw не распознан — вернуть None + залогировать WARNING один раз. - - Args: - raw: сырое значение из парсера (e.g. «euro», «without», «design») - или None. - - Returns: - Одно из needs_repair / standard / good / excellent, либо None. - """ - if raw is None: - return None - # Pass-through: уже нормализованный enum (idempotency — нужно при ре-обработке) - if raw in _VALID_ENUM: - return raw - # Cian отдаёт camelCase ('preFine', 'whiteBox') — ключи карты в нижнем регистре, - # сравниваем case-insensitive, чтобы не плодить дубль-ключи на каждый регистр. - result = _RAW_TO_ENUM.get(raw) or _RAW_TO_ENUM.get(raw.lower()) - if result is None: - logger.warning("repair_state_normalizer: unknown raw value %r — stored as NULL", raw) - return result - - -# ── Inference из текста описания (#622) ────────────────────────────────────── -# Когда структурное поле repair отсутствует (avito ~1% / cian ~5% / yandex 0%), -# извлекаем состояние из текста описания регэкспами по русским фразам. -# -# Порядок ВАЖЕН: проверяем от самого сильного сигнала к слабому -# excellent → good → needs_repair → standard, -# чтобы «после черновой сделали дизайнерский ремонт» дал excellent, а не -# needs_repair. Первое совпадение по порядку выигрывает. -# -# Целевой enum выровнен по normalize_repair_state() — те же 4 значения. -_TEXT_PATTERNS: tuple[tuple[re.Pattern[str], str], ...] = ( - # ── excellent: дизайнерский / премиум / эксклюзивный ремонт ────────────── - ( - re.compile( - r"дизайнерск(?:ий|ого|ом|ая)\s+ремонт" - r"|дизайнерск(?:ая|ой)\s+отделк" - r"|премиальн\w*\s+ремонт" - r"|эксклюзивн\w*\s+ремонт" - r"|авторск\w*\s+ремонт", - re.IGNORECASE, - ), - "excellent", - ), - # ── good: евроремонт / отличный / свежий / современный ремонт ──────────── - ( - re.compile( - r"евроремонт" - r"|евро[\s-]?ремонт" - r"|отличн\w*\s+ремонт" - r"|свеж\w*\s+ремонт" - r"|современн\w*\s+ремонт" - r"|качественн\w*\s+ремонт" - r"|ремонт\s+в\s+отличн", - re.IGNORECASE, - ), - "good", - ), - # ── needs_repair: без отделки / черновая / требует ремонта / под ремонт ── - ( - re.compile( - r"без\s+отделк" - r"|без\s+ремонт" - r"|черновая\s+отделк" - r"|черновой\s+ремонт" - r"|требует(?:ся)?\s+ремонт" - r"|требует(?:ся)?\s+космет" - r"|нужен\s+ремонт" - r"|под\s+ремонт" - r"|под\s+чистов\w+\s+отделк" - r"|предчистов", - re.IGNORECASE, - ), - "needs_repair", - ), - # ── standard: косметический / обычный / жилое состояние ───────────────── - ( - re.compile( - r"косметическ\w*\s+ремонт" - r"|косметическ\w*\s+отделк" - r"|обычн\w*\s+ремонт" - r"|жил(?:ое|ом)\s+состоян" - r"|хорош\w*\s+состоян", - re.IGNORECASE, - ), - "standard", - ), -) - - -def infer_repair_state_from_text(description: str | None) -> str | None: - """Извлечь repair_state из текста описания объявления (#622). - - Fallback-путь для повышения покрытия listings.repair_state, когда - структурное поле источника отсутствует. Проверяет паттерны в порядке - убывания силы сигнала (excellent → good → needs_repair → standard); - первое совпадение выигрывает. - - Args: - description: текст описания объявления (avito/cian/yandex) или None. - - Returns: - Одно из needs_repair / standard / good / excellent, либо None если - ни один паттерн не сработал (НЕ фабрикуем — unknown остаётся NULL). - """ - if not description: - return None - for pattern, enum_value in _TEXT_PATTERNS: - if pattern.search(description): - return enum_value - return None diff --git a/tradein-mvp/backend/app/services/scrapers/snapshot_writer.py b/tradein-mvp/backend/app/services/scrapers/snapshot_writer.py deleted file mode 100644 index 624637bd..00000000 --- a/tradein-mvp/backend/app/services/scrapers/snapshot_writer.py +++ /dev/null @@ -1,92 +0,0 @@ -"""listings_snapshots writer — point-in-time observation per listing per day. - -Таблица listings_snapshots (016_listings_snapshots.sql): - PRIMARY KEY (listing_id, snapshot_date) — максимум 1 snapshot в сутки. - ON CONFLICT DO UPDATE — берём последний за день (перезаписываем при повторном run'е). - -Используется двумя путями: - 1. SERP scrape (save_listings в base.py) — записывает цену + позицию в выдаче - каждый раз когда listing появляется в поиске. - 2. Detail backfill (save_detail_enrichment в cian_detail.py) — записывает цену - из detail-страницы для listings которые раньше не имели snapshot'а. -""" -from __future__ import annotations - -import logging -from datetime import date - -from sqlalchemy import text -from sqlalchemy.orm import Session - -logger = logging.getLogger(__name__) - - -def upsert_listing_snapshot( - db: Session, - *, - listing_id: int, - price_rub: int, - price_per_m2: int | None = None, - run_id: int | None = None, - snapshot_date: date | None = None, - position_in_serp: int | None = None, - status: str | None = "active", -) -> None: - """Записать / обновить snapshot для listing за текущий (или указанный) день. - - Идемпотентно: повторный вызов с теми же listing_id + snapshot_date перезаписывает - строку (берём последние данные за день — ON CONFLICT DO UPDATE). - - Args: - db: открытая SQLAlchemy session. Commit делает CALLER. - listing_id: PK из таблицы listings. - price_rub: текущая цена в рублях. - price_per_m2: цена за кв.м (вычисляется в caller'е, None если нет). - run_id: FK scrape_runs.id — с каким run'ом связан snapshot (None если backfill - запускается вне run'а). - snapshot_date: дата наблюдения; если None — используется CURRENT_DATE (по БД). - position_in_serp: позиция в SERP (1..60+), только при SERP-scrape. - status: 'active' / 'closed' / None. Default 'active' при обычном scrape. - """ - db.execute( - text(""" - INSERT INTO listings_snapshots - (listing_id, snapshot_date, run_id, - price_rub, price_per_m2, position_in_serp, status, observed_at) - VALUES ( - CAST(:lid AS bigint), - COALESCE(CAST(:snap_date AS date), CURRENT_DATE), - CAST(:run_id AS bigint), - CAST(:price AS bigint), - CAST(:ppm2 AS int), - CAST(:pos AS int), - :status, - NOW() - ) - ON CONFLICT (listing_id, snapshot_date) DO UPDATE SET - run_id = COALESCE(EXCLUDED.run_id, listings_snapshots.run_id), - price_rub = EXCLUDED.price_rub, - price_per_m2 = COALESCE(EXCLUDED.price_per_m2, listings_snapshots.price_per_m2), - position_in_serp = COALESCE( - EXCLUDED.position_in_serp, listings_snapshots.position_in_serp - ), - status = COALESCE(EXCLUDED.status, listings_snapshots.status), - observed_at = EXCLUDED.observed_at - """), - { - "lid": listing_id, - "snap_date": snapshot_date, - "run_id": run_id, - "price": price_rub, - "ppm2": price_per_m2, - "pos": position_in_serp, - "status": status, - }, - ) - logger.debug( - "upsert_listing_snapshot: listing_id=%s date=%s price=%s run_id=%s", - listing_id, - snapshot_date or "CURRENT_DATE", - price_rub, - run_id, - ) diff --git a/tradein-mvp/backend/app/services/scrapers/yandex_helpers.py b/tradein-mvp/backend/app/services/scrapers/yandex_helpers.py deleted file mode 100644 index a33d4501..00000000 --- a/tradein-mvp/backend/app/services/scrapers/yandex_helpers.py +++ /dev/null @@ -1,355 +0,0 @@ -"""Yandex Realty parsing helpers — JSON-LD + regex + DOM utilities for SERP, detail, newbuilding, -and valuation parsers.""" - -from __future__ import annotations - -import json -import re -from datetime import date, timedelta -from typing import Any - -from selectolax.parser import HTMLParser - -__all__ = [ - "RE_AGENCY_FOUNDED", - "RE_AGENCY_OBJECTS", - "RE_FLOOR", - "RE_JK_ID", - "RE_METRO_WALK", - "RE_OFFER_ID", - "RE_PPM2", - "RE_PRICE", - "RE_STREET_ID", - "RE_TITLE_AREA", - "RE_TITLE_ROOMS", - "RE_VIEWS", - "RE_YEAR", - "RU_MONTHS", - "RU_MONTH_NOMINATIVE", - "extract_json_ld", - "find_ld_by_type", - "parse_dmy", - "parse_house_class", - "parse_house_type", - "parse_listing_date", - "parse_ru_date", - "parse_rub", -] - -# --------------------------------------------------------------------------- -# Section 1 — JSON-LD extraction -# --------------------------------------------------------------------------- - - -def extract_json_ld(html: str) -> list[dict[str, Any]]: - """Extract all <script type='application/ld+json'> blocks from HTML. - - Returns a list of parsed JSON-LD dicts. Skips blocks that fail to parse. - """ - tree = HTMLParser(html) - results: list[dict[str, Any]] = [] - for script in tree.css('script[type="application/ld+json"]'): - text = script.text() or "" - if not text.strip(): - continue - try: - parsed = json.loads(text) - except json.JSONDecodeError: - continue - if isinstance(parsed, dict): - results.append(parsed) - elif isinstance(parsed, list): - # Some pages wrap multiple LDs in a JSON array - results.extend(item for item in parsed if isinstance(item, dict)) - return results - - -def find_ld_by_type(html: str, type_name: str) -> dict[str, Any] | None: - """Return the first JSON-LD dict whose @type equals type_name. - - If @type is a list, returns the dict if type_name is contained in it. - """ - for ld in extract_json_ld(html): - t = ld.get("@type") - if t == type_name: - return ld - if isinstance(t, list) and type_name in t: - return ld - return None - - -# --------------------------------------------------------------------------- -# Section 2 — Regex extractors (module-level compiled) -# --------------------------------------------------------------------------- - -# URL pattern extractors -RE_OFFER_ID = re.compile(r"/offer/(\d+)/?") -RE_JK_ID = re.compile(r"/novostrojka/([\w-]+?)-(\d+)/?") # group(1)=slug, group(2)=id -RE_STREET_ID = re.compile(r"/kupit/kvartira/st-([\w-]+?)-(\d+)/?") - -# Card / title text extractors -RE_TITLE_AREA = re.compile(r"(\d+[.,]?\d*)\s*м²") -RE_TITLE_ROOMS = re.compile(r"(\d+)\s*-?\s*комнатн|(студи[яюй])", re.IGNORECASE) -RE_FLOOR = re.compile(r"(\d+)\s+этаж\s+из\s+(\d+)", re.IGNORECASE) -RE_PRICE = re.compile(r"(\d[\d\s]+\d)\s*₽") -RE_PPM2 = re.compile(r"(\d[\d\s]+)\s*₽\s*за\s*м²", re.IGNORECASE) -RE_VIEWS = re.compile(r"(\d+)\s+просмотр", re.IGNORECASE) - -# Detail / NLP extractors -RE_METRO_WALK = re.compile( - r"(\d+)\s+минут\s+(?:неспешной\s+прогулки|ходьбы|пешком)", re.IGNORECASE -) -RE_YEAR = re.compile(r"\b(19\d{2}|20[0-2]\d)\b") - -# Agency block ("Год основания 1995", "150 объектов") -RE_AGENCY_FOUNDED = re.compile(r"Год\s+основания\s+(\d{4})", re.IGNORECASE) -RE_AGENCY_OBJECTS = re.compile(r"(\d+)\s+объект", re.IGNORECASE) - -# --------------------------------------------------------------------------- -# Section 3 — Russian date parsing -# --------------------------------------------------------------------------- - -RU_MONTHS: dict[str, int] = { - "января": 1, "февраля": 2, "марта": 3, "апреля": 4, "мая": 5, "июня": 6, - "июля": 7, "августа": 8, "сентября": 9, "октября": 10, "ноября": 11, "декабря": 12, -} -RU_MONTH_NOMINATIVE: dict[str, int] = { - "январь": 1, "февраль": 2, "март": 3, "апрель": 4, "май": 5, "июнь": 6, - "июль": 7, "август": 8, "сентябрь": 9, "октябрь": 10, "ноябрь": 11, "декабрь": 12, -} -RE_RU_DATE = re.compile(r"(\d{1,2})\s+(\w+)\s+(\d{4})") -RE_DMY = re.compile(r"(\d{2})\.(\d{2})\.(\d{4})") - - -def parse_ru_date(text: str | None) -> date | None: - """Parse Russian date string like '9 мая 2026' → date(2026, 5, 9). Genitive month form only.""" - if not text: - return None - m = RE_RU_DATE.search(text) - if not m: - return None - day_s, month_ru, year_s = m.groups() - month = RU_MONTHS.get(month_ru.lower()) - if not month: - return None - try: - return date(int(year_s), month, int(day_s)) - except (ValueError, TypeError): - return None - - -def parse_dmy(text: str | None) -> date | None: - """Parse DD.MM.YYYY format (used by Yandex Valuation tool history) → date.""" - if not text: - return None - m = RE_DMY.search(text) - if not m: - return None - try: - return date(int(m.group(3)), int(m.group(2)), int(m.group(1))) - except (ValueError, TypeError): - return None - - -# Relative date parsing (Yandex SERP shows "N дней назад" / "N часов назад") -_RE_REL_DATE = re.compile( - r"(?P<n>\d+)\s+(?P<unit>" - r"секунд[ауы]?|" - r"минут[ауы]?|" - r"час(?:а|ов)?|" - r"день|дн(?:я|ей?)|" - r"недел[июяь]+|" - r"месяц(?:а|ев)?|" - r"год(?:а|ов)?" - r")\s+назад", - flags=re.I, -) -_REL_UNIT_DAYS: dict[str, int] = { - "секунд": 0, "минут": 0, "час": 0, - "день": 1, "дн": 1, - "недел": 7, - "месяц": 30, - "год": 365, -} - - -def _parse_relative_date_yandex(s: str | None) -> date | None: - """Parse 'N дней/недель/месяцев назад' → date. Returns None if not matched.""" - if not s: - return None - m = _RE_REL_DATE.search(s) - if not m: - return None - n = int(m["n"]) - unit_raw = m["unit"].lower() - for prefix, days in _REL_UNIT_DAYS.items(): - if unit_raw.startswith(prefix): - return date.today() - timedelta(days=n * days) - return None - - -RE_RU_DATE_NO_YEAR = re.compile( - r"(\d{1,2})\s+(январ\w*|феврал\w*|март\w*|апрел\w*|ма[яй]|" - r"июн\w*|июл\w*|август\w*|сентябр\w*|октябр\w*|ноябр\w*|декабр\w*)" - r"(?!\s+\d{4})", - flags=re.I, -) -_RU_MONTH_PREFIXES: dict[str, int] = { - "январ": 1, "феврал": 2, "март": 3, "апрел": 4, - "ма": 5, "июн": 6, "июл": 7, "август": 8, - "сентябр": 9, "октябр": 10, "ноябр": 11, "декабр": 12, -} - - -def parse_listing_date(text: str | None) -> date | None: - """Parse listing date from Yandex card text. - - Tries in order: - 1. Keyword shortcut: 'сегодня' / 'вчера' / 'позавчера' → relative day - 2. Absolute Russian date with year: '9 мая 2026' → date(2026, 5, 9) - 3. Absolute without year: '18 мая' → current year (rollback to prev - year if month >30 дней в будущем) - 4. Relative date: 'N дней назад' → date.today() - N days - - Returns None if none match. - """ - if not text: - return None - text_lower = text.lower() - - # 1. Keyword shortcuts (common on Yandex SERP for recent cards). - if "позавчера" in text_lower: - return date.today() - timedelta(days=2) - if "вчера" in text_lower: - return date.today() - timedelta(days=1) - if "сегодня" in text_lower or "только что" in text_lower: - return date.today() - - # 2. Absolute with year (existing). - result = parse_ru_date(text) - if result is not None: - return result - - # 3. Absolute without year — infer current, rollback to prev if future. - m = RE_RU_DATE_NO_YEAR.search(text) - if m: - day = int(m.group(1)) - month_raw = m.group(2).lower() - month: int | None = None - for prefix, num in _RU_MONTH_PREFIXES.items(): - if month_raw.startswith(prefix): - month = num - break - if month is not None and 1 <= day <= 31: - today = date.today() - try: - parsed = date(today.year, month, day) - except ValueError: - parsed = None - if parsed and (parsed - today).days > 30: - try: - parsed = date(today.year - 1, month, day) - except ValueError: - parsed = None - if parsed: - return parsed - - # 4. Relative numeric. - return _parse_relative_date_yandex(text) - - -# --------------------------------------------------------------------------- -# Section 4 — House type / class NLP from description text -# --------------------------------------------------------------------------- - - -def parse_house_type(text: str | None) -> str | None: - """Extract house material type from free-form description. - - Returns: 'panel' | 'monolith_brick' | 'monolith' | 'brick' | 'block' | 'wood' | None. - Order matters: 'monolith_brick' is checked before 'monolith' and 'brick' alone. - """ - if not text: - return None - t = text.lower() - if "панельн" in t: - return "panel" - if "монолит" in t and "кирпич" in t: - return "monolith_brick" - if "монолит" in t: - return "monolith" - if "кирпичн" in t: - return "brick" - if "блочн" in t: - return "block" - if "деревянн" in t: - return "wood" - return None - - -def parse_house_class(text: str | None) -> str | None: - """Extract ЖК class from description. - - Returns: 'elite' | 'premium' | 'business' | 'comfort_plus' | 'comfort' | 'economy' | None. - """ - if not text: - return None - t = text.lower() - if re.search(r"класса?\s+элит|элитный\s+класс", t): - return "elite" - if re.search(r"класса?\s+премиум|премиум.{0,5}класс", t): - return "premium" - if re.search(r"класса?\s+бизнес", t): - return "business" - if re.search(r"класса?\s+комфорт\s*\+", t): - return "comfort_plus" - if re.search(r"класса?\s+комфорт(?!\s*\+)", t): - return "comfort" - if re.search(r"класса?\s+эконом", t): - return "economy" - return None - - -# --------------------------------------------------------------------------- -# Section 5 — Money parsing -# --------------------------------------------------------------------------- - -RE_RUB_MLN = re.compile(r"([\d,.]+)\s*млн\s*₽", re.IGNORECASE) -RE_RUB_RAW = re.compile(r"(\d[\d\s]*\d|\d)") - - -def parse_rub(text: str | None) -> int | None: - """Parse Russian price string → integer rubles. - - Supports: - '4 399 000 ₽' → 4_399_000 - '4,4 млн ₽' → 4_400_000 - '4.4 млн ₽' → 4_400_000 - '117 500' → 117_500 (raw digits without currency, e.g. ppm2) - None / 'без цены' / '' → None - """ - if not text: - return None - text = text.strip() - if not text: - return None - - m = RE_RUB_MLN.search(text) - if m: - try: - # `\s` strips also NBSP (\xa0), thin space, и т.д. — Yandex SERP - # форматирует «9\xa0800\xa0000» через NBSP, не regular space. - cleaned = re.sub(r"\s", "", m.group(1)).replace(",", ".") - return int(float(cleaned) * 1_000_000) - except (ValueError, TypeError): - pass - - m = RE_RUB_RAW.search(text) - if m: - # Same NBSP issue: `.replace(" ", "")` оставляет \xa0 → int() raises. - cleaned = re.sub(r"\s", "", m.group(1)) - try: - return int(cleaned) - except ValueError: - return None - return None diff --git a/tradein-mvp/backend/app/services/scrapers/yandex_newbuilding.py b/tradein-mvp/backend/app/services/scrapers/yandex_newbuilding.py deleted file mode 100644 index 81b666b0..00000000 --- a/tradein-mvp/backend/app/services/scrapers/yandex_newbuilding.py +++ /dev/null @@ -1,432 +0,0 @@ -"""Yandex Realty ЖК landing page parser. - -URL pattern: /{city}/kupit/novostrojka/<slug>-<id>/ -Reference target: ЖК Татлин (id=1592987, slug=tatlin) — comfort+, June 2023, -PRINZIP, rating 4.3, 1505 ratings, 353 text reviews, coords (56.855312, 60.576668). - -Fetch strategy (#974): - - Все network-запросы (ЖК-лендинг + SERP slug-resolve) идут через BrowserFetcher - (tradein-browser camoufox), а НЕ через httpx/_http_get. - - Yandex Realty — JS-heavy / anti-bot; curl_cffi / httpx не получают данные. - - BrowserFetcher.fetch(url) → str (полный HTML); вызывающий код парсит через parse(). -""" - -from __future__ import annotations - -import logging -import re -from typing import Any - -from pydantic import BaseModel, Field -from selectolax.parser import HTMLParser, Node - -from app.services.scraper_settings import get_scraper_delay -from app.services.scrapers.base import BaseScraper -from app.services.scrapers.yandex_helpers import ( - RE_JK_ID, - parse_house_class, - parse_house_type, -) - -logger = logging.getLogger(__name__) - - -# ── Models ─────────────────────────────────────────────────────────────────── - - -class JKMetroStation(BaseModel): - name: str - walk_min: int | None = None - - -class YandexNewbuildingInfo(BaseModel): - """ЖК landing scrape payload.""" - - ext_id: str # '1592987' - ext_slug: str # 'tatlin' - source_url: str - - # Identification - name: str | None = None # 'ЖК «Татлин»' - address: str | None = None # 'Екатеринбург, ул. Черепанова / ул. Готвальда' - - # Coords (inline in HTML — Yandex unique vs Avito/Cian) - lat: float | None = None - lon: float | None = None - - # Classification + commission - house_class: str | None = None # 'comfort_plus' etc. - commission_year: int | None = None - commission_month: str | None = None # 'июнь' (RU) - - # Footprint - total_floors: int | None = None # 35 - corpus_count: int | None = None # 3 - total_area_ha: float | None = None # 1.5 - - # Developer - developer_name: str | None = None # 'PRINZIP' - developer_url: str | None = None - developer_other_jk: list[str] = Field(default_factory=list) - - # Reviews - rating: float | None = None # 4.3 - ratings_count: int | None = None # 1505 — оценок - text_reviews_count: int | None = None # 353 — текстовых отзывов - - # NLP / description - description: str | None = None - - # Metro - metro_stations: list[JKMetroStation] = Field(default_factory=list) - - # House type (best-effort NLP from description) - house_type: str | None = None - - raw_payload: dict[str, Any] | None = None - - -# ── Regex constants ─────────────────────────────────────────────────────────── - -RE_FLOORS_TOWERS = re.compile(r"(\d+)-этажн\w+\s+башн", re.IGNORECASE) -RE_AREA_HA = re.compile(r"участке\s+([\d,]+)\s*га", re.IGNORECASE) -RE_COMMISSION = re.compile( - r"(?:введ[её]н|сдан)\s+в\s+эксплуатацию\s+в\s+(\w+)\s+(\d{4})", re.IGNORECASE -) -# Yandex drift (#974 follow-up): рейтинг рендерится без пробела перед «из» -# ("4.3из 5" / "4,5из 5", nbsp внутри) → \s* (а не \s+) перед «из», иначе rating=NULL -# при заполненных ratings_count/text_reviews (verified live 2026-06-15, ЖК Татлин/Рио). -RE_RATING = re.compile(r"(\d[.,]\d)\s*из\s+5") -RE_RATINGS_COUNT = re.compile(r"(\d+)\s+оценок", re.IGNORECASE) -RE_TEXT_REVIEWS = re.compile(r"Смотреть\s+все\s+(\d+)\s+отзыв", re.IGNORECASE) -RE_CORPUS_COUNT_WORD = re.compile( - r"(одн[ау]|две|три|четыре|пять|шесть|семь|восемь|\d+)\s+(?:[\d\s-]*)\s*-?\s*этажн", - re.IGNORECASE, -) -RE_METRO_INLINE = re.compile( - r"([А-ЯЁ][А-Яа-яё\s-]{2,30}?)\s+(\d+)\s*мин", -) - -# Slug из href: /{city}/kupit/novostrojka/<slug>-<id>/ -# Используется в resolve_yandex_jk_slug для извлечения slug из SERP href. -_JK_SLUG_RE = re.compile( - r"/(?:ekaterinburg|moskva|spb|[a-z-]+)/kupit/novostrojka/([a-z0-9-]+)-(\d+)/" -) - -# Ekb-specific coord ranges (extend per-city later) -LAT_RANGE = (55.5, 57.5) -LON_RANGE = (59.5, 61.5) - -# Word → number map -_WORD_NUM: dict[str, int] = { - "одна": 1, - "одну": 1, - "одной": 1, - "две": 2, - "три": 3, - "четыре": 4, - "пять": 5, - "шесть": 6, - "семь": 7, - "восемь": 8, -} - - -# ── Scraper class ───────────────────────────────────────────────────────────── - - -class YandexNewbuildingScraper(BaseScraper): - name = "yandex_realty_nb" - base_url = "https://realty.yandex.ru" - request_delay_sec = 5.0 # class default; instance value loaded from scraper_settings - - def __init__(self) -> None: - super().__init__() - self.request_delay_sec = get_scraper_delay(self.name) - - async def fetch_around(self, lat: float, lon: float, radius_m: int = 1000) -> list: # type: ignore[override] - raise NotImplementedError( - "YandexNewbuildingScraper is JK-slug-based; use fetch_jk(slug, id) instead." - ) - - async def fetch_jk( - self, jk_slug: str, jk_id: str, city: str = "ekaterinburg" - ) -> YandexNewbuildingInfo | None: - """Загрузить ЖК-лендинг через BrowserFetcher и распарсить. - - Yandex Realty — JS/anti-bot: curl/httpx не получают данные. Запрос идёт - через tradein-browser (camoufox) контейнер — единственный рабочий путь (#974). - """ - from app.services.scrapers.browser_fetcher import BrowserFetcher - - url = f"{self.base_url}/{city}/kupit/novostrojka/{jk_slug}-{jk_id}/" - try: - async with BrowserFetcher(source="yandex") as fetcher: - html = await fetcher.fetch(url) - except Exception: - logger.exception("yandex nb browser fetch failed: %s", url) - return None - if not html or len(html) < 500: - logger.warning( - "yandex nb browser returned empty/tiny HTML (%d bytes): %s", - len(html) if html else 0, - url, - ) - return None - result = self.parse(html, jk_slug=jk_slug, jk_id=jk_id, source_url=url) - return result - - def parse(self, html: str, jk_slug: str, jk_id: str, source_url: str) -> YandexNewbuildingInfo: - tree = HTMLParser(html) - body = tree.body - body_text = body.text(strip=True) if body else "" - - # Header - h1 = tree.css_first("h1") - name = h1.text(strip=True) if h1 else None - - # Sections (heading-based extraction) - location_text = _find_section_text(tree, "Расположение") or "" - description_section = ( - _find_section_text(tree, "О комплексе") - or _find_section_text(tree, "Расположение, транспортная доступность") - or location_text - ) - - # Coords — inline scan within plausible range - lat, lon = _extract_coords(html) - - # Class - house_class = parse_house_class(description_section) or parse_house_class(body_text) - - # Commission - commission_year: int | None = None - commission_month: str | None = None - c_m = RE_COMMISSION.search(description_section) or RE_COMMISSION.search(body_text) - if c_m: - commission_month = c_m.group(1).lower() - try: - commission_year = int(c_m.group(2)) - except ValueError: - pass - - # Floors + corpus + area - floors_m = RE_FLOORS_TOWERS.search(description_section) or RE_FLOORS_TOWERS.search( - body_text - ) - total_floors = int(floors_m.group(1)) if floors_m else None - - corpus_count = _parse_corpus_count(description_section or body_text) - - area_m = RE_AREA_HA.search(description_section) or RE_AREA_HA.search(body_text) - total_area_ha = float(area_m.group(1).replace(",", ".")) if area_m else None - - # Developer - dev_link = tree.css_first('[data-test="CARD_DEV_BADGE_DEVELOPER_LINK"]') - developer_name: str | None = None - developer_url: str | None = None - if dev_link is not None: - developer_name = dev_link.text(strip=True) or None - href = dev_link.attributes.get("href", "") - if href: - developer_url = ( - href if href.startswith("http") else f"https://realty.yandex.ru{href}" - ) - - # Developer's other JKs - other_jk_block = tree.css_first('[data-test="CardDevSites"]') - developer_other_jk: list[str] = [] - if other_jk_block is not None: - for jk_a in other_jk_block.css("a"): - t = jk_a.text(strip=True) - if t and t not in developer_other_jk: - developer_other_jk.append(t) - - # Reviews - rating_m = RE_RATING.search(body_text) - rating = float(rating_m.group(1).replace(",", ".")) if rating_m else None - rcount_m = RE_RATINGS_COUNT.search(body_text) - ratings_count = int(rcount_m.group(1)) if rcount_m else None - treviews_m = RE_TEXT_REVIEWS.search(body_text) - text_reviews_count = int(treviews_m.group(1)) if treviews_m else None - - # Address (header area before lat/lon block) - address = _extract_jk_address(body_text, name) - - # Metro stations - metro_stations = _parse_metro(body_text) - - # NLP house_type fallback - house_type = parse_house_type(description_section) or parse_house_type(body_text) - - return YandexNewbuildingInfo( - ext_id=jk_id, - ext_slug=jk_slug, - source_url=source_url, - name=name, - address=address, - lat=lat, - lon=lon, - house_class=house_class, - commission_year=commission_year, - commission_month=commission_month, - total_floors=total_floors, - corpus_count=corpus_count, - total_area_ha=total_area_ha, - developer_name=developer_name, - developer_url=developer_url, - developer_other_jk=developer_other_jk[:10], - rating=rating, - ratings_count=ratings_count, - text_reviews_count=text_reviews_count, - description=description_section or None, - metro_stations=metro_stations, - house_type=house_type, - raw_payload={ - "description_len": len(description_section or ""), - "body_len": len(body_text), - }, - ) - - -# ── Slug resolution via SERP ────────────────────────────────────────────────── - - -async def resolve_yandex_jk_slug( - jk_id: str, - city: str = "ekaterinburg", -) -> str | None: - """Найти Yandex Realty slug для ЖК по его ext_id (jk_id) через SERP. - - Стратегия (#974 — зеркало resolve_cian_zhk_url_via_search): - 1. Запросить поисковую страницу Yandex Realty через BrowserFetcher. - URL: /ekaterinburg/kupit/novostrojka/?siteId=<jk_id> - 2. В HTML найти первую ссылку вида /<city>/kupit/novostrojka/<slug>-<jk_id>/ - через regex _JK_SLUG_RE. - 3. Вернуть slug или None при любой ошибке. - - Caller несёт ответственность за anti-bot sleep (зеркало cian_newbuilding.py). - BrowserFetcher обязателен — Yandex Realty JS/anti-bot, httpx/curl не работают. - - Returns: - slug (str без id-суффикса), или None при ошибке / не найден. - """ - from app.services.scrapers.browser_fetcher import BrowserFetcher - - # Yandex Realty SERP: фильтр по siteId → первый результат = нужный ЖК. - # Альтернативный путь через Яндекс Поиск (web SERP) менее надёжен из-за - # вариативности разметки. Прямой realty.yandex.ru SERP — стабильнее. - serp_url = f"https://realty.yandex.ru/{city}/kupit/novostrojka/?siteId={jk_id}" - try: - async with BrowserFetcher(source="yandex") as fetcher: - html = await fetcher.fetch(serp_url) - except Exception as exc: - logger.warning("resolve_yandex_jk_slug jk_id=%s browser fetch failed: %s", jk_id, exc) - return None - - if not html: - logger.warning("resolve_yandex_jk_slug jk_id=%s: empty HTML from browser", jk_id) - return None - - # Ищем ссылку вида /{city}/kupit/novostrojka/<slug>-<id>/ - # Ограничиваем: id в ссылке должен совпадать с искомым jk_id. - for m in _JK_SLUG_RE.finditer(html): - if m.group(2) == str(jk_id): - slug = m.group(1) - logger.info("resolve_yandex_jk_slug jk_id=%s → slug=%s", jk_id, slug) - return slug - - logger.warning( - "resolve_yandex_jk_slug jk_id=%s: no matching slug in SERP HTML (markup drift?)", - jk_id, - ) - return None - - -# ── helpers ─────────────────────────────────────────────────────────────────── - - -def _extract_coords(html: str) -> tuple[float | None, float | None]: - """Scan HTML for any `<lat>.\\d+` and `<lon>.\\d+` matching plausible city range.""" - lats = [ - float(m) - for m in re.findall(r"\b(\d{2}\.\d{4,8})\b", html) - if LAT_RANGE[0] <= float(m) <= LAT_RANGE[1] - ] - lons = [ - float(m) - for m in re.findall(r"\b(\d{2}\.\d{4,8})\b", html) - if LON_RANGE[0] <= float(m) <= LON_RANGE[1] - ] - lat = lats[0] if lats else None - lon = lons[0] if lons else None - return lat, lon - - -def _find_section_text(tree: HTMLParser, heading: str) -> str | None: - for h in tree.css("h2, h3"): - if heading.lower() in (h.text(strip=True) or "").lower(): - parts: list[str] = [] - node: Node | None = h.next - while node is not None: - tag = (node.tag or "").lower() - if tag in {"h2", "h3"}: - break - txt = node.text(strip=True) if hasattr(node, "text") else "" - if txt: - parts.append(txt) - node = node.next - return " ".join(parts).strip() or None - return None - - -def _parse_corpus_count(text: str) -> int | None: - m = RE_CORPUS_COUNT_WORD.search(text) - if not m: - return None - token = m.group(1).lower() - if token.isdigit(): - return int(token) - return _WORD_NUM.get(token) - - -def _parse_metro(text: str) -> list[JKMetroStation]: - stations: list[JKMetroStation] = [] - for m in RE_METRO_INLINE.finditer(text): - name = m.group(1).strip() - # filter common noise - if not 2 <= len(name) <= 40 or "ходьб" in name.lower() or "минут" in name.lower(): - continue - try: - stations.append(JKMetroStation(name=name, walk_min=int(m.group(2)))) - except ValueError: - continue - if len(stations) >= 5: - break - return stations - - -def _extract_jk_address(body_text: str, name: str | None) -> str | None: - """Find address line — typically right after JK name, before metro entries.""" - if name and name in body_text: - after_name = body_text.split(name, 1)[1] - # Match 'Город, ул. X / ул. Y' until first metro mention - m = re.match( - r"\s*([А-ЯЁ][^•]{5,150}?)(?:\s+[А-ЯЁ][а-яё]+\s+\d+\s*мин|\d+\s+оценок|$)", - after_name, - ) - if m: - addr = m.group(1).strip().rstrip(",").strip() - if 10 < len(addr) < 200: - return addr - return None - - -# Expose RE_JK_ID at module level for external use (e.g. SERP link → newbuilding detail) -__all__ = [ - "RE_JK_ID", - "JKMetroStation", - "YandexNewbuildingInfo", - "YandexNewbuildingScraper", - "resolve_yandex_jk_slug", -] diff --git a/tradein-mvp/backend/app/services/yandex_price_history.py b/tradein-mvp/backend/app/services/yandex_price_history.py index a01caa93..42f364a5 100644 --- a/tradein-mvp/backend/app/services/yandex_price_history.py +++ b/tradein-mvp/backend/app/services/yandex_price_history.py @@ -29,10 +29,10 @@ import logging from datetime import UTC, datetime, timedelta # #2306: ScrapedLot migrated to scraper_kit (byte-identical field/method surface to -# legacy app.services.scrapers.base, см. tests/test_scraper_kit_pricehistory_session_parity.py). -# record_yandex_price_history reads lot.* via duck-typing only — no isinstance checks — -# так что kit- и legacy-скрапперы (оба конструируют ScrapedLot по (source, source_id, -# price_rub, price_previous_rub, price_trend)) остаются взаимозаменяемы для этой функции. +# legacy base-module ScrapedLot, deleted #2397 финальный шаг E — kit единственный +# живой путь). record_yandex_price_history reads lot.* via duck-typing only — no +# isinstance checks — так что любой ScrapedLot-совместимый объект (source, source_id, +# price_rub, price_previous_rub, price_trend) остаётся взаимозаменяем для этой функции. from scraper_kit.base import ScrapedLot from sqlalchemy import text from sqlalchemy.orm import Session diff --git a/tradein-mvp/backend/tests/scrapers/test_admin_domclick_ingest_kit_parity.py b/tradein-mvp/backend/tests/scrapers/test_admin_domclick_ingest_kit_parity.py deleted file mode 100644 index b37e5d57..00000000 --- a/tradein-mvp/backend/tests/scrapers/test_admin_domclick_ingest_kit_parity.py +++ /dev/null @@ -1,78 +0,0 @@ -"""Parity-proof: `scripts/ingest_domclick_jsonl.py` (#2305, Group A) — legacy vs scraper_kit. - -`scripts/ingest_domclick_jsonl.py` теперь строит `ScrapedLot` из -`scraper_kit.base` (вместо `app.services.scrapers.base`) и вызывает -`scraper_kit.base.save_listings` (matcher/region_code инжектируются — см. -`app/services/scraper_adapters.RealMatcherAdapter` + `DEFAULT_REGION_CODE`). - -`DomClickDetailEnrichment`/`save_detail_enrichment` изначально планировались -остаться на legacy: на момент начала миграции admin.py `domclick_detail.py` не -имел kit-эквивалента (см. audit `Scraper_Kit_Legacy_Dependency_Audit_0703`). -Issue #2307 (Group D, PR #2318) портировал `domclick_detail.py` в kit ПАРАЛЛЕЛЬНО -с этой задачей и явно оставил `scripts/ingest_domclick_jsonl.py` untouched -("the only other domclick_detail caller is scripts/ingest_domclick_jsonl.py, -also untouched") — поэтому переключение этого caller'а сделано ЗДЕСЬ, в рамках -#2305, теперь когда kit-эквивалент существует. - -Легаси `app.services.scrapers.domclick_detail` удалён (#2277 финальный шаг -scraper_kit-миграции, 0 runtime importers, `scripts/ingest_domclick_jsonl.py` уже -100% на kit-стороне для `DomClickDetailEnrichment`) — legacy-vs-kit parity для -`DomClickDetailEnrichment`-конструирования убрана вместе с ним; `ScrapedLot` -parity (`app.services.scrapers.base`, всё ещё легаси/живой модуль) остаётся. -""" - -from __future__ import annotations - -import os - -os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db") - -from scraper_kit.base import ScrapedLot as KitScrapedLot - -from app.services.scrapers.base import ScrapedLot as LegacyScrapedLot -from tests.support.parity import assert_parity - -# Аналог одной ok-записи JSONL-раннера (см. ingest_domclick_jsonl._build_lot). -_LOT_KWARGS = { - "source": "domklik", - "source_url": "https://ekaterinburg.domclick.ru/card/sale__flat__2075729321", - "source_id": "2075729321", - "address": "Екатеринбург, ул. Тестовая, 1", - "lat": 56.83, - "lon": 60.6, - "rooms": 2, - "area_m2": 54.3, - "floor": 5, - "total_floors": 9, - "year_built": 2015, - "price_rub": 8500000, - "listing_segment": "vtorichka", - "repair_state": "euro", - "raw_payload": {"is_sber_collateral": False, "square_price": 156538, "avm": {}}, -} - - -def _build_legacy_lot(rec: dict[str, object]) -> LegacyScrapedLot: - return LegacyScrapedLot(**rec) # type: ignore[arg-type] - - -def _build_kit_lot(rec: dict[str, object]) -> KitScrapedLot: - return KitScrapedLot(**rec) # type: ignore[arg-type] - - -def test_scraped_lot_are_class_distinct() -> None: - """Sanity: legacy/kit ScrapedLot — разные классы (pydantic), обычный `==` даёт False.""" - legacy_result = _build_legacy_lot(_LOT_KWARGS) - kit_result = _build_kit_lot(_LOT_KWARGS) - - assert type(legacy_result) is not type(kit_result) - assert legacy_result.model_dump() == kit_result.model_dump() - - -def test_parity_ingest_domclick_scraped_lot() -> None: - """`ingest_domclick_jsonl._build_lot()` kwargs → ScrapedLot(legacy) vs ScrapedLot(kit).""" - assert_parity( - legacy_fn=_build_legacy_lot, - kit_fn=_build_kit_lot, - fixtures=[_LOT_KWARGS], - ) diff --git a/tradein-mvp/backend/tests/scrapers/test_avito_403_backconnect_reconnect.py b/tradein-mvp/backend/tests/scrapers/test_avito_403_backconnect_reconnect.py deleted file mode 100644 index b9fcb6dd..00000000 --- a/tradein-mvp/backend/tests/scrapers/test_avito_403_backconnect_reconnect.py +++ /dev/null @@ -1,220 +0,0 @@ -"""curl_cffi 403/firewall reconnect-retry под backconnect-прокси. - -Под backconnect rotating-прокси (mproxy.site, нет changeip-URL) одиночный HTTP 403 -или firewall-интерстициал ронял весь avito_full_load sweep через AvitoBlockedError → -mark_banned. Каждое новое соединение через backconnect даёт свежий exit-IP, поэтому -ретраим пересозданием curl_cffi-сессии до _AVITO_403_MAX_RETRIES раз. - -curl_cffi-сессия всегда egress через scraper_proxy_url (backconnect mproxy), поэтому -reconnect-retry гейтим ТОЛЬКО на наличие proxy — независимо от того, задан ли changeip-URL -(тот относится к ДРУГОМУ, browser/auv-прокси). Включаем через scraper_proxy_url_env -(scraper_proxy_url — read-only property, его не monkeypatch'ишь). -""" - -from unittest.mock import AsyncMock, MagicMock, patch - -import pytest - -from app.services.scrapers import avito as avito_mod -from app.services.scrapers.avito import AvitoScraper -from app.services.scrapers.avito_exceptions import AvitoBlockedError - -# Валидная карточка SERP (одна) — successful page после reconnect. -_CARD_HTML = ( - "<html><body>" - '<div data-marker="item" data-item-id="123456789">' - '<a data-marker="item-title" href="/ekaterinburg/kvartiry/test-123456789">' - "2-к. квартира, 54,3 м², 5/9 эт." - "</a>" - '<meta itemprop="price" content="4500000"/>' - "</div>" - "</body></html>" -) - -# Firewall-интерстициал (HTTP 200, не 403). -_FIREWALL_HTML = ( - "<html><head><title>Доступ ограничен: проблема с IP" - '

...
' -) - - -def _enable_backconnect(monkeypatch: pytest.MonkeyPatch) -> None: - """scraper_proxy_url задан (через env-поле), changeip-URL — нет → backconnect.""" - monkeypatch.setattr(avito_mod.settings, "scraper_proxy_url_env", "http://u:p@mproxy.site:14619") - monkeypatch.setattr(avito_mod.settings, "avito_proxy_url", None) - monkeypatch.setattr(avito_mod.settings, "avito_proxy_rotate_url", None) - - -@pytest.mark.asyncio -async def test_backconnect_403_403_200_reconnect_succeeds( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """backconnect: 403 → 403 → 200 → reconnect-retry возвращает HTML, _reset_cffi дёрнут 2×.""" - _enable_backconnect(monkeypatch) - - scraper = AvitoScraper() - r403 = MagicMock() - r403.status_code = 403 - r403.text = "" - ok = MagicMock() - ok.status_code = 200 - ok.text = _CARD_HTML - mock_session = AsyncMock() - mock_session.get = AsyncMock(side_effect=[r403, r403, ok]) - scraper._cffi = mock_session - - # _reset_cffi не должен реально пересоздавать сессию (иначе .get-side_effect - # потеряется) — патчим на no-op счётчик, проверяем что вызвался. - reset = AsyncMock() - with ( - patch("app.services.scrapers.avito.asyncio.sleep", AsyncMock()), - patch.object(AvitoScraper, "_reset_cffi", reset), - patch.object(AvitoScraper, "_rotate_ip", AsyncMock(return_value=True)) as rot, - ): - html = await scraper._fetch_serp_html("https://www.avito.ru/x", page=1) - - assert html == _CARD_HTML - assert reset.await_count == 2 # 403, 403 → два reconnect перед 200 - rot.assert_not_awaited() # backconnect не дёргает changeip-ротацию - assert mock_session.get.await_count == 3 - - -@pytest.mark.asyncio -async def test_backconnect_firewall_reconnect_succeeds( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """backconnect: firewall-200 → 200 → reconnect-retry (firewall тоже триггерит).""" - _enable_backconnect(monkeypatch) - - scraper = AvitoScraper() - fw = MagicMock() - fw.status_code = 200 - fw.text = _FIREWALL_HTML - ok = MagicMock() - ok.status_code = 200 - ok.text = _CARD_HTML - mock_session = AsyncMock() - mock_session.get = AsyncMock(side_effect=[fw, ok]) - scraper._cffi = mock_session - - reset = AsyncMock() - with ( - patch("app.services.scrapers.avito.asyncio.sleep", AsyncMock()), - patch.object(AvitoScraper, "_reset_cffi", reset), - ): - html = await scraper._fetch_serp_html("https://www.avito.ru/x", page=1) - - assert html == _CARD_HTML - assert reset.await_count == 1 - - -@pytest.mark.asyncio -async def test_backconnect_403_exhausted_raises_blocked( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """backconnect: 403 _AVITO_403_MAX_RETRIES+1 раз → всё равно AvitoBlockedError.""" - _enable_backconnect(monkeypatch) - - scraper = AvitoScraper() - r403 = MagicMock() - r403.status_code = 403 - r403.text = "" - mock_session = AsyncMock() - mock_session.get = AsyncMock(return_value=r403) - scraper._cffi = mock_session - - reset = AsyncMock() - with ( - patch("app.services.scrapers.avito.asyncio.sleep", AsyncMock()), - patch.object(AvitoScraper, "_reset_cffi", reset), - ): - with pytest.raises(AvitoBlockedError, match="IP banned"): - await scraper._fetch_serp_html("https://www.avito.ru/x", page=1) - - # ровно _AVITO_403_MAX_RETRIES reconnect, потом raise - assert reset.await_count == avito_mod._AVITO_403_MAX_RETRIES - # get: 1 первичный + _AVITO_403_MAX_RETRIES повторов - assert mock_session.get.await_count == avito_mod._AVITO_403_MAX_RETRIES + 1 - - -@pytest.mark.asyncio -async def test_reconnect_fires_even_when_rotate_url_set( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """Прод-конфиг run-212 регрессия: scraper_proxy_url + leftover avito_proxy_rotate_url + - max_rotations=0 → reconnect-retry ВСЁ РАВНО срабатывает (gate только на наличие proxy). - - changeip-URL относится к ДРУГОМУ (auv/browser) прокси; curl_cffi egress через - scraper_proxy_url, поэтому пересоздание сессии меняет exit-IP независимо от changeip. - С max_rotations=0 changeip-путь — no-op, fallback-reconnect снимает 403. - """ - monkeypatch.setattr( - avito_mod.settings, "avito_proxy_rotate_url", "https://changeip.example/?proxy_key=x" - ) - monkeypatch.setattr(avito_mod.settings, "avito_proxy_max_rotations", 0) - monkeypatch.setattr(avito_mod.settings, "scraper_proxy_url_env", "http://u:p@mproxy.site:14619") - - scraper = AvitoScraper() - r403 = MagicMock() - r403.status_code = 403 - r403.text = "" - ok = MagicMock() - ok.status_code = 200 - ok.text = _CARD_HTML - mock_session = AsyncMock() - mock_session.get = AsyncMock(side_effect=[r403, r403, ok]) - scraper._cffi = mock_session - - reset = AsyncMock() - with ( - patch("app.services.scrapers.avito.asyncio.sleep", AsyncMock()), - patch.object(AvitoScraper, "_reset_cffi", reset), - patch.object(AvitoScraper, "_rotate_ip", AsyncMock(return_value=True)) as rot, - ): - html = await scraper._fetch_serp_html("https://www.avito.ru/x", page=1) - - assert html == _CARD_HTML # 403→403→200 снят reconnect'ом, нет raise - assert reset.await_count == 2 # reconnect сработал несмотря на rotate_url - rot.assert_not_awaited() # max_rotations=0 → changeip-путь не дёргается - assert mock_session.get.await_count == 3 - - -@pytest.mark.asyncio -async def test_changeip_path_tried_first_when_rotations_available( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """changeip precedence: scraper_proxy_url + rotate_url + max_rotations=1 → на 403 - changeip-ротация пробуется ПЕРЕД reconnect-fallback. - - Гарантирует, что новый gate не сломал приоритет legacy changeip-пути. - """ - monkeypatch.setattr( - avito_mod.settings, "avito_proxy_rotate_url", "https://changeip.example/?proxy_key=x" - ) - monkeypatch.setattr(avito_mod.settings, "avito_proxy_max_rotations", 1) - monkeypatch.setattr(avito_mod.settings, "scraper_proxy_url_env", "http://u:p@mproxy.site:14619") - - scraper = AvitoScraper() - r403 = MagicMock() - r403.status_code = 403 - r403.text = "" - ok = MagicMock() - ok.status_code = 200 - ok.text = _CARD_HTML - mock_session = AsyncMock() - mock_session.get = AsyncMock(side_effect=[r403, ok]) - scraper._cffi = mock_session - - reset = AsyncMock() - rot = AsyncMock(return_value=True) - with ( - patch("app.services.scrapers.avito.asyncio.sleep", AsyncMock()), - patch.object(AvitoScraper, "_reset_cffi", reset), - patch.object(AvitoScraper, "_rotate_ip", rot), - ): - html = await scraper._fetch_serp_html("https://www.avito.ru/x", page=1) - - assert html == _CARD_HTML - rot.assert_awaited_once() # changeip-путь использован первым на 403 - reset.assert_not_awaited() # до reconnect-fallback не дошло - assert mock_session.get.await_count == 2 diff --git a/tradein-mvp/backend/tests/scrapers/test_avito_429_bucket_skip.py b/tradein-mvp/backend/tests/scrapers/test_avito_429_bucket_skip.py deleted file mode 100644 index 6b40270e..00000000 --- a/tradein-mvp/backend/tests/scrapers/test_avito_429_bucket_skip.py +++ /dev/null @@ -1,242 +0,0 @@ -"""429-hardening: widened short-retry + reconnect-fallback в _fetch_serp_html и -per-bucket skip в fetch_all_secondary. - -Run 216 (прод): merged 403-reconnect-retry восстановил два 403, но прогон всё равно -забанился на HTTP 429 на page 36 (deep-pagination dense-бакета). 429 под backconnect = -transient conn-limit (лимит 5 соединений), не IP-ban. - -Две защиты: - A. _fetch_serp_html: short-retry того же session расширен до _AVITO_429_MAX_RETRIES=8; - на исчерпании под backconnect — ещё _AVITO_429_RECONNECT_RETRIES попыток через - _reset_cffi (освобождает conn-slot) перед AvitoRateLimitedError. - B. fetch_all_secondary: одиночный заблокированный бакет скипается (continue), sweep - продолжается; только _AVITO_SWEEP_MAX_CONSECUTIVE_BLOCKED ПОДРЯД блоков = hard ban - → re-raise (сохраняет real-ban detection). -""" - -from unittest.mock import AsyncMock, MagicMock, patch - -import pytest - -from app.services.scrapers import avito as avito_mod -from app.services.scrapers.avito import AvitoScraper -from app.services.scrapers.avito_exceptions import AvitoBlockedError, AvitoRateLimitedError - -# Валидная карточка SERP (одна) — successful page. -_CARD_HTML = ( - "" - '" - "" -) - - -def _enable_backconnect(monkeypatch: pytest.MonkeyPatch) -> None: - """scraper_proxy_url задан (через env-поле), changeip-URL — нет → backconnect.""" - monkeypatch.setattr(avito_mod.settings, "scraper_proxy_url_env", "http://u:p@mproxy.site:14619") - monkeypatch.setattr(avito_mod.settings, "avito_proxy_url", None) - monkeypatch.setattr(avito_mod.settings, "avito_proxy_rotate_url", None) - - -def _resp(status: int, text: str = "") -> MagicMock: - r = MagicMock() - r.status_code = status - r.text = text - return r - - -# --- A. _fetch_serp_html 429 short-retry + reconnect-fallback ------------------- - - -@pytest.mark.asyncio -async def test_429_short_retries_then_200_within_widened_budget( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """429×6 (< _AVITO_429_MAX_RETRIES=8) → 200: short-retry того же session снимает, - _reset_cffi НЕ дёргается (reconnect-fallback не нужен).""" - _enable_backconnect(monkeypatch) - n_429 = 6 - assert n_429 < avito_mod._AVITO_429_MAX_RETRIES - - scraper = AvitoScraper() - seq = [_resp(429) for _ in range(n_429)] + [_resp(200, _CARD_HTML)] - mock_session = AsyncMock() - mock_session.get = AsyncMock(side_effect=seq) - scraper._cffi = mock_session - - reset = AsyncMock() - with ( - patch("app.services.scrapers.avito.asyncio.sleep", AsyncMock()), - patch.object(AvitoScraper, "_reset_cffi", reset), - ): - html = await scraper._fetch_serp_html("https://www.avito.ru/x", page=36) - - assert html == _CARD_HTML - reset.assert_not_awaited() # short-retry хватило, reconnect не понадобился - assert mock_session.get.await_count == n_429 + 1 - - -@pytest.mark.asyncio -async def test_429_exhausted_then_reconnect_then_200( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """short-retry исчерпан (429×_AVITO_429_MAX_RETRIES) → reconnect-fallback (_reset_cffi) - → 200 успех. Подтверждает: одна reconnect-фаза освобождает conn-slot.""" - _enable_backconnect(monkeypatch) - n_short = avito_mod._AVITO_429_MAX_RETRIES - - scraper = AvitoScraper() - # _AVITO_429_MAX_RETRIES коротких retry (все 429), ещё один 429 проваливается в - # reconnect-block, после _reset_cffi следующий ответ 200. - seq = [_resp(429) for _ in range(n_short + 1)] + [_resp(200, _CARD_HTML)] - mock_session = AsyncMock() - mock_session.get = AsyncMock(side_effect=seq) - scraper._cffi = mock_session - - reset = AsyncMock() - with ( - patch("app.services.scrapers.avito.asyncio.sleep", AsyncMock()), - patch.object(AvitoScraper, "_reset_cffi", reset), - ): - html = await scraper._fetch_serp_html("https://www.avito.ru/x", page=36) - - assert html == _CARD_HTML - reset.assert_awaited() # reconnect-fallback сработал - assert reset.await_count == 1 - - -@pytest.mark.asyncio -async def test_429_fully_exhausted_raises_rate_limited( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """429 на каждый запрос: short-retry + все reconnect-fallback исчерпаны → - AvitoRateLimitedError. _reset_cffi дёрнут ровно _AVITO_429_RECONNECT_RETRIES раз.""" - _enable_backconnect(monkeypatch) - - scraper = AvitoScraper() - mock_session = AsyncMock() - mock_session.get = AsyncMock(return_value=_resp(429)) - scraper._cffi = mock_session - - reset = AsyncMock() - with ( - patch("app.services.scrapers.avito.asyncio.sleep", AsyncMock()), - patch.object(AvitoScraper, "_reset_cffi", reset), - ): - with pytest.raises(AvitoRateLimitedError, match="rate limited"): - await scraper._fetch_serp_html("https://www.avito.ru/x", page=36) - - assert reset.await_count == avito_mod._AVITO_429_RECONNECT_RETRIES - # get: 1 первичный + _AVITO_429_MAX_RETRIES short-retry + _AVITO_429_RECONNECT_RETRIES - assert ( - mock_session.get.await_count - == 1 + avito_mod._AVITO_429_MAX_RETRIES + avito_mod._AVITO_429_RECONNECT_RETRIES - ) - - -# --- B. fetch_all_secondary per-bucket skip ----------------------------------- - - -def _single_room() -> list[tuple[str, str]]: - """Один room-bucket → число брекетов = len(_AVITO_PRICE_SEED_BRACKETS).""" - return [("2-комнатные", "2-komnatnye")] - - -@pytest.mark.asyncio -async def test_one_blocked_bucket_skipped_sweep_completes( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """_walk_price_range кидает AvitoRateLimitedError на 1-м брекете, успех на остальных → - sweep завершается без raise, заблокированный бакет посчитан.""" - n_brackets = len(avito_mod._AVITO_PRICE_SEED_BRACKETS) - assert n_brackets >= 2 - - scraper = AvitoScraper() - calls = {"n": 0} - - async def fake_walk(self: object, **kwargs: object) -> None: - calls["n"] += 1 - if calls["n"] == 1: - raise AvitoRateLimitedError("429 deep pagination") - # успех: ничего не добавляем в seen (не нужно для теста контракта) - - with patch.object(AvitoScraper, "_walk_price_range", fake_walk): - result = await scraper.fetch_all_secondary(rooms_buckets=_single_room()) - - assert result == [] # seen пуст, но без raise - assert calls["n"] == n_brackets # все брекеты пройдены (1 скипнут, остальные ок) - - -@pytest.mark.asyncio -async def test_consecutive_blocked_threshold_reraises( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """_walk_price_range кидает _AVITO_SWEEP_MAX_CONSECUTIVE_BLOCKED раз ПОДРЯД → - последнее исключение пробрасывается (real hard-ban detection сохранена).""" - thr = avito_mod._AVITO_SWEEP_MAX_CONSECUTIVE_BLOCKED - assert len(avito_mod._AVITO_PRICE_SEED_BRACKETS) >= thr - - scraper = AvitoScraper() - - async def always_block(self: object, **kwargs: object) -> None: - raise AvitoBlockedError("IP banned") - - with patch.object(AvitoScraper, "_walk_price_range", always_block): - with pytest.raises(AvitoBlockedError, match="IP banned"): - await scraper.fetch_all_secondary(rooms_buckets=_single_room()) - - -@pytest.mark.asyncio -async def test_success_between_blocks_resets_counter( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """Чередование block/success НЕ срабатывает порог: успех между блоками сбрасывает - consecutive_blocked. Паттерн block,success,block,success,... → sweep завершается.""" - thr = avito_mod._AVITO_SWEEP_MAX_CONSECUTIVE_BLOCKED - n_brackets = len(avito_mod._AVITO_PRICE_SEED_BRACKETS) - # Нужно достаточно брекетов чтобы суммарно было > thr блоков, но не подряд. - assert n_brackets > thr - - scraper = AvitoScraper() - calls = {"n": 0} - - async def alternate(self: object, **kwargs: object) -> None: - calls["n"] += 1 - # нечётные вызовы блокируются, чётные — успех → никогда не thr подряд - if calls["n"] % 2 == 1: - raise AvitoRateLimitedError("429 transient") - - with patch.object(AvitoScraper, "_walk_price_range", alternate): - result = await scraper.fetch_all_secondary(rooms_buckets=_single_room()) - - assert result == [] # без raise — порог не пробит - assert calls["n"] == n_brackets - - -@pytest.mark.asyncio -async def test_incremental_path_also_skips_blocked_bucket( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """Per-bucket skip применяется и к incremental-пути (_paginate_incremental_bracket).""" - from datetime import date - - n_brackets = len(avito_mod._AVITO_PRICE_SEED_BRACKETS) - scraper = AvitoScraper() - calls = {"n": 0} - - async def fake_incr(self: object, **kwargs: object) -> None: - calls["n"] += 1 - if calls["n"] == 1: - raise AvitoRateLimitedError("429") - - with patch.object(AvitoScraper, "_paginate_incremental_bracket", fake_incr): - result = await scraper.fetch_all_secondary( - rooms_buckets=_single_room(), since=date(2026, 1, 1) - ) - - assert result == [] - assert calls["n"] == n_brackets diff --git a/tradein-mvp/backend/tests/scrapers/test_avito_anti_bot.py b/tradein-mvp/backend/tests/scrapers/test_avito_anti_bot.py deleted file mode 100644 index 219ffed3..00000000 --- a/tradein-mvp/backend/tests/scrapers/test_avito_anti_bot.py +++ /dev/null @@ -1,425 +0,0 @@ -"""Test anti-bot hardening: blocked exceptions + early abort.""" - -from unittest.mock import AsyncMock, MagicMock, patch - -import pytest - -from app.services.scrapers.avito_exceptions import ( - AvitoBlockedError, - AvitoContentBlockedError, - AvitoRateLimitedError, -) - -# ── fetch_detail tests ────────────────────────────────────────── - - -@pytest.mark.asyncio -async def test_fetch_detail_raises_blocked_on_403() -> None: - """fetch_detail должен raise AvitoBlockedError на HTTP 403.""" - from app.services.scrapers.avito_detail import fetch_detail - - mock_session = AsyncMock() - mock_response = MagicMock() - mock_response.status_code = 403 - mock_session.get = AsyncMock(return_value=mock_response) - - with pytest.raises(AvitoBlockedError, match="403"): - await fetch_detail("/ekaterinburg/kvartiry/test-123", cffi_session=mock_session) - - -@pytest.mark.asyncio -async def test_fetch_detail_raises_rate_limited_on_429() -> None: - """fetch_detail должен raise AvitoRateLimitedError на HTTP 429.""" - from app.services.scrapers.avito_detail import fetch_detail - - mock_session = AsyncMock() - mock_response = MagicMock() - mock_response.status_code = 429 - mock_session.get = AsyncMock(return_value=mock_response) - - with pytest.raises(AvitoRateLimitedError, match="429"): - await fetch_detail("/ekaterinburg/kvartiry/test-123", cffi_session=mock_session) - - -@pytest.mark.asyncio -async def test_fetch_detail_raises_value_error_on_other_non_200() -> None: - """fetch_detail должен raise ValueError на прочие non-200 (не 403/429).""" - from app.services.scrapers.avito_detail import fetch_detail - - mock_session = AsyncMock() - mock_response = MagicMock() - mock_response.status_code = 503 - mock_session.get = AsyncMock(return_value=mock_response) - - with pytest.raises(ValueError, match="503"): - await fetch_detail("/ekaterinburg/kvartiry/test-123", cffi_session=mock_session) - - -# ── SERP (avito.py) tests ─────────────────────────────────────── - - -@pytest.mark.asyncio -async def test_fetch_around_raises_content_blocked_on_page1_empty_html() -> None: - """fetch_around должен raise AvitoContentBlockedError на HTTP 200 + 0 карточек на странице 1.""" - from app.services.scrapers.avito import AvitoScraper - - scraper = AvitoScraper() - mock_session = AsyncMock() - mock_response = MagicMock() - mock_response.status_code = 200 - # Garbage HTML: нет data-marker="item" → _parse_html вернёт [] - mock_response.text = "

captcha challenge

" - mock_session.get = AsyncMock(return_value=mock_response) - scraper._cffi = mock_session - - with pytest.raises(AvitoContentBlockedError, match="content-block"): - await scraper.fetch_around(56.84, 60.60, 1000, pages=1) - - -@pytest.mark.asyncio -async def test_fetch_around_content_blocked_is_subtype_of_blocked() -> None: - """AvitoContentBlockedError должен быть поймать как AvitoBlockedError (наследование).""" - from app.services.scrapers.avito import AvitoScraper - - scraper = AvitoScraper() - mock_session = AsyncMock() - mock_response = MagicMock() - mock_response.status_code = 200 - mock_response.text = "" - mock_session.get = AsyncMock(return_value=mock_response) - scraper._cffi = mock_session - - with pytest.raises(AvitoBlockedError): - await scraper.fetch_around(56.84, 60.60, 1000, pages=1) - - -@pytest.mark.asyncio -async def test_fetch_around_page_gt1_empty_returns_gracefully() -> None: - """fetch_around на page>1 с 0 карточек должен тихо завершить пагинацию (не raise).""" - from app.services.scrapers.avito import AvitoScraper - - scraper = AvitoScraper() - mock_session = AsyncMock() - - # Страница 1: нормальный HTML с одной карточкой - valid_html = ( - "" - '" - "" - ) - # Страница 2: пустой HTML → end of pagination - empty_html = "

no listings

" - - page1_resp = MagicMock() - page1_resp.status_code = 200 - page1_resp.text = valid_html - - page2_resp = MagicMock() - page2_resp.status_code = 200 - page2_resp.text = empty_html - - mock_session.get = AsyncMock(side_effect=[page1_resp, page2_resp]) - scraper._cffi = mock_session - - with patch("app.services.scrapers.avito.AvitoScraper.sleep_between_requests", AsyncMock()): - result = await scraper.fetch_around(56.84, 60.60, 1000, pages=2) - - # Должен вернуть лоты со страницы 1, не raise на странице 2 - assert len(result) >= 1 - - -@pytest.mark.asyncio -async def test_fetch_around_raises_blocked_on_403() -> None: - """AvitoScraper.fetch_around должен raise AvitoBlockedError на HTTP 403.""" - from app.services.scrapers.avito import AvitoScraper - - scraper = AvitoScraper() - mock_session = AsyncMock() - mock_response = MagicMock() - mock_response.status_code = 403 - mock_session.get = AsyncMock(return_value=mock_response) - scraper._cffi = mock_session - - with pytest.raises(AvitoBlockedError): - await scraper.fetch_around(56.84, 60.60, 1000, pages=1) - - -@pytest.mark.asyncio -async def test_fetch_around_raises_rate_limited_on_429() -> None: - """AvitoScraper.fetch_around должен raise AvitoRateLimitedError на persistent HTTP 429. - - 429 (backconnect conn-limit) сначала проходит короткие retry; при отсутствии - rotate-url после исчерпания retry сохраняется старый fallback — AvitoRateLimitedError. - """ - from app.services.scrapers.avito import AvitoScraper - - scraper = AvitoScraper() - mock_session = AsyncMock() - mock_response = MagicMock() - mock_response.status_code = 429 - mock_session.get = AsyncMock(return_value=mock_response) - scraper._cffi = mock_session - - with patch("app.services.scrapers.avito.asyncio.sleep", AsyncMock()): - with pytest.raises(AvitoRateLimitedError): - await scraper.fetch_around(56.84, 60.60, 1000, pages=1) - - -# ── #623 firewall-детект + ротация IP ──────────────────────────── -# -# Legacy `run_avito_pipeline` / `run_avito_city_sweep` orchestration-level tests -# (pipeline consecutive-block abort, inter-detail sleep, city-sweep mark_banned) -# удалены вместе с `app.services.scrape_pipeline` (#2397 Part E1) — эквивалентная -# regression-coverage теперь в `test_scraper_kit_pipeline_parity.py` (kit -# `run_avito_city_sweep`: ban/rotation state-machine, detail-block propagation). - -_CARD_HTML = ( - "" - '" - "" -) - -# Firewall-интерстициал Avito (HTTP 200, не 403) — IP-блок datacenter. -_FIREWALL_HTML = ( - "Доступ ограничен: проблема с IP" - '
...
' -) - - -@pytest.mark.asyncio -async def test_fetch_around_raises_blocked_on_firewall_html_200() -> None: - """HTTP 200 + firewall-страница «Доступ ограничен» → AvitoBlockedError (#623). - - Avito банит datacenter-IP отдачей firewall-интерстициала с кодом 200 (не 403) — - раньше это молча проходило как «0 cards / content-block», теперь явный IP-блок. - """ - from app.services.scrapers.avito import AvitoScraper - - scraper = AvitoScraper() - mock_session = AsyncMock() - mock_response = MagicMock() - mock_response.status_code = 200 - mock_response.text = _FIREWALL_HTML - mock_session.get = AsyncMock(return_value=mock_response) - scraper._cffi = mock_session - - with pytest.raises(AvitoBlockedError, match="firewall"): - await scraper.fetch_around(56.84, 60.60, 1000, pages=1) - - -@pytest.mark.asyncio -async def test_fetch_around_rotates_ip_then_succeeds(monkeypatch: pytest.MonkeyPatch) -> None: - """При firewall + заданной changeip-ссылке: ротация IP и повтор страницы (#623).""" - from app.services.scrapers import avito as avito_mod - from app.services.scrapers.avito import AvitoScraper - - monkeypatch.setattr( - avito_mod.settings, "avito_proxy_rotate_url", "https://changeip.example/?proxy_key=x" - ) - monkeypatch.setattr(avito_mod.settings, "avito_proxy_max_rotations", 1) - - scraper = AvitoScraper() - fw = MagicMock() - fw.status_code = 200 - fw.text = _FIREWALL_HTML - ok = MagicMock() - ok.status_code = 200 - ok.text = _CARD_HTML - mock_session = AsyncMock() - mock_session.get = AsyncMock(side_effect=[fw, ok]) - scraper._cffi = mock_session - - with ( - patch.object(AvitoScraper, "_rotate_ip", AsyncMock(return_value=True)) as rot, - patch.object(AvitoScraper, "sleep_between_requests", AsyncMock()), - ): - result = await scraper.fetch_around(56.84, 60.60, 1000, pages=1) - - rot.assert_awaited_once() - assert len(result) == 1 - assert mock_session.get.await_count == 2 # firewall → rotate → retry OK - - -@pytest.mark.asyncio -async def test_fetch_around_blocked_when_rotation_exhausted( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """Если ротация не сняла бан (max_rotations исчерпан) → AvitoBlockedError (#623).""" - from app.services.scrapers import avito as avito_mod - from app.services.scrapers.avito import AvitoScraper - - monkeypatch.setattr( - avito_mod.settings, "avito_proxy_rotate_url", "https://changeip.example/?proxy_key=x" - ) - monkeypatch.setattr(avito_mod.settings, "avito_proxy_max_rotations", 1) - - scraper = AvitoScraper() - fw = MagicMock() - fw.status_code = 200 - fw.text = _FIREWALL_HTML - mock_session = AsyncMock() - mock_session.get = AsyncMock(return_value=fw) # всегда firewall - scraper._cffi = mock_session - - with patch.object(AvitoScraper, "_rotate_ip", AsyncMock(return_value=True)) as rot: - with pytest.raises(AvitoBlockedError, match="firewall"): - await scraper.fetch_around(56.84, 60.60, 1000, pages=1) - - assert rot.await_count == 1 # одна попытка ротации, потом сдаёмся - - -# ── #623 sweep-pipeline proxy (SERP+detail через mobile proxy) ─── - - -def test_avito_proxies_helper() -> None: - """Kit _avito_proxies(config): None без proxy, dict {http,https} при заданном URL. - - Регрессия: sweep создаёт shared-сессию, через неё идут и SERP, и detail — - без прокси detail летит с datacenter-IP → 429 → run banned (#623 followup). - """ - from types import SimpleNamespace - - from scraper_kit.orchestration.pipeline import _avito_proxies - - assert _avito_proxies(SimpleNamespace(scraper_proxy_url=None)) is None - - url = "http://u:p@mproxy.site:16928" - assert _avito_proxies(SimpleNamespace(scraper_proxy_url=url)) == {"http": url, "https": url} - - -# ── curl_cffi 429 short-retry (backconnect conn-limit transient) ────────────── - - -@pytest.mark.asyncio -async def test_fetch_around_429_short_retry_then_succeeds() -> None: - """429 дважды → 200: короткий retry снимает transient conn-limit БЕЗ ротации IP. - - backconnect-прокси (mproxy.site, лимит 5 соединений) отдаёт 429 при параллельной - пагинации — это transient, проходит на коротком retry. _rotate_ip НЕ должен дёргаться. - """ - from app.services.scrapers.avito import AvitoScraper - - scraper = AvitoScraper() - r429 = MagicMock() - r429.status_code = 429 - ok = MagicMock() - ok.status_code = 200 - ok.text = _CARD_HTML - mock_session = AsyncMock() - mock_session.get = AsyncMock(side_effect=[r429, r429, ok]) - scraper._cffi = mock_session - - with ( - patch("app.services.scrapers.avito.asyncio.sleep", AsyncMock()) as slp, - patch.object(AvitoScraper, "_rotate_ip", AsyncMock(return_value=True)) as rot, - patch.object(AvitoScraper, "sleep_between_requests", AsyncMock()), - ): - result = await scraper.fetch_around(56.84, 60.60, 1000, pages=1) - - assert len(result) == 1 - assert mock_session.get.await_count == 3 # 429, 429, 200 - rot.assert_not_awaited() # короткий retry, не ротация - # короткий backoff (_AVITO_429_BACKOFF_SEC ~1.5s), не 9с-settle ротации - assert slp.await_count >= 2 - for c in slp.await_args_list: - assert c.args[0] < 5.0 - - -@pytest.mark.asyncio -async def test_fetch_around_429_exhausted_no_rotate_url_raises() -> None: - """429 постоянно + НЕТ rotate-url → после исчерпания коротких retry: старый fallback.""" - from app.services.scrapers import avito as avito_mod - from app.services.scrapers.avito import AvitoScraper - - scraper = AvitoScraper() - r429 = MagicMock() - r429.status_code = 429 - mock_session = AsyncMock() - mock_session.get = AsyncMock(return_value=r429) - scraper._cffi = mock_session - - with ( - patch("app.services.scrapers.avito.asyncio.sleep", AsyncMock()) as slp, - patch.object(AvitoScraper, "_rotate_ip", AsyncMock(return_value=True)) as rot, - ): - with pytest.raises(AvitoRateLimitedError): - await scraper.fetch_around(56.84, 60.60, 1000, pages=1) - - rot.assert_not_awaited() # нет rotate-url → ротация не дёргается - # ровно _AVITO_429_MAX_RETRIES коротких retry, потом fallback - assert slp.await_count == avito_mod._AVITO_429_MAX_RETRIES - - -@pytest.mark.asyncio -async def test_fetch_around_429_exhausted_with_rotate_url_rotates( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """429 постоянно + ЕСТЬ rotate-url → короткие retry, затем ротация (старый путь).""" - from app.services.scrapers import avito as avito_mod - from app.services.scrapers.avito import AvitoScraper - - monkeypatch.setattr( - avito_mod.settings, "avito_proxy_rotate_url", "https://changeip.example/?proxy_key=x" - ) - monkeypatch.setattr(avito_mod.settings, "avito_proxy_max_rotations", 1) - - scraper = AvitoScraper() - r429 = MagicMock() - r429.status_code = 429 - ok = MagicMock() - ok.status_code = 200 - ok.text = _CARD_HTML - mock_session = AsyncMock() - # 429 на всех коротких retry (1 первичный + _AVITO_429_MAX_RETRIES), после - # исчерпания short-retry — ротация → 200. backconnect-reconnect здесь НЕ - # срабатывает (scraper_proxy_url не задан), поэтому сразу rotate-путь. - n_429 = 1 + avito_mod._AVITO_429_MAX_RETRIES - mock_session.get = AsyncMock(side_effect=[r429] * n_429 + [ok]) - scraper._cffi = mock_session - - with ( - patch("app.services.scrapers.avito.asyncio.sleep", AsyncMock()), - patch.object(AvitoScraper, "_rotate_ip", AsyncMock(return_value=True)) as rot, - patch.object(AvitoScraper, "sleep_between_requests", AsyncMock()), - ): - result = await scraper.fetch_around(56.84, 60.60, 1000, pages=1) - - rot.assert_awaited_once() # после исчерпания коротких retry — ротация - assert len(result) == 1 - - -@pytest.mark.asyncio -async def test_fetch_around_403_no_429_retry_path(monkeypatch: pytest.MonkeyPatch) -> None: - """403 (реальный IP-ban) НЕ проходит через 429-retry — поведение без изменений.""" - from app.services.scrapers import avito as avito_mod - from app.services.scrapers.avito import AvitoScraper - - monkeypatch.setattr(avito_mod.settings, "avito_proxy_rotate_url", None) - - scraper = AvitoScraper() - r403 = MagicMock() - r403.status_code = 403 - mock_session = AsyncMock() - mock_session.get = AsyncMock(return_value=r403) - scraper._cffi = mock_session - - with ( - patch("app.services.scrapers.avito.asyncio.sleep", AsyncMock()) as slp, - patch.object(AvitoScraper, "_rotate_ip", AsyncMock(return_value=True)), - ): - with pytest.raises(AvitoBlockedError): - await scraper.fetch_around(56.84, 60.60, 1000, pages=1) - - # 403 без rotate-url → немедленный raise, ни одного 429-backoff sleep - slp.assert_not_awaited() - assert mock_session.get.await_count == 1 diff --git a/tradein-mvp/backend/tests/scrapers/test_avito_detail_403_reconnect.py b/tradein-mvp/backend/tests/scrapers/test_avito_detail_403_reconnect.py deleted file mode 100644 index 3c469222..00000000 --- a/tradein-mvp/backend/tests/scrapers/test_avito_detail_403_reconnect.py +++ /dev/null @@ -1,191 +0,0 @@ -"""curl_cffi 403/firewall reconnect-retry на detail-странице под backconnect-прокси. - -fetch_detail ронял весь detail-backfill на первом 403/429 через AvitoBlockedError / -AvitoRateLimitedError (run 191 обогатил 48 → zombie). Зеркало SERP-фикса (#1765): под -backconnect rotating-прокси (mproxy.site) каждое НОВОЕ соединение даёт свежий exit-IP, -поэтому 403/firewall ретраим пересозданием эфемерной curl_cffi-сессии до -_AVITO_DETAIL_403_MAX_RETRIES раз. 429 — короткий retry той же сессии без reconnect. - -Ownership-правило: caller-provided cffi_session НИКОГДА не закрывается/не пересоздаётся — -retries идут через свою эфемерную retry_session (закрываем в finally). - -Включаем backconnect через scraper_proxy_url_env (scraper_proxy_url — read-only property, -его не monkeypatch'ишь). Зеркало fixture-стиля test_avito_403_backconnect_reconnect.py. -""" - -from unittest.mock import AsyncMock, MagicMock, patch - -import pytest - -from app.core.config import settings -from app.services.scrapers import avito_detail as detail_mod -from app.services.scrapers.avito_detail import fetch_detail -from app.services.scrapers.avito_exceptions import AvitoBlockedError - -_SENTINEL = object() -_FIREWALL_HTML = ( - "Доступ ограничен: проблема с IP" - '
...
' -) - - -def _resp(status_code: int, text: str = "") -> MagicMock: - r = MagicMock() - r.status_code = status_code - r.text = text - return r - - -def _enable_backconnect(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(settings, "scraper_proxy_url_env", "http://u:p@mproxy.site:14619") - - -def _disable_backconnect(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(settings, "scraper_proxy_url_env", None) - - -@pytest.fixture(autouse=True) -def _patch_common() -> object: - """asyncio.sleep → no-op; parse_detail_html → sentinel (нет нужды в реальном HTML).""" - with ( - patch("app.services.scrapers.avito_detail.asyncio.sleep", AsyncMock()), - patch( - "app.services.scrapers.avito_detail.parse_detail_html", return_value=_SENTINEL - ) as parse, - ): - yield parse - - -@pytest.mark.asyncio -async def test_403_403_200_caller_session_reconnect_succeeds( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """caller-provided session: 403 → 403 → 200 → reconnect-retry возвращает enrichment. - - _build_detail_session дёрнут для retries; caller-session НЕ закрыта. - """ - _enable_backconnect(monkeypatch) - - caller = AsyncMock() - caller.get = AsyncMock(return_value=_resp(403)) - - # Каждый retry → свежая эфемерная сессия. Первая отдаёт 403 (для 2-го reconnect), - # вторая — 200. - retry1 = AsyncMock() - retry1.get = AsyncMock(return_value=_resp(403)) - retry2 = AsyncMock() - retry2.get = AsyncMock(return_value=_resp(200, "")) - build = MagicMock(side_effect=[retry1, retry2]) - - with patch.object(detail_mod, "_build_detail_session", build): - result = await fetch_detail("/ekaterinburg/kvartiry/test-123", cffi_session=caller) - - assert result is _SENTINEL - assert build.call_count == 2 # 403, 403 → два reconnect перед 200 - caller.close.assert_not_called() # caller-provided session НЕ закрывается - assert caller.get.await_count == 1 # первый attempt - retry1.close.assert_awaited() # промежуточная retry-session закрыта - retry2.close.assert_awaited() # последняя retry-session закрыта в finally - - -@pytest.mark.asyncio -async def test_firewall_200_reconnect_succeeds(monkeypatch: pytest.MonkeyPatch) -> None: - """firewall(HTTP 200) → 200 → новый HTTP-200 firewall-detect триггерит reconnect.""" - _enable_backconnect(monkeypatch) - - caller = AsyncMock() - caller.get = AsyncMock(return_value=_resp(200, _FIREWALL_HTML)) - retry1 = AsyncMock() - retry1.get = AsyncMock(return_value=_resp(200, "")) - build = MagicMock(side_effect=[retry1]) - - with patch.object(detail_mod, "_build_detail_session", build): - result = await fetch_detail("/x", cffi_session=caller) - - assert result is _SENTINEL - assert build.call_count == 1 - caller.close.assert_not_called() - - -@pytest.mark.asyncio -async def test_403_exhausted_raises_blocked(monkeypatch: pytest.MonkeyPatch) -> None: - """403 _AVITO_DETAIL_403_MAX_RETRIES+1 раз → AvitoBlockedError.""" - _enable_backconnect(monkeypatch) - - caller = AsyncMock() - caller.get = AsyncMock(return_value=_resp(403)) - - def _make_retry() -> AsyncMock: - s = AsyncMock() - s.get = AsyncMock(return_value=_resp(403)) - return s - - build = MagicMock(side_effect=lambda: _make_retry()) - - with patch.object(detail_mod, "_build_detail_session", build): - with pytest.raises(AvitoBlockedError, match="HTTP 403"): - await fetch_detail("/x", cffi_session=caller) - - assert build.call_count == detail_mod._AVITO_DETAIL_403_MAX_RETRIES - caller.close.assert_not_called() - - -@pytest.mark.asyncio -async def test_429_short_retry_then_200_same_session( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """429 → 429 → 200: short-retry той же сессии, без reconnect.""" - _enable_backconnect(monkeypatch) - - caller = AsyncMock() - caller.get = AsyncMock(side_effect=[_resp(429), _resp(429), _resp(200, "")]) - build = MagicMock() - - with patch.object(detail_mod, "_build_detail_session", build): - result = await fetch_detail("/x", cffi_session=caller) - - assert result is _SENTINEL - build.assert_not_called() # 429 не дёргает reconnect - assert caller.get.await_count == 3 # та же сессия переиспользована - caller.close.assert_not_called() - - -@pytest.mark.asyncio -async def test_no_proxy_single_403_raises_immediately( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """scraper_proxy_url unset → нет reconnect-пути → первый 403 сразу raise.""" - _disable_backconnect(monkeypatch) - - caller = AsyncMock() - caller.get = AsyncMock(return_value=_resp(403)) - build = MagicMock() - - with patch.object(detail_mod, "_build_detail_session", build): - with pytest.raises(AvitoBlockedError, match="HTTP 403"): - await fetch_detail("/x", cffi_session=caller) - - build.assert_not_called() - assert caller.get.await_count == 1 - - -@pytest.mark.asyncio -async def test_own_session_403_200_closes_both(monkeypatch: pytest.MonkeyPatch) -> None: - """own-session path (cffi_session=None): 403 → 200; own session + retry session - обе закрыты в finally.""" - _enable_backconnect(monkeypatch) - - own = AsyncMock() - own.get = AsyncMock(return_value=_resp(403)) - retry1 = AsyncMock() - retry1.get = AsyncMock(return_value=_resp(200, "")) - # Первый _build_detail_session → own (own_session-ветка), второй → retry1 (reconnect). - build = MagicMock(side_effect=[own, retry1]) - - with patch.object(detail_mod, "_build_detail_session", build): - result = await fetch_detail("/x", cffi_session=None) - - assert result is _SENTINEL - own.close.assert_awaited() # own session закрыта (own_session=True) - retry1.close.assert_awaited() # retry session закрыта в finally - assert build.call_count == 2 diff --git a/tradein-mvp/backend/tests/scrapers/test_avito_detail_429_reconnect.py b/tradein-mvp/backend/tests/scrapers/test_avito_detail_429_reconnect.py deleted file mode 100644 index 3284c326..00000000 --- a/tradein-mvp/backend/tests/scrapers/test_avito_detail_429_reconnect.py +++ /dev/null @@ -1,153 +0,0 @@ -"""curl_cffi 429 reconnect-retry на detail-странице под backconnect-прокси. - -fetch_detail имел только short-retry той же сессии на 429 (_AVITO_DETAIL_429_MAX_RETRIES), -затем raise AvitoRateLimitedError. Но keep-alive detail-fetch переиспользует ОДИН -backconnect exit-IP на последовательные запросы → avito лочит этот IP, и short-retry той -же сессии не помогает (run 217: 5 подряд 429, abort на 19 enriched). Зеркало SERP-фикса -(#1769): при исчерпании short-retry под backconnect делаем ещё -_AVITO_DETAIL_429_RECONNECT_RETRIES попыток через эфемерную сессию (свежий exit-IP), -переиспользуя ту же retry_session-машинерию что и 403-путь. - -Ownership-правило: caller-provided cffi_session НИКОГДА не закрывается/не пересоздаётся — -reconnect идёт через свою эфемерную retry_session (закрываем в finally). - -Включаем backconnect через scraper_proxy_url_env (scraper_proxy_url — read-only property). -Зеркало fixture-стиля test_avito_detail_403_reconnect.py. -""" - -from unittest.mock import AsyncMock, MagicMock, patch - -import pytest - -from app.core.config import settings -from app.services.scrapers import avito_detail as detail_mod -from app.services.scrapers.avito_detail import fetch_detail -from app.services.scrapers.avito_exceptions import AvitoRateLimitedError - -_SENTINEL = object() - - -def _resp(status_code: int, text: str = "") -> MagicMock: - r = MagicMock() - r.status_code = status_code - r.text = text - return r - - -def _enable_backconnect(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(settings, "scraper_proxy_url_env", "http://u:p@mproxy.site:14619") - - -def _disable_backconnect(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(settings, "scraper_proxy_url_env", None) - - -@pytest.fixture(autouse=True) -def _patch_common() -> object: - """asyncio.sleep → no-op; parse_detail_html → sentinel (нет нужды в реальном HTML).""" - with ( - patch("app.services.scrapers.avito_detail.asyncio.sleep", AsyncMock()), - patch( - "app.services.scrapers.avito_detail.parse_detail_html", return_value=_SENTINEL - ) as parse, - ): - yield parse - - -@pytest.mark.asyncio -async def test_429_short_retry_exhausted_then_reconnect_200( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """429×(short-retry budget) → 429 → reconnect (свежая сессия) → 200. - - После _AVITO_DETAIL_429_MAX_RETRIES short-retry той же сессии ещё один 429 исчерпывает - short-budget → reconnect через _build_detail_session → 200. Caller-session НЕ закрыта. - """ - _enable_backconnect(monkeypatch) - - # short-budget short-retries (каждый получает 429 и continue) + ещё один 429, который - # исчерпывает budget и триггерит reconnect. Итого budget+1 ответов 429 на caller-сессии. - n429 = detail_mod._AVITO_DETAIL_429_MAX_RETRIES + 1 - caller = AsyncMock() - caller.get = AsyncMock(side_effect=[_resp(429)] * n429) - - retry1 = AsyncMock() - retry1.get = AsyncMock(return_value=_resp(200, "")) - build = MagicMock(side_effect=[retry1]) - - with patch.object(detail_mod, "_build_detail_session", build): - result = await fetch_detail("/ekaterinburg/kvartiry/test-123", cffi_session=caller) - - assert result is _SENTINEL - assert build.call_count == 1 # один reconnect перед 200 - assert caller.get.await_count == n429 # short-budget + исчерпавший 429 - caller.close.assert_not_called() # caller-provided session НЕ закрывается - retry1.close.assert_awaited() # эфемерная retry-session закрыта в finally - - -@pytest.mark.asyncio -async def test_429_fully_exhausted_raises_rate_limited( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """429 на short-retry budget + всех reconnect-попытках → AvitoRateLimitedError.""" - _enable_backconnect(monkeypatch) - - # short-budget+1 (исчерпание short) на caller; затем каждая reconnect-сессия тоже 429. - n429 = detail_mod._AVITO_DETAIL_429_MAX_RETRIES + 1 - caller = AsyncMock() - caller.get = AsyncMock(side_effect=[_resp(429)] * n429) - - def _make_retry() -> AsyncMock: - s = AsyncMock() - s.get = AsyncMock(return_value=_resp(429)) - return s - - build = MagicMock(side_effect=lambda: _make_retry()) - - with patch.object(detail_mod, "_build_detail_session", build): - with pytest.raises(AvitoRateLimitedError, match="HTTP 429"): - await fetch_detail("/x", cffi_session=caller) - - # Все reconnect-попытки исчерпаны. - assert build.call_count == detail_mod._AVITO_DETAIL_429_RECONNECT_RETRIES - caller.close.assert_not_called() - - -@pytest.mark.asyncio -async def test_429_short_retry_then_200_no_reconnect( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """429 → 200 в рамках short-retry budget: та же сессия, БЕЗ reconnect (regression guard).""" - _enable_backconnect(monkeypatch) - - caller = AsyncMock() - caller.get = AsyncMock(side_effect=[_resp(429), _resp(200, "")]) - build = MagicMock() - - with patch.object(detail_mod, "_build_detail_session", build): - result = await fetch_detail("/x", cffi_session=caller) - - assert result is _SENTINEL - build.assert_not_called() # short-retry успешен → reconnect не дёрнут - assert caller.get.await_count == 2 # та же сессия переиспользована - caller.close.assert_not_called() - - -@pytest.mark.asyncio -async def test_no_proxy_429_short_exhausted_raises_immediately( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """scraper_proxy_url unset → нет reconnect-пути → исчерпание short-retry сразу raise.""" - _disable_backconnect(monkeypatch) - - n429 = detail_mod._AVITO_DETAIL_429_MAX_RETRIES + 1 - caller = AsyncMock() - caller.get = AsyncMock(side_effect=[_resp(429)] * n429) - build = MagicMock() - - with patch.object(detail_mod, "_build_detail_session", build): - with pytest.raises(AvitoRateLimitedError, match="HTTP 429"): - await fetch_detail("/x", cffi_session=caller) - - build.assert_not_called() # no proxy → нет reconnect - assert caller.get.await_count == n429 diff --git a/tradein-mvp/backend/tests/scrapers/test_avito_detail_browser_only.py b/tradein-mvp/backend/tests/scrapers/test_avito_detail_browser_only.py deleted file mode 100644 index 6d78306b..00000000 --- a/tradein-mvp/backend/tests/scrapers/test_avito_detail_browser_only.py +++ /dev/null @@ -1,109 +0,0 @@ -"""Browser-only режим fetch_detail (#1814). - -Когда передан browser_fetcher, detail-fetch идёт СТРОГО через camoufox-сервис. -Никакой curl_cffi-сессии (AsyncSession) и никакого curl-fallback — avito-антибот -банит curl-фингерпринт под нагрузкой (429 CONNECT tunnel failed). Любой -firewall/ошибка браузера → AvitoBlockedError; caller (backfill) метит failed и идёт -дальше, НЕ создавая curl-сессию. - -Тесты проверяют именно отсутствие инстанцирования AsyncSession и поднятие -AvitoBlockedError на browser-firewall/ошибке. -""" - -from unittest.mock import AsyncMock, MagicMock, patch - -import httpx -import pytest - -from app.services.scrapers.avito_detail import fetch_detail -from app.services.scrapers.avito_exceptions import AvitoBlockedError - -_SENTINEL = object() -_FIREWALL_HTML = ( - "Доступ ограничен: проблема с IP" - '
...
' -) -_OK_HTML = "ok" - -# AsyncSession (curl_cffi) в модуле avito_detail — НЕ должна инстанцироваться в browser-mode. -_ASYNC_SESSION = "app.services.scrapers.avito_detail.AsyncSession" -_BUILD_SESSION = "app.services.scrapers.avito_detail._build_detail_session" -_PARSE = "app.services.scrapers.avito_detail.parse_detail_html" - - -def _browser(*, fetch_side: object) -> MagicMock: - bf = MagicMock() - if isinstance(fetch_side, BaseException): - bf.fetch = AsyncMock(side_effect=fetch_side) - else: - bf.fetch = AsyncMock(return_value=fetch_side) - return bf - - -@pytest.mark.asyncio -async def test_browser_mode_success_no_curl_session() -> None: - """browser_fetcher OK → parse, AsyncSession НЕ создаётся.""" - bf = _browser(fetch_side=_OK_HTML) - with ( - patch(_ASYNC_SESSION) as async_session, - patch(_BUILD_SESSION) as build_session, - patch(_PARSE, return_value=_SENTINEL) as parse, - ): - result = await fetch_detail("/items/123", browser_fetcher=bf) - - assert result is _SENTINEL - bf.fetch.assert_awaited_once() - parse.assert_called_once() - async_session.assert_not_called() - build_session.assert_not_called() - - -@pytest.mark.asyncio -async def test_browser_firewall_raises_blocked_no_curl_session() -> None: - """browser вернул firewall-HTML (status 200) → AvitoBlockedError, без curl.""" - bf = _browser(fetch_side=_FIREWALL_HTML) - with ( - patch(_ASYNC_SESSION) as async_session, - patch(_BUILD_SESSION) as build_session, - patch(_PARSE, return_value=_SENTINEL), - ): - with pytest.raises(AvitoBlockedError): - await fetch_detail("/items/123", browser_fetcher=bf) - - async_session.assert_not_called() - build_session.assert_not_called() - - -@pytest.mark.asyncio -async def test_browser_http_error_raises_blocked_no_curl_session() -> None: - """browser_fetcher.fetch кидает HTTPError (502/firewall сервиса) → AvitoBlockedError, - AsyncSession НЕ инстанцируется (НЕ откатываемся в curl-фингерпринт → 429-бан).""" - bf = _browser(fetch_side=httpx.HTTPError("browser service 502")) - with ( - patch(_ASYNC_SESSION) as async_session, - patch(_BUILD_SESSION) as build_session, - patch(_PARSE, return_value=_SENTINEL), - ): - with pytest.raises(AvitoBlockedError): - await fetch_detail("/items/123", browser_fetcher=bf) - - async_session.assert_not_called() - build_session.assert_not_called() - - -@pytest.mark.asyncio -async def test_browser_blocked_error_propagates_unchanged() -> None: - """Если browser_fetcher сам кинул AvitoBlockedError — пробрасываем как есть, без curl.""" - original = AvitoBlockedError("already blocked") - bf = _browser(fetch_side=original) - with ( - patch(_ASYNC_SESSION) as async_session, - patch(_BUILD_SESSION) as build_session, - patch(_PARSE, return_value=_SENTINEL), - ): - with pytest.raises(AvitoBlockedError) as exc_info: - await fetch_detail("/items/123", browser_fetcher=bf) - - assert exc_info.value is original - async_session.assert_not_called() - build_session.assert_not_called() diff --git a/tradein-mvp/backend/tests/scrapers/test_avito_detail_kit_parity.py b/tradein-mvp/backend/tests/scrapers/test_avito_detail_kit_parity.py deleted file mode 100644 index 7c0f2e3b..00000000 --- a/tradein-mvp/backend/tests/scrapers/test_avito_detail_kit_parity.py +++ /dev/null @@ -1,129 +0,0 @@ -"""Smoke-proof для parity-harness'а (tests/support/parity.py) — issue #2304. - -Доказывает, что harness реально ловит эквивалентность legacy ↔ scraper_kit на -РЕАЛЬНОЙ паре функций, а не на синтетике: `parse_detail_html` — parsing-ядро, -которое `app/api/v1/admin.py::scrape_avito_detail` (debug-роут ручного запуска -парсера) вызывает косвенно через `fetch_detail()` (fetch_detail = HTTP GET + -`parse_detail_html(html, url)`; сам parse — чистая функция, без сети/БД, что и -позволяет сравнить её offline на фиксированной HTML-фикстуре). - -`app/services/scrapers/avito_detail.py::parse_detail_html` (legacy) и -`scraper_kit/providers/avito/detail.py::parse_detail_html` (kit) на момент -написания теста ИДЕНТИЧНЫ по телу функции (см. audit -`Scraper_Kit_Legacy_Dependency_Audit_0703` в vault) — различаются только -import-пути модулей, из которых берутся зависимости (`app.services.scrapers.*` -vs `scraper_kit.*`) и Strangler-инжекция `ScraperConfig` в соседних функциях -(`fetch_detail`, `_build_detail_session`), которую сам `parse_detail_html` не -использует. Это НЕ миграция admin.py (issue #2305+) — только доказательство -инструмента. - -HTML-фикстура — тот же minimal-marker набор, что и в -`tests/test_avito_detail_parse.py::test_minimal_parse` (независимая копия -литерала, чтобы этот тест не зависел от другого test-модуля). -""" - -# ruff: noqa: E501 # HTML-фикстура содержит длинные строки разметки - -from __future__ import annotations - -import dataclasses -import os - -import pytest - -# app.services.scrapers.avito_detail импортирует app.core.config.settings=Settings(), -# которому нужен DATABASE_URL. Офлайн-парсинг БД не трогает — фиктивный DSN достаточен -# (mirror tests/test_scraper_kit_avito_golden_parity.py — тест должен жить и без -# CI-уровневого DATABASE_URL из ci-tradein.yml/deploy-tradein.yml). -os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db") - -from scraper_kit.providers.avito.detail import ( - parse_detail_html as kit_parse_detail_html, -) - -from app.services.scrapers.avito_detail import ( - parse_detail_html as legacy_parse_detail_html, -) -from tests.support.parity import ParityMismatchError, assert_parity - -_DETAIL_HTML = """ - -
№ 7986882804 · 18 мая в 13:40 · 1899 просмотров (+ 12 сегодня)
-11 990 000 ₽ -
-
-
    -
  • Количество комнат: 3
    • -
  • Общая площадь: 75 м²
    • -
  • Площадь кухни: 15 м²
    • -
  • Этаж: 20 из 26
    • -
  • Балкон или лоджия: лоджия
    • -
  • Ремонт: евро
    • -
  • Способ продажи: свободная
    • -
  • Условия продажи: возможна ипотека
    • -
-
    -
  • Тип дома: монолитный
    • -
  • Этажей в доме: 26
    • -
-
-Дом -
-

1 собственник

-

Последняя смена собственника 5 мая 2020

-

Не найдены ограничения и обременения

-

Совпадают площадь, адрес и этаж

-
-
Отличная квартира в новом ЖК. Чкаловская 11-15 мин пешком.
- -""" -_SOURCE_URL = "https://www.avito.ru/ekaterinburg/kvartiry/test_7986882804" - - -def test_legacy_and_kit_parse_detail_html_are_class_distinct() -> None: - """Sanity-check ДО harness'а: подтверждает, зачем harness вообще нужен. - - legacy DetailEnrichment и kit DetailEnrichment — РАЗНЫЕ классы (разные - модули) → обычный `==` даёт False, хотя данные идентичны. Именно это - harness должен обходить через структурное сравнение (dataclasses.fields). - """ - legacy_result = legacy_parse_detail_html(_DETAIL_HTML, _SOURCE_URL) - kit_result = kit_parse_detail_html(_DETAIL_HTML, _SOURCE_URL) - - assert type(legacy_result) is not type(kit_result) - assert legacy_result != kit_result # dataclass __eq__ проверяет class identity первым - assert dataclasses.asdict(legacy_result) == dataclasses.asdict(kit_result) - - -def test_parity_harness_confirms_avito_detail_parse_equivalence() -> None: - """Реальное proof-of-concept использование harness'а (issues #2305-#2310 template).""" - assert_parity( - legacy_fn=legacy_parse_detail_html, - kit_fn=kit_parse_detail_html, - fixtures=[(_DETAIL_HTML, _SOURCE_URL)], - ) - - -def test_parity_harness_catches_real_field_divergence() -> None: - """Доказывает, что harness реально ЛОВИТ расхождение на этой же реальной - 30+-полевой dataclass-форме — а не только сообщает "нет разницы" на двух - genuinely-identical инстансах, как предыдущий тест. - - Мутируем один field (``price_rub``) в kit-результате через - ``dataclasses.replace`` и убеждаемся, что harness падает с - ``ParityMismatchError``, называющим именно этот field. - """ - legacy_result = legacy_parse_detail_html(_DETAIL_HTML, _SOURCE_URL) - kit_result = kit_parse_detail_html(_DETAIL_HTML, _SOURCE_URL) - assert kit_result.price_rub is not None # sanity: поле реально распарсилось из фикстуры - - mutated_kit_result = dataclasses.replace(kit_result, price_rub=kit_result.price_rub + 1) - - with pytest.raises(ParityMismatchError) as exc_info: - assert_parity( - legacy_fn=lambda: legacy_result, - kit_fn=lambda: mutated_kit_result, - fixtures=[()], - ) - - assert "price_rub" in str(exc_info.value) diff --git a/tradein-mvp/backend/tests/scrapers/test_avito_detail_not_found.py b/tradein-mvp/backend/tests/scrapers/test_avito_detail_not_found.py deleted file mode 100644 index 912b7885..00000000 --- a/tradein-mvp/backend/tests/scrapers/test_avito_detail_not_found.py +++ /dev/null @@ -1,82 +0,0 @@ -"""404 / removed-listing detection на detail-странице (#2034). - -Lat-null pending-очередь backfill в основном содержит МЁРТВЫЕ листинги (404 — потому -они и стали координатными дырами). В browser-mode 404-страница Avito («Ошибка 404. -Страница не найдена», тело тоже содержит generic «Объявления на сайте Авито») не имеет -item-view → раньше ловилась как soft-block → AvitoBlockedError → backfill крутил -consecutive-block breaker и абортил run ДО live-листингов (run 458: attempted=5 -enriched=0 blocked=5 → abort). - -_is_detail_not_found детектит её ОТДЕЛЬНО → fetch_detail кидает AvitoListingGoneError -(НЕ AvitoBlockedError), backfill метит листинг is_active=FALSE и продолжает без -ротации/ретрая. - -Зеркало fixture-стиля test_avito_detail_soft_block.py / test_avito_detail_browser_only.py. -""" - -from unittest.mock import AsyncMock, MagicMock, patch - -import pytest - -from app.services.scrapers import avito_detail as detail_mod -from app.services.scrapers.avito_detail import fetch_detail -from app.services.scrapers.avito_exceptions import AvitoBlockedError, AvitoListingGoneError - -_SENTINEL = object() - -# 404-страница Avito: специфичный ; тело тоже содержит generic-витринный текст, -# который сам по себе тригерит soft-block — поэтому not-found проверяется ПЕРВЫМ. -_NOT_FOUND_HTML = ( - '<!DOCTYPE html><html lang="ru"><head>' - "<title>Ошибка 404. Страница не найдена" - "
Объявления на сайте Авито
" -) -# Soft-block deflect: generic-витрина БЕЗ 404-текста — это блок, НЕ gone. -_SOFT_BLOCK_HTML = ( - '' - "Авито — Объявления на сайте Авито" - "
...
" -) -_ITEM_VIEW_HTML = "№ 123" - -_ASYNC_SESSION = "app.services.scrapers.avito_detail.AsyncSession" -_BUILD_SESSION = "app.services.scrapers.avito_detail._build_detail_session" -_PARSE = "app.services.scrapers.avito_detail.parse_detail_html" - - -def test_is_detail_not_found_unit() -> None: - """Юнит _is_detail_not_found: 404 → True; soft-block deflect / item-view / пусто → False.""" - assert detail_mod._is_detail_not_found(_NOT_FOUND_HTML) is True - # soft-block deflect (generic-витрина без 404-текста) НЕ должен ловиться как gone - assert detail_mod._is_detail_not_found(_SOFT_BLOCK_HTML) is False - # реальное объявление с item-view → False - assert detail_mod._is_detail_not_found(_ITEM_VIEW_HTML) is False - assert detail_mod._is_detail_not_found("") is False - - -def _browser_returning(html: str) -> MagicMock: - bf = MagicMock() - bf.fetch = AsyncMock(return_value=html) - return bf - - -@pytest.mark.asyncio -async def test_browser_mode_404_raises_listing_gone_not_blocked() -> None: - """browser_fetcher вернул 404-HTML → AvitoListingGoneError (НЕ AvitoBlockedError), - без curl-сессии.""" - bf = _browser_returning(_NOT_FOUND_HTML) - with ( - patch(_ASYNC_SESSION) as async_session, - patch(_BUILD_SESSION) as build_session, - patch(_PARSE, return_value=_SENTINEL) as parse, - ): - with pytest.raises(AvitoListingGoneError) as exc_info: - await fetch_detail("/items/123", browser_fetcher=bf) - - # Тип строго AvitoListingGoneError и НЕ подкласс AvitoBlockedError — - # иначе backfill-ловушка `except (AvitoBlockedError, ...)` перехватила бы его. - assert not isinstance(exc_info.value, AvitoBlockedError) - bf.fetch.assert_awaited_once() - parse.assert_not_called() - async_session.assert_not_called() - build_session.assert_not_called() diff --git a/tradein-mvp/backend/tests/scrapers/test_avito_detail_soft_block.py b/tradein-mvp/backend/tests/scrapers/test_avito_detail_soft_block.py deleted file mode 100644 index 1d974a46..00000000 --- a/tradein-mvp/backend/tests/scrapers/test_avito_detail_soft_block.py +++ /dev/null @@ -1,101 +0,0 @@ -"""Soft-block (HTTP 200 generic-витрина) detection на detail-странице (#1967 / #1950). - -Под нагрузкой Avito иногда отдаёт HTTP 200 с ОБЩЕЙ витриной -(Авито — Объявления на сайте Авито) вместо запрошенного объявления — -мягкий анти-бот deflect. У такой страницы нет ни firewall-маркеров, ни item-view → -parse_detail_html ронял ValueError → generic failed, IP НЕ ротировался, enriched -каскадно падал к 0. _is_detail_soft_block детектит её как блок → существующая -403/firewall reconnect-машинерия пересоздаёт эфемерную сессию (свежий exit-IP). - -Зеркало fixture-стиля test_avito_detail_403_reconnect.py. -""" - -from unittest.mock import AsyncMock, MagicMock, patch - -import pytest - -from app.core.config import settings -from app.services.scrapers import avito_detail as detail_mod -from app.services.scrapers.avito_detail import fetch_detail -from app.services.scrapers.avito_exceptions import AvitoBlockedError - -_SENTINEL = object() -_SOFT_BLOCK_HTML = ( - '' - "Авито — Объявления на сайте Авито" - "
...
" -) - - -def _resp(status_code: int, text: str = "") -> MagicMock: - r = MagicMock() - r.status_code = status_code - r.text = text - return r - - -def _enable_backconnect(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(settings, "scraper_proxy_url_env", "http://u:p@mproxy.site:14619") - - -@pytest.fixture(autouse=True) -def _patch_common() -> object: - """asyncio.sleep → no-op; parse_detail_html → sentinel (нет нужды в реальном HTML).""" - with ( - patch("app.services.scrapers.avito_detail.asyncio.sleep", AsyncMock()), - patch( - "app.services.scrapers.avito_detail.parse_detail_html", return_value=_SENTINEL - ) as parse, - ): - yield parse - - -@pytest.mark.asyncio -async def test_soft_block_200_reconnect_succeeds(monkeypatch: pytest.MonkeyPatch) -> None: - """soft-block(HTTP 200) → 200: generic-витрина детектится как блок → reconnect.""" - _enable_backconnect(monkeypatch) - - caller = AsyncMock() - caller.get = AsyncMock(return_value=_resp(200, _SOFT_BLOCK_HTML)) - retry1 = AsyncMock() - retry1.get = AsyncMock(return_value=_resp(200, "")) - build = MagicMock(side_effect=[retry1]) - - with patch.object(detail_mod, "_build_detail_session", build): - result = await fetch_detail("/x", cffi_session=caller) - - assert result is _SENTINEL - assert build.call_count == 1 - caller.close.assert_not_called() - - -@pytest.mark.asyncio -async def test_soft_block_exhausted_raises_blocked(monkeypatch: pytest.MonkeyPatch) -> None: - """soft-block на каждом attempt _AVITO_DETAIL_403_MAX_RETRIES+1 раз → AvitoBlockedError.""" - _enable_backconnect(monkeypatch) - - caller = AsyncMock() - caller.get = AsyncMock(return_value=_resp(200, _SOFT_BLOCK_HTML)) - - def _make_retry() -> AsyncMock: - s = AsyncMock() - s.get = AsyncMock(return_value=_resp(200, _SOFT_BLOCK_HTML)) - return s - - build = MagicMock(side_effect=lambda: _make_retry()) - - with patch.object(detail_mod, "_build_detail_session", build): - with pytest.raises(AvitoBlockedError): - await fetch_detail("/x", cffi_session=caller) - - assert build.call_count == detail_mod._AVITO_DETAIL_403_MAX_RETRIES - caller.close.assert_not_called() - - -def test_is_detail_soft_block_unit() -> None: - """Юнит _is_detail_soft_block: generic-витрина → True; item-view / пусто → False.""" - assert detail_mod._is_detail_soft_block(_SOFT_BLOCK_HTML) is True - item_view = "№ 123" - assert detail_mod._is_detail_soft_block(item_view) is False - assert detail_mod._is_detail_soft_block("") is False - assert detail_mod._is_detail_soft_block("") is False diff --git a/tradein-mvp/backend/tests/scrapers/test_avito_detail_warmup.py b/tradein-mvp/backend/tests/scrapers/test_avito_detail_warmup.py deleted file mode 100644 index b741e0f4..00000000 --- a/tradein-mvp/backend/tests/scrapers/test_avito_detail_warmup.py +++ /dev/null @@ -1,140 +0,0 @@ -"""Живой прогрев сессии + батч-на-сессию (#1551). - -warm_up_session: GET Yandex SERP (best-effort) -> GET Avito EKB search-страница с -Referer=yandex сеет антибот-куки. Прогретая сессия потом держит батч detail-GET'ов -без 403 (доказано пробами на проде: 22/22 карточек подряд, 0 блоков). - -fetch_detail(referer=..., reconnect_on_block=False): шлёт Referer detail-GET'а и при -403/firewall поднимает AvitoBlockedError СРАЗУ — без холодных per-card reconnect'ов -(их делает outer-loop через rebuild+rewarm прогретой сессии в avito_detail_backfill). - -Стиль фикстур/моков зеркалит test_avito_detail_403_reconnect.py. -""" - -from unittest.mock import AsyncMock, MagicMock, patch - -import pytest - -from app.core.config import settings -from app.services.scrapers import avito_detail as detail_mod -from app.services.scrapers.avito_detail import ( - fetch_detail, - research_in_session, - warm_up_session, -) -from app.services.scrapers.avito_exceptions import AvitoBlockedError - - -def _resp(status_code: int, text: str = "") -> MagicMock: - r = MagicMock() - r.status_code = status_code - r.text = text - return r - - -def _enable_backconnect(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(settings, "scraper_proxy_url_env", "http://u:p@mproxy.site:14619") - - -# ── warm_up_session ─────────────────────────────────────────────────────────── -@pytest.mark.asyncio -async def test_warm_up_session_search_with_yandex_referer_true_on_200() -> None: - """warm_up_session шлёт GET на _AVITO_WARM_SEARCH_URL с Referer=yandex и - возвращает True на 200/non-firewall.""" - session = AsyncMock() - session.get = AsyncMock(return_value=_resp(200, "ok")) - - with ( - patch("app.services.scrapers.avito_detail.asyncio.sleep", AsyncMock()), - patch("app.services.scrapers.avito_detail._is_firewall_page", return_value=False), - ): - ok = await warm_up_session(session) - - assert ok is True - # search-GET (на avito-search URL) ушёл ровно один раз и с Referer=yandex. - search_calls = [ - c - for c in session.get.await_args_list - if c.args and c.args[0] == detail_mod._AVITO_WARM_SEARCH_URL - ] - assert len(search_calls) == 1 - assert search_calls[0].kwargs["headers"]["Referer"] == detail_mod._AVITO_WARM_YANDEX_REFERER - # Yandex SERP — органический referer-источник — тоже фетчился (best-effort, до search). - assert any( - c.args and c.args[0] == detail_mod._AVITO_WARM_YANDEX_URL - for c in session.get.await_args_list - ) - - -@pytest.mark.asyncio -async def test_warm_up_session_false_on_403() -> None: - """search-страница вернула 403 → warm_up_session возвращает False.""" - session = AsyncMock() - session.get = AsyncMock(return_value=_resp(403)) - - with ( - patch("app.services.scrapers.avito_detail.asyncio.sleep", AsyncMock()), - patch("app.services.scrapers.avito_detail._is_firewall_page", return_value=False), - ): - ok = await warm_up_session(session) - - assert ok is False - - -# ── fetch_detail referer + reconnect_on_block=False ─────────────────────────── -@pytest.mark.asyncio -async def test_fetch_detail_referer_forwarded_and_block_raises_immediately( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """fetch_detail(referer="X", reconnect_on_block=False): Referer="X" уходит в - detail-GET, а 403 поднимает AvitoBlockedError СРАЗУ — без холодного reconnect - (даже когда backconnect-proxy включён → гейт = reconnect_on_block, не наличие proxy). - """ - _enable_backconnect(monkeypatch) - - caller = AsyncMock() - caller.get = AsyncMock(return_value=_resp(403)) - build = MagicMock() # _build_detail_session НЕ должна дёргаться (нет reconnect'а) - - with patch.object(detail_mod, "_build_detail_session", build): - with pytest.raises(AvitoBlockedError, match="HTTP 403"): - await fetch_detail( - "/ekaterinburg/kvartiry/test-123", - cffi_session=caller, - referer="X", - reconnect_on_block=False, - ) - - build.assert_not_called() # никаких холодных reconnect'ов - assert caller.get.await_count == 1 # один detail-GET → сразу raise - assert caller.get.await_args.kwargs["headers"] == {"Referer": "X"} - - -# ── research_in_session (in-session перепоиск, #1551 follow-up) ──────────────── -@pytest.mark.asyncio -async def test_research_in_session_re_get_search_with_yandex_referer_true_on_200() -> None: - """research_in_session re-GET'ит avito-search на ТОЙ ЖЕ сессии с Referer=yandex - и возвращает True на 200/non-firewall (свежесть куки без смены exit-IP).""" - session = AsyncMock() - session.get = AsyncMock(return_value=_resp(200, "ok")) - - with patch("app.services.scrapers.avito_detail._is_firewall_page", return_value=False): - ok = await research_in_session(session) - - assert ok is True - assert session.get.await_count == 1 # ровно один re-GET, без yandex-фазы (in-session) - assert session.get.await_args.args[0] == detail_mod._AVITO_WARM_SEARCH_URL - sent_referer = session.get.await_args.kwargs["headers"]["Referer"] - assert sent_referer == detail_mod._AVITO_WARM_YANDEX_REFERER - - -@pytest.mark.asyncio -async def test_research_in_session_false_on_403() -> None: - """search вернул 403 → research_in_session возвращает False (best-effort, не fatal).""" - session = AsyncMock() - session.get = AsyncMock(return_value=_resp(403)) - - with patch("app.services.scrapers.avito_detail._is_firewall_page", return_value=False): - ok = await research_in_session(session) - - assert ok is False diff --git a/tradein-mvp/backend/tests/scrapers/test_avito_exhaustive.py b/tradein-mvp/backend/tests/scrapers/test_avito_exhaustive.py deleted file mode 100644 index f86ffa33..00000000 --- a/tradein-mvp/backend/tests/scrapers/test_avito_exhaustive.py +++ /dev/null @@ -1,921 +0,0 @@ -"""Тесты exhaustive-загрузки Avito: fetch_all_secondary + _walk_price_range. - -Зеркало tests/scrapers/test_cian_exhaustive.py. Мокаем сетевые вызовы -(_fetch_rooms_page_html, _extract_total_count, _parse_html) для детерминированного -контроля бисекции без реального HTTP. - -Дополнительно: _build_rooms_url с pmin/pmax (backward-compat) и _extract_total_count -на реальном span-фрагменте `data-marker="page-title/count"`. -""" - -from __future__ import annotations - -import os -from unittest.mock import MagicMock, patch - -# Settings requires DATABASE_URL at import time — set dummy DSN before any app import. -os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test") - -import httpx -import pytest - -from app.services.scrapers.avito import ( - _AVITO_MAX_PRICE, - _AVITO_MIN_BRACKET, - _AVITO_OFFERS_PER_PAGE, - _AVITO_PRICE_SEED_BRACKETS, - AvitoScraper, -) -from app.services.scrapers.avito_exceptions import AvitoBlockedError -from app.services.scrapers.base import ScrapedLot - - -def _make_lot(source_id: str, price: int = 3_000_000, segment: str = "vtorichka") -> ScrapedLot: - """Минимальный ScrapedLot для теста.""" - return ScrapedLot( - source="avito", - source_url=f"https://www.avito.ru/ekaterinburg/kvartiry/{source_id}", - source_id=source_id, - address="Екатеринбург", - price_rub=price, - listing_segment=segment, - ) - - -@pytest.fixture -def scraper() -> AvitoScraper: - """AvitoScraper без реального __aenter__ (без HTTP/IP-rotation).""" - s = AvitoScraper.__new__(AvitoScraper) - s.name = "avito" - s.base_url = "https://www.avito.ru" - s.request_delay_sec = 0.0 - s._cffi = MagicMock() # не None - s._browser = None - s.parse_failures = 0 - return s - - -# ── _build_rooms_url с pmin/pmax ────────────────────────────────────────────── - - -def test_build_rooms_url_both_prices(scraper: AvitoScraper) -> None: - """min_price + max_price → pmin/pmax оба в query.""" - url = scraper._build_rooms_url( - "1-komnatnye-X", page=2, min_price=3_000_000, max_price=4_000_000 - ) - assert "pmin=3000000" in url - assert "pmax=4000000" in url - assert "p=2" in url - assert "s=104" in url - assert "/kvartiry/prodam/1-komnatnye-X?" in url - - -def test_build_rooms_url_min_only(scraper: AvitoScraper) -> None: - """Только min_price → pmin в query, pmax отсутствует.""" - url = scraper._build_rooms_url("1-komnatnye-X", min_price=3_000_000) - assert "pmin=3000000" in url - assert "pmax" not in url - - -def test_build_rooms_url_no_prices_backward_compat(scraper: AvitoScraper) -> None: - """Без цен URL не меняется (backward-compat для fetch_by_rooms).""" - url = scraper._build_rooms_url("1-komnatnye-X", page=1) - assert "pmin" not in url - assert "pmax" not in url - assert url.endswith("/kvartiry/prodam/1-komnatnye-X?s=104&p=1") - - -# ── _extract_total_count ────────────────────────────────────────────────────── - -# Реальный span-фрагмент из живого Avito SERP (NBSP-разделитель разрядов). -_REAL_COUNT_SPAN = '1\xa0178' - - -def test_extract_total_count_real_span(scraper: AvitoScraper) -> None: - """Реальный span с NBSP `1\\xa0178` → 1178.""" - html = f"

Квартиры

{_REAL_COUNT_SPAN}
" - assert scraper._extract_total_count(html) == 1178 - - -def test_extract_total_count_nbsp_entity(scraper: AvitoScraper) -> None: - """Вариант с ` `-entity → корректно очищается.""" - html = '12 345' - assert scraper._extract_total_count(html) == 12345 - - -def test_extract_total_count_zero(scraper: AvitoScraper) -> None: - """0 результатов → 0 (не None).""" - html = '0' - assert scraper._extract_total_count(html) == 0 - - -def test_extract_total_count_missing_returns_none(scraper: AvitoScraper) -> None: - """Captcha/firewall (нет span, нет no-results-маркера) → None.""" - html = "Доступ ограничен" - assert scraper._extract_total_count(html) is None - - -def test_extract_total_count_no_results_marker_returns_zero(scraper: AvitoScraper) -> None: - """Валидная пустая выборка: нет `page-title/count`, но есть «ничего не найдено» → 0. - - Даже если на странице есть fallback-«похожие» объявления (item-id'ы) — это - не подходящие под фильтр листинги, total всё равно 0. Это предотвращает - дробление пустого брекета + ротацию IP (прод-инцидент run 209). - """ - html = ( - '
' - '
' - "

По вашему запросу ничего не найдено

" - ) - assert scraper._extract_total_count(html) == 0 - - -def test_extract_total_count_no_results_marker_case_insensitive(scraper: AvitoScraper) -> None: - """Маркер регистронезависим: «Ничего не найдено» с заглавной → тоже 0.""" - html = "

Ничего не найдено

Попробуйте изменить фильтры

" - assert scraper._extract_total_count(html) == 0 - - -def test_extract_total_count_count_wins_over_no_results(scraper: AvitoScraper) -> None: - """Непустая выборка с count имеет приоритет, даже если где-то ниже есть маркер.""" - html = ( - f"

Квартиры

{_REAL_COUNT_SPAN}
" - "
Если ничего не найдено, измените запрос
" - ) - assert scraper._extract_total_count(html) == 1178 - - -# ── Бисекция: split / paginate / dedup ──────────────────────────────────────── - - -@pytest.mark.asyncio -async def test_fetch_all_secondary_splits_on_cap(scraper: AvitoScraper) -> None: - """Внутри seed-брекета total > cap → бакет делится пополам; суб-бакеты пагинируются.""" - # Первый закрытый seed-брекет (0, 4_000_000) → walk_hi=3_999_999. - br_lo, br_hi_raw = _AVITO_PRICE_SEED_BRACKETS[0] - assert br_hi_raw is not None - br_hi = br_hi_raw - 1 # эксклюзивный верх - mid = (br_lo + br_hi) // 2 - call_log: list[tuple] = [] - - async def fake_fetch( - room_slug: str, page: int, min_price: int | None, max_price: int | None - ) -> str: - call_log.append((page, min_price, max_price)) - return f"page={page} min={min_price} max={max_price}" - - def fake_total(html: str) -> int | None: - import re - - min_m = re.search(r"min=(\w+)", html) - max_m = re.search(r"max=(\w+)", html) - min_p = int(min_m.group(1)) if min_m and min_m.group(1) != "None" else 0 - max_p = int(max_m.group(1)) if max_m and max_m.group(1) != "None" else None - # Внутри первого брекета [0, 3_999_999]: весь брекет > cap → split; - # половинки < cap → пагинируются. Прочие seed-брекеты — пусто (0). - if min_p == 0 and max_p == br_hi: - return 3000 # весь первый брекет — split - if min_p == 0 and max_p == mid: - return 800 # левая половина — пагинировать - if min_p == mid + 1 and max_p == br_hi: - return 700 # правая половина — пагинировать - return 0 # остальные seed-брекеты пусты - - def fake_parse(html: str, source_url_base: str) -> list[ScrapedLot]: - import re - - page_m = re.search(r"page=(\d+)", html) - page = int(page_m.group(1)) if page_m else 1 - min_m = re.search(r"min=(\w+)", html) - max_m = re.search(r"max=(\w+)", html) - min_p = min_m.group(1) if min_m else "0" - max_p = max_m.group(1) if max_m else "MAX" - if page == 1: - return [ - _make_lot(f"lot_{min_p}_{max_p}_1"), - _make_lot(f"lot_{min_p}_{max_p}_2"), - ] - return [] - - with ( - patch.object(scraper, "_fetch_rooms_page_html", side_effect=fake_fetch), - patch.object(scraper, "_extract_total_count", side_effect=fake_total), - patch.object(scraper, "_parse_html", side_effect=fake_parse), - patch.object(scraper, "_rotate_ip", return_value=False), - ): - lots = await scraper.fetch_all_secondary( - rooms_buckets=[("1-комн.", "1-komnatnye-X")], - price_cap_per_bucket=1400, - ) - - assert len(lots) > 0 - # Должны быть пробы суб-бакетов (после split весь брекет не пагинируется). - probed_max = [mx for (p, _mn, mx) in call_log if p == 1] - assert any(mx == mid for mx in probed_max), f"Ожидался probe левой половины, got {probed_max}" - - -@pytest.mark.asyncio -async def test_fetch_all_secondary_paginates_when_under_cap(scraper: AvitoScraper) -> None: - """total <= cap → пагинирует бакет полностью.""" - pages_fetched: list[int] = [] - - async def fake_fetch( - room_slug: str, page: int, min_price: int | None, max_price: int | None - ) -> str: - pages_fetched.append(page) - return f"page={page}" - - def fake_total(html: str) -> int | None: - return 100 # ceil(100/50) = 2 страницы - - def fake_parse(html: str, source_url_base: str) -> list[ScrapedLot]: - import re - - page_m = re.search(r"page=(\d+)", html) - page = int(page_m.group(1)) if page_m else 1 - if page <= 2: - return [_make_lot(f"lot_p{page}_{i}") for i in range(_AVITO_OFFERS_PER_PAGE)] - return [] - - with ( - patch.object(scraper, "_fetch_rooms_page_html", side_effect=fake_fetch), - patch.object(scraper, "_extract_total_count", side_effect=fake_total), - patch.object(scraper, "_parse_html", side_effect=fake_parse), - patch.object(scraper, "_rotate_ip", return_value=False), - ): - lots = await scraper.fetch_all_secondary( - rooms_buckets=[("2-комн.", "2-komnatnye-X")], - price_cap_per_bucket=1400, - ) - - assert len(lots) == 100, f"Ожидалось 100 уникальных лотов, получено {len(lots)}" - assert 2 in pages_fetched, "Должна быть запрошена страница 2" - - -@pytest.mark.asyncio -async def test_fetch_all_secondary_min_bracket_guard(scraper: AvitoScraper) -> None: - """hi - lo < MIN_BRACKET → пагинируем как есть (нет бесконечной рекурсии).""" - call_count = 0 - - async def fake_fetch( - room_slug: str, page: int, min_price: int | None, max_price: int | None - ) -> str: - nonlocal call_count - call_count += 1 - if call_count > 200: - raise RuntimeError("Бесконечная рекурсия — MIN_BRACKET guard не сработал") - return f"page={page}" - - def fake_total(html: str) -> int | None: - return 5000 # всегда > cap → без guard делилось бы бесконечно - - def fake_parse(html: str, source_url_base: str) -> list[ScrapedLot]: - return [] - - with ( - patch.object(scraper, "_fetch_rooms_page_html", side_effect=fake_fetch), - patch.object(scraper, "_extract_total_count", side_effect=fake_total), - patch.object(scraper, "_parse_html", side_effect=fake_parse), - patch.object(scraper, "_rotate_ip", return_value=False), - ): - await scraper._walk_price_range( - room_slug="1-komnatnye-X", - room_label="room_1_komn", - lo=1_000_000, - hi=1_000_000 + _AVITO_MIN_BRACKET - 1, - seen={}, - price_cap_per_bucket=1400, - max_pages_per_bucket=100, - ) - - # Guard сработал: рекурсии нет, только один проход пагинации (probe + до cap страниц). - assert call_count <= 101, f"Слишком много вызовов ({call_count}) — guard не сработал" - - -@pytest.mark.asyncio -async def test_fetch_all_secondary_concurrent_pages_deduped(scraper: AvitoScraper) -> None: - """Параллельная пагинация не ломает дедуп по source_id.""" - - async def fake_fetch( - room_slug: str, page: int, min_price: int | None, max_price: int | None - ) -> str: - return f"page={page}" - - def fake_total(html: str) -> int | None: - return 150 # ceil(150/50) = 3 страницы - - def fake_parse(html: str, source_url_base: str) -> list[ScrapedLot]: - import re - - page_m = re.search(r"page=(\d+)", html) - page = int(page_m.group(1)) if page_m else 1 - if page == 3: - return [_make_lot(f"lot_p1_{i}") for i in range(10)] # дубликаты стр 1 - return [_make_lot(f"lot_p{page}_{i}") for i in range(10)] - - with ( - patch.object(scraper, "_fetch_rooms_page_html", side_effect=fake_fetch), - patch.object(scraper, "_extract_total_count", side_effect=fake_total), - patch.object(scraper, "_parse_html", side_effect=fake_parse), - patch.object(scraper, "_rotate_ip", return_value=False), - ): - lots = await scraper.fetch_all_secondary( - rooms_buckets=[("1-комн.", "1-komnatnye-X")], - price_cap_per_bucket=1400, - concurrency=5, - ) - - # Страницы 1+2 = 20 уникальных, страница 3 дублирует → 20 - assert len(lots) == 20, f"Ожидалось 20 уникальных лотов (дедуп), получено {len(lots)}" - - -@pytest.mark.asyncio -async def test_on_bucket_receives_key_and_lots(scraper: AvitoScraper) -> None: - """on_bucket получает (bucket_key, lots) на каждый seed-брекет. - - bucket_key закрытого брекета = "room_label:lo:hi" (hi = br_hi-1, эксклюзивный); - открытого (последнего) = "room_label:lo:open". Одна on_bucket-нотификация - на каждый непустой seed-брекет (здесь все непустые → len(brackets) вызовов). - """ - received: list[tuple[str, int]] = [] - - def fake_on_bucket(bucket_key: str, lots: list[ScrapedLot]) -> None: - received.append((bucket_key, len(lots))) - - # Уникальные source_id на каждый брекет, чтобы on_bucket получал непустые - # списки во ВСЕХ брекетах (дедуп в seen не должен опустошать поздние брекеты). - async def fake_fetch( - room_slug: str, page: int, min_price: int | None, max_price: int | None - ) -> str: - return f"page={page} min={min_price} max={max_price}" - - def fake_total(html: str) -> int | None: - return 5 # 1 страница - - def fake_parse(html: str, source_url_base: str) -> list[ScrapedLot]: - import re - - min_m = re.search(r"min=(\w+)", html) - max_m = re.search(r"max=(\w+)", html) - min_p = min_m.group(1) if min_m else "0" - max_p = max_m.group(1) if max_m else "open" - page_m = re.search(r"page=(\d+)", html) - page = int(page_m.group(1)) if page_m else 1 - return [_make_lot(f"lot_{min_p}_{max_p}_p{page}_{i}") for i in range(5)] - - with ( - patch.object(scraper, "_fetch_rooms_page_html", side_effect=fake_fetch), - patch.object(scraper, "_extract_total_count", side_effect=fake_total), - patch.object(scraper, "_parse_html", side_effect=fake_parse), - patch.object(scraper, "_rotate_ip", return_value=False), - ): - await scraper.fetch_all_secondary( - rooms_buckets=[("2-комн.", "2-komnatnye-X")], - price_cap_per_bucket=1400, - on_bucket=fake_on_bucket, - ) - - # Один on_bucket на каждый seed-брекет (все непустые). - assert len(received) == len( - _AVITO_PRICE_SEED_BRACKETS - ), f"Ожидалось {len(_AVITO_PRICE_SEED_BRACKETS)} вызовов on_bucket, получено {len(received)}" - keys = [k for (k, _c) in received] - # Закрытые брекеты: hi = br_hi - 1. Открытый (последний): ':open'. - first_lo, first_hi = _AVITO_PRICE_SEED_BRACKETS[0] - assert first_hi is not None - assert keys[0] == f"room_2_komn:{first_lo}:{first_hi - 1}", f"Неверный ключ: {keys[0]!r}" - last_lo, last_hi = _AVITO_PRICE_SEED_BRACKETS[-1] - assert last_hi is None - assert keys[-1] == f"room_2_komn:{last_lo}:open", f"Открытый ключ неверен: {keys[-1]!r}" - assert all(count == 5 for (_k, count) in received) - - -@pytest.mark.asyncio -async def test_skip_buckets_skips_pagination(scraper: AvitoScraper) -> None: - """skip_buckets: для skip-бакета probe выполняется, пагинация и on_bucket — нет.""" - fetch_calls: list[tuple[int, int | None, int | None]] = [] - on_bucket_calls: list[str] = [] - - async def fake_fetch( - room_slug: str, page: int, min_price: int | None, max_price: int | None - ) -> str: - fetch_calls.append((page, min_price, max_price)) - return f"page={page} min={min_price} max={max_price}" - - def fake_total(html: str) -> int | None: - return 5 - - def fake_parse(html: str, source_url_base: str) -> list[ScrapedLot]: - import re - - min_m = re.search(r"min=(\w+)", html) - max_m = re.search(r"max=(\w+)", html) - return [_make_lot(f"lot_{min_m.group(1) if min_m else 0}_{max_m.group(1) if max_m else 0}")] - - def fake_on_bucket(bucket_key: str, lots: list[ScrapedLot]) -> None: - on_bucket_calls.append(bucket_key) - - # Skip ВСЕ брекеты → ни одной on_bucket-нотификации, но каждый брекет - # всё равно делает probe (page=1). - first_lo, first_hi = _AVITO_PRICE_SEED_BRACKETS[0] - assert first_hi is not None - first_key = f"room_1_komn:{first_lo}:{first_hi - 1}" - skip_all: set[str] = set() - for br_lo, br_hi in _AVITO_PRICE_SEED_BRACKETS: - hi_repr = "open" if br_hi is None else str(br_hi - 1) - skip_all.add(f"room_1_komn:{br_lo}:{hi_repr}") - - with ( - patch.object(scraper, "_fetch_rooms_page_html", side_effect=fake_fetch), - patch.object(scraper, "_extract_total_count", side_effect=fake_total), - patch.object(scraper, "_parse_html", side_effect=fake_parse), - patch.object(scraper, "_rotate_ip", return_value=False), - ): - lots = await scraper.fetch_all_secondary( - rooms_buckets=[("1-комн.", "1-komnatnye-X")], - price_cap_per_bucket=1400, - on_bucket=fake_on_bucket, - skip_buckets=skip_all, - ) - - # Ровно один probe (page=1) на каждый seed-брекет, без доп. страниц. - assert first_key in skip_all - assert all( - p == 1 for (p, _mn, _mx) in fetch_calls - ), f"Ожидались только probe-вызовы: {fetch_calls}" - assert len(fetch_calls) == len( - _AVITO_PRICE_SEED_BRACKETS - ), f"Ожидался 1 probe на брекет ({len(_AVITO_PRICE_SEED_BRACKETS)}), got {len(fetch_calls)}" - assert on_bucket_calls == [], "on_bucket не должен вызываться для skip-бакетов" - assert lots == [] - - -@pytest.mark.asyncio -async def test_secondary_only_drops_novostroyki(scraper: AvitoScraper) -> None: - """secondary_only=True: новостройки (listing_segment=='novostroyki') отброшены.""" - mixed = [ - _make_lot("nb_1", segment="novostroyki"), - _make_lot("vt_1", segment="vtorichka"), - _make_lot("vt_2", segment="vtorichka"), - ] - - async def fake_fetch( - room_slug: str, page: int, min_price: int | None, max_price: int | None - ) -> str: - return f"page={page}" - - def fake_total(html: str) -> int | None: - return 3 - - def fake_parse(html: str, source_url_base: str) -> list[ScrapedLot]: - import re - - page_m = re.search(r"page=(\d+)", html) - page = int(page_m.group(1)) if page_m else 1 - return mixed if page == 1 else [] - - with ( - patch.object(scraper, "_fetch_rooms_page_html", side_effect=fake_fetch), - patch.object(scraper, "_extract_total_count", side_effect=fake_total), - patch.object(scraper, "_parse_html", side_effect=fake_parse), - patch.object(scraper, "_rotate_ip", return_value=False), - ): - lots = await scraper.fetch_all_secondary( - rooms_buckets=[("1-комн.", "1-komnatnye-X")], - price_cap_per_bucket=1400, - secondary_only=True, - ) - - ids = {lot.source_id for lot in lots} - assert "nb_1" not in ids - assert ids == {"vt_1", "vt_2"} - - -@pytest.mark.asyncio -async def test_on_bucket_cancel_stops_run(scraper: AvitoScraper) -> None: - """on_bucket кидает RuntimeError('cancelled') → прогон прерывается.""" - - def cancel_on_bucket(bucket_key: str, lots: list[ScrapedLot]) -> None: - raise RuntimeError("cancelled") - - async def fake_fetch( - room_slug: str, page: int, min_price: int | None, max_price: int | None - ) -> str: - return f"page={page}" - - def fake_total(html: str) -> int | None: - return 5 - - def fake_parse(html: str, source_url_base: str) -> list[ScrapedLot]: - return [_make_lot("lot_1")] - - with ( - patch.object(scraper, "_fetch_rooms_page_html", side_effect=fake_fetch), - patch.object(scraper, "_extract_total_count", side_effect=fake_total), - patch.object(scraper, "_parse_html", side_effect=fake_parse), - patch.object(scraper, "_rotate_ip", return_value=False), - ): - with pytest.raises(RuntimeError, match="cancelled"): - await scraper.fetch_all_secondary( - rooms_buckets=[("1-комн.", "1-komnatnye-X"), ("2-комн.", "2-komnatnye-X")], - price_cap_per_bucket=1400, - on_bucket=cancel_on_bucket, - ) - - -# ── Probe-устойчивость к browser-ошибкам (HTTP 500 / timeout) ───────────────── - - -def _http_500_error() -> httpx.HTTPStatusError: - """Сборка httpx.HTTPStatusError(500) как из browser_fetcher.raise_for_status().""" - request = httpx.Request("POST", "http://tradein-browser:3000/fetch") - response = httpx.Response(500, request=request) - return httpx.HTTPStatusError("Server error '500'", request=request, response=response) - - -@pytest.mark.asyncio -async def test_probe_browser_500_on_wide_bucket_splits_not_crash( - scraper: AvitoScraper, -) -> None: - """Probe бросает HTTP 500 на ШИРОКОМ bucket → НЕ крах, происходит split. - - Воспроизводит прод-инцидент run 199: студии pmax=200M таймаутят на goto → - browser HTTP 500. Раньше это роняло весь full-load с 0 uniq. Теперь широкий - bucket принудительно делится пополам; под-бакеты пагинируются нормально. - """ - # Первый закрытый seed-брекет (0, 4_000_000) → walk_hi=3_999_999. - _br_lo, br_hi_raw = _AVITO_PRICE_SEED_BRACKETS[0] - assert br_hi_raw is not None - br_hi = br_hi_raw - 1 - probe_calls: list[tuple[int | None, int | None]] = [] - - async def fake_fetch( - room_slug: str, page: int, min_price: int | None, max_price: int | None - ) -> str: - # Probe верхней границы ШИРОКОГО первого брекета падает (имитация HTTP 500 - # от tradein-browser на тяжёлой выдаче) → должен произойти split. - if page == 1 and (min_price in (None, 0)) and max_price == br_hi: - probe_calls.append((min_price, max_price)) - raise _http_500_error() - return f"page={page} min={min_price} max={max_price}" - - def fake_total(html: str) -> int | None: - return 50 - - def fake_parse(html: str, source_url_base: str) -> list[ScrapedLot]: - import re - - page_m = re.search(r"page=(\d+)", html) - page = int(page_m.group(1)) if page_m else 1 - min_m = re.search(r"min=(\w+)", html) - max_m = re.search(r"max=(\w+)", html) - min_p = min_m.group(1) if min_m else "0" - max_p = max_m.group(1) if max_m else "MAX" - if page == 1: - return [_make_lot(f"lot_{min_p}_{max_p}_{i}") for i in range(3)] - return [] - - with ( - patch.object(scraper, "_fetch_rooms_page_html", side_effect=fake_fetch), - patch.object(scraper, "_extract_total_count", side_effect=fake_total), - patch.object(scraper, "_parse_html", side_effect=fake_parse), - patch.object(scraper, "_rotate_ip", return_value=False), - ): - lots = await scraper.fetch_all_secondary( - rooms_buckets=[("студии", "studii-X")], - price_cap_per_bucket=1400, - ) - - assert probe_calls, "Probe первого (широкого) брекета должен был вызваться и упасть" - assert len(lots) > 0, f"Ожидались лоты из под-бакетов после split, got {len(lots)}" - - -@pytest.mark.asyncio -async def test_probe_timeout_on_wide_bucket_splits_not_crash( - scraper: AvitoScraper, -) -> None: - """httpx.TimeoutException на широком bucket обрабатывается как HTTP 500 → split.""" - _br_lo, br_hi_raw = _AVITO_PRICE_SEED_BRACKETS[0] - assert br_hi_raw is not None - br_hi = br_hi_raw - 1 - probe_failed = False - - async def fake_fetch( - room_slug: str, page: int, min_price: int | None, max_price: int | None - ) -> str: - nonlocal probe_failed - if page == 1 and (min_price in (None, 0)) and max_price == br_hi: - probe_failed = True - raise httpx.TimeoutException("Page.goto timeout") - return f"page={page} min={min_price} max={max_price}" - - def fake_total(html: str) -> int | None: - return 40 - - def fake_parse(html: str, source_url_base: str) -> list[ScrapedLot]: - import re - - page_m = re.search(r"page=(\d+)", html) - page = int(page_m.group(1)) if page_m else 1 - min_m = re.search(r"min=(\w+)", html) - max_m = re.search(r"max=(\w+)", html) - min_p = min_m.group(1) if min_m else "0" - max_p = max_m.group(1) if max_m else "MAX" - return [_make_lot(f"lot_{min_p}_{max_p}_{i}") for i in range(2)] if page == 1 else [] - - with ( - patch.object(scraper, "_fetch_rooms_page_html", side_effect=fake_fetch), - patch.object(scraper, "_extract_total_count", side_effect=fake_total), - patch.object(scraper, "_parse_html", side_effect=fake_parse), - patch.object(scraper, "_rotate_ip", return_value=False), - ): - lots = await scraper.fetch_all_secondary( - rooms_buckets=[("студии", "studii-X")], - price_cap_per_bucket=1400, - ) - - assert probe_failed, "Probe должен был таймаутнуть" - assert len(lots) > 0, "После split под-бакеты должны собрать лоты" - - -@pytest.mark.asyncio -async def test_probe_browser_500_on_narrow_bucket_skips_not_crash( - scraper: AvitoScraper, -) -> None: - """Probe бросает HTTP 500 на УЗКОМ bucket (< MIN_BRACKET) → skip+return, без краха. - - Узкий bucket делить бесполезно — хвост теряется осознанно. Прогон не падает, - остальные комнатности продолжаются. - """ - second_room_paginated = False - - async def fake_fetch( - room_slug: str, page: int, min_price: int | None, max_price: int | None - ) -> str: - nonlocal second_room_paginated - if room_slug == "narrow-X": - raise _http_500_error() - if room_slug == "ok-X" and page == 1: - second_room_paginated = True - return f"page={page}" - - def fake_total(html: str) -> int | None: - return 5 - - def fake_parse(html: str, source_url_base: str) -> list[ScrapedLot]: - return [_make_lot("ok_lot")] if "page=1" in html else [] - - narrow_lo = 1_000_000 - narrow_hi = narrow_lo + _AVITO_MIN_BRACKET - 1 - seen: dict[str, ScrapedLot] = {} - - with ( - patch.object(scraper, "_fetch_rooms_page_html", side_effect=fake_fetch), - patch.object(scraper, "_extract_total_count", side_effect=fake_total), - patch.object(scraper, "_parse_html", side_effect=fake_parse), - patch.object(scraper, "_rotate_ip", return_value=False), - ): - await scraper._walk_price_range( - room_slug="narrow-X", - room_label="room_narrow", - lo=narrow_lo, - hi=narrow_hi, - seen=seen, - price_cap_per_bucket=1400, - max_pages_per_bucket=100, - ) - assert seen == {}, "Узкий bucket с упавшим probe не должен дать лотов" - - await scraper._walk_price_range( - room_slug="ok-X", - room_label="room_ok", - lo=0, - hi=_AVITO_MAX_PRICE, - seen=seen, - price_cap_per_bucket=1400, - max_pages_per_bucket=100, - ) - - assert second_room_paginated, "Вторая комнатность должна пагинироваться после skip" - assert "ok_lot" in seen - - -@pytest.mark.asyncio -async def test_probe_avito_blocked_propagates_not_swallowed( - scraper: AvitoScraper, -) -> None: - """AvitoBlockedError из probe ВСПЛЫВАЕТ (hard-block), а не глушится как total=None.""" - - async def fake_fetch( - room_slug: str, page: int, min_price: int | None, max_price: int | None - ) -> str: - raise AvitoBlockedError("Avito SERP firewall (browser-mode) — IP banned") - - def fake_total(html: str) -> int | None: - return 100 - - with ( - patch.object(scraper, "_fetch_rooms_page_html", side_effect=fake_fetch), - patch.object(scraper, "_extract_total_count", side_effect=fake_total), - patch.object(scraper, "_parse_html", side_effect=lambda h, source_url_base: []), - patch.object(scraper, "_rotate_ip", return_value=False), - ): - with pytest.raises(AvitoBlockedError): - await scraper.fetch_all_secondary( - rooms_buckets=[("1-комн.", "1-komnatnye-X")], - price_cap_per_bucket=1400, - ) - - -# ── Seed price-brackets entry point (perf: open top bucket) ─────────────────── - - -@pytest.mark.asyncio -async def test_fetch_all_secondary_walks_seed_brackets_per_room( - scraper: AvitoScraper, -) -> None: - """fetch_all_secondary зовёт _walk_price_range РОВНО len(seed) раз на комнатность. - - Аргументы lo/hi совпадают с seed-брекетами: закрытые передаются с hi=br_hi-1 - (эксклюзивный верх, без overlap на стыках), последний (открытый) — hi=None. - """ - walk_calls: list[tuple[int, int | None]] = [] - - async def spy_walk(*args: object, **kwargs: object) -> None: - walk_calls.append((kwargs["lo"], kwargs["hi"])) # type: ignore[arg-type] - # Не уходим в реальную рекурсию — фиксируем только верхнеуровневые входы. - return None - - with patch.object(scraper, "_walk_price_range", side_effect=spy_walk): - await scraper.fetch_all_secondary( - rooms_buckets=[("1-комн.", "1-komnatnye-X")], - price_cap_per_bucket=2000, - ) - - expected = [ - (br_lo, (br_hi - 1) if br_hi is not None else None) - for (br_lo, br_hi) in _AVITO_PRICE_SEED_BRACKETS - ] - assert walk_calls == expected, f"Неверные входы _walk_price_range: {walk_calls}" - # Последний — открытый (hi=None). - assert walk_calls[-1][1] is None, "Последний seed-брекет должен быть открытым (hi=None)" - # Ровно один вызов на seed-брекет (1 комнатность). - assert len(walk_calls) == len(_AVITO_PRICE_SEED_BRACKETS) - - -@pytest.mark.asyncio -async def test_open_bracket_probes_pmax_none_and_paginates_without_split( - scraper: AvitoScraper, -) -> None: - """Открытый брекет (hi=None): probe с pmax=None, пагинация БЕЗ попытки split. - - Даже если total > cap (мокнуто) — НЕ должно быть рекурсивного деления: - при hi=None mid=(lo+hi)//2 невозможен, поэтому leaf пагинируется напрямую. - Проверяем: (1) probe идёт с max_price=None (pmax не ставится), (2) НЕТ - под-вызовов с пересчитанным mid (никаких max_price != None в fetch'ах). - """ - open_lo = _AVITO_PRICE_SEED_BRACKETS[-1][0] - assert _AVITO_PRICE_SEED_BRACKETS[-1][1] is None - - fetch_args: list[tuple[int, int | None, int | None]] = [] - - async def fake_fetch( - room_slug: str, page: int, min_price: int | None, max_price: int | None - ) -> str: - fetch_args.append((page, min_price, max_price)) - return f"page={page} min={min_price} max={max_price}" - - # total > cap для ОТКРЫТОГО брекета — провоцируем split-соблазн. Закрытые - # брекеты возвращают 0 (не пагинируются), чтобы изолировать открытый. - def fake_total(html: str) -> int | None: - import re - - max_m = re.search(r"max=(\w+)", html) - max_raw = max_m.group(1) if max_m else "None" - if max_raw == "None": # открытый брекет (pmax отсутствует) - return 5000 # > cap=2000 — но split НЕ должен случиться - return 0 - - def fake_parse(html: str, source_url_base: str) -> list[ScrapedLot]: - import re - - page_m = re.search(r"page=(\d+)", html) - page = int(page_m.group(1)) if page_m else 1 - # Только открытый брекет (max=None) отдаёт лоты. - if "max=None" not in html: - return [] - return [_make_lot(f"open_lot_p{page}_{i}") for i in range(3)] - - with ( - patch.object(scraper, "_fetch_rooms_page_html", side_effect=fake_fetch), - patch.object(scraper, "_extract_total_count", side_effect=fake_total), - patch.object(scraper, "_parse_html", side_effect=fake_parse), - patch.object(scraper, "_rotate_ip", return_value=False), - ): - lots = await scraper.fetch_all_secondary( - rooms_buckets=[("1-комн.", "1-komnatnye-X")], - price_cap_per_bucket=2000, - max_pages_per_bucket=100, - ) - - # Probe открытого брекета: min_price=open_lo, max_price=None. - open_probes = [ - (mn, mx) for (p, mn, mx) in fetch_args if p == 1 and mn == open_lo and mx is None - ] - assert open_probes, f"Ожидался probe открытого брекета с pmax=None: {fetch_args}" - # Все fetch'и открытого брекета (min=open_lo) идут с max_price=None — НЕТ - # под-брекетов с вычисленным mid (что означало бы split). - open_fetches = [(mn, mx) for (p, mn, mx) in fetch_args if mn == open_lo] - assert all( - mx is None for (_mn, mx) in open_fetches - ), f"Открытый брекет НЕ должен порождать closed-под-брекеты (split): {open_fetches}" - # Открытый брекет всё же собрал лоты (пагинация прошла, total=5000 → cap pages). - assert len(lots) > 0, "Открытый брекет должен пагинироваться и собрать лоты" - - -@pytest.mark.asyncio -async def test_closed_bracket_under_cap_single_probe_then_paginate( - scraper: AvitoScraper, -) -> None: - """Закрытый брекет total str: - fetch_args.append((page, min_price, max_price)) - return f"page={page} min={min_price} max={max_price}" - - def fake_total(html: str) -> int | None: - import re - - min_m = re.search(r"min=(\w+)", html) - max_m = re.search(r"max=(\w+)", html) - min_p = min_m.group(1) if min_m else "0" - max_p = max_m.group(1) if max_m else "None" - if min_p == str(target_lo) and max_p == str(target_hi): - return 60 # ceil(60/50)=2 страницы, < cap → без split - return 0 - - def fake_parse(html: str, source_url_base: str) -> list[ScrapedLot]: - import re - - if f"min={target_lo} max={target_hi}" not in html: - return [] - page_m = re.search(r"page=(\d+)", html) - page = int(page_m.group(1)) if page_m else 1 - if page <= 2: - return [_make_lot(f"t_lot_p{page}_{i}") for i in range(30)] - return [] - - with ( - patch.object(scraper, "_fetch_rooms_page_html", side_effect=fake_fetch), - patch.object(scraper, "_extract_total_count", side_effect=fake_total), - patch.object(scraper, "_parse_html", side_effect=fake_parse), - patch.object(scraper, "_rotate_ip", return_value=False), - ): - lots = await scraper.fetch_all_secondary( - rooms_buckets=[("1-комн.", "1-komnatnye-X")], - price_cap_per_bucket=2000, - ) - - # Целевой брекет: probe (page=1) + страница 2, без под-брекетов (split). - target_pages = sorted(p for (p, mn, mx) in fetch_args if mn == target_lo and mx == target_hi) - assert target_pages == [1, 2], f"Ожидались probe+page2 без split: {target_pages}" - assert len(lots) == 60, f"Ожидалось 60 лотов (2 страницы × 30), got {len(lots)}" - - -@pytest.mark.asyncio -async def test_dedup_source_id_across_brackets(scraper: AvitoScraper) -> None: - """Дедуп по source_id работает между разными seed-брекетами (общий seen).""" - - async def fake_fetch( - room_slug: str, page: int, min_price: int | None, max_price: int | None - ) -> str: - return f"page={page} min={min_price} max={max_price}" - - def fake_total(html: str) -> int | None: - return 3 # 1 страница на каждый брекет - - def fake_parse(html: str, source_url_base: str) -> list[ScrapedLot]: - # ВСЕ брекеты отдают ОДИН и тот же source_id → должен остаться 1 уникальный. - return [_make_lot("shared_dup_id")] - - with ( - patch.object(scraper, "_fetch_rooms_page_html", side_effect=fake_fetch), - patch.object(scraper, "_extract_total_count", side_effect=fake_total), - patch.object(scraper, "_parse_html", side_effect=fake_parse), - patch.object(scraper, "_rotate_ip", return_value=False), - ): - lots = await scraper.fetch_all_secondary( - rooms_buckets=[("1-комн.", "1-komnatnye-X")], - price_cap_per_bucket=2000, - ) - - assert len(lots) == 1, f"Дедуп между брекетами должен дать 1 уникальный лот, got {len(lots)}" - assert lots[0].source_id == "shared_dup_id" diff --git a/tradein-mvp/backend/tests/scrapers/test_avito_incremental_pagination.py b/tradein-mvp/backend/tests/scrapers/test_avito_incremental_pagination.py deleted file mode 100644 index b112e7cc..00000000 --- a/tradein-mvp/backend/tests/scrapers/test_avito_incremental_pagination.py +++ /dev/null @@ -1,315 +0,0 @@ -"""INCREMENTAL SERP-пагинация Avito с date early-stop (param-gated, default off). - -`fetch_all_secondary(since=date)` на каждый (комнатность × seed-брекет) идёт -ПОСЛЕДОВАТЕЛЬНО newest-first и останавливается, как только страница уходит ниже -watermark `since` (SERP s=104 newest-first). `since=None` сохраняет точный -exhaustive bisection-обход (_walk_price_range) — без изменений. - -Тестируем через monkeypatch page-fetch seam (`_fetch_rooms_page_html`) + -`_parse_html`, отдавая контролируемые страницы ScrapedLot с заданными listing_date. -asyncio.sleep мокается в no-op для скорости (паттерн из -test_avito_403_backconnect_reconnect.py). -""" - -from datetime import date -from unittest.mock import AsyncMock, patch - -import pytest - -from app.services.scrapers import avito as avito_mod -from app.services.scrapers.avito import AvitoScraper -from app.services.scrapers.avito_exceptions import AvitoBlockedError -from app.services.scrapers.base import ScrapedLot - -SINCE = date(2026, 6, 10) - - -def _lot(item_id: str, listing_date: date | None, segment: str = "vtorichka") -> ScrapedLot: - """Минимальный валидный ScrapedLot с заданной датой/сегментом.""" - return ScrapedLot( - source="avito", - source_url=f"https://www.avito.ru/ekaterinburg/kvartiry/{item_id}", - source_id=item_id, - rooms=2, - price_rub=4_500_000, - listing_date=listing_date, - listing_segment=segment, - ) - - -def _make_scraper() -> AvitoScraper: - scraper = AvitoScraper() - scraper.request_delay_sec = 0.0 - return scraper - - -def _install_pages( - scraper: AvitoScraper, - monkeypatch: pytest.MonkeyPatch, - pages: list[list[ScrapedLot] | None], -) -> dict[str, int]: - """Подменяет page-fetch seam: страница p (1-based) → pages[p-1]. - - pages[i] is None → _fetch_rooms_page_html вернёт None (end-of-pages). - pages[i] == [] → HTML непустой, но _parse_html вернёт [] (end-of-pages). - Возвращает счётчик {"fetched": N} — сколько раз _fetch_rooms_page_html вызван. - """ - counter = {"fetched": 0} - - async def fake_fetch(room_slug: str, page: int, lo: int | None, hi: int | None) -> str | None: - counter["fetched"] += 1 - if page > len(pages): - return None - canned = pages[page - 1] - if canned is None: - return None - # Непустой HTML-маркер; реальный парс подменён ниже. - return f"" - - def fake_parse(html: str, source_url_base: str) -> list[ScrapedLot]: - # Достаём номер страницы из маркера, отдаём соответствующие лоты. - marker = html.split("data-page='")[1].split("'")[0] - page = int(marker) - canned = pages[page - 1] - return list(canned) if canned else [] - - monkeypatch.setattr(scraper, "_fetch_rooms_page_html", fake_fetch) - monkeypatch.setattr(scraper, "_parse_html", fake_parse) - return counter - - -@pytest.mark.asyncio -async def test_early_stop_on_all_older_page(monkeypatch: pytest.MonkeyPatch) -> None: - """page1 fresh, page2 mixed (fresh+old), page3 all older → стоп на page3, page4 НЕ фетчится.""" - scraper = _make_scraper() - pages: list[list[ScrapedLot] | None] = [ - [_lot("p1a", date(2026, 6, 15)), _lot("p1b", date(2026, 6, 14))], # все свежие - [_lot("p2a", date(2026, 6, 11)), _lot("p2b", date(2026, 6, 5))], # mixed (один >= since) - [_lot("p3a", date(2026, 6, 3)), _lot("p3b", date(2026, 6, 1))], # все старше since - [_lot("p4a", date(2026, 5, 20))], # НЕ должна фетчиться - ] - counter = _install_pages(scraper, monkeypatch, pages) - seen: dict[str, ScrapedLot] = {} - captured: list[tuple[str, list[ScrapedLot]]] = [] - - with patch.object(avito_mod.asyncio, "sleep", AsyncMock()): - await scraper._paginate_incremental_bracket( - room_slug="rooms-slug", - room_label="2к", - lo=0, - hi=None, - since=SINCE, - seen=seen, - max_pages_per_bucket=100, - secondary_only=True, - on_bucket=lambda k, lots: captured.append((k, lots)), - ) - - # page3 обработана (она old → break ПОСЛЕ неё), page4 не фетчилась. - assert counter["fetched"] == 3 - # Собраны лоты страниц 1-3 (включая boundary-страницу 3), но не 4. - assert set(seen.keys()) == {"p1a", "p1b", "p2a", "p2b", "p3a", "p3b"} - assert "p4a" not in seen - assert captured and captured[0][0] == "2к:0:open" - - -@pytest.mark.asyncio -async def test_no_early_stop_when_all_fresh(monkeypatch: pytest.MonkeyPatch) -> None: - """Все страницы свежее since до естественного конца (page возвращает None) → walk до конца.""" - scraper = _make_scraper() - pages: list[list[ScrapedLot] | None] = [ - [_lot("p1", date(2026, 6, 18))], - [_lot("p2", date(2026, 6, 16))], - [_lot("p3", date(2026, 6, 12))], - None, # end-of-pages - ] - counter = _install_pages(scraper, monkeypatch, pages) - seen: dict[str, ScrapedLot] = {} - - with patch.object(avito_mod.asyncio, "sleep", AsyncMock()): - await scraper._paginate_incremental_bracket( - room_slug="rooms-slug", - room_label="2к", - lo=0, - hi=None, - since=SINCE, - seen=seen, - max_pages_per_bucket=100, - secondary_only=True, - on_bucket=None, - ) - - # 3 страницы с лотами + 4-я (None) → 4 фетча, затем стоп. - assert counter["fetched"] == 4 - assert set(seen.keys()) == {"p1", "p2", "p3"} - - -@pytest.mark.asyncio -async def test_all_none_dates_grace_stop(monkeypatch: pytest.MonkeyPatch) -> None: - """Страницы all-None дат: page1 не стопит сразу, но 2 подряд undated → grace-стоп на page2.""" - scraper = _make_scraper() - pages: list[list[ScrapedLot] | None] = [ - [_lot("p1", None)], # undated → не стоп, streak=1 - [_lot("p2", None)], # undated → streak=2 → grace-стоп ПОСЛЕ page2 - [_lot("p3", None)], # НЕ должна фетчиться - ] - counter = _install_pages(scraper, monkeypatch, pages) - seen: dict[str, ScrapedLot] = {} - - with patch.object(avito_mod.asyncio, "sleep", AsyncMock()): - await scraper._paginate_incremental_bracket( - room_slug="rooms-slug", - room_label="2к", - lo=0, - hi=None, - since=SINCE, - seen=seen, - max_pages_per_bucket=100, - secondary_only=True, - on_bucket=None, - ) - - assert counter["fetched"] == 2 # page3 не фетчилась (grace bound) - assert set(seen.keys()) == {"p1", "p2"} - - -@pytest.mark.asyncio -async def test_none_date_does_not_stop_then_fresh_continues( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """Одна undated-страница между свежими не триггерит стоп (streak сбрасывается).""" - scraper = _make_scraper() - pages: list[list[ScrapedLot] | None] = [ - [_lot("p1", date(2026, 6, 15))], # fresh - [_lot("p2", None)], # undated → streak=1, продолжаем - [_lot("p3", date(2026, 6, 12))], # fresh → streak сброшен - [_lot("p4", date(2026, 6, 2))], # all older → стоп после page4 - [_lot("p5", date(2026, 5, 1))], # НЕ фетчится - ] - counter = _install_pages(scraper, monkeypatch, pages) - seen: dict[str, ScrapedLot] = {} - - with patch.object(avito_mod.asyncio, "sleep", AsyncMock()): - await scraper._paginate_incremental_bracket( - room_slug="rooms-slug", - room_label="2к", - lo=0, - hi=None, - since=SINCE, - seen=seen, - max_pages_per_bucket=100, - secondary_only=True, - on_bucket=None, - ) - - assert counter["fetched"] == 4 - assert set(seen.keys()) == {"p1", "p2", "p3", "p4"} - - -@pytest.mark.asyncio -async def test_secondary_only_filters_novostroyki(monkeypatch: pytest.MonkeyPatch) -> None: - """secondary_only=True отбрасывает novostroyki до дедупа в seen (как leaf-bucket).""" - scraper = _make_scraper() - pages: list[list[ScrapedLot] | None] = [ - [ - _lot("vt1", date(2026, 6, 15), segment="vtorichka"), - _lot("nb1", date(2026, 6, 15), segment="novostroyki"), - ], - None, - ] - _install_pages(scraper, monkeypatch, pages) - seen: dict[str, ScrapedLot] = {} - - with patch.object(avito_mod.asyncio, "sleep", AsyncMock()): - await scraper._paginate_incremental_bracket( - room_slug="rooms-slug", - room_label="2к", - lo=0, - hi=None, - since=SINCE, - seen=seen, - max_pages_per_bucket=100, - secondary_only=True, - on_bucket=None, - ) - - assert set(seen.keys()) == {"vt1"} # novostroyki отфильтрована - - -@pytest.mark.asyncio -async def test_blocked_error_propagates(monkeypatch: pytest.MonkeyPatch) -> None: - """AvitoBlockedError из page-фетча пробрасывается наверх (не глушится).""" - scraper = _make_scraper() - - async def fake_fetch(room_slug: str, page: int, lo: int | None, hi: int | None) -> str | None: - raise AvitoBlockedError("hard block") - - monkeypatch.setattr(scraper, "_fetch_rooms_page_html", fake_fetch) - seen: dict[str, ScrapedLot] = {} - - with patch.object(avito_mod.asyncio, "sleep", AsyncMock()): - with pytest.raises(AvitoBlockedError): - await scraper._paginate_incremental_bracket( - room_slug="rooms-slug", - room_label="2к", - lo=0, - hi=None, - since=SINCE, - seen=seen, - max_pages_per_bucket=100, - secondary_only=True, - on_bucket=None, - ) - - -@pytest.mark.asyncio -async def test_since_none_uses_exhaustive_path(monkeypatch: pytest.MonkeyPatch) -> None: - """since=None → НЕ зовёт _paginate_incremental_bracket; идёт бисекция (_walk_price_range).""" - scraper = _make_scraper() - - walk_calls = {"n": 0} - incr_calls = {"n": 0} - - async def fake_walk(**kwargs: object) -> None: - walk_calls["n"] += 1 - - async def fake_incr(**kwargs: object) -> None: - incr_calls["n"] += 1 - - monkeypatch.setattr(scraper, "_walk_price_range", fake_walk) - monkeypatch.setattr(scraper, "_paginate_incremental_bracket", fake_incr) - - await scraper.fetch_all_secondary( - rooms_buckets=[("2-комнатные", "rooms-slug")], - since=None, - ) - - # Один room × N seed-брекетов → _walk_price_range зовётся, incremental — нет. - assert walk_calls["n"] == len(avito_mod._AVITO_PRICE_SEED_BRACKETS) - assert incr_calls["n"] == 0 - - -@pytest.mark.asyncio -async def test_since_date_uses_incremental_path(monkeypatch: pytest.MonkeyPatch) -> None: - """since=date → зовёт _paginate_incremental_bracket вместо _walk_price_range.""" - scraper = _make_scraper() - - walk_calls = {"n": 0} - incr_calls = {"n": 0} - - async def fake_walk(**kwargs: object) -> None: - walk_calls["n"] += 1 - - async def fake_incr(**kwargs: object) -> None: - incr_calls["n"] += 1 - - monkeypatch.setattr(scraper, "_walk_price_range", fake_walk) - monkeypatch.setattr(scraper, "_paginate_incremental_bracket", fake_incr) - - await scraper.fetch_all_secondary( - rooms_buckets=[("2-комнатные", "rooms-slug")], - since=SINCE, - ) - - assert incr_calls["n"] == len(avito_mod._AVITO_PRICE_SEED_BRACKETS) - assert walk_calls["n"] == 0 diff --git a/tradein-mvp/backend/tests/scrapers/test_avito_sweep_retry.py b/tradein-mvp/backend/tests/scrapers/test_avito_sweep_retry.py deleted file mode 100644 index 269eca06..00000000 --- a/tradein-mvp/backend/tests/scrapers/test_avito_sweep_retry.py +++ /dev/null @@ -1,225 +0,0 @@ -"""P2 resilience — page-level retry для avito sweep (citywide / byrooms). - -Покрывает _fetch_serp_html_with_retry и интеграцию в fetch_city_wide: -один транзиентный сбой страницы больше НЕ обрывает весь прогон. - -Тесты: -1. transient на 1-й попытке → успех на 2-й → возвращает html (1 retry). -2. все попытки кидают generic → re-raise после исчерпания. -3. AvitoBlockedError на 1-й попытке → немедленный re-raise (НЕ retry, call_count==1). -4. None (non-200) → возвращается без retry. -5. integration fetch_city_wide: page=2 транзиентный 1 раз потом ок → обе страницы. -6. integration fetch_city_wide: page=3 persistent-fail → break после retry, - страницы 1-2 сохранены. - -Без сети, без БД, без curl_cffi. -""" - -from __future__ import annotations - -import os -from unittest.mock import AsyncMock, patch - -import pytest - -os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db") - -from app.services.scrapers.avito import AvitoScraper -from app.services.scrapers.avito_exceptions import AvitoBlockedError, AvitoRateLimitedError -from app.services.scrapers.base import ScrapedLot - - -def _make_lot(source_id: str) -> ScrapedLot: - return ScrapedLot( - source="avito", - source_url=f"https://www.avito.ru/ekaterinburg/kvartiry/{source_id}", - source_id=source_id, - price_rub=5_000_000, - ) - - -# --------------------------------------------------------------------------- -# 1. transient → retry → success -# --------------------------------------------------------------------------- - - -@pytest.mark.asyncio -async def test_retry_recovers_after_one_transient() -> None: - s = AvitoScraper() - calls = 0 - - async def flaky(url: str, page: int) -> str: - nonlocal calls - calls += 1 - if calls == 1: - raise RuntimeError("transient network blip") - return "ok" - - with patch.object(s, "_fetch_serp_html", side_effect=flaky): - with patch("app.services.scrapers.avito.asyncio.sleep", new=AsyncMock()): - html = await s._fetch_serp_html_with_retry("u", 1, max_retries=2) - - assert html == "ok" - assert calls == 2 # 1 fail + 1 success - - -# --------------------------------------------------------------------------- -# 2. all attempts fail → re-raise -# --------------------------------------------------------------------------- - - -@pytest.mark.asyncio -async def test_retry_reraises_after_exhaustion() -> None: - s = AvitoScraper() - calls = 0 - - async def always_fail(url: str, page: int) -> str: - nonlocal calls - calls += 1 - raise RuntimeError("persistent failure") - - with patch.object(s, "_fetch_serp_html", side_effect=always_fail): - with patch("app.services.scrapers.avito.asyncio.sleep", new=AsyncMock()): - with pytest.raises(RuntimeError, match="persistent failure"): - await s._fetch_serp_html_with_retry("u", 1, max_retries=2) - - assert calls == 3 # max_retries=2 → 3 attempts total - - -# --------------------------------------------------------------------------- -# 3. hard-block → immediate re-raise, no retry -# --------------------------------------------------------------------------- - - -@pytest.mark.asyncio -async def test_blocked_error_not_retried() -> None: - s = AvitoScraper() - calls = 0 - - async def blocked(url: str, page: int) -> str: - nonlocal calls - calls += 1 - raise AvitoBlockedError("IP banned") - - with patch.object(s, "_fetch_serp_html", side_effect=blocked): - with patch("app.services.scrapers.avito.asyncio.sleep", new=AsyncMock()): - with pytest.raises(AvitoBlockedError): - await s._fetch_serp_html_with_retry("u", 1, max_retries=2) - - assert calls == 1 # hard-block: no retry - - -@pytest.mark.asyncio -async def test_rate_limited_error_not_retried() -> None: - s = AvitoScraper() - calls = 0 - - async def rl(url: str, page: int) -> str: - nonlocal calls - calls += 1 - raise AvitoRateLimitedError("429") - - with patch.object(s, "_fetch_serp_html", side_effect=rl): - with patch("app.services.scrapers.avito.asyncio.sleep", new=AsyncMock()): - with pytest.raises(AvitoRateLimitedError): - await s._fetch_serp_html_with_retry("u", 1, max_retries=2) - - assert calls == 1 - - -# --------------------------------------------------------------------------- -# 4. None (non-200) → returned without retry -# --------------------------------------------------------------------------- - - -@pytest.mark.asyncio -async def test_none_returned_without_retry() -> None: - s = AvitoScraper() - calls = 0 - - async def returns_none(url: str, page: int) -> None: - nonlocal calls - calls += 1 - return None - - with patch.object(s, "_fetch_serp_html", side_effect=returns_none): - with patch("app.services.scrapers.avito.asyncio.sleep", new=AsyncMock()): - result = await s._fetch_serp_html_with_retry("u", 1, max_retries=2) - - assert result is None - assert calls == 1 # None is end-of-pagination, not an error - - -# --------------------------------------------------------------------------- -# 5. integration: transient page recovers, run does NOT abort -# --------------------------------------------------------------------------- - - -@pytest.mark.asyncio -async def test_fetch_city_wide_transient_page_does_not_abort() -> None: - """page=2 кидает transient 1 раз, потом ок → собираются обе страницы.""" - s = AvitoScraper() - page_fail_budget = {2: 1} # page=2 фейлит ровно один раз - - async def flaky_html(url: str, page: int) -> str: - if page_fail_budget.get(page, 0) > 0: - page_fail_budget[page] -= 1 - raise RuntimeError(f"transient on page {page}") - return f"page{page}" - - page_lots: dict[int, list[ScrapedLot]] = { - 1: [_make_lot("P1a"), _make_lot("P1b")], - 2: [_make_lot("P2a")], - 3: [], # end of pagination - } - - def mock_parse(html: str, source_url_base: str) -> list[ScrapedLot]: - for p, lots in page_lots.items(): - if f"page{p}" in html: - return lots - return [] - - with patch.object(s, "_fetch_serp_html", side_effect=flaky_html): - with patch.object(s, "_parse_html", side_effect=mock_parse): - with patch.object(s, "sleep_between_requests", new=AsyncMock()): - with patch("app.services.scrapers.avito.asyncio.sleep", new=AsyncMock()): - result = await s.fetch_city_wide(pages=10, delay_override_sec=0) - - ids = {lot.source_id for lot in result} - assert ids == {"P1a", "P1b", "P2a"} # page=2 survived the transient blip - - -# --------------------------------------------------------------------------- -# 6. integration: persistent page failure → break after retry, head kept -# --------------------------------------------------------------------------- - - -@pytest.mark.asyncio -async def test_fetch_city_wide_persistent_page_breaks_keeping_head() -> None: - """page=3 фейлит всегда → break после исчерпания retry; страницы 1-2 сохранены.""" - s = AvitoScraper() - - async def html(url: str, page: int) -> str: - if page >= 3: - raise RuntimeError(f"persistent on page {page}") - return f"page{page}" - - page_lots: dict[int, list[ScrapedLot]] = { - 1: [_make_lot("P1a")], - 2: [_make_lot("P2a")], - } - - def mock_parse(html_str: str, source_url_base: str) -> list[ScrapedLot]: - for p, lots in page_lots.items(): - if f"page{p}" in html_str: - return lots - return [] - - with patch.object(s, "_fetch_serp_html", side_effect=html): - with patch.object(s, "_parse_html", side_effect=mock_parse): - with patch.object(s, "sleep_between_requests", new=AsyncMock()): - with patch("app.services.scrapers.avito.asyncio.sleep", new=AsyncMock()): - result = await s.fetch_city_wide(pages=10, delay_override_sec=0) - - ids = {lot.source_id for lot in result} - assert ids == {"P1a", "P2a"} # pages 1-2 kept; page=3 persistent → graceful break diff --git a/tradein-mvp/backend/tests/scrapers/test_avito_unix_date_tz_consistency.py b/tradein-mvp/backend/tests/scrapers/test_avito_unix_date_tz_consistency.py index 4ee0ab61..cae92483 100644 --- a/tradein-mvp/backend/tests/scrapers/test_avito_unix_date_tz_consistency.py +++ b/tradein-mvp/backend/tests/scrapers/test_avito_unix_date_tz_consistency.py @@ -3,25 +3,21 @@ До фикса было 3 независимые копии с разными timezone: - avito.py (inline в `_build_sort_timestamp_map`) — MSK (документировано #726) - - avito_houses.py (легаси, удалён #2397 Part E2) — UTC - - avito_imv.py (легаси, удалён #2277) — naive/system-local - (`datetime.fromtimestamp(ts)` без tz) + - avito_houses.py — UTC + - avito_imv.py — naive/system-local (`datetime.fromtimestamp(ts)` без tz) Один и тот же epoch мог давать РАЗНЫЕ `date` в разных путях (пример из живой fixture avito_imv_getdata.json: removedDate=1756077885 → UTC 2025-08-24, MSK 2025-08-25). Офлайн: без сети/curl_cffi/БД, только импорт + чистые функции. -Легаси `app.services.scrapers.avito_imv` удалён (#2277 финальный шаг scraper_kit- -миграции, 0 runtime importers) — IMV-ветка consolidation-проверки переведена на -kit-эквивалент `scraper_kit.providers.avito.imv`, который использует ТОТ ЖЕ паттерн -(`from scraper_kit.providers.avito.shared import _unix_to_date`) — регрессия -"3 независимые копии с разными TZ" остаётся под наблюдением и на kit-стороне. - -Легаси `app.services.scrapers.avito_houses` удалён (#2397 Part E2, 0 runtime -importers) — houses-ветка consolidation-проверки переведена на kit-эквивалент -`scraper_kit.providers.avito.houses`, который использует ТОТ ЖЕ паттерн -(`from scraper_kit.providers.avito.shared import RUS_MONTHS, _unix_to_date`). +Легаси `app.services.scrapers.avito` / `avito_shared` / `avito_detail` / `avito_imv` / +`avito_houses` удалены (#2397 финальный шаг E, 0 runtime importers) — вся +consolidation-проверка переведена на kit-эквиваленты (`scraper_kit.providers.avito. +serp/shared/detail/houses/imv`), которые используют ТОТ ЖЕ единый источник правды +(`scraper_kit.providers.avito.shared._unix_to_date` / `RUS_MONTHS`) — регрессия +"3 независимые копии с разными TZ" остаётся под наблюдением на kit-стороне (теперь +единственной живой). """ from __future__ import annotations @@ -31,12 +27,12 @@ from datetime import date os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db") +from scraper_kit.providers.avito import detail as kit_avito_detail from scraper_kit.providers.avito import houses as kit_avito_houses from scraper_kit.providers.avito import imv as kit_avito_imv +from scraper_kit.providers.avito import serp as kit_avito_serp from scraper_kit.providers.avito import shared as kit_avito_shared -from app.services.scrapers import avito, avito_shared - # Живой пример из fixtures/avito_imv_getdata.json (placementHistory.items[0].removedDate) — # именно тот timestamp, на котором UTC и MSK расходились в date() ДО фикса # (UTC → 2025-08-24, MSK → 2025-08-25). @@ -45,54 +41,50 @@ _EXPECTED_MSK_DATE = date(2025, 8, 25) def test_shared_unix_to_date_uses_msk() -> None: - """Канонический источник правды — MSK (обоснование в avito_shared.py).""" - assert avito_shared._unix_to_date(_EPOCH_SECONDS) == _EXPECTED_MSK_DATE + """Канонический источник правды — MSK (обоснование в shared.py).""" + assert kit_avito_shared._unix_to_date(_EPOCH_SECONDS) == _EXPECTED_MSK_DATE def test_avito_houses_reexports_shared_unix_to_date() -> None: - """kit houses.py не держит свою копию — импортирует общую функцию (тот же паттерн, - что и легаси avito_houses.py до удаления, #2397 Part E2).""" + """kit houses.py не держит свою копию — импортирует общую функцию.""" assert kit_avito_houses._unix_to_date is kit_avito_shared._unix_to_date assert kit_avito_houses._unix_to_date(_EPOCH_SECONDS) == _EXPECTED_MSK_DATE def test_kit_avito_imv_reexports_shared_unix_to_date() -> None: """kit imv.py не держит свою копию — импортирует общую функцию (тот же паттерн, - что и kit houses.py/avito_shared.py после фикса #2129).""" + что и kit houses.py/shared.py после фикса #2129).""" assert kit_avito_imv._unix_to_date is kit_avito_shared._unix_to_date assert kit_avito_imv._unix_to_date(_EPOCH_SECONDS) == _EXPECTED_MSK_DATE def test_avito_serp_sort_timestamp_matches_shared_date() -> None: - """avito.py путь (SERP `sortTimeStamp`, epoch-МС) для того же момента времени + """serp.py путь (SERP `sortTimeStamp`, epoch-МС) для того же момента времени даёт ТУ ЖЕ дату, что и houses/IMV путь (epoch-секунды) — единственная разница - в единицах измерения epoch, TZ теперь общая (MSK) во всех путях (легаси и kit).""" + в единицах измерения epoch, TZ общая (MSK) во всех путях.""" html = ( f'
{{"id":8043936560,"sortTimeStamp":{_EPOCH_SECONDS * 1000},' f'"addressDetailed":{{"foo":"bar"}}}}
' ) - ts_map = avito._build_sort_timestamp_map(html) + ts_map = kit_avito_serp._build_sort_timestamp_map(html) assert ts_map["8043936560"] == _EXPECTED_MSK_DATE - # Легаси avito/avito_shared + kit avito_houses/avito_imv все согласованы на одном - # моменте времени: + # avito/shared + avito_houses/avito_imv все согласованы на одном моменте времени: assert ( ts_map["8043936560"] == kit_avito_houses._unix_to_date(_EPOCH_SECONDS) == kit_avito_imv._unix_to_date(_EPOCH_SECONDS) - == avito_shared._unix_to_date(_EPOCH_SECONDS) + == kit_avito_shared._unix_to_date(_EPOCH_SECONDS) ) def test_rus_months_single_source() -> None: """RUS_MONTHS (тоже была продублирована ×3) — общий словарь, не 3 копии.""" - from app.services.scrapers import avito_detail - - assert avito_detail.RUS_MONTHS is avito_shared.RUS_MONTHS + assert kit_avito_detail.RUS_MONTHS is kit_avito_shared.RUS_MONTHS assert kit_avito_houses.RUS_MONTHS is kit_avito_shared.RUS_MONTHS - assert avito_shared.RUS_MONTHS["декабря"] == 12 + assert kit_avito_shared.RUS_MONTHS["декабря"] == 12 def test_unix_to_date_none_and_zero() -> None: """None и 0 — sentinel «нет даты» → None (сохранено поведение старых копий).""" - assert avito_shared._unix_to_date(None) is None - assert avito_shared._unix_to_date(0) is None + assert kit_avito_shared._unix_to_date(None) is None + assert kit_avito_shared._unix_to_date(0) is None diff --git a/tradein-mvp/backend/tests/scrapers/test_browser_fetcher.py b/tradein-mvp/backend/tests/scrapers/test_browser_fetcher.py deleted file mode 100644 index 73c10a90..00000000 --- a/tradein-mvp/backend/tests/scrapers/test_browser_fetcher.py +++ /dev/null @@ -1,375 +0,0 @@ -"""Tests for BrowserFetcher (#884/#905, HTTP-клиент). - -Полностью замоканы через monkeypatch на httpx.AsyncClient.post: -не требуют установленного Firefox, camoufox или реального HTTP-сервера. - -Покрывают: - - fetch(): POST к правильному endpoint + разбор {"html": "..."} из ответа - - HTTPError триггерит один retry, затем поднимает исключение - - URL строится из settings.browser_http_endpoint - - импорт модуля не ломается без playwright/camoufox -""" - -from __future__ import annotations - -import os -from types import SimpleNamespace -from unittest.mock import AsyncMock, MagicMock, patch - -import httpx -import pytest - -os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test") - - -# ── helpers ──────────────────────────────────────────────────────────────────── - - -def _mock_settings( - browser_http_endpoint: str = "http://fake-browser:3000", -) -> SimpleNamespace: - return SimpleNamespace(browser_http_endpoint=browser_http_endpoint) - - -def _make_ok_response(html: str = "ok") -> MagicMock: - """Создаёт мок httpx.Response с JSON {"html": html} и статусом 200.""" - resp = MagicMock(spec=httpx.Response) - resp.status_code = 200 - resp.json = MagicMock(return_value={"html": html}) - resp.raise_for_status = MagicMock() # нет исключения - return resp - - -def _make_ok_json_response(status: int = 200, body: str = "{}") -> MagicMock: - """Создаёт мок httpx.Response с JSON {"status": ..., "body": ...} (для /fetch-json).""" - resp = MagicMock(spec=httpx.Response) - resp.status_code = 200 - resp.json = MagicMock(return_value={"status": status, "body": body}) - resp.raise_for_status = MagicMock() # нет исключения - return resp - - -def _make_error_response(status: int = 500) -> MagicMock: - """Создаёт мок httpx.Response который кидает HTTPStatusError при raise_for_status.""" - resp = MagicMock(spec=httpx.Response) - resp.status_code = status - resp.raise_for_status = MagicMock( - side_effect=httpx.HTTPStatusError( - f"Server error {status}", - request=MagicMock(), - response=resp, - ) - ) - return resp - - -# ── импорт не ломается ───────────────────────────────────────────────────────── - - -def test_import_browser_fetcher() -> None: - """import app.services.scrapers.browser_fetcher не поднимает исключений.""" - import app.services.scrapers.browser_fetcher # noqa: F401 - - -# ── fetch(): возвращает HTML из JSON-ответа ──────────────────────────────────── - - -@pytest.mark.asyncio -async def test_fetch_returns_html_from_json_response() -> None: - """fetch() разбирает {'html': ...} из 200-ответа и возвращает строку.""" - from app.services.scrapers.browser_fetcher import BrowserFetcher - - expected_html = "avito" - ok_resp = _make_ok_response(html=expected_html) - ms = _mock_settings() - - with patch("app.core.config.settings", ms): - fetcher = BrowserFetcher() - async with fetcher: - assert fetcher._client is not None - fetcher._client.post = AsyncMock(return_value=ok_resp) # type: ignore[method-assign] - result = await fetcher.fetch("https://avito.ru/test") - - assert result == expected_html - assert isinstance(result, str) - - -@pytest.mark.asyncio -async def test_fetch_posts_to_correct_endpoint() -> None: - """fetch() делает POST к {browser_http_endpoint}/fetch с {"url", "source"}. - - Дефолтный source="avito" — обратная совместимость со старым поведением, когда - сервер игнорирует source при выключенном FEATURE_BROWSER_POOL_ENABLED. - """ - from app.services.scrapers.browser_fetcher import BrowserFetcher - - endpoint = "http://fake-browser:3000" - ok_resp = _make_ok_response() - ms = _mock_settings(browser_http_endpoint=endpoint) - - with patch("app.core.config.settings", ms): - fetcher = BrowserFetcher() - async with fetcher: - assert fetcher._client is not None - post_mock = AsyncMock(return_value=ok_resp) - fetcher._client.post = post_mock # type: ignore[method-assign] - await fetcher.fetch("https://example.com/page") - - post_mock.assert_called_once_with( - f"{endpoint}/fetch", - json={"url": "https://example.com/page", "source": "avito"}, - ) - - -@pytest.mark.asyncio -async def test_fetch_posts_source_field_when_set() -> None: - """BrowserFetcher(source="yandex")._post_fetch шлёт body с "source": "yandex".""" - from app.services.scrapers.browser_fetcher import BrowserFetcher - - endpoint = "http://fake-browser:3000" - ok_resp = _make_ok_response() - ms = _mock_settings(browser_http_endpoint=endpoint) - - with patch("app.core.config.settings", ms): - fetcher = BrowserFetcher(source="yandex") - async with fetcher: - assert fetcher._client is not None - post_mock = AsyncMock(return_value=ok_resp) - fetcher._client.post = post_mock # type: ignore[method-assign] - await fetcher.fetch("https://realty.yandex.ru/ekb/") - - post_mock.assert_called_once_with( - f"{endpoint}/fetch", - json={"url": "https://realty.yandex.ru/ekb/", "source": "yandex"}, - ) - - -# ── fetch_json(): in-page fetch через /fetch-json (#915 Stage 2) ─────────────── - - -@pytest.mark.asyncio -async def test_fetch_json_returns_dict_from_json_response() -> None: - """fetch_json() возвращает {"status","body"} dict из 200-ответа /fetch-json.""" - from app.services.scrapers.browser_fetcher import BrowserFetcher - - ok_resp = _make_ok_json_response(status=200, body='{"items": []}') - ms = _mock_settings() - - with patch("app.core.config.settings", ms): - fetcher = BrowserFetcher() - async with fetcher: - assert fetcher._client is not None - fetcher._client.post = AsyncMock(return_value=ok_resp) # type: ignore[method-assign] - result = await fetcher.fetch_json("https://avito.ru/api/x") - - assert result == {"status": 200, "body": '{"items": []}'} - - -@pytest.mark.asyncio -async def test_fetch_json_posts_full_payload() -> None: - """fetch_json() POST'ит url/source/method/headers/body/origin к /fetch-json.""" - from app.services.scrapers.browser_fetcher import BrowserFetcher - - endpoint = "http://fake-browser:3000" - ok_resp = _make_ok_json_response() - ms = _mock_settings(browser_http_endpoint=endpoint) - - with patch("app.core.config.settings", ms): - fetcher = BrowserFetcher(source="avito") - async with fetcher: - assert fetcher._client is not None - post_mock = AsyncMock(return_value=ok_resp) - fetcher._client.post = post_mock # type: ignore[method-assign] - await fetcher.fetch_json( - "https://avito.ru/api/listings", - method="POST", - headers={"content-type": "application/json"}, - body='{"q": 1}', - origin="https://avito.ru/moskva/kvartiry", - ) - - post_mock.assert_called_once_with( - f"{endpoint}/fetch-json", - json={ - "url": "https://avito.ru/api/listings", - "source": "avito", - "method": "POST", - "headers": {"content-type": "application/json"}, - "body": '{"q": 1}', - "origin": "https://avito.ru/moskva/kvartiry", - }, - ) - - -@pytest.mark.asyncio -async def test_fetch_json_defaults_get_empty_headers_none_body() -> None: - """Дефолты: method=GET, headers={}, body=None, origin=None.""" - from app.services.scrapers.browser_fetcher import BrowserFetcher - - endpoint = "http://fake-browser:3000" - ok_resp = _make_ok_json_response() - ms = _mock_settings(browser_http_endpoint=endpoint) - - with patch("app.core.config.settings", ms): - fetcher = BrowserFetcher(source="cian") - async with fetcher: - assert fetcher._client is not None - post_mock = AsyncMock(return_value=ok_resp) - fetcher._client.post = post_mock # type: ignore[method-assign] - await fetcher.fetch_json("https://cian.ru/api/x") - - post_mock.assert_called_once_with( - f"{endpoint}/fetch-json", - json={ - "url": "https://cian.ru/api/x", - "source": "cian", - "method": "GET", - "headers": {}, - "body": None, - "origin": None, - }, - ) - - -@pytest.mark.asyncio -async def test_fetch_json_retries_once_on_transport_error() -> None: - """TransportError первый раз → один retry → успешный второй вызов.""" - from app.services.scrapers.browser_fetcher import BrowserFetcher - - ok_resp = _make_ok_json_response(status=200, body="retry-ok") - connect_err = httpx.ConnectError("Connection refused") - ms = _mock_settings() - - with patch("app.core.config.settings", ms): - fetcher = BrowserFetcher() - async with fetcher: - assert fetcher._client is not None - post_mock = AsyncMock(side_effect=[connect_err, ok_resp]) - fetcher._client.post = post_mock # type: ignore[method-assign] - - with patch("app.services.scrapers.browser_fetcher.asyncio.sleep", AsyncMock()): - result = await fetcher.fetch_json("https://avito.ru/api/retry") - - assert result == {"status": 200, "body": "retry-ok"} - assert post_mock.await_count == 2 - - -@pytest.mark.asyncio -async def test_fetch_json_raises_after_two_http_errors() -> None: - """Оба вызова /fetch-json кидают HTTPError → исключение пробрасывается.""" - from app.services.scrapers.browser_fetcher import BrowserFetcher - - error_resp_1 = _make_error_response(500) - error_resp_2 = _make_error_response(500) - ms = _mock_settings() - - with patch("app.core.config.settings", ms): - fetcher = BrowserFetcher() - async with fetcher: - assert fetcher._client is not None - post_mock = AsyncMock(side_effect=[error_resp_1, error_resp_2]) - fetcher._client.post = post_mock # type: ignore[method-assign] - - with patch("app.services.scrapers.browser_fetcher.asyncio.sleep", AsyncMock()): - with pytest.raises(httpx.HTTPStatusError): - await fetcher.fetch_json("https://avito.ru/api/fail") - - assert post_mock.await_count == 2 - - -# ── retry при HTTPError ──────────────────────────────────────────────────────── - - -@pytest.mark.asyncio -async def test_fetch_retries_once_on_http_error() -> None: - """При HTTPError первый вызов → retry → успешный второй вызов.""" - from app.services.scrapers.browser_fetcher import BrowserFetcher - - html = "retry-ok" - ok_resp = _make_ok_response(html=html) - error_resp = _make_error_response(500) - ms = _mock_settings() - - with patch("app.core.config.settings", ms): - fetcher = BrowserFetcher() - async with fetcher: - assert fetcher._client is not None - # Первый вызов кидает HTTPStatusError, второй возвращает OK - post_mock = AsyncMock(side_effect=[error_resp, ok_resp]) - fetcher._client.post = post_mock # type: ignore[method-assign] - - with patch("app.services.scrapers.browser_fetcher.asyncio.sleep", AsyncMock()): - result = await fetcher.fetch("https://avito.ru/retry") - - assert result == html - assert post_mock.await_count == 2 - - -@pytest.mark.asyncio -async def test_fetch_raises_after_two_http_errors() -> None: - """Если оба вызова кидают HTTPError — исключение пробрасывается.""" - from app.services.scrapers.browser_fetcher import BrowserFetcher - - error_resp_1 = _make_error_response(500) - error_resp_2 = _make_error_response(500) - ms = _mock_settings() - - with patch("app.core.config.settings", ms): - fetcher = BrowserFetcher() - async with fetcher: - assert fetcher._client is not None - post_mock = AsyncMock(side_effect=[error_resp_1, error_resp_2]) - fetcher._client.post = post_mock # type: ignore[method-assign] - - with patch("app.services.scrapers.browser_fetcher.asyncio.sleep", AsyncMock()): - with pytest.raises(httpx.HTTPStatusError): - await fetcher.fetch("https://avito.ru/fail") - - assert post_mock.await_count == 2 - - -@pytest.mark.asyncio -async def test_fetch_retries_on_connect_error() -> None: - """ConnectError (подкласс TransportError) тоже триггерит retry.""" - from app.services.scrapers.browser_fetcher import BrowserFetcher - - html = "connect-retry-ok" - ok_resp = _make_ok_response(html=html) - connect_err = httpx.ConnectError("Connection refused") - ms = _mock_settings() - - with patch("app.core.config.settings", ms): - fetcher = BrowserFetcher() - async with fetcher: - assert fetcher._client is not None - post_mock = AsyncMock(side_effect=[connect_err, ok_resp]) - fetcher._client.post = post_mock # type: ignore[method-assign] - - with patch("app.services.scrapers.browser_fetcher.asyncio.sleep", AsyncMock()): - result = await fetcher.fetch("https://avito.ru/connect") - - assert result == html - assert post_mock.await_count == 2 - - -# ── non-http exception пробрасывается без перехвата ─────────────────────────── - - -@pytest.mark.asyncio -async def test_fetch_propagates_unexpected_exception() -> None: - """Не-HTTP исключение (ValueError и т.п.) не подавляется и не вызывает retry.""" - from app.services.scrapers.browser_fetcher import BrowserFetcher - - ms = _mock_settings() - - with patch("app.core.config.settings", ms): - fetcher = BrowserFetcher() - async with fetcher: - assert fetcher._client is not None - post_mock = AsyncMock(side_effect=ValueError("unexpected")) - fetcher._client.post = post_mock # type: ignore[method-assign] - - with pytest.raises(ValueError, match="unexpected"): - await fetcher.fetch("https://avito.ru/bad") - - assert post_mock.await_count == 1 diff --git a/tradein-mvp/backend/tests/scrapers/test_price_brackets.py b/tradein-mvp/backend/tests/scrapers/test_price_brackets.py deleted file mode 100644 index 8f91eac9..00000000 --- a/tradein-mvp/backend/tests/scrapers/test_price_brackets.py +++ /dev/null @@ -1,48 +0,0 @@ -"""Тесты общего seed price-brackets констант (avito/cian/yandex).""" - -from itertools import pairwise - -from app.services.scrapers.price_brackets import ( - _EKB_SECONDARY, - CITY_PRICE_SEED_BRACKETS, - get_price_seed_brackets, -) - - -def test_default_returns_ekb(): - """get_price_seed_brackets() без аргумента возвращает EKB-сетку.""" - assert get_price_seed_brackets() == _EKB_SECONDARY - assert get_price_seed_brackets("ekaterinburg") == _EKB_SECONDARY - - -def test_unknown_city_falls_back_to_ekb(): - """Неизвестный город → fallback на EKB (не KeyError).""" - assert get_price_seed_brackets("atlantis") == _EKB_SECONDARY - - -def test_last_bracket_is_open(): - """Последний брекет ОТКРЫТ (hi=None) — catch-all хвоста люкса без потолка.""" - brackets = get_price_seed_brackets() - assert brackets[-1][1] is None - # Только последний брекет открыт — остальные закрыты. - for lo, hi in brackets[:-1]: - assert hi is not None - assert hi > lo - - -def test_brackets_are_contiguous(): - """Закрытые брекеты соприкасаются: hi[i] == lo[i+1] (без дыр/перекрытий).""" - brackets = get_price_seed_brackets() - for (lo, hi), (next_lo, _next_hi) in pairwise(brackets): - assert hi == next_lo, f"разрыв между [{lo},{hi}] и [{next_lo},...]" - - -def test_first_bracket_starts_at_zero(): - """Первый брекет стартует с 0 (вся вторичка с нижнего края).""" - assert get_price_seed_brackets()[0][0] == 0 - - -def test_city_dict_contains_ekaterinburg(): - """City-dict готов под multi-city — сейчас единственный ключ ekaterinburg.""" - assert "ekaterinburg" in CITY_PRICE_SEED_BRACKETS - assert CITY_PRICE_SEED_BRACKETS["ekaterinburg"] is _EKB_SECONDARY diff --git a/tradein-mvp/backend/tests/services/scrapers/test_cian_resolve_zhk_url.py b/tradein-mvp/backend/tests/services/scrapers/test_cian_resolve_zhk_url.py index 14550942..f3b3519e 100644 --- a/tradein-mvp/backend/tests/services/scrapers/test_cian_resolve_zhk_url.py +++ b/tradein-mvp/backend/tests/services/scrapers/test_cian_resolve_zhk_url.py @@ -4,7 +4,12 @@ The legacy resolve_cian_zhk_url hit https://cian.ru/zhk// which now 404s; th search-based resolver fetches the cat.php newbuilding-SERP and extracts the canonical zhk-.cian.ru URL from its markup. No real network: the curl_cffi session is mocked; the regex is exercised against a trimmed real-prod SERP fixture. + +Легаси `app.services.scrapers.cian_newbuilding` удалён (#2397 финальный шаг E, 0 +runtime-импортёров) — переведено на kit-эквивалент `scraper_kit.providers.cian. +newbuilding` (byte-identical strangler-копия). """ + from __future__ import annotations import os @@ -12,21 +17,18 @@ import sys from pathlib import Path from unittest.mock import AsyncMock, MagicMock -# DATABASE_URL required by config before any app import (cian_newbuilding → app.core.config). +# DATABASE_URL required by config before any app import. os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test") # WeasyPrint stub — not installed in CI without GTK; pulled in transitively by app imports. sys.modules.setdefault("weasyprint", MagicMock()) import pytest # noqa: E402 - -from app.services.scrapers.cian_newbuilding import ( # noqa: E402 +from scraper_kit.providers.cian.newbuilding import ( # noqa: E402 _extract_zhk_url_from_serp, resolve_cian_zhk_url_via_search, ) -_FIXTURE = ( - Path(__file__).resolve().parents[2] / "fixtures" / "cian_catph_newbuilding_serp.html" -) +_FIXTURE = Path(__file__).resolve().parents[2] / "fixtures" / "cian_catph_newbuilding_serp.html" def _serp_fixture() -> str: @@ -72,7 +74,7 @@ def test_extract_zhk_url_anchors_on_title_not_recommendation() -> None: # Banner / recommendation block rendered BEFORE the title, carrying a foreign slug. '
' 'похожий ЖК (реклама)' - 'and a bare one too https://zhk-another-promo-ekb-i.cian.ru in text
' + "and a bare one too https://zhk-another-promo-ekb-i.cian.ru in text" # The canonical title block — its anchor is the queried ЖК. '
' '

Купить квартиру в ' @@ -93,7 +95,7 @@ def test_extract_zhk_url_fallback_first_match_when_no_title() -> None: """ html = ( 'noise A ' - 'and later https://zhk-second-ekb-i.cian.ru more' + "and later https://zhk-second-ekb-i.cian.ru more" ) assert _extract_zhk_url_from_serp(html) == "https://zhk-first-ekb-i.cian.ru" diff --git a/tradein-mvp/backend/tests/support/README.md b/tradein-mvp/backend/tests/support/README.md index 1b48c877..2ec30445 100644 --- a/tradein-mvp/backend/tests/support/README.md +++ b/tradein-mvp/backend/tests/support/README.md @@ -2,16 +2,23 @@ ## parity.py — legacy → scraper_kit parity harness (issue #2304) -Инструмент для issues #2305-#2310 (миграция 12 неймигрированных importers +Инструмент для issues #2305-#2310 (миграция неймигрированных importers `app/services/scrapers/*` → `scraper_kit` эквиваленты, см. audit `Scraper_Kit_Legacy_Dependency_Audit_0703` в vault). Каждая такая миграция -должна доказать, что kit-путь даёт ТОТ ЖЕ результат, что и legacy-путь на -одном и том же входе — для этого используйте `assert_parity`. +должна была доказать, что kit-путь даёт ТОТ ЖЕ результат, что и legacy-путь на +одном и том же входе — для этого использовался `assert_parity`. -### Быстрый старт +Легаси `app/services/scrapers/` каталог полностью удалён (#2397 финальный шаг +E, миграция завершена) — все golden-parity/legacy-vs-kit тесты, построенные на +`assert_parity`, удалены вместе с ним (kit — единственный живой путь). Harness +(`parity.py`) остаётся в дереве на случай будущих strangler-миграций +(например legacy-кода за пределами `app/services/scrapers/`). + +### Быстрый старт (иллюстративный пример; `legacy_fn` — гипотетическая функция +### из будущего strangler-миграции модуля, не из уже удалённого `app/services/scrapers/`) ```python -from app.services.scrapers.cian_valuation import evaluate_via_cian as legacy_fn +from some_legacy_module import evaluate_via_cian as legacy_fn from scraper_kit.providers.cian.valuation import evaluate_via_cian as kit_fn from tests.support.parity import assert_parity @@ -37,8 +44,9 @@ def test_cian_valuation_parity() -> None: - любое другое значение (str, dict, ...) → передаётся как единственный позиционный аргумент: `legacy_fn(fixture)`. -Начните с 1-2 hardcoded HTML-фикстур/dict'ов (см. -`tests/scrapers/test_avito_detail_kit_parity.py` — референсный пример). +Начните с 1-2 hardcoded HTML-фикстур/dict'ов (git-история +`tests/scrapers/test_avito_detail_kit_parity.py`, удалён вместе с legacy +`app/services/scrapers/` в #2397, содержит референсный пример). DB-фикстуры НЕ нужны для чистых parse/compute-функций — используйте их только если сама legacy/kit-функция реально требует Session. diff --git a/tradein-mvp/backend/tests/test_781_quality_gate.py b/tradein-mvp/backend/tests/test_781_quality_gate.py index f266f0c2..75a39f9b 100644 --- a/tradein-mvp/backend/tests/test_781_quality_gate.py +++ b/tradein-mvp/backend/tests/test_781_quality_gate.py @@ -219,7 +219,7 @@ def _make_scraped_lot( source_id: str | None = None, price_rub: int = 5_000_000, ) -> Any: - from app.services.scrapers.base import ScrapedLot + from scraper_kit.base import ScrapedLot return ScrapedLot( source=source, @@ -399,10 +399,12 @@ async def test_754_block_page_http200_page1_raises() -> None: Before #754 this scenario caused a silent empty return ([]) masking a content block. Refs: PR #779 / commit 489666c. """ - from app.services.scrapers.avito import AvitoScraper - from app.services.scrapers.avito_exceptions import AvitoContentBlockedError + from scraper_kit.avito_exceptions import AvitoContentBlockedError + from scraper_kit.providers.avito.serp import AvitoScraper - scraper = AvitoScraper() + from app.services.scraper_adapters import RealScraperConfig + + scraper = AvitoScraper(RealScraperConfig()) with patch.object(scraper, "_fetch_serp_html", new=AsyncMock(return_value=_BLOCKPAGE_HTML)): with pytest.raises(AvitoContentBlockedError): await scraper.fetch_around(56.838, 60.605, radius_m=1000, pages=1) @@ -416,9 +418,11 @@ async def test_754_block_page_http200_page_gt1_graceful_return() -> None: end-of-pagination break). The call must NOT raise. Refs: PR #779 / commit 489666c. """ - from app.services.scrapers.avito import AvitoScraper + from scraper_kit.providers.avito.serp import AvitoScraper - scraper = AvitoScraper() + from app.services.scraper_adapters import RealScraperConfig + + scraper = AvitoScraper(RealScraperConfig()) async def _serp_html_side_effect(url: str, page: int) -> str: return _SERP_HTML if page == 1 else _BLOCKPAGE_HTML diff --git a/tradein-mvp/backend/tests/test_849_matching_savepoint.py b/tradein-mvp/backend/tests/test_849_matching_savepoint.py deleted file mode 100644 index b67ae4f4..00000000 --- a/tradein-mvp/backend/tests/test_849_matching_savepoint.py +++ /dev/null @@ -1,337 +0,0 @@ -"""Tests for issue #849 Parts 2 and 3. - -Part 2: same normalized_address → exactly 1 house and 1 alias key. -Part 3: batch save_listings with one failing lot → siblings persist, - no orphan SAVEPOINT / no whole-batch rollback. - -All tests use mock DB sessions — no real Postgres required. -""" - -from __future__ import annotations - -import logging -from contextlib import contextmanager -from unittest.mock import MagicMock, patch - -import pytest - -from app.services.matching.houses import _insert_alias, match_or_create_house -from app.services.matching.normalize import address_fingerprint, normalize_address -from app.services.scrapers.base import ScrapedLot, save_listings - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - - -def _make_db(rows: list[dict | None]) -> MagicMock: - """Build a Session mock returning rows sequentially for each execute() call.""" - db = MagicMock() - side_effects = [] - for row in rows: - result = MagicMock() - mappings = MagicMock() - if row is None: - mappings.first.return_value = None - mappings.one.return_value = None - else: - mappings.first.return_value = row - mappings.one.return_value = row - result.mappings.return_value = mappings - side_effects.append(result) - db.execute.side_effect = side_effects - return db - - -def _mock_save_db(inserted: bool = True, listing_id: int = 42) -> MagicMock: - """Mock Session for save_listings: INSERT RETURNING + SAVEPOINTs as no-op.""" - row = MagicMock() - row.id = listing_id - row.inserted = inserted - - result = MagicMock() - result.fetchone.return_value = row - - db = MagicMock() - db.execute.return_value = result - - @contextmanager - def _nested(): - yield MagicMock() - - db.begin_nested.side_effect = _nested - return db - - -def _base_lot(**overrides) -> ScrapedLot: - defaults = { - "source": "avito", - "source_url": "https://www.avito.ru/ekaterinburg/kvartiry/1-k_11111", - "source_id": "11111", - "price_rub": 4_000_000, - } - defaults.update(overrides) - return ScrapedLot(**defaults) - - -# --------------------------------------------------------------------------- -# Part 2: fingerprint↔alias sync -# --------------------------------------------------------------------------- - - -class TestFingerprintAliasSync: - """Two listings with the same address → exactly 1 house and 1 alias key. - - Scenario: two scrapers submit the same street address but with coords that - differ beyond the 4-decimal rounding boundary — they produce different - fingerprints. The matcher must still resolve to the same canonical house - and NOT create a second alias for the same normalized_address. - """ - - def test_same_normalized_address_resolves_to_same_house_via_tier2b(self): - """First call misses all tiers → creates house + alias (fp=fp1). - Second call with same address but different coords: - - Tier 2a: fp2 ≠ fp1 → miss - - Tier 2b: normalized_address hit → returns existing house_id - Result: same house_id, NOT a new house row. - """ - address = "Екатеринбург, ул. Ленина, 5" - lat1, lon1 = 56.838000, 60.594600 # yields fp1 - lat2, lon2 = 56.839100, 60.594600 # 4dp rounds to 56.8391 ≠ 56.8380 → fp2 ≠ fp1 - - fp1 = address_fingerprint(address, lat1, lon1) - fp2 = address_fingerprint(address, lat2, lon2) - assert fp1 != fp2, "precondition: coords produce different fingerprints" - - # ── First call: all tiers miss → INSERT new house ─────────────────── - db1 = _make_db( - [ - None, # pg_advisory_xact_lock - None, # house_sources miss (Tier 1) - None, # fingerprint miss (Tier 2a) - None, # normalized_address miss (Tier 2b) — first time - None, # geo miss (Tier 3; no lat/lon provided here) - {"id": 7}, # INSERT RETURNING id - None, # _upsert_house_source - None, # _insert_alias - ] - ) - house_id_1, _, method_1 = match_or_create_house( - db1, - "avito", - "ext-001", - address=address, - lat=lat1, - lon=lon1, - ) - assert method_1 == "new" - assert house_id_1 == 7 - - # ── Second call (same address, different coords) ──────────────────── - # Tier 2a misses (fp2 ≠ stored fp1), Tier 2b hits on normalized_address. - db2 = _make_db( - [ - None, # pg_advisory_xact_lock - None, # house_sources miss (Tier 1) - None, # fingerprint miss (Tier 2a — fp2 not registered yet) - {"house_id": 7}, # normalized_address hit (Tier 2b) - None, # _upsert_house_source - None, # _insert_alias (register fp2) - ] - ) - house_id_2, _, method_2 = match_or_create_house( - db2, - "cian", - "ext-002", - address=address, - lat=lat2, - lon=lon2, - ) - assert ( - method_2 == "fingerprint" - ), f"Expected 'fingerprint' (Tier 2b match), got {method_2!r}" - assert ( - house_id_2 == 7 - ), f"Expected same house_id=7, got {house_id_2} — second scraper created a duplicate" - - def test_same_normalized_address_tier2a_hits_directly_when_fp_matches(self): - """When the fingerprint IS already registered (same address + coords within 11 m), - Tier 2a hits directly and normalized_address fallback is not reached.""" - address = "ул Мира 10" - lat, lon = 56.84, 60.59 - - db = _make_db( - [ - None, # pg_advisory_xact_lock - None, # house_sources miss - {"house_id": 99}, # fingerprint hit (Tier 2a) - None, # _upsert_house_source - None, # _insert_alias - ] - ) - house_id, conf, method = match_or_create_house( - db, - "yandex", - "ext-yz", - address=address, - lat=lat, - lon=lon, - ) - assert method == "fingerprint" - assert house_id == 99 - assert conf == 0.9 - - def test_insert_alias_upsert_keeps_fingerprint_in_sync(self): - """_insert_alias with ON CONFLICT DO UPDATE ensures the stored fingerprint - is refreshed, not left stale from the first writer. - - Verify the emitted SQL contains 'DO UPDATE SET' (not 'DO NOTHING'). - """ - db = MagicMock() - result = MagicMock() - db.execute.return_value = result - - _insert_alias( - db, - house_id=5, - address="ул. Ленина 5", - fp="aabbccdd" * 4, - source="avito", - ) - - assert db.execute.called - sql_str = str(db.execute.call_args[0][0]) - assert ( - "DO UPDATE SET" in sql_str - ), "_insert_alias must use ON CONFLICT DO UPDATE SET fingerprint, not DO NOTHING" - assert "fingerprint" in sql_str - - def test_two_listings_same_address_exactly_one_alias_key(self): - """Integration check: normalize_address produces identical key for two scrapers - submitting the same address in different formats, so at most one alias row can - exist (UNIQUE normalized_address constraint holds). - - This test verifies the normalization layer — the DB constraint is structural. - addr_a uses the abbreviated form 'ул.' which expands to 'улица'; - addr_b already uses the full form. Both should normalize identically. - """ - addr_a = "ул. Ленина 5" # abbreviated 'ул.' → expands to 'улица' - addr_b = "улица ленина 5" # already expanded, lowercase - assert normalize_address(addr_a) == normalize_address(addr_b), ( - "normalize_address must produce identical key for both address forms " - "so the UNIQUE(normalized_address) constraint prevents duplicate aliases" - ) - - -# --------------------------------------------------------------------------- -# Part 3: single SAVEPOINT level — batch isolation -# --------------------------------------------------------------------------- - - -class TestSingleSavepointBatchIsolation: - """save_listings: one failing lot → siblings persist; no orphan SAVEPOINT.""" - - @pytest.fixture(autouse=True) - def _patch_matching(self): - """Stub matching so save_listings tests don't hit real DB queries.""" - with ( - patch("app.services.scrapers.base.match_or_create_house") as m_house, - patch("app.services.scrapers.base.upsert_listing_source") as m_link, - ): - m_house.return_value = (101, 1.0, "new") - m_link.return_value = None - self.m_house = m_house - self.m_link = m_link - yield - - def test_failing_lot_does_not_abort_batch(self, caplog): - """Lot whose match_or_create_house raises → that lot's hook is rolled back - (outer SAVEPOINT), but other lots in the batch are fully inserted. - - No whole-batch rollback, no orphan SAVEPOINTs. - """ - # First lot: house match succeeds; second lot: house match raises - self.m_house.side_effect = [ - (10, 1.0, "new"), - RuntimeError("simulated house-match DB error"), - (12, 1.0, "new"), - ] - - db = _mock_save_db(inserted=True, listing_id=50) - - lots = [ - _base_lot( - source_id="ok-1", - address="Екатеринбург, Ленина 5", - lat=56.84, - lon=60.59, - ), - _base_lot( - source_id="fail-1", - address="Екатеринбург, Мира 10", - lat=56.85, - lon=60.60, - ), - _base_lot( - source_id="ok-2", - address="Екатеринбург, Куйбышева 3", - lat=56.83, - lon=60.58, - ), - ] - - with caplog.at_level(logging.WARNING, logger="app.services.scrapers.base"): - inserted, _updated = save_listings(db, lots) - - # All 3 listings INSERT'd (listing INSERT is outside any SAVEPOINT) - assert inserted == 3 - - # House match attempted on all 3 (all have address+coords) - assert self.m_house.call_count == 3 - - # upsert_listing_source: lots ok-1 and ok-2 succeed; fail-1's hook was aborted - # by the outer SAVEPOINT so link not called for it. - assert self.m_link.call_count == 2 - - # Warning logged for the failing lot - warns = [r for r in caplog.records if "match_failed" in r.getMessage()] - assert len(warns) == 1 - - def test_no_begin_nested_inside_link_listing_to_house(self): - """Verify that _link_listing_to_house does NOT open its own begin_nested. - - The outer per-row SAVEPOINT (one call per lot in save_listings) is the only - SAVEPOINT level allowed. If begin_nested is called more than N lots times, - that indicates a double-nesting. - """ - db = _mock_save_db(inserted=True, listing_id=77) - lots = [_base_lot(source_id=str(i)) for i in range(3)] - - save_listings(db, lots) - - # save_listings calls begin_nested THREE times per lot: - # 1) upsert SAVEPOINT — ловит IntegrityError на UNIQUE(source,source_id) - # (migration 133) при dedup_hash-дрейфе → rollback + прямой UPDATE (#1921); - # 2) snapshot SAVEPOINT; - # 3) hook SAVEPOINT (_link_listing_to_house). - # If _link_listing_to_house opened an INNER begin_nested too, - # the count would be > 3 * len(lots). The intent of this test (no double-nesting - # INSIDE the hook) holds — base.py keeps exactly one SAVEPOINT level in the hook. - assert db.begin_nested.call_count == 3 * len(lots), ( - f"Expected exactly 3 begin_nested per lot (upsert + snapshot + hook outer), " - f"got {db.begin_nested.call_count} — inner SAVEPOINT may still be present" - ) - - def test_whole_batch_not_rolled_back_on_single_hook_failure(self, caplog): - """When one lot's hook fails, db.commit() is still called (batch persisted).""" - self.m_house.side_effect = RuntimeError("boom") - - db = _mock_save_db(inserted=True, listing_id=10) - lots = [_base_lot(source_id="only-one", address="x", lat=56.8, lon=60.5)] - - with caplog.at_level(logging.WARNING, logger="app.services.scrapers.base"): - save_listings(db, lots) - - # db.commit() called despite the hook failure — no whole-batch rollback - db.commit.assert_called_once() - db.rollback.assert_not_called() diff --git a/tradein-mvp/backend/tests/test_avito_address_clean.py b/tradein-mvp/backend/tests/test_avito_address_clean.py deleted file mode 100644 index c00805cf..00000000 --- a/tradein-mvp/backend/tests/test_avito_address_clean.py +++ /dev/null @@ -1,82 +0,0 @@ -import pytest - -from app.services.scrapers.avito import _clean_address - - -def test_strips_css_noise(): - raw = ( - "ул. Токарей, 56к1Площадь 1905 года.css-39hgr0{fill:currentColor;" - "height:1em;box-sizing:border-box;}.css-39hgr0:focus{outline:none;}" - "от 31 мин.Геологическая.css-39hgr0{...}" - ) - assert _clean_address(raw) == "ул. Токарей, 56к1" - - -def test_clean_address_passthrough(): - assert _clean_address("ул. Малышева, 1") == "ул. Малышева, 1" - - -def test_clean_handles_none(): - assert _clean_address(None) is None - assert _clean_address("") is None - assert _clean_address(" ") is None - - -def test_clean_strips_trailing_punctuation(): - assert _clean_address("ул. Ленина, 5,") == "ул. Ленина, 5" - - -# ── #1773: де-слип номера дома и маркера района/нас.пункта ──────────────────── -@pytest.mark.parametrize( - "raw,expected", - [ - # реальный кейс с прода - ( - "Свердловская обл., Екатеринбург, ул. Евгения Савкова, 29р-н Академический", - "Свердловская обл., Екатеринбург, ул. Евгения Савкова, 29, р-н Академический", - ), - # литера дома слиплась с маркером - ("ул. X, 17Ар-н Пионерский", "ул. X, 17А, р-н Пионерский"), - ("ул. X, 44Бр-н Запад", "ул. X, 44Б, р-н Запад"), - # корпус через дробь - ("ул. X, 35/2р-н Юг", "ул. X, 35/2, р-н Юг"), - # корпус через «сN» - ("ул. X, 33с1р-н Центр", "ул. X, 33с1, р-н Центр"), - # прочие маркеры - ("ул. X, 12мкр Солнечный", "ул. X, 12, мкр Солнечный"), - ("ул. X, 5район Кировский", "ул. X, 5, район Кировский"), - ("ул. X, 100к2р-н Чкаловский", "ул. X, 100к2, р-н Чкаловский"), - # маркеры населённых пунктов - ("ул. X, 12с. Кашино", "ул. X, 12, с. Кашино"), - ("ул. X, 10д. Малое", "ул. X, 10, д. Малое"), - ("ул. X, 25г. Берёзовский", "ул. X, 25, г. Берёзовский"), - ], -) -def test_deglue_house_marker(raw, expected): - assert _clean_address(raw) == expected - - -@pytest.mark.parametrize( - "raw", - [ - # уже расклеено — идемпотентность - "ул. Евгения Савкова, 29, р-н Академический", - # нормальные адреса без склейки — разделять НЕ надо - "ул. Малышева, 1", - "ул. Токарей, 56к1", - "ул. Ленина, 5", - "ул. Серова, 45", - "г. Екатеринбург, ул. Ленина, 10", - "снт Дружба, уч. 12", - "ул. Декабристов, 16", - ], -) -def test_deglue_negative_no_change(raw): - # для этих строк де-слип не должен ничего менять (кроме штатного strip пунктуации) - assert _clean_address(raw) == raw.strip(" ,.\n\t") - - -def test_deglue_idempotent(): - once = _clean_address("ул. Евгения Савкова, 29р-н Академический") - twice = _clean_address(once) - assert once == twice == "ул. Евгения Савкова, 29, р-н Академический" diff --git a/tradein-mvp/backend/tests/test_avito_blockpage.py b/tradein-mvp/backend/tests/test_avito_blockpage.py deleted file mode 100644 index 1d99cc8c..00000000 --- a/tradein-mvp/backend/tests/test_avito_blockpage.py +++ /dev/null @@ -1,125 +0,0 @@ -"""#823 — детект HTTP-200 block/captcha страницы и parse_failures счётчик. - -Тест 1: _is_firewall_page возвращает True на fixtures/avito_blockpage_sample.html. -Тест 2: _parse_html на блок-странице возвращает 0 лотов (нет карточек). -Тест 3: fetch_around бросает AvitoContentBlockedError когда page=1 даёт 0 лотов - (гарантия что silent 0-done НЕ происходит — основной DoD #823). -Тест 4: _dom_card_to_lot инкрементирует parse_failures при сломанной карточке. - -Без сети, без БД, без curl_cffi. -""" - -import os -from pathlib import Path -from unittest.mock import AsyncMock, MagicMock, patch - -import pytest -from selectolax.parser import HTMLParser - -os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db") - -from app.services.scrapers.avito import AvitoScraper, _is_firewall_page -from app.services.scrapers.avito_exceptions import AvitoContentBlockedError - -FIXTURES = Path(__file__).parent / "fixtures" -BLOCKPAGE_HTML = (FIXTURES / "avito_blockpage_sample.html").read_text(encoding="utf-8") -SERP_HTML = (FIXTURES / "avito_serp_sample.html").read_text(encoding="utf-8") - - -# --------------------------------------------------------------------------- -# 1. _is_firewall_page детектит блок-маркер в HTML -# --------------------------------------------------------------------------- - - -def test_is_firewall_page_detects_block_fixture() -> None: - """Блок-страница с 'доступ ограничен' и 'firewall-container' → True.""" - assert _is_firewall_page(BLOCKPAGE_HTML) is True - - -def test_is_firewall_page_false_for_normal_serp() -> None: - """Нормальная SERP страница не является блок-страницей.""" - assert _is_firewall_page(SERP_HTML) is False - - -def test_is_firewall_page_false_for_empty_html() -> None: - assert _is_firewall_page("") is False - - -# --------------------------------------------------------------------------- -# 2. _parse_html на блок-странице даёт 0 лотов (нет data-marker="item") -# --------------------------------------------------------------------------- - - -def test_parse_html_block_page_returns_empty_list() -> None: - """Блок-страница не содержит карточек — парсер возвращает [].""" - scraper = AvitoScraper() - lots = scraper._parse_html(BLOCKPAGE_HTML, "https://www.avito.ru/x") - assert lots == [] - - -# --------------------------------------------------------------------------- -# 3. fetch_around raises AvitoContentBlockedError при 0 лотах на page=1 -# (гарантия что silent 0-done НЕ происходит — основной DoD #823) -# --------------------------------------------------------------------------- - - -@pytest.mark.asyncio -async def test_fetch_around_raises_on_zero_lots_page1() -> None: - """HTTP-200 + пустой SERP (блок/captcha) на page=1 → AvitoContentBlockedError. - - Мокаем _fetch_serp_html чтобы вернуть блок-страницу без сети. - """ - scraper = AvitoScraper() - with patch.object( - scraper, - "_fetch_serp_html", - new=AsyncMock(return_value=BLOCKPAGE_HTML), - ): - with pytest.raises(AvitoContentBlockedError): - await scraper.fetch_around(56.838, 60.605, radius_m=1000, pages=1) - - -# --------------------------------------------------------------------------- -# 4. parse_failures счётчик инкрементируется при сломанной карточке -# --------------------------------------------------------------------------- - - -def test_parse_failures_starts_at_zero() -> None: - scraper = AvitoScraper() - assert scraper.parse_failures == 0 - - -def test_parse_failures_not_incremented_on_link_missing() -> None: - """Карточка без item-title link → ранний return None, НЕ через except.""" - scraper = AvitoScraper() - broken_html = '
no link
' - tree = HTMLParser(broken_html) - cards = tree.css('[data-marker="item"]') - result = scraper._dom_card_to_lot(cards[0], "https://www.avito.ru/x") - assert result is None - assert scraper.parse_failures == 0 # ранний return, не exception path - - -def test_parse_failures_incremented_on_exception() -> None: - """css_first бросает TypeError → except path → parse_failures++.""" - scraper = AvitoScraper() - bad_card = MagicMock() - bad_card.css_first.side_effect = TypeError("simulated DOM error") - bad_card.attributes = {} - - result = scraper._dom_card_to_lot(bad_card, "https://www.avito.ru/x") - assert result is None - assert scraper.parse_failures == 1 # #823: счётчик вырос - - -def test_parse_failures_accumulates_across_cards() -> None: - """Несколько сломанных карточек аккумулируются в одном счётчике.""" - scraper = AvitoScraper() - - for _ in range(3): - bad_card = MagicMock() - bad_card.css_first.side_effect = RuntimeError("bad card") - bad_card.attributes = {} - scraper._dom_card_to_lot(bad_card, "https://www.avito.ru/x") - - assert scraper.parse_failures == 3 diff --git a/tradein-mvp/backend/tests/test_avito_browser_mode.py b/tradein-mvp/backend/tests/test_avito_browser_mode.py deleted file mode 100644 index 0311bb64..00000000 --- a/tradein-mvp/backend/tests/test_avito_browser_mode.py +++ /dev/null @@ -1,520 +0,0 @@ -"""#901/#1805 — avito SERP fetch via BrowserFetcher (camoufox), curl_cffi fallback. - -Tests: -1. browser branch default (#1805): __aenter__ → _browser set AND _cffi fallback armed. -2. curl branch unchanged: _browser is None → _cffi session used directly. -3. firewall-detect in browser-mode: firewall HTML + no rotate URL + no cffi → AvitoBlockedError. -3b. #1805 curl_cffi fallback: browser firewall → curl retry returns HTML / re-raises if also banned. -4. pagination terminates via empty-parse: browser HTML with 0 lots → caller loop breaks. - -Без сети, без БД, без curl_cffi. -""" - -from __future__ import annotations - -import os -from unittest.mock import AsyncMock, MagicMock, patch - -import httpx -import pytest - -os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db") - -from app.services.scrapers.avito import _FIREWALL_MARKERS, AvitoScraper -from app.services.scrapers.avito_exceptions import AvitoBlockedError, AvitoRateLimitedError -from app.services.scrapers.base import ScrapedLot - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - -_NORMAL_HTML = "normal avito SERP" -_FIREWALL_HTML = f"{_FIREWALL_MARKERS[0]}" - - -def _sidecar_error(status: int, error: str) -> httpx.HTTPStatusError: - """Build a tradein-browser sidecar HTTPStatusError with a real JSON body. - - Mirrors what BrowserFetcher.fetch re-raises: an httpx.HTTPStatusError whose - `.response` is intact so `.response.json()["error"]` and `.response.status_code` - work the same as in production. - """ - request = httpx.Request("POST", "http://tradein-browser:3000/fetch") - response = httpx.Response(status, json={"error": error}, request=request) - return httpx.HTTPStatusError(f"{status} error", request=request, response=response) - - -def _make_mock_browser(html: str = _NORMAL_HTML) -> MagicMock: - """Return a mock BrowserFetcher context manager that yields html from fetch().""" - mock = MagicMock() - mock.__aenter__ = AsyncMock(return_value=mock) - mock.__aexit__ = AsyncMock(return_value=None) - mock.fetch = AsyncMock(return_value=html) - return mock - - -# --------------------------------------------------------------------------- -# 1. browser branch chosen -# --------------------------------------------------------------------------- - - -@pytest.mark.asyncio -async def test_aenter_browser_mode_arms_cffi_fallback() -> None: - """#1805: mode=browser → __aenter__ поднимает BrowserFetcher И curl_cffi-fallback. - - Раньше browser-mode оставлял _cffi=None; теперь curl_cffi-сессия строится тут - же как fallback на firewall/пусто (см. _fetch_serp_html_browser). - """ - mock_browser = _make_mock_browser() - scraper = AvitoScraper() - - with patch("app.services.scrapers.avito.settings") as mock_settings: - mock_settings.scraper_fetch_mode = "browser" - mock_settings.scraper_proxy_url = None - mock_settings.avito_proxy_rotate_url = None - mock_settings.avito_proxy_max_rotations = 2 - - with patch( - "app.services.scrapers.avito.BrowserFetcher", - return_value=mock_browser, - ): - result = await scraper.__aenter__() - - assert result is scraper - assert scraper._browser is mock_browser - # curl_cffi fallback armed (real AsyncSession built, no network/proxy). - assert scraper._cffi is not None - await scraper._cffi.close() - - -@pytest.mark.asyncio -async def test_aenter_proactive_rotate_when_rotate_url_set() -> None: - """#1731: rotate_url configured → __aenter__ calls _rotate_ip exactly once at start. - - _rotate_ip is mocked (no real changeip HTTP / no 9s sleep). Browser mode used - to keep BaseScraper setup out of the picture. - """ - mock_browser = _make_mock_browser() - scraper = AvitoScraper() - mock_rotate = AsyncMock(return_value=True) - - with patch("app.services.scrapers.avito.settings") as mock_settings: - mock_settings.scraper_fetch_mode = "browser" - mock_settings.scraper_proxy_url = None - mock_settings.avito_proxy_rotate_url = "http://proxy.example.com/changeip" - mock_settings.avito_proxy_max_rotations = 4 - - with patch( - "app.services.scrapers.avito.BrowserFetcher", - return_value=mock_browser, - ): - with patch.object(scraper, "_rotate_ip", new=mock_rotate): - result = await scraper.__aenter__() - - assert result is scraper - mock_rotate.assert_awaited_once() - - -@pytest.mark.asyncio -async def test_aenter_no_proactive_rotate_when_rotate_url_none() -> None: - """#1731: rotate_url is None → __aenter__ must NOT call _rotate_ip.""" - mock_browser = _make_mock_browser() - scraper = AvitoScraper() - mock_rotate = AsyncMock(return_value=True) - - with patch("app.services.scrapers.avito.settings") as mock_settings: - mock_settings.scraper_fetch_mode = "browser" - mock_settings.scraper_proxy_url = None - mock_settings.avito_proxy_rotate_url = None - mock_settings.avito_proxy_max_rotations = 4 - - with patch( - "app.services.scrapers.avito.BrowserFetcher", - return_value=mock_browser, - ): - with patch.object(scraper, "_rotate_ip", new=mock_rotate): - await scraper.__aenter__() - - mock_rotate.assert_not_awaited() - - -@pytest.mark.asyncio -async def test_aenter_proactive_rotate_failure_does_not_abort() -> None: - """#1731: a failed proactive rotate (returns False / raises) must NOT abort __aenter__.""" - mock_browser = _make_mock_browser() - scraper = AvitoScraper() - - with patch("app.services.scrapers.avito.settings") as mock_settings: - mock_settings.scraper_fetch_mode = "browser" - mock_settings.scraper_proxy_url = None - mock_settings.avito_proxy_rotate_url = "http://proxy.example.com/changeip" - mock_settings.avito_proxy_max_rotations = 4 - - with patch( - "app.services.scrapers.avito.BrowserFetcher", - return_value=mock_browser, - ): - # _rotate_ip raises — sweep must proceed (browser still attached) - with patch.object( - scraper, "_rotate_ip", new=AsyncMock(side_effect=RuntimeError("changeip down")) - ): - result = await scraper.__aenter__() - - assert result is scraper - assert scraper._browser is mock_browser - - -@pytest.mark.asyncio -async def test_fetch_serp_html_browser_mode_returns_html() -> None: - """_fetch_serp_html with _browser set returns HTML string, never touches _cffi.""" - scraper = AvitoScraper() - mock_browser = _make_mock_browser(_NORMAL_HTML) - scraper._browser = mock_browser - - result = await scraper._fetch_serp_html("https://www.avito.ru/test", page=1) - - assert result == _NORMAL_HTML - mock_browser.fetch.assert_awaited_once_with("https://www.avito.ru/test") - # _cffi not armed (direct _browser assignment bypasses __aenter__, which would - # otherwise build the curl_cffi fallback session); the browser path never touches it. - assert scraper._cffi is None - - -# --------------------------------------------------------------------------- -# 2. curl branch unchanged -# --------------------------------------------------------------------------- - - -@pytest.mark.asyncio -async def test_fetch_serp_html_curl_mode_uses_cffi() -> None: - """Default mode (_browser is None) → _fetch_serp_html goes through curl path.""" - scraper = AvitoScraper() - - # Mock the curl session returning HTTP 200 with normal HTML - mock_response = MagicMock() - mock_response.status_code = 200 - mock_response.text = _NORMAL_HTML - - mock_cffi = MagicMock() - mock_cffi.get = AsyncMock(return_value=mock_response) - scraper._cffi = mock_cffi - - # _browser must be None (default curl path) - assert scraper._browser is None - - result = await scraper._fetch_serp_html("https://www.avito.ru/test", page=1) - - assert result == _NORMAL_HTML - mock_cffi.get.assert_awaited_once_with("https://www.avito.ru/test") - - -# --------------------------------------------------------------------------- -# 3. firewall-detect in browser-mode → AvitoBlockedError (no rotate URL) -# --------------------------------------------------------------------------- - - -@pytest.mark.asyncio -async def test_browser_mode_firewall_raises_blocked_error() -> None: - """Browser firewall + no rotate URL + no curl fallback (_cffi=None) → AvitoBlockedError.""" - scraper = AvitoScraper() - mock_browser = _make_mock_browser(_FIREWALL_HTML) - scraper._browser = mock_browser - scraper._cffi = None # no curl_cffi fallback available (pipeline shared-browser path) - - with patch("app.services.scrapers.avito.settings") as mock_settings: - mock_settings.avito_proxy_rotate_url = None # no rotation configured - mock_settings.avito_proxy_max_rotations = 2 - - with pytest.raises(AvitoBlockedError, match="browser-mode"): - await scraper._fetch_serp_html("https://www.avito.ru/test", page=1) - - -# --------------------------------------------------------------------------- -# 3b. #1805 — curl_cffi fallback when browser returns firewall -# --------------------------------------------------------------------------- - - -@pytest.mark.asyncio -async def test_browser_firewall_falls_back_to_cffi_returns_html() -> None: - """#1805: browser firewall + curl_cffi fallback armed → curl returns clean HTML.""" - scraper = AvitoScraper() - scraper._browser = _make_mock_browser(_FIREWALL_HTML) - - # curl_cffi fallback: HTTP 200 with clean (non-firewall) HTML. - mock_resp = MagicMock() - mock_resp.status_code = 200 - mock_resp.text = _NORMAL_HTML - mock_cffi = MagicMock() - mock_cffi.get = AsyncMock(return_value=mock_resp) - scraper._cffi = mock_cffi - - with patch("app.services.scrapers.avito.settings") as mock_settings: - mock_settings.avito_proxy_rotate_url = None - mock_settings.avito_proxy_max_rotations = 0 - mock_settings.scraper_proxy_url = None - - result = await scraper._fetch_serp_html("https://www.avito.ru/test", page=1) - - assert result == _NORMAL_HTML - mock_cffi.get.assert_awaited_once_with("https://www.avito.ru/test") - - -@pytest.mark.asyncio -async def test_browser_firewall_cffi_fallback_also_firewall_raises() -> None: - """#1805: browser firewall AND curl_cffi fallback also firewall → AvitoBlockedError.""" - scraper = AvitoScraper() - scraper._browser = _make_mock_browser(_FIREWALL_HTML) - - # curl_cffi fallback: HTTP 200 but ALSO firewall HTML → no escape. - mock_resp = MagicMock() - mock_resp.status_code = 200 - mock_resp.text = _FIREWALL_HTML - mock_cffi = MagicMock() - mock_cffi.get = AsyncMock(return_value=mock_resp) - scraper._cffi = mock_cffi - - with patch("app.services.scrapers.avito.settings") as mock_settings: - mock_settings.avito_proxy_rotate_url = None - mock_settings.avito_proxy_max_rotations = 0 - mock_settings.scraper_proxy_url = None # no backconnect reconnect retries - - # curl fallback raises its own AvitoBlockedError (firewall на HTTP 200); - # browser branch re-raises it as a hard-block. - with pytest.raises(AvitoBlockedError): - await scraper._fetch_serp_html("https://www.avito.ru/test", page=1) - - -@pytest.mark.asyncio -async def test_browser_mode_firewall_rotates_and_retries() -> None: - """Browser returns firewall on first call, normal HTML after rotation → returns HTML.""" - scraper = AvitoScraper() - - # First call → firewall; second call → normal HTML - mock_browser = MagicMock() - mock_browser.fetch = AsyncMock(side_effect=[_FIREWALL_HTML, _NORMAL_HTML]) - scraper._browser = mock_browser - - with patch("app.services.scrapers.avito.settings") as mock_settings: - mock_settings.avito_proxy_rotate_url = "http://proxy.example.com/rotate" - mock_settings.avito_proxy_max_rotations = 1 - - with patch.object(scraper, "_rotate_ip", new=AsyncMock(return_value=True)): - result = await scraper._fetch_serp_html("https://www.avito.ru/test", page=2) - - assert result == _NORMAL_HTML - assert mock_browser.fetch.await_count == 2 - - -# --------------------------------------------------------------------------- -# 4. pagination terminates via empty-parse -# --------------------------------------------------------------------------- - - -@pytest.mark.asyncio -async def test_fetch_city_wide_browser_mode_stops_on_empty_parse() -> None: - """fetch_city_wide with browser mode: page=1 returns lots, page=2 returns [] → stops. - - Verifies the documented invariant: callers break on `not lots` regardless of - fetch mode. _fetch_serp_html is mocked at method level (browser vs curl is - already covered by unit tests above); here we confirm the caller loop behaviour. - """ - scraper = AvitoScraper() - - html_p1 = "page1" - html_p2 = "page2" - - lot = ScrapedLot( - source="avito", - source_url="https://www.avito.ru/ekaterinburg/kvartiry/test", - source_id="LOT_1", - price_rub=5_000_000, - ) - - call_count = 0 - - async def mock_fetch_html(url: str, page: int) -> str: - return html_p1 if page == 1 else html_p2 - - def mock_parse(html: str, source_url_base: str) -> list[ScrapedLot]: - nonlocal call_count - call_count += 1 - return [lot] if "page1" in html else [] - - with patch.object(scraper, "_fetch_serp_html", side_effect=mock_fetch_html): - with patch.object(scraper, "_parse_html", side_effect=mock_parse): - with patch.object(scraper, "sleep_between_requests", new=AsyncMock()): - result = await scraper.fetch_city_wide(pages=10, delay_override_sec=0) - - assert len(result) == 1 - assert result[0].source_id == "LOT_1" - # page1 parsed (returned lot) + page2 parsed (returned []) → stopped; page3-10 never called - assert call_count == 2 - - -# --------------------------------------------------------------------------- -# 5. rotation returns False → immediate AvitoBlockedError -# --------------------------------------------------------------------------- - - -@pytest.mark.asyncio -async def test_browser_mode_rotation_returns_false_raises_blocked() -> None: - """rotate_url set but _rotate_ip() returns False → must immediately raise AvitoBlockedError. - - Covers the branch: firewall detected + rotate_url configured + _rotate_ip() == False - (e.g. rotation API timeout/failure) → we cannot recover → blocked. - """ - scraper = AvitoScraper() - scraper._browser = _make_mock_browser(_FIREWALL_HTML) - - with patch("app.services.scrapers.avito.settings") as mock_settings: - mock_settings.avito_proxy_rotate_url = "http://proxy/rotate" - mock_settings.avito_proxy_max_rotations = 2 - with patch.object(scraper, "_rotate_ip", new=AsyncMock(return_value=False)): - with pytest.raises(AvitoBlockedError, match="browser-mode"): - await scraper._fetch_serp_html("https://avito.ru/test", page=1) - - -# --------------------------------------------------------------------------- -# 6. #356 — sidecar HTTPStatusError (500/503) → graceful AvitoRateLimitedError -# --------------------------------------------------------------------------- - - -@pytest.mark.asyncio -async def test_sidecar_proxy_too_many_requests_rotates_then_succeeds() -> None: - """#356: sidecar 500 NS_ERROR_PROXY_TOO_MANY_REQUESTS, rotation available → - after IP rotation a retry succeeds and the clean HTML is returned.""" - scraper = AvitoScraper() - - mock_browser = MagicMock() - # First fetch raises the soft-ban 500; after rotation the second fetch is clean. - mock_browser.fetch = AsyncMock( - side_effect=[ - _sidecar_error(500, "Error: Page.goto: NS_ERROR_PROXY_TOO_MANY_REQUESTS"), - _NORMAL_HTML, - ] - ) - scraper._browser = mock_browser - mock_rotate = AsyncMock(return_value=True) - - with patch("app.services.scrapers.avito.settings") as mock_settings: - mock_settings.avito_proxy_rotate_url = "http://proxy/rotate" - mock_settings.avito_proxy_max_rotations = 2 - with patch.object(scraper, "_rotate_ip", new=mock_rotate): - result = await scraper._fetch_serp_html("https://www.avito.ru/test", page=3) - - assert result == _NORMAL_HTML - mock_rotate.assert_awaited_once() - assert mock_browser.fetch.await_count == 2 - - -@pytest.mark.asyncio -async def test_sidecar_proxy_too_many_requests_exhausted_raises_rate_limited() -> None: - """#356: sidecar 500 proxy-too-many with rotation budget exhausted → - AvitoRateLimitedError (NOT a bare HTTPStatusError, NOT AvitoBlockedError).""" - scraper = AvitoScraper() - - mock_browser = MagicMock() - mock_browser.fetch = AsyncMock( - side_effect=_sidecar_error(500, "Error: Page.goto: NS_ERROR_PROXY_TOO_MANY_REQUESTS") - ) - scraper._browser = mock_browser - mock_rotate = AsyncMock(return_value=True) - - with patch("app.services.scrapers.avito.settings") as mock_settings: - mock_settings.avito_proxy_rotate_url = "http://proxy/rotate" - mock_settings.avito_proxy_max_rotations = 2 - with patch.object(scraper, "_rotate_ip", new=mock_rotate): - with pytest.raises(AvitoRateLimitedError, match="browser-sidecar"): - await scraper._fetch_serp_html("https://www.avito.ru/test", page=3) - - # 2 rotations consumed + 1 final fetch that exhausts the budget. - assert mock_rotate.await_count == 2 - assert mock_browser.fetch.await_count == 3 - - -@pytest.mark.asyncio -async def test_sidecar_transient_timeout_exhausts_budget_no_infinite_loop() -> None: - """#356: sidecar 500 transient (Timeout) repeated past the transient budget with - NO rotations → AvitoRateLimitedError after bounded retries (no infinite loop).""" - scraper = AvitoScraper() - - mock_browser = MagicMock() - mock_browser.fetch = AsyncMock( - side_effect=_sidecar_error(500, "TimeoutError: Page.goto: Timeout 60000ms exceeded") - ) - scraper._browser = mock_browser - mock_rotate = AsyncMock(return_value=True) - - with patch("app.services.scrapers.avito.settings") as mock_settings: - mock_settings.avito_proxy_rotate_url = None # no rotation → transient budget only - mock_settings.avito_proxy_max_rotations = 0 - # Patch sleep so the bounded backoff doesn't slow the test. - with patch("app.services.scrapers.avito.asyncio.sleep", new=AsyncMock()): - with patch.object(scraper, "_rotate_ip", new=mock_rotate): - with pytest.raises(AvitoRateLimitedError, match="browser-sidecar"): - await scraper._fetch_serp_html("https://www.avito.ru/test", page=5) - - # Transient is NOT a soft-ban → never rotates. - mock_rotate.assert_not_awaited() - # Bounded: _AVITO_SIDECAR_TRANSIENT_RETRIES (2) retries + final attempt = 3. - assert mock_browser.fetch.await_count == 3 - - -@pytest.mark.asyncio -async def test_sidecar_503_browser_unavailable_exhausted_raises_rate_limited() -> None: - """#356: sidecar 503 'browser unavailable (proxy may be down)' with rotation - exhausted → AvitoRateLimitedError (soft-ban classification via status==503).""" - scraper = AvitoScraper() - - mock_browser = MagicMock() - mock_browser.fetch = AsyncMock( - side_effect=_sidecar_error(503, "browser unavailable (proxy may be down)") - ) - scraper._browser = mock_browser - mock_rotate = AsyncMock(return_value=True) - - with patch("app.services.scrapers.avito.settings") as mock_settings: - mock_settings.avito_proxy_rotate_url = "http://proxy/rotate" - mock_settings.avito_proxy_max_rotations = 1 - with patch.object(scraper, "_rotate_ip", new=mock_rotate): - with pytest.raises(AvitoRateLimitedError, match="browser-sidecar"): - await scraper._fetch_serp_html("https://www.avito.ru/test", page=7) - - # 1 rotation consumed then exhausted. - assert mock_rotate.await_count == 1 - assert mock_browser.fetch.await_count == 2 - - -@pytest.mark.asyncio -async def test_sidecar_transient_then_soft_ban_independent_budgets() -> None: - """#356: смешанная последовательность transient → soft-ban пинит, что бюджеты - независимы и корректно чередуются в одном while-цикле. - - attempt 1: transient Timeout → тратит transient_left (1 из 2), backoff, continue. - attempt 2: too_many_requests → soft-ban, тратит ротацию (1 из 1), continue. - attempt 3: too_many_requests, ротация исчерпана → AvitoRateLimitedError. - """ - scraper = AvitoScraper() - - mock_browser = MagicMock() - mock_browser.fetch = AsyncMock( - side_effect=[ - _sidecar_error(500, "TimeoutError: Page.goto: Timeout 60000ms exceeded"), - _sidecar_error(500, "Error: Page.goto: NS_ERROR_PROXY_TOO_MANY_REQUESTS"), - _sidecar_error(500, "Error: Page.goto: NS_ERROR_PROXY_TOO_MANY_REQUESTS"), - ] - ) - scraper._browser = mock_browser - mock_rotate = AsyncMock(return_value=True) - - with patch("app.services.scrapers.avito.settings") as mock_settings: - mock_settings.avito_proxy_rotate_url = "http://proxy/rotate" - mock_settings.avito_proxy_max_rotations = 1 - with patch("app.services.scrapers.avito.asyncio.sleep", new=AsyncMock()): - with patch.object(scraper, "_rotate_ip", new=mock_rotate): - with pytest.raises(AvitoRateLimitedError, match="browser-sidecar"): - await scraper._fetch_serp_html("https://www.avito.ru/test", page=9) - - # 1 transient retry + 1 rotation, оба бюджета тронуты до исчерпания. - assert mock_rotate.await_count == 1 - assert mock_browser.fetch.await_count == 3 diff --git a/tradein-mvp/backend/tests/test_avito_citywide.py b/tradein-mvp/backend/tests/test_avito_citywide.py deleted file mode 100644 index 4c58df39..00000000 --- a/tradein-mvp/backend/tests/test_avito_citywide.py +++ /dev/null @@ -1,288 +0,0 @@ -"""T6 — avito citywide + byrooms sweep mode. - -Тесты: -1. _build_citywide_url — без geoCoords/radius, sort=104, page param. -2. _build_rooms_url — room-slug в пути, sort=104, page param. -3. ROOM_SLUGS — 7 категорий, структура кортежей. -4. fetch_city_wide break-on-empty — останавливается когда _parse_html вернул []. -5. fetch_city_wide dedup — дублирующийся source_id не дублируется в результате. -6. fetch_by_rooms break-on-empty per-category — каждая категория прерывается. -7. fetch_by_rooms global dedup — cross-category дедуп по source_id. -8. fetch_around не затронут — сигнатура и URL-режим не изменились. - -Без сети, без БД, без curl_cffi. -""" - -from __future__ import annotations - -import os -from unittest.mock import AsyncMock, patch - -import pytest - -os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db") - -from app.services.scrapers.avito import ROOM_SLUGS, AvitoScraper -from app.services.scrapers.base import ScrapedLot - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - - -def _make_lot(source_id: str) -> ScrapedLot: - return ScrapedLot( - source="avito", - source_url=f"https://www.avito.ru/ekaterinburg/kvartiry/{source_id}", - source_id=source_id, - price_rub=5_000_000, - ) - - -# --------------------------------------------------------------------------- -# 1. _build_citywide_url — без geo, s=104 -# --------------------------------------------------------------------------- - - -def test_build_citywide_url_no_geocoords() -> None: - s = AvitoScraper() - url = s._build_citywide_url(page=1) - assert "geoCoords" not in url - assert "radius" not in url - assert "s=104" in url - assert "p=1" in url - assert "ekaterinburg/kvartiry/prodam-ASgBAgICAUSSA8YQ" in url - - -def test_build_citywide_url_page_param() -> None: - s = AvitoScraper() - assert "p=3" in s._build_citywide_url(page=3) - assert "p=10" in s._build_citywide_url(page=10) - - -def test_build_citywide_url_default_page() -> None: - s = AvitoScraper() - assert "p=1" in s._build_citywide_url() - - -# --------------------------------------------------------------------------- -# 2. _build_rooms_url — slug в пути, без geo -# --------------------------------------------------------------------------- - - -def test_build_rooms_url_contains_slug() -> None: - s = AvitoScraper() - slug = "1-komnatnye-ASgBAgICAkSSA8YQygiAWQ" - url = s._build_rooms_url(slug, page=1) - assert slug in url - assert "geoCoords" not in url - assert "radius" not in url - assert "s=104" in url - assert "p=1" in url - assert "ekaterinburg/kvartiry/prodam/" in url - - -def test_build_rooms_url_page_param() -> None: - s = AvitoScraper() - slug = "studii-ASgBAgICAkSSA8YQygj~WA" - assert "p=5" in s._build_rooms_url(slug, page=5) - - -# --------------------------------------------------------------------------- -# 3. ROOM_SLUGS structure -# --------------------------------------------------------------------------- - - -def test_room_slugs_count() -> None: - assert len(ROOM_SLUGS) == 7 - - -def test_room_slugs_are_pairs() -> None: - for name, slug in ROOM_SLUGS: - assert isinstance(name, str) and name - assert isinstance(slug, str) and slug - # Каждый slug содержит ASgB-кодировку - assert "ASgB" in slug - - -def test_room_slugs_names_unique() -> None: - names = [name for name, _ in ROOM_SLUGS] - assert len(names) == len(set(names)) - - -# --------------------------------------------------------------------------- -# 4. fetch_city_wide — break-on-empty -# --------------------------------------------------------------------------- - - -@pytest.mark.asyncio -async def test_fetch_city_wide_break_on_empty() -> None: - """Если _parse_html вернул [] — цикл прерывается, не идёт на следующие страницы.""" - s = AvitoScraper() - # page=1 возвращает 2 лота, page=2 возвращает [] → стоп - page_lots: dict[int, list[ScrapedLot]] = { - 1: [_make_lot("AAA"), _make_lot("BBB")], - 2: [], - } - call_count = 0 - - async def mock_fetch_html(url: str, page: int) -> str: - return f"page{page}" - - def mock_parse(html: str, source_url_base: str) -> list[ScrapedLot]: - nonlocal call_count - call_count += 1 - # Derive page number from html string - for p, lots in page_lots.items(): - if f"page{p}" in html: - return lots - return [] - - with patch.object(s, "_fetch_serp_html", side_effect=mock_fetch_html): - with patch.object(s, "_parse_html", side_effect=mock_parse): - with patch.object(s, "sleep_between_requests", new=AsyncMock()): - result = await s.fetch_city_wide(pages=10, delay_override_sec=0) - - assert len(result) == 2 - assert call_count == 2 # page1 + page2(empty) → stop; page3..10 не вызваны - - -# --------------------------------------------------------------------------- -# 5. fetch_city_wide — dedup по source_id -# --------------------------------------------------------------------------- - - -@pytest.mark.asyncio -async def test_fetch_city_wide_dedup_by_source_id() -> None: - """Лот с повторяющимся source_id добавляется только один раз.""" - s = AvitoScraper() - dup_lot = _make_lot("SAME_ID") - # pages 1 и 2 возвращают один и тот же lot - pages_data: list[list[ScrapedLot]] = [[dup_lot], [dup_lot], []] - - async def mock_html(url: str, page: int) -> str: - return f"{page}" - - call_n = 0 - - def mock_parse(html: str, source_url_base: str) -> list[ScrapedLot]: - nonlocal call_n - idx = call_n - call_n += 1 - return pages_data[idx] if idx < len(pages_data) else [] - - with patch.object(s, "_fetch_serp_html", side_effect=mock_html): - with patch.object(s, "_parse_html", side_effect=mock_parse): - with patch.object(s, "sleep_between_requests", new=AsyncMock()): - result = await s.fetch_city_wide(pages=10, delay_override_sec=0) - - assert len(result) == 1 - assert result[0].source_id == "SAME_ID" - - -# --------------------------------------------------------------------------- -# 6. fetch_by_rooms — break-on-empty per category -# --------------------------------------------------------------------------- - - -@pytest.mark.asyncio -async def test_fetch_by_rooms_break_on_empty_per_category() -> None: - """Каждая категория прерывается на пустой странице, следующая категория продолжается.""" - s = AvitoScraper() - slugs = [("1-комн.", "slug-1k"), ("2-комн.", "slug-2k")] - - # category 1: page1=2 lots, page2=[] → стоп на этой категории - # category 2: page1=3 lots, page2=[] → стоп - parse_responses: list[list[ScrapedLot]] = [ - [_make_lot("C1P1a"), _make_lot("C1P1b")], # cat1 p1 - [], # cat1 p2 → break - [_make_lot("C2P1a"), _make_lot("C2P1b"), _make_lot("C2P1c")], # cat2 p1 - [], # cat2 p2 → break - ] - parse_call = 0 - - async def mock_html(url: str, page: int) -> str: - return f"{url}{page}" - - def mock_parse(html: str, source_url_base: str) -> list[ScrapedLot]: - nonlocal parse_call - idx = parse_call - parse_call += 1 - return parse_responses[idx] if idx < len(parse_responses) else [] - - with patch.object(s, "_fetch_serp_html", side_effect=mock_html): - with patch.object(s, "_parse_html", side_effect=mock_parse): - with patch.object(s, "sleep_between_requests", new=AsyncMock()): - result = await s.fetch_by_rooms(pages=10, delay_override_sec=0, room_slugs=slugs) - - assert len(result) == 5 # 2 + 3 (нет дублей) - assert parse_call == 4 # 2 pages per category × 2 categories - - -# --------------------------------------------------------------------------- -# 7. fetch_by_rooms — global dedup cross-category -# --------------------------------------------------------------------------- - - -@pytest.mark.asyncio -async def test_fetch_by_rooms_global_dedup() -> None: - """Лот присутствующий в двух категориях добавляется только один раз.""" - s = AvitoScraper() - slugs = [("A", "slug-a"), ("B", "slug-b")] - - shared = _make_lot("SHARED") - unique_a = _make_lot("UNIQUE_A") - unique_b = _make_lot("UNIQUE_B") - - parse_data: list[list[ScrapedLot]] = [ - [shared, unique_a], # cat A p1 - [], # cat A p2 → break - [shared, unique_b], # cat B p1 (shared — дубль) - [], # cat B p2 → break - ] - call_idx = 0 - - async def mock_html(url: str, page: int) -> str: - return "" - - def mock_parse(html: str, source_url_base: str) -> list[ScrapedLot]: - nonlocal call_idx - i = call_idx - call_idx += 1 - return parse_data[i] if i < len(parse_data) else [] - - with patch.object(s, "_fetch_serp_html", side_effect=mock_html): - with patch.object(s, "_parse_html", side_effect=mock_parse): - with patch.object(s, "sleep_between_requests", new=AsyncMock()): - result = await s.fetch_by_rooms(pages=10, delay_override_sec=0, room_slugs=slugs) - - ids = [lot.source_id for lot in result] - assert ids.count("SHARED") == 1 - assert "UNIQUE_A" in ids - assert "UNIQUE_B" in ids - assert len(result) == 3 - - -# --------------------------------------------------------------------------- -# 8. fetch_around не затронут — URL содержит geoCoords -# --------------------------------------------------------------------------- - - -def test_build_web_url_still_has_geocoords() -> None: - """Существующий _build_web_url сохраняет geoCoords+radius — geo-режим не сломан.""" - s = AvitoScraper() - url = s._build_web_url(56.838, 60.605, radius_km=2, page=1) - assert "geoCoords=56.838%2C60.605" in url or "geoCoords=56.838,60.605" in url - assert "radius=2" in url - assert "s=104" in url - assert "p=1" in url - - -def test_citywide_url_differs_from_geo_url() -> None: - """citywide URL не содержит geoCoords — это принципиально разные URL.""" - s = AvitoScraper() - geo_url = s._build_web_url(56.838, 60.605, radius_km=1, page=1) - cw_url = s._build_citywide_url(page=1) - assert geo_url != cw_url - assert "geoCoords" not in cw_url - assert "geoCoords" in geo_url diff --git a/tradein-mvp/backend/tests/test_avito_detail_parse.py b/tradein-mvp/backend/tests/test_avito_detail_parse.py deleted file mode 100644 index 1e4ca0ca..00000000 --- a/tradein-mvp/backend/tests/test_avito_detail_parse.py +++ /dev/null @@ -1,400 +0,0 @@ -"""Smoke тесты для avito_detail parser. - -Загружает minimal HTML fixture и проверяет selector + conversion logic -без сетевых запросов. -""" - -# ruff: noqa: E501 # HTML-фикстуры содержат длинные строки разметки - -import pytest - -from app.services.scrapers.avito_detail import ( - DetailEnrichment, - _parse_height_m, - parse_detail_html, -) - -MINIMAL_HTML = """ - -
№ 7986882804 · 18 мая в 13:40 · 1899 просмотров (+ 12 сегодня)
-11 990 000 ₽ -
-
-
    -
  • Количество комнат: 3
    • -
  • Общая площадь: 75 м²
    • -
  • Площадь кухни: 15 м²
    • -
  • Этаж: 20 из 26
    • -
  • Балкон или лоджия: лоджия
    • -
  • Ремонт: евро
    • -
  • Способ продажи: свободная
    • -
  • Условия продажи: возможна ипотека
    • -
-
    -
  • Тип дома: монолитный
    • -
  • Этажей в доме: 26
    • -
-
-Дом -
-

1 собственник

-

Последняя смена собственника 5 мая 2020

-

Не найдены ограничения и обременения

-

Совпадают площадь, адрес и этаж

-
-
Отличная квартира в новом ЖК. Чкаловская 11-15 мин пешком.
- -""" - -SOURCE_URL = "https://www.avito.ru/ekaterinburg/kvartiry/test_7986882804" - - -def test_minimal_parse() -> None: - result = parse_detail_html(MINIMAL_HTML, SOURCE_URL) - assert isinstance(result, DetailEnrichment) - assert result.item_id == "7986882804" - assert result.price_rub == 11990000 - assert result.rooms == 3 - assert result.area_m2 == 75.0 - assert result.kitchen_area_m2 == 15.0 - assert result.floor == 20 - assert result.total_floors == 26 - assert result.balcony_loggia == "loggia" - assert result.repair_state == "good" # «евро» → good (normalized enum, #621) - assert result.sale_type == "free" - assert result.mortgage_available is True - assert result.house_type == "monolith" - assert result.total_floors_house == 26 - assert result.lat == 56.797713 - assert result.lon == 60.609135 - assert result.avito_location_id == 654070 - assert result.house_catalog_url is not None and "3171365" in result.house_catalog_url - assert result.owners_count == 1 - assert result.owners_at_least is False - assert result.last_owner_change_date is not None and result.last_owner_change_date.year == 2020 - assert result.encumbrances_clean is True - assert result.registry_match is True - assert result.views_total == 1899 - assert result.views_today == 12 - - -def test_sale_type_assignment_pereustupka() -> None: - """«Способ продажи: переуступка» → sale_type='assignment' (ДДУ переуступка, - сигнал новостройки). До фикса map не содержал ключ → None.""" - html = """ - -
№ 8043003287
- 9 500 000 ₽ -
-
    -
  • Количество комнат: 2
    • -
  • Способ продажи: переуступка
    • -
-
- - """ - result = parse_detail_html(html, "https://www.avito.ru/test_8043003287") - assert result.sale_type == "assignment" - - -def _html_with_coords(item_id: str, lat: str, lon: str) -> str: - return f""" - -
№ {item_id}
- 9 500 000 ₽ -
- - """ - - -def test_coord_guard_keeps_ekb_coords() -> None: - """ЕКБ-координаты (внутри wide-bbox) пишутся как есть → попадут в UPDATE (#1871).""" - html = _html_with_coords("8043000001", "56.797713", "60.609135") - result = parse_detail_html(html, "https://www.avito.ru/test_8043000001") - assert result.lat == 56.797713 - assert result.lon == 60.609135 - assert result.avito_location_id == 654070 - - -def test_coord_guard_drops_piter_coords() -> None: - """Питерские координаты (вне wide-bbox) → lat/lon сбрасываются в None (#1871). - - Это значение уходит в save_detail_enrichment UPDATE COALESCE(:lat, lat) → NULL, - то есть существующее значение в БД не перетирается мусором, листинг уходит - в geocode-cron путь. - """ - html = _html_with_coords("8043000002", "59.9386", "30.3141") # СПб - result = parse_detail_html(html, "https://www.avito.ru/test_8043000002") - assert result.lat is None - assert result.lon is None - # avito_location_id НЕ трогаем — фикс только на координаты - assert result.avito_location_id == 654070 - - -def test_coord_guard_no_map_element() -> None: - """Без map-элемента координат нет — guard не падает, lat/lon=None.""" - html = """ - -
№ 8043000003
- 9 500 000 ₽ - - """ - result = parse_detail_html(html, "https://www.avito.ru/test_8043000003") - assert result.lat is None - assert result.lon is None - - -def test_metro_extraction() -> None: - result = parse_detail_html(MINIMAL_HTML, SOURCE_URL) - # "Чкаловская 11-15 мин пешком" → metro_stations - assert len(result.metro_stations) >= 1 - station = result.metro_stations[0] - assert station["name"] == "Чкаловская" - assert station["min_from"] == 11 - assert station["min_to"] == 15 - assert station["mode"] == "walk" - - -def test_owners_at_least_false() -> None: - """Один собственник (без 'или больше') → owners_at_least=False.""" - result = parse_detail_html(MINIMAL_HTML, SOURCE_URL) - assert result.owners_count == 1 - assert result.owners_at_least is False - - -def test_owners_at_least_true() -> None: - """'3 собственника или больше' → owners_at_least=True.""" - html = """ - -
№ 1234567890
- 5 000 000 ₽ -
-

3 собственника или больше

-
- - """ - result = parse_detail_html(html, "https://www.avito.ru/test_1234567890") - assert result.owners_count == 3 - assert result.owners_at_least is True - - -def test_missing_domoteka_all_none() -> None: - """Если нет domoteka-блока — все 5 полей None.""" - html = """ - -
№ 9999999999
- 3 000 000 ₽ - - """ - result = parse_detail_html(html, "https://www.avito.ru/test_9999999999") - assert result.owners_count is None - assert result.owners_at_least is None - assert result.last_owner_change_date is None - assert result.encumbrances_clean is None - assert result.registry_match is None - - -def test_invalid_html_raises_value_error() -> None: - """HTML без item_id → ValueError.""" - html = "

Нет айди

" - with pytest.raises(ValueError, match="Cannot extract item_id"): - parse_detail_html(html, "https://www.avito.ru/test") - - -def test_house_catalog_url_full() -> None: - """house_catalog_url должен быть absolute URL.""" - result = parse_detail_html(MINIMAL_HTML, SOURCE_URL) - assert result.house_catalog_url is not None - assert result.house_catalog_url.startswith("https://www.avito.ru") - - -def test_house_catalog_url_secondary_marker_no_nd_jk() -> None: - """#871: вторичка без nd-jk-details-button → fallback на /catalog/houses/ якорь; - ?context-трекинг отброшен (verified live: house-link есть в detail SSR).""" - html = """ - -
№ 7683141312
- 4 000 000 ₽ - - Узнать больше о доме - - """ - result = parse_detail_html(html, "https://www.avito.ru/test_7683141312") - assert ( - result.house_catalog_url - == "https://www.avito.ru/catalog/houses/ekaterinburg/ul_gotvalda_6k1/142792" - ) - - -def test_bathroom_type_combined() -> None: - html = """ - -
№ 1111111111
- 4 000 000 ₽ -
-
    -
  • Санузел: совмещенный
    • -
  • Окна: во двор и на улицу
    • -
  • Тип комнат: изолированные
    • -
-
- - """ - result = parse_detail_html(html, "https://www.avito.ru/test_1111111111") - assert result.bathroom_type == "combined" - assert result.windows_view == "both" - assert result.room_layout == "isolated" - - -def test_photo_urls_extracted() -> None: - html = """ - -
№ 2222222222
- 7 000 000 ₽ -
-
- - """ - result = parse_detail_html(html, "https://www.avito.ru/test_2222222222") - assert len(result.photo_urls) == 2 - assert result.photo_urls[0] == "https://img.avito.ru/img1.jpg" - - -def test_source_url_preserved() -> None: - result = parse_detail_html(MINIMAL_HTML, SOURCE_URL) - assert result.source_url == SOURCE_URL - - -# ── repair_state fallback из описания (#622) ────────────────────────────────── - - -def test_repair_state_structured_wins_over_description() -> None: - """Структурное «Ремонт: евро» (good) в приоритете даже при сигнале в тексте.""" - html = """ - -
№ 3333333333
- 6 000 000 ₽ -
-
  • Ремонт: евро
-
-
Квартира без отделки, под чистовую
- - """ - result = parse_detail_html(html, "https://www.avito.ru/test_3333333333") - assert result.repair_state == "good" # структурное поле, не needs_repair из текста - - -def test_repair_state_inferred_from_description_when_structured_absent() -> None: - """Нет блока «Ремонт» → инференс из описания (#622).""" - html = """ - -
№ 4444444444
- 8 000 000 ₽ -
-
  • Количество комнат: 2
-
-
Сделан свежий евроремонт, заезжай и живи
- - """ - result = parse_detail_html(html, "https://www.avito.ru/test_4444444444") - assert result.repair_state == "good" - - -def test_repair_state_none_when_no_signal() -> None: - """Нет структурного поля и нет сигнала в тексте → None (не фабрикуем).""" - html = """ - -
№ 5555555555
- 9 000 000 ₽ -
Светлая квартира рядом с метро
- - """ - result = parse_detail_html(html, "https://www.avito.ru/test_5555555555") - assert result.repair_state is None - - -# ── Новые поля: living_area_m2, ceiling_height_m, finishing ────────────────── - - -_NEW_FIELDS_HTML = """ - -
№ 6666666666
-8 500 000 ₽ -
-
    -
  • Количество комнат: 1
    • -
  • Общая площадь: 43.7\xa0м²
    • -
  • Площадь кухни: 17.1\xa0м²
    • -
  • Жилая площадь: 12.6\xa0м²
    • -
  • Высота потолков: 2.7\xa0м
    • -
  • Отделка: чистовая
    • -
  • Этаж: 6 из 16
    • -
-
- -""" - - -def test_new_apartment_fields_populated() -> None: - """living_area_m2 / ceiling_height_m / finishing парсятся из params-блока.""" - result = parse_detail_html(_NEW_FIELDS_HTML, "https://www.avito.ru/test_6666666666") - assert result.living_area_m2 == 12.6 - assert result.ceiling_height_m == 2.7 - assert result.finishing == "чистовая" - - -def test_new_fields_absent_when_not_in_params() -> None: - """Если полей нет в params — None, не дефолт.""" - result = parse_detail_html( - """ - -
№ 7777777777
- 4 000 000 ₽ -
-
  • Количество комнат: 2
-
- - """, - "https://www.avito.ru/test_7777777777", - ) - assert result.living_area_m2 is None - assert result.ceiling_height_m is None - assert result.finishing is None - - -def test_finishing_stored_lowercase_stripped() -> None: - """finishing → lowercase trimmed (' Чистовая ' → 'чистовая').""" - html = """ - -
№ 8888888888
- 5 000 000 ₽ -
-
  • Отделка: Чистовая
-
- - """ - result = parse_detail_html(html, "https://www.avito.ru/test_8888888888") - assert result.finishing == "чистовая" - - -# ── _parse_height_m unit tests ──────────────────────────────────────────────── - - -def test_parse_height_m_dot() -> None: - assert _parse_height_m("2.7\xa0м") == 2.7 - - -def test_parse_height_m_comma() -> None: - assert _parse_height_m("2,7\xa0м") == 2.7 - - -def test_parse_height_m_integer() -> None: - assert _parse_height_m("3 м") == 3.0 - - -def test_parse_height_m_junk() -> None: - assert _parse_height_m("нет данных") is None - - -def test_parse_height_m_empty() -> None: - assert _parse_height_m("") is None diff --git a/tradein-mvp/backend/tests/test_avito_ekb_geo_filter.py b/tradein-mvp/backend/tests/test_avito_ekb_geo_filter.py deleted file mode 100644 index 6096803b..00000000 --- a/tradein-mvp/backend/tests/test_avito_ekb_geo_filter.py +++ /dev/null @@ -1,154 +0,0 @@ -"""Тест per-card ЕКБ гео-фильтра в AvitoScraper._parse_html. - -Avito при редких комбо (4+ комн., дорогой сегмент) добивает выдачу карточками -«по всей России» (Москва, Уфа, Челябинск и т.д.). Каждая карточка несёт -СВОЙ href с city-slug (/ekaterinburg/, /moskva/, ...). Фильтр (за флагом -settings.avito_serp_ekb_only) отбрасывает non-EKB карточки. - -Тесты: -1. EKB + non-EKB в SERP → только ЕКБ остаётся при ekb_only=True. -2. ekb_only=False → все карточки проходят (старое поведение). -3. Карточка без распознанного city-slug → пропускается консервативно (не дропается). -4. Только EKB карточки → ничего не дропается (нет лишних логов). -""" - -from __future__ import annotations - -import os - -import pytest - -os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db") - - -def _make_card_html( - item_id: str, - city_slug: str, - price: int = 5_000_000, - title: str = "2-к. квартира, 50 м², 3/9 эт.", -) -> str: - """Минимальная Avito SERP-карточка с указанным city-slug в href.""" - href = f"/{city_slug}/kvartiry/{item_id}-12345" - return f""" -
- {title} - -

ул. Тестовая, 1

-
- """ - - -def _build_serp_html(*cards: str) -> str: - """Обернуть карточки в минимальный SERP-HTML.""" - inner = "".join(cards) - return f"{inner}" - - -# --------------------------------------------------------------------------- -# 1. EKB + non-EKB → только ЕКБ при ekb_only=True -# --------------------------------------------------------------------------- - - -def test_geo_filter_drops_non_ekb_cards(monkeypatch: pytest.MonkeyPatch) -> None: - """При avito_serp_ekb_only=True карточки других городов отсеиваются.""" - from app.core.config import settings - from app.services.scrapers.avito import AvitoScraper - - monkeypatch.setattr(settings, "avito_serp_ekb_only", True) - - ekb_card = _make_card_html("EKB1", "ekaterinburg") - msk_card = _make_card_html("MSK1", "moskva") - ufa_card = _make_card_html("UFA1", "ufa") - chelya_card = _make_card_html("CHE1", "chelyabinsk") - - html = _build_serp_html(ekb_card, msk_card, ufa_card, chelya_card) - - scraper = AvitoScraper() - lots = scraper._parse_html(html, source_url_base="https://www.avito.ru/ekaterinburg/kvartiry/") - - source_ids = {lot.source_id for lot in lots} - assert "EKB1" in source_ids, "ЕКБ-карточка должна пройти фильтр" - assert "MSK1" not in source_ids, "Московская карточка должна быть отброшена" - assert "UFA1" not in source_ids, "Уфимская карточка должна быть отброшена" - assert "CHE1" not in source_ids, "Челябинская карточка должна быть отброшена" - assert len(lots) == 1 - - -# --------------------------------------------------------------------------- -# 2. ekb_only=False → все карточки проходят -# --------------------------------------------------------------------------- - - -def test_geo_filter_disabled_passes_all(monkeypatch: pytest.MonkeyPatch) -> None: - """При avito_serp_ekb_only=False все карточки проходят (старое поведение).""" - from app.core.config import settings - from app.services.scrapers.avito import AvitoScraper - - monkeypatch.setattr(settings, "avito_serp_ekb_only", False) - - ekb_card = _make_card_html("EKB1", "ekaterinburg") - msk_card = _make_card_html("MSK1", "moskva") - - html = _build_serp_html(ekb_card, msk_card) - - scraper = AvitoScraper() - lots = scraper._parse_html(html, source_url_base="https://www.avito.ru/ekaterinburg/kvartiry/") - - source_ids = {lot.source_id for lot in lots} - assert "EKB1" in source_ids - assert "MSK1" in source_ids - assert len(lots) == 2 - - -# --------------------------------------------------------------------------- -# 3. Карточка без city-slug → консервативно пропускается (не дропается) -# --------------------------------------------------------------------------- - - -def test_geo_filter_keeps_card_without_city_slug(monkeypatch: pytest.MonkeyPatch) -> None: - """Карточка с href без city-slug (/kvartiry/...) пропускается консервативно.""" - from app.core.config import settings - from app.services.scrapers.avito import AvitoScraper - - monkeypatch.setattr(settings, "avito_serp_ekb_only", True) - - # href без city-slug: первый сегмент = "kvartiry" → non-city, консервативно берём - no_city_card = """ -
- 2-к. квартира, 50 м², 3/9 эт. - -

ул. Тестовая, 1

-
- """ - msk_card = _make_card_html("MSK1", "moskva") - - html = _build_serp_html(no_city_card, msk_card) - - scraper = AvitoScraper() - lots = scraper._parse_html(html, source_url_base="https://www.avito.ru/ekaterinburg/kvartiry/") - - source_ids = {lot.source_id for lot in lots} - assert "NOCITY" in source_ids, "Карточка без city-slug должна быть сохранена консервативно" - assert "MSK1" not in source_ids, "Московская карточка должна быть отброшена" - - -# --------------------------------------------------------------------------- -# 4. Только EKB карточки → всё проходит -# --------------------------------------------------------------------------- - - -def test_geo_filter_all_ekb_no_drops(monkeypatch: pytest.MonkeyPatch) -> None: - """Если все карточки из ЕКБ — dropped=0, все лоты сохраняются.""" - from app.core.config import settings - from app.services.scrapers.avito import AvitoScraper - - monkeypatch.setattr(settings, "avito_serp_ekb_only", True) - - cards = [_make_card_html(f"EKB{i}", "ekaterinburg") for i in range(3)] - html = _build_serp_html(*cards) - - scraper = AvitoScraper() - lots = scraper._parse_html(html, source_url_base="https://www.avito.ru/ekaterinburg/kvartiry/") - - assert len(lots) == 3 - assert all("EKB" in (lot.source_id or "") for lot in lots) diff --git a/tradein-mvp/backend/tests/test_avito_newbuilding_sweep.py b/tradein-mvp/backend/tests/test_avito_newbuilding_sweep.py index 5aa05d27..f0b393aa 100644 --- a/tradein-mvp/backend/tests/test_avito_newbuilding_sweep.py +++ b/tradein-mvp/backend/tests/test_avito_newbuilding_sweep.py @@ -12,6 +12,11 @@ save_listings + mark_done) удалён вместе с `app.services.scrape_pip Part E1) — эквивалентная regression-coverage теперь в `test_scraper_kit_pipeline_parity2.py::test_avito_newbuilding_sweep` (kit `run_avito_newbuilding_sweep`). + +Легаси `app.services.scrapers.avito`/`base` удалены (#2397 финальный шаг E, 0 +runtime-импортёров) — AvitoScraper/ScrapedLot переведены на kit-эквиваленты +`scraper_kit.providers.avito.serp`/`scraper_kit.base` (byte-identical +strangler-копия). """ from __future__ import annotations @@ -24,8 +29,10 @@ import pytest os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test") -from app.services.scrapers.avito import NOVOSTROYKA_SLUG, AvitoScraper -from app.services.scrapers.base import ScrapedLot +from scraper_kit.base import ScrapedLot +from scraper_kit.providers.avito.serp import NOVOSTROYKA_SLUG, AvitoScraper + +from app.services.scraper_adapters import RealScraperConfig FIXTURE = Path(__file__).parent / "fixtures" / "avito_serp_novostroyka.html" @@ -44,7 +51,7 @@ def _make_lot(source_id: str) -> ScrapedLot: def test_build_newbuilding_url_contains_slug_and_sort() -> None: - s = AvitoScraper() + s = AvitoScraper(RealScraperConfig()) url = s._build_newbuilding_url(page=1) assert NOVOSTROYKA_SLUG in url assert "novostroyka-" in url @@ -56,14 +63,14 @@ def test_build_newbuilding_url_contains_slug_and_sort() -> None: def test_build_newbuilding_url_page_param() -> None: - s = AvitoScraper() + s = AvitoScraper(RealScraperConfig()) url = s._build_newbuilding_url(page=2) assert "novostroyka-" in url assert "p=2" in url def test_build_newbuilding_url_differs_from_citywide() -> None: - s = AvitoScraper() + s = AvitoScraper(RealScraperConfig()) assert s._build_newbuilding_url(page=1) != s._build_citywide_url(page=1) @@ -72,7 +79,7 @@ def test_build_newbuilding_url_differs_from_citywide() -> None: def test_parse_novostroyka_fixture_segments_and_newbuilding_id() -> None: html = FIXTURE.read_text(encoding="utf-8") - s = AvitoScraper() + s = AvitoScraper(RealScraperConfig()) lots = s._parse_html(html, source_url_base=s._build_newbuilding_url(page=1)) assert len(lots) == 2 @@ -91,7 +98,7 @@ def test_parse_novostroyka_fixture_segments_and_newbuilding_id() -> None: @pytest.mark.asyncio async def test_fetch_newbuildings_break_on_empty() -> None: - s = AvitoScraper() + s = AvitoScraper(RealScraperConfig()) pages_data: list[list[ScrapedLot]] = [[_make_lot("A"), _make_lot("B")], []] call_n = 0 @@ -116,7 +123,7 @@ async def test_fetch_newbuildings_break_on_empty() -> None: @pytest.mark.asyncio async def test_fetch_newbuildings_dedup_by_source_id() -> None: - s = AvitoScraper() + s = AvitoScraper(RealScraperConfig()) dup = _make_lot("SAME") pages_data: list[list[ScrapedLot]] = [[dup], [dup], []] call_n = 0 @@ -141,7 +148,7 @@ async def test_fetch_newbuildings_dedup_by_source_id() -> None: def test_fetch_city_wide_url_unchanged() -> None: """fetch_city_wide остаётся citywide-URL — рефактор не сменил его поведение.""" - s = AvitoScraper() + s = AvitoScraper(RealScraperConfig()) assert "novostroyka-" not in s._build_citywide_url(page=1) assert "prodam-ASgBAgICAUSSA8YQ" in s._build_citywide_url(page=1) diff --git a/tradein-mvp/backend/tests/test_avito_relative_date.py b/tradein-mvp/backend/tests/test_avito_relative_date.py deleted file mode 100644 index 65d80596..00000000 --- a/tradein-mvp/backend/tests/test_avito_relative_date.py +++ /dev/null @@ -1,69 +0,0 @@ -from datetime import date, timedelta - -from app.services.scrapers.avito import _parse_relative_date - - -def test_days_ago(): - assert _parse_relative_date("5 дней назад") == date.today() - timedelta(days=5) - - -def test_week_ago(): - assert _parse_relative_date("2 недели назад") == date.today() - timedelta(days=14) - - -def test_hour_ago_is_today(): - assert _parse_relative_date("3 часа назад") == date.today() - - -def test_none_passthrough(): - assert _parse_relative_date(None) is None - assert _parse_relative_date("") is None - assert _parse_relative_date("random text") is None - - -# Keyword shortcuts — без \d+ (Avito SERP формат для свежих cards) -def test_today_keyword(): - assert _parse_relative_date("сегодня в 12:30") == date.today() - - -def test_yesterday_keyword(): - assert _parse_relative_date("вчера в 18:45") == date.today() - timedelta(days=1) - - -def test_day_before_yesterday(): - assert _parse_relative_date("позавчера") == date.today() - timedelta(days=2) - - -def test_just_now(): - assert _parse_relative_date("только что") == date.today() - - -# Absolute dates — «18 мая», «5 июня в 12:30» -def test_absolute_date_recent(): - today = date.today() - # Take a day-month combo near current (last month) — gradening для CI stability - if today.month >= 2: - target = date(today.year, today.month - 1, 15) - else: - target = date(today.year - 1, 12, 15) - ru_months = { - 1: "января", 2: "февраля", 3: "марта", 4: "апреля", - 5: "мая", 6: "июня", 7: "июля", 8: "августа", - 9: "сентября", 10: "октября", 11: "ноября", 12: "декабря", - } - text = f"{target.day} {ru_months[target.month]} в 12:30" - assert _parse_relative_date(text) == target - - -def test_absolute_date_future_month_rolls_back(): - """Если today январь, а текст «5 декабря» — это прошлый год.""" - today = date.today() - if today.month == 1: - # Today already January, «декабря» → previous year December - result = _parse_relative_date("5 декабря") - assert result == date(today.year - 1, 12, 5) - else: - # Skip when not January — would change semantics - result = _parse_relative_date("5 января") - # Either matches current year January (если уже прошло) или previous - assert result is not None and result.month == 1 and result.day == 5 diff --git a/tradein-mvp/backend/tests/test_avito_serp_date.py b/tradein-mvp/backend/tests/test_avito_serp_date.py deleted file mode 100644 index 71f2b546..00000000 --- a/tradein-mvp/backend/tests/test_avito_serp_date.py +++ /dev/null @@ -1,135 +0,0 @@ -"""#726 — avito SERP listing_date extraction (sortTimeStamp JSON + DOM fallback). - -Offline: парсит фикстуру через AvitoScraper._parse_html, без сети/curl_cffi/БД. -Регрессия на дрейф разметки: дата у lazy-карточек (без DOM item-date) берётся из -per-item JSON sortTimeStamp; у верхних карточек — из DOM data-marker="item-date". -""" - -import os -from datetime import date, datetime, timedelta, timezone -from pathlib import Path - -os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db") - -from app.services.scrapers.avito import ( - AvitoScraper, - _build_sort_timestamp_map, -) - -FIXTURE = Path(__file__).parent / "fixtures" / "avito_serp_sample.html" - -# Из фикстуры (sortTimeStamp, epoch-ms). -_TS_LAZY = 1700000000000 -_TS_TOP = 1700086400000 - -# #726: дата берётся в MSK (UTC+3), чтобы совпадать с тем, что Avito показывает на -# сайте (sortTimeStamp — UTC epoch, но отображается по Москве). -_MSK = timezone(timedelta(hours=3)) - - -def _expected(ms: int) -> date: - return datetime.fromtimestamp(ms / 1000, tz=_MSK).date() - - -def test_sort_timestamp_map_built_from_json() -> None: - html = FIXTURE.read_text(encoding="utf-8") - ts_map = _build_sort_timestamp_map(html) - assert ts_map["8043936560"] == _expected(_TS_LAZY) - assert ts_map["8163084615"] == _expected(_TS_TOP) - - -def test_parse_html_sets_listing_date_for_all_cards() -> None: - """Обе карточки получают listing_date != None (DoD #726).""" - html = FIXTURE.read_text(encoding="utf-8") - lots = AvitoScraper()._parse_html(html, "https://www.avito.ru/ekaterinburg/kvartiry") - assert len(lots) == 2 - by_url = {lot.source_url: lot for lot in lots} - assert all(lot.listing_date is not None for lot in lots), "every card must carry a date" - - # Lazy card (нет DOM item-date) → дата из JSON sortTimeStamp. - lazy = next(lot for lot in lots if "8043936560" in (lot.source_url or "")) - assert lazy.listing_date == _expected(_TS_LAZY) - assert by_url # touch to keep linter calm if structure changes - - -def test_lazy_card_date_comes_from_json_not_dom() -> None: - """Карточка без data-marker='item-date' всё равно датируется (JSON primary).""" - html = FIXTURE.read_text(encoding="utf-8") - assert 'data-item-id="8043936560"' in html - # у этой карточки в фикстуре НЕТ DOM item-date — проверяем, что дата не потерялась - lots = AvitoScraper()._parse_html(html, "https://www.avito.ru/x") - lazy = next(lot for lot in lots if "8043936560" in (lot.source_url or "")) - assert lazy.listing_date == _expected(_TS_LAZY) - - -# ── listing_segment по DOM-маркеру item-development-name ─────────────────────── -# Каждый avito-URL квартиры содержит /kvartiry/ (вкл. новостройки), поэтому -# сегмент определяется не по URL, а по наличию data-marker="item-development-name" -# (название ЖК/застройщика рендерится только у карточек новостроек). - - -def _serp_card(item_id: str, *, with_dev_name: bool, with_zhk_anchor: bool = False) -> str: - dev = '
ЖК «Федерация»
' if with_dev_name else "" - zhk = ( - 'ЖК Федерация' - if with_zhk_anchor - else "" - ) - return f""" - - """ - - -def test_segment_novostroyki_when_development_name_present() -> None: - """Карточка с data-marker='item-development-name' → listing_segment='novostroyki'.""" - html = f"{_serp_card('8043003287', with_dev_name=True)}" - lots = AvitoScraper()._parse_html(html, "https://www.avito.ru/ekaterinburg/kvartiry") - assert len(lots) == 1 - assert lots[0].listing_segment == "novostroyki" - - -def test_segment_vtorichka_when_no_development_name() -> None: - """Карточка без маркера ЖК → listing_segment='vtorichka' (даже с /kvartiry/ в URL).""" - html = f"{_serp_card('8043936560', with_dev_name=False)}" - lots = AvitoScraper()._parse_html(html, "https://www.avito.ru/ekaterinburg/kvartiry") - assert len(lots) == 1 - assert lots[0].listing_segment == "vtorichka" - - -# ── newbuilding (ЖК) id/url из zhk-сабдомена ────────────────────────────────── - - -def test_newbuilding_captured_from_zhk_anchor() -> None: - """Новостройка с якорем zhk-*.avito.ru → newbuilding_url без ?context, id = slug.""" - card = _serp_card("8043003287", with_dev_name=True, with_zhk_anchor=True) - html = f"{card}" - lots = AvitoScraper()._parse_html(html, "https://www.avito.ru/ekaterinburg/kvartiry") - assert len(lots) == 1 - assert lots[0].newbuilding_url == "https://zhk-fed-ekaterinburg.avito.ru" - assert lots[0].newbuilding_id == "fed-ekaterinburg" - - -def test_newbuilding_none_when_novostroyka_without_anchor() -> None: - """Новостройка без zhk-якоря → оба поля None (сегмент по dev-name остаётся).""" - card = _serp_card("8043003287", with_dev_name=True, with_zhk_anchor=False) - html = f"{card}" - lots = AvitoScraper()._parse_html(html, "https://www.avito.ru/ekaterinburg/kvartiry") - assert len(lots) == 1 - assert lots[0].listing_segment == "novostroyki" - assert lots[0].newbuilding_id is None - assert lots[0].newbuilding_url is None - - -def test_newbuilding_none_for_vtorichka() -> None: - """Вторичка → newbuilding_id/url оба None.""" - html = f"{_serp_card('8043936560', with_dev_name=False)}" - lots = AvitoScraper()._parse_html(html, "https://www.avito.ru/ekaterinburg/kvartiry") - assert len(lots) == 1 - assert lots[0].newbuilding_id is None - assert lots[0].newbuilding_url is None diff --git a/tradein-mvp/backend/tests/test_base_save_listings.py b/tradein-mvp/backend/tests/test_base_save_listings.py deleted file mode 100644 index 9904107d..00000000 --- a/tradein-mvp/backend/tests/test_base_save_listings.py +++ /dev/null @@ -1,1357 +0,0 @@ -"""Тесты для base.py save_listings — проверяем что Cian-специфичные колонки -передаются в INSERT и ON CONFLICT DO UPDATE SET. - -Используем unittest.mock для замены db.execute — без реальной БД. -""" - -import json -from contextlib import contextmanager -from unittest.mock import MagicMock, patch - -import pytest - -from app.services.scrapers.base import ScrapedLot, save_listings - -# ── Autouse fixture: patch matching service so all save_listings tests are deterministic ── - - -@pytest.fixture(autouse=True) -def _patch_matching(): - """Stub matching service for all tests in this module. - - save_listings() now calls match_or_create_house + upsert_listing_source after - each INSERT. Without patching, those would issue extra db.execute() calls with - real SQL (and MagicMock returns truthy mappings() that would simulate Tier 0 - matches with fake house_ids). Tests assert via call_args_list[0] (the listings - INSERT) and call_count on the patched stubs to verify the hook fires. - """ - with ( - patch("app.services.scrapers.base.match_or_create_house") as m_house, - patch("app.services.scrapers.base.upsert_listing_source") as m_link, - ): - m_house.return_value = (101, 1.0, "new") # (house_id, conf, method) - m_link.return_value = None - yield {"house": m_house, "link": m_link} - - -# ── Helpers ────────────────────────────────────────────────────────────────── - - -def _mock_db(inserted: bool = True, listing_id: int = 42) -> MagicMock: - """Mock Session: INSERT listings RETURNING id, inserted=; SAVEPOINTs no-op.""" - row = MagicMock() - row.id = listing_id - row.inserted = inserted - - result = MagicMock() - result.fetchone.return_value = row - - db = MagicMock() - db.execute.return_value = result - - # begin_nested() returns a context manager; SAVEPOINT semantics — no-op here - @contextmanager - def _nested(): - yield MagicMock() - - db.begin_nested.side_effect = _nested - return db - - -def _base_lot(**overrides) -> ScrapedLot: - """Минимальный валидный ScrapedLot — только обязательные поля.""" - defaults = { - "source": "avito", - "source_url": "https://www.avito.ru/ekaterinburg/kvartiry/1-k_987654", - "source_id": "987654", - "price_rub": 4_500_000, - } - defaults.update(overrides) - return ScrapedLot(**defaults) - - -def _cian_lot(**overrides) -> ScrapedLot: - """ScrapedLot с полным набором Cian-специфичных полей.""" - base = { - "source": "cian", - "source_url": "https://ekb.cian.ru/sale/flat/123456/", - "source_id": "123456", - "price_rub": 6_200_000, - "address": "Екатеринбург, улица Постовского, 17А", - "lat": 56.8385, - "lon": 60.5940, - "rooms": 2, - "area_m2": 52.0, - "floor": 7, - "total_floors": 16, - # Cian-specific - "living_area_m2": 35.0, - "bedrooms_count": 2, - "balconies_count": 1, - "loggias_count": 0, - "description_minhash": "abc123def456abc1", - "cadastral_number": "66:41:0204016:1234", - "building_cadastral_number": "66:41:0204016:100", - # PII purge (#2217): phones больше не заполняется — по умолчанию пусто - # (ScrapedLot.phones default_factory=list, сериализуется в None). - "is_homeowner": True, - "is_pro_seller": False, - "bargain_allowed": True, - "sale_type": "free", - "metro_stations": [{"name": "Геологическая", "time": 10, "mode": "walk"}], - "house_source": "cian", - "house_ext_id": None, - "listing_segment": "vtorichka", - } - base.update(overrides) - return ScrapedLot(**base) - - -# ── Utility: extract params from mock db call ──────────────────────────────── - - -def _find_listings_insert_call(db: MagicMock): - """Найти (sql, params) вызова INSERT INTO listings. - - save_listings теперь делает лёгкий pre-read `SELECT card_hash FROM listings` - ПЕРЕД upsert'ом, поэтому INSERT больше не обязательно call_args_list[0]. - Ищем по SQL-тексту, а не по фиксированному индексу. - """ - assert db.execute.called, "db.execute was not called" - for call in db.execute.call_args_list: - sql = str(call.args[0]) - if "INSERT INTO listings (" in sql: - return sql, call.args[1] - raise AssertionError("INSERT INTO listings call not found") - - -def _get_execute_params(db: MagicMock) -> dict: - """Получить params dict из вызова INSERT listings.""" - _sql, params = _find_listings_insert_call(db) - return params - - -def _get_listings_insert_sql(db: MagicMock) -> str: - """Получить SQL-текст INSERT INTO listings.""" - sql, _params = _find_listings_insert_call(db) - return sql - - -# ── Test 1: Cian lot → все новые колонки попадают в params ────────────────── - - -def test_save_listings_persists_cian_specific_fields(): - """Cian lot с заполненными полями → все Cian колонки в params INSERT.""" - db = _mock_db(inserted=True) - lot = _cian_lot() - - inserted, updated = save_listings(db, [lot]) - - assert inserted == 1 - assert updated == 0 - - params = _get_execute_params(db) - - assert params["living_area_m2"] == 35.0 - assert params["bedrooms_count"] == 2 - assert params["balconies_count"] == 1 - assert params["loggias_count"] == 0 - assert params["description_minhash"] == "abc123def456abc1" - assert params["cadastral_number"] == "66:41:0204016:1234" - assert params["building_cadastral_number"] == "66:41:0204016:100" - assert params["is_homeowner"] is True - assert params["is_pro_seller"] is False - assert params["bargain_allowed"] is True - assert params["sale_type"] == "free" - - # PII purge (#2217): phones больше не заполняется cian-фикстурой → None. - assert params["phones"] is None - - # metro_stations — JSON-строка, проверяем десериализацией - metro = json.loads(params["metro_stations"]) - assert metro[0]["name"] == "Геологическая" - assert metro[0]["time"] == 10 - - -# ── Test 2: Avito lot → Cian-специфичные params = None (NULL) ─────────────── - - -def test_save_listings_avito_compat_cian_cols_are_null(): - """Avito lot не имеет Cian-полей → все Cian params = None (→ SQL NULL). - - Проверяет backward compat — Avito / DomRF scrapers не ломаются. - """ - db = _mock_db(inserted=True) - lot = _base_lot(source="avito", source_id="987654") - - inserted, _updated = save_listings(db, [lot]) - - assert inserted == 1 - - params = _get_execute_params(db) - - # ScrapedLot.living_area_m2 defaults to None - assert params["living_area_m2"] is None - assert params["bedrooms_count"] is None - assert params["balconies_count"] is None - assert params["loggias_count"] is None - assert params["description_minhash"] is None - assert params["cadastral_number"] is None - assert params["building_cadastral_number"] is None - assert params["is_homeowner"] is None - assert params["is_pro_seller"] is None - assert params["bargain_allowed"] is None - assert params["sale_type"] is None - # phones=[] (default_factory=list) → None (пустой список → None в helper) - assert params["phones"] is None - # metro_stations=[] → None - assert params["metro_stations"] is None - - -# ── Test 3: ON CONFLICT — при updated=False все новые поля в UPDATE SET ────── - - -def test_save_listings_on_conflict_updates_cian_cols(): - """ON CONFLICT path: xmax != 0 → updated counter. Params верны для UPDATE. - - Проверяем что SQL-текст содержит все новые колонки в DO UPDATE SET. - """ - db = _mock_db(inserted=False) # xmax != 0 → updated - lot = _cian_lot( - source_id="111", - cadastral_number="66:41:0000000:NEW", - description_minhash="newhash", - ) - - inserted, updated = save_listings(db, [lot]) - - assert inserted == 0 - assert updated == 1 - - # Проверяем SQL-текст на наличие DO UPDATE SET с новыми колонками - sql_text = _get_listings_insert_sql(db) - - assert "cadastral_number = EXCLUDED.cadastral_number" in sql_text - assert "description_minhash = EXCLUDED.description_minhash" in sql_text - assert "building_cadastral_number = EXCLUDED.building_cadastral_number" in sql_text - assert "phones = EXCLUDED.phones" in sql_text - assert "is_homeowner = EXCLUDED.is_homeowner" in sql_text - assert "is_pro_seller = EXCLUDED.is_pro_seller" in sql_text - assert "bargain_allowed = EXCLUDED.bargain_allowed" in sql_text - assert "sale_type = EXCLUDED.sale_type" in sql_text - assert "metro_stations = EXCLUDED.metro_stations" in sql_text - assert "living_area_m2 = EXCLUDED.living_area_m2" in sql_text - assert "bedrooms_count = EXCLUDED.bedrooms_count" in sql_text - assert "balconies_count = EXCLUDED.balconies_count" in sql_text - assert "loggias_count = EXCLUDED.loggias_count" in sql_text - - # Params с новым значением cadastral_number - params = _get_execute_params(db) - assert params["cadastral_number"] == "66:41:0000000:NEW" - assert params["description_minhash"] == "newhash" - - -# ── Test 3b: kitchen/ceiling promote (#2007) ───────────────────────────────── - - -def test_save_listings_promotes_kitchen_ceiling_to_columns(): - """yandex lot с kitchen_area_m2/ceiling_height_m → params + обе ceiling-колонки. - - ceiling пишется ОДНИМ param :ceiling_height_m в ДВЕ колонки: ceiling_height - (019, coverage-дашборд + yandex_detail) и ceiling_height_m (111, avito-колонка). - """ - db = _mock_db(inserted=True) - lot = _base_lot( - source="yandex", - source_id=None, - source_url="https://realty.yandex.ru/offer/123/", - kitchen_area_m2=12.7, - ceiling_height_m=2.7, - ) - - save_listings(db, [lot]) - - params = _get_execute_params(db) - assert params["kitchen_area_m2"] == 12.7 - assert params["ceiling_height_m"] == 2.7 - - sql_text = _get_listings_insert_sql(db) - # обе колонки в списке INSERT - assert "kitchen_area_m2" in sql_text - assert "ceiling_height" in sql_text - assert "ceiling_height_m" in sql_text - # один param :ceiling_height_m питает обе ceiling-колонки (дважды в VALUES) - assert sql_text.count(":ceiling_height_m") >= 2 - # ON CONFLICT — COALESCE, чтобы re-scrape без полей не затирал detail-enrichment - assert "kitchen_area_m2 = COALESCE(" in sql_text - assert "ceiling_height = COALESCE(" in sql_text - assert "ceiling_height_m = COALESCE(" in sql_text - - -def test_save_listings_kitchen_ceiling_none_for_serp_without_them(): - """Avito SERP lot без kitchen/ceiling → params None (COALESCE сохранит detail-значение).""" - db = _mock_db(inserted=True) - lot = _base_lot(source="avito", source_id="987654") - - save_listings(db, [lot]) - - params = _get_execute_params(db) - assert params["kitchen_area_m2"] is None - assert params["ceiling_height_m"] is None - - -# ── Test 3c: Class B promote (#2008) — mortgage/apartments/rosreestr ────────── - - -def test_save_listings_promotes_class_b_cian_fields(): - """#2008: mortgage_available/is_apartments/is_rosreestr_checked → params + SQL. - - Колонки + ON CONFLICT COALESCE (re-scrape источника без полей не затирает - значение, ранее записанное cian-ом). - """ - db = _mock_db(inserted=True) - lot = _cian_lot( - mortgage_available=True, - is_apartments=False, - is_rosreestr_checked=True, - ) - - save_listings(db, [lot]) - - params = _get_execute_params(db) - assert params["mortgage_available"] is True - assert params["is_apartments"] is False - assert params["is_rosreestr_checked"] is True - - sql_text = _get_listings_insert_sql(db) - for col in ("mortgage_available", "is_apartments", "is_rosreestr_checked"): - assert col in sql_text, f"Column '{col}' missing from INSERT SQL" - assert "mortgage_available = COALESCE(" in sql_text - assert "is_apartments = COALESCE(" in sql_text - assert "is_rosreestr_checked = COALESCE(" in sql_text - - -def test_save_listings_class_b_fields_none_for_non_cian(): - """#2008: avito lot без Class B-полей → params None (COALESCE сохранит prior).""" - db = _mock_db(inserted=True) - lot = _base_lot(source="avito", source_id="987654") - - save_listings(db, [lot]) - - params = _get_execute_params(db) - assert params["mortgage_available"] is None - assert params["is_apartments"] is None - assert params["is_rosreestr_checked"] is None - - -# ── Test 4: empty list → early return, no db.execute call ─────────────────── - - -def test_save_listings_empty_lots(): - """Пустой список → (0, 0), db.execute не вызывается.""" - db = _mock_db() - - inserted, updated = save_listings(db, []) - - assert inserted == 0 - assert updated == 0 - db.execute.assert_not_called() - - -# ── Test 5: phones=[] и metro_stations=[] → None в params ─────────────────── - - -def test_save_listings_empty_phones_and_metro_become_null(): - """Если phones=[] / metro_stations=[] — передаём None (не '[]' строку).""" - db = _mock_db(inserted=True) - lot = _cian_lot(phones=[], metro_stations=[]) - - save_listings(db, [lot]) - - params = _get_execute_params(db) - assert params["phones"] is None - assert params["metro_stations"] is None - - -# ── Test 6: INSERT SQL содержит все новые колонки ──────────────────────────── - - -def test_save_listings_sql_contains_all_new_columns(): - """SQL INSERT перечисляет все Cian-специфичные колонки.""" - db = _mock_db(inserted=True) - lot = _base_lot() - - save_listings(db, [lot]) - - sql_text = _get_listings_insert_sql(db) - - for col in ( - "living_area_m2", - "bedrooms_count", - "balconies_count", - "loggias_count", - "description_minhash", - "cadastral_number", - "building_cadastral_number", - "phones", - "is_homeowner", - "is_pro_seller", - "bargain_allowed", - "sale_type", - "metro_stations", - ): - assert col in sql_text, f"Column '{col}' missing from INSERT SQL" - - -# ── Test 7: INSERT RETURNING id — used downstream by matching hook ────────── - - -def test_save_listings_returning_id_for_matching_hook(): - """INSERT … RETURNING id, (xmax = 0) — needs both columns for the hook.""" - db = _mock_db(inserted=True, listing_id=12345) - lot = _base_lot() - - save_listings(db, [lot]) - - sql_text = _get_listings_insert_sql(db) - assert "RETURNING id, (xmax = 0) AS inserted" in sql_text - - -# ── Test 8: matching hook fires exactly once per saved listing ────────────── - - -def test_save_listings_invokes_matching_hook_once_per_lot(_patch_matching): - """Each saved lot triggers exactly one upsert_listing_source call. - - match_or_create_house may or may not be called (depends on address/lat/lon - presence). upsert_listing_source MUST always be called. - """ - db = _mock_db(inserted=True, listing_id=777) - lots = [ - _base_lot(source_id="a1"), - _base_lot(source_id="a2", address="Екатеринбург, Ленина 1", lat=56.84, lon=60.6), - _cian_lot(source_id="c1"), - ] - - save_listings(db, lots) - - # Hook fires once per lot - assert _patch_matching["link"].call_count == 3 - - # House resolution attempted only when address/coords present (2 of 3) - assert _patch_matching["house"].call_count == 2 - - # All upsert_listing_source calls use ext_source = listing.source - for idx, call_args in enumerate(_patch_matching["link"].call_args_list): - kwargs = call_args.kwargs - assert kwargs["listing_id"] == 777 - assert kwargs["ext_source"] == lots[idx].source - assert kwargs["ext_id"] == lots[idx].source_id - assert kwargs["method"] == "source_link" - assert kwargs["confidence"] == 1.0 - - -# ── Test 9: matching hook receives full listing context ───────────────────── - - -def test_save_listings_matching_hook_passes_listing_context(_patch_matching): - """upsert_listing_source receives price/area/floor/rooms/url from the lot.""" - db = _mock_db(inserted=True, listing_id=555) - lot = _cian_lot( - source_id="cian-999", - price_rub=7_500_000, - area_m2=54.5, - floor=3, - rooms=2, - ) - - save_listings(db, [lot]) - - kwargs = _patch_matching["link"].call_args.kwargs - assert kwargs["listing_id"] == 555 - assert kwargs["ext_source"] == "cian" - assert kwargs["ext_id"] == "cian-999" - assert kwargs["price_rub"] == 7_500_000 - assert kwargs["area_m2"] == 54.5 - assert kwargs["floor"] == 3 - assert kwargs["rooms_count"] == 2 - assert kwargs["source_url"] == lot.source_url - - -# ── Test 10: matching hook idempotent on re-scrape (ON CONFLICT updated path) ─ - - -def test_save_listings_matching_hook_fires_on_update_path(_patch_matching): - """ON CONFLICT DO UPDATE (re-scrape) — hook still fires so last_seen_at refreshes.""" - db = _mock_db(inserted=False, listing_id=12) # updated, not new - lot = _base_lot() - - save_listings(db, [lot]) - - assert _patch_matching["link"].call_count == 1 - - -# ── Test 11: matching failure logs warning but does not abort the batch ───── - - -def test_save_listings_matching_failure_does_not_abort_batch(_patch_matching, caplog): - """When upsert_listing_source raises, the loop logs and continues.""" - import logging - - _patch_matching["link"].side_effect = RuntimeError("simulated DB blip") - - db = _mock_db(inserted=True, listing_id=99) - lots = [_base_lot(source_id="a1"), _base_lot(source_id="a2")] - - with caplog.at_level(logging.WARNING, logger="app.services.scrapers.base"): - inserted, updated = save_listings(db, lots) - - # Both listings still INSERTed; only the hook failed - assert inserted == 2 - assert updated == 0 - - # 2 warnings — one per lot - warns = [r for r in caplog.records if "match_failed" in r.getMessage()] - assert len(warns) == 2 - - # Hook attempted on both - assert _patch_matching["link"].call_count == 2 - - -# ── Test 12: lot without source_id falls back to dedup_hash as ext_id ─────── - - -def test_save_listings_matching_hook_dedup_hash_fallback(_patch_matching): - """Yandex-style lot without stable source_id → ext_id := dedup_hash.""" - db = _mock_db(inserted=True, listing_id=44) - lot = _base_lot( - source="yandex", - source_url="https://realty.yandex.ru/offer/some-offer/", - source_id=None, - address="Екатеринбург, Малышева 51", - lat=56.84, - lon=60.6, - ) - - save_listings(db, [lot]) - - kwargs = _patch_matching["link"].call_args.kwargs - assert kwargs["ext_source"] == "yandex" - # ext_id should be the dedup_hash (sha256 hex), not None - assert kwargs["ext_id"] - assert len(kwargs["ext_id"]) == 64 # sha256 hex - assert kwargs["ext_id"] == lot.compute_dedup_hash() - - -# ── Test 13: house_match failure aborts the hook (single SAVEPOINT level) ───── - - -def test_save_listings_house_match_failure_aborts_hook(_patch_matching, caplog): - """If match_or_create_house raises, the outer per-row SAVEPOINT rolls back the - entire hook call (_link_listing_to_house). upsert_listing_source is NOT called - because there is no inner SAVEPOINT to shelter it — a single SAVEPOINT level - (per backend.md) means one failure = one hook roll-back. The listings INSERT - that happened before the SAVEPOINT is unaffected (inserted counter still 1). - - This is a behaviour change from the old double-SAVEPOINT design where house - failure was silently swallowed and listing_sources was still upserted. With a - single SAVEPOINT the outer except at save_listings:386 logs match_failed and - continues to the next lot. (#849 Part 3) - """ - import logging - - _patch_matching["house"].side_effect = RuntimeError("address parse boom") - - db = _mock_db(inserted=True, listing_id=88) - lot = _base_lot( - source_id="x1", - address="Bad addr", - lat=56.84, - lon=60.6, - ) - - with caplog.at_level(logging.WARNING, logger="app.services.scrapers.base"): - inserted, _ = save_listings(db, [lot]) - - assert inserted == 1 - # House match attempted — hook aborted, listing_source NOT recorded - assert _patch_matching["house"].call_count == 1 - assert _patch_matching["link"].call_count == 0 - # Outer except logs the failure - warns = [r for r in caplog.records if "match_failed" in r.getMessage()] - assert len(warns) == 1 - - -# ── Test 14: empty list → matching service not called ─────────────────────── - - -def test_save_listings_empty_lots_skips_matching(_patch_matching): - """Empty input → no hook calls (verifies early return preserves behaviour).""" - db = _mock_db() - save_listings(db, []) - - assert _patch_matching["house"].call_count == 0 - assert _patch_matching["link"].call_count == 0 - - -# ── #753: dedup_hash стабильность (Avito ?context= + смена цены) ───────────── - - -def test_dedup_hash_stable_across_price_and_context_query(): - """#753: тот же Avito item (один source_id) с РАЗНОЙ ценой и РАЗНЫМ ?context= - в URL → ОДИНАКОВЫЙ dedup_hash. Иначе смена цены / ре-скрейп = дубль.""" - lot_a = _base_lot( - source_url="https://www.avito.ru/ekaterinburg/kvartiry/1-k_987654?context=AAAA", - source_id="987654", - price_rub=4_500_000, - ) - lot_b = _base_lot( - source_url="https://www.avito.ru/ekaterinburg/kvartiry/1-k_987654?context=ZZZZ", - source_id="987654", - price_rub=4_300_000, # цена снижена - ) - assert lot_a.compute_dedup_hash() == lot_b.compute_dedup_hash() - - -def test_dedup_hash_strips_query_when_no_source_id(): - """#753: при отсутствии source_id ключ = source_url без query-string — - волатильный ?context= не должен влиять на hash.""" - lot_a = _base_lot( - source_url="https://www.avito.ru/ekaterinburg/kvartiry/1-k_55?context=AAAA", - source_id=None, - price_rub=5_000_000, - ) - lot_b = _base_lot( - source_url="https://www.avito.ru/ekaterinburg/kvartiry/1-k_55?context=BBBB", - source_id=None, - price_rub=5_900_000, - ) - assert lot_a.compute_dedup_hash() == lot_b.compute_dedup_hash() - - -def test_dedup_hash_distinguishes_different_listings(): - """#753 guard: разные объявления (разные source_id) → РАЗНЫЙ hash - (фикс не должен схлопывать несвязанные listings).""" - lot_a = _base_lot(source_id="111", source_url="https://www.avito.ru/a_111") - lot_b = _base_lot(source_id="222", source_url="https://www.avito.ru/a_222") - assert lot_a.compute_dedup_hash() != lot_b.compute_dedup_hash() - - -def test_dedup_hash_distinguishes_source_even_same_id(): - """#753 guard: одинаковый source_id у разных источников → РАЗНЫЙ hash - (source — часть ключа).""" - lot_avito = _base_lot(source="avito", source_id="500") - lot_cian = _base_lot(source="cian", source_id="500") - assert lot_avito.compute_dedup_hash() != lot_cian.compute_dedup_hash() - - -# ── Tests 15-17: #821 COALESCE backfill listing_date / area_m2 ────────────── - - -def test_save_listings_sql_contains_coalesce_listing_date_area_m2(): - """#821: ON CONFLICT SET block содержит COALESCE для listing_date и area_m2.""" - db = _mock_db(inserted=False) # conflict path - lot = _cian_lot() - - save_listings(db, [lot]) - - sql_text = _get_listings_insert_sql(db) - assert "COALESCE(EXCLUDED.listing_date, listings.listing_date)" in sql_text - assert "COALESCE(EXCLUDED.area_m2, listings.area_m2)" in sql_text - - -def test_save_listings_params_listing_date_area_m2_passed_through(): - """#821: listing_date и area_m2 попадают в params INSERT (VALUES side).""" - import datetime - - db = _mock_db(inserted=True) - lot = _base_lot( - source_id="ld_test", - area_m2=47.5, - listing_date=datetime.date(2024, 3, 15), - ) - - save_listings(db, [lot]) - - params = _get_execute_params(db) - assert params["area_m2"] == 47.5 - assert params["listing_date"] == datetime.date(2024, 3, 15) - - -def test_save_listings_null_area_and_date_pass_none_in_params(): - """#821: re-scrape без area_m2/listing_date → params содержат None. - - COALESCE в SQL сохранит существующее значение в БД — здесь проверяем - только что params не ломают вызов (None передаётся без ошибки). - """ - db = _mock_db(inserted=False) # conflict/update path - lot = _base_lot( - source_id="null_test", - area_m2=None, - listing_date=None, - ) - - inserted, updated = save_listings(db, [lot]) - - # Ожидаем updated (xmax != 0) - assert inserted == 0 - assert updated == 1 - - params = _get_execute_params(db) - assert params["area_m2"] is None - assert params["listing_date"] is None - - -# ── Tests: house_id_fk realtime link write (avito/cian unlinked fix) ───────── - - -def _find_house_fk_update(db: MagicMock): - """Return (sql, params) of the UPDATE listings SET house_id_fk call, or None. - - match_or_create_house / upsert_listing_source are patched (no db.execute), so - the only db.execute calls are the listings INSERT (call 0) and — when a house - is resolved — the house_id_fk UPDATE added by the realtime link hook. - """ - for call in db.execute.call_args_list: - sql = str(call.args[0]) - if "house_id_fk" in sql and "UPDATE listings" in sql: - return sql, call.args[1] - return None - - -def test_save_listings_writes_house_id_fk_when_house_resolved(_patch_matching): - """House resolved (address/coords present) → UPDATE listings.house_id_fk runs - with the resolved house_id and the saved listing id.""" - _patch_matching["house"].return_value = (909, 1.0, "geo") - - db = _mock_db(inserted=True, listing_id=4242) - lot = _cian_lot(source_id="c-fk-1") # has address + lat/lon → house resolved - - save_listings(db, [lot]) - - found = _find_house_fk_update(db) - assert found is not None, "UPDATE listings SET house_id_fk was not issued" - _sql, params = found - assert params["hid"] == 909 - assert params["lid"] == 4242 - - -def test_save_listings_no_house_fk_update_when_house_not_resolved(_patch_matching): - """No address and no coords → match_or_create_house skipped → no FK UPDATE.""" - db = _mock_db(inserted=True, listing_id=4243) - # base lot has no address / lat / lon → house resolution skipped - lot = _base_lot(source="avito", source_id="a-fk-no", address=None, lat=None, lon=None) - - save_listings(db, [lot]) - - assert _patch_matching["house"].call_count == 0 - assert _find_house_fk_update(db) is None - - -def test_save_listings_house_fk_update_guards_no_op_write(_patch_matching): - """The UPDATE is idempotency-guarded (IS DISTINCT FROM) and casts to bigint - (psycopg v3 CAST, never ::bigint).""" - _patch_matching["house"].return_value = (55, 1.0, "cadastr") - - db = _mock_db(inserted=True, listing_id=1) - lot = _cian_lot(source_id="c-fk-guard") - - save_listings(db, [lot]) - - found = _find_house_fk_update(db) - assert found is not None - sql, _params = found - assert "IS DISTINCT FROM" in sql - assert "CAST(:hid AS bigint)" in sql - assert "::bigint" not in sql - - -# ── Tests 18-20: Yandex rich fields (predictedPrice, trend, description, agency) ── - - -def _yandex_rich_lot(**overrides) -> ScrapedLot: - """ScrapedLot с заполненными Yandex rich-полями.""" - import datetime - - base = { - "source": "yandex", - "source_url": "https://realty.yandex.ru/offer/5500000000000000001/", - "source_id": "5500000000000000001", - "price_rub": 7_200_000, - "address": "Екатеринбург, улица Мира, 10", - "rooms": 2, - "area_m2": 48.0, - "listing_date": datetime.date(2026, 5, 24), - "publish_date": datetime.date(2026, 5, 24), - "days_on_market": 24, - "description": "Светлая квартира.", - "is_homeowner": True, - "agency_name": None, - "metro_stations": [{"name": "Динамо", "min": 7}], - "yandex_offer_id": "5500000000000000001", - "predicted_price_rub": 7_554_000, - "predicted_price_min": 7_100_000, - "predicted_price_max": 8_000_000, - "price_trend": "DECREASED", - "price_previous_rub": 7_500_000, - } - base.update(overrides) - return ScrapedLot(**base) - - -def test_save_listings_persists_yandex_rich_fields(): - """Yandex rich-поля попадают в params INSERT.""" - db = _mock_db(inserted=True) - lot = _yandex_rich_lot() - - save_listings(db, [lot]) - - params = _get_execute_params(db) - assert params["description"] == "Светлая квартира." - assert params["days_on_market"] == 24 - assert params["yandex_offer_id"] == "5500000000000000001" - assert params["predicted_price_rub"] == 7_554_000 - assert params["predicted_price_min"] == 7_100_000 - assert params["predicted_price_max"] == 8_000_000 - assert params["price_trend"] == "DECREASED" - assert params["price_previous_rub"] == 7_500_000 - assert params["agency_name"] is None - metro = json.loads(params["metro_stations"]) - assert metro == [{"name": "Динамо", "min": 7}] - - -def test_save_listings_yandex_rich_cols_in_sql(): - """INSERT + ON CONFLICT SET содержат новые Yandex rich колонки.""" - db = _mock_db(inserted=False) - lot = _yandex_rich_lot(agency_name="Агентство «Диал»", is_pro_seller=True, is_homeowner=None) - - save_listings(db, [lot]) - - sql_text = _get_listings_insert_sql(db) - for col in ( - "description", - "publish_date", - "agency_name", - "yandex_offer_id", - "predicted_price_rub", - "predicted_price_min", - "predicted_price_max", - "price_trend", - "price_previous_rub", - ): - assert col in sql_text, f"Column '{col}' missing from INSERT SQL" - - # COALESCE backfill in DO UPDATE SET (never wipe a prior value) - assert "description = COALESCE(EXCLUDED.description, listings.description)" in sql_text - assert "agency_name = COALESCE(EXCLUDED.agency_name, listings.agency_name)" in sql_text - assert "predicted_price_rub = COALESCE(" in sql_text - assert "price_trend = COALESCE(EXCLUDED.price_trend, listings.price_trend)" in sql_text - - -def test_save_listings_non_yandex_rich_cols_are_null(): - """Avito lot не имеет Yandex rich-полей → params = None (backward compat).""" - db = _mock_db(inserted=True) - lot = _base_lot(source="avito", source_id="987654") - - save_listings(db, [lot]) - - params = _get_execute_params(db) - assert params["description"] is None - assert params["publish_date"] is None - assert params["agency_name"] is None - assert params["yandex_offer_id"] is None - assert params["predicted_price_rub"] is None - assert params["predicted_price_min"] is None - assert params["predicted_price_max"] is None - assert params["price_trend"] is None - assert params["price_previous_rub"] is None - - -# ── newbuilding (ЖК) id/url persistence ────────────────────────────────────── - - -def test_save_listings_persists_newbuilding_fields(): - """Lot с newbuilding_id/url → оба попадают в params INSERT + есть в SQL.""" - db = _mock_db(inserted=True) - lot = _base_lot( - source="avito", - source_id="nb_test", - newbuilding_id="federatsiya-ekaterinburg", - newbuilding_url="https://zhk-federatsiya-ekaterinburg.avito.ru", - ) - - save_listings(db, [lot]) - - params = _get_execute_params(db) - assert params["newbuilding_id"] == "federatsiya-ekaterinburg" - assert params["newbuilding_url"] == "https://zhk-federatsiya-ekaterinburg.avito.ru" - - sql_text = _get_listings_insert_sql(db) - assert "newbuilding_id" in sql_text - assert "newbuilding_url" in sql_text - # COALESCE backfill in DO UPDATE SET (never wipe a prior value). - # SQL wraps long COALESCE calls; assert on a whitespace-normalised copy. - sql_norm = " ".join(sql_text.split()) - assert ( - "newbuilding_id = COALESCE( EXCLUDED.newbuilding_id, listings.newbuilding_id )" in sql_norm - ) - assert ( - "newbuilding_url = COALESCE( EXCLUDED.newbuilding_url, listings.newbuilding_url )" - in sql_norm - ) - - -def test_save_listings_newbuilding_fields_default_none(): - """Lot без ЖК-полей → params newbuilding_id/url = None (backward compat).""" - db = _mock_db(inserted=True) - lot = _base_lot(source="avito", source_id="987654") - - save_listings(db, [lot]) - - params = _get_execute_params(db) - assert params["newbuilding_id"] is None - assert params["newbuilding_url"] is None - - -# ── card_hash: ScrapedLot.compute_card_hash determinism + sensitivity ──────── - - -def test_compute_card_hash_deterministic(): - """Тот же lot → тот же hash (вызов дважды и два эквивалентных lot).""" - lot = _cian_lot() - assert lot.compute_card_hash() == lot.compute_card_hash() - - lot_b = _cian_lot() - assert lot.compute_card_hash() == lot_b.compute_card_hash() - # sha256 hex - assert len(lot.compute_card_hash()) == 64 - - -def test_compute_card_hash_changes_on_price(): - """Смена price_rub → другой hash.""" - a = _base_lot(price_rub=4_500_000) - b = _base_lot(price_rub=4_300_000) - assert a.compute_card_hash() != b.compute_card_hash() - - -def test_compute_card_hash_changes_on_area_floor(): - """Смена area_m2, floor, total_floors, rooms → другой hash.""" - base = _cian_lot() - assert base.compute_card_hash() != _cian_lot(area_m2=53.0).compute_card_hash() - assert base.compute_card_hash() != _cian_lot(floor=8).compute_card_hash() - assert base.compute_card_hash() != _cian_lot(total_floors=17).compute_card_hash() - assert base.compute_card_hash() != _cian_lot(rooms=3).compute_card_hash() - - -def test_compute_card_hash_changes_on_address_and_photos(): - """Смена address (после нормализации) и числа фото → другой hash.""" - base = _base_lot(address="Екатеринбург, Ленина 1", photo_urls=["a", "b"]) - assert ( - base.compute_card_hash() - != _base_lot(address="Екатеринбург, Ленина 2", photo_urls=["a", "b"]).compute_card_hash() - ) - assert ( - base.compute_card_hash() - != _base_lot( - address="Екатеринбург, Ленина 1", photo_urls=["a", "b", "c"] - ).compute_card_hash() - ) - - -def test_compute_card_hash_address_normalized(): - """address нормализуется (strip + lower) — регистр/пробелы не меняют hash.""" - a = _base_lot(address=" Екатеринбург, Ленина 1 ") - b = _base_lot(address="екатеринбург, ленина 1") - assert a.compute_card_hash() == b.compute_card_hash() - - -def test_compute_card_hash_none_address_stable(): - """address=None → "" — два lot без адреса дают одинаковый hash.""" - a = _base_lot(address=None) - b = _base_lot(address=None) - assert a.compute_card_hash() == b.compute_card_hash() - - -def test_compute_card_hash_stable_when_only_timestamps_or_detail_change(): - """Изменение только timestamp/detail-only полей (не входящих в карточку) → - hash НЕ меняется.""" - import datetime - - base = _cian_lot() - # detail-only / metadata поля — НЕ часть card_hash - variant = _cian_lot( - listing_date=datetime.date(2026, 1, 1), - publish_date=datetime.date(2026, 1, 2), - days_on_market=42, - description="совсем другое описание", - cadastral_number="66:41:9999999:9999", - is_homeowner=False, - agency_name="Другое агентство", - raw_payload={"foo": "bar"}, - ) - assert base.compute_card_hash() == variant.compute_card_hash() - - -# ── save_listings: card_hash persisted on insert + update ──────────────────── - - -def test_save_listings_card_hash_in_sql_and_params(): - """INSERT SQL содержит колонку card_hash + EXCLUDED в DO UPDATE SET; - params несут вычисленный hash.""" - db = _mock_db(inserted=True) - lot = _base_lot() - - save_listings(db, [lot]) - - sql_text = _get_listings_insert_sql(db) - assert "card_hash" in sql_text - assert "card_hash = EXCLUDED.card_hash" in sql_text - - params = _get_execute_params(db) - assert params["card_hash"] == lot.compute_card_hash() - assert len(params["card_hash"]) == 64 - - -def test_save_listings_card_hash_on_update_path(): - """ON CONFLICT (update) — card_hash тоже передаётся в params.""" - db = _mock_db(inserted=False) - lot = _cian_lot(source_id="ch-upd") - - inserted, updated = save_listings(db, [lot]) - assert inserted == 0 - assert updated == 1 - - params = _get_execute_params(db) - assert params["card_hash"] == lot.compute_card_hash() - - -def test_save_listings_pre_reads_existing_card_hash(): - """Перед upsert'ом делается лёгкий SELECT card_hash, last_seen_at FROM listings.""" - db = _mock_db(inserted=True) - lot = _base_lot() - - save_listings(db, [lot]) - - pre_reads = [ - c - for c in db.execute.call_args_list - if "SELECT card_hash" in str(c.args[0]) and "WHERE dedup_hash" in str(c.args[0]) - ] - assert len(pre_reads) == 1 - assert pre_reads[0].args[1]["dedup"] == lot.compute_dedup_hash() - - -def test_save_listings_skips_snapshot_when_card_unchanged(): - """Если prior card_hash == new card_hash И snapshot за сегодня уже есть — - upsert_listing_snapshot НЕ вызывается (snapshot-dedup).""" - lot = _base_lot() - new_hash = lot.compute_card_hash() - - # row для INSERT RETURNING (id, inserted) - insert_row = MagicMock() - insert_row.id = 42 - insert_row.inserted = False # update path - - # pre-read row с тем же card_hash → карточка не изменилась - prior_row = MagicMock() - prior_row.card_hash = new_hash - prior_row.last_seen_at = None # не сегодня → skip_seen_today не срабатывает - - # today-snapshot EXISTS row (любой truthy) - today_row = MagicMock() - - db = MagicMock() - - def _execute(sql, params=None): - s = str(sql) - res = MagicMock() - if "SELECT card_hash" in s and "WHERE dedup_hash" in s: - res.fetchone.return_value = prior_row - elif "FROM listings_snapshots" in s: - res.fetchone.return_value = today_row - elif "INSERT INTO listings (" in s: - res.fetchone.return_value = insert_row - else: - res.fetchone.return_value = None - return res - - db.execute.side_effect = _execute - - @contextmanager - def _nested(): - yield MagicMock() - - db.begin_nested.side_effect = _nested - - with patch("app.services.scrapers.base.upsert_listing_snapshot") as m_snap: - save_listings(db, [lot]) - m_snap.assert_not_called() - - -def test_save_listings_writes_snapshot_when_card_changed(): - """Если card_hash изменился — snapshot пишется даже если сегодня уже был - (когда в сомнениях — пишем).""" - lot = _base_lot(price_rub=4_300_000) # новая цена → новый hash - - insert_row = MagicMock() - insert_row.id = 42 - insert_row.inserted = False - - prior_row = MagicMock() - prior_row.card_hash = "оld-different-hash" # отличается - - db = MagicMock() - - def _execute(sql, params=None): - s = str(sql) - res = MagicMock() - if "SELECT card_hash" in s and "WHERE dedup_hash" in s: - res.fetchone.return_value = prior_row - elif "INSERT INTO listings (" in s: - res.fetchone.return_value = insert_row - else: - res.fetchone.return_value = None - return res - - db.execute.side_effect = _execute - - @contextmanager - def _nested(): - yield MagicMock() - - db.begin_nested.side_effect = _nested - - with patch("app.services.scrapers.base.upsert_listing_snapshot") as m_snap: - save_listings(db, [lot]) - assert m_snap.call_count == 1 - - -def test_save_listings_writes_snapshot_on_fresh_insert(): - """Свежий insert (prior_card_hash is None) → snapshot всегда пишется.""" - lot = _base_lot() - - insert_row = MagicMock() - insert_row.id = 42 - insert_row.inserted = True - - db = MagicMock() - - def _execute(sql, params=None): - s = str(sql) - res = MagicMock() - if "SELECT card_hash" in s and "WHERE dedup_hash" in s: - res.fetchone.return_value = None # row не существует - elif "INSERT INTO listings (" in s: - res.fetchone.return_value = insert_row - else: - res.fetchone.return_value = None - return res - - db.execute.side_effect = _execute - - @contextmanager - def _nested(): - yield MagicMock() - - db.begin_nested.side_effect = _nested - - with patch("app.services.scrapers.base.upsert_listing_snapshot") as m_snap: - save_listings(db, [lot]) - assert m_snap.call_count == 1 - - -def test_save_listings_writes_snapshot_when_unchanged_but_no_today_snapshot(): - """card_hash не изменился, НО snapshot за сегодня ещё нет (первый скрейп дня) → - snapshot всё равно пишется (не теряем сегодняшнее наблюдение).""" - lot = _base_lot() - new_hash = lot.compute_card_hash() - - insert_row = MagicMock() - insert_row.id = 42 - insert_row.inserted = False - - prior_row = MagicMock() - prior_row.card_hash = new_hash # тот же - - db = MagicMock() - - def _execute(sql, params=None): - s = str(sql) - res = MagicMock() - if "SELECT card_hash" in s and "WHERE dedup_hash" in s: - res.fetchone.return_value = prior_row - elif "FROM listings_snapshots" in s: - res.fetchone.return_value = None # сегодня snapshot'а ещё нет - elif "INSERT INTO listings (" in s: - res.fetchone.return_value = insert_row - else: - res.fetchone.return_value = None - return res - - db.execute.side_effect = _execute - - @contextmanager - def _nested(): - yield MagicMock() - - db.begin_nested.side_effect = _nested - - with patch("app.services.scrapers.base.upsert_listing_snapshot") as m_snap: - save_listings(db, [lot]) - assert m_snap.call_count == 1 - - -# ── Tests: skip_seen_today ──────────────────────────────────────────────────── - - -def _mock_db_with_last_seen( - last_seen_at, - inserted: bool = False, - listing_id: int = 42, -) -> MagicMock: - """Mock Session где prior SELECT возвращает строку с card_hash и last_seen_at. - - last_seen_at: datetime с tzinfo (timezone-aware) или None. - """ - insert_row = MagicMock() - insert_row.id = listing_id - insert_row.inserted = inserted - - prior_row = MagicMock() - prior_row.card_hash = "somehash" - prior_row.last_seen_at = last_seen_at - - db = MagicMock() - - def _execute(sql, params=None): - s = str(sql) - res = MagicMock() - if "SELECT card_hash" in s and "WHERE dedup_hash" in s: - res.fetchone.return_value = prior_row - elif "FROM listings_snapshots" in s: - res.fetchone.return_value = None - elif "INSERT INTO listings (" in s: - res.fetchone.return_value = insert_row - else: - res.fetchone.return_value = None - return res - - db.execute.side_effect = _execute - - @contextmanager - def _nested(): - yield MagicMock() - - db.begin_nested.side_effect = _nested - return db - - -def test_skip_seen_today_skips_when_last_seen_is_today(_patch_matching): - """skip_seen_today=True + last_seen_at=сегодня по МСК → лот пропускается (ins=upd=0).""" - from datetime import datetime - from zoneinfo import ZoneInfo - - msk_tz = ZoneInfo("Europe/Moscow") - now_msk = datetime.now(msk_tz) - - db = _mock_db_with_last_seen(last_seen_at=now_msk, inserted=False) - lot = _base_lot(source_id="skip-today") - - inserted, updated = save_listings(db, [lot], skip_seen_today=True) - - assert inserted == 0 - assert updated == 0 - # INSERT INTO listings не должен вызываться - insert_calls = [ - c for c in db.execute.call_args_list if "INSERT INTO listings (" in str(c.args[0]) - ] - assert len(insert_calls) == 0 - # matching hook не должен срабатывать - assert _patch_matching["link"].call_count == 0 - - -def test_skip_seen_today_processes_when_last_seen_is_yesterday(_patch_matching): - """skip_seen_today=True + last_seen_at=вчера по МСК → лот обрабатывается.""" - from datetime import datetime, timedelta - from zoneinfo import ZoneInfo - - msk_tz = ZoneInfo("Europe/Moscow") - yesterday_msk = datetime.now(msk_tz) - timedelta(days=1) - - db = _mock_db_with_last_seen(last_seen_at=yesterday_msk, inserted=False) - lot = _base_lot(source_id="process-yesterday") - - inserted, updated = save_listings(db, [lot], skip_seen_today=True) - - # updated=1 (upsert path) - assert inserted == 0 - assert updated == 1 - - -def test_skip_seen_today_new_lot_always_inserted(_patch_matching): - """skip_seen_today=True + prior_row=None (новый листинг) → вставляется.""" - insert_row = MagicMock() - insert_row.id = 10 - insert_row.inserted = True - - db = MagicMock() - - def _execute(sql, params=None): - s = str(sql) - res = MagicMock() - if "SELECT card_hash" in s and "WHERE dedup_hash" in s: - res.fetchone.return_value = None # строки нет — новый листинг - elif "INSERT INTO listings (" in s: - res.fetchone.return_value = insert_row - else: - res.fetchone.return_value = None - return res - - db.execute.side_effect = _execute - - @contextmanager - def _nested(): - yield MagicMock() - - db.begin_nested.side_effect = _nested - - lot = _base_lot(source_id="brand-new") - - inserted, updated = save_listings(db, [lot], skip_seen_today=True) - - assert inserted == 1 - assert updated == 0 - - -def test_skip_seen_today_false_always_updates(_patch_matching): - """skip_seen_today=False (дефолт) + last_seen_at=сегодня → upsert как обычно.""" - from datetime import datetime - from zoneinfo import ZoneInfo - - msk_tz = ZoneInfo("Europe/Moscow") - now_msk = datetime.now(msk_tz) - - db = _mock_db_with_last_seen(last_seen_at=now_msk, inserted=False) - lot = _base_lot(source_id="no-skip-flag") - - # skip_seen_today не передаём (дефолт False) - inserted, updated = save_listings(db, [lot]) - - assert inserted == 0 - assert updated == 1 - - -def test_skip_seen_today_skips_snapshot_for_skipped_lot(_patch_matching): - """Пропущенный лот (seen_today) не вызывает upsert_listing_snapshot.""" - from datetime import datetime - from zoneinfo import ZoneInfo - - msk_tz = ZoneInfo("Europe/Moscow") - now_msk = datetime.now(msk_tz) - - db = _mock_db_with_last_seen(last_seen_at=now_msk, inserted=False) - lot = _base_lot(source_id="no-snap-skip") - - with patch("app.services.scrapers.base.upsert_listing_snapshot") as m_snap: - save_listings(db, [lot], skip_seen_today=True) - m_snap.assert_not_called() diff --git a/tradein-mvp/backend/tests/test_cian_newbuilding.py b/tradein-mvp/backend/tests/test_cian_newbuilding.py deleted file mode 100644 index 2364ba0b..00000000 --- a/tradein-mvp/backend/tests/test_cian_newbuilding.py +++ /dev/null @@ -1,382 +0,0 @@ -"""Tests for cian_newbuilding scraper (Stage 6).""" - -from __future__ import annotations - -import pytest - -from app.services.scrapers.cian_newbuilding import ( - NewbuildingEnrichment, - _extract_address, - _extract_advantages, - _extract_chart, - _extract_deadline_year, - _extract_nested_offers, - _extract_reliability_checks, - fetch_newbuilding, -) - -# ---- unit tests for extractors ---- - - -def test_extract_chart_primary_shape(): - """realtyValuation primary shape: data.priceDynamics.chart.data.{labels, values}.""" - rv_state = { - "data": { - "priceDynamics": { - "chart": { - "data": { - "labels": ["2025-11-01", "2025-12-01", "2026-01-01"], - "values": [13800000, 14100000, 14200000], - } - } - } - } - } - points = _extract_chart(rv_state) - assert len(points) == 3 - assert points[0]["month_date"] == "2025-11-01" - assert points[0]["price_per_sqm"] == 13800000.0 - assert points[0]["room_count"] == "all" - assert points[0]["prices_type"] == "price" - assert points[0]["period"] == "halfYear" - assert points[2]["month_date"] == "2026-01-01" - assert points[2]["price_per_sqm"] == 14200000.0 - - -def test_extract_chart_normalizes_points(): - """Fallback shape: rv_state.chart[] with {month, price, roomType, period}.""" - rv_state = { - "chart": [ - {"month": "2025-12-01", "price": 150000, "roomType": "1", "period": "monthly"}, - {"month": "2026-01-01", "price": 152000, "roomType": "1", "period": "monthly"}, - ] - } - points = _extract_chart(rv_state) - assert len(points) == 2 - assert points[0]["month_date"] == "2025-12-01" - assert points[0]["price_per_sqm"] == 150000.0 - assert points[0]["room_count"] == "1" - assert points[1]["month_date"] == "2026-01-01" - - -def test_extract_chart_handles_empty(): - assert _extract_chart({}) == [] - assert _extract_chart({"chart": None}) == [] - assert _extract_chart({"data": None}) == [] - assert _extract_chart({"data": {"priceDynamics": None}}) == [] - - -def test_extract_chart_handles_none_values(): - """Entries with None month or value are skipped.""" - rv_state = { - "chart": [ - {"month": "2025-12-01", "price": 150000}, - {"month": None, "price": 152000}, - {"month": "2026-01-01", "price": None}, - ] - } - points = _extract_chart(rv_state) - assert len(points) == 1 - assert points[0]["month_date"] == "2025-12-01" - - -def test_extract_chart_truncates_date(): - """Only first 10 chars of date string used (YYYY-MM-DD prefix).""" - rv_state = { - "chart": [ - {"month": "2025-12-01T00:00:00Z", "price": 100000}, - ] - } - points = _extract_chart(rv_state) - assert points[0]["month_date"] == "2025-12-01" - - -def test_extract_reliability_primary_shape(): - """Primary shape: {checkStatus: {status, title}, details: [{type, title, iconType}]}.""" - rel = { - "checkStatus": { - "status": "reliable", - "title": "Проверено", - "mobileTitle": "Проверено нашдом.рф", - "date": "Обновлено 5 февраля 2026", - }, - "details": [ - { - "title": "Нет в реестре проблемных объектов", - "iconType": "success", - "type": "problemHousesRegistry", - }, - { - "title": "Застройщик не банкрот", - "iconType": "success", - "type": "developerBankruptStatus", - }, - ], - "banner": {"title": "ЖК проверен нашдом.рф"}, - } - result = _extract_reliability_checks(rel) - assert len(result) == 1 - assert result[0]["check_status"] == "reliable" - assert result[0]["check_name"] == "Проверено" - assert isinstance(result[0]["details"], list) - assert len(result[0]["details"]) == 2 - assert result[0]["details"][0]["type"] == "problemHousesRegistry" - - -def test_extract_reliability_handles_empty(): - assert _extract_reliability_checks({}) == [] - assert _extract_reliability_checks({"checkStatus": None}) == [] - - -def test_extract_reliability_fallback_shape(): - """Fallback: {checks: [{name, status, details}]}.""" - rel = { - "checks": [ - {"name": "License", "status": "ok", "details": {"doc": "lic-123"}}, - {"name": "Permission", "status": "expired", "details": {}}, - ] - } - result = _extract_reliability_checks(rel) - assert len(result) == 2 - assert result[0]["check_name"] == "License" - assert result[0]["check_status"] == "ok" - assert result[1]["check_name"] == "Permission" - assert result[1]["check_status"] == "expired" - - -def test_extract_address_handles_string_and_dict(): - assert _extract_address({"fullAddress": "Екатеринбург"}) == "Екатеринбург" - assert _extract_address({"address": "ул. Ленина"}) == "ул. Ленина" - assert _extract_address({"address": {"full": "ЕКБ, ул. Ленина 1"}}) == "ЕКБ, ул. Ленина 1" - assert _extract_address({"address": {"display": "ЕКБ"}}) == "ЕКБ" - assert _extract_address({}) is None - - -def test_extract_address_prefers_full_address(): - """fullAddress wins over address.""" - nb = {"fullAddress": "Полный адрес", "address": "Краткий"} - assert _extract_address(nb) == "Полный адрес" - - -def test_extract_deadline_year_dict_or_int(): - assert _extract_deadline_year({"deadline": {"year": 2027}}) == 2027 - assert _extract_deadline_year({"buildingDeadline": {"year": 2028}}) == 2028 - assert _extract_deadline_year({"deadline": 2029}) == 2029 - assert _extract_deadline_year({}) is None - - -def test_extract_advantages_returns_dicts_only(): - nb = { - "advantages": [ - {"title": "Отделка", "advantageType": "Finish"}, - "not_a_dict", - None, - {"title": "Парковка", "advantageType": "Parking"}, - ] - } - result = _extract_advantages(nb) - assert len(result) == 2 - assert result[0]["advantageType"] == "Finish" - assert result[1]["advantageType"] == "Parking" - - -def test_extract_advantages_handles_empty(): - assert _extract_advantages({}) == [] - assert _extract_advantages({"advantages": None}) == [] - assert _extract_advantages({"advantages": "invalid"}) == [] - - -def test_extract_nested_offers_developer_rooms_shape(): - """data.total.fromDeveloperRooms[].layouts[] → flat list with room_count.""" - offers_state = { - "data": { - "total": { - "fromDeveloperRooms": [ - { - "roomType": "oneRoom", - "layouts": [ - {"offerId": 100, "price": 10000000}, - {"offerId": 101, "price": 10500000}, - ], - }, - { - "roomType": "twoRoom", - "layouts": [ - {"offerId": 200, "price": 15000000}, - ], - }, - ] - } - } - } - result = _extract_nested_offers(offers_state) - assert len(result) == 3 - assert result[0]["room_count"] == "oneRoom" - assert result[0]["offerId"] == 100 - assert result[2]["room_count"] == "twoRoom" - assert result[2]["offerId"] == 200 - - -def test_extract_nested_offers_items_shape(): - """items[] fast path.""" - offers_state = {"items": [{"offerId": 1}, {"offerId": 2}]} - result = _extract_nested_offers(offers_state) - assert len(result) == 2 - assert result[0]["offerId"] == 1 - - -def test_extract_nested_offers_handles_empty(): - assert _extract_nested_offers({}) == [] - assert _extract_nested_offers({"data": {}}) == [] - assert _extract_nested_offers({"data": {"total": {}}}) == [] - - -def test_dataclass_defaults(): - """NewbuildingEnrichment initialises with safe defaults.""" - nb = NewbuildingEnrichment() - assert nb.cian_internal_house_id is None - assert nb.realty_valuation_chart == [] - assert nb.reliability_checks == [] - assert nb.advantages == [] - assert nb.builders == [] - assert nb.banks == [] - assert nb.houses_by_turn == [] - assert nb.reviews == [] - assert nb.nested_offers == [] - assert nb.raw_state is None - assert nb.raw_sister_states == {} - - -# ---- async integration-style tests ---- -# Transport: fetch_newbuilding теперь использует BrowserFetcher (camoufox) вместо -# curl_cffi AsyncSession (#972). Мокируем BrowserFetcher.fetch через monkeypatch. - - -def _make_browser_fetcher_mock(html: str, monkeypatch: pytest.MonkeyPatch) -> None: - """Мокирует BrowserFetcher так, чтобы fetch() возвращал заданный html.""" - from unittest.mock import AsyncMock, MagicMock - - mock_fetcher = MagicMock() - mock_fetcher.fetch = AsyncMock(return_value=html) - mock_fetcher.__aenter__ = AsyncMock(return_value=mock_fetcher) - mock_fetcher.__aexit__ = AsyncMock(return_value=None) - - monkeypatch.setattr( - "app.services.scrapers.cian_newbuilding.BrowserFetcher", - lambda *a, **k: mock_fetcher, - ) - - -@pytest.mark.asyncio -async def test_fetch_newbuilding_returns_none_on_no_state(monkeypatch): - """Если state extraction fails → return None gracefully.""" - _make_browser_fetcher_mock("", monkeypatch) - monkeypatch.setattr( - "app.services.scrapers.cian_newbuilding.extract_state", - lambda html, mfe, key: None, - ) - - result = await fetch_newbuilding("https://zhk-test-ekb-i.cian.ru/") - assert result is None - - -@pytest.mark.asyncio -async def test_fetch_newbuilding_browser_error_propagates(monkeypatch): - """BrowserFetcher.fetch raises → fetch_newbuilding propagates the exception.""" - from unittest.mock import AsyncMock, MagicMock - - mock_fetcher = MagicMock() - mock_fetcher.fetch = AsyncMock(side_effect=RuntimeError("browser timeout")) - mock_fetcher.__aenter__ = AsyncMock(return_value=mock_fetcher) - mock_fetcher.__aexit__ = AsyncMock(return_value=None) - monkeypatch.setattr( - "app.services.scrapers.cian_newbuilding.BrowserFetcher", - lambda *a, **k: mock_fetcher, - ) - - with pytest.raises(RuntimeError, match="browser timeout"): - await fetch_newbuilding("https://zhk-test-ekb-i.cian.ru/") - - -@pytest.mark.asyncio -async def test_fetch_newbuilding_session_param_ignored(monkeypatch): - """session= param принимается для backward-compat, но не используется для page-fetch.""" - _make_browser_fetcher_mock("", monkeypatch) - monkeypatch.setattr( - "app.services.scrapers.cian_newbuilding.extract_state", - lambda html, mfe, key: None, - ) - - from unittest.mock import MagicMock - - sentinel_session = MagicMock(name="should_not_be_called") - # Передаём session= — должно работать без ошибок, session.get НЕ вызывается - result = await fetch_newbuilding("https://zhk-test-ekb-i.cian.ru/", session=sentinel_session) - assert result is None - sentinel_session.get.assert_not_called() - - -@pytest.mark.asyncio -async def test_fetch_newbuilding_parses_state(monkeypatch): - """Happy-path: valid state → NewbuildingEnrichment populated.""" - fake_state = { - "newbuilding": { - "id": 54016, - "name": "ЖК Тест", - "fullAddress": "Екатеринбург, ул. Теста", - "newbuildingClassName": "comfort", - "transportAccessibilityRate": 7, - "advantages": [{"title": "Парковка", "advantageType": "Parking"}], - "builders": [{"name": "Застройщик Тест"}], - "banks": [{"name": "Банк Тест"}], - "housesByTurn": [{"turn": 1, "houses": []}], - }, - "realtyValuation": { - "data": { - "priceDynamics": { - "chart": { - "data": { - "labels": ["2025-11-01", "2025-12-01"], - "values": [14000000, 14500000], - } - } - } - } - }, - "reliabilityState": { - "reliability": { - "checkStatus": {"status": "reliable", "title": "Проверено"}, - "details": [{"type": "problemHousesRegistry", "iconType": "success"}], - }, - }, - } - - _make_browser_fetcher_mock("fake", monkeypatch) - monkeypatch.setattr( - "app.services.scrapers.cian_newbuilding.extract_state", - lambda html, mfe, key: fake_state, - ) - monkeypatch.setattr( - "app.services.scrapers.cian_newbuilding.extract_all_states", - lambda html: {}, - ) - - result = await fetch_newbuilding("https://zhk-test-ekb-i.cian.ru/") - - assert result is not None - assert result.cian_internal_house_id == 54016 - assert result.name == "ЖК Тест" - assert result.address == "Екатеринбург, ул. Теста" - assert result.building_class == "comfort" - assert result.transport_accessibility_rate == 7 - assert len(result.advantages) == 1 - assert len(result.builders) == 1 - assert len(result.banks) == 1 - assert len(result.houses_by_turn) == 1 - # realtyValuation chart - assert len(result.realty_valuation_chart) == 2 - assert result.realty_valuation_chart[0]["month_date"] == "2025-11-01" - assert result.realty_valuation_chart[0]["price_per_sqm"] == 14000000.0 - # reliability - assert len(result.reliability_checks) == 1 - assert result.reliability_checks[0]["check_status"] == "reliable" diff --git a/tradein-mvp/backend/tests/test_cian_state_parser.py b/tradein-mvp/backend/tests/test_cian_state_parser.py deleted file mode 100644 index f1c15365..00000000 --- a/tradein-mvp/backend/tests/test_cian_state_parser.py +++ /dev/null @@ -1,122 +0,0 @@ -"""Tests for cian_state_parser.""" - -from app.services.scrapers.cian_state_parser import extract_state - - -def test_extract_state_simple_json(): - html = """ - - - - """ - state = extract_state(html, mfe="frontend-serp", key="initialState") - assert state is not None - assert state["results"]["offers"][0]["id"] == 1 - - -def test_extract_state_json_parse_form(): - html = r""" - window._cianConfig['frontend-offer-card'].push({ - key: 'defaultState', - value: JSON.parse("{\"offerData\":{\"offer\":{\"cianId\":123}}}"), - priority: 1 - }); - """ - state = extract_state(html, mfe="frontend-offer-card", key="defaultState") - assert state is not None - assert state["offerData"]["offer"]["cianId"] == 123 - - -def test_extract_state_missing_returns_none(): - html = "" - assert extract_state(html, mfe="frontend-serp") is None - - -def test_extract_state_wrong_mfe_returns_none(): - html = """ - window._cianConfig['other-mfe'].push({key: 'initialState', value: {}, priority: 1}); - """ - assert extract_state(html, mfe="frontend-serp", key="initialState") is None - - -def test_extract_state_backslash_in_description() -> None: - """Backslash in a listing description must not break page parsing (Part D fix). - - The hand-rolled .replace() chain was order-dependent: '\\\\' → '\\' after - '\\"' → '"' left stray backslashes that broke json.loads on the inner payload. - The fix uses json.loads on the outer quoted string atomically. - """ - # Build a JSON payload where the description contains a literal backslash. - # In the wire format (JSON.parse("...")), the inner JSON is double-escaped: - # inner JSON: {"desc": "с \\ слешем"} - # outer string: "{\"desc\": \"с \\\\ слешем\"}" - import json - - inner_payload = {"offerData": {"offer": {"cianId": 42, "description": r"описание с \ слешем"}}} - inner_json_str = json.dumps(inner_payload, ensure_ascii=False) - # Simulate what Cian emits: JSON.parse("") - outer_escaped = json.dumps(inner_json_str) # produces "\"...\"" with double escaping - html = f""" - window._cianConfig['frontend-offer-card'].push({{ - key: 'defaultState', - value: JSON.parse({outer_escaped}), - priority: 1 - }}); - """ - state = extract_state(html, mfe="frontend-offer-card", key="defaultState") - assert state is not None, "extract_state returned None — page parse failed" - assert state["offerData"]["offer"]["cianId"] == 42 - assert ( - "\\" in state["offerData"]["offer"]["description"] - ), "Backslash in description was lost during unescape" - - -# ── #639: новый формат Cian (2026-05) — .concat([{"key":..,"value":..}]) ── - - -def test_extract_state_concat_format() -> None: - """Новый формат: .concat([{...}]) с JSON-кавычками вместо .push({key:..}).""" - html = ( - "" - ) - state = extract_state(html, mfe="frontend-serp", key="initialState") - assert state is not None - assert state["results"]["offers"][0]["id"] == 7 - - -def test_extract_state_concat_bracket_in_string() -> None: - """Балансировщик массива не ломается на ']'/'[' внутри JSON-строк.""" - html = ( - "window._cianConfig['header-frontend'] = " - "(window._cianConfig['header-frontend'] || []).concat(" - '[{"key":"initialState","value":{"user":{"isAuthenticated":true,"userId":42,' - '"note":"text with ] and [ inside"}},"priority":1}]' - ");" - ) - state = extract_state(html, mfe="header-frontend", key="initialState") - assert state is not None - assert state["user"]["isAuthenticated"] is True - assert state["user"]["userId"] == 42 - - -def test_extract_state_legacy_push_still_works() -> None: - """Старый .push формат остаётся рабочим (fallback).""" - html = ( - "window._cianConfig['frontend-serp'].push({" - 'key: \'initialState\', value: {"results":{"offers":[{"id":3}]}}, priority: 1});' - ) - state = extract_state(html, mfe="frontend-serp", key="initialState") - assert state is not None - assert state["results"]["offers"][0]["id"] == 3 diff --git a/tradein-mvp/backend/tests/test_cian_valuation.py b/tradein-mvp/backend/tests/test_cian_valuation.py deleted file mode 100644 index e2df1181..00000000 --- a/tradein-mvp/backend/tests/test_cian_valuation.py +++ /dev/null @@ -1,477 +0,0 @@ -"""Tests for cian_valuation.py — Stage 7 Cian Valuation Calculator scraper.""" - -import os - -# Settings требует DATABASE_URL при инициализации (fail-fast). -# Для offline unit-тестов задаём dummy DSN до любого импорта из app. -os.environ.setdefault("DATABASE_URL", "postgresql://test:test@localhost/test_db") -os.environ.setdefault("COOKIE_ENCRYPTION_KEY", "test-key-for-tests") - -from unittest.mock import MagicMock - -import pytest - -from app.services.scrapers.cian_valuation import ( - CianValuationResult, - _is_price_sane, - _parse_change_pct, - _parse_num, - _parse_valuation_state, - compute_cache_key, - estimate_via_cian_valuation, -) - -# --------------------------------------------------------------------------- -# compute_cache_key -# --------------------------------------------------------------------------- - - -def test_cache_key_stable(): - """Нормализация address: lower + strip дают одинаковый ключ.""" - k1 = compute_cache_key("ЕКБ ул. Ленина 1", 50.0, 2, 5, "cosmetic", "sale") - k2 = compute_cache_key("екб ул. ленина 1 ", 50.0, 2, 5, "cosmetic", "sale") - assert k1 == k2 - - -def test_cache_key_distinct_on_deal_type(): - k1 = compute_cache_key("addr", 50.0, 2, 5, "cosmetic", "sale") - k2 = compute_cache_key("addr", 50.0, 2, 5, "cosmetic", "rent") - assert k1 != k2 - - -def test_cache_key_distinct_on_area(): - k1 = compute_cache_key("addr", 50.0, 2, 5, "cosmetic", "sale") - k2 = compute_cache_key("addr", 51.0, 2, 5, "cosmetic", "sale") - assert k1 != k2 - - -def test_cache_key_is_hex_string(): - k = compute_cache_key("addr", 50.0, 2, 5, "cosmetic", "sale") - assert len(k) == 64 - assert all(c in "0123456789abcdef" for c in k) - - -# --------------------------------------------------------------------------- -# _parse_num -# --------------------------------------------------------------------------- - - -def test_parse_num_float(): - assert _parse_num(5.5) == 5.5 - - -def test_parse_num_str(): - assert _parse_num("100") == 100.0 - - -def test_parse_num_none(): - assert _parse_num(None) is None - - -def test_parse_num_bad_str(): - assert _parse_num("bad") is None - - -def test_parse_num_int(): - assert _parse_num(0) == 0.0 - - -# --------------------------------------------------------------------------- -# _parse_change_pct -# --------------------------------------------------------------------------- - - -def test_parse_change_pct_positive(): - assert _parse_change_pct("8,3%") == pytest.approx(8.3) - - -def test_parse_change_pct_negative(): - assert _parse_change_pct("-2,5%") == pytest.approx(-2.5) - - -def test_parse_change_pct_empty(): - assert _parse_change_pct("") is None - - -def test_parse_change_pct_dot_format(): - assert _parse_change_pct("1.5%") == pytest.approx(1.5) - - -# --------------------------------------------------------------------------- -# _parse_valuation_state — основные сценарии -# --------------------------------------------------------------------------- - - -def _make_state( - is_authenticated: bool = True, - user_id: int = 12345, - sale_price: int = 7500000, - sale_accuracy: int = 94, - rent_price: int = 35000, - rent_accuracy: int = 93, - chart_change: str = "increase", - chart_change_value: str = "8,3%", - house_id: int = 54016, -) -> dict: - return { - "user": {"isAuthenticated": is_authenticated, "userId": user_id}, - "estimation": { - "sale": { - "data": { - "price": sale_price, - "accuracy": sale_accuracy, - "priceFrom": 6900000, - "priceTo": 8100000, - "priceSqm": 150000, - "dealType": "sale", - "filtersHash": "abc123", - }, - "isError": False, - "isFetching": False, - }, - "rent": { - "data": { - "price": rent_price, - "accuracy": rent_accuracy, - "priceFrom": 32000, - "priceTo": 38000, - "taxPrice": 1400, - "dealType": "rent", - }, - }, - }, - "estimationChart": { - "data": { - "title": { - "title": "Стоимость этой квартиры увеличилась на {{change}} за полгода", - "changeValue": chart_change_value, - "change": chart_change, - }, - "chartData": { - "maxXValue": 1777593600000, - "minXValue": 1761955200000, - "data": [ - {"date": 1761955200000, "price": 6033438, "priceFormatted": "6,0 млн ₽"}, - {"date": 1764547200000, "price": 6013573, "priceFormatted": "6,0 млн ₽"}, - {"date": 1767225600000, "price": 6116587, "priceFormatted": "6,1 млн ₽"}, - {"date": 1769904000000, "price": 6236983, "priceFormatted": "6,2 млн ₽"}, - {"date": 1772323200000, "price": 6232172, "priceFormatted": "6,2 млн ₽"}, - {"date": 1775001600000, "price": 6331112, "priceFormatted": "6,3 млн ₽"}, - {"date": 1777593600000, "price": 6532872, "priceFormatted": "6,5 млн ₽"}, - ], - }, - }, - }, - "houseInfo": { - "data": { - "items": [ - {"title": "Год постройки", "value": 2005}, - {"title": "Тип дома", "value": "Кирпичный"}, - ], - "isRenovation": False, - "isEmergency": False, - }, - }, - "filters": {"houseId": house_id, "roomsCount": 2}, - "managementCompany": { - "data": { - "name": "ТСЖ «Учителей, 18»", - "address": "ул. Учителей, 18", - "phones": ["+7 (343) 000-00-00"], - "email": "tsj@example.ru", - "chiefName": "Иванов И.И.", - "openingHours": [{"Пн–Пт": ["9:00", "18:00"]}], - }, - }, - "page": {"pageType": "valuationPage"}, - } - - -def test_parse_valuation_state_extracts_sale(): - state = _make_state() - result = _parse_valuation_state(state) - - assert result.is_authenticated is True - assert result.user_id == 12345 - assert result.sale_price_rub == 7500000.0 - assert result.sale_accuracy == 94.0 - assert result.sale_price_from == 6900000.0 - assert result.sale_price_to == 8100000.0 - assert result.sale_price_sqm == 150000.0 - assert result.filters_hash == "abc123" - - -def test_parse_valuation_state_extracts_rent(): - state = _make_state() - result = _parse_valuation_state(state) - - assert result.rent_price_rub == 35000.0 - assert result.rent_accuracy == 93.0 - assert result.rent_price_from == 32000.0 - assert result.rent_price_to == 38000.0 - assert result.rent_tax_price == 1400.0 - - -def test_parse_valuation_state_chart_7_points(): - state = _make_state() - result = _parse_valuation_state(state) - - assert len(result.chart) == 7 - # First point: unix ms 1761955200000 → 2025-11-01 (UTC) - assert result.chart[0]["month_date"] == "2025-11-01" - assert result.chart[0]["price"] == 6033438.0 - - -def test_parse_valuation_state_chart_change_increase(): - state = _make_state(chart_change="increase", chart_change_value="8,3%") - result = _parse_valuation_state(state) - - assert result.chart_change_pct == pytest.approx(8.3) - assert result.chart_change_direction == "increase" - - -def test_parse_valuation_state_chart_change_decrease(): - state = _make_state(chart_change="decrease", chart_change_value="-2,5%") - result = _parse_valuation_state(state) - - assert result.chart_change_pct == pytest.approx(-2.5) - assert result.chart_change_direction == "decrease" - - -def test_parse_valuation_state_house_info(): - state = _make_state() - result = _parse_valuation_state(state) - - assert len(result.house_info) == 2 - assert result.house_info[0]["title"] == "Год постройки" - assert result.house_info[0]["value"] == 2005 - - -def test_parse_valuation_state_external_house_id(): - state = _make_state(house_id=54016) - result = _parse_valuation_state(state) - - assert result.external_house_id == 54016 - - -def test_parse_valuation_state_management_company(): - state = _make_state() - result = _parse_valuation_state(state) - - assert result.management_company is not None - assert result.management_company["name"] == "ТСЖ «Учителей, 18»" - assert result.management_company["chiefName"] == "Иванов И.И." - - -def test_parse_valuation_state_no_auth(): - state = _make_state(is_authenticated=False) - result = _parse_valuation_state(state) - - assert result.is_authenticated is False - # Values still parsed — auth check is done by caller, not parser - assert result.sale_price_rub == 7500000.0 - - -def test_parse_valuation_state_empty_estimation(): - state = {"user": {"isAuthenticated": True, "userId": 1}, "estimation": {}} - result = _parse_valuation_state(state) - - assert result.sale_price_rub is None - assert result.rent_price_rub is None - assert result.chart == [] - - -def test_parse_valuation_state_missing_management_company(): - state = _make_state() - del state["managementCompany"] - result = _parse_valuation_state(state) - - assert result.management_company is None - - -# --------------------------------------------------------------------------- -# CianValuationResult dataclass defaults -# --------------------------------------------------------------------------- - - -def test_dataclass_defaults(): - r = CianValuationResult() - - assert r.chart == [] - assert r.house_info == [] - assert r.is_authenticated is False - assert r.sale_price_rub is None - assert r.management_company is None - - -# --------------------------------------------------------------------------- -# estimate_via_cian_valuation — integration (mocked) -# --------------------------------------------------------------------------- - - -@pytest.mark.asyncio -async def test_estimate_returns_none_when_no_cookies(monkeypatch): - """Если load_session returns None → graceful return None без crash.""" - monkeypatch.setattr( - "app.services.scrapers.cian_valuation.load_session", - lambda db: None, - ) - db = MagicMock() - # cache miss - db.execute.return_value.mappings.return_value.first.return_value = None - - result = await estimate_via_cian_valuation( - db, - address="ЕКБ", - total_area=50.0, - rooms_count=2, - floor=5, - use_cache=True, - ) - assert result is None - - -@pytest.mark.asyncio -async def test_cache_hit_returns_cached(monkeypatch): - """Если cache hit → return без fetching (load_session не вызывается).""" - db = MagicMock() - cached_row = { - "raw_payload": {"cached": True}, - "sale_price_rub": 7000000, - "sale_accuracy": 80, - "rent_price_rub": 30000, - "rent_accuracy": 65, - "chart": [{"month_date": "2026-05-01", "price": 145000}], - "chart_change_pct": 0.5, - "chart_change_direction": "increase", - "external_house_id": 999, - "filters_hash": "xyz789", - "low_price": 6500000, # required by _load_from_cache → sale_price_from - "high_price": 7500000, # required by _load_from_cache → sale_price_to - } - db.execute.return_value.mappings.return_value.first.return_value = cached_row - - # load_session должен НЕ вызываться при cache hit - monkeypatch.setattr( - "app.services.scrapers.cian_valuation.load_session", - lambda db: pytest.fail("load_session should not be called on cache hit"), - ) - - result = await estimate_via_cian_valuation( - db, - address="ЕКБ", - total_area=50.0, - rooms_count=2, - floor=5, - use_cache=True, - ) - - assert result is not None - assert result.sale_price_rub == 7000000.0 - assert result.rent_price_rub == 30000.0 - assert result.external_house_id == 999 - assert result.filters_hash == "xyz789" - assert result.is_authenticated is True # cache implies authenticated - assert len(result.chart) == 1 - - -# --------------------------------------------------------------------------- -# Mera-audit fix-1: _is_price_sane — bounds validation -# --------------------------------------------------------------------------- - - -def test_price_sane_valid_price() -> None: - """Обычная цена ЕКБ-квартиры — проходит санитарную проверку.""" - r = CianValuationResult( - sale_price_rub=7_500_000, sale_price_from=6_900_000, sale_price_to=8_100_000 - ) - assert _is_price_sane(r) is True - - -def test_price_sane_none_price() -> None: - """sale_price_rub=None — проверка цены не судит None (graceful).""" - r = CianValuationResult(sale_price_rub=None) - assert _is_price_sane(r) is True - - -def test_price_sane_below_min_returns_false() -> None: - """Цена 999_999 < дефолтный min (500_000... нет, 999_999 > 500_000). - Тест: цена ниже min=500_000 — задаём 100_000 → False.""" - r = CianValuationResult(sale_price_rub=100_000) - assert _is_price_sane(r) is False - - -def test_price_sane_above_max_returns_false() -> None: - """Цена 9_999_999_999 (> 500М) → False.""" - r = CianValuationResult(sale_price_rub=9_999_999_999) - assert _is_price_sane(r) is False - - -def test_price_sane_at_min_boundary() -> None: - """Цена ровно на нижней границе дефолта (500_000) → True.""" - r = CianValuationResult(sale_price_rub=500_000) - assert _is_price_sane(r) is True - - -def test_price_sane_at_max_boundary() -> None: - """Цена ровно на верхней границе дефолта (500_000_000) → True.""" - r = CianValuationResult(sale_price_rub=500_000_000) - assert _is_price_sane(r) is True - - -def test_price_sane_inverted_range_returns_false() -> None: - """low > high (инвертированный диапазон) → False.""" - r = CianValuationResult( - sale_price_rub=7_000_000, - sale_price_from=8_000_000, # from > to - sale_price_to=6_000_000, - ) - assert _is_price_sane(r) is False - - -def test_price_sane_negative_low_returns_false() -> None: - """sale_price_from < 0 → False.""" - r = CianValuationResult(sale_price_rub=7_000_000, sale_price_from=-100) - assert _is_price_sane(r) is False - - -def test_price_sane_negative_high_returns_false() -> None: - """sale_price_to < 0 → False.""" - r = CianValuationResult(sale_price_rub=7_000_000, sale_price_to=-1) - assert _is_price_sane(r) is False - - -def test_price_sane_none_bounds_ok() -> None: - """sale_price_from/to=None при валидной цене → True (bounds не обязательны).""" - r = CianValuationResult(sale_price_rub=7_000_000, sale_price_from=None, sale_price_to=None) - assert _is_price_sane(r) is True - - -@pytest.mark.asyncio -async def test_estimate_use_cache_false_skips_cache(monkeypatch): - """use_cache=False → cache lookup не вызывается, идём сразу в load_session.""" - load_session_called = [] - - def fake_load_session(db: object) -> None: - load_session_called.append(True) - return None # simulate no cookies → None result - - monkeypatch.setattr( - "app.services.scrapers.cian_valuation.load_session", - fake_load_session, - ) - - db = MagicMock() - result = await estimate_via_cian_valuation( - db, - address="ЕКБ", - total_area=50.0, - rooms_count=2, - floor=5, - use_cache=False, - ) - - assert result is None - assert load_session_called # load_session был вызван (cache bypass worked) - # db.execute НЕ должен быть вызван для cache lookup - db.execute.assert_not_called() diff --git a/tradein-mvp/backend/tests/test_estimator_cian_integration.py b/tradein-mvp/backend/tests/test_estimator_cian_integration.py index 86ee65f2..4d67eab4 100644 --- a/tradein-mvp/backend/tests/test_estimator_cian_integration.py +++ b/tradein-mvp/backend/tests/test_estimator_cian_integration.py @@ -1,4 +1,11 @@ -"""Tests for Cian Valuation integration in estimator.py (Stage 9).""" +"""Tests for Cian Valuation integration in estimator.py (Stage 9). + +Легаси `app.services.scrapers.cian_valuation` удалён (#2397 финальный шаг E, 0 +runtime-импортёров) — переведено на kit-эквивалент `scraper_kit.providers.cian. +valuation`, тот же тип/функции, что `estimator.py` реально использует. +`estimate_via_cian_valuation` требует mandatory `config=` (kit strangler DI, +#2337) — добавлен `RealScraperConfig()` там, где легаси-сигнатура его не требовала. +""" import os @@ -8,8 +15,7 @@ os.environ.setdefault("DATABASE_URL", "postgresql://test:test@localhost/test_db" from unittest.mock import AsyncMock, MagicMock, patch import anyio - -from app.services.scrapers.cian_valuation import CianValuationResult +from scraper_kit.providers.cian.valuation import CianValuationResult def _fake_cian_result(**kwargs) -> CianValuationResult: @@ -31,7 +37,7 @@ def _fake_cian_result(**kwargs) -> CianValuationResult: def test_cian_valuation_cache_key_deterministic() -> None: """compute_cache_key produces same hash for identical inputs.""" - from app.services.scrapers.cian_valuation import compute_cache_key + from scraper_kit.providers.cian.valuation import compute_cache_key k1 = compute_cache_key("ЕКБ, ул. Учителей, 18", 38.8, 1, 4, "cosmetic", "sale") k2 = compute_cache_key("ЕКБ, ул. Учителей, 18", 38.8, 1, 4, "cosmetic", "sale") @@ -40,7 +46,7 @@ def test_cian_valuation_cache_key_deterministic() -> None: def test_cian_valuation_cache_key_differs_for_different_inputs() -> None: - from app.services.scrapers.cian_valuation import compute_cache_key + from scraper_kit.providers.cian.valuation import compute_cache_key a = compute_cache_key("addr A", 38.8, 1, 4, "cosmetic", "sale") b = compute_cache_key("addr B", 38.8, 1, 4, "cosmetic", "sale") @@ -54,19 +60,22 @@ def test_cian_valuation_cache_key_differs_for_different_inputs() -> None: def test_estimate_via_cian_returns_none_when_no_cookies() -> None: """estimate_via_cian_valuation returns None when no cookies in DB (graceful).""" + from app.services.scraper_adapters import RealScraperConfig + db = MagicMock() # cache miss db.execute.return_value.mappings.return_value.first.return_value = None async def _run() -> None: with patch( - "app.services.scrapers.cian_valuation.load_session", + "scraper_kit.providers.cian.valuation.load_session", return_value=None, # no cookies stored ): - from app.services.scrapers.cian_valuation import estimate_via_cian_valuation + from scraper_kit.providers.cian.valuation import estimate_via_cian_valuation result = await estimate_via_cian_valuation( db, + config=RealScraperConfig(), address="ЕКБ, ул. Учителей, 18", total_area=38.8, rooms_count=1, diff --git a/tradein-mvp/backend/tests/test_extval_house_id_write_path.py b/tradein-mvp/backend/tests/test_extval_house_id_write_path.py index 2038ae30..9de56709 100644 --- a/tradein-mvp/backend/tests/test_extval_house_id_write_path.py +++ b/tradein-mvp/backend/tests/test_extval_house_id_write_path.py @@ -7,11 +7,11 @@ Best-effort: house_id=None → колонка NULL, сохранение НЕ п DB — MagicMock, записываем параметры db.execute (никакого реального Postgres, как в test_yandex_valuation_save / test_backfill_house_coords). -Легаси `app.services.scrapers.yandex_valuation` удалён (#2277 финальный шаг -scraper_kit-миграции) — `_yandex_result()` переведён на kit-эквивалент -`scraper_kit.providers.yandex.valuation` (тот же тип, что `estimator.py` реально -использует через `YandexValuationScraper`, который здесь пропатчен). `cian_valuation` -остаётся легаси (не мигрирован) — его импорт не тронут. +Легаси `app.services.scrapers.yandex_valuation` / `cian_valuation` удалены (#2277 / +#2397 финальный шаг E, 0 runtime-импортёров) — `_yandex_result()` и cian-импорты +переведены на kit-эквиваленты `scraper_kit.providers.yandex.valuation` / +`scraper_kit.providers.cian.valuation` (те же типы, что `estimator.py` реально +использует). """ from __future__ import annotations @@ -23,7 +23,7 @@ os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/ from unittest.mock import AsyncMock, MagicMock, patch -from app.services.scrapers.cian_valuation import CianValuationResult, _save_to_cache +from scraper_kit.providers.cian.valuation import CianValuationResult, _save_to_cache # --------------------------------------------------------------------------- # Cian valuation — _save_to_cache прокидывает house_id в INSERT params diff --git a/tradein-mvp/backend/tests/test_geo_precision.py b/tradein-mvp/backend/tests/test_geo_precision.py index ae5cc040..f0f2da67 100644 --- a/tradein-mvp/backend/tests/test_geo_precision.py +++ b/tradein-mvp/backend/tests/test_geo_precision.py @@ -177,7 +177,7 @@ def test_fetch_analogs_tier_h_excludes_city_precision() -> None: def test_scraped_lot_has_geo_precision_field() -> None: """ScrapedLot has geo_precision field with default None.""" - from app.services.scrapers.base import ScrapedLot + from scraper_kit.base import ScrapedLot lot = ScrapedLot( source="cian", @@ -190,7 +190,7 @@ def test_scraped_lot_has_geo_precision_field() -> None: def test_scraped_lot_accepts_city_geo_precision() -> None: """ScrapedLot accepts geo_precision='city'.""" - from app.services.scrapers.base import ScrapedLot + from scraper_kit.base import ScrapedLot lot = ScrapedLot( source="n1", diff --git a/tradein-mvp/backend/tests/test_house_type_normalizer.py b/tradein-mvp/backend/tests/test_house_type_normalizer.py deleted file mode 100644 index a52a2029..00000000 --- a/tradein-mvp/backend/tests/test_house_type_normalizer.py +++ /dev/null @@ -1,127 +0,0 @@ -"""Unit-тесты для house_type_normalizer (#2007, #2009). - -Покрывают: -- normalize_house_type(): yandex SCREAMING + cian camelCase + avito русские - лейблы вокабуляры → канон, -- идемпотентность (normalize(normalize(x)) == normalize(x)), -- None / '' / whitespace / 'other' / unknown → None. -""" - -import pytest - -from app.services.scrapers.house_type_normalizer import ( - _CANON, - normalize_house_type, -) - -# ── yandex SERP (SCREAMING_SNAKE buildingType) ─────────────────────────────── - - -@pytest.mark.parametrize( - ("raw", "expected"), - [ - ("MONOLIT", "monolith"), - ("BRICK", "brick"), - ("PANEL", "panel"), - ("MONOLIT_BRICK", "monolith_brick"), - ("BLOCK", "block"), - ("WOOD", "wood"), - ], -) -def test_yandex_screaming_vocab(raw: str, expected: str) -> None: - assert normalize_house_type(raw) == expected - - -# ── cian SERP (camelCase materialType) — reuse #2008 ───────────────────────── - - -@pytest.mark.parametrize( - ("raw", "expected"), - [ - ("monolith", "monolith"), - ("brick", "brick"), - ("panel", "panel"), - ("block", "block"), - ("wood", "wood"), - ("monolithBrick", "monolith_brick"), - ("gasSilicateBlock", "block"), - ("aerocreteBlock", "block"), - ("foamConcreteBlock", "block"), - ("stalin", "brick"), - ], -) -def test_cian_camelcase_vocab(raw: str, expected: str) -> None: - assert normalize_house_type(raw) == expected - - -# ── avito detail page («Тип дома», русские лейблы) — reuse #2009 ──────────── - - -@pytest.mark.parametrize( - ("raw", "expected"), - [ - ("монолитный", "monolith"), - ("панельный", "panel"), - ("кирпичный", "brick"), - ("монолитно-кирпичный", "monolith_brick"), - ("блочный", "block"), - ("деревянный", "wood"), - ], -) -def test_avito_russian_vocab(raw: str, expected: str) -> None: - assert normalize_house_type(raw) == expected - - -# ── регистронезависимость ──────────────────────────────────────────────────── - - -@pytest.mark.parametrize( - ("raw", "expected"), - [ - ("monolit", "monolith"), # lower yandex-token - ("Brick", "brick"), - ("MONOLITHBRICK", "monolith_brick"), - (" PANEL ", "panel"), # окружающие пробелы - ], -) -def test_case_insensitive_and_strip(raw: str, expected: str) -> None: - assert normalize_house_type(raw) == expected - - -# ── идемпотентность ────────────────────────────────────────────────────────── - - -@pytest.mark.parametrize("canon", sorted(_CANON)) -def test_canonical_passthrough(canon: str) -> None: - """Уже каноничное значение возвращается as-is (в т.ч. monolith_brick).""" - assert normalize_house_type(canon) == canon - - -@pytest.mark.parametrize( - "raw", - [ - "MONOLIT", - "MONOLIT_BRICK", - "monolithBrick", - "gasSilicateBlock", - "stalin", - "monolith_brick", - "panel", - "other", - None, - "", - ], -) -def test_idempotency(raw: str | None) -> None: - """normalize(normalize(x)) == normalize(x) для любого входа.""" - once = normalize_house_type(raw) - assert normalize_house_type(once) == once - - -# ── None / unknown → None ──────────────────────────────────────────────────── - - -@pytest.mark.parametrize("raw", [None, "", " ", "other", "OTHER", "unknown", " keramzit "]) -def test_unknown_returns_none(raw: str | None) -> None: - """Неизвестное / 'other' / пустое → None (НЕ 'other'): NULL нейтрально для estimator.""" - assert normalize_house_type(raw) is None diff --git a/tradein-mvp/backend/tests/test_pipeline_browser_routing.py b/tradein-mvp/backend/tests/test_pipeline_browser_routing.py index 48530e6b..b5f23ea9 100644 --- a/tradein-mvp/backend/tests/test_pipeline_browser_routing.py +++ b/tradein-mvp/backend/tests/test_pipeline_browser_routing.py @@ -16,6 +16,10 @@ Legacy `run_avito_pipeline` browser/curl-mode integration tests (было ран `fetch_house_catalog` ретаргетирован на `scraper_kit.providers.avito.houses` (#2397 Part E2) — legacy `app.services.scrapers.avito_houses` удалён (0 runtime-импортёров, kit — единственный живой путь). + +`fetch_detail`/`AvitoScraper`/`AvitoBlockedError` ретаргетированы на +`scraper_kit.providers.avito.detail`/`serp`/`avito_exceptions` (#2397 финальный шаг +E) — legacy `app.services.scrapers.avito*` удалён (0 runtime-импортёров). """ from __future__ import annotations @@ -30,12 +34,10 @@ import pytest os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db") -from scraper_kit.avito_exceptions import AvitoBlockedError as KitAvitoBlockedError +from scraper_kit.avito_exceptions import AvitoBlockedError +from scraper_kit.providers.avito.detail import fetch_detail from scraper_kit.providers.avito.houses import fetch_house_catalog -from app.services.scrapers.avito_detail import fetch_detail -from app.services.scrapers.avito_exceptions import AvitoBlockedError - FIXTURES = Path(__file__).parent / "fixtures" # Минимальный HTML для fetch_detail — достаточен для parse_detail_html @@ -214,7 +216,7 @@ async def test_fetch_house_catalog_browser_firewall_raises() -> None: """Firewall HTML через browser path → AvitoBlockedError (kit).""" mock_fetcher = make_mock_fetcher(FIREWALL_HTML) - with pytest.raises(KitAvitoBlockedError, match="browser-mode"): + with pytest.raises(AvitoBlockedError, match="browser-mode"): await fetch_house_catalog( "/catalog/houses/ekaterinburg/ul_test/9999", browser_fetcher=mock_fetcher, @@ -238,8 +240,10 @@ async def test_fetch_detail_curl_mode_no_browser_fetcher() -> None: mock_session.get = AsyncMock(return_value=mock_resp) mock_session.close = AsyncMock() + # Kit _build_detail_session delegates to scraper_kit.providers._base.build_document_session + # (shared helper), which constructs AsyncSession there — not in the detail module itself. with _patch( - "app.services.scrapers.avito_detail.AsyncSession", + "scraper_kit.providers._base.AsyncSession", return_value=mock_session, ): result = await fetch_detail("/ekaterinburg/kvartiry/test_99887766-1234") @@ -259,9 +263,11 @@ async def test_fetch_serp_html_routes_through_browser() -> None: SERP routing on top of #901's browser branch.""" from unittest.mock import AsyncMock - from app.services.scrapers.avito import AvitoScraper + from scraper_kit.providers.avito.serp import AvitoScraper - scraper = AvitoScraper() + from app.services.scraper_adapters import RealScraperConfig + + scraper = AvitoScraper(RealScraperConfig()) assert scraper._cffi is None scraper._browser = AsyncMock() scraper._browser.fetch = AsyncMock(return_value="real serp listing") diff --git a/tradein-mvp/backend/tests/test_repair_state_normalizer.py b/tradein-mvp/backend/tests/test_repair_state_normalizer.py deleted file mode 100644 index e134926e..00000000 --- a/tradein-mvp/backend/tests/test_repair_state_normalizer.py +++ /dev/null @@ -1,188 +0,0 @@ -"""Unit-тесты для repair_state_normalizer (#621, #622). - -Покрывают: -- normalize_repair_state(): все known raw → enum, idempotency, None-guard (#621). -- infer_repair_state_from_text(): русские фразы → enum, приоритет сигналов, - unknown → None, None-guard (#622). -""" - -import logging - -import pytest - -from app.services.scrapers.repair_state_normalizer import ( - infer_repair_state_from_text, - normalize_repair_state, -) - -# ── needs_repair ───────────────────────────────────────────────────────────── - - -@pytest.mark.parametrize("raw", ["required", "without", "no", "rough", "prefine"]) -def test_needs_repair_values(raw: str) -> None: - assert normalize_repair_state(raw) == "needs_repair" - - -def test_cian_camelcase_decoration_is_case_insensitive() -> None: - """Cian detail отдаёт offer.decoration='preFine' (camelCase) — #1791.""" - assert normalize_repair_state("preFine") == "needs_repair" - - -# ── standard ───────────────────────────────────────────────────────────────── - - -def test_cosmetic_maps_to_standard() -> None: - assert normalize_repair_state("cosmetic") == "standard" - - -# ── good ───────────────────────────────────────────────────────────────────── - - -@pytest.mark.parametrize("raw", ["euro", "fine"]) -def test_good_values(raw: str) -> None: - assert normalize_repair_state(raw) == "good" - - -# ── excellent ──────────────────────────────────────────────────────────────── - - -@pytest.mark.parametrize("raw", ["designer", "design"]) -def test_excellent_values(raw: str) -> None: - assert normalize_repair_state(raw) == "excellent" - - -# ── None guard ─────────────────────────────────────────────────────────────── - - -def test_none_returns_none() -> None: - assert normalize_repair_state(None) is None - - -# ── idempotency ────────────────────────────────────────────────────────────── - - -@pytest.mark.parametrize("enum_val", ["needs_repair", "standard", "good", "excellent"]) -def test_already_normalized_is_passthrough(enum_val: str) -> None: - """Уже нормализованное значение возвращается as-is (idempotent при ре-обработке).""" - assert normalize_repair_state(enum_val) == enum_val - - -# ── unknown value ──────────────────────────────────────────────────────────── - - -def test_unknown_raw_returns_none(caplog: pytest.LogCaptureFixture) -> None: - """Нераспознанное значение → None + WARNING в лог.""" - with caplog.at_level( - logging.WARNING, - logger="app.services.scrapers.repair_state_normalizer", - ): - result = normalize_repair_state("whitebox") - assert result is None - assert "whitebox" in caplog.text - - -# ═══════════════════════════════════════════════════════════════════════════════ -# infer_repair_state_from_text() — извлечение из описания (#622) -# ═══════════════════════════════════════════════════════════════════════════════ - - -# ── excellent ──────────────────────────────────────────────────────────────── - - -@pytest.mark.parametrize( - "text", - [ - "Сделан дизайнерский ремонт с авторской мебелью", - "Квартира с дизайнерской отделкой премиум-класса", - "Премиальный ремонт от застройщика", - "Эксклюзивный ремонт по индивидуальному проекту", - "Авторский ремонт, всё новое", - ], -) -def test_infer_excellent(text: str) -> None: - assert infer_repair_state_from_text(text) == "excellent" - - -# ── good ───────────────────────────────────────────────────────────────────── - - -@pytest.mark.parametrize( - "text", - [ - "Свежий евроремонт, заезжай и живи", - "Сделан евро-ремонт в 2024 году", - "Современный ремонт, всё в отличном состоянии — ремонт в отличном", - "Отличный ремонт, продаётся с мебелью", - "Качественный ремонт от собственника", - ], -) -def test_infer_good(text: str) -> None: - assert infer_repair_state_from_text(text) == "good" - - -# ── needs_repair ───────────────────────────────────────────────────────────── - - -@pytest.mark.parametrize( - "text", - [ - "Квартира без отделки, под чистовую отделку", - "Сдаётся без ремонта, черновая отделка", - "Черновой ремонт, нужен ремонт", - "Требует ремонта, торг уместен", - "Требуется косметический ремонт", - "Квартира под ремонт, цена снижена", - "Предчистовая отделка от застройщика", - ], -) -def test_infer_needs_repair(text: str) -> None: - assert infer_repair_state_from_text(text) == "needs_repair" - - -# ── standard ───────────────────────────────────────────────────────────────── - - -@pytest.mark.parametrize( - "text", - [ - "Косметический ремонт, чисто и уютно", - "Сделана косметическая отделка", - "Обычный ремонт, всё работает", - "Квартира в жилом состоянии", - "Хорошее состояние, заезжай и живи", - ], -) -def test_infer_standard(text: str) -> None: - assert infer_repair_state_from_text(text) == "standard" - - -# ── приоритет сигналов (порядок excellent > good > needs_repair > standard) ─── - - -def test_infer_excellent_beats_needs_repair() -> None: - """Дизайнерский ремонт после черновой → excellent, не needs_repair.""" - text = "Была черновая отделка, теперь сделан дизайнерский ремонт" - assert infer_repair_state_from_text(text) == "excellent" - - -def test_infer_good_beats_standard() -> None: - """Евроремонт + косметический → good (сильнее).""" - text = "Косметический ремонт обновлён до евроремонта" - assert infer_repair_state_from_text(text) == "good" - - -# ── None / unknown guard ───────────────────────────────────────────────────── - - -def test_infer_none_returns_none() -> None: - assert infer_repair_state_from_text(None) is None - - -def test_infer_empty_returns_none() -> None: - assert infer_repair_state_from_text("") is None - - -def test_infer_unknown_text_returns_none() -> None: - """Текст без repair-сигналов → None (не фабрикуем).""" - text = "Светлая квартира рядом с метро, развитая инфраструктура, школа во дворе" - assert infer_repair_state_from_text(text) is None diff --git a/tradein-mvp/backend/tests/test_save_listings_upsert_reconcile.py b/tradein-mvp/backend/tests/test_save_listings_upsert_reconcile.py deleted file mode 100644 index ddcb4385..00000000 --- a/tradein-mvp/backend/tests/test_save_listings_upsert_reconcile.py +++ /dev/null @@ -1,327 +0,0 @@ -"""Тесты Fix 1: save_listings — SAVEPOINT вокруг INSERT + reconcile по (source,source_id). - -Корень бага: при dedup_hash-дрейфе (контент ad изменился, новый source_id не меняется) -INSERT не находит existing row по dedup_hash → пробует свежий INSERT → UniqueViolation -по listings_source_source_id_uq (source, source_id) → весь прогон падает. - -Фикс: per-lot SAVEPOINT + IntegrityError catch + UPDATE by (source, source_id). -""" - -from __future__ import annotations - -import os -from contextlib import contextmanager -from unittest.mock import MagicMock, patch - -import psycopg.errors -import pytest -from sqlalchemy.exc import IntegrityError - -os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db") - -from app.services.scrapers.base import ScrapedLot, save_listings - -# ── Fixtures ───────────────────────────────────────────────────────────────── - - -@pytest.fixture(autouse=True) -def _patch_matching(): - """Stub matching + snapshot so tests focus on the upsert/reconcile logic.""" - with ( - patch("app.services.scrapers.base.match_or_create_house") as m_house, - patch("app.services.scrapers.base.upsert_listing_source") as m_link, - patch("app.services.scrapers.base.upsert_listing_snapshot") as m_snap, - ): - m_house.return_value = (101, 1.0, "new") - m_link.return_value = None - m_snap.return_value = None - yield {"house": m_house, "link": m_link, "snap": m_snap} - - -def _base_lot(**overrides) -> ScrapedLot: - defaults = { - "source": "avito", - "source_url": "https://www.avito.ru/ekaterinburg/test/1", - "source_id": "7960764619", - "price_rub": 5_000_000, - } - defaults.update(overrides) - return ScrapedLot(**defaults) - - -@contextmanager -def _nested_ctx(): - yield MagicMock() - - -def _mock_db_normal(inserted: bool = True, listing_id: int = 42) -> MagicMock: - """DB mock для нормального пути (нет UniqueViolation).""" - insert_row = MagicMock() - insert_row.id = listing_id - insert_row.inserted = inserted - - db = MagicMock() - - def _execute(sql, params=None): - s = str(sql) - res = MagicMock() - if "SELECT card_hash" in s and "WHERE dedup_hash" in s: - res.fetchone.return_value = None # нет prior row → fresh insert - elif "FROM listings_snapshots" in s: - res.fetchone.return_value = None - elif "INSERT INTO listings (" in s: - res.fetchone.return_value = insert_row - else: - res.fetchone.return_value = None - return res - - db.execute.side_effect = _execute - db.begin_nested.side_effect = _nested_ctx - return db - - -def _mock_db_with_unique_violation(reconcile_id: int = 77) -> MagicMock: - """DB mock: первый INSERT вызывает UniqueViolation (dedup_hash drift). - - Симулирует кейс: INSERT по новому dedup_hash → коллизия по (source, source_id). - Reconcile UPDATE by (source, source_id) → возвращает id=reconcile_id. - """ - uv_orig = psycopg.errors.UniqueViolation() - integrity_err = IntegrityError( - statement="INSERT INTO listings ...", - params={}, - orig=uv_orig, - ) - - rec_row = MagicMock() - rec_row.id = reconcile_id - - db = MagicMock() - - def _execute(sql, params=None): - s = str(sql) - res = MagicMock() - if "SELECT card_hash" in s and "WHERE dedup_hash" in s: - # prior_row=None → INSERT будет сделан (no prior dedup_hash match) - res.fetchone.return_value = None - elif "FROM listings_snapshots" in s: - res.fetchone.return_value = None - elif "INSERT INTO listings (" in s: - # Имитируем исключение внутри begin_nested() context - raise integrity_err - elif "UPDATE listings" in s and "SET dedup_hash" in s: - # reconcile UPDATE by (source, source_id) - res.fetchone.return_value = rec_row - elif "UPDATE listings" in s and "house_id_fk" in s: - res.fetchone.return_value = None - else: - res.fetchone.return_value = None - return res - - db.execute.side_effect = _execute - - # begin_nested() нужен для per-lot SAVEPOINT + snapshot + matching hook - # Используем реальный contextmanager который НЕ перехватывает IntegrityError - # (savepoint rollback симулируется тем, что db.execute после rollback'а - # уже вызывает reconcile UPDATE, а не снова INSERT). - savepoint_entered = [False] - - @contextmanager - def _nested(): - savepoint_entered[0] = True - try: - yield MagicMock() - except IntegrityError: - # SAVEPOINT rollback — пробрасываем наверх (SQLAlchemy делает rollback) - raise - - db.begin_nested.side_effect = _nested - return db - - -# ── Tests ───────────────────────────────────────────────────────────────────── - - -def test_normal_insert_path_unchanged(): - """Нормальный путь (нет UniqueViolation) → inserted=1, updated=0.""" - db = _mock_db_normal(inserted=True, listing_id=10) - lot = _base_lot() - - inserted, updated = save_listings(db, [lot]) - - assert inserted == 1 - assert updated == 0 - - -def test_normal_update_path_unchanged(): - """Нормальный update path (ON CONFLICT DO UPDATE) → inserted=0, updated=1.""" - insert_row = MagicMock() - insert_row.id = 42 - insert_row.inserted = False # xmax != 0 → update - - db = MagicMock() - - def _execute(sql, params=None): - s = str(sql) - res = MagicMock() - if "SELECT card_hash" in s and "WHERE dedup_hash" in s: - res.fetchone.return_value = None - elif "FROM listings_snapshots" in s: - res.fetchone.return_value = None - elif "INSERT INTO listings (" in s: - res.fetchone.return_value = insert_row - else: - res.fetchone.return_value = None - return res - - db.execute.side_effect = _execute - db.begin_nested.side_effect = _nested_ctx - - lot = _base_lot(source_id="upd_test") - inserted, updated = save_listings(db, [lot]) - - assert inserted == 0 - assert updated == 1 - - -def test_null_source_id_no_savepoint_needed(): - """source_id=None (yandex) → обычный INSERT, UniqueViolation не может возникнуть. - - Проверяем, что нормальный путь не изменился при source_id=None. - """ - db = _mock_db_normal(inserted=True, listing_id=99) - lot = _base_lot( - source="yandex", - source_url="https://realty.yandex.ru/offer/123/", - source_id=None, - ) - - inserted, updated = save_listings(db, [lot]) - - assert inserted == 1 - assert updated == 0 - - -def test_unique_violation_triggers_reconcile_update(caplog, _patch_matching): - """UniqueViolation по (source, source_id) → SAVEPOINT rollback → UPDATE by (s,sid). - - После reconcile UPDATE: listing_id определён, matching hook вызывается. - inserted=0, updated=0 (reconciled не входит в возврат, но лог есть). - """ - import logging - - db = _mock_db_with_unique_violation(reconcile_id=77) - lot = _base_lot(source="avito", source_id="7960764619") - - with caplog.at_level(logging.WARNING, logger="app.services.scrapers.base"): - inserted, updated = save_listings(db, [lot]) - - # Основные counters: reconcile не добавляет к inserted/updated - assert inserted == 0 - assert updated == 0 - - # Лог предупреждения о reconcile - warns = [r for r in caplog.records if "dedup_hash_drift" in r.getMessage()] - assert len(warns) == 1 - assert "7960764619" in warns[0].getMessage() - - # Matching hook должен вызваться (listing_id=77 из reconcile UPDATE) - assert _patch_matching["link"].call_count == 1 - - -def test_unique_violation_reconcile_uses_update_by_source_source_id(): - """Reconcile UPDATE содержит WHERE source = :source AND source_id = :source_id.""" - db = _mock_db_with_unique_violation(reconcile_id=55) - lot = _base_lot(source="cian", source_id="330200428") - - save_listings(db, [lot]) - - # Ищем вызов reconcile UPDATE (содержит SET dedup_hash) - reconcile_calls = [ - c - for c in db.execute.call_args_list - if "UPDATE listings" in str(c.args[0]) and "SET dedup_hash" in str(c.args[0]) - ] - assert len(reconcile_calls) == 1 - _sql, params = str(reconcile_calls[0].args[0]), reconcile_calls[0].args[1] - assert "WHERE source = :source" in _sql - assert "AND source_id = :source_id" in _sql - assert params["source"] == "cian" - assert params["source_id"] == "330200428" - - -def test_reconcile_includes_new_dedup_hash(): - """Reconcile UPDATE обновляет dedup_hash на новый (дрейфнувший) хэш.""" - db = _mock_db_with_unique_violation(reconcile_id=66) - lot = _base_lot(source="avito", source_id="7960764619", price_rub=5_500_000) - - expected_dedup = lot.compute_dedup_hash() - save_listings(db, [lot]) - - reconcile_calls = [ - c - for c in db.execute.call_args_list - if "UPDATE listings" in str(c.args[0]) and "SET dedup_hash" in str(c.args[0]) - ] - assert len(reconcile_calls) == 1 - params = reconcile_calls[0].args[1] - assert params["dedup"] == expected_dedup - - -def test_reconcile_run_does_not_crash_full_batch(_patch_matching): - """Один лот с UniqueViolation в batch из 3 → остальные 2 успешно обрабатываются. - - Ключевая проверка что один битый lot НЕ роняет прогон. - """ - rec_row = MagicMock() - rec_row.id = 77 - - ok_insert_row = MagicMock() - ok_insert_row.id = 88 - ok_insert_row.inserted = True - - uv_orig = psycopg.errors.UniqueViolation() - integrity_err = IntegrityError("INSERT", {}, uv_orig) - - def _execute(sql, params=None): - s = str(sql) - res = MagicMock() - if "SELECT card_hash" in s and "WHERE dedup_hash" in s: - res.fetchone.return_value = None - elif "FROM listings_snapshots" in s: - res.fetchone.return_value = None - elif "INSERT INTO listings (" in s: - if params and params.get("source_id") == "CONFLICT_ID": - raise integrity_err - res.fetchone.return_value = ok_insert_row - elif "UPDATE listings" in s and "SET dedup_hash" in s: - res.fetchone.return_value = rec_row - elif "UPDATE listings" in s and "house_id_fk" in s: - res.fetchone.return_value = None - else: - res.fetchone.return_value = None - return res - - db = MagicMock() - db.execute.side_effect = _execute - - @contextmanager - def _nested(): - try: - yield MagicMock() - except IntegrityError: - raise - - db.begin_nested.side_effect = _nested - - lot_conflict = _base_lot(source_id="CONFLICT_ID") - lot_ok1 = _base_lot(source_id="OK_1", source_url="https://www.avito.ru/ok1") - lot_ok2 = _base_lot(source_id="OK_2", source_url="https://www.avito.ru/ok2") - - inserted, updated = save_listings(db, [lot_conflict, lot_ok1, lot_ok2]) - - # 2 нормальных INSERT + 1 reconcile (не считается в inserted/updated) - assert inserted == 2 - assert updated == 0 - # Все 3 лота дошли до matching hook - assert _patch_matching["link"].call_count == 3 diff --git a/tradein-mvp/backend/tests/test_scraped_at_bump_2206.py b/tradein-mvp/backend/tests/test_scraped_at_bump_2206.py index 4725e76f..f5e3b247 100644 --- a/tradein-mvp/backend/tests/test_scraped_at_bump_2206.py +++ b/tradein-mvp/backend/tests/test_scraped_at_bump_2206.py @@ -7,8 +7,9 @@ эстиматору было видно лишь ~45.6% активного инвентаря (CIAN — 5.9%). Фикс: и ON CONFLICT DO UPDATE, и dedup-drift reconcile UPDATE теперь выставляют -scraped_at = NOW() рядом с last_seen_at = NOW(). Проверяем оба пути в ОБОИХ -зеркальных модулях (app.services.scrapers.base + scraper_kit.base) и свойства +scraped_at = NOW() рядом с last_seen_at = NOW(). Проверяем оба пути в +`scraper_kit.base` (единственный живой модуль — `app.services.scrapers.base` +удалён #2397 финальный шаг E, mirror-тесты через legacy убраны) и свойства ретро-бэкфилл-миграции 161. """ @@ -22,7 +23,6 @@ from typing import Any from unittest.mock import MagicMock, patch import psycopg.errors -import pytest from sqlalchemy.exc import IntegrityError os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db") @@ -30,9 +30,6 @@ os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/ from scraper_kit.base import ScrapedLot as KitLot from scraper_kit.base import save_listings as kit_save_listings -from app.services.scrapers.base import ScrapedLot as AppLot -from app.services.scrapers.base import save_listings as app_save_listings - @contextmanager def _nested_ctx() -> Any: @@ -111,59 +108,8 @@ def _find_sql(db: MagicMock, needle: str) -> str: raise AssertionError(f"SQL containing {needle!r} not found") -# ── app.services.scrapers.base ──────────────────────────────────────────────── - - -@pytest.fixture -def _patch_app_matching() -> Any: - with ( - patch("app.services.scrapers.base.match_or_create_house") as m_house, - patch("app.services.scrapers.base.upsert_listing_source") as m_link, - patch("app.services.scrapers.base.upsert_listing_snapshot") as m_snap, - ): - m_house.return_value = (101, 1.0, "new") - m_link.return_value = None - m_snap.return_value = None - yield - - -def test_app_on_conflict_bumps_scraped_at(_patch_app_matching: Any) -> None: - """ON CONFLICT DO UPDATE (ре-подтверждение) двигает scraped_at = NOW().""" - db = _mock_db_update_path(inserted=False) - lot = AppLot( - source="cian", - source_url="https://ekb.cian.ru/sale/flat/1/", - source_id="1", - price_rub=5_000_000, - ) - - inserted, updated = app_save_listings(db, [lot]) - - assert (inserted, updated) == (0, 1) - sql = _find_sql(db, "INSERT INTO listings (") - # scraped_at должен обновляться в DO UPDATE SET рядом с last_seen_at - assert "scraped_at = NOW()" in sql - assert "last_seen_at = NOW()" in sql - - -def test_app_reconcile_bumps_scraped_at(_patch_app_matching: Any) -> None: - """dedup-drift reconcile UPDATE by (source, source_id) двигает scraped_at = NOW().""" - db = _mock_db_reconcile(reconcile_id=88) - lot = AppLot( - source="avito", - source_url="https://www.avito.ru/ekaterinburg/kvartiry/1-k_1", - source_id="7960764619", - price_rub=5_000_000, - ) - - app_save_listings(db, [lot]) - - sql = _find_sql(db, "SET dedup_hash") - assert "scraped_at = NOW()" in sql - assert "last_seen_at = NOW()" in sql - - -# ── scraper_kit.base (БОЕВОЙ путь: USE_KIT_SCHEDULER=true) ───────────────────── +# ── scraper_kit.base (единственный живой путь — legacy app.services.scrapers.base +# удалён #2397 финальный шаг E) ───────────────────────────────────────────────── def _kit_matcher() -> MagicMock: diff --git a/tradein-mvp/backend/tests/test_scraper_kit_avito_golden_parity.py b/tradein-mvp/backend/tests/test_scraper_kit_avito_golden_parity.py deleted file mode 100644 index de2bea71..00000000 --- a/tradein-mvp/backend/tests/test_scraper_kit_avito_golden_parity.py +++ /dev/null @@ -1,206 +0,0 @@ -"""Golden-parity: `scraper_kit.providers.avito.*` парсинг ≡ `app.services.scrapers.avito*`. - -Strangler-инвариант (#2133): новая scraper_kit-копия avito-провайдера должна давать -БАЙТ-ИДЕНТИЧНЫЙ результат парсинга старому боевому коду на одинаковом входе. -Развязка (settings→ScraperConfig, get_scraper_delay→delay_provider, -geocoder→scraper_kit.geo, matcher→HouseMatcher) НЕ меняет распарсенные данные. - -Гоняем ОБА модуля на одних фикстурах и сравниваем результат: - - SERP `_parse_html` (DOM-карточки → list[ScrapedLot]) на avito_serp_sample.html, - при ekb_only=False и ekb_only=True (фильтр padding-карточек); - - `_extract_total_count` (счётчик выдачи); - - `_build_sort_timestamp_map` + `_unix_to_date` (sortTimeStamp → date, MSK); - - detail `parse_detail_html` (item-view HTML → DetailEnrichment); - - чистые title/address-хелперы (_clean_address, _extract_rooms/area/floor_from_title). - -Идентичность результата = kit-парсер верно скопирован (развязка не задела логику). -""" - -from __future__ import annotations - -import dataclasses -import os -from pathlib import Path -from types import SimpleNamespace -from typing import Any -from unittest.mock import patch - -# Старый app.services.scrapers.avito импортирует app.core.config.settings=Settings(), -# которому нужен DATABASE_URL. Офлайн-парсинг БД не трогает — фиктивный DSN достаточен. -os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db") - -# НОВЫЙ (scraper_kit) и СТАРЫЙ (app.services.scrapers) avito-парсеры — оба в одном -# тесте; ruff сортирует scraper_kit (first-party) выше app-импортов. -from scraper_kit.providers.avito.detail import parse_detail_html as new_parse_detail -from scraper_kit.providers.avito.serp import AvitoScraper as NewAvitoScraper -from scraper_kit.providers.avito.serp import ( - _build_sort_timestamp_map as new_build_ts_map, -) -from scraper_kit.providers.avito.serp import _clean_address as new_clean_address -from scraper_kit.providers.avito.serp import _extract_area_from_title as new_area -from scraper_kit.providers.avito.serp import _extract_floor_from_title as new_floor -from scraper_kit.providers.avito.serp import _extract_rooms_from_title as new_rooms -from scraper_kit.providers.avito.shared import _unix_to_date as new_unix_to_date - -from app.services.scrapers.avito import AvitoScraper as OldAvitoScraper -from app.services.scrapers.avito import ( - _build_sort_timestamp_map as old_build_ts_map, -) -from app.services.scrapers.avito import _clean_address as old_clean_address -from app.services.scrapers.avito import _extract_area_from_title as old_area -from app.services.scrapers.avito import _extract_floor_from_title as old_floor -from app.services.scrapers.avito import _extract_rooms_from_title as old_rooms -from app.services.scrapers.avito import _unix_to_date as old_unix_to_date -from app.services.scrapers.avito_detail import parse_detail_html as old_parse_detail - -_FIXTURES = Path(__file__).parent / "fixtures" -_SERP_HTML = (_FIXTURES / "avito_serp_sample.html").read_text(encoding="utf-8") - -# Минимальный detail item-view HTML (зеркало test_avito_detail_parse.MINIMAL_HTML). -_DETAIL_HTML = """ - -
№ 7986882804 · 18 мая в 13:40 · 1899 просмотров (+ 12 сегодня)
-11 990 000 ₽ -
-
-
    -
  • Количество комнат: 3
    • -
  • Общая площадь: 75 м²
    • -
  • Площадь кухни: 15 м²
    • -
  • Этаж: 20 из 26
    • -
  • Балкон или лоджия: лоджия
    • -
  • Ремонт: евро
    • -
-
    -
  • Тип дома: монолитный
    • -
  • Этажей в доме: 26
    • -
-
-Дом -
Отличная квартира в новом ЖК.
- -""" # noqa: E501 - -_SOURCE_URL = "https://www.avito.ru/ekaterinburg/kvartiry/test_7986882804" - - -def _make_old(ekb_only: bool) -> tuple[OldAvitoScraper, SimpleNamespace]: - """Старый AvitoScraper без DB: patch get_scraper_delay + settings.avito_serp_ekb_only.""" - with patch("app.services.scrapers.avito.get_scraper_delay", return_value=7.0): - scraper = OldAvitoScraper() - # _parse_html читает settings.avito_serp_ekb_only — подменяем на детерминированное. - scraper_settings = SimpleNamespace(avito_serp_ekb_only=ekb_only) - return scraper, scraper_settings - - -def _make_new(ekb_only: bool) -> NewAvitoScraper: - """Новый AvitoScraper с инжектируемым ScraperConfig (duck-typed namespace).""" - config = SimpleNamespace(avito_serp_ekb_only=ekb_only) - return NewAvitoScraper(config=config) - - -def _dump_lots(lots: list[Any]) -> list[dict[str, Any]]: - return [lot.model_dump() for lot in lots] - - -# ── SERP _parse_html parity ────────────────────────────────────────────────── - - -def _run_old_parse_html(html: str, base: str, ekb_only: bool) -> list[Any]: - old, old_settings = _make_old(ekb_only) - with patch("app.services.scrapers.avito.settings", old_settings): - return old._parse_html(html, base) - - -def test_serp_parse_html_parity_ekb_only_false() -> None: - """DOM-карточки → ScrapedLot идентичны (фильтр padding выключен).""" - base = "https://www.avito.ru/ekaterinburg/kvartiry" - old_lots = _run_old_parse_html(_SERP_HTML, base, ekb_only=False) - new_lots = _make_new(ekb_only=False)._parse_html(_SERP_HTML, base) - - assert len(old_lots) == len(new_lots) - assert len(new_lots) > 0, "фикстура должна содержать хотя бы одну карточку" - assert _dump_lots(old_lots) == _dump_lots(new_lots) - - -def test_serp_parse_html_parity_ekb_only_true() -> None: - """С включённым ekb_only-фильтром padding-дроп идентичен.""" - base = "https://www.avito.ru/ekaterinburg/kvartiry" - old_lots = _run_old_parse_html(_SERP_HTML, base, ekb_only=True) - new_lots = _make_new(ekb_only=True)._parse_html(_SERP_HTML, base) - - assert len(old_lots) == len(new_lots) - assert _dump_lots(old_lots) == _dump_lots(new_lots) - - -def test_serp_extract_total_count_parity() -> None: - """_extract_total_count на SERP-фикстуре идентичен.""" - old, _ = _make_old(ekb_only=False) - new = _make_new(ekb_only=False) - assert old._extract_total_count(_SERP_HTML) == new._extract_total_count(_SERP_HTML) - - -def test_build_sort_timestamp_map_parity() -> None: - """_build_sort_timestamp_map (sortTimeStamp → date, MSK) идентичен.""" - assert old_build_ts_map(_SERP_HTML) == new_build_ts_map(_SERP_HTML) - - -def test_unix_to_date_parity() -> None: - """_unix_to_date (MSK) идентичен на репрезентативных epoch-значениях.""" - for ts in [None, 0, 1756077885, 1735689600, 1700000000.0, 1_600_000_000]: - assert old_unix_to_date(ts) == new_unix_to_date(ts) - - -# ── detail parse_detail_html parity ────────────────────────────────────────── - - -def test_parse_detail_html_parity() -> None: - """parse_detail_html → DetailEnrichment идентичен (все поля).""" - old_res = old_parse_detail(_DETAIL_HTML, _SOURCE_URL) - new_res = new_parse_detail(_DETAIL_HTML, _SOURCE_URL) - assert dataclasses.asdict(old_res) == dataclasses.asdict(new_res) - - -# ── чистые title/address-хелперы ────────────────────────────────────────────── - - -def test_pure_title_helpers_parity() -> None: - titles = [ - "3-к. квартира, 75 м², 20/26 эт.", - "Квартира-студия, 24,5 м², 12/25 эт.", - "1-к. квартира, 38 м², 5/9 эт.", - "2-к. квартира, 52,3 м², 7/16 эт.", - "Комната, 18 м², 3/5 эт.", - ] - for t in titles: - assert old_rooms(t) == new_rooms(t) - assert old_area(t) == new_area(t) - assert old_floor(t) == new_floor(t) - - -def test_clean_address_parity() -> None: - addrs = [ - None, - "Екатеринбург, ул. Постовского, 17А", - "Свердловская область, Екатеринбург, Малышева, 51", - " Екатеринбург, Ленина 1 ", - ] - for a in addrs: - assert old_clean_address(a) == new_clean_address(a) - - -# ── strangler-guard: kit-провайдер не импортирует app.* ────────────────────── - - -def test_kit_avito_has_no_app_imports() -> None: - """Ни один модуль scraper_kit.providers.avito.* не импортирует app.* (развязка).""" - import inspect - - from scraper_kit.providers.avito import detail, houses, imv, serp, shared - - for mod in (serp, detail, houses, imv, shared): - source = inspect.getsource(mod) - for line in source.splitlines(): - stripped = line.strip() - assert not stripped.startswith("from app"), f"{mod.__name__}: {line!r}" - assert not stripped.startswith("import app"), f"{mod.__name__}: {line!r}" diff --git a/tradein-mvp/backend/tests/test_scraper_kit_base_golden_parity.py b/tradein-mvp/backend/tests/test_scraper_kit_base_golden_parity.py deleted file mode 100644 index 7c70152c..00000000 --- a/tradein-mvp/backend/tests/test_scraper_kit_base_golden_parity.py +++ /dev/null @@ -1,363 +0,0 @@ -"""Golden-parity: `scraper_kit.base.save_listings` ≡ `app.services.scrapers.base.save_listings`. - -Strangler-инвариант (#2132): новая scraper_kit-копия базового слоя должна давать -БАЙТ-ИДЕНТИЧНЫЙ эффект старому боевому коду на одинаковом входе. Единственное -намеренное расхождение — развязка `region_code`: старый хардкодит литерал `66` в -INSERT SQL, новый получает его параметром `:region_code` (kit не знает про регион). -При region_code=66 итоговый INSERT идентичен — что этот тест и доказывает, -нормализуя `:region_code` → `66` на новой стороне перед сравнением. - -Метод: гоняем ОБА модуля на одном списке `ScrapedLot`, перехватывая через -recording-mock всю последовательность `db.execute(sql, params)` + аргументы hook'а -матчинга. Сравниваем: - - последовательность (SQL, params) вызовов execute (INSERT / SELECT / reconcile - UPDATE / house_id_fk UPDATE) — после нормализации region_code; - - kwargs каждого вызова matcher.match_or_create_house / upsert_listing_source; - - compute_dedup_hash / compute_card_hash / compute_price_per_m2 старого и нового - класса ScrapedLot на идентичном входе. - -Фикстуры покрывают: avito (без адреса), cian (адрес+coords → house resolve → FK -UPDATE), студию rooms=0, yandex без source_id (ext_id := dedup_hash fallback) и -dedup-коллизию (INSERT → UniqueViolation → reconcile UPDATE by source,source_id). -""" - -from __future__ import annotations - -import contextlib -from typing import Any -from unittest.mock import MagicMock, patch - -import psycopg.errors -from scraper_kit.base import ScrapedLot as NewLot -from scraper_kit.base import save_listings as new_save_listings -from sqlalchemy.exc import IntegrityError - -# СТАРЫЙ (боевой) и НОВЫЙ (scraper_kit) модули — импортируем оба в одном тесте. -from app.services.scrapers.base import ScrapedLot as OldLot -from app.services.scrapers.base import save_listings as old_save_listings - -_REGION_CODE = 66 - - -# ── recording session ──────────────────────────────────────────────────────── - - -class _Row: - def __init__(self, **kw: Any) -> None: - self.__dict__.update(kw) - - -class _Result: - def __init__(self, row: _Row | None) -> None: - self._row = row - - def fetchone(self) -> _Row | None: - return self._row - - -@contextlib.contextmanager -def _nested_cm() -> Any: - yield MagicMock() - - -class RecordingSession: - """Мок Session: пишет (sql, params) всех execute, детерминированно отвечает. - - - SELECT card_hash … WHERE dedup_hash → None (prior_row отсутствует, fresh) - - INSERT INTO listings … → row(id, inserted); если source_id в - drift_ids → raise IntegrityError(UniqueViolation) (dedup-коллизия) - - reconcile UPDATE listings SET dedup_hash … → row(id=reconcile_id) - - UPDATE listings … house_id_fk / snapshot EXISTS → None - """ - - def __init__( - self, - *, - drift_ids: frozenset[str] = frozenset(), - update_ids: frozenset[str] = frozenset(), - insert_id: int = 42, - reconcile_id: int = 99, - ) -> None: - self.calls: list[tuple[str, dict[str, Any]]] = [] - self._drift = drift_ids - self._update = update_ids - self._insert_id = insert_id - self._reconcile_id = reconcile_id - - def execute(self, sql: Any, params: dict[str, Any] | None = None) -> _Result: - s = str(sql) - p = dict(params) if params else {} - self.calls.append((s, p)) - if "INSERT INTO listings (" in s: - sid = p.get("source_id") - if sid in self._drift: - raise IntegrityError("INSERT", p, psycopg.errors.UniqueViolation("dup")) - inserted = sid not in self._update - return _Result(_Row(id=self._insert_id, inserted=inserted)) - if "UPDATE listings" in s and "SET dedup_hash" in s: - return _Result(_Row(id=self._reconcile_id)) - return _Result(None) - - def begin_nested(self) -> Any: - return _nested_cm() - - def commit(self) -> None: - pass - - -# ── fixtures ───────────────────────────────────────────────────────────────── - - -def _fixture_data() -> list[dict[str, Any]]: - """Реалистичные входные dict'ы (общие для old/new ScrapedLot).""" - return [ - # 1. avito без адреса → house resolve пропускается (нет FK UPDATE) - { - "source": "avito", - "source_url": "https://www.avito.ru/ekaterinburg/kvartiry/1-k_987654?context=AB", - "source_id": "987654", - "price_rub": 4_500_000, - "area_m2": 38.0, - "rooms": 1, - "floor": 5, - "total_floors": 9, - }, - # 2. cian full: адрес + coords → house resolve → house_id_fk UPDATE - { - "source": "cian", - "source_url": "https://ekb.cian.ru/sale/flat/123456/", - "source_id": "123456", - "price_rub": 6_200_000, - "address": "Екатеринбург, улица Постовского, 17А", - "lat": 56.8385, - "lon": 60.5940, - "rooms": 2, - "area_m2": 52.0, - "floor": 7, - "total_floors": 16, - "living_area_m2": 35.0, - "bedrooms_count": 2, - "cadastral_number": "66:41:0204016:1234", - "building_cadastral_number": "66:41:0204016:100", - "phones": [{"countryCode": "7", "number": "9012345678"}], - "is_homeowner": True, - "metro_stations": [{"name": "Геологическая", "time": 10}], - "house_source": "cian", - "listing_segment": "vtorichka", - }, - # 3. студия rooms=0 (avito) с адресом → house resolve - { - "source": "avito", - "source_url": "https://www.avito.ru/ekaterinburg/kvartiry/studiya_555", - "source_id": "555", - "price_rub": 3_100_000, - "address": "Екатеринбург, Малышева 51", - "lat": 56.84, - "lon": 60.60, - "rooms": 0, - "area_m2": 24.5, - "floor": 12, - "total_floors": 25, - }, - # 4. yandex без source_id → ext_id := dedup_hash fallback - { - "source": "yandex", - "source_url": "https://realty.yandex.ru/offer/5500000000000000001/?from=serp", - "source_id": None, - "price_rub": 7_200_000, - "address": "Екатеринбург, улица Мира, 10", - "lat": 56.83, - "lon": 60.65, - "rooms": 2, - "area_m2": 48.0, - "yandex_offer_id": "5500000000000000001", - "predicted_price_rub": 7_554_000, - "price_trend": "DECREASED", - "price_previous_rub": 7_500_000, - }, - # 5. dedup-коллизия: INSERT → UniqueViolation → reconcile UPDATE - { - "source": "cian", - "source_url": "https://ekb.cian.ru/sale/flat/DRIFT-1/", - "source_id": "DRIFT-1", - "price_rub": 5_900_000, - "address": "Екатеринбург, Ленина 1", - "lat": 56.838, - "lon": 60.597, - "rooms": 3, - "area_m2": 72.0, - }, - ] - - -_DRIFT_IDS = frozenset({"DRIFT-1"}) - - -# ── run helpers ────────────────────────────────────────────────────────────── - - -def _run_old(lots: list[Any]) -> tuple[RecordingSession, MagicMock, MagicMock]: - db = RecordingSession(drift_ids=_DRIFT_IDS) - with ( - patch("app.services.scrapers.base.match_or_create_house") as m_house, - patch("app.services.scrapers.base.upsert_listing_source") as m_link, - patch("app.services.scrapers.base.upsert_listing_snapshot"), - ): - m_house.return_value = (101, 1.0, "new") - m_link.return_value = None - old_save_listings(db, lots) - return db, m_house, m_link - - -def _run_new(lots: list[Any]) -> tuple[RecordingSession, MagicMock]: - db = RecordingSession(drift_ids=_DRIFT_IDS) - matcher = MagicMock() - matcher.match_or_create_house.return_value = (101, 1.0, "new") - matcher.upsert_listing_source.return_value = None - with patch("scraper_kit.base.upsert_listing_snapshot"): - new_save_listings(db, lots, matcher=matcher, region_code=_REGION_CODE) - return db, matcher - - -def _normalize_new_calls( - calls: list[tuple[str, dict[str, Any]]], -) -> list[tuple[str, dict[str, Any]]]: - """Нормализуем расхождение region_code: `:region_code` → `66`, убираем ключ. - - Это единственное намеренное отличие новой копии; после нормализации при - region_code=66 SQL и params должны быть байт-идентичны старому боевому коду. - """ - out: list[tuple[str, dict[str, Any]]] = [] - for sql, params in calls: - norm_sql = sql.replace(":region_code", str(_REGION_CODE)) - norm_params = {k: v for k, v in params.items() if k != "region_code"} - out.append((norm_sql, norm_params)) - return out - - -def _kwargs_list(mock_or_method: Any) -> list[dict[str, Any]]: - return [c.kwargs for c in mock_or_method.call_args_list] - - -# ── tests ──────────────────────────────────────────────────────────────────── - - -def test_save_listings_execute_sequence_parity() -> None: - """Последовательность (SQL, params) execute идентична (после region_code-нормализации).""" - lots = [NewLot(**d) for d in _fixture_data()] - - old_db, _oh, _ol = _run_old(lots) - new_db, _nm = _run_new(lots) - - old_calls = old_db.calls - new_calls = _normalize_new_calls(new_db.calls) - - assert len(new_calls) == len( - old_calls - ), f"разное число execute: old={len(old_calls)} new={len(new_calls)}" - for i, (old_c, new_c) in enumerate(zip(old_calls, new_calls, strict=True)): - assert old_c[0] == new_c[0], f"SQL расходится на вызове #{i}" - assert old_c[1] == new_c[1], f"params расходятся на вызове #{i}" - - -def test_insert_sql_region_code_injection() -> None: - """Старый хардкодит `66` в VALUES; новый — параметр `:region_code`, params[region_code]=66.""" - lots = [NewLot(**_fixture_data()[0])] - - old_db, _oh, _ol = _run_old(lots) - new_db, _nm = _run_new(lots) - - old_insert = next(s for s, _ in old_db.calls if "INSERT INTO listings (" in s) - new_insert, new_params = next((s, p) for s, p in new_db.calls if "INSERT INTO listings (" in s) - - assert ", 66," in old_insert and ":region_code" not in old_insert - assert ":region_code," in new_insert and ", 66," not in new_insert - assert new_params["region_code"] == 66 - # После нормализации → идентичны - assert new_insert.replace(":region_code", "66") == old_insert - - -def test_matching_hook_kwargs_parity() -> None: - """kwargs match_or_create_house / upsert_listing_source идентичны old ↔ new.""" - lots = [NewLot(**d) for d in _fixture_data()] - - _old_db, m_house, m_link = _run_old(lots) - _new_db, matcher = _run_new(lots) - - assert _kwargs_list(m_house) == _kwargs_list(matcher.match_or_create_house) - assert _kwargs_list(m_link) == _kwargs_list(matcher.upsert_listing_source) - - # sanity: hook действительно сработал (не пустой прогон) - assert m_link.call_count == len(lots) - # house resolve только когда есть address/coords (4 из 5 фикстур) - assert m_house.call_count == 4 - assert matcher.match_or_create_house.call_count == 4 - - -def test_dedup_collision_triggers_reconcile_update_in_both() -> None: - """Drift-лот → INSERT UniqueViolation → reconcile UPDATE by (source, source_id) в обоих.""" - lots = [NewLot(**_fixture_data()[4])] # DRIFT-1 - - old_db, _oh, _ol = _run_old(lots) - new_db, _nm = _run_new(lots) - - old_reconcile = [s for s, _ in old_db.calls if "UPDATE listings" in s and "SET dedup_hash" in s] - new_reconcile = [s for s, _ in new_db.calls if "UPDATE listings" in s and "SET dedup_hash" in s] - assert len(old_reconcile) == 1 - assert len(new_reconcile) == 1 - assert old_reconcile[0] == new_reconcile[0] - - -def test_return_counts_parity() -> None: - """(inserted, updated) идентичны old ↔ new на одном входе.""" - lots = [NewLot(**d) for d in _fixture_data()] - - old_db = RecordingSession(drift_ids=_DRIFT_IDS) - with ( - patch("app.services.scrapers.base.match_or_create_house", return_value=(101, 1.0, "new")), - patch("app.services.scrapers.base.upsert_listing_source", return_value=None), - patch("app.services.scrapers.base.upsert_listing_snapshot"), - ): - old_result = old_save_listings(old_db, lots) - - new_db = RecordingSession(drift_ids=_DRIFT_IDS) - matcher = MagicMock() - matcher.match_or_create_house.return_value = (101, 1.0, "new") - matcher.upsert_listing_source.return_value = None - with patch("scraper_kit.base.upsert_listing_snapshot"): - new_result = new_save_listings(new_db, lots, matcher=matcher, region_code=_REGION_CODE) - - assert old_result == new_result - - -def test_hash_parity_old_vs_new_scraped_lot() -> None: - """compute_dedup_hash / compute_card_hash / compute_price_per_m2 идентичны у обоих классов.""" - for data in _fixture_data(): - old_lot = OldLot(**data) - new_lot = NewLot(**data) - assert old_lot.compute_dedup_hash() == new_lot.compute_dedup_hash() - assert old_lot.compute_card_hash() == new_lot.compute_card_hash() - assert old_lot.compute_price_per_m2() == new_lot.compute_price_per_m2() - - -def test_kit_base_public_api_importable() -> None: - """Acceptance: from scraper_kit.base import ScrapedLot, save_listings, BaseScraper.""" - from scraper_kit.base import BaseScraper, ScrapedLot, save_listings - - assert ScrapedLot is not None - assert callable(save_listings) - assert BaseScraper is not None - - -def test_kit_base_has_no_app_imports() -> None: - """scraper_kit.base не должен импортировать app.* (strangler-развязка).""" - import inspect - - import scraper_kit.base as kit_base - - source = inspect.getsource(kit_base) - # Игнорируем docstring-упоминания: проверяем именно import-стейтменты. - for line in source.splitlines(): - stripped = line.strip() - assert not stripped.startswith("from app"), f"app import найден: {line!r}" - assert not stripped.startswith("import app"), f"app import найден: {line!r}" diff --git a/tradein-mvp/backend/tests/test_scraper_kit_bisection_parity.py b/tradein-mvp/backend/tests/test_scraper_kit_bisection_parity.py deleted file mode 100644 index 44e1508c..00000000 --- a/tradein-mvp/backend/tests/test_scraper_kit_bisection_parity.py +++ /dev/null @@ -1,225 +0,0 @@ -"""Golden-parity бисекции: старый боевой обход `app.services.scrapers.avito` -≡ новый единый движок `scraper_kit.pricing.walk_price_range` (#2134). - -Четыре независимые копии адаптивной бисекции «комнатность × цена» были -консолидированы в один движок (`scraper_kit/pricing/bisection.py`). Провайдеры -cian/domclick/yandex_realty удалены (#2397 Part E2, 0 runtime-импортёров) — -их legacy-vs-kit секции обхода убраны отсюда вместе с ними (легаси-эталон -исчез). `avito` остаётся легаси (не мигрирован) — его parity-проверка ниже -доказывает, что unification НЕ изменил ОБХОД: при одинаковом входе -(детерминированный мок счётчика по диапазону) старая боевая реализация и -новая (движок) посещают ИДЕНТИЧНУЮ последовательность бакетов — probe-запросы, -leaf-пагинации, деления, ранние возвраты на нуле, probe-fail-ветки. - -Механика: на ОБЕ реализации (старую и новую) ставятся ОДИНАКОВЫЕ моки на -низкоуровневые методы (probe-fetch + leaf-пагинатор), которые (а) возвращают -счётчик из общего оракула `count(lo, hi)` и (б) пишут событие в общий список. -Совпадение списков событий = идентичный обход. -""" - -from __future__ import annotations - -import os -from collections.abc import Callable -from types import SimpleNamespace -from typing import Any -from unittest.mock import AsyncMock, patch - -import pytest - -os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db") - -from scraper_kit.price_brackets import get_price_seed_brackets -from scraper_kit.providers.avito.serp import AvitoScraper as NewAvito - -from app.services.scrapers.avito import AvitoScraper as OldAvito - -# ── Оракул счётчиков ───────────────────────────────────────────────────────── - -Oracle = Callable[[Any, Any], int | None] - - -def make_oracle( - *, - divisor: int, - open_ceiling: int, - zeros: frozenset[tuple[Any, Any]] = frozenset(), - nones: frozenset[tuple[Any, Any]] = frozenset(), -) -> Oracle: - """Детерминированный счётчик от (lo, hi): count = span // divisor. - - span считается по клампнутым границам (None-lo → 0, None-hi → open_ceiling), - чтобы широкие брекеты делились, а узкие уходили в leaf. `zeros` форсит 0 (ранний - возврат), `nones` форсит None (probe-fail). - """ - - def oracle(lo: Any, hi: Any) -> int | None: - if (lo, hi) in nones: - return None - if (lo, hi) in zeros: - return 0 - eff_lo = lo if lo is not None else 0 - eff_hi = hi if hi is not None else open_ceiling - span = eff_hi - eff_lo - return max(0, span // divisor) - - return oracle - - -# ── avito «сборщик событий» ────────────────────────────────────────────────── -# -# Ставит одинаковые моки на старый/новый экземпляр и возвращает список событий -# обхода. `("probe", lo, hi)` — запрос счётчика бакета; `("leaf", ...)` — -# пагинация leaf-бакета. - - -def _patch_avito(scraper: Any, oracle: Oracle, events: list[Any]) -> None: - def probe(room_slug: str, min_price: Any, max_price: Any) -> tuple[str | None, int | None]: - events.append(("probe", min_price, max_price)) - return "H", oracle(min_price, max_price) - - def leaf(**kw: Any) -> None: - events.append(("leaf", kw["lo"], kw["hi"], kw["max_pages"])) - - scraper._probe_total = AsyncMock(side_effect=probe) - scraper._paginate_leaf_bucket = AsyncMock(side_effect=leaf) - scraper._rotate_ip = AsyncMock(return_value=False) - - -# ── Конструкторы старых/новых экземпляров (offline, без DB) ─────────────────── - -_KIT_CONFIG = SimpleNamespace( - scraper_proxy_url=None, - yandex_proxy_rotate_url=None, - avito_proxy_rotate_url=None, - avito_proxy_max_rotations=0, - scraper_fetch_mode="browser", - browser_http_endpoint="http://tradein-browser:9000", - glitchtip_dsn=None, - avito_serp_ekb_only=False, -) - - -def _old_avito() -> Any: - with patch("app.services.scrapers.avito.get_scraper_delay", return_value=7.0): - return OldAvito() - - -# ── avito: probe → split → leaf ─────────────────────────────────────────────── - -_AVITO_WALK_EXTRA = {"room_slug": "room1", "room_label": "1к"} - - -@pytest.mark.asyncio -async def test_bisection_parity_avito() -> None: - """avito: split-дерево + leaf + ранний возврат на нуле — обход идентичен. - - Оракул: divisor=1000, cap=1400 → брекет 8М делится 3 уровня (8М→4М→2М→1М-leaf); - один под-брекет форсим в 0 (ранний возврат без leaf). - """ - # После первого split (0,7_999_999): mid=3_999_999 → дети (None,3_999_999) и - # (4_000_000, 7_999_999). Форсим 0 на (4_000_000, 5_999_999) — под-брекет после - # второго split правой ветки. - oracle = make_oracle( - divisor=1000, - open_ceiling=30_000_000, - zeros=frozenset({(4_000_000, 5_999_999)}), - ) - brackets: list[tuple[int, int | None]] = [(0, 7_999_999), (25_000_000, None)] - - async def run(scraper: Any) -> list[Any]: - events: list[Any] = [] - _patch_avito(scraper, oracle, events) - for lo, hi in brackets: - await scraper._walk_price_range( - lo=lo, - hi=hi, - seen={}, - price_cap_per_bucket=1400, - max_pages_per_bucket=100, - concurrency=5, - secondary_only=True, - on_bucket=None, - skip_buckets=None, - **_AVITO_WALK_EXTRA, - ) - return events - - old_events = await run(_old_avito()) - new_events = await run(NewAvito(config=_KIT_CONFIG)) - assert old_events == new_events - # Санити: обход реально нетривиален (несколько probe + несколько leaf). - assert sum(1 for e in old_events if e[0] == "leaf") >= 4 - assert sum(1 for e in old_events if e[0] == "probe") >= 6 - - -@pytest.mark.asyncio -async def test_bisection_parity_avito_probe_fail() -> None: - """avito probe-fail (total=None): SPLIT_OR_SKIP — принудительный split делимого - закрытого брекета, best-effort leaf открытого — идентичен старому.""" - # (0, 7_999_999): probe None → делим (bracket>=MIN). Дети (None,4M),(4000001,...): - # им дадим реальные счётчики. Открытый (25M, None) probe None → best-effort leaf. - oracle = make_oracle( - divisor=1000, - open_ceiling=30_000_000, - nones=frozenset({(None, 7_999_999), (25_000_000, None)}), - ) - brackets: list[tuple[int, int | None]] = [(0, 7_999_999), (25_000_000, None)] - - async def run(scraper: Any) -> list[Any]: - events: list[Any] = [] - _patch_avito(scraper, oracle, events) - for lo, hi in brackets: - await scraper._walk_price_range( - room_slug="room1", - room_label="1к", - lo=lo, - hi=hi, - seen={}, - price_cap_per_bucket=1400, - max_pages_per_bucket=100, - concurrency=5, - secondary_only=True, - on_bucket=None, - skip_buckets=None, - ) - return events - - old_events = await run(_old_avito()) - new_events = await run(NewAvito(config=_KIT_CONFIG)) - assert old_events == new_events - # Открытый брекет с probe-fail → ровно один best-effort leaf на (25M, None). - assert ("leaf", 25_000_000, None, 100) in old_events - - -# ── Full seed-grid обход (все seed-брекеты одной комнатности) ───────────────── - - -@pytest.mark.asyncio -async def test_bisection_parity_full_seed_grid() -> None: - """Полный обход всех seed-брекетов (как в fetch_all_secondary) — включая - открытый catch-all — идентичен между старым и новым для avito.""" - oracle = make_oracle(divisor=1500, open_ceiling=300_000_000) - - async def run(scraper: Any) -> list[Any]: - events: list[Any] = [] - _patch_avito(scraper, oracle, events) - for br_lo, br_hi in get_price_seed_brackets(): - walk_hi = br_hi - 1 if br_hi is not None else None - await scraper._walk_price_range( - lo=br_lo, - hi=walk_hi, - seen={}, - price_cap_per_bucket=1400, - max_pages_per_bucket=100, - concurrency=5, - secondary_only=True, - on_bucket=None, - skip_buckets=None, - **_AVITO_WALK_EXTRA, - ) - return events - - old_events = await run(_old_avito()) - new_events = await run(NewAvito(config=_KIT_CONFIG)) - assert old_events == new_events diff --git a/tradein-mvp/backend/tests/test_scraper_kit_cian_golden_parity.py b/tradein-mvp/backend/tests/test_scraper_kit_cian_golden_parity.py deleted file mode 100644 index 15378f5c..00000000 --- a/tradein-mvp/backend/tests/test_scraper_kit_cian_golden_parity.py +++ /dev/null @@ -1,369 +0,0 @@ -"""Golden-parity: `scraper_kit.providers.cian.*` парсинг ≡ ещё живым `app.services.scrapers.cian*`. - -Strangler-инвариант (#2133 cian): новая scraper_kit-копия cian-провайдера должна давать -БАЙТ-ИДЕНТИЧНЫЙ результат парсинга старому боевому коду на одинаковом входе. Развязка -(settings→ScraperConfig, get_scraper_delay→delay_provider, cian_session→локальная копия) -НЕ меняет распарсенные данные. - -`app.services.scrapers.cian` / `cian_detail` удалены (#2397 Part E2, 0 runtime- -импортёров — admin.py/cian_history_backfill.py уже на kit). Parity-тесты SERP -`_parse_serp_html`/`_offer_to_lot`/`_extract_total_offers`/`_format_address` и -detail `fetch_detail` удалены вместе с ними (легаси-эталон исчез). - -Оставшееся здесь сравнивает kit ≡ ЕЩЁ ЖИВЫЕ легаси-модули: - - `extract_state` — `app.services.scrapers.cian_state_parser` (живой, общий парсер state); - - valuation `_parse_valuation_state` / `estimate_via_cian_valuation` — - `app.services.scrapers.cian_valuation` (живой); - - newbuilding pure-экстракторы — `app.services.scrapers.cian_newbuilding` (живой); - - kit mandatory-config guards (`estimate_via_cian_valuation`, `cian.session`) — kit-only; - - strangler-guard: kit-провайдер не импортирует `app.*`. - -Идентичность результата = kit-парсер верно скопирован (развязка не задела логику). -""" - -from __future__ import annotations - -import asyncio -import dataclasses -import json -import os -from types import SimpleNamespace -from typing import Any -from unittest.mock import MagicMock - -import pytest - -# Старый app.services.scrapers.cian_state_parser/cian_valuation/cian_newbuilding -# импортируют app.core.config.settings=Settings(), которому нужен DATABASE_URL. -# Офлайн-парсинг БД не трогает — фиктивный DSN достаточен. -os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db") - -# НОВЫЙ (scraper_kit) cian-провайдер. -from scraper_kit.cian_state_parser import extract_state as new_extract_state -from scraper_kit.providers.cian.newbuilding import _extract_chart as new_extract_chart -from scraper_kit.providers.cian.newbuilding import ( - _extract_nested_offers as new_extract_nested_offers, -) -from scraper_kit.providers.cian.newbuilding import ( - _extract_reliability_checks as new_extract_reliability, -) -from scraper_kit.providers.cian.newbuilding import ( - _extract_zhk_url_from_serp as new_extract_zhk_url, -) -from scraper_kit.providers.cian.valuation import ( - _parse_valuation_state as new_parse_valuation, -) -from scraper_kit.providers.cian.valuation import ( - estimate_via_cian_valuation as new_estimate_via_cian_valuation, -) - -# #2335 (Group E2): estimate_via_cian_valuation full-pipeline parity + mandatory-config -# regression guard. RealScraperConfig — тот же read-only адаптер над settings, что и -# #2306's cian_price_history.py (см. app/services/scraper_adapters.py). -from app.services.scraper_adapters import RealScraperConfig - -# СТАРЫЙ (app.services.scrapers) — ещё живые cian-модули (cian.py/cian_detail.py удалены). -from app.services.scrapers.cian_newbuilding import _extract_chart as old_extract_chart -from app.services.scrapers.cian_newbuilding import ( - _extract_nested_offers as old_extract_nested_offers, -) -from app.services.scrapers.cian_newbuilding import ( - _extract_reliability_checks as old_extract_reliability, -) -from app.services.scrapers.cian_newbuilding import ( - _extract_zhk_url_from_serp as old_extract_zhk_url, -) -from app.services.scrapers.cian_state_parser import extract_state as old_extract_state -from app.services.scrapers.cian_valuation import ( - _parse_valuation_state as old_parse_valuation, -) -from app.services.scrapers.cian_valuation import ( - estimate_via_cian_valuation as old_estimate_via_cian_valuation, -) -from tests.support.parity import assert_parity - -# ── extract_state parity (state-парсер, живой в cian_state_parser.py) ──────── - - -def _serp_html_for_state(state: dict[str, Any]) -> str: - """Обернуть произвольный state в валидный _cianConfig['frontend-serp'].push() HTML.""" - state_json = json.dumps(state, ensure_ascii=False) - return ( - "" - ) - - -def test_extract_state_parity() -> None: - """app.extract_state ≡ scraper_kit.extract_state на SERP-фикстуре.""" - html = _serp_html_for_state({"results": {"totalOffers": 1, "offers": []}}) - old_state = old_extract_state(html, mfe="frontend-serp", key="initialState") - new_state = new_extract_state(html, mfe="frontend-serp", key="initialState") - assert old_state == new_state - assert new_state is not None - - -# ── valuation _parse_valuation_state parity (живой cian_valuation.py) ──────── - - -def _valuation_state() -> dict[str, Any]: - return { - "user": {"isAuthenticated": True, "userId": 42}, - "estimation": { - "sale": { - "data": { - "price": 6800000, - "accuracy": 87, - "priceFrom": 6400000, - "priceTo": 7200000, - "priceSqm": 130000, - "filtersHash": "hash-abc", - } - }, - "rent": { - "data": { - "price": 42000, - "accuracy": 80, - "priceFrom": 39000, - "priceTo": 45000, - "taxPrice": 5460, - } - }, - }, - "estimationChart": { - "data": { - "title": {"change": "increase", "changeValue": "8,3%"}, - "chartData": { - "data": [ - {"date": 1714521600000, "price": 6700000, "priceFormatted": "6,7 млн"}, - {"date": 1717200000000, "price": 6800000, "priceFormatted": "6,8 млн"}, - ] - }, - } - }, - "houseInfo": { - "data": { - "items": [ - {"title": "Год постройки", "value": "2010"}, - {"title": "Тип дома", "value": "монолит"}, - ] - } - }, - "filters": {"houseId": 998877}, - "managementCompany": {"data": {"name": "УК Ромашка", "phones": ["+73430000000"]}}, - } - - -def test_valuation_parse_state_parity() -> None: - """_parse_valuation_state → CianValuationResult идентичен (все поля).""" - state = _valuation_state() - old_res = old_parse_valuation(state) - new_res = new_parse_valuation(state) - assert dataclasses.asdict(old_res) == dataclasses.asdict(new_res) - - -# ── estimate_via_cian_valuation full-pipeline parity (#2335, Group E2) ─────── -# -# test_valuation_parse_state_parity above only covers the pure parsing helper. -# These tests exercise the FULL async call path (cache lookup → load_session → -# curl_cffi fetch → extract_state → auth-check → parse → sanity-check → cache -# write), with config=RealScraperConfig() wired per the #2306 convention -# (app/services/cian_price_history.py), proving the kit path is safe to switch -# to whenever #2337 wires estimator.py's call site. - - -def _valuation_mfe_html(state: dict[str, Any]) -> str: - """Wrap a valuation state dict as a valid _cianConfig push() HTML page.""" - state_json = json.dumps(state, ensure_ascii=False) - return ( - "" - ) - - -def _make_fake_async_session_cls(html: str) -> type: - """Stand-in for curl_cffi.requests.AsyncSession — returns canned HTML on .get().""" - - class _FakeAsyncSession: - def __init__(self, *args: Any, **kwargs: Any) -> None: - pass - - async def __aenter__(self) -> _FakeAsyncSession: - return self - - async def __aexit__(self, *exc: object) -> bool: - return False - - async def get(self, url: str, allow_redirects: bool = True) -> SimpleNamespace: - return SimpleNamespace(status_code=200, text=html) - - return _FakeAsyncSession - - -def test_estimate_via_cian_valuation_full_pipeline_parity( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """Full estimate_via_cian_valuation() pipeline parity — not just the pure parser. - - Mocks curl_cffi.AsyncSession (canned MFE HTML) + load_session (fixed cookies) on - BOTH legacy and kit modules, cache-miss on a MagicMock db, and drives both async - call paths with identical inputs (kit wired with config=RealScraperConfig()). - """ - html = _valuation_mfe_html(_valuation_state()) - fake_session_cls = _make_fake_async_session_cls(html) - - def _run_legacy() -> Any: - monkeypatch.setattr("app.services.scrapers.cian_valuation.AsyncSession", fake_session_cls) - monkeypatch.setattr( - "app.services.scrapers.cian_valuation.load_session", - lambda db: {"DMIR_AUTH": "token"}, - ) - db = MagicMock() - db.execute.return_value.mappings.return_value.first.return_value = None # cache miss - return asyncio.run( - old_estimate_via_cian_valuation( - db, - address="Свердловская обл, Екатеринбург", - total_area=50.0, - rooms_count=2, - floor=5, - total_floors=16, - use_cache=False, - ) - ) - - def _run_kit() -> Any: - monkeypatch.setattr("scraper_kit.providers.cian.valuation.AsyncSession", fake_session_cls) - monkeypatch.setattr( - "scraper_kit.providers.cian.valuation.load_session", - lambda db, *, config: {"DMIR_AUTH": "token"}, - ) - db = MagicMock() - db.execute.return_value.mappings.return_value.first.return_value = None # cache miss - return asyncio.run( - new_estimate_via_cian_valuation( - db, - config=RealScraperConfig(), - address="Свердловская обл, Екатеринбург", - total_area=50.0, - rooms_count=2, - floor=5, - total_floors=16, - use_cache=False, - ) - ) - - assert_parity( - legacy_fn=_run_legacy, - kit_fn=_run_kit, - fixtures=[()], - ) - - -def test_kit_estimate_via_cian_valuation_requires_config() -> None: - """Kit's estimate_via_cian_valuation MUST require config= (no default) — mandatory. - - Regression guard (#2335): config: ScraperConfig has NO default in the kit - signature, so calling it without config= raises TypeError immediately (at - argument-binding time, before any coroutine runs). This is intentional and - must stay this way: estimator.py's call site (#2337) sits inside a blanket - `except Exception` used for graceful multi-source degradation — a future - change that gave config= a default would let a wiring mistake silently - degrade into "source unavailable" instead of raising loudly at call-time. - See #2306 for the precedent this guards against (silent-degradation footgun - from an optional dependency parameter). - """ - db = MagicMock() - with pytest.raises(TypeError, match="config"): - new_estimate_via_cian_valuation( # not awaited — TypeError raises at call time - db, - address="Свердловская обл, Екатеринбург", - total_area=50.0, - rooms_count=2, - floor=5, - ) - - -def test_kit_cian_session_config_requirement_asymmetry() -> None: - """scraper_kit.providers.cian.session: load_session requires config=, but - mark_session_invalid does NOT. - - Confirmed by direct inspection for #2335 — the two helpers estimate_via_cian_valuation - calls internally are NOT symmetric re: config, unlike the assumption in the original - task brief ("the kit version of those also requires config mandatorily"). Documented - here as a regression guard so a future refactor that changes either signature is a - deliberate, visible decision rather than an accidental drift. - """ - from scraper_kit.providers.cian.session import load_session, mark_session_invalid - - db = MagicMock() - with pytest.raises(TypeError, match="config"): - load_session(db) # missing mandatory config= - - db2 = MagicMock() - mark_session_invalid(db2, 42) # no config parameter at all — must NOT raise - db2.execute.assert_called_once() - - -# ── newbuilding pure-экстракторы parity (живой cian_newbuilding.py) ────────── - - -def test_newbuilding_extractors_parity() -> None: - rv_state = { - "data": { - "priceDynamics": { - "chart": { - "data": {"labels": ["2026-01-01", "2026-02-01"], "values": [120000, 125000]} - } - } - } - } - assert old_extract_chart(rv_state) == new_extract_chart(rv_state) - - rel_state = { - "checkStatus": {"status": "reliable", "title": "Надёжный застройщик"}, - "details": [{"title": "Проектная декларация", "type": "doc"}], - } - assert old_extract_reliability(rel_state) == new_extract_reliability(rel_state) - - offers_state = { - "data": { - "total": { - "fromDeveloperRooms": [ - {"roomType": "1", "layouts": [{"id": 1, "area": 38}]}, - {"roomType": "2", "layouts": [{"id": 2, "area": 55}]}, - ] - } - } - } - assert old_extract_nested_offers(offers_state) == new_extract_nested_offers(offers_state) - - serp_html = ( - '

Купить квартиру в ' - 'ЖК «Парковый квартал»

' - ) - assert old_extract_zhk_url(serp_html) == new_extract_zhk_url(serp_html) - - -# ── strangler-guard: kit-провайдер не импортирует app.* ────────────────────── - - -def test_kit_cian_has_no_app_imports() -> None: - """Ни один модуль scraper_kit.providers.cian.* не импортирует app.* (развязка).""" - import inspect - - from scraper_kit.providers.cian import detail, newbuilding, serp, session, valuation - - for mod in (serp, detail, newbuilding, valuation, session): - source = inspect.getsource(mod) - for line in source.splitlines(): - stripped = line.strip() - assert not stripped.startswith("from app"), f"{mod.__name__}: {line!r}" - assert not stripped.startswith("import app"), f"{mod.__name__}: {line!r}" diff --git a/tradein-mvp/backend/tests/test_scraper_kit_group_c_backfill_kit_parity.py b/tradein-mvp/backend/tests/test_scraper_kit_group_c_backfill_kit_parity.py index 98d5e3f8..fc51ffd6 100644 --- a/tradein-mvp/backend/tests/test_scraper_kit_group_c_backfill_kit_parity.py +++ b/tradein-mvp/backend/tests/test_scraper_kit_group_c_backfill_kit_parity.py @@ -43,18 +43,21 @@ cian_detail.fetch_detail` vs kit, browser_fetcher call shape) removed (#2397 Par — `app.services.scrapers.cian_detail` deleted, 0 runtime importers (cian_history_backfill.py already calls the kit `fetch_detail` exclusively, proven by `test_cian_history_backfill_browser_fetcher_uses_settings_endpoint` below). + +`test_avito_fetch_detail_via_browser_fetcher_parity` (legacy `app.services.scrapers. +avito_detail.fetch_detail` vs kit, browser_fetcher call shape) removed (#2397 финальный +шаг E) — `app.services.scrapers.avito_detail` deleted, 0 runtime importers. Kit-only +avito_detail_backfill.py coverage of the browser_fetcher branch remains via +`test_pipeline_browser_routing.py::test_fetch_detail_browser_returns_enrichment`. """ from __future__ import annotations -import asyncio -import dataclasses import os from typing import Any from unittest.mock import AsyncMock, MagicMock, patch -# Legacy app.services.scrapers.* / app.core.config.settings require DATABASE_URL at -# import time (same pattern as tests/scrapers/test_avito_detail_kit_parity.py). +# app.core.config.settings requires DATABASE_URL at import time. os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db") from scraper_kit.avito_exceptions import AvitoBlockedError as KitAvitoBlockedError @@ -63,8 +66,6 @@ from scraper_kit.providers.avito.serp import AvitoScraper as KitAvitoScraper from app.core.config import settings from app.services.scraper_adapters import RealScraperConfig -from app.services.scrapers.avito_detail import fetch_detail as legacy_avito_fetch_detail -from tests.support.parity import assert_parity # ── avito fetch_detail: backconnect config-gate (avito_detail_backfill.py delta) ── @@ -149,35 +150,6 @@ def test_avito_scraper_rotate_ip_reads_injected_config() -> None: assert scraper._config.avito_proxy_rotate_url == settings.avito_proxy_rotate_url -# ── avito fetch_detail via browser_fetcher (avito_detail_backfill.py browser_mode) ── - - -def _fake_browser_fetcher(html: str) -> Any: - bf = MagicMock() - bf.fetch = AsyncMock(return_value=html) - return bf - - -def test_avito_fetch_detail_via_browser_fetcher_parity() -> None: - """browser_mode branch of avito_detail_backfill.py calls fetch_detail(url, - browser_fetcher=browser_fetcher) — config is never read on this path (early - return before the config-gated backconnect logic). Confirms legacy/kit agree.""" - html = _detail_html() - url = "https://www.avito.ru/ekaterinburg/kvartiry/test_1" - - def _run_legacy() -> Any: - return dataclasses.asdict( - asyncio.run(legacy_avito_fetch_detail(url, browser_fetcher=_fake_browser_fetcher(html))) - ) - - def _run_kit() -> Any: - return dataclasses.asdict( - asyncio.run(kit_avito_fetch_detail(url, browser_fetcher=_fake_browser_fetcher(html))) - ) - - assert_parity(legacy_fn=_run_legacy, kit_fn=_run_kit, fixtures=[()]) - - # ── BrowserFetcher(endpoint=) wiring — house_imv_backfill.py / cian_history_backfill.py ── diff --git a/tradein-mvp/backend/tests/test_scraper_kit_pricehistory_session_parity.py b/tradein-mvp/backend/tests/test_scraper_kit_pricehistory_session_parity.py deleted file mode 100644 index 6daf481e..00000000 --- a/tradein-mvp/backend/tests/test_scraper_kit_pricehistory_session_parity.py +++ /dev/null @@ -1,109 +0,0 @@ -"""Parity tests for issue #2306 (Group B of scraper_kit migration epic #2277). - -Migrates the INTERNAL legacy-scraper imports of three product-side files to their -`scraper_kit` equivalents (public API of the three files is unchanged — no caller -elsewhere in the codebase needed updating, see vault audit -`Scraper_Kit_Legacy_Dependency_Audit_0703`): - - - app/services/cian_price_history.py → fetch_detail/save_detail_enrichment now from - `scraper_kit.providers.cian.detail` (was `app.services.scrapers.cian_detail`). - - app/services/cian_session.py → extract_state now from - `scraper_kit.cian_state_parser` (was `app.services.scrapers.cian_state_parser`). - - app/services/yandex_price_history.py → ScrapedLot now from `scraper_kit.base` - (was `app.services.scrapers.base`). - -This file proves the DELTA introduced by the migration specifically — general -cian-provider / ScrapedLot golden-parity is already covered by -`tests/test_scraper_kit_cian_golden_parity.py` and -`tests/test_scraper_kit_base_golden_parity.py`. Uses `tests/support/parity.assert_parity` -per the epic's parity-gate convention (issue #2304). - -`test_fetch_detail_with_config_injection_parity` / `test_fetch_detail_own_session_proxy_ -wiring_parity` (legacy `app.services.scrapers.cian_detail.fetch_detail` vs kit) removed -(#2397 Part E2) — `cian_detail` deleted, 0 runtime importers. `extract_state` -(`cian_state_parser`, still legacy-live) and `ScrapedLot` (`base`, still legacy-live) -parity below is unaffected. -""" - -from __future__ import annotations - -import json -import os -from typing import Any - -# Старый app.services.scrapers.cian_state_parser/base / app.core.config.settings -# требует DATABASE_URL при импорте (тот же паттерн, что в -# test_scraper_kit_cian_golden_parity.py). -os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db") - -# Kit — то, на что теперь ссылаются мигрированные файлы (#2306). -from scraper_kit.base import ScrapedLot as KitScrapedLot -from scraper_kit.cian_state_parser import extract_state as kit_extract_state - -# Legacy — эталон сравнения (эти модули НЕ меняются в #2306, остаются live). -from app.services.scrapers.base import ScrapedLot as LegacyScrapedLot -from app.services.scrapers.cian_state_parser import extract_state as legacy_extract_state -from tests.support.parity import assert_parity - -# ── extract_state parity (cian_session.py dependency) ───────────────────────── - - -def _auth_state_html() -> str: - state = {"user": {"isAuthenticated": True, "userId": 42}} - state_json = json.dumps(state, ensure_ascii=False) - return ( - "" - ) - - -def test_extract_state_parity_on_match() -> None: - """cian_session.py._classify_verify_response reads this exact (mfe, key) pair.""" - html = _auth_state_html() - assert_parity( - legacy_fn=legacy_extract_state, - kit_fn=kit_extract_state, - fixtures=[(html, "header-frontend", "initialState")], - ) - - -def test_extract_state_parity_on_miss() -> None: - """Missing mfe/key → both return None identically (not just on the happy path).""" - html = _auth_state_html() - assert_parity( - legacy_fn=legacy_extract_state, - kit_fn=kit_extract_state, - fixtures=[(html, "frontend-offer-card", "defaultState")], - ) - - -# ── ScrapedLot parity (yandex_price_history.py dependency) ──────────────────── - - -def _lot_kwargs() -> dict[str, Any]: - return { - "source": "yandex", - "source_url": "https://realty.yandex.ru/offer/123/", - "source_id": "123", - "price_rub": 5_000_000, - "price_previous_rub": 5_200_000, - "price_trend": "DECREASED", - } - - -def test_scraped_lot_model_dump_parity() -> None: - """record_yandex_price_history duck-types lot.source/.source_id/.price_rub/... - - Legacy- и kit-ScrapedLot должны давать идентичный model_dump() на одном входе — - доказывает, что смена типа импорта в yandex_price_history.py не меняет runtime- - поведение (функция принимает оба класса взаимозаменяемо). - """ - kwargs = _lot_kwargs() - assert_parity( - legacy_fn=lambda kw: LegacyScrapedLot(**kw).model_dump(), - kit_fn=lambda kw: KitScrapedLot(**kw).model_dump(), - fixtures=[kwargs], - ) diff --git a/tradein-mvp/backend/tests/test_scraper_kit_yandex_golden_parity.py b/tradein-mvp/backend/tests/test_scraper_kit_yandex_golden_parity.py deleted file mode 100644 index eec7fd3f..00000000 --- a/tradein-mvp/backend/tests/test_scraper_kit_yandex_golden_parity.py +++ /dev/null @@ -1,123 +0,0 @@ -"""Golden-parity: `scraper_kit.providers.yandex.*` парсинг ≡ ещё живым -`app.services.scrapers.yandex_*`. - -Strangler-инвариант (#2133): новая scraper_kit-копия yandex-провайдера должна давать -БАЙТ-ИДЕНТИЧНЫЙ результат парсинга старому боевому коду на одинаковом входе. -Развязка (settings→ScraperConfig, get_scraper_delay→delay_provider, -house_type_normalizer→providers.yandex.shared, base/browser_fetcher/helpers→scraper_kit.*) -НЕ меняет распарсенные данные. - -`app.services.scrapers.yandex_realty` удалён (#2397 Part E2, 0 runtime-импортёров). -SERP `_entity_to_lot`/`_parse_gate_json`/`_build_url` parity-тесты (сравнение с -`YandexRealtyScraper`) удалены вместе с ним (легаси-эталон исчез). - -Оставшееся сравнивает kit ≡ ЕЩЁ ЖИВЫЕ легаси-модули: - - `normalize_house_type` — `app.services.scrapers.house_type_normalizer` (живой); - - newbuilding `parse` (ЖК-лендинг → YandexNewbuildingInfo) — - `app.services.scrapers.yandex_newbuilding` (живой); - - strangler-guard: kit-провайдер не импортирует `app.*`. - -Легаси `app.services.scrapers.yandex_detail` / `yandex_valuation` удалены (#2277 -финальный шаг scraper_kit-миграции, 0 runtime importers) — detail/valuation -parity-проверки убраны отсюда ранее. Kit-side coverage для detail/valuation -`.parse()` сохранена без изменений в `tests/test_yandex_detail.py` / -`tests/test_yandex_detail_structural.py` / `tests/test_yandex_valuation.py` / -`tests/test_yandex_valuation_save.py` (переведены на kit-эквиваленты тем же PR). - -Идентичность результата = kit-парсер верно скопирован (развязка не задела логику). -""" - -from __future__ import annotations - -import inspect -import os - -# Старый app.services.scrapers.house_type_normalizer/yandex_newbuilding импортирует -# app.core.config.settings=Settings(), которому нужен DATABASE_URL. Офлайн-парсинг -# БД не трогает — фиктивный DSN достаточен. -os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db") - -# НОВЫЙ (scraper_kit) и СТАРЫЙ (app.services.scrapers) yandex-парсеры — оба в одном -# тесте; ruff сортирует scraper_kit (first-party) выше app-импортов. -from scraper_kit.providers.yandex.newbuilding import ( - YandexNewbuildingScraper as NewNewbuildingScraper, -) -from scraper_kit.providers.yandex.shared import normalize_house_type as new_normalize_house_type - -from app.services.scrapers.house_type_normalizer import ( - normalize_house_type as old_normalize_house_type, -) -from app.services.scrapers.yandex_newbuilding import ( - YandexNewbuildingScraper as OldNewbuildingScraper, -) - -# ── normalize_house_type parity ────────────────────────────────────────────── - - -def test_normalize_house_type_parity() -> None: - """SCREAMING / camelCase / канон / мусор → идентичный результат.""" - raws = [ - None, - "", - "MONOLIT", - "BRICK", - "PANEL", - "MONOLIT_BRICK", - "monolithBrick", - "gasSilicateBlock", - "monolith_brick", - "monolith", - "stalin", - "OTHER", - " MONOLIT ", - ] - for raw in raws: - assert old_normalize_house_type(raw) == new_normalize_house_type(raw) - - -# ── newbuilding parse parity (синтетика) ───────────────────────────────────── - -_NEWBUILDING_HTML = """ - -

ЖК «Татлин»

-Екатеринбург, ул. Черепанова 4.3 из 5 1505 оценок Смотреть все 353 отзыва -56.855312 60.576668 -

О комплексе

-
Дом комфорт-класса, монолитный. Введён в эксплуатацию в июнь 2023. -35-этажные башни на участке 1,5 га. Три корпуса.
- -""" - - -def test_newbuilding_parse_parity() -> None: - """ЖК-лендинг → YandexNewbuildingInfo идентичен.""" - old_scraper = OldNewbuildingScraper() - new_scraper = NewNewbuildingScraper() - old_res = old_scraper.parse( - _NEWBUILDING_HTML, - jk_slug="tatlin", - jk_id="1592987", - source_url="https://realty.yandex.ru/ekaterinburg/kupit/novostrojka/tatlin-1592987/", - ) - new_res = new_scraper.parse( - _NEWBUILDING_HTML, - jk_slug="tatlin", - jk_id="1592987", - source_url="https://realty.yandex.ru/ekaterinburg/kupit/novostrojka/tatlin-1592987/", - ) - assert old_res.model_dump() == new_res.model_dump() - - -# ── strangler-guard: kit-провайдер не импортирует app.* ────────────────────── - - -def test_kit_yandex_has_no_app_imports() -> None: - """Ни один модуль scraper_kit.providers.yandex.* не импортирует app.* (развязка).""" - from scraper_kit.providers.yandex import detail, newbuilding, serp, shared, valuation - - for mod in (serp, detail, valuation, newbuilding, shared): - source = inspect.getsource(mod) - for line in source.splitlines(): - stripped = line.strip() - assert not stripped.startswith("from app"), f"{mod.__name__}: {line!r}" - assert not stripped.startswith("import app"), f"{mod.__name__}: {line!r}" diff --git a/tradein-mvp/backend/tests/test_scraper_proxy.py b/tradein-mvp/backend/tests/test_scraper_proxy.py index bb52143f..eb05c6aa 100644 --- a/tradein-mvp/backend/tests/test_scraper_proxy.py +++ b/tradein-mvp/backend/tests/test_scraper_proxy.py @@ -16,6 +16,15 @@ Covers: эквивалентная и более точная kit-регрессия (config footgun: с/без `config=`) уже покрыта в `tests/scrapers/test_avito_imv_kit_parity.py:: test_kit_evaluate_via_imv_{with,without}_config_{uses,drops}_proxy`. + +Легаси `app.services.scrapers.cian_newbuilding` / `avito_detail` удалены (#2397 +финальный шаг E, 0 runtime-импортёров) — CianNewbuilding/AvitoDetail proxy-wiring +тесты переведены на kit-эквиваленты (`scraper_kit.providers.cian.newbuilding` / +`scraper_kit.providers.avito.detail`). Kit `cian_newbuilding` больше не импортирует +`AsyncSession` на уровне модуля рантайма (только `TYPE_CHECKING` — страница ЖК тянется +исключительно через BrowserFetcher), поэтому patch.object(..., "AsyncSession", ...) +заменён на `hasattr`-проверку отсутствия атрибута (та же гарантия: page-fetch НЕ +использует curl_cffi AsyncSession). """ from __future__ import annotations @@ -243,13 +252,13 @@ async def test_cian_newbuilding_own_session_receives_proxies(): Проверяем: - BrowserFetcher.fetch(url) вызывается с правильным URL - - curl_cffi AsyncSession НЕ создаётся для page-fetch (прокси — browser-level) + - curl_cffi AsyncSession НЕ создаётся для page-fetch (прокси — browser-level; + kit-модуль вообще не импортирует AsyncSession на уровне рантайма) """ - from app.services.scrapers import cian_newbuilding + from scraper_kit.providers.cian import newbuilding as cian_newbuilding zhk_url = "https://zhk-test.cian.ru/" fetch_calls: list[str] = [] - session_ctor_calls: list[dict] = [] mock_fetcher = MagicMock() mock_fetcher.fetch = AsyncMock( @@ -258,18 +267,14 @@ async def test_cian_newbuilding_own_session_receives_proxies(): mock_fetcher.__aenter__ = AsyncMock(return_value=mock_fetcher) mock_fetcher.__aexit__ = AsyncMock(return_value=None) - def _fake_async_session(*args, **kwargs): - session_ctor_calls.append(kwargs) - return MagicMock() - with patch.object(cian_newbuilding, "BrowserFetcher", lambda *a, **k: mock_fetcher): - with patch.object(cian_newbuilding, "AsyncSession", _fake_async_session): - await cian_newbuilding.fetch_newbuilding(zhk_url) + await cian_newbuilding.fetch_newbuilding(zhk_url) # BrowserFetcher.fetch вызван с ЖК-url assert fetch_calls == [zhk_url] - # curl_cffi AsyncSession НЕ создавался для page-fetch - assert session_ctor_calls == [] + # curl_cffi AsyncSession НЕ создавался для page-fetch (нет такого атрибута на + # уровне рантайма — только BrowserFetcher). + assert not hasattr(cian_newbuilding, "AsyncSession") @pytest.mark.asyncio @@ -279,11 +284,10 @@ async def test_cian_newbuilding_shared_session_not_recreated(): Даже когда передан внешний session — страница ЖК тянется через BrowserFetcher, AsyncSession не пересоздаётся, session.get не вызывается. """ - from app.services.scrapers import cian_newbuilding + from scraper_kit.providers.cian import newbuilding as cian_newbuilding zhk_url = "https://zhk-test.cian.ru/" fetch_calls: list[str] = [] - session_ctor_calls: list[dict] = [] mock_fetcher = MagicMock() mock_fetcher.fetch = AsyncMock( @@ -292,20 +296,16 @@ async def test_cian_newbuilding_shared_session_not_recreated(): mock_fetcher.__aenter__ = AsyncMock(return_value=mock_fetcher) mock_fetcher.__aexit__ = AsyncMock(return_value=None) - def _fake_async_session(*args, **kwargs): - session_ctor_calls.append(kwargs) - return MagicMock() - sentinel_session = MagicMock(name="external_session") with patch.object(cian_newbuilding, "BrowserFetcher", lambda *a, **k: mock_fetcher): - with patch.object(cian_newbuilding, "AsyncSession", _fake_async_session): - await cian_newbuilding.fetch_newbuilding(zhk_url, session=sentinel_session) + await cian_newbuilding.fetch_newbuilding(zhk_url, session=sentinel_session) # BrowserFetcher.fetch вызван — именно через него тянется страница assert fetch_calls == [zhk_url] - # Ни нового AsyncSession, ни вызова переданного session.get — proxy browser-level - assert session_ctor_calls == [] + # Ни нового AsyncSession (атрибута вообще нет на рантайме), ни вызова переданного + # session.get — proxy browser-level + assert not hasattr(cian_newbuilding, "AsyncSession") sentinel_session.get.assert_not_called() @@ -315,7 +315,7 @@ async def test_cian_newbuilding_shared_session_not_recreated(): @pytest.mark.asyncio async def test_avito_detail_own_session_receives_proxies(): """fetch_detail creates own session with proxies= when no cffi_session passed.""" - from app.services.scrapers import avito_detail + from scraper_kit.providers.avito import detail as avito_detail mock_session = MagicMock() mock_resp = MagicMock() @@ -331,12 +331,15 @@ async def test_avito_detail_own_session_receives_proxies(): captured_kwargs.update(kwargs) return mock_session - with patch.object(avito_detail, "AsyncSession", _fake_session): - with patch("app.core.config.settings", _mock_settings(_PROXY_URL)): - try: - await avito_detail.fetch_detail("/ekaterinburg/kvartiry/test-1234") - except ValueError: - pass # parse_detail_html raises ValueError (no item_id) — that's ok + # Kit _build_detail_session delegates to scraper_kit.providers._base.build_document_session + # (shared helper), which constructs AsyncSession there — not in the detail module itself. + # config= (strangler DI, #2133) replaces the old app.core.config.settings patch. + cfg = SimpleNamespace(scraper_proxy_url=_PROXY_URL) + with patch("scraper_kit.providers._base.AsyncSession", _fake_session): + try: + await avito_detail.fetch_detail("/ekaterinburg/kvartiry/test-1234", config=cfg) + except ValueError: + pass # parse_detail_html raises ValueError (no item_id) — that's ok assert captured_kwargs.get("proxies") == _EXPECTED_PROXIES @@ -344,7 +347,7 @@ async def test_avito_detail_own_session_receives_proxies(): @pytest.mark.asyncio async def test_avito_detail_shared_session_not_patched(): """fetch_detail does NOT build own session when cffi_session is provided.""" - from app.services.scrapers import avito_detail + from scraper_kit.providers.avito import detail as avito_detail mock_session = MagicMock() mock_resp = MagicMock() @@ -359,15 +362,16 @@ async def test_avito_detail_shared_session_not_patched(): ctor_calls.append(kwargs) return MagicMock() - with patch.object(avito_detail, "AsyncSession", _fake_session): - with patch("app.core.config.settings", _mock_settings(_PROXY_URL)): - try: - await avito_detail.fetch_detail( - "/ekaterinburg/kvartiry/test-1234", - cffi_session=mock_session, - ) - except ValueError: - pass + cfg = SimpleNamespace(scraper_proxy_url=_PROXY_URL) + with patch("scraper_kit.providers._base.AsyncSession", _fake_session): + try: + await avito_detail.fetch_detail( + "/ekaterinburg/kvartiry/test-1234", + cffi_session=mock_session, + config=cfg, + ) + except ValueError: + pass # No own session created — shared session was used as-is assert ctor_calls == [] diff --git a/tradein-mvp/backend/tests/test_snapshot_writer.py b/tradein-mvp/backend/tests/test_snapshot_writer.py index 8c023c64..df1aafde 100644 --- a/tradein-mvp/backend/tests/test_snapshot_writer.py +++ b/tradein-mvp/backend/tests/test_snapshot_writer.py @@ -12,10 +12,15 @@ `app.services.scrapers.cian_detail` удалён (#2397 Part E2, 0 runtime-импортёров) — `DetailEnrichment`/`save_detail_enrichment` ретаргетированы на -`scraper_kit.providers.cian.detail` (byte-identical strangler-копия; `snapshot_writer` -тестируется по-прежнему через живой `app.services.scrapers.snapshot_writer` — -kit и app-версии этого модуля идентичны, kit's `save_detail_enrichment` импортирует -`upsert_listing_snapshot` из `scraper_kit.snapshot_writer`). +`scraper_kit.providers.cian.detail` (byte-identical strangler-копия). + +`app.services.scrapers.base`/`snapshot_writer` удалены (#2397 финальный шаг E, 0 +runtime-импортёров) — `upsert_listing_snapshot`/`ScrapedLot`/`save_listings` +переведены на `scraper_kit.snapshot_writer`/`scraper_kit.base`. Kit `save_listings` +принимает инжектируемый `matcher: HouseMatcher` (mandatory kwarg, strangler DI +#2133) вместо прямого импорта `match_or_create_house`/`upsert_listing_source` — +тесты используют `MagicMock()` matcher (тот же паттерн, что +`test_scraped_at_bump_2206.py::_kit_matcher`), плюс mandatory `region_code=66`. """ from __future__ import annotations @@ -24,9 +29,18 @@ from contextlib import contextmanager from datetime import date from unittest.mock import MagicMock, patch +from scraper_kit.base import ScrapedLot, save_listings from scraper_kit.providers.cian.detail import DetailEnrichment, save_detail_enrichment +from scraper_kit.snapshot_writer import upsert_listing_snapshot + + +def _mock_matcher() -> MagicMock: + """MagicMock HouseMatcher — mirror test_scraped_at_bump_2206.py::_kit_matcher.""" + matcher = MagicMock() + matcher.match_or_create_house.return_value = (1, 1.0, "new") + matcher.upsert_listing_source.return_value = None + return matcher -from app.services.scrapers.snapshot_writer import upsert_listing_snapshot # ── Helpers ─────────────────────────────────────────────────────────────────── @@ -321,7 +335,6 @@ def _mock_db_for_save_listings( def test_save_listings_writes_snapshot_per_lot(): """save_listings вызывает upsert_listing_snapshot для каждого сохранённого lot.""" - from app.services.scrapers.base import ScrapedLot, save_listings db = _mock_db_for_save_listings(inserted=True, listing_id=77) lots = [ @@ -339,12 +352,8 @@ def test_save_listings_writes_snapshot_per_lot(): ), ] - with ( - patch("app.services.scrapers.base.match_or_create_house", return_value=(1, 1.0, "new")), - patch("app.services.scrapers.base.upsert_listing_source"), - patch("app.services.scrapers.base.upsert_listing_snapshot") as mock_snap, - ): - save_listings(db, lots, run_id=10) + with patch("scraper_kit.base.upsert_listing_snapshot") as mock_snap: + save_listings(db, lots, matcher=_mock_matcher(), region_code=66, run_id=10) assert mock_snap.call_count == 2, "snapshot должен вызваться для каждого lot" # Первый вызов @@ -357,7 +366,6 @@ def test_save_listings_writes_snapshot_per_lot(): def test_save_listings_snapshot_receives_run_id(): """Если передан run_id — он пробрасывается в upsert_listing_snapshot.""" - from app.services.scrapers.base import ScrapedLot, save_listings db = _mock_db_for_save_listings(listing_id=88) lot = ScrapedLot( @@ -367,12 +375,8 @@ def test_save_listings_snapshot_receives_run_id(): price_rub=3_000_000, ) - with ( - patch("app.services.scrapers.base.match_or_create_house", return_value=(1, 1.0, "new")), - patch("app.services.scrapers.base.upsert_listing_source"), - patch("app.services.scrapers.base.upsert_listing_snapshot") as mock_snap, - ): - save_listings(db, [lot], run_id=55) + with patch("scraper_kit.base.upsert_listing_snapshot") as mock_snap: + save_listings(db, [lot], matcher=_mock_matcher(), region_code=66, run_id=55) kwargs = mock_snap.call_args.kwargs assert kwargs["run_id"] == 55 @@ -380,7 +384,6 @@ def test_save_listings_snapshot_receives_run_id(): def test_save_listings_snapshot_none_run_id_by_default(): """Без run_id — upsert_listing_snapshot получает run_id=None.""" - from app.services.scrapers.base import ScrapedLot, save_listings db = _mock_db_for_save_listings(listing_id=9) lot = ScrapedLot( @@ -390,12 +393,8 @@ def test_save_listings_snapshot_none_run_id_by_default(): price_rub=2_000_000, ) - with ( - patch("app.services.scrapers.base.match_or_create_house", return_value=(1, 1.0, "new")), - patch("app.services.scrapers.base.upsert_listing_source"), - patch("app.services.scrapers.base.upsert_listing_snapshot") as mock_snap, - ): - save_listings(db, [lot]) # no run_id + with patch("scraper_kit.base.upsert_listing_snapshot") as mock_snap: + save_listings(db, [lot], matcher=_mock_matcher(), region_code=66) # no run_id kwargs = mock_snap.call_args.kwargs assert kwargs["run_id"] is None @@ -403,7 +402,6 @@ def test_save_listings_snapshot_none_run_id_by_default(): def test_save_listings_snapshot_failure_does_not_abort_batch(): """Ошибка snapshot не прерывает основной INSERT; все lots сохраняются.""" - from app.services.scrapers.base import ScrapedLot, save_listings db = _mock_db_for_save_listings(inserted=True, listing_id=5) lots = [ @@ -416,15 +414,11 @@ def test_save_listings_snapshot_failure_does_not_abort_batch(): for i in range(1, 4) ] - with ( - patch("app.services.scrapers.base.match_or_create_house", return_value=(1, 1.0, "new")), - patch("app.services.scrapers.base.upsert_listing_source"), - patch( - "app.services.scrapers.base.upsert_listing_snapshot", - side_effect=RuntimeError("snapshot fail"), - ), + with patch( + "scraper_kit.base.upsert_listing_snapshot", + side_effect=RuntimeError("snapshot fail"), ): - inserted, _updated = save_listings(db, lots) + inserted, _updated = save_listings(db, lots, matcher=_mock_matcher(), region_code=66) # INSERT listings всё равно прошёл для всех 3 lots assert inserted == 3 @@ -432,12 +426,11 @@ def test_save_listings_snapshot_failure_does_not_abort_batch(): def test_save_listings_empty_no_snapshot_call(): """Пустой список → upsert_listing_snapshot не вызывается.""" - from app.services.scrapers.base import save_listings db = _mock_db_for_save_listings() - with patch("app.services.scrapers.base.upsert_listing_snapshot") as mock_snap: - save_listings(db, []) + with patch("scraper_kit.base.upsert_listing_snapshot") as mock_snap: + save_listings(db, [], matcher=_mock_matcher(), region_code=66) mock_snap.assert_not_called() diff --git a/tradein-mvp/backend/tests/test_yandex_helpers.py b/tradein-mvp/backend/tests/test_yandex_helpers.py deleted file mode 100644 index 87dc6504..00000000 --- a/tradein-mvp/backend/tests/test_yandex_helpers.py +++ /dev/null @@ -1,271 +0,0 @@ -"""Unit tests for Yandex Realty parsing helpers.""" - -from datetime import date - -from app.services.scrapers.yandex_helpers import ( - RE_FLOOR, - RE_JK_ID, - RE_OFFER_ID, - RE_TITLE_AREA, - RE_TITLE_ROOMS, - extract_json_ld, - find_ld_by_type, - parse_dmy, - parse_house_class, - parse_house_type, - parse_ru_date, - parse_rub, -) - - -# --- house_type --- -def test_parse_house_type_panel(): - assert parse_house_type("в тёплом, панельном доме 1981 года") == "panel" - - -def test_parse_house_type_monolith_brick(): - assert parse_house_type("кирпично-монолитный дом") == "monolith_brick" - - -def test_parse_house_type_monolith_only(): - assert parse_house_type("монолитное здание") == "monolith" - - -def test_parse_house_type_brick_only(): - assert parse_house_type("кирпичный дом 1995") == "brick" - - -def test_parse_house_type_unknown(): - assert parse_house_type("из железобетона") is None - assert parse_house_type("") is None - assert parse_house_type(None) is None - - -# --- house_class --- -def test_parse_house_class_comfort_plus(): - assert parse_house_class("дома класса комфорт+") == "comfort_plus" - - -def test_parse_house_class_comfort(): - assert parse_house_class("ЖК класса комфорт в центре") == "comfort" - - -def test_parse_house_class_business(): - assert parse_house_class("проект класса бизнес") == "business" - - -def test_parse_house_class_premium(): - assert parse_house_class("ЖК премиум-класса") == "premium" - - -def test_parse_house_class_elite(): - assert parse_house_class("элитный класс жилья") == "elite" - - -def test_parse_house_class_none(): - assert parse_house_class("обычный жилой комплекс") is None - assert parse_house_class(None) is None - - -# --- parse_rub --- -def test_parse_rub_full_price(): - assert parse_rub("4 399 000 ₽") == 4_399_000 - - -def test_parse_rub_mln_comma(): - assert parse_rub("4,4 млн ₽") == 4_400_000 - - -def test_parse_rub_mln_dot(): - assert parse_rub("4.4 млн ₽") == 4_400_000 - - -def test_parse_rub_ppm2(): - assert parse_rub("117 500 ₽ за м²") == 117_500 - - -def test_parse_rub_raw_digits(): - assert parse_rub("117500") == 117_500 - - -def test_parse_rub_empty(): - assert parse_rub("") is None - assert parse_rub(None) is None - assert parse_rub("без цены") is None - - -def test_parse_rub_nbsp_separator(): - """Yandex SERP форматирует «9\\xa0800\\xa0000» через NBSP, не regular space. - - До fix'а `.replace(" ", "")` оставлял \\xa0 → int() падал ValueError → - parse_rub() возвращал None → весь scraper не парсил lots (0/23 на live HTML). - """ - assert parse_rub("9\xa0800\xa0000\xa0₽") == 9_800_000 - assert parse_rub("4\xa0399\xa0000") == 4_399_000 - - -def test_parse_rub_mln_with_nbsp(): - """«4,4\\xa0млн\\xa0₽» — NBSP вокруг unit-keyword тоже встречается.""" - assert parse_rub("4,4\xa0млн\xa0₽") == 4_400_000 - - -# --- parse_ru_date --- -def test_parse_ru_date_basic(): - assert parse_ru_date("9 мая 2026") == date(2026, 5, 9) - - -def test_parse_ru_date_in_sentence(): - assert parse_ru_date("опубликовано 17 февраля 2024") == date(2024, 2, 17) - - -def test_parse_ru_date_invalid_month(): - assert parse_ru_date("9 invalidmonth 2026") is None - - -def test_parse_ru_date_dmy_not_supported(): - # parse_ru_date doesn't handle DD.MM.YYYY (that's parse_dmy's job) - assert parse_ru_date("01.02.2026") is None - - -def test_parse_ru_date_empty(): - assert parse_ru_date("") is None - assert parse_ru_date(None) is None - - -# --- parse_listing_date (extended formats) --- -def test_listing_date_today_keyword(): - from datetime import timedelta - - from app.services.scrapers.yandex_helpers import parse_listing_date - assert parse_listing_date("сегодня в 12:30") == date.today() - assert parse_listing_date("вчера в 18:45") == date.today() - timedelta(days=1) - assert parse_listing_date("позавчера") == date.today() - timedelta(days=2) - assert parse_listing_date("только что") == date.today() - - -def test_listing_date_absolute_no_year(): - """Yandex SERP формат «18 мая» без года — infer current year.""" - from app.services.scrapers.yandex_helpers import parse_listing_date - today = date.today() - # Pick a date в прошлом этого года, чтобы тест стабильный - if today.month >= 2: - target = date(today.year, today.month - 1, 15) - else: - target = date(today.year - 1, 12, 15) - ru_months = { - 1: "января", 2: "февраля", 3: "марта", 4: "апреля", - 5: "мая", 6: "июня", 7: "июля", 8: "августа", - 9: "сентября", 10: "октября", 11: "ноября", 12: "декабря", - } - text = f"{target.day} {ru_months[target.month]}" - assert parse_listing_date(text) == target - - -def test_listing_date_relative_still_works(): - from datetime import timedelta - - from app.services.scrapers.yandex_helpers import parse_listing_date - assert parse_listing_date("5 дней назад") == date.today() - timedelta(days=5) - - -# --- parse_dmy --- -def test_parse_dmy_basic(): - assert parse_dmy("17.02.2026") == date(2026, 2, 17) - - -def test_parse_dmy_in_sentence(): - assert parse_dmy("Дата 31.12.2025 — конец года") == date(2025, 12, 31) - - -def test_parse_dmy_invalid(): - assert parse_dmy("99.99.9999") is None - assert parse_dmy("") is None - assert parse_dmy(None) is None - - -# --- JSON-LD --- -def test_extract_json_ld_single(): - html = ( - '" - ) - ld = extract_json_ld(html) - assert len(ld) == 1 - assert ld[0]["@type"] == "Product" - - -def test_extract_json_ld_multiple(): - html = ( - '' - '' - ) - ld = extract_json_ld(html) - assert len(ld) == 2 - - -def test_extract_json_ld_array_wrapper(): - html = '' - ld = extract_json_ld(html) - assert len(ld) == 2 - - -def test_extract_json_ld_invalid_skipped(): - html = ( - '' - '' - ) - ld = extract_json_ld(html) - assert len(ld) == 1 - assert ld[0]["@type"] == "Valid" - - -def test_find_ld_by_type_simple(): - html = ( - '' - '' - ) - product = find_ld_by_type(html, "Product") - assert product is not None - assert product["price"] == 100 - - -def test_find_ld_by_type_list_form(): - # @type can be a list - html = '' - assert find_ld_by_type(html, "Offer") is not None - - -def test_find_ld_by_type_missing(): - html = '' - assert find_ld_by_type(html, "Product") is None - - -# --- regex sanity --- -def test_offer_id(): - m = RE_OFFER_ID.search("https://realty.yandex.ru/offer/7567094292504417257/") - assert m and m.group(1) == "7567094292504417257" - - -def test_jk_id(): - m = RE_JK_ID.search("/ekaterinburg/kupit/novostrojka/tatlin-1592987/") - assert m - assert m.group(1) == "tatlin" - assert m.group(2) == "1592987" - - -def test_title_parsing(): - title = "39 м² · 1-комнатная квартира · 6 этаж из 15" - area = RE_TITLE_AREA.search(title) - rooms = RE_TITLE_ROOMS.search(title) - floor = RE_FLOOR.search(title) - assert area and area.group(1) == "39" - assert rooms and rooms.group(1) == "1" - assert floor and floor.group(1) == "6" and floor.group(2) == "15" - - -def test_title_studio(): - rooms = RE_TITLE_ROOMS.search("Студия 28 м²") - # group(2) captures студия/студию/студий when no number-room pattern present - assert rooms is not None - matched = rooms.group(1) or rooms.group(2) - assert matched is not None and matched.lower().startswith("студи") diff --git a/tradein-mvp/backend/tests/test_yandex_newbuilding.py b/tradein-mvp/backend/tests/test_yandex_newbuilding.py deleted file mode 100644 index 789f938d..00000000 --- a/tradein-mvp/backend/tests/test_yandex_newbuilding.py +++ /dev/null @@ -1,441 +0,0 @@ -"""Unit tests for yandex_newbuilding.py — ЖК landing parser (Worker B, Wave 4). - -Reference target: ЖК Татлин (slug=tatlin, id=1592987) — comfort+, June 2023, -PRINZIP developer, rating 4.3, 1505 ratings, 353 text reviews, -coords (56.855312, 60.576668). - -All tests are offline — no network calls. Parser receives hand-crafted fixture HTML. -""" - -from __future__ import annotations - -import pytest - -from app.services.scrapers.yandex_newbuilding import ( - JKMetroStation, - YandexNewbuildingInfo, - YandexNewbuildingScraper, - _extract_coords, - _find_section_text, - _parse_corpus_count, - _parse_metro, -) - -# ── Fixture HTML ────────────────────────────────────────────────────────────── - -TATLIN_FIXTURE_HTML = """ - -ЖК «Татлин» - -

ЖК «Татлин»

-
Екатеринбург, ул. Черепанова / ул. Готвальда
-PRINZIP - -
56.855312, 60.576668
-
Уральская 11 мин. Динамо 16 мин.
-
4.3 из 5 1505 оценок Смотреть все 353 отзыва
-

О комплексе

-

ЖК «Татлин» — три 35-этажные башни в Заречном микрорайоне Екатеринбурга. - Комплекс класса комфорт+. Введён в эксплуатацию в июне 2023 года. - Расположен на участке 1,5 га. Монолитный тип дома.

-

Расположение

-

В 5 минутах от центра

- -""" - -# Minimal fixture — missing optional sections -MINIMAL_FIXTURE_HTML = """ - -

ЖК Минимальный

-""" - - -# ── Full happy-path test ────────────────────────────────────────────────────── - - -def test_parse_jk_full_fixture(): - """Happy path: all major fields extracted from Татлин-like HTML.""" - scraper = YandexNewbuildingScraper() - info = scraper.parse( - TATLIN_FIXTURE_HTML, - jk_slug="tatlin", - jk_id="1592987", - source_url="https://realty.yandex.ru/ekaterinburg/kupit/novostrojka/tatlin-1592987/", - ) - - assert isinstance(info, YandexNewbuildingInfo) - assert info.ext_id == "1592987" - assert info.ext_slug == "tatlin" - assert "Татлин" in (info.name or "") - - # coords - assert info.lat == pytest.approx(56.855312, abs=1e-5) - assert info.lon == pytest.approx(60.576668, abs=1e-5) - - # class + commission - assert info.house_class == "comfort_plus" - assert info.commission_year == 2023 - assert info.commission_month == "июне" - - # footprint - assert info.total_floors == 35 - assert info.corpus_count == 3 - assert info.total_area_ha == pytest.approx(1.5, abs=0.01) - - # developer - assert info.developer_name == "PRINZIP" - assert info.developer_url is not None and "prinzip" in info.developer_url - assert "ЖК Кислород" in info.developer_other_jk - assert "ЖК Нова" in info.developer_other_jk - - # reviews - assert info.rating == pytest.approx(4.3, abs=0.01) - assert info.ratings_count == 1505 - assert info.text_reviews_count == 353 - - # description present - assert info.description is not None - assert len(info.description) > 10 - - # metro - assert len(info.metro_stations) >= 1 - station_names = [s.name for s in info.metro_stations] - assert any("Уральская" in n for n in station_names) - - # house type from "монолитный" - assert info.house_type == "monolith" - - # raw_payload - assert info.raw_payload is not None - assert "body_len" in info.raw_payload - - -# ── Coord extraction tests ──────────────────────────────────────────────────── - - -def test_extract_coords_within_ekb_range(): - html = "
56.855312, 60.576668
" - lat, lon = _extract_coords(html) - assert lat == pytest.approx(56.855312, abs=1e-5) - assert lon == pytest.approx(60.576668, abs=1e-5) - - -def test_extract_coords_outside_range_returns_none(): - # Moscow coords — outside EKB range - html = "
55.7558, 37.6176
" - _lat, lon = _extract_coords(html) - # 55.75 is in LAT_RANGE but 37.61 is NOT in LON_RANGE - assert lon is None - - -def test_extract_coords_no_coords_returns_none(): - html = "
Текст без координат
" - lat, lon = _extract_coords(html) - assert lat is None - assert lon is None - - -def test_extract_coords_multiple_takes_first(): - """When multiple valid coords present, first is returned.""" - html = "
56.855312, 60.576668
56.900000, 60.600000
" - lat, lon = _extract_coords(html) - assert lat == pytest.approx(56.855312, abs=1e-5) - assert lon == pytest.approx(60.576668, abs=1e-5) - - -# ── Corpus count tests ──────────────────────────────────────────────────────── - - -def test_parse_corpus_count_word_three(): - text = "три 35-этажные башни в центре" - result = _parse_corpus_count(text) - assert result == 3 - - -def test_parse_corpus_count_digit(): - text = "5 35-этажных корпусов" - result = _parse_corpus_count(text) - assert result == 5 - - -def test_parse_corpus_count_two(): - text = "две 20-этажные башни" - result = _parse_corpus_count(text) - assert result == 2 - - -def test_parse_corpus_count_none_when_no_match(): - text = "Обычный текст без этажей" - result = _parse_corpus_count(text) - assert result is None - - -# ── House class test ────────────────────────────────────────────────────────── - - -def test_parse_house_class_comfort_plus_from_description(): - from app.services.scrapers.yandex_helpers import parse_house_class - - text = "Жилой комплекс класса комфорт+ в центре города" - result = parse_house_class(text) - assert result == "comfort_plus" - - -def test_parse_house_class_business(): - from app.services.scrapers.yandex_helpers import parse_house_class - - text = "ЖК класса бизнес" - result = parse_house_class(text) - assert result == "business" - - -# ── Commission year/month test ──────────────────────────────────────────────── - - -def test_parse_commission_year_month(): - """Full pipeline: parse() correctly extracts commission_year and commission_month.""" - scraper = YandexNewbuildingScraper() - info = scraper.parse( - TATLIN_FIXTURE_HTML, - jk_slug="tatlin", - jk_id="1592987", - source_url="https://realty.yandex.ru/ekaterinburg/kupit/novostrojka/tatlin-1592987/", - ) - assert info.commission_year == 2023 - assert info.commission_month == "июне" - - -def test_parse_commission_sdan_variant(): - """Regex supports 'сдан в эксплуатацию' phrasing as well.""" - html = """ -

ЖК Тест

-

О комплексе

-

Сдан в эксплуатацию в марте 2025 года.

- """ - scraper = YandexNewbuildingScraper() - info = scraper.parse(html, jk_slug="test", jk_id="999", source_url="http://x") - assert info.commission_year == 2025 - assert info.commission_month == "марте" - - -# ── Rating + counts test ────────────────────────────────────────────────────── - - -def test_parse_rating_and_counts(): - scraper = YandexNewbuildingScraper() - info = scraper.parse( - TATLIN_FIXTURE_HTML, - jk_slug="tatlin", - jk_id="1592987", - source_url="https://realty.yandex.ru/ekaterinburg/kupit/novostrojka/tatlin-1592987/", - ) - assert info.rating == pytest.approx(4.3, abs=0.01) - assert info.ratings_count == 1505 - assert info.text_reviews_count == 353 - - -def test_parse_rating_comma_separator(): - """Rating '4,3 из 5' with comma handled correctly.""" - html = "
4,3 из 5 200 оценок
" - scraper = YandexNewbuildingScraper() - info = scraper.parse(html, jk_slug="x", jk_id="1", source_url="http://x") - assert info.rating == pytest.approx(4.3, abs=0.01) - assert info.ratings_count == 200 - - -def test_parse_rating_no_space_before_iz(): - """Yandex drift (#974): '4.3из 5' / '4,5из\xa05' — без пробела перед «из», nbsp внутри. - - Регрессия 2026-06-15: live-страницы рендерили рейтинг как «4.3из 5» (без - разделителя), из-за чего \\s+ перед «из» не матчил → rating=NULL при - заполненных ratings_count. \\s* должен извлечь рейтинг в обеих формах. - """ - scraper = YandexNewbuildingScraper() - info = scraper.parse( - "
Отзывы4.3из 5Всего 1508 оценок
", - jk_slug="x", - jk_id="1", - source_url="http://x", - ) - assert info.rating == pytest.approx(4.3, abs=0.01) - assert info.ratings_count == 1508 - # nbsp-вариант (как в реальном body_text) - info2 = scraper.parse( - "
4,5из\xa05Всего 850 оценок
", - jk_slug="y", - jk_id="2", - source_url="http://y", - ) - assert info2.rating == pytest.approx(4.5, abs=0.01) - - -# ── Developer link test ─────────────────────────────────────────────────────── - - -def test_developer_name_extracted_from_data_test_link(): - scraper = YandexNewbuildingScraper() - info = scraper.parse( - TATLIN_FIXTURE_HTML, - jk_slug="tatlin", - jk_id="1592987", - source_url="https://realty.yandex.ru/ekaterinburg/kupit/novostrojka/tatlin-1592987/", - ) - assert info.developer_name == "PRINZIP" - assert info.developer_url == "https://realty.yandex.ru/developer/prinzip" - - -def test_developer_absolute_url_kept_as_is(): - html = """ - АБС Групп - """ - scraper = YandexNewbuildingScraper() - info = scraper.parse(html, jk_slug="x", jk_id="1", source_url="http://x") - assert info.developer_name == "АБС Групп" - assert info.developer_url == "https://realty.yandex.ru/developer/abc" - - -# ── Metro stations test ─────────────────────────────────────────────────────── - - -def test_metro_stations_parsed(): - scraper = YandexNewbuildingScraper() - info = scraper.parse( - TATLIN_FIXTURE_HTML, - jk_slug="tatlin", - jk_id="1592987", - source_url="https://realty.yandex.ru/ekaterinburg/kupit/novostrojka/tatlin-1592987/", - ) - assert len(info.metro_stations) >= 1 - uralskaya = next((s for s in info.metro_stations if "Уральская" in s.name), None) - assert uralskaya is not None - assert uralskaya.walk_min == 11 - - -def test_parse_metro_direct(): - text = "Уральская 11 мин. Динамо 16 мин." - stations = _parse_metro(text) - assert len(stations) == 2 - assert stations[0].name == "Уральская" - assert stations[0].walk_min == 11 - assert stations[1].name == "Динамо" - assert stations[1].walk_min == 16 - - -def test_parse_metro_caps_at_five(): - text = "Альфа 5 мин. Бета 6 мин. Гамма 7 мин. " "Дельта 8 мин. Эпсилон 9 мин. Зета 10 мин." - stations = _parse_metro(text) - assert len(stations) == 5 - - -# ── No description section test ─────────────────────────────────────────────── - - -def test_no_description_section_returns_none(): - """When no О комплексе / Расположение section, description is None.""" - scraper = YandexNewbuildingScraper() - info = scraper.parse( - MINIMAL_FIXTURE_HTML, - jk_slug="minimal", - jk_id="42", - source_url="http://example.com/", - ) - assert info.description is None - assert info.house_class is None - assert info.commission_year is None - assert info.total_floors is None - - -# ── fetch_around raises NotImplementedError ─────────────────────────────────── - - -@pytest.mark.asyncio -async def test_fetch_around_raises_not_implemented(): - scraper = YandexNewbuildingScraper() - with pytest.raises(NotImplementedError, match="JK-slug-based"): - await scraper.fetch_around(56.855, 60.576) - - -# ── _find_section_text helper ───────────────────────────────────────────────── - - -def test_find_section_text_returns_content_after_heading(): - from selectolax.parser import HTMLParser - - html = """ -

О комплексе

-

Описание комплекса здесь.

-

Ещё абзац.

-

Другой раздел

-

Не должен попасть.

- """ - tree = HTMLParser(html) - result = _find_section_text(tree, "О комплексе") - assert result is not None - assert "Описание комплекса" in result - assert "Не должен" not in result - - -def test_find_section_text_returns_none_when_missing(): - from selectolax.parser import HTMLParser - - html = "

Текст

" - tree = HTMLParser(html) - result = _find_section_text(tree, "О комплексе") - assert result is None - - -# ── Developer other JK dedup ────────────────────────────────────────────────── - - -def test_developer_other_jk_dedup(): - """Duplicate JK names are not added twice.""" - html = """ - - """ - scraper = YandexNewbuildingScraper() - info = scraper.parse(html, jk_slug="x", jk_id="1", source_url="http://x") - assert info.developer_other_jk.count("ЖК Один") == 1 - assert "ЖК Два" in info.developer_other_jk - - -def test_developer_other_jk_capped_at_10(): - """Only first 10 other JK entries are kept.""" - links = "\n".join(f'ЖК {i}' for i in range(15)) - html = f""" -
{links}
- """ - scraper = YandexNewbuildingScraper() - info = scraper.parse(html, jk_slug="x", jk_id="1", source_url="http://x") - assert len(info.developer_other_jk) == 10 - - -# ── model defaults ──────────────────────────────────────────────────────────── - - -def test_yandex_newbuilding_info_defaults(): - """Model initialises with safe defaults for all optional fields.""" - info = YandexNewbuildingInfo(ext_id="1", ext_slug="x", source_url="http://x") - assert info.name is None - assert info.lat is None - assert info.lon is None - assert info.house_class is None - assert info.developer_other_jk == [] - assert info.metro_stations == [] - assert info.raw_payload is None - - -def test_jk_metro_station_model(): - station = JKMetroStation(name="Уральская", walk_min=11) - assert station.name == "Уральская" - assert station.walk_min == 11 - - station_no_time = JKMetroStation(name="Геологическая") - assert station_no_time.walk_min is None diff --git a/tradein-mvp/backend/tests/test_yandex_price_history.py b/tradein-mvp/backend/tests/test_yandex_price_history.py index 9a6af431..3ebd30e0 100644 --- a/tradein-mvp/backend/tests/test_yandex_price_history.py +++ b/tradein-mvp/backend/tests/test_yandex_price_history.py @@ -18,11 +18,11 @@ import re os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test") import pytest +from scraper_kit.base import ScrapedLot from sqlalchemy import create_engine, text from sqlalchemy.orm import Session, sessionmaker from app.services import yandex_price_history as yph -from app.services.scrapers.base import ScrapedLot _SERVICE_SRC = inspect.getsource(yph) # Concatenated text of every actual SQL statement the service emits (not the docstring, diff --git a/tradein-mvp/backend/tests/test_yandex_scrapers_delay_wiring.py b/tradein-mvp/backend/tests/test_yandex_scrapers_delay_wiring.py index 43d7f54e..72e3d07c 100644 --- a/tradein-mvp/backend/tests/test_yandex_scrapers_delay_wiring.py +++ b/tradein-mvp/backend/tests/test_yandex_scrapers_delay_wiring.py @@ -1,23 +1,23 @@ """Smoke: each Yandex scraper instance reads request_delay_sec from scraper_settings. Легаси `app.services.scrapers.yandex_detail` / `yandex_valuation` удалены (#2277 -финальный шаг scraper_kit-миграции). `yandex_newbuilding` пока остаётся легаси (не -мигрирован) — его delay-wiring тест не тронут. `yandex_realty` удалён (#2397 Part -E2, 0 runtime-импортёров) — его delay-wiring тест убран (kit `YandexRealtyScraper` -delay-provider wiring покрыт `test_kit_serp_proxy_pool.py`). +финальный шаг scraper_kit-миграции). `yandex_realty` удалён (#2397 Part E2, 0 +runtime-импортёров) — его delay-wiring тест убран (kit `YandexRealtyScraper` +delay-provider wiring покрыт `test_kit_serp_proxy_pool.py`). `yandex_newbuilding` +удалён (#2397 финальный шаг E) — переведён на kit-эквивалент, который тоже +использует инжектируемый `delay_provider=` (strangler DI, #2133) вместо прямого +импорта `get_scraper_delay`. `yandex_detail`/`yandex_valuation` переведены на kit-эквиваленты, которые используют инжектируемый `delay_provider=` (strangler DI, #2133) вместо прямого импорта `get_scraper_delay` — тест адаптирован под реальный контракт kit-класса. """ from types import SimpleNamespace -from unittest.mock import patch from scraper_kit.providers.yandex.detail import YandexDetailScraper +from scraper_kit.providers.yandex.newbuilding import YandexNewbuildingScraper from scraper_kit.providers.yandex.valuation import YandexValuationScraper -from app.services.scrapers.yandex_newbuilding import YandexNewbuildingScraper - _KIT_CONFIG = SimpleNamespace( scraper_proxy_url=None, yandex_proxy_rotate_url=None, @@ -32,9 +32,9 @@ def test_yandex_detail_picks_up_delay(): def test_yandex_newbuilding_picks_up_delay(): - with patch("app.services.scrapers.yandex_newbuilding.get_scraper_delay", return_value=9.5): - s = YandexNewbuildingScraper() - assert s.request_delay_sec == 9.5 + """Kit YandexNewbuildingScraper: delay_provider callable вызывается с self.name.""" + s = YandexNewbuildingScraper(delay_provider=lambda _name: 9.5) + assert s.request_delay_sec == 9.5 def test_yandex_valuation_picks_up_delay(): diff --git a/tradein-mvp/scripts/avito-playwright-sweep.py b/tradein-mvp/scripts/avito-playwright-sweep.py deleted file mode 100644 index 66344d89..00000000 --- a/tradein-mvp/scripts/avito-playwright-sweep.py +++ /dev/null @@ -1,371 +0,0 @@ -"""Playwright-based Avito sweep — обходит Qrator JS challenge через real headless Chromium. - -Use case: curl_cffi заблочен Qrator anti-DDoS (429 + captcha challenge). Real -browser proходит JS challenge автоматически. - -Setup: - cd tradein-mvp/backend - uv pip install playwright - uv run playwright install chromium - -Usage: - export DATABASE_URL='postgresql+psycopg://tradein:@localhost:15433/tradein' - uv run --project tradein-mvp/backend python tradein-mvp/scripts/avito-playwright-sweep.py - # options: --pages N --delay S --mode {citywide|byrooms} --headless -""" -from __future__ import annotations - -# Banner FIRST — даже до imports чтобы видно что скрипт стартанул: -import sys as _sys - -print("=" * 60, flush=True) -print("AVITO PLAYWRIGHT SWEEP — starting...", flush=True) -print("=" * 60, flush=True) -_sys.stdout.flush() - -import argparse -import asyncio -import json -import os -import random -import sys -import time -import types -from pathlib import Path -from urllib.parse import urlencode - -# Force unbuffered output -os.environ["PYTHONUNBUFFERED"] = "1" - -_BACKEND = Path(__file__).resolve().parent.parent / "backend" -if str(_BACKEND) not in sys.path: - sys.path.insert(0, str(_BACKEND)) - -# WeasyPrint stub (Windows без GTK) -for _m in ("weasyprint", "weasyprint.text", "weasyprint.text.ffi", "weasyprint.css", "weasyprint.css.targets"): - sys.modules[_m] = types.ModuleType(_m) -sys.modules["weasyprint"].HTML = type("HTML", (), {"__init__": lambda self, *a, **kw: None}) -sys.modules["weasyprint"].CSS = type("CSS", (), {"__init__": lambda self, *a, **kw: None}) - -from playwright.async_api import Browser, BrowserContext, async_playwright # noqa: E402 - -try: - from playwright_stealth import Stealth # noqa: E402 - _STEALTH_AVAILABLE = True -except ImportError: - _STEALTH_AVAILABLE = False - print("[warn] playwright-stealth not installed — без stealth, может triggered больше CAPTCHA", flush=True) - -from app.core.db import SessionLocal # noqa: E402 -from app.services.scrapers.avito import AvitoScraper # noqa: E402 -from app.services.scrapers.base import save_listings # noqa: E402 - -COOKIES_FILE = Path(__file__).resolve().parent / ".avito-cookies.json" -AVITO_BASE = "https://www.avito.ru" -UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36" - -# Reuse AvitoScraper's parser (DOM cards extraction) -_parser_instance = AvitoScraper() -_parser_instance._anchor_lat = 0.0 # noqa: SLF001 -_parser_instance._anchor_lon = 0.0 # noqa: SLF001 -_parser_instance._anchor_radius_m = 1000 # noqa: SLF001 - - -def load_cookies() -> list[dict]: - """Load DevTools-exported cookies → Playwright format.""" - if not COOKIES_FILE.exists(): - print(f"WARN: cookies file not found at {COOKIES_FILE} — будем без cookies") - return [] - raw = json.loads(COOKIES_FILE.read_text(encoding="utf-8")) - out = [] - for c in raw: - ck = { - "name": c["name"], - "value": c["value"], - "domain": c["domain"], - "path": c.get("path", "/"), - "secure": c.get("secure", True), - "httpOnly": c.get("httpOnly", False), - } - if c.get("expirationDate"): - ck["expires"] = int(c["expirationDate"]) - ss = c.get("sameSite") - if ss == "no_restriction": - ck["sameSite"] = "None" - elif ss == "lax": - ck["sameSite"] = "Lax" - elif ss == "strict": - ck["sameSite"] = "Strict" - out.append(ck) - return out - - -def build_citywide_url(page: int) -> str: - return f"{AVITO_BASE}/ekaterinburg/kvartiry/prodam-ASgBAgICAUSSA8YQ?{urlencode({'s':104,'p':page})}" - - -ROOM_SLUGS: list[tuple[str, str]] = [ - ("студии", "studii-ASgBAgICAkSSA8YQygj~WA"), - ("1-комн.", "1-komnatnye-ASgBAgICAkSSA8YQygiAWQ"), - ("2-комн.", "2-komnatnye-ASgBAgICAkSSA8YQygiCWQ"), - ("3-комн.", "3-komnatnye-ASgBAgICAkSSA8YQygiEWQ"), - ("4-комн.", "4-komnatnye-ASgBAgICAkSSA8YQygiGWQ"), - ("5+комн.", "5-komnatnye-ASgBAgICAkSSA8YQygiIWQ"), - ("своб.планир.", "svobodnaya_planirovka-ASgBAgICAkSSA8YQygj8zzI"), -] - - -def build_rooms_url(slug: str, page: int) -> str: - return f"{AVITO_BASE}/ekaterinburg/kvartiry/prodam/{slug}?{urlencode({'s':104,'p':page})}" - - -async def fetch_page(context: BrowserContext, url: str) -> tuple[int, str | None]: - """Open page in Playwright, wait for content, return (status, html or None). - - Strategy: - 1. goto с wait_until='domcontentloaded' (быстро, не ждём всех scripts) - 2. Если status 200 → wait_for_selector data-marker=item (Qrator challenge должен resolve пока) - 3. Иначе если 429/403 — challenge не прошёл, дать ещё 5s + retry content (maybe challenge resolves async) - """ - page = await context.new_page() - try: - try: - response = await page.goto(url, wait_until="domcontentloaded", timeout=45000) - except Exception as e: # noqa: BLE001 - print(f" goto FAIL: {type(e).__name__}: {str(e)[:80]}") - return 0, None - status = response.status if response else 0 - if status in (429, 403): - # Qrator challenge — wait для JS auto-resolve, потом reload - await asyncio.sleep(5) - try: - response = await page.reload(wait_until="domcontentloaded", timeout=30000) - status = response.status if response else status - except Exception: # noqa: BLE001 - pass - if status != 200: - return status, None - # Wait for item cards - try: - await page.wait_for_selector('[data-marker="item"]', timeout=15000) - except Exception: # noqa: BLE001 - pass - html = await page.content() - return status, html - finally: - await page.close() - - -async def sweep_urls( - urls_with_label: list[tuple[str, str]], - delay: float, - headless: bool, -) -> int: - """Generic sweep: list of (label, url) → fetch + parse + save per URL.""" - cookies = load_cookies() - print(f"loaded {len(cookies)} cookies") - total_fetched = 0 - total_ins = 0 - total_upd = 0 - challenge_hits = 0 - print("=== Launching Playwright browser ===", flush=True) - async with async_playwright() as p: - print(f" headless={headless}", flush=True) - browser: Browser = await p.chromium.launch( - headless=headless, - args=[ - "--disable-blink-features=AutomationControlled", - "--no-sandbox", - ], - ) - print(" browser launched, creating context...", flush=True) - context: BrowserContext = await browser.new_context( - user_agent=UA, - locale="ru-RU", - timezone_id="Asia/Yekaterinburg", - viewport={"width": 1366, "height": 900}, - ) - # Stealth — optional, skip on error - if _STEALTH_AVAILABLE: - try: - stealth = Stealth() - await stealth.apply_stealth_async(context) - print(" stealth applied", flush=True) - except Exception as e: # noqa: BLE001 - print(f" stealth skipped (err): {type(e).__name__}: {e}", flush=True) - if cookies: - await context.add_cookies(cookies) - print(f" cookies added ({len(cookies)})", flush=True) - - # Initial warmup: open main Avito + wait для user pass CAPTCHA если нужно - print("\n=== WARMUP: opening Avito initial page ===", flush=True) - print(" creating new page tab...", flush=True) - warmup_page = await context.new_page() - print(" navigating to Avito (timeout 30s)...", flush=True) - try: - await warmup_page.goto( - "https://www.avito.ru/ekaterinburg/kvartiry/prodam-ASgBAgICAUSSA8YQ?s=104&p=1", - wait_until="domcontentloaded", - timeout=30000, - ) - print(" goto done, reading content...", flush=True) - except Exception as e: # noqa: BLE001 - print(f" goto FAIL: {type(e).__name__}: {e}", flush=True) - # Check if challenge page — reliable detector: NO listing cards в HTML. - # Слово "captcha" может быть в JS embed страницы normally (false positive). - warmup_html = await warmup_page.content() - print(f" content read: {len(warmup_html)} bytes", flush=True) - try: - await warmup_page.wait_for_selector('[data-marker="item"]', timeout=5000) - has_items = True - except Exception: # noqa: BLE001 - has_items = False - is_qrator = "qrator" in warmup_html.lower() and len(warmup_html) < 100000 - challenge_present = (not has_items) and (is_qrator or len(warmup_html) < 50000) - print(f" has_items={has_items} qrator={is_qrator} size={len(warmup_html)} challenge={challenge_present}", flush=True) - if challenge_present: - print("\n🚨 CAPTCHA detected на initial page!") - print("👉 Окно браузера открыто — пройди капчу руками,") - print("👉 затем нажми Enter здесь чтобы продолжить sweep...") - try: - input(">>> Press ENTER when ready: ") - except (EOFError, KeyboardInterrupt): - print("aborted") - await browser.close() - return 0 - # Verify challenge passed - await warmup_page.reload(wait_until="domcontentloaded", timeout=30000) - new_html = await warmup_page.content() - if any(m in new_html.lower() for m in ("captcha", "qrator")): - print("⚠️ всё ещё CAPTCHA — повторно пройди или Ctrl+C") - input(">>> Retry ENTER: ") - else: - print("✅ CAPTCHA пройдена, продолжаю sweep") - else: - print("✅ Initial page OK (no challenge)") - await warmup_page.close() - - try: - # Per-category skip flag: 0 lots на странице категории = end of pagination - # ДЛЯ ТЕКУЩЕЙ КАТЕГОРИИ, а не для всего sweep'а (был bug: break ломал всё). - current_cat: str | None = None - skip_until_next_cat = False - for i, (label, url) in enumerate(urls_with_label, 1): - # Категория = всё до " page=" в label ("1-комн. page=5/60" → "1-комн.") - cat = label.split(" page=")[0] if " page=" in label else label - if cat != current_cat: - current_cat = cat - skip_until_next_cat = False - challenge_hits = 0 # reset 429-counter на границе категорий - if skip_until_next_cat: - continue - - print(f"[{i}/{len(urls_with_label)}] {label}", flush=True) - status, html = await fetch_page(context, url) - if status != 200 or not html: - print(f" HTTP {status} → skip", flush=True) - if status in (429, 403): - challenge_hits += 1 - if challenge_hits >= 3: - print(f" ⚠️ {challenge_hits}x challenge hits — sleep 60s extra", flush=True) - print(" если CAPTCHA в окне — пройди и нажми ENTER:", flush=True) - try: - input(" >>> ") - except (EOFError, KeyboardInterrupt): - break - challenge_hits = 0 - await asyncio.sleep(delay) - continue - # Parse + save - lots = _parser_instance._parse_html(html, source_url_base=url) # noqa: SLF001 - if not lots: - print(f" 0 lots — end of pagination for {cat!r}", flush=True) - if " page=" in label: - # 0 lots на пагинированной странице = end of category, не sweep - skip_until_next_cat = True - continue - db = SessionLocal() - try: - ins, upd = save_listings(db, lots) - total_fetched += len(lots) - total_ins += ins - total_upd += upd - print( - f" {len(lots)} lots → ins={ins} upd={upd} " - f"[cum: tot={total_fetched} new={total_ins}]", - flush=True, - ) - except Exception as e: # noqa: BLE001 - print(f" save FAIL: {type(e).__name__}: {e}", flush=True) - finally: - db.close() - # Jittered delay - if i < len(urls_with_label): - j = delay * random.uniform(0.7, 1.3) - await asyncio.sleep(j) - finally: - await context.close() - await browser.close() - print( - f"\n=== TOTAL: fetched={total_fetched} ins={total_ins} upd={total_upd} ===", - flush=True, - ) - return total_fetched - - -async def main() -> None: - # Diagnostic — show env vars used: - db_url = os.environ.get("DATABASE_URL", "") - # Mask password for logging - if "://" in db_url and "@" in db_url: - before_at, after_at = db_url.rsplit("@", 1) - scheme_user = before_at.split("://", 1) - if len(scheme_user) == 2 and ":" in scheme_user[1]: - user = scheme_user[1].split(":", 1)[0] - masked = f"{scheme_user[0]}://{user}:***@{after_at}" - else: - masked = db_url[:50] + "..." - else: - masked = db_url - print(f"DATABASE_URL: {masked}", flush=True) - print(f"ENVIRONMENT: {os.environ.get('ENVIRONMENT', '')}", flush=True) - - p = argparse.ArgumentParser() - p.add_argument("--mode", choices=["citywide", "byrooms"], default="citywide") - p.add_argument("--pages", type=int, default=100, help="pages per query") - p.add_argument("--delay", type=float, default=7, help="sec между requests") - p.add_argument("--headless", action="store_true", default=True) - p.add_argument("--no-headless", dest="headless", action="store_false") - p.add_argument( - "--start-cat", - type=int, - default=1, - help="byrooms: с какой категории начать (1-based). 1=студии, 2=1-комн, " - "3=2-комн, 4=3-комн, 5=4-комн, 6=5+комн, 7=своб.планир.", - ) - args = p.parse_args() - - urls: list[tuple[str, str]] = [] - if args.mode == "citywide": - for page in range(1, args.pages + 1): - urls.append((f"citywide page={page}/{args.pages}", build_citywide_url(page))) - else: # byrooms - cats = ROOM_SLUGS[args.start_cat - 1:] - if not cats: - print(f"ERROR: --start-cat={args.start_cat} > {len(ROOM_SLUGS)} cats", flush=True) - return - print(f"byrooms: starting from cat #{args.start_cat} ({cats[0][0]!r}), " - f"{len(cats)} cats total", flush=True) - for cat_name, slug in cats: - for page in range(1, args.pages + 1): - urls.append((f"{cat_name} page={page}/{args.pages}", build_rooms_url(slug, page))) - - print(f"Mode: {args.mode}, pages: {args.pages}, delay: {args.delay}s, total URLs: {len(urls)}") - print(f"Estimated time: {len(urls) * args.delay / 60:.0f} min (без challenge)") - t0 = time.perf_counter() - await sweep_urls(urls, args.delay, args.headless) - print(f"\nDONE in {(time.perf_counter() - t0) / 60:.1f} min") - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/tradein-mvp/scripts/backfill-house-imv.py b/tradein-mvp/scripts/backfill-house-imv.py deleted file mode 100644 index 83aabccd..00000000 --- a/tradein-mvp/scripts/backfill-house-imv.py +++ /dev/null @@ -1,467 +0,0 @@ -"""Phase C: backfill Avito IMV per house. - -Iterates `houses` with imv_status='pending' AND lat/lon NOT NULL. -For each: pick median lot-params from linked listings -> call evaluate_via_imv() --> save house_imv_evaluations + house_placement_history + house_suggestions. - -Resumable: marks houses.imv_status = 'ok' / 'not_found' / 'transient_error' / 'error'. - -Usage: - # Locally with SSH tunnel (LocalForward 35432 -> tradein-postgres): - DATABASE_URL=postgresql+psycopg://tradein:@localhost:35432/tradein \\ - python scripts/backfill-house-imv.py --batch 50 --delay 5 - - # Inside container: - docker exec tradein-backend python /app/scripts/backfill-house-imv.py --batch 100 --delay 3 - -Args: - --batch N How many houses to process this run (default 50) - --delay SEC Sleep between houses (default 5 to avoid Avito captcha) - --limit-total Stop after N successful evaluations (debug; default no limit) - --only-status Process only houses with this imv_status (default 'pending'; - also accepts 'transient_error') -""" -from __future__ import annotations - -import argparse -import asyncio -import json -import logging -import os -import sys -from pathlib import Path - -# Ensure repo backend in path -sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "backend")) - -from sqlalchemy import create_engine, text -from sqlalchemy.orm import Session - -from app.services.scrapers.avito_imv import ( - IMVAddressNotFoundError, - IMVAuthError, - IMVEvaluation, - IMVTransientError, - evaluate_via_imv, -) - -logging.basicConfig( - level=logging.INFO, - format="%(asctime)s %(levelname)s %(message)s", -) -logger = logging.getLogger("phase-c") - -# Avito raw log — pipe everything from app.services.scrapers.avito_imv to a file -# so we keep full request/response history for diagnosing contract changes. -_AVITO_LOG_PATH = os.environ.get("AVITO_LOG_PATH", "avito-raw.log") -_avito_fh = logging.FileHandler(_AVITO_LOG_PATH, encoding="utf-8") -_avito_fh.setFormatter(logging.Formatter("%(asctime)s %(levelname)s [%(name)s] %(message)s")) -_avito_fh.setLevel(logging.DEBUG) -_avito_logger = logging.getLogger("app.services.scrapers.avito_imv") -_avito_logger.addHandler(_avito_fh) -_avito_logger.setLevel(logging.DEBUG) -# Also pipe phase-c progress lines to the file -logger.addHandler(_avito_fh) -logger.info("avito raw log writing to %s", os.path.abspath(_AVITO_LOG_PATH)) - -# Avito IMV API accepts only these house_type values (verified 2026-05-24). -# Our `listings.house_type` may contain pre-2026 values like 'monolith' / 'monolith_brick'. -_HOUSE_TYPE_TO_IMV = { - "panel": "panel", - "brick": "brick", - "monolith": "monolithic", # API renamed; raw value rejected with 400 - "monolithic": "monolithic", - "monolith_brick": "monolithic", # API doesn't accept hybrid → fall back to monolithic - "block": "block", - "wood": "wood", -} -_HOUSE_TYPE_DEFAULT = "panel" # most common in EKB; safe default for unknown - - -def _map_house_type(raw: str | None) -> str: - if not raw: - return _HOUSE_TYPE_DEFAULT - return _HOUSE_TYPE_TO_IMV.get(raw.lower().strip(), _HOUSE_TYPE_DEFAULT) - - -# Bounding boxes for known regions in our DB. -# When house.lat/lon falls inside one, we prepend the region name to address -# before sending to Avito (resolves ambiguity — e.g. "Орджоникидзевский район" exists -# in EKB AND in Khakasia republic). -_REGION_BBOX = [ - # (region_name, lat_min, lat_max, lon_min, lon_max) - ("Свердловская область, Екатеринбург", 56.5, 57.1, 59.9, 61.0), - ("Свердловская область", 56.0, 61.0, 58.0, 65.0), # rest of region - ("Кировская область, Киров", 58.4, 58.8, 49.4, 49.9), - ("Ставропольский край, Ставрополь", 44.9, 45.2, 41.7, 42.2), - ("Тюменская область, Тюмень", 57.0, 57.3, 65.3, 65.8), - ("Челябинская область, Челябинск", 55.0, 55.4, 61.2, 61.7), -] -_REGION_DEFAULT = "Свердловская область, Екатеринбург" # bulk of our data is EKB - - -def _detect_region_prefix(lat: float | None, lon: float | None) -> str: - if lat is None or lon is None: - return _REGION_DEFAULT - for name, lat_min, lat_max, lon_min, lon_max in _REGION_BBOX: - if lat_min <= lat <= lat_max and lon_min <= lon <= lon_max: - return name - return _REGION_DEFAULT - - -def _enrich_address_for_imv(raw: str, lat: float | None, lon: float | None) -> str: - """Prepend region prefix if address lacks one (ambiguity fix for Avito geocoder).""" - addr = (raw or "").strip() - # Already has region marker? skip - addr_lower = addr.lower() - if any( - m in addr_lower - for m in ("область,", " обл.,", " обл,", "край,", "респ.,", "республика,") - ): - return addr - prefix = _detect_region_prefix(lat, lon) - return f"{prefix}, {addr}" - - -DATABASE_URL = os.environ.get("DATABASE_URL") -if not DATABASE_URL: - raise SystemExit( - "Set DATABASE_URL env" - " (e.g. postgresql+psycopg://tradein:...@localhost:35432/tradein)" - ) - - -def pick_lot_params(db: Session, house_id: int) -> dict: - """Pick representative lot-params from linked listings — median values.""" - row = db.execute( - text(""" - SELECT - CAST(percentile_cont(0.5) WITHIN GROUP (ORDER BY rooms) AS integer) AS rooms, - CAST(percentile_cont(0.5) WITHIN GROUP (ORDER BY area_m2) AS numeric(8,2)) - AS area_m2, - CAST(percentile_cont(0.5) WITHIN GROUP (ORDER BY floor) AS integer) AS floor, - CAST(percentile_cont(0.5) WITHIN GROUP (ORDER BY total_floors) AS integer) - AS total_floors, - mode() WITHIN GROUP (ORDER BY house_type) AS house_type - FROM listings - WHERE house_id_fk = :hid - AND rooms IS NOT NULL - AND area_m2 IS NOT NULL - """), - {"hid": house_id}, - ).mappings().first() - - if row is None or row["rooms"] is None: - return {} # no listings with required fields, can't pick params - - # Fall back to house's own type/floors if listings don't have them - house = db.execute( - text("SELECT house_type, total_floors FROM houses WHERE id = :hid"), - {"hid": house_id}, - ).mappings().first() - - # Avito IMV: rooms=0 (студия) rejected with HTTP 400. Clamp to 1. - rooms_raw = int(row["rooms"]) - return { - "rooms": max(rooms_raw, 1), - "area_m2": float(row["area_m2"]), - "floor": int(row["floor"] or 1), - "floor_at_home": int(row["total_floors"] or (house and house["total_floors"]) or 9), - "house_type": _map_house_type(row["house_type"] or (house and house["house_type"])), - "renovation_type": "cosmetic", - "has_balcony": True, - "has_loggia": False, - } - - -def save_imv_result(db: Session, house_id: int, params: dict, result: IMVEvaluation) -> None: - """Save 3 things: house_imv_evaluations + history items + suggestions. - - Uses already-parsed dataclass fields from IMVEvaluation (placement_history, - suggestions) instead of re-parsing raw_response to avoid duplication and - ensure consistency with _parse_placement_item / _parse_suggestion logic. - """ - # 1. House-level IMV evaluation (UPSERT on house_id) - db.execute( - text(""" - INSERT INTO house_imv_evaluations ( - house_id, cache_key, recommended_price, lower_price, higher_price, - market_count, raw_response, fetched_at, - rooms, area_m2, floor, floor_at_home, house_type, - renovation_type, has_balcony, has_loggia - ) VALUES ( - :hid, :ck, :rec, :lo, :hi, - :mc, CAST(:raw AS jsonb), NOW(), - :rooms, :area, :floor, :fah, :htype, - :rtype, :balcony, :loggia - ) - ON CONFLICT (house_id) DO UPDATE SET - cache_key = EXCLUDED.cache_key, - recommended_price = EXCLUDED.recommended_price, - lower_price = EXCLUDED.lower_price, - higher_price = EXCLUDED.higher_price, - market_count = EXCLUDED.market_count, - raw_response = EXCLUDED.raw_response, - fetched_at = NOW(), - rooms = EXCLUDED.rooms, - area_m2 = EXCLUDED.area_m2, - floor = EXCLUDED.floor, - floor_at_home = EXCLUDED.floor_at_home, - house_type = EXCLUDED.house_type, - renovation_type = EXCLUDED.renovation_type, - has_balcony = EXCLUDED.has_balcony, - has_loggia = EXCLUDED.has_loggia - """), - { - "hid": house_id, - "ck": result.cache_key, - "rec": result.recommended_price, - "lo": result.lower_price, - "hi": result.higher_price, - "mc": result.market_count, - "raw": ( - json.dumps(result.raw_response, ensure_ascii=False) - if result.raw_response - else None - ), - "rooms": params["rooms"], - "area": params["area_m2"], - "floor": params["floor"], - "fah": params["floor_at_home"], - "htype": params["house_type"], - "rtype": params["renovation_type"], - "balcony": params["has_balcony"], - "loggia": params["has_loggia"], - }, - ) - - # 2. Placement history items (already parsed into IMVPlacementHistoryItem dataclasses) - for item in result.placement_history: - db.execute( - text(""" - INSERT INTO house_placement_history ( - source, house_id, ext_item_id, title, rooms, area_m2, - floor, total_floors, start_price, start_price_date, - last_price, last_price_date, removed_date, exposure_days, - raw_payload, scraped_at - ) VALUES ( - 'avito_imv', :hid, :ext, :title, :rooms, :area, - :floor, :total_floors, :sp, :spd, :lp, :lpd, :rmd, :exp, - CAST(:raw AS jsonb), NOW() - ) - ON CONFLICT (source, ext_item_id) DO UPDATE SET - house_id = EXCLUDED.house_id, - title = EXCLUDED.title, - rooms = EXCLUDED.rooms, - area_m2 = EXCLUDED.area_m2, - floor = EXCLUDED.floor, - total_floors = EXCLUDED.total_floors, - start_price = EXCLUDED.start_price, - start_price_date = EXCLUDED.start_price_date, - last_price = EXCLUDED.last_price, - last_price_date = EXCLUDED.last_price_date, - removed_date = EXCLUDED.removed_date, - exposure_days = EXCLUDED.exposure_days, - raw_payload = EXCLUDED.raw_payload, - scraped_at = NOW() - """), - { - "hid": house_id, - "ext": item.ext_item_id, - "title": item.title, - "rooms": item.rooms, - "area": item.area_m2, - "floor": item.floor, - "total_floors": item.total_floors, - "sp": item.start_price, - "spd": item.start_price_date, - "lp": item.last_price, - "lpd": item.last_price_date, - "rmd": item.removed_date, - "exp": item.exposure_days, - "raw": ( - json.dumps(item.raw_payload, ensure_ascii=False) - if item.raw_payload - else None - ), - }, - ) - - # 3. Suggestions (already parsed into IMVSuggestion dataclasses) - for sug in result.suggestions: - db.execute( - text(""" - INSERT INTO house_suggestions ( - house_id, ext_item_id, title, address, price_rub, - exposure_days, publish_date, - item_link, metro_name, metro_distance, - has_good_price_badge, raw_payload, fetched_at - ) VALUES ( - :hid, :ext, :title, :addr, :price, - :exp, :pdate, - :link, :mname, :mdist, - :gpb, CAST(:raw AS jsonb), NOW() - ) - ON CONFLICT (house_id, ext_item_id) DO UPDATE SET - title = EXCLUDED.title, - price_rub = EXCLUDED.price_rub, - exposure_days = EXCLUDED.exposure_days, - publish_date = EXCLUDED.publish_date, - item_link = EXCLUDED.item_link, - metro_name = EXCLUDED.metro_name, - metro_distance = EXCLUDED.metro_distance, - has_good_price_badge = EXCLUDED.has_good_price_badge, - raw_payload = EXCLUDED.raw_payload, - fetched_at = NOW() - """), - { - "hid": house_id, - "ext": sug.ext_item_id, - "title": sug.title, - "addr": sug.address, - "price": sug.price_rub, - "exp": sug.exposure_days, - "pdate": sug.publish_date, - "link": sug.item_url, - "mname": sug.metro_name, - "mdist": sug.metro_distance, - "gpb": sug.has_good_price_badge, - "raw": ( - json.dumps(sug.raw_payload, ensure_ascii=False) - if sug.raw_payload - else None - ), - }, - ) - - # 4. Mark house success - db.execute( - text(""" - UPDATE houses - SET imv_status = 'ok', last_imv_attempt_at = NOW(), imv_error_reason = NULL - WHERE id = :hid - """), - {"hid": house_id}, - ) - - -def mark_status(db: Session, house_id: int, status: str, reason: str | None = None) -> None: - db.execute( - text(""" - UPDATE houses - SET imv_status = :s, last_imv_attempt_at = NOW(), imv_error_reason = :r - WHERE id = :hid - """), - {"hid": house_id, "s": status, "r": reason}, - ) - - -async def process_one(db: Session, house: dict) -> str: - """Process a single house: geocode + IMV call + save. - - Returns final status string: - 'ok' / 'no_params' / 'no_address' / 'not_found' / 'auth_error' / 'transient' / 'error' - """ - params = pick_lot_params(db, house["id"]) - if not params: - mark_status(db, house["id"], "no_params", "no listings with rooms+area") - return "no_params" - - address = house["address"] or house["full_address"] - if not address: - mark_status(db, house["id"], "no_address", "house.address is NULL") - return "no_address" - - # Enrich with region prefix to prevent Avito geocoder mis-resolving ambiguous - # street names to wrong regions (e.g. "улица Победы" exists in EKB and Khakasia). - enriched = _enrich_address_for_imv(address, house.get("lat"), house.get("lon")) - if enriched != address: - logger.debug("enriched address house=%d %r -> %r", house["id"], address, enriched) - - try: - result = await evaluate_via_imv(address=enriched, **params) - except IMVAddressNotFoundError as e: - mark_status(db, house["id"], "not_found", str(e)[:200]) - return "not_found" - except IMVAuthError as e: - mark_status(db, house["id"], "transient_error", f"auth: {e!s}"[:200]) - logger.warning("AUTH ERROR house=%d — pausing 60s", house["id"]) - await asyncio.sleep(60) # long back-off on auth block - return "auth_error" - except IMVTransientError as e: - mark_status(db, house["id"], "transient_error", str(e)[:200]) - return "transient" - except Exception as e: - mark_status(db, house["id"], "error", repr(e)[:200]) - logger.error("Unexpected error for house=%d: %r", house["id"], e) - return "error" - - save_imv_result(db, house["id"], params, result) - return "ok" - - -async def main() -> None: - parser = argparse.ArgumentParser(description="Phase C: backfill Avito IMV per house") - parser.add_argument("--batch", type=int, default=50, help="Houses to process this run") - parser.add_argument("--delay", type=float, default=5.0, help="Sleep between houses (s)") - parser.add_argument( - "--limit-total", type=int, default=0, - help="Stop after N successful evaluations (0 = no limit)" - ) - parser.add_argument( - "--only-status", default="pending", - help="Process only houses with this imv_status (default: pending)" - ) - args = parser.parse_args() - - engine = create_engine(DATABASE_URL, future=True) - - with Session(engine) as db: - rows = db.execute( - text(""" - SELECT id, address, full_address, lat, lon - FROM houses - WHERE imv_status = :status - AND lat IS NOT NULL - AND lon IS NOT NULL - AND address IS NOT NULL - ORDER BY last_imv_attempt_at NULLS FIRST, id - LIMIT :batch - """), - {"status": args.only_status, "batch": args.batch}, - ).mappings().all() - - if not rows: - logger.info("nothing to process — all done or no houses match filter") - return - - logger.info( - "processing %d houses (status=%r delay=%.1fs)", - len(rows), args.only_status, args.delay - ) - counters: dict[str, int] = { - "ok": 0, "not_found": 0, "no_params": 0, "no_address": 0, - "transient": 0, "auth_error": 0, "error": 0, - } - - for i, house in enumerate(rows, 1): - status = await process_one(db, dict(house)) - counters[status] = counters.get(status, 0) + 1 - db.commit() - logger.info( - "[%d/%d] house=%d status=%s addr=%r", - i, len(rows), house["id"], status, (house["address"] or "")[:50], - ) - - if args.limit_total and counters["ok"] >= args.limit_total: - logger.info("hit --limit-total=%d; stopping", args.limit_total) - break - - if i < len(rows): - await asyncio.sleep(args.delay) - - logger.info("done: %s", counters) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/tradein-mvp/scripts/backfill_cian_zhk_url.py b/tradein-mvp/scripts/backfill_cian_zhk_url.py deleted file mode 100644 index 5da7bb99..00000000 --- a/tradein-mvp/scripts/backfill_cian_zhk_url.py +++ /dev/null @@ -1,122 +0,0 @@ -"""Backfill houses.cian_zhk_url for rows that have cian_internal_house_id but no URL. - -Requires migration 071_houses_cian_zhk_url.sql to be applied first. - -What it does: - SELECT houses WHERE cian_internal_house_id IS NOT NULL AND cian_zhk_url IS NULL - For each row: resolve https://cian.ru/zhk// → follow redirect → store final URL. - -Cian redirects the canonical /zhk// path to the slug URL -(e.g. https://zhk-park-ekb-i.cian.ru/). We capture the final redirect destination. - -Usage: - cd tradein-mvp/backend - DATABASE_URL=postgresql://... uv run python ../scripts/backfill_cian_zhk_url.py - -Env vars: - DATABASE_URL required — psycopg v3 DSN - DELAY_SEC default 5.0 — seconds between requests (Cian rate limit) - BATCH_LIMIT default 500 — max houses to process per run (safety cap) - -Idempotent — skips houses where cian_zhk_url IS NOT NULL. Safe to re-run. -""" -from __future__ import annotations - -import asyncio -import logging -import os -import sys -import time - -import psycopg - -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "backend")) - -from app.services.scrapers.cian_newbuilding import resolve_cian_zhk_url - -logging.basicConfig( - format="%(asctime)s [%(levelname)s] %(message)s", - level=logging.INFO, -) -logger = logging.getLogger("backfill_cian_zhk_url") - -DATABASE_URL = os.environ.get("DATABASE_URL", "") -DELAY_SEC = float(os.environ.get("DELAY_SEC", "5.0")) -BATCH_LIMIT = int(os.environ.get("BATCH_LIMIT", "500")) - - -async def main() -> None: - if not DATABASE_URL: - logger.error("DATABASE_URL env var not set") - sys.exit(1) - - conn = await psycopg.AsyncConnection.connect(DATABASE_URL) - - async with conn: - rows = await ( - await conn.execute( - """ - SELECT id, cian_internal_house_id - FROM houses - WHERE cian_internal_house_id IS NOT NULL - AND cian_zhk_url IS NULL - ORDER BY id - LIMIT %(lim)s - """, - {"lim": BATCH_LIMIT}, - ) - ).fetchall() - - total = len(rows) - logger.info( - "Found %d houses with cian_internal_house_id and no cian_zhk_url (limit=%d)", - total, - BATCH_LIMIT, - ) - if total == 0: - logger.info("Nothing to backfill.") - return - - succeeded = 0 - failed = 0 - t0 = time.time() - - conn = await psycopg.AsyncConnection.connect(DATABASE_URL) - async with conn: - for i, (house_id, cian_id) in enumerate(rows, start=1): - url = await resolve_cian_zhk_url(cian_id) - if url: - await conn.execute( - """ - UPDATE houses - SET cian_zhk_url = %(url)s - WHERE id = %(hid)s - AND cian_zhk_url IS NULL - """, - {"url": url, "hid": house_id}, - ) - await conn.commit() - succeeded += 1 - logger.info("[%d/%d] house_id=%s id=%s → %s", i, total, house_id, cian_id, url) - else: - failed += 1 - logger.warning( - "[%d/%d] house_id=%s id=%s — could not resolve URL", - i, total, house_id, cian_id, - ) - - if i < total: - await asyncio.sleep(DELAY_SEC) - - elapsed = time.time() - t0 - logger.info( - "Backfill complete: %d succeeded, %d failed, %.1fs total (%.1fs/req avg)", - succeeded, - failed, - elapsed, - elapsed / max(succeeded + failed, 1), - ) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/tradein-mvp/scripts/probe-avito-1anchor.py b/tradein-mvp/scripts/probe-avito-1anchor.py deleted file mode 100644 index cdcbbbbc..00000000 --- a/tradein-mvp/scripts/probe-avito-1anchor.py +++ /dev/null @@ -1,103 +0,0 @@ -"""Avito probe: 1 anchor × 1 page с настраиваемой задержкой. - -Минимальный dry-run чтобы проверить bans / detection после anti-bot -hardening (PR #487) + global delay setting (PR #485). - -Setup: - ssh -f -N -L 15433:172.20.0.2:5432 gendesign - cd tradein-mvp/backend - $env:DATABASE_URL = 'postgresql+psycopg://tradein:@localhost:15433/tradein' - -Usage: - uv run python ../scripts/probe-avito-1anchor.py - uv run python ../scripts/probe-avito-1anchor.py --delay 5 --radius 1000 -""" -from __future__ import annotations - -import argparse -import asyncio -import sys -import time -import types -from pathlib import Path - -_BACKEND_DIR = Path(__file__).resolve().parent.parent / "backend" -if str(_BACKEND_DIR) not in sys.path: - sys.path.insert(0, str(_BACKEND_DIR)) - -for _m in ( - "weasyprint", - "weasyprint.text", - "weasyprint.text.ffi", - "weasyprint.css", - "weasyprint.css.targets", -): - sys.modules[_m] = types.ModuleType(_m) -sys.modules["weasyprint"].HTML = type("HTML", (), {"__init__": lambda self, *a, **kw: None}) -sys.modules["weasyprint"].CSS = type("CSS", (), {"__init__": lambda self, *a, **kw: None}) - -from app.core.db import SessionLocal # noqa: E402 -from app.services.scrapers.avito import AvitoScraper # noqa: E402 -from app.services.scrapers.base import save_listings # noqa: E402 - -ANCHOR_NAME = "Центр ЕКБ" -ANCHOR_LAT = 56.8400 -ANCHOR_LON = 60.6050 - - -async def main() -> None: - p = argparse.ArgumentParser() - p.add_argument("--delay", type=float, default=3.0, help="sec между HTTP requests") - p.add_argument("--radius", type=int, default=1500, help="метры") - p.add_argument("--pages", type=int, default=1) - args = p.parse_args() - - print( - f"=== Avito probe ===\n" - f" anchor: {ANCHOR_NAME} ({ANCHOR_LAT}, {ANCHOR_LON})\n" - f" radius: {args.radius}m\n" - f" pages: {args.pages}\n" - f" delay: {args.delay}s\n", - flush=True, - ) - - t0 = time.perf_counter() - try: - async with AvitoScraper() as s: - lots = await s.fetch_around( - ANCHOR_LAT, ANCHOR_LON, args.radius, - pages=args.pages, delay_override_sec=args.delay, - ) - except Exception as e: # noqa: BLE001 - print(f"FETCH FAIL: {type(e).__name__}: {e}", flush=True) - return - - fetched = len(lots) - print(f"fetched: {fetched} lots in {time.perf_counter() - t0:.1f}s", flush=True) - if fetched == 0: - print("⚠️ 0 lots — вероятно 429/ban. Смотри backend logs.", flush=True) - return - - db = SessionLocal() - try: - ins, upd = save_listings(db, lots) - print(f"saved: ins={ins} upd={upd}", flush=True) - except Exception as e: # noqa: BLE001 - print(f"SAVE FAIL: {type(e).__name__}: {e}", flush=True) - finally: - db.close() - - sample = lots[:3] - print("\nsample listings:", flush=True) - for lot in sample: - addr = (lot.address or "")[:60] - print( - f" source_id={lot.source_id} price={lot.price_rub}₽ " - f"rooms={lot.rooms} area={lot.area_m2}м² floor={lot.floor}/{lot.total_floors} " - f"addr={addr!r}", - flush=True, - ) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/tradein-mvp/scripts/probe-imv-by-address.py b/tradein-mvp/scripts/probe-imv-by-address.py deleted file mode 100644 index 2cc7f272..00000000 --- a/tradein-mvp/scripts/probe-imv-by-address.py +++ /dev/null @@ -1,83 +0,0 @@ -"""Diagnose IMV API behavior for a specific address. - -Usage: - python scripts/probe-imv-by-address.py "Свердловская область, г. Екатеринбург, Склад, ул. Заводская, д. 44-а" - -Prints: raw API response, error category, cleaned-address retry result. -""" -import asyncio -import logging -import sys - -logging.basicConfig(level=logging.DEBUG, format="%(asctime)s %(levelname)s %(message)s") - - -async def main(address: str) -> None: - from app.services.scrapers.avito_imv import ( - IMVAddressNotFoundError, - IMVAuthError, - IMVTransientError, - evaluate_via_imv, - ) - - # Default params for ЕКБ 3-room 60м² - params = dict( - rooms=3, - area_m2=60.0, - floor=5, - floor_at_home=17, - house_type="panel", - renovation_type="cosmetic", - has_balcony=False, - has_loggia=False, - ) - - print(f"\n=== Attempt 1: original address ===\n{address}\n") - try: - result = await evaluate_via_imv(address=address, **params) # type: ignore[arg-type] - print(f"OK: recommended={result.recommended_price}") - return - except IMVAddressNotFoundError as e: - print(f"FAIL: address not found (IMVAddressNotFoundError): {e}") - except IMVAuthError as e: - print(f"FAIL: auth/quota error (IMVAuthError): {e}") - return - except IMVTransientError as e: - print(f"FAIL: transient/network error (IMVTransientError): {e}") - return - except Exception as e: - print(f"FAIL: {type(e).__name__}: {e}") - return - - import re - - cleaned = re.sub( - r"(Склад|Гараж|Подсобка|Нежилое|Помещение|Цех),\s*", - "", - address, - flags=re.IGNORECASE, - ) - if cleaned != address: - print(f"\n=== Attempt 2: cleaned address ===\n{cleaned}\n") - try: - result = await evaluate_via_imv(address=cleaned, **params) # type: ignore[arg-type] - print(f"OK on retry: recommended={result.recommended_price}") - except IMVAddressNotFoundError as e: - print(f"FAIL on retry (IMVAddressNotFoundError): {e}") - except IMVAuthError as e: - print(f"FAIL on retry (IMVAuthError — quota/auth): {e}") - except IMVTransientError as e: - print(f"FAIL on retry (IMVTransientError — network/5xx): {e}") - except Exception as e: - print(f"FAIL on retry: {type(e).__name__}: {e}") - else: - print("\nNo noise prefix found — cleaned address identical, skipping retry.") - - -if __name__ == "__main__": - _address = ( - sys.argv[1] - if len(sys.argv) > 1 - else "Свердловская область, г. Екатеринбург, Склад, ул. Заводская, д. 44-а" - ) - asyncio.run(main(_address)) diff --git a/tradein-mvp/scripts/probe-parser-region-filter.py b/tradein-mvp/scripts/probe-parser-region-filter.py deleted file mode 100644 index e0ef87b5..00000000 --- a/tradein-mvp/scripts/probe-parser-region-filter.py +++ /dev/null @@ -1,49 +0,0 @@ -"""Прогнать current parser на HTML из Playwright debug-страницы (своб.планир p=10). - -Проверяет что URL-filter работает: до фикса parser возвращал ~30 карточек других -городов, после — должен 0 или только ЕКБ. -""" -from __future__ import annotations - -import asyncio -import sys -import types -from pathlib import Path - -_BACKEND = Path(__file__).resolve().parent.parent / "backend" -if str(_BACKEND) not in sys.path: - sys.path.insert(0, str(_BACKEND)) -for _m in ("weasyprint", "weasyprint.text", "weasyprint.text.ffi", "weasyprint.css", "weasyprint.css.targets"): - sys.modules[_m] = types.ModuleType(_m) -sys.modules["weasyprint"].HTML = type("HTML", (), {"__init__": lambda self, *a, **kw: None}) -sys.modules["weasyprint"].CSS = type("CSS", (), {"__init__": lambda self, *a, **kw: None}) - -from app.services.scrapers.avito import AvitoScraper # noqa: E402 - -URL = "https://www.avito.ru/ekaterinburg/kvartiry/prodam/svobodnaya_planirovka-ASgBAgICAkSSA8YQygj8zzI?p=10&s=104" - - -async def main() -> None: - async with AvitoScraper() as s: - assert s._cffi is not None - # Use scraper's own session — может попасть в 403 (curl_cffi), это OK, - # тогда переключимся на reading from Playwright MCP page. - print(f"Fetching {URL[:80]}...") - try: - r = await s._cffi.get(URL) - print(f" HTTP {r.status_code}, html size: {len(r.text)}") - if r.status_code != 200: - print(" curl_cffi 403 — Avito не отдал. Пробни через MCP / browser save.") - return - except Exception as e: # noqa: BLE001 - print(f" fetch fail: {type(e).__name__}: {e}") - return - lots = s._parse_html(r.text, source_url_base=URL) - print(f"\nParsed {len(lots)} EKB lots (после фильтра).") - for lot in lots[:5]: - print(f" {lot.source_id} {lot.source_url[:80]}") - print(f" {lot.address or '-'!r}") - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/tradein-mvp/scripts/sweep-yandex-valuation-ekb.py b/tradein-mvp/scripts/sweep-yandex-valuation-ekb.py deleted file mode 100644 index f96c8303..00000000 --- a/tradein-mvp/scripts/sweep-yandex-valuation-ekb.py +++ /dev/null @@ -1,314 +0,0 @@ -"""Bulk sweep of Yandex.Недвижимость house-history for all EKB yandex addresses. - -Iterates `SELECT DISTINCT address FROM listings WHERE source='yandex' AND region_code=66 -AND address IS NOT NULL AND is_active=TRUE` and for each address calls the same -cached fetch + save pipeline used by estimator. Idempotent — repeat runs deduplicate -via house_placement_history UNIQUE (source, ext_item_id) and external_valuations cache. - -Why this script: - - 160 orphan rows in house_placement_history have house_id=NULL because match - pipeline was wired only after PRs #526 + #531 (2026-05-24). - - 3704 yandex listings on prod with ~hundreds of unique addresses → after one - sweep we get a population of house-history rows linked to houses + with - removed_date / total_floors / source_confidence filled. - -Setup (same as local-sweep-ekb-yandex.py): - ssh -f -N -L 15433:172.20.0.2:5432 gendesign # tradein-postgres tunnel - cd tradein-mvp/backend - export DATABASE_URL='postgresql+psycopg://tradein:@localhost:15433/tradein' - export ENVIRONMENT='local' - export GLITCHTIP_DSN='' - -Usage: - uv run python ../scripts/sweep-yandex-valuation-ekb.py # full - uv run python ../scripts/sweep-yandex-valuation-ekb.py --limit 10 # smoke - uv run python ../scripts/sweep-yandex-valuation-ekb.py --max-pages 3 - uv run python ../scripts/sweep-yandex-valuation-ekb.py --skip-recent-days 0 # force re-scrape - uv run python ../scripts/sweep-yandex-valuation-ekb.py \ - --address "Екатеринбург, улица Куйбышева, 106" -""" - -from __future__ import annotations - -import argparse -import asyncio -import random -import sys -import time -import types -from pathlib import Path - -# Auto-inject backend/ in sys.path -_BACKEND_DIR = Path(__file__).resolve().parent.parent / "backend" -if str(_BACKEND_DIR) not in sys.path: - sys.path.insert(0, str(_BACKEND_DIR)) - -# Stub weasyprint (Windows GTK-less import chain compat) -for _m in ( - "weasyprint", - "weasyprint.text", - "weasyprint.text.ffi", - "weasyprint.css", - "weasyprint.css.targets", -): - sys.modules[_m] = types.ModuleType(_m) -sys.modules["weasyprint"].HTML = type("HTML", (), {"__init__": lambda self, *a, **kw: None}) -sys.modules["weasyprint"].CSS = type("CSS", (), {"__init__": lambda self, *a, **kw: None}) - -# Backend imports -from sqlalchemy import text # noqa: E402 - -from app.core.db import SessionLocal # noqa: E402 -from app.services.estimator import ( # noqa: E402 - _get_or_fetch_yandex_valuation_cached, - _save_yandex_history_items, -) - - -def _addresses_to_sweep( - limit: int | None, - skip_recent_days: int, - single_address: str | None, - require_house_number: bool = True, -) -> list[str]: - """Pull unique active EKB yandex addresses, optionally skipping recently-cached ones. - - require_house_number: filter to addresses containing a house number. Yandex's - valuation endpoint needs a specific building — bare street names ('Академическая - улица') return 0 items. Heuristic: address must contain a digit after a comma - OR end with a digit token. Default True. - """ - if single_address: - return [single_address.strip()] - - # Postgres regex for house-number tokens — used when require_house_number=True. - # Matches ", 12" / ", 12к3" / ", 5Б" anywhere in the address. - house_num_clause = " AND address ~ ',\\s*\\d+' " if require_house_number else "" - - db = SessionLocal() - try: - if skip_recent_days > 0: - sql = text( - f""" - SELECT DISTINCT l.address - FROM listings l - WHERE l.source = 'yandex' - AND l.region_code = 66 - AND l.is_active = TRUE - AND l.address IS NOT NULL - {house_num_clause} - AND NOT EXISTS ( - SELECT 1 FROM external_valuations ev - WHERE ev.source = 'yandex_valuation' - AND ev.address = l.address - AND ev.fetched_at > NOW() - (:days || ' days')::interval - ) - ORDER BY l.address - """ - ) - rows = db.execute(sql, {"days": skip_recent_days}).mappings().all() - else: - sql = text( - f""" - SELECT DISTINCT address - FROM listings - WHERE source = 'yandex' - AND region_code = 66 - AND is_active = TRUE - AND address IS NOT NULL - {house_num_clause} - ORDER BY address - """ - ) - rows = db.execute(sql).mappings().all() - addrs = [r["address"] for r in rows if r.get("address")] - finally: - db.close() - - if limit is not None and limit > 0: - addrs = addrs[:limit] - return addrs - - -async def _sweep_one_address( - address: str, - max_pages: int, - delay: float, -) -> tuple[int, int]: - """Fetch+save up to max_pages for one address. - - Pagination stop conditions (whichever fires first): - 1. Page returns 0 items (empty / past last page) - 2. Cumulative fetched >= total_objects from page-1 header (when known) - 3. max_pages reached - - Returns (items_fetched_total, items_saved_total). - """ - total_fetched = 0 - total_saved = 0 - total_objects_expected: int | None = None - seen_keys: set[tuple] = set() # dedup within sweep (page overlap protection) - - for page in range(1, max_pages + 1): - db = SessionLocal() - try: - # _get_or_fetch_yandex_valuation_cached only fetches page 1 (no `page` - # arg); cache key includes address only. Pages >= 2 go direct to scraper. - if page == 1: - result = await _get_or_fetch_yandex_valuation_cached( - db, - address=address, - ) - else: - from app.services.scrapers.yandex_valuation import YandexValuationScraper - - async with YandexValuationScraper() as scraper: - result = await scraper.fetch_house_history( - address=address, - offer_category="APARTMENT", - offer_type="SELL", - page=page, - ) - if result is None or not result.history_items: - break - - # Capture total_objects on first response (subsequent pages may not carry it) - if total_objects_expected is None and result.house.total_objects: - total_objects_expected = result.house.total_objects - - # Dedup against prior pages (Yandex sometimes repeats items at page boundaries) - new_items = [] - for it in result.history_items: - key = (it.publish_date, it.area_m2, it.floor, it.start_price) - if key not in seen_keys: - seen_keys.add(key) - new_items.append(it) - - saved = _save_yandex_history_items(db, result) - total_fetched += len(new_items) - total_saved += saved - exp_str = f"/{total_objects_expected}" if total_objects_expected is not None else "" - print( - f" page={page}/{max_pages}: items={len(result.history_items)} " - f"new={len(new_items)} saved={saved} cum={total_fetched}{exp_str}", - flush=True, - ) - # Stop if we've collected at least what the header promised - if total_objects_expected is not None and total_fetched >= total_objects_expected: - break - # Or if this page yielded zero NEW items (pure duplicates → end of feed) - if not new_items: - break - except Exception as e: - print( - f" page={page} FAIL: {type(e).__name__}: {e}", - flush=True, - ) - break - finally: - db.close() - - if page < max_pages: - await asyncio.sleep(delay * random.uniform(0.85, 1.3)) - - return total_fetched, total_saved - - -async def main() -> None: - p = argparse.ArgumentParser( - description="Bulk sweep Yandex.Недвижимость house-history for EKB yandex addresses" - ) - p.add_argument("--limit", type=int, default=None, help="Cap number of addresses (default all)") - p.add_argument( - "--max-pages", - type=int, - default=12, - help="Max pages per address (Yandex pagination cap; default 12)", - ) - p.add_argument( - "--delay", - type=float, - default=5.0, - help="Sec between page requests within one address (default 5)", - ) - p.add_argument( - "--address-delay", - type=float, - default=15.0, - help="Sec between switching to next address (default 15)", - ) - p.add_argument( - "--skip-recent-days", - type=int, - default=1, - help="Skip addresses cached within N days (default 1; 0 = no skip)", - ) - p.add_argument( - "--address", - type=str, - default=None, - help="Single address mode — sweep only this address", - ) - p.add_argument( - "--require-house-number", - dest="require_house_number", - action="store_true", - default=True, - help="Skip addresses without a house number (default: True). " - "Yandex valuation endpoint needs a specific building; street-only addresses " - "return 0 items. Use --no-require-house-number to disable.", - ) - p.add_argument( - "--no-require-house-number", - dest="require_house_number", - action="store_false", - help="Include all addresses (sweep street-only entries too — most return 0).", - ) - args = p.parse_args() - - addresses = _addresses_to_sweep( - args.limit, - args.skip_recent_days, - args.address, - require_house_number=args.require_house_number, - ) - if not addresses: - print("No addresses to sweep (all recently cached or none match query).", flush=True) - return - - print( - f"=== Sweep {len(addresses)} addresses, max_pages={args.max_pages}, " - f"delay={args.delay}s, address-delay={args.address_delay}s, " - f"skip-recent={args.skip_recent_days}d ===", - flush=True, - ) - - t0 = time.perf_counter() - grand_fetched = 0 - grand_saved = 0 - for i, addr in enumerate(addresses, 1): - print(f"\n[{i}/{len(addresses)}] {addr[:90]}", flush=True) - try: - fetched, saved = await _sweep_one_address(addr, args.max_pages, args.delay) - except Exception as e: - print(f" FATAL on this address: {type(e).__name__}: {e}", flush=True) - continue - grand_fetched += fetched - grand_saved += saved - print( - f" done: fetched={fetched} saved={saved} [grand: f={grand_fetched} s={grand_saved}]", - flush=True, - ) - if i < len(addresses): - await asyncio.sleep(args.address_delay * random.uniform(0.85, 1.3)) - - elapsed = time.perf_counter() - t0 - print( - f"\n=== ALL DONE in {elapsed / 60:.1f} min. " - f"addresses={len(addresses)} fetched={grand_fetched} saved={grand_saved} ===", - flush=True, - ) - - -if __name__ == "__main__": - asyncio.run(main()) -- 2.45.3