From ea24289f47b55cd3ebc6d0bd7ac8d9b5303c9a88 Mon Sep 17 00:00:00 2001 From: bot-backend Date: Sat, 4 Jul 2026 00:27:01 +0300 Subject: [PATCH] feat(tradein/scraper-kit): migrate admin.py debug routes + domclick ingest to kit (#2305) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Group A of the scraper_kit migration epic (#2277). Switches admin.py's manual "run parser" debug endpoints and scripts/ingest_domclick_jsonl.py from direct app.services.scrapers.* imports to their scraper_kit.providers.* equivalents, using the DI adapters (RealScraperConfig/RealMatcherAdapter/RealProxyProvider, app.services.scraper_settings.get_scraper_delay) already established by app.scheduler_main._run_kit_scheduler. Migrated: /scrape (AvitoScraper/CianScraper/YandexRealtyScraper + save_listings), scrape_avito_house, scrape_avito_detail, scrape_avito_imv, scrape_yandex_detail, scrape_yandex_valuation, scrape_cian_detail, cian_auto_login's BrowserFetcher. scripts/ingest_domclick_jsonl.py: ScrapedLot/save_listings + (now that #2307/ Group D ported a kit equivalent while this was in flight) DomClickDetailEnrichment/ save_detail_enrichment. Deliberately NOT migrated (documented in admin.py): scrape_yandex_newbuilding / scrape_cian_newbuilding — their kit equivalents (providers/{yandex,cian}/newbuilding.py) construct an internal BrowserFetcher(...) without the mandatory `endpoint` kwarg, so any call crashes/silently-fails regardless of caller-side DI. Bug lives in scraper_kit provider code, out of scope here (only consuming, not touching provider logic) — flagged as follow-up. Parity proven via tests/support/parity.py (assert_parity) against the exact functions each debug route now calls, on offline fixtures — no live network/DB. Refs #2305 --- tradein-mvp/backend/app/api/v1/admin.py | 107 +++++++++---- .../backend/scripts/ingest_domclick_jsonl.py | 16 +- .../scrapers/test_admin_avito_kit_parity.py | 129 +++++++++++++++ .../scrapers/test_admin_cian_kit_parity.py | 51 ++++++ .../test_admin_domclick_ingest_kit_parity.py | 119 ++++++++++++++ .../scrapers/test_admin_yandex_kit_parity.py | 151 ++++++++++++++++++ 6 files changed, 536 insertions(+), 37 deletions(-) create mode 100644 tradein-mvp/backend/tests/scrapers/test_admin_avito_kit_parity.py create mode 100644 tradein-mvp/backend/tests/scrapers/test_admin_cian_kit_parity.py create mode 100644 tradein-mvp/backend/tests/scrapers/test_admin_domclick_ingest_kit_parity.py create mode 100644 tradein-mvp/backend/tests/scrapers/test_admin_yandex_kit_parity.py diff --git a/tradein-mvp/backend/app/api/v1/admin.py b/tradein-mvp/backend/app/api/v1/admin.py index ff45e927..b26e3bf1 100644 --- a/tradein-mvp/backend/app/api/v1/admin.py +++ b/tradein-mvp/backend/app/api/v1/admin.py @@ -16,6 +16,34 @@ from uuid import uuid4 import httpx from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Query from pydantic import BaseModel, Field, field_validator + +# scraper_kit-миграция (#2305, Group A): debug run-parser роуты этого файла переключены +# на scraper_kit.providers.* эквиваленты (парность доказана tests/scrapers/parity — см. +# vault Scraper_Kit_Legacy_Dependency_Audit_0703). ИСКЛЮЧЕНИЯ (остаются на legacy, +# намеренно НЕ мигрированы): YandexNewbuildingScraper.fetch_jk и cian_newbuilding +# .fetch_newbuilding — их kit-эквиваленты (providers/yandex/newbuilding.py, +# providers/cian/newbuilding.py) внутри себя строят BrowserFetcher(source=...) БЕЗ +# обязательного kwarg'а endpoint → TypeError / silently-swallowed exception при любом +# вызове, независимо от DI на стороне caller'а. Это баг в scraper_kit provider-коде +# (endpoint не прокинут через config, в отличие от SERP-классов и BaseScraper.__aenter__ +# соседних провайдеров) — вне scope этой задачи (см. "только consume, не трогать +# scraper_kit provider-логику"), фиксируем как follow-up. +from scraper_kit.base import save_listings +from scraper_kit.browser_fetcher import BrowserFetcher +from scraper_kit.orchestration.pipeline import DEFAULT_REGION_CODE +from scraper_kit.providers.avito.detail import fetch_detail, save_detail_enrichment +from scraper_kit.providers.avito.houses import fetch_house_catalog, save_house_catalog_enrichment +from scraper_kit.providers.avito.imv import ( + IMVAddressNotFoundError, + evaluate_via_imv, + save_imv_evaluation, + save_imv_placement_history, +) +from scraper_kit.providers.avito.serp import AvitoScraper +from scraper_kit.providers.cian.serp import CianScraper +from scraper_kit.providers.yandex.detail import YandexDetailScraper +from scraper_kit.providers.yandex.serp import YandexRealtyScraper +from scraper_kit.providers.yandex.valuation import YandexValuationScraper from sqlalchemy import text from sqlalchemy.orm import Session @@ -33,23 +61,9 @@ from app.services.scrape_pipeline import ( run_yandex_city_sweep, run_yandex_full_load, ) -from app.services.scraper_settings import invalidate_cache -from app.services.scrapers.avito import AvitoScraper -from app.services.scrapers.avito_detail import fetch_detail, save_detail_enrichment -from app.services.scrapers.avito_houses import fetch_house_catalog, save_house_catalog_enrichment -from app.services.scrapers.avito_imv import ( - IMVAddressNotFoundError, - evaluate_via_imv, - save_imv_evaluation, - save_imv_placement_history, -) -from app.services.scrapers.base import save_listings -from app.services.scrapers.browser_fetcher import BrowserFetcher -from app.services.scrapers.cian import CianScraper -from app.services.scrapers.yandex_detail import YandexDetailScraper +from app.services.scraper_adapters import RealMatcherAdapter, RealProxyProvider, RealScraperConfig +from app.services.scraper_settings import get_scraper_delay, invalidate_cache from app.services.scrapers.yandex_newbuilding import YandexNewbuildingScraper -from app.services.scrapers.yandex_realty import YandexRealtyScraper -from app.services.scrapers.yandex_valuation import YandexValuationScraper from app.tasks.avito_detail_backfill import run_avito_detail_backfill from app.tasks.geocode_missing import GeocodeBackfillResult, geocode_missing_listings @@ -58,6 +72,18 @@ logger = logging.getLogger(__name__) router = APIRouter() +def _kit_proxy_provider() -> RealProxyProvider | None: + """RealProxyProvider только когда proxy-pool включён (#2163/#2164) — иначе None. + + Зеркалит гейт `app.scheduler_main._run_kit_scheduler`: оба флага дефолтно False + (ship-dark), поэтому debug-роуты этого файла ведут себя как раньше (env-прокси), + пока пул явно не включён. + """ + if settings.use_proxy_pool_curl or settings.use_proxy_pool_browser: + return RealProxyProvider() + return None + + def _assert_allowed_url(url: str) -> None: """SSRF guard: отклоняем абсолютные URL с хостом не из allowlist (#756). @@ -130,16 +156,28 @@ async def scrape_around( -d '{"lat":56.8332,"lon":60.5944,"radius_m":1000,"sources":["avito"]}' """ results: list[ScrapeResult] = [] + # scraper_kit DI (#2305): config/delay_provider/proxy_provider/matcher инжектируются + # снаружи вместо прямого импорта app.core.config.settings внутри скрапперов — + # тот же паттерн, что и в app.scheduler_main._run_kit_scheduler. + config = RealScraperConfig() + proxy_provider = _kit_proxy_provider() + matcher = RealMatcherAdapter() for source in payload.sources: - scraper_cls = { - "avito": AvitoScraper, - "cian": CianScraper, - "yandex": YandexRealtyScraper, - }.get(source) - if scraper_cls is None: + scraper_ctx: AvitoScraper | CianScraper | YandexRealtyScraper + if source == "avito": + scraper_ctx = AvitoScraper(config, delay_provider=get_scraper_delay) + elif source == "cian": + scraper_ctx = CianScraper( + config, delay_provider=get_scraper_delay, proxy_provider=proxy_provider + ) + elif source == "yandex": + scraper_ctx = YandexRealtyScraper( + config, delay_provider=get_scraper_delay, proxy_provider=proxy_provider + ) + else: continue - async with scraper_cls() as scraper: + async with scraper_ctx as scraper: if source == "yandex" and payload.deep_yandex: lots = await scraper.fetch_around_multi_room( payload.lat, @@ -158,7 +196,9 @@ async def scrape_around( ) else: lots = await scraper.fetch_around(payload.lat, payload.lon, payload.radius_m) - inserted, updated = save_listings(db, lots) + inserted, updated = save_listings( + db, lots, matcher=matcher, region_code=DEFAULT_REGION_CODE + ) results.append( ScrapeResult(source=source, fetched=len(lots), inserted=inserted, updated=updated) ) @@ -361,7 +401,9 @@ async def cian_auto_login( ) try: - async with BrowserFetcher(source="cian") as fetcher: + async with BrowserFetcher( + source="cian", endpoint=settings.browser_http_endpoint + ) as fetcher: raw_cookies = await fetcher.login( url=settings.cian_login_url, email=email, @@ -569,7 +611,7 @@ async def scrape_avito_house( ) from e try: - counters = save_house_catalog_enrichment(db, enrichment) + counters = save_house_catalog_enrichment(db, enrichment, matcher=RealMatcherAdapter()) except Exception as e: raise _safe_http_error( 500, "save failed", f"avito-house: save failed for {house_url}" @@ -593,7 +635,7 @@ async def scrape_avito_detail( """ _assert_allowed_url(item_url) try: - enrichment = await fetch_detail(item_url) + enrichment = await fetch_detail(item_url, config=RealScraperConfig()) except Exception as e: raise _safe_http_error( 502, "fetch failed", f"avito-detail: fetch failed for {item_url}" @@ -686,6 +728,7 @@ async def scrape_avito_imv( renovation_type=renovation_type, has_balcony=has_balcony, has_loggia=has_loggia, + config=RealScraperConfig(), ) except IMVAddressNotFoundError as e: # Ожидаемое клиентское условие (адрес не в базе Avito), НЕ сбой — logger.warning @@ -1427,7 +1470,7 @@ async def scrape_yandex_detail( debug; main pipeline writes via estimator on /estimate flow). """ _assert_allowed_url(offer_url) - async with YandexDetailScraper() as scraper: + async with YandexDetailScraper(delay_provider=get_scraper_delay) as scraper: result = await scraper.fetch_detail(offer_url) if result is None: raise HTTPException(404, f"Could not parse Yandex detail page: {offer_url}") @@ -1590,7 +1633,9 @@ async def scrape_yandex_valuation( Anonymous GET - works without auth. """ - async with YandexValuationScraper() as scraper: + async with YandexValuationScraper( + RealScraperConfig(), delay_provider=get_scraper_delay, proxy_provider=_kit_proxy_provider() + ) as scraper: result = await scraper.fetch_house_history( address=address, offer_category=offer_category, @@ -1632,9 +1677,9 @@ async def scrape_cian_detail( Without it → debug-only (no DB write). """ _assert_allowed_url(offer_url) - from app.services.scrapers.cian_detail import fetch_detail, save_detail_enrichment + from scraper_kit.providers.cian.detail import fetch_detail, save_detail_enrichment - enrichment = await fetch_detail(offer_url) + enrichment = await fetch_detail(offer_url, config=RealScraperConfig()) if enrichment is None: raise HTTPException(404, f"Could not parse Cian detail page: {offer_url}") diff --git a/tradein-mvp/backend/scripts/ingest_domclick_jsonl.py b/tradein-mvp/backend/scripts/ingest_domclick_jsonl.py index a963818d..ff0b65ce 100644 --- a/tradein-mvp/backend/scripts/ingest_domclick_jsonl.py +++ b/tradein-mvp/backend/scripts/ingest_domclick_jsonl.py @@ -20,14 +20,16 @@ import json import logging from typing import Any -from sqlalchemy import text - -from app.core.db import SessionLocal -from app.services.scrapers.base import ScrapedLot, save_listings -from app.services.scrapers.domclick_detail import ( +from scraper_kit.base import ScrapedLot, save_listings +from scraper_kit.orchestration.pipeline import DEFAULT_REGION_CODE +from scraper_kit.providers.domclick.detail import ( DomClickDetailEnrichment, save_detail_enrichment, ) +from sqlalchemy import text + +from app.core.db import SessionLocal +from app.services.scraper_adapters import RealMatcherAdapter logger = logging.getLogger(__name__) @@ -188,7 +190,9 @@ def run(jsonl_path: str, limit: int | None, dry_run: bool) -> dict[str, int]: db = SessionLocal() try: if lots: - inserted, updated = save_listings(db, lots) + inserted, updated = save_listings( + db, lots, matcher=RealMatcherAdapter(), region_code=DEFAULT_REGION_CODE + ) logger.info("save_listings: inserted=%d updated=%d", inserted, updated) # Достаём listing.id + house_id_fk одним SELECT по source_id. diff --git a/tradein-mvp/backend/tests/scrapers/test_admin_avito_kit_parity.py b/tradein-mvp/backend/tests/scrapers/test_admin_avito_kit_parity.py new file mode 100644 index 00000000..8540169f --- /dev/null +++ b/tradein-mvp/backend/tests/scrapers/test_admin_avito_kit_parity.py @@ -0,0 +1,129 @@ +"""Parity-proof: admin.py Avito debug-роуты (#2305, Group A) — legacy vs scraper_kit. + +`app/api/v1/admin.py::scrape_avito_house` / `scrape_avito_imv` теперь зовут +`scraper_kit.providers.avito.{houses,imv}` вместо `app.services.scrapers.avito_*` +(тот же `/scrape` root-роут — `scraper_kit.providers.avito.serp.AvitoScraper`). +Каждая пара функций ниже — чистое, offline-тестируемое parsing-ядро, которое +реально используется соответствующим debug-эндпоинтом (не сам HTTP-fetch — +см. `tests/support/README.md`: "Live network/DB в parity-тестах избегайте"). + +- `_parse_house_page` — `scrape_avito_house` → `fetch_house_catalog` парсит виджет + `housePage` этой функцией. +- `_parse_price` — `scrape_avito_imv` → `evaluate_via_imv` парсит top-level `price` + блок ответа `realty-imv/get-data` этой функцией. +- `_extract_rooms_from_title` / `_extract_area_from_title` — использует + `AvitoScraper` (SERP-парсинг `/scrape` root-роута) для title-fallback полей. +""" + +from __future__ import annotations + +import dataclasses +import os + +# Соседние модули app.services.scrapers.* импортируют app.core.config при импорте +# пакета — фиктивный DSN мостит офлайн-запуск (mirror test_avito_detail_kit_parity.py). +os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db") + +from scraper_kit.providers.avito.houses import _parse_house_page as kit_parse_house_page +from scraper_kit.providers.avito.imv import _parse_price as kit_parse_price +from scraper_kit.providers.avito.serp import ( + _extract_area_from_title as kit_extract_area_from_title, +) +from scraper_kit.providers.avito.serp import ( + _extract_rooms_from_title as kit_extract_rooms_from_title, +) + +from app.services.scrapers.avito import _extract_area_from_title as legacy_extract_area_from_title +from app.services.scrapers.avito import ( + _extract_rooms_from_title as legacy_extract_rooms_from_title, +) +from app.services.scrapers.avito_houses import _parse_house_page as legacy_parse_house_page +from app.services.scrapers.avito_imv import _parse_price as legacy_parse_price +from tests.support.parity import assert_parity + +_HOUSE_WIDGET = { + "props": { + "developmentData": { + "avitoId": 3171365, + "id": "abc123hash", + "title": "ЖК Тестовый", + "address": "ул. Тестовая, 1", + "fullAddress": "Екатеринбург, ул. Тестовая, 1", + "coords": {"lat": 56.8, "lng": 60.6}, + "aboutDevelopment": { + "expandParams": { + "items": [ + { + "params": [ + {"type": "Год постройки", "value": "2020"}, + {"type": "Этажей", "value": "16"}, + {"type": "Пассажирский лифт", "value": "2"}, + ] + } + ] + } + }, + "ratingBadge": {"info": {"score": 4.5, "scoreString": "4.5"}}, + "ratingSummaryStat": {"reviewCount": 10, "ratingStat": []}, + "mapPreview": {"objects": "школа рядом", "distance": "5 мин", "pins": []}, + "developer": {"name": "ООО Застройщик", "key": "dev1"}, + } + } +} + +_IMV_GETDATA = { + "price": { + "recommendedPrice": 8500000, + "lowerPrice": 7800000, + "higherPrice": 9200000, + "count": 42, + "addItemUrl": "/add", + } +} + + +def test_avito_house_page_parse_are_class_distinct() -> None: + """Sanity: legacy/kit HouseInfo — разные классы, обычный `==` даёт False.""" + legacy_result = legacy_parse_house_page(_HOUSE_WIDGET) + kit_result = kit_parse_house_page(_HOUSE_WIDGET) + + assert type(legacy_result) is not type(kit_result) + assert legacy_result != kit_result + assert dataclasses.asdict(legacy_result) == dataclasses.asdict(kit_result) + + +def test_parity_avito_house_page() -> None: + """`scrape_avito_house` (admin.py) → fetch_house_catalog → _parse_house_page.""" + assert_parity( + legacy_fn=legacy_parse_house_page, + kit_fn=kit_parse_house_page, + fixtures=[(_HOUSE_WIDGET,)], + ) + + +def test_parity_avito_imv_price() -> None: + """`scrape_avito_imv` (admin.py) → evaluate_via_imv → _parse_price.""" + assert_parity( + legacy_fn=legacy_parse_price, + kit_fn=kit_parse_price, + fixtures=[(_IMV_GETDATA,)], + ) + + +def test_parity_avito_serp_title_extract() -> None: + """`/scrape` root-роут (admin.py) → AvitoScraper SERP title-fallback fields.""" + titles = [ + "2-к. квартира, 54.3 м², 3/9 эт.", + "Студия, 25 м², 1/5 эт.", + "3-к. кв., 75,5 м², 12/16 эт.", + ] + assert_parity( + legacy_fn=legacy_extract_rooms_from_title, + kit_fn=kit_extract_rooms_from_title, + fixtures=titles, + ) + assert_parity( + legacy_fn=legacy_extract_area_from_title, + kit_fn=kit_extract_area_from_title, + fixtures=titles, + ) diff --git a/tradein-mvp/backend/tests/scrapers/test_admin_cian_kit_parity.py b/tradein-mvp/backend/tests/scrapers/test_admin_cian_kit_parity.py new file mode 100644 index 00000000..39fdc616 --- /dev/null +++ b/tradein-mvp/backend/tests/scrapers/test_admin_cian_kit_parity.py @@ -0,0 +1,51 @@ +"""Parity-proof: admin.py `scrape_cian_detail` (#2305, Group A) — legacy vs scraper_kit. + +`app/api/v1/admin.py::scrape_cian_detail` теперь зовёт +`scraper_kit.providers.cian.detail.fetch_detail/save_detail_enrichment` вместо +`app.services.scrapers.cian_detail`. `_extract_price_changes` — чистая, +offline-тестируемая normalize-функция, которую `fetch_detail` вызывает при +сборке `DetailEnrichment.price_changes` (см. `tests/support/README.md`: +"Live network/DB в parity-тестах избегайте"). +""" + +from __future__ import annotations + +import os + +# cian_detail.py импортирует app.core.config при module-level import — фиктивный +# DSN мостит офлайн-запуск (mirror test_avito_detail_kit_parity.py). +os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db") + +from scraper_kit.providers.cian.detail import _extract_price_changes as kit_extract_price_changes + +from app.services.scrapers.cian_detail import ( + _extract_price_changes as legacy_extract_price_changes, +) +from tests.support.parity import assert_parity + +# Три реальных формы priceChanges: curl_cffi flat, browser-rendered nested, оба-пусто. +_OFFER_FLAT = { + "priceChanges": [ + {"changeTime": "2026-01-15T00:00:00", "price": 9000000, "diffPercent": -2.5}, + {"changeTime": "2026-02-01T00:00:00", "priceRub": 8800000, "diff_percent": -2.2}, + ] +} +_OFFER_DATA_NESTED = { + "priceChanges": [ + {"changeTime": "2026-01-15T00:00:00", "priceData": {"price": 9000000}}, + ] +} +_STATE_EMPTY: dict[str, object] = {} + + +def test_parity_cian_detail_price_changes() -> None: + """`scrape_cian_detail` (admin.py) → fetch_detail → _extract_price_changes.""" + assert_parity( + legacy_fn=legacy_extract_price_changes, + kit_fn=kit_extract_price_changes, + fixtures=[ + (_OFFER_FLAT, {}, _STATE_EMPTY), + ({}, _OFFER_DATA_NESTED, _STATE_EMPTY), + ({}, {}, _STATE_EMPTY), + ], + ) diff --git a/tradein-mvp/backend/tests/scrapers/test_admin_domclick_ingest_kit_parity.py b/tradein-mvp/backend/tests/scrapers/test_admin_domclick_ingest_kit_parity.py new file mode 100644 index 00000000..5da9e12e --- /dev/null +++ b/tradein-mvp/backend/tests/scrapers/test_admin_domclick_ingest_kit_parity.py @@ -0,0 +1,119 @@ +"""Parity-proof: `scripts/ingest_domclick_jsonl.py` (#2305, Group A) — legacy vs scraper_kit. + +`scripts/ingest_domclick_jsonl.py` теперь строит `ScrapedLot` из +`scraper_kit.base` (вместо `app.services.scrapers.base`) и вызывает +`scraper_kit.base.save_listings` (matcher/region_code инжектируются — см. +`app/services/scraper_adapters.RealMatcherAdapter` + `DEFAULT_REGION_CODE`). + +`DomClickDetailEnrichment`/`save_detail_enrichment` изначально планировались +остаться на legacy: на момент начала миграции admin.py `domclick_detail.py` не +имел kit-эквивалента (см. audit `Scraper_Kit_Legacy_Dependency_Audit_0703`). +Issue #2307 (Group D, PR #2318) портировал `domclick_detail.py` в kit ПАРАЛЛЕЛЬНО +с этой задачей и явно оставил `scripts/ingest_domclick_jsonl.py` untouched +("the only other domclick_detail caller is scripts/ingest_domclick_jsonl.py, +also untouched") — поэтому переключение этого caller'а сделано ЗДЕСЬ, в рамках +#2305, теперь когда kit-эквивалент существует. `parse_detail_html`/exception- +parity для domclick_detail уже исчерпывающе доказаны в +`tests/scrapers/test_domclick_detail_kit_parity.py` (issue #2307) — этот файл +НЕ дублирует то доказательство, а фокусируется на конкретной точке потребления: +`ScrapedLot`/`DomClickDetailEnrichment`, сконструированные ИЗ ТЕХ ЖЕ kwargs, что +и `_build_lot()`/`_build_enrichment()` в самом скрипте. +""" + +from __future__ import annotations + +import os + +os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db") + +from scraper_kit.base import ScrapedLot as KitScrapedLot +from scraper_kit.providers.domclick.detail import ( + DomClickDetailEnrichment as KitDomClickDetailEnrichment, +) + +from app.services.scrapers.base import ScrapedLot as LegacyScrapedLot +from app.services.scrapers.domclick_detail import ( + DomClickDetailEnrichment as LegacyDomClickDetailEnrichment, +) +from tests.support.parity import assert_parity + +# Аналог одной ok-записи JSONL-раннера (см. ingest_domclick_jsonl._build_lot). +_LOT_KWARGS = { + "source": "domklik", + "source_url": "https://ekaterinburg.domclick.ru/card/sale__flat__2075729321", + "source_id": "2075729321", + "address": "Екатеринбург, ул. Тестовая, 1", + "lat": 56.83, + "lon": 60.6, + "rooms": 2, + "area_m2": 54.3, + "floor": 5, + "total_floors": 9, + "year_built": 2015, + "price_rub": 8500000, + "listing_segment": "vtorichka", + "repair_state": "euro", + "raw_payload": {"is_sber_collateral": False, "square_price": 156538, "avm": {}}, +} + +# Аналог одной ok-записи JSONL-раннера (см. ingest_domclick_jsonl._build_enrichment). +_ENRICHMENT_KWARGS = { + "item_id": "2075729321", + "source_url": "https://ekaterinburg.domclick.ru/card/sale__flat__2075729321", + "repair_state": "euro", + "repair_type": "евроремонт", + "living_area_m2": 32.1, + "kitchen_area_m2": 9.5, + "sale_type": "free", + "owners_count": 1, + "encumbrances_clean": True, + "year_built": 2015, + "views_total": 338, + "price_changes": [{"change_time": "2026-01-15T00:00:00", "price_rub": 8500000}], + "raw_extra": {"wall_type": "кирпично-монолитный", "calls": 5}, +} + + +def _build_legacy_lot(rec: dict[str, object]) -> LegacyScrapedLot: + return LegacyScrapedLot(**rec) # type: ignore[arg-type] + + +def _build_kit_lot(rec: dict[str, object]) -> KitScrapedLot: + return KitScrapedLot(**rec) # type: ignore[arg-type] + + +def _build_legacy_enrichment(rec: dict[str, object]) -> LegacyDomClickDetailEnrichment: + return LegacyDomClickDetailEnrichment(**rec) # type: ignore[arg-type] + + +def _build_kit_enrichment(rec: dict[str, object]) -> KitDomClickDetailEnrichment: + return KitDomClickDetailEnrichment(**rec) # type: ignore[arg-type] + + +def test_scraped_lot_are_class_distinct() -> None: + """Sanity: legacy/kit ScrapedLot — разные классы (pydantic), обычный `==` даёт False.""" + legacy_result = _build_legacy_lot(_LOT_KWARGS) + kit_result = _build_kit_lot(_LOT_KWARGS) + + assert type(legacy_result) is not type(kit_result) + assert legacy_result.model_dump() == kit_result.model_dump() + + +def test_parity_ingest_domclick_scraped_lot() -> None: + """`ingest_domclick_jsonl._build_lot()` kwargs → ScrapedLot(legacy) vs ScrapedLot(kit).""" + assert_parity( + legacy_fn=_build_legacy_lot, + kit_fn=_build_kit_lot, + fixtures=[_LOT_KWARGS], + ) + + +def test_parity_ingest_domclick_enrichment() -> None: + """`ingest_domclick_jsonl._build_enrichment()` kwargs → + DomClickDetailEnrichment(legacy) vs DomClickDetailEnrichment(kit) — + см. module docstring про переключение этого caller'а в рамках #2305.""" + assert_parity( + legacy_fn=_build_legacy_enrichment, + kit_fn=_build_kit_enrichment, + fixtures=[_ENRICHMENT_KWARGS], + ) diff --git a/tradein-mvp/backend/tests/scrapers/test_admin_yandex_kit_parity.py b/tradein-mvp/backend/tests/scrapers/test_admin_yandex_kit_parity.py new file mode 100644 index 00000000..92470a0e --- /dev/null +++ b/tradein-mvp/backend/tests/scrapers/test_admin_yandex_kit_parity.py @@ -0,0 +1,151 @@ +"""Parity-proof: admin.py Yandex debug-роуты (#2305, Group A) — legacy vs scraper_kit. + +`app/api/v1/admin.py::scrape_yandex_detail` / `scrape_yandex_valuation` теперь +зовут `scraper_kit.providers.yandex.{detail,valuation}` вместо +`app.services.scrapers.yandex_{detail,valuation}` (тот же `/scrape` root-роут — +`scraper_kit.providers.yandex.serp.YandexRealtyScraper`). Каждая функция ниже — +чистое, offline-тестируемое parsing-ядро (см. `tests/support/README.md`: +"Live network/DB в parity-тестах избегайте"). + +- `_parse_card_fields` — `scrape_yandex_detail` → `YandexDetailScraper.fetch_detail` + извлекает (rooms, area, living, kitchen, ceiling, floor, total_floors) этой функцией. +- `YandexValuationScraper._parse_house_meta` (@staticmethod) — `scrape_yandex_valuation` + → `fetch_house_history` извлекает house-level метаданные этой функцией. +- `_entity_to_lot` — `/scrape` root-роут → `YandexRealtyScraper.fetch_around` + конвертирует gate-API entity → ScrapedLot этой функцией (также доказывает + ScrapedLot-контракт, общий с `save_listings`/`scripts/ingest_domclick_jsonl.py`). +""" + +from __future__ import annotations + +import os + +os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db") + +from scraper_kit.providers.yandex.detail import _parse_card_fields as kit_parse_card_fields +from scraper_kit.providers.yandex.serp import _entity_to_lot as kit_entity_to_lot +from scraper_kit.providers.yandex.valuation import ( + YandexValuationScraper as KitYandexValuationScraper, +) + +from app.services.scrapers.yandex_detail import _parse_card_fields as legacy_parse_card_fields +from app.services.scrapers.yandex_realty import _entity_to_lot as legacy_entity_to_lot +from app.services.scrapers.yandex_valuation import ( + YandexValuationScraper as LegacyYandexValuationScraper, +) +from tests.support.parity import assert_parity + +_CARD = { + "roomsTotal": 2, + "area": {"value": 54.3}, + "livingSpace": {"value": 32.1}, + "kitchenSpace": {"value": 9.5}, + "ceilingHeight": 2.7, + "floorsOffered": [5], + "floorsTotal": 9, +} +_CARD_STUDIO = { + "house": {"studio": True}, + "area": {"value": 25.0}, + "floorsOffered": [1], + "floorsTotal": 5, +} + +_VALUATION_BODY_TEXT = ( + "Дом 2015 года. 9 этажей. 2,7 м потолки. Лифт есть. 120 объектов продажи. Панорама двора." +) + +# gate-API entity (realty.yandex.ru), без creationDate/author — исключает +# time-dependent поля (days_on_market) из фикстуры (deterministic offline). +_YANDEX_ENTITY = { + "offerId": "12345", + "url": "//realty.yandex.ru/offer/12345", + "price": {"value": 8500000, "valuePerPart": 157000}, + "area": {"value": 54.3}, + "livingSpace": {"value": 32.1}, + "kitchenSpace": {"value": 9.5}, + "roomsTotal": 2, + "floorsOffered": [5], + "floorsTotal": 9, + "ceilingHeight": 2.7, + "building": {"builtYear": 2015, "buildingType": "MONOLIT"}, + "location": { + "point": {"latitude": 56.83, "longitude": 60.6}, + "geocoderAddress": "Екатеринбург, ул. Тестовая, 1", + }, + "mainImages": ["//avatars.mds.yandex.net/img1.jpg"], +} + + +def test_parity_yandex_detail_card_fields() -> None: + """`scrape_yandex_detail` (admin.py) → fetch_detail → _parse_card_fields.""" + assert_parity( + legacy_fn=legacy_parse_card_fields, + kit_fn=kit_parse_card_fields, + fixtures=[(_CARD,), (_CARD_STUDIO,)], + ) + + +def test_yandex_valuation_house_meta_are_class_distinct() -> None: + """Sanity: legacy/kit ValuationHouseMeta — разные классы, обычный `==` даёт False.""" + legacy_result = LegacyYandexValuationScraper._parse_house_meta(_VALUATION_BODY_TEXT) + kit_result = KitYandexValuationScraper._parse_house_meta(_VALUATION_BODY_TEXT) + + assert type(legacy_result) is not type(kit_result) + assert legacy_result.model_dump() == kit_result.model_dump() + + +def test_parity_yandex_valuation_house_meta() -> None: + """`scrape_yandex_valuation` (admin.py) → fetch_house_history → _parse_house_meta.""" + assert_parity( + legacy_fn=LegacyYandexValuationScraper._parse_house_meta, + kit_fn=KitYandexValuationScraper._parse_house_meta, + fixtures=[_VALUATION_BODY_TEXT], + ) + + +def test_yandex_entity_to_lot_are_class_distinct() -> None: + """Sanity: legacy/kit ScrapedLot — разные классы (pydantic), обычный `==` даёт False.""" + legacy_result = legacy_entity_to_lot(_YANDEX_ENTITY) + kit_result = kit_entity_to_lot(_YANDEX_ENTITY) + + assert legacy_result is not None + assert kit_result is not None + assert type(legacy_result) is not type(kit_result) + assert legacy_result.model_dump() == kit_result.model_dump() + + +def test_parity_yandex_serp_entity_to_lot() -> None: + """`/scrape` root-роут (admin.py) → YandexRealtyScraper.fetch_around → _entity_to_lot. + + Также доказывает ScrapedLot-контракт, общий с `save_listings` + (используется и `/scrape`, и `scripts/ingest_domclick_jsonl.py`). + """ + assert_parity( + legacy_fn=legacy_entity_to_lot, + kit_fn=kit_entity_to_lot, + fixtures=[(_YANDEX_ENTITY,)], + ) + + +def test_parity_harness_catches_yandex_entity_divergence() -> None: + """Harness реально ЛОВИТ расхождение — мутируем price_rub в kit-результате.""" + import pytest + + from tests.support.parity import ParityMismatchError + + legacy_result = legacy_entity_to_lot(_YANDEX_ENTITY) + kit_result = kit_entity_to_lot(_YANDEX_ENTITY) + assert legacy_result is not None + assert kit_result is not None + + # ScrapedLot — pydantic BaseModel (НЕ dataclass) → model_copy, не dataclasses.replace. + mutated_kit_result = kit_result.model_copy(update={"price_rub": kit_result.price_rub + 1}) + + with pytest.raises(ParityMismatchError) as exc_info: + assert_parity( + legacy_fn=lambda: legacy_result, + kit_fn=lambda: mutated_kit_result, + fixtures=[()], + ) + assert "price_rub" in str(exc_info.value)