lekss361
b7abaeb569
Some checks failed
Deploy Trade-In / changes (push) Successful in 12s
Deploy Trade-In / build-frontend (push) Has been skipped
Deploy Trade-In / build-browser (push) Has been skipped
Deploy Trade-In / test (push) Successful in 1m28s
Deploy Trade-In / build-backend (push) Successful in 1m24s
Deploy Trade-In / deploy (push) Has been cancelled
183 lines
8.8 KiB
Python
183 lines
8.8 KiB
Python
"""Golden-parity: `scraper_kit.providers.domclick.serp` ≡ `app.services.scrapers.domclick`.
|
||
|
||
Strangler-инвариант (#2133): новая scraper_kit-копия domclick-провайдера должна
|
||
давать БАЙТ-ИДЕНТИЧНЫЙ результат парсинга старому боевому коду на одинаковом входе.
|
||
Развязка (get_scraper_delay→delay_provider, base/exceptions/repair-normalizer→
|
||
scraper_kit.*) НЕ меняет распарсенные данные.
|
||
|
||
Гоняем ОБА модуля на одной BFF-JSON-фикстуре и сравниваем:
|
||
- `_extract_json` (JSON из HTML-обёртки + QRATOR-block detection);
|
||
- `_map_item` (offer → ScrapedLot): студия rooms=0, sentinel 0→None, agency;
|
||
- `_is_geo_ok` (bbox + offerRegionName guard);
|
||
- url-билдеры и чистые хелперы (_build_offers_url/_build_count_url/
|
||
_parse_publish_date/_extract_agency_name).
|
||
|
||
Идентичность результата = kit-парсер верно скопирован (развязка не задела логику).
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import os
|
||
from pathlib import Path
|
||
from types import SimpleNamespace
|
||
from typing import Any
|
||
from unittest.mock import patch
|
||
|
||
# Старый app.services.scrapers.domclick тянет app.core.config.settings=Settings(),
|
||
# которому нужен DATABASE_URL. Офлайн-парсинг БД не трогает — фиктивный DSN достаточен.
|
||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
|
||
|
||
# НОВЫЙ (scraper_kit) — ruff сортирует scraper_kit (first-party) выше app-импортов.
|
||
from scraper_kit.providers.domclick.serp import DomClickScraper as NewDomClickScraper
|
||
from scraper_kit.providers.domclick.serp import _build_count_url as new_build_count_url
|
||
from scraper_kit.providers.domclick.serp import _build_offers_url as new_build_offers_url
|
||
from scraper_kit.providers.domclick.serp import _extract_agency_name as new_extract_agency
|
||
from scraper_kit.providers.domclick.serp import _extract_json as new_extract_json
|
||
from scraper_kit.providers.domclick.serp import _parse_publish_date as new_parse_publish_date
|
||
|
||
from app.services.scrapers.domclick import DomClickScraper as OldDomClickScraper
|
||
from app.services.scrapers.domclick import _build_count_url as old_build_count_url
|
||
from app.services.scrapers.domclick import _build_offers_url as old_build_offers_url
|
||
from app.services.scrapers.domclick import _extract_agency_name as old_extract_agency
|
||
from app.services.scrapers.domclick import _extract_json as old_extract_json
|
||
from app.services.scrapers.domclick import _parse_publish_date as old_parse_publish_date
|
||
|
||
_FIXTURES = Path(__file__).parent / "fixtures"
|
||
_BFF_JSON = (_FIXTURES / "domclick_bff_offers_sample.json").read_text(encoding="utf-8")
|
||
_BFF_ITEMS: list[dict[str, Any]] = json.loads(_BFF_JSON)["result"]["items"]
|
||
|
||
|
||
def _make_old() -> OldDomClickScraper:
|
||
"""Старый DomClickScraper без DB: patch get_scraper_delay."""
|
||
with patch("app.services.scrapers.domclick.get_scraper_delay", return_value=8.0):
|
||
return OldDomClickScraper()
|
||
|
||
|
||
def _make_new() -> NewDomClickScraper:
|
||
"""Новый DomClickScraper с инжектируемым ScraperConfig + delay_provider.
|
||
|
||
Парсинг-тесты не дёргают fetch_city → browser_http_endpoint не используется,
|
||
но конструктор требует config (duck-typed namespace достаточен).
|
||
"""
|
||
config = SimpleNamespace(browser_http_endpoint="http://tradein-browser:9000")
|
||
return NewDomClickScraper(config, delay_provider=lambda _name: 8.0)
|
||
|
||
|
||
# ── _extract_json parity ──────────────────────────────────────────────────────
|
||
|
||
|
||
def test_extract_json_parity_bare() -> None:
|
||
"""_extract_json на bare JSON-теле идентичен."""
|
||
assert old_extract_json(_BFF_JSON) == new_extract_json(_BFF_JSON)
|
||
|
||
|
||
def test_extract_json_parity_pre_wrapped() -> None:
|
||
"""_extract_json на <pre>-обёрнутом JSON идентичен (реальная форма ответа)."""
|
||
wrapped = f"<html><body><pre>{_BFF_JSON}</pre></body></html>"
|
||
assert old_extract_json(wrapped) == new_extract_json(wrapped)
|
||
|
||
|
||
def test_extract_json_parity_qrator_block() -> None:
|
||
"""QRATOR block-страница → DomClickBlockedError в обоих (идентичное поведение)."""
|
||
from scraper_kit.domclick_exceptions import DomClickBlockedError as NewBlocked
|
||
|
||
from app.services.scrapers.domclick_exceptions import DomClickBlockedError as OldBlocked
|
||
|
||
block_html = "<html><body>Система защиты QRATOR: captcha</body></html>"
|
||
old_raised = new_raised = False
|
||
try:
|
||
old_extract_json(block_html)
|
||
except OldBlocked:
|
||
old_raised = True
|
||
try:
|
||
new_extract_json(block_html)
|
||
except NewBlocked:
|
||
new_raised = True
|
||
assert old_raised and new_raised
|
||
|
||
|
||
# ── _map_item parity (offer → ScrapedLot) ─────────────────────────────────────
|
||
|
||
|
||
def test_map_item_parity_all_items() -> None:
|
||
"""Каждый offer-item → ScrapedLot идентичен (студия rooms=0, sentinel 0→None)."""
|
||
old = _make_old()
|
||
new = _make_new()
|
||
for item in _BFF_ITEMS:
|
||
# force_rooms=0 для студий (st-бакет) — проверяем обе ветки маппинга.
|
||
for force_rooms in (None, 0):
|
||
old_lot = old._map_item(item, force_rooms=force_rooms)
|
||
new_lot = new._map_item(item, force_rooms=force_rooms)
|
||
if old_lot is None or new_lot is None:
|
||
assert old_lot is None and new_lot is None
|
||
continue
|
||
assert old_lot.model_dump() == new_lot.model_dump()
|
||
# Счётчик parse-ошибок эволюционирует одинаково.
|
||
assert old.parse_failures == new.parse_failures
|
||
|
||
|
||
# ── _is_geo_ok parity (bbox + region guard) ───────────────────────────────────
|
||
|
||
|
||
def test_is_geo_ok_parity_all_items() -> None:
|
||
"""Гео-гард (offerRegionName + bbox) идентичен на всех items."""
|
||
old = _make_old()
|
||
new = _make_new()
|
||
for item in _BFF_ITEMS:
|
||
assert old._is_geo_ok(item) == new._is_geo_ok(item)
|
||
|
||
|
||
# ── url-билдеры + чистые хелперы ──────────────────────────────────────────────
|
||
|
||
|
||
def test_url_builders_parity() -> None:
|
||
"""_build_offers_url / _build_count_url идентичны (вкл. '5+' → '5%2B')."""
|
||
cases: list[tuple[str, int | None, int | None]] = [
|
||
("st", None, None),
|
||
("2", 3_000_000, 8_000_000),
|
||
("5+", 15_000_000, None),
|
||
("1", None, 5_000_000),
|
||
]
|
||
for rooms, gte, lte in cases:
|
||
assert old_build_count_url(rooms, gte, lte) == new_build_count_url(rooms, gte, lte)
|
||
for offset in (0, 20, 1980):
|
||
assert old_build_offers_url(rooms, gte, lte, offset) == new_build_offers_url(
|
||
rooms, gte, lte, offset
|
||
)
|
||
|
||
|
||
def test_parse_publish_date_parity() -> None:
|
||
"""_parse_publish_date идентичен на валидных/битых ISO-строках."""
|
||
for iso in [None, "", "2026-06-28T09:15:00+05:00", "2026-06-27", "not-a-date", "2026-13-40"]:
|
||
assert old_parse_publish_date(iso) == new_parse_publish_date(iso)
|
||
|
||
|
||
def test_extract_agency_name_parity() -> None:
|
||
"""_extract_agency_name идентичен на строковых/dict/пустых seller-блоках."""
|
||
sellers: list[dict[str, Any]] = [
|
||
{},
|
||
{"company": "АН Этажи", "agent": "Иванов"},
|
||
{"company": " ", "agent": {"name": "Резерв"}},
|
||
{"agent": {"fullName": "Петров П."}},
|
||
{"company": {"title": "СберРиелт"}},
|
||
{"agent": 12345},
|
||
]
|
||
for seller in sellers:
|
||
assert old_extract_agency(seller) == new_extract_agency(seller)
|
||
|
||
|
||
# ── strangler-guard: kit-провайдер не импортирует app.* ───────────────────────
|
||
|
||
|
||
def test_kit_domclick_has_no_app_imports() -> None:
|
||
"""Ни один модуль scraper_kit.providers.domclick.* не импортирует app.* (развязка)."""
|
||
import inspect
|
||
|
||
from scraper_kit.providers.domclick import serp
|
||
|
||
source = inspect.getsource(serp)
|
||
for line in source.splitlines():
|
||
stripped = line.strip()
|
||
assert not stripped.startswith("from app"), f"serp: {line!r}"
|
||
assert not stripped.startswith("import app"), f"serp: {line!r}"
|