lekss361
df5e9003df
All checks were successful
Deploy Trade-In / changes (push) Successful in 11s
Deploy Trade-In / build-frontend (push) Has been skipped
Deploy Trade-In / build-browser (push) Has been skipped
Deploy Trade-In / test (push) Successful in 1m30s
Deploy Trade-In / build-backend (push) Successful in 1m47s
Deploy Trade-In / deploy (push) Successful in 1m1s
206 lines
9.9 KiB
Python
206 lines
9.9 KiB
Python
"""Golden-parity: `scraper_kit.providers.avito.*` парсинг ≡ `app.services.scrapers.avito*`.
|
||
|
||
Strangler-инвариант (#2133): новая scraper_kit-копия avito-провайдера должна давать
|
||
БАЙТ-ИДЕНТИЧНЫЙ результат парсинга старому боевому коду на одинаковом входе.
|
||
Развязка (settings→ScraperConfig, get_scraper_delay→delay_provider,
|
||
geocoder→scraper_kit.geo, matcher→HouseMatcher) НЕ меняет распарсенные данные.
|
||
|
||
Гоняем ОБА модуля на одних фикстурах и сравниваем результат:
|
||
- SERP `_parse_html` (DOM-карточки → list[ScrapedLot]) на avito_serp_sample.html,
|
||
при ekb_only=False и ekb_only=True (фильтр padding-карточек);
|
||
- `_extract_total_count` (счётчик выдачи);
|
||
- `_build_sort_timestamp_map` + `_unix_to_date` (sortTimeStamp → date, MSK);
|
||
- detail `parse_detail_html` (item-view HTML → DetailEnrichment);
|
||
- чистые title/address-хелперы (_clean_address, _extract_rooms/area/floor_from_title).
|
||
|
||
Идентичность результата = kit-парсер верно скопирован (развязка не задела логику).
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import dataclasses
|
||
import os
|
||
from pathlib import Path
|
||
from types import SimpleNamespace
|
||
from typing import Any
|
||
from unittest.mock import patch
|
||
|
||
# Старый app.services.scrapers.avito импортирует app.core.config.settings=Settings(),
|
||
# которому нужен DATABASE_URL. Офлайн-парсинг БД не трогает — фиктивный DSN достаточен.
|
||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
|
||
|
||
# НОВЫЙ (scraper_kit) и СТАРЫЙ (app.services.scrapers) avito-парсеры — оба в одном
|
||
# тесте; ruff сортирует scraper_kit (first-party) выше app-импортов.
|
||
from scraper_kit.providers.avito.detail import parse_detail_html as new_parse_detail
|
||
from scraper_kit.providers.avito.serp import AvitoScraper as NewAvitoScraper
|
||
from scraper_kit.providers.avito.serp import (
|
||
_build_sort_timestamp_map as new_build_ts_map,
|
||
)
|
||
from scraper_kit.providers.avito.serp import _clean_address as new_clean_address
|
||
from scraper_kit.providers.avito.serp import _extract_area_from_title as new_area
|
||
from scraper_kit.providers.avito.serp import _extract_floor_from_title as new_floor
|
||
from scraper_kit.providers.avito.serp import _extract_rooms_from_title as new_rooms
|
||
from scraper_kit.providers.avito.shared import _unix_to_date as new_unix_to_date
|
||
|
||
from app.services.scrapers.avito import AvitoScraper as OldAvitoScraper
|
||
from app.services.scrapers.avito import (
|
||
_build_sort_timestamp_map as old_build_ts_map,
|
||
)
|
||
from app.services.scrapers.avito import _clean_address as old_clean_address
|
||
from app.services.scrapers.avito import _extract_area_from_title as old_area
|
||
from app.services.scrapers.avito import _extract_floor_from_title as old_floor
|
||
from app.services.scrapers.avito import _extract_rooms_from_title as old_rooms
|
||
from app.services.scrapers.avito import _unix_to_date as old_unix_to_date
|
||
from app.services.scrapers.avito_detail import parse_detail_html as old_parse_detail
|
||
|
||
_FIXTURES = Path(__file__).parent / "fixtures"
|
||
_SERP_HTML = (_FIXTURES / "avito_serp_sample.html").read_text(encoding="utf-8")
|
||
|
||
# Минимальный detail item-view HTML (зеркало test_avito_detail_parse.MINIMAL_HTML).
|
||
_DETAIL_HTML = """
|
||
<html><body>
|
||
<div data-marker="item-view/item-id">№ 7986882804 · 18 мая в 13:40 · 1899 просмотров (+ 12 сегодня)</div>
|
||
<span itemprop="price" content="11990000">11 990 000 ₽</span>
|
||
<div data-marker="item-map-wrapper" data-map-lat="56.797713" data-map-lon="60.609135" data-location-id="654070"></div>
|
||
<div data-marker="item-view/item-params">
|
||
<ul>
|
||
<li>Количество комнат: 3</li>
|
||
<li>Общая площадь: 75 м²</li>
|
||
<li>Площадь кухни: 15 м²</li>
|
||
<li>Этаж: 20 из 26</li>
|
||
<li>Балкон или лоджия: лоджия</li>
|
||
<li>Ремонт: евро</li>
|
||
</ul>
|
||
<ul>
|
||
<li>Тип дома: монолитный</li>
|
||
<li>Этажей в доме: 26</li>
|
||
</ul>
|
||
</div>
|
||
<a data-marker="nd-jk-details-button" href="/catalog/houses/ekaterinburg/ul_postovskogo_17a/3171365">Дом</a>
|
||
<div data-marker="item-view/item-description">Отличная квартира в новом ЖК.</div>
|
||
</body></html>
|
||
""" # noqa: E501
|
||
|
||
_SOURCE_URL = "https://www.avito.ru/ekaterinburg/kvartiry/test_7986882804"
|
||
|
||
|
||
def _make_old(ekb_only: bool) -> tuple[OldAvitoScraper, SimpleNamespace]:
|
||
"""Старый AvitoScraper без DB: patch get_scraper_delay + settings.avito_serp_ekb_only."""
|
||
with patch("app.services.scrapers.avito.get_scraper_delay", return_value=7.0):
|
||
scraper = OldAvitoScraper()
|
||
# _parse_html читает settings.avito_serp_ekb_only — подменяем на детерминированное.
|
||
scraper_settings = SimpleNamespace(avito_serp_ekb_only=ekb_only)
|
||
return scraper, scraper_settings
|
||
|
||
|
||
def _make_new(ekb_only: bool) -> NewAvitoScraper:
|
||
"""Новый AvitoScraper с инжектируемым ScraperConfig (duck-typed namespace)."""
|
||
config = SimpleNamespace(avito_serp_ekb_only=ekb_only)
|
||
return NewAvitoScraper(config=config)
|
||
|
||
|
||
def _dump_lots(lots: list[Any]) -> list[dict[str, Any]]:
|
||
return [lot.model_dump() for lot in lots]
|
||
|
||
|
||
# ── SERP _parse_html parity ──────────────────────────────────────────────────
|
||
|
||
|
||
def _run_old_parse_html(html: str, base: str, ekb_only: bool) -> list[Any]:
|
||
old, old_settings = _make_old(ekb_only)
|
||
with patch("app.services.scrapers.avito.settings", old_settings):
|
||
return old._parse_html(html, base)
|
||
|
||
|
||
def test_serp_parse_html_parity_ekb_only_false() -> None:
|
||
"""DOM-карточки → ScrapedLot идентичны (фильтр padding выключен)."""
|
||
base = "https://www.avito.ru/ekaterinburg/kvartiry"
|
||
old_lots = _run_old_parse_html(_SERP_HTML, base, ekb_only=False)
|
||
new_lots = _make_new(ekb_only=False)._parse_html(_SERP_HTML, base)
|
||
|
||
assert len(old_lots) == len(new_lots)
|
||
assert len(new_lots) > 0, "фикстура должна содержать хотя бы одну карточку"
|
||
assert _dump_lots(old_lots) == _dump_lots(new_lots)
|
||
|
||
|
||
def test_serp_parse_html_parity_ekb_only_true() -> None:
|
||
"""С включённым ekb_only-фильтром padding-дроп идентичен."""
|
||
base = "https://www.avito.ru/ekaterinburg/kvartiry"
|
||
old_lots = _run_old_parse_html(_SERP_HTML, base, ekb_only=True)
|
||
new_lots = _make_new(ekb_only=True)._parse_html(_SERP_HTML, base)
|
||
|
||
assert len(old_lots) == len(new_lots)
|
||
assert _dump_lots(old_lots) == _dump_lots(new_lots)
|
||
|
||
|
||
def test_serp_extract_total_count_parity() -> None:
|
||
"""_extract_total_count на SERP-фикстуре идентичен."""
|
||
old, _ = _make_old(ekb_only=False)
|
||
new = _make_new(ekb_only=False)
|
||
assert old._extract_total_count(_SERP_HTML) == new._extract_total_count(_SERP_HTML)
|
||
|
||
|
||
def test_build_sort_timestamp_map_parity() -> None:
|
||
"""_build_sort_timestamp_map (sortTimeStamp → date, MSK) идентичен."""
|
||
assert old_build_ts_map(_SERP_HTML) == new_build_ts_map(_SERP_HTML)
|
||
|
||
|
||
def test_unix_to_date_parity() -> None:
|
||
"""_unix_to_date (MSK) идентичен на репрезентативных epoch-значениях."""
|
||
for ts in [None, 0, 1756077885, 1735689600, 1700000000.0, 1_600_000_000]:
|
||
assert old_unix_to_date(ts) == new_unix_to_date(ts)
|
||
|
||
|
||
# ── detail parse_detail_html parity ──────────────────────────────────────────
|
||
|
||
|
||
def test_parse_detail_html_parity() -> None:
|
||
"""parse_detail_html → DetailEnrichment идентичен (все поля)."""
|
||
old_res = old_parse_detail(_DETAIL_HTML, _SOURCE_URL)
|
||
new_res = new_parse_detail(_DETAIL_HTML, _SOURCE_URL)
|
||
assert dataclasses.asdict(old_res) == dataclasses.asdict(new_res)
|
||
|
||
|
||
# ── чистые title/address-хелперы ──────────────────────────────────────────────
|
||
|
||
|
||
def test_pure_title_helpers_parity() -> None:
|
||
titles = [
|
||
"3-к. квартира, 75 м², 20/26 эт.",
|
||
"Квартира-студия, 24,5 м², 12/25 эт.",
|
||
"1-к. квартира, 38 м², 5/9 эт.",
|
||
"2-к. квартира, 52,3 м², 7/16 эт.",
|
||
"Комната, 18 м², 3/5 эт.",
|
||
]
|
||
for t in titles:
|
||
assert old_rooms(t) == new_rooms(t)
|
||
assert old_area(t) == new_area(t)
|
||
assert old_floor(t) == new_floor(t)
|
||
|
||
|
||
def test_clean_address_parity() -> None:
|
||
addrs = [
|
||
None,
|
||
"Екатеринбург, ул. Постовского, 17А",
|
||
"Свердловская область, Екатеринбург, Малышева, 51",
|
||
" Екатеринбург, Ленина 1 ",
|
||
]
|
||
for a in addrs:
|
||
assert old_clean_address(a) == new_clean_address(a)
|
||
|
||
|
||
# ── strangler-guard: kit-провайдер не импортирует app.* ──────────────────────
|
||
|
||
|
||
def test_kit_avito_has_no_app_imports() -> None:
|
||
"""Ни один модуль scraper_kit.providers.avito.* не импортирует app.* (развязка)."""
|
||
import inspect
|
||
|
||
from scraper_kit.providers.avito import detail, houses, imv, serp, shared
|
||
|
||
for mod in (serp, detail, houses, imv, shared):
|
||
source = inspect.getsource(mod)
|
||
for line in source.splitlines():
|
||
stripped = line.strip()
|
||
assert not stripped.startswith("from app"), f"{mod.__name__}: {line!r}"
|
||
assert not stripped.startswith("import app"), f"{mod.__name__}: {line!r}"
|