8c8289cdad
All checks were successful
CI / changes (pull_request) Successful in 10s
CI Trade-In / changes (pull_request) Successful in 10s
CI Trade-In / frontend-checks (pull_request) Has been skipped
CI / backend-tests (pull_request) Has been skipped
CI / frontend-tests (pull_request) Has been skipped
CI / openapi-codegen-check (pull_request) Has been skipped
CI Trade-In / backend-tests (pull_request) Successful in 1m44s
Group D of the scraper_kit migration epic (#2277). Both legacy modules had no kit-side equivalent yet (per audit Scraper_Kit_Legacy_Dependency_Audit_0703), unlike the other 24/26 files which were already strangler-duplicated. - scraper_kit/providers/domclick/detail.py — Layer B detail-card enrichment (parse_detail_html/fetch_detail/save_detail_enrichment), no ScraperConfig injection needed (fetch_detail takes an already-open browser_fetcher). - scraper_kit/providers/ekb_geoportal/{__init__,client}.py — new subpackage, first non-listing-site provider in scraper_kit. Legacy module had zero app.* coupling (only httpx), so the port is near-verbatim. Parity proven via tests/support/parity.py (assert_parity) against the exact legacy functions on realistic fixtures (live-card SSR JSON, GeoJSON FeatureCollection). Callers of both legacy modules are unchanged (out of scope — ekb_geoportal_ingest switch is #2310; the only other domclick_detail caller is scripts/ingest_domclick_jsonl.py, also untouched).
189 lines
7.8 KiB
Python
189 lines
7.8 KiB
Python
"""Parity-proof: `app.services.scrapers.domclick_detail` vs
|
||
`scraper_kit.providers.domclick.detail` (issue #2307, Group D).
|
||
|
||
`parse_detail_html` — parsing-ядро без сети/БД (SSR-стейт → dataclass), поэтому
|
||
сравнивается offline на фиксированной HTML-фикстуре через
|
||
`tests.support.parity.assert_parity` (harness #2304). Legacy и kit тела функции
|
||
на момент написания теста ИДЕНТИЧНЫ (см. audit
|
||
`Scraper_Kit_Legacy_Dependency_Audit_0703` в vault) — различаются только
|
||
import-пути зависимостей (`app.services.scrapers.*` vs `scraper_kit.*`).
|
||
|
||
HTML-фикстура — независимая копия литерала из
|
||
`tests/scrapers/test_domclick_detail.py::_HTML` (тот же live-card ground truth
|
||
2075729321, 2026-06-27), чтобы этот тест не зависел от другого test-модуля.
|
||
"""
|
||
|
||
# SSR-JSON фикстура содержит длинные строки
|
||
|
||
from __future__ import annotations
|
||
|
||
import dataclasses
|
||
import os
|
||
|
||
import pytest
|
||
|
||
# app.services.scrapers.domclick_detail не читает settings напрямую, но соседние
|
||
# модули пакета app.services.scrapers делают это при импорте app.core.config —
|
||
# фиктивный DSN мостит офлайн-запуск без CI-уровневого DATABASE_URL (mirror
|
||
# tests/scrapers/test_avito_detail_kit_parity.py).
|
||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
|
||
|
||
from scraper_kit.domclick_exceptions import (
|
||
DomClickBlockedError as KitDomClickBlockedError,
|
||
)
|
||
from scraper_kit.domclick_exceptions import (
|
||
DomClickParseError as KitDomClickParseError,
|
||
)
|
||
from scraper_kit.providers.domclick.detail import (
|
||
parse_detail_html as kit_parse_detail_html,
|
||
)
|
||
|
||
from app.services.scrapers.domclick_detail import (
|
||
parse_detail_html as legacy_parse_detail_html,
|
||
)
|
||
from app.services.scrapers.domclick_exceptions import (
|
||
DomClickBlockedError as LegacyDomClickBlockedError,
|
||
)
|
||
from app.services.scrapers.domclick_exceptions import (
|
||
DomClickParseError as LegacyDomClickParseError,
|
||
)
|
||
from tests.support.parity import ParityMismatchError, assert_parity
|
||
|
||
_CARD_URL = "https://ekaterinburg.domclick.ru/card/sale__flat__2075729321"
|
||
|
||
# Real __SSR_STATE__ shape (ground truth, live card 2075729321, 2026-06-27).
|
||
# bare `undefined` (value-position), in-string "undefined", '{' внутри строки и
|
||
# эскейп-кавычки — написан вручную (json.dumps использовать нельзя, нужен
|
||
# именно НЕвалидный-для-json undefined).
|
||
_SSR_LITERAL = """{
|
||
"productCard": {
|
||
"objectInfo": {
|
||
"area": 38.0,
|
||
"rooms": 1,
|
||
"floor": 5,
|
||
"isApartment": false,
|
||
"renovation": "евроремонт",
|
||
"livingArea": 22.37,
|
||
"kitchenArea": 10.13,
|
||
"description": "Состояние пока undefined.",
|
||
"note": "угол {da}",
|
||
"quote": "сказал \\"привет\\" сосед"
|
||
},
|
||
"legalOptions": {"saleType": "Свободная продажа"},
|
||
"egrnData": {
|
||
"area": 38.2,
|
||
"floor": 5,
|
||
"owners_count": 1,
|
||
"collateral": true,
|
||
"collateral_sber": false
|
||
},
|
||
"priceInfo": {
|
||
"priceHistory": [
|
||
{"date": "2026-04-15T09:02:26.548728+03:00", "price": 13400000, "diff": -400000, "state": "less"},
|
||
{"date": "not-a-date", "price": 4800000, "diff": -1.0, "state": "less"},
|
||
{"date": "2026-05-01T10:00:00Z", "price": null, "diff": 0, "state": "more"}
|
||
]
|
||
},
|
||
"viewsCount": 338,
|
||
"callsCount": 5,
|
||
"favoriteOfferUsersCount": 12,
|
||
"duplicatesOfferCount": 0,
|
||
"address": {"guid": "abc-guid-123"},
|
||
"unknownField": undefined
|
||
},
|
||
"houseInfo": {
|
||
"info": {
|
||
"buildYear": 2015,
|
||
"wallType": "Кирпично-монолитный",
|
||
"floorType": "Железобетонный",
|
||
"entranceCount": 3,
|
||
"quartersCount": 120,
|
||
"energyEfficiency": "B",
|
||
"buildingSeries": "индивидуальный"
|
||
}
|
||
},
|
||
"pricePrediction": {
|
||
"market_price": 13000000,
|
||
"min_market_price": 12000000,
|
||
"max_market_price": 14000000,
|
||
"rent_short": 50000,
|
||
"rent_long": 35000,
|
||
"repair_quality": "good"
|
||
}
|
||
}"""
|
||
|
||
_HTML = (
|
||
"<html><head></head><body><script>"
|
||
f"window.__SSR_STATE__ = {_SSR_LITERAL};"
|
||
"</script></body></html>"
|
||
)
|
||
|
||
|
||
def test_legacy_and_kit_parse_detail_html_are_class_distinct() -> None:
|
||
"""Sanity-check ДО harness'а: legacy/kit DomClickDetailEnrichment — РАЗНЫЕ
|
||
классы (разные модули) → обычный `==` даёт False, хотя данные идентичны.
|
||
Именно это harness обходит через структурное сравнение (dataclasses.fields).
|
||
"""
|
||
legacy_result = legacy_parse_detail_html(_HTML, _CARD_URL)
|
||
kit_result = kit_parse_detail_html(_HTML, _CARD_URL)
|
||
|
||
assert type(legacy_result) is not type(kit_result)
|
||
assert legacy_result != kit_result # dataclass __eq__ проверяет class identity первым
|
||
assert dataclasses.asdict(legacy_result) == dataclasses.asdict(kit_result)
|
||
|
||
|
||
def test_parity_harness_confirms_domclick_detail_parse_equivalence() -> None:
|
||
"""Реальное использование harness'а для DomClick Layer B (issue #2307)."""
|
||
assert_parity(
|
||
legacy_fn=legacy_parse_detail_html,
|
||
kit_fn=kit_parse_detail_html,
|
||
fixtures=[(_HTML, _CARD_URL)],
|
||
)
|
||
|
||
|
||
def test_parity_harness_catches_real_field_divergence() -> None:
|
||
"""Harness реально ЛОВИТ расхождение на реальной форме — мутируем один
|
||
field (``living_area_m2``) в kit-результате и убеждаемся, что harness
|
||
падает с ``ParityMismatchError``, называющим именно этот field.
|
||
"""
|
||
legacy_result = legacy_parse_detail_html(_HTML, _CARD_URL)
|
||
kit_result = kit_parse_detail_html(_HTML, _CARD_URL)
|
||
assert kit_result.living_area_m2 is not None # sanity: поле реально распарсилось
|
||
|
||
mutated_kit_result = dataclasses.replace(
|
||
kit_result, living_area_m2=kit_result.living_area_m2 + 1
|
||
)
|
||
|
||
with pytest.raises(ParityMismatchError) as exc_info:
|
||
assert_parity(
|
||
legacy_fn=lambda: legacy_result,
|
||
kit_fn=lambda: mutated_kit_result,
|
||
fixtures=[()],
|
||
)
|
||
|
||
assert "living_area_m2" in str(exc_info.value)
|
||
|
||
|
||
def test_legacy_and_kit_exceptions_behave_identically_on_challenge_page() -> None:
|
||
"""Anti-bot challenge page (no __SSR_STATE__, block-маркеры) → оба пути
|
||
поднимают семантически эквивалентный DomClickBlockedError (разные классы,
|
||
те же условия детекции — зеркало `test_scraper_kit_domclick_golden_parity.py`
|
||
для Layer A exceptions).
|
||
"""
|
||
challenge_html = "<html><body>Access denied by DataDome captcha</body></html>"
|
||
|
||
with pytest.raises(LegacyDomClickBlockedError):
|
||
legacy_parse_detail_html(challenge_html, _CARD_URL)
|
||
with pytest.raises(KitDomClickBlockedError):
|
||
kit_parse_detail_html(challenge_html, _CARD_URL)
|
||
|
||
|
||
def test_legacy_and_kit_exceptions_behave_identically_on_missing_state() -> None:
|
||
"""HTML без __SSR_STATE__ и без block-маркеров → оба пути поднимают
|
||
DomClickParseError (дрейф схемы / повреждённый ответ, не блок)."""
|
||
no_state_html = "<html><body>no ssr state here</body></html>"
|
||
|
||
with pytest.raises(LegacyDomClickParseError):
|
||
legacy_parse_detail_html(no_state_html, _CARD_URL)
|
||
with pytest.raises(KitDomClickParseError):
|
||
kit_parse_detail_html(no_state_html, _CARD_URL)
|