lekss361/gendesign
1
0
Fork 0
Code Issues 57 Pull requests Projects Releases Packages Wiki Activity Actions
gendesign/tradein-mvp/backend/tests/test_avito_houses_parse.py
Light1YT a3f45a6722 fix(tradein): decode Avito houses URL-encoded __preloadedState__ (#566) 
Avito switched the houses-catalog state blob from JSON.parse("<js-escaped>")
to a URL-encoded string literal, wrapped the 10 widgets in
placeholders[0].widgets, и re-nested developmentData под props.housePage.
JS-unescape path silently failed → houses rows перестали наполняться
(starve'ит #6 same-building + #569 deal centroids).

Extract decode в pure _extract_preloaded_state (urllib.parse.unquote,
JSON.parse fallback для старой формы), add _flatten_widgets, и сделать
_parse_house_page tolerant к обеим developmentData/ratingPreview формам.
Real-HTML fixture (Yumasheva-6 EKB) + 4 regression-теста.

code-reviewer: ✅ APPROVE no critical (regex anchoring verified, decode
safe — no eval/unicode_escape, backward-compat реальный, delisted-house
envelope degrade'ит gracefully, fixture без PII/secrets). Tests: 22 passed.

Closes #566
2026-05-28 15:58:19 +05:00

460 lines
18 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""Offline smoke for avito_houses parser.
Uses minimal JSON fixture mimicking Avito __preloadedState__ structure.
Не требует сети, БД или curl_cffi — тестирует только чистый парсер.
"""
from datetime import date
from pathlib import Path
import pytest
from app.services.scrapers.avito_houses import (
HouseCatalogEnrichment,
HouseInfo,
HouseReview,
MiniSerpListing,
PlacementHistoryItem,
RecommendationItem,
SellerInfo,
_extract_preloaded_state,
_parse_ru_date,
parse_houses_state,
)
FIXTURES_DIR = Path(__file__).parent / "fixtures"
# ---------------------------------------------------------------------------
# Minimal __preloadedState__ fixture (Python dict — не raw HTML)
# ---------------------------------------------------------------------------
MINIMAL_STATE: dict = {
"data": {"data": {"page": {"placeholders": [
{"type": "breadcrumbs"},
{"type": "title"},
{"type": "gallery", "props": {}},
{
"type": "housePage",
"props": {"developmentData": {
"avitoId": 3171365,
"id": "MTE3LjM3MDg5MA",
"title": "Akademika Postovskogo 17a",
"address": "ул. Постовского, 17а",
"fullAddress": "Свердловская обл., Екатеринбург, ул. Постовского, 17а",
"coords": {"lat": 56.790699, "lng": 60.580191},
"aboutDevelopment": {"expandParams": {"items": [
{"title": "Параметры дома", "params": [
{"type": "Год постройки", "value": "2019"},
{"type": "Этажей", "value": "25"},
{"type": "Тип дома", "value": "Монолитный"},
{"type": "Пассажирский лифт", "value": "2"},
{"type": "Консьерж", "value": "Да"},
{"type": "Закрытый двор", "value": "Да"},
{"type": "Детская площадка", "value": "Да"},
]},
]}},
"developer": {"name": "Атомстройкомплекс", "key": "atomstroy"},
"mapPreview": {
"distance": "в 5 минутах ходьбы",
"objects": "школа, парк, аптека",
"pins": [{"lat": 56.79, "lng": 60.58, "type": "school"}],
},
"ratingBadge": {"info": {"score": 4.666, "scoreString": "4,7"}},
"ratingSummaryStat": {
"ratingStat": [{"score": 5, "count": 4}, {"score": 4, "count": 2}],
"reviewCount": 6,
},
}},
},
{
"type": "reviews",
"props": {"entries": [
{"type": "rating", "value": {
"id": 12345,
"author": {"title": "Иван"},
"reviewTitle": "Хорошо",
"score": 5,
"modelExperience": "Живу в своей квартире",
"rated": "9 октября 2025",
"textSections": [
{"title": "", "text": "Общий отзыв"},
{"title": "Преимущества", "text": "Парк рядом"},
{"title": "Недостатки", "text": "Парковки мало"},
],
}},
{"type": "pages", "value": {"nextPageUrl": "/reviews?page=2"}},
]},
},
{
"type": "miniSerp",
"props": {"items": [
{
"id": 7986882804,
"title": "3-к. квартира",
"description": "Описание",
"url": "/ekaterinburg/kvartiry/test_7986882804",
"price": "11 990 000 ₽",
"geo": {"content": "Чкаловская, 2,7 км", "colors": ["#cf2734"]},
"seller": {
"name": "DOMRF66",
"type": "Компания",
"from": "На Авито с июля 2012",
"url": "/brands/domrf66",
"logos": [],
},
},
]},
},
{
"type": "housePlacementHistory",
"props": {"items": [
{
"id": 7000000001,
"title": "1-к. квартира",
"startPrice": 5000000,
"startPriceDate": 1697000000,
"lastPrice": 4900000,
"lastPriceDate": 1715000000,
"exposure": 210,
"itemImage": {},
},
]},
},
{"type": "uxFeedback"},
{
"type": "recommendations",
"props": {"items": [
{
"title": "Похожая",
"priceRange": "от 6 млн",
"additionalInfo": ["Адрес 1"],
"url": "/rec1",
"images": [{"208x156": "https://avito.ru/img1.jpg"}],
},
]},
},
{"type": "padding"},
]}}}
}
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def test_ru_date_parse() -> None:
assert _parse_ru_date("9 октября 2025") == date(2025, 10, 9)
assert _parse_ru_date("31 декабря 2020") == date(2020, 12, 31)
assert _parse_ru_date("1 января 2000") == date(2000, 1, 1)
assert _parse_ru_date(None) is None
assert _parse_ru_date("bogus") is None
assert _parse_ru_date("") is None
# ---------------------------------------------------------------------------
# Main parse test
# ---------------------------------------------------------------------------
def test_parse_minimal_state() -> None:
e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
assert isinstance(e, HouseCatalogEnrichment)
assert e.house_url == "/catalog/houses/ekaterinburg/test/3171365"
def test_house_info() -> None:
e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
h = e.house
assert isinstance(h, HouseInfo)
assert h.ext_id == 3171365
assert h.ext_id_hash == "MTE3LjM3MDg5MA"
assert h.title == "Akademika Postovskogo 17a"
assert h.short_address == "ул. Постовского, 17а"
assert h.full_address == "Свердловская обл., Екатеринбург, ул. Постовского, 17а"
assert h.lat == 56.790699
assert h.lon == 60.580191
def test_house_expand_params() -> None:
e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
h = e.house
assert h.year_built == 2019
assert h.total_floors == 25
assert h.house_type == "monolith"
assert h.passenger_elevators == 2
assert h.has_concierge is True
assert h.closed_yard is True
assert h.has_playground is True
def test_house_developer_and_rating() -> None:
e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
h = e.house
assert h.developer_name == "Атомстройкомплекс"
assert h.developer_key == "atomstroy"
assert h.rating_score == 4.666
assert h.rating_string == "4,7"
assert h.reviews_count == 6
assert len(h.rating_distribution) == 2
def test_house_map_and_infra() -> None:
e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
h = e.house
assert h.infrastructure_walk_distance == "в 5 минутах ходьбы"
assert h.infrastructure_summary == "школа, парк, аптека"
assert len(h.map_pins) == 1
assert h.map_pins[0]["type"] == "school"
def test_house_raw_characteristics() -> None:
e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
# raw_characteristics = полный expandParams items (1 категория)
assert len(e.house.raw_characteristics) == 1
assert e.house.raw_characteristics[0]["title"] == "Параметры дома"
def test_reviews() -> None:
e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
assert len(e.reviews) == 1
assert e.reviews_next_page_url == "/reviews?page=2"
r = e.reviews[0]
assert isinstance(r, HouseReview)
assert r.ext_review_id == 12345
assert r.author_name == "Иван"
assert r.review_title == "Хорошо"
assert r.score == 5
assert r.model_experience == "Живу в своей квартире"
assert r.rated_date == date(2025, 10, 9)
assert r.text_main == "Общий отзыв"
assert r.text_pros == "Парк рядом"
assert r.text_cons == "Парковки мало"
assert r.raw_payload is not None
def test_mini_serp() -> None:
e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
assert len(e.mini_serp) == 1
m = e.mini_serp[0]
assert isinstance(m, MiniSerpListing)
assert m.ext_item_id == 7986882804
assert m.price_rub == 11990000
assert m.metro_text == "Чкаловская, 2,7 км"
assert m.metro_color == "#cf2734"
assert m.description == "Описание"
def test_seller_extraction() -> None:
e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
m = e.mini_serp[0]
assert m.seller is not None
s = m.seller
assert isinstance(s, SellerInfo)
assert s.ext_seller_id == "domrf66"
assert s.name == "DOMRF66"
assert s.seller_type == "Компания"
assert s.on_platform_since == "На Авито с июля 2012"
assert s.profile_url == "/brands/domrf66"
def test_placement_history() -> None:
e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
assert len(e.placement_history) == 1
p = e.placement_history[0]
assert isinstance(p, PlacementHistoryItem)
assert p.ext_item_id == "7000000001"
assert p.title == "1-к. квартира"
assert p.start_price == 5000000
assert p.last_price == 4900000
assert p.exposure_days == 210
# removed_date ВСЕГДА None — виджет не отдаёт!
assert p.removed_date is None
# raw_payload не содержит itemImage (скипается)
assert p.raw_payload is not None
assert "itemImage" not in p.raw_payload
# start_price_date и last_price_date — из unix timestamp
assert isinstance(p.start_price_date, date)
assert isinstance(p.last_price_date, date)
def test_recommendations() -> None:
e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
assert len(e.recommendations) == 1
rec = e.recommendations[0]
assert isinstance(rec, RecommendationItem)
assert rec.title == "Похожая"
assert rec.price_text == "от 6 млн"
assert rec.address == "Адрес 1"
assert rec.url == "/rec1"
assert rec.image_url == "https://avito.ru/img1.jpg"
def test_missing_optional_widgets() -> None:
"""Парсер не падает если опциональные виджеты отсутствуют."""
state_minimal: dict = {
"data": {"data": {"page": {"placeholders": [
{
"type": "housePage",
"props": {"developmentData": {
"avitoId": 9999,
"coords": {"lat": 56.0, "lng": 60.0},
"aboutDevelopment": {"expandParams": {"items": []}},
"developer": {},
"mapPreview": {},
"ratingBadge": {"info": {}},
"ratingSummaryStat": {},
}},
},
]}}}
}
e = parse_houses_state(state_minimal, "/catalog/houses/test/9999")
assert e.house.ext_id == 9999
assert e.reviews == []
assert e.mini_serp == []
assert e.placement_history == []
assert e.recommendations == []
assert e.reviews_next_page_url is None
def test_invalid_state_raises() -> None:
"""ValueError если placeholders не найден."""
with pytest.raises(ValueError, match="placeholders"):
parse_houses_state({}, "/catalog/houses/test/1")
def test_missing_house_page_raises() -> None:
"""ValueError если housePage виджет отсутствует."""
state: dict = {
"data": {"data": {"page": {"placeholders": [
{"type": "breadcrumbs"},
]}}}
}
with pytest.raises(ValueError, match="housePage"):
parse_houses_state(state, "/catalog/houses/test/1")
def test_house_type_normalization() -> None:
"""Все варианты типа дома нормализуются корректно."""
from app.services.scrapers.avito_houses import _normalize_house_type
assert _normalize_house_type("Монолитный") == "monolith"
assert _normalize_house_type("Панельный") == "panel"
assert _normalize_house_type("Кирпичный") == "brick"
assert _normalize_house_type("Монолитно-кирпичный") == "monolith_brick"
assert _normalize_house_type("Блочный") == "block"
assert _normalize_house_type("Деревянный") == "wood"
assert _normalize_house_type("Неизвестный") == "other"
assert _normalize_house_type(None) is None
def test_strip_price() -> None:
"""_strip_price корректно извлекает числа из строк Avito."""
from app.services.scrapers.avito_houses import _strip_price
assert _strip_price("11 990 000 ₽") == 11990000
assert _strip_price("5 000 000 ₽") == 5000000
assert _strip_price(None) is None
assert _strip_price("") is None
def test_mini_serp_no_seller() -> None:
"""Объявление без продавца не вызывает ошибку."""
state: dict = {
"data": {"data": {"page": {"placeholders": [
{
"type": "housePage",
"props": {"developmentData": {
"avitoId": 111,
"coords": {"lat": 56.0, "lng": 60.0},
"aboutDevelopment": {"expandParams": {"items": []}},
"developer": {},
"mapPreview": {},
"ratingBadge": {"info": {}},
"ratingSummaryStat": {},
}},
},
{
"type": "miniSerp",
"props": {"items": [
{"id": 12345, "title": "1-к.", "url": "/test", "price": "3 000 000 ₽",
"geo": {"content": "Центр", "colors": []}},
]},
},
]}}}
}
e = parse_houses_state(state, "/catalog/houses/test/111")
assert len(e.mini_serp) == 1
assert e.mini_serp[0].seller is None
assert e.mini_serp[0].price_rub == 3000000
# ---------------------------------------------------------------------------
# Real-HTML fixture (regression for issue #566 — state-blob encoding/shape change)
# ---------------------------------------------------------------------------
#
# Fixture: реальная страница /catalog/houses/ekaterinburg/ul_yumasheva_6/115188,
# захвачена curl_cffi chrome120 (2026-05-28). Avito сменил формат __preloadedState__
# на URL-encoded литерал + обернул виджеты в placeholders[0]['widgets'] +
# вложил developmentData под props['housePage']. Этот тест ловит регрессию декода/shape.
def test_extract_preloaded_state_from_real_html() -> None:
"""_extract_preloaded_state декодирует текущий URL-encoded __preloadedState__."""
html = (FIXTURES_DIR / "avito_houses_sample.html").read_text(encoding="utf-8")
state = _extract_preloaded_state(html)
assert state is not None, "URL-encoded __preloadedState__ должен распарситься"
# Avito SSR обёртка — успешный ответ
assert state["data"]["status"] == 200
# Inner path до placeholders сохранился
placeholders = state["data"]["data"]["page"]["placeholders"]
assert isinstance(placeholders, list) and placeholders, "placeholders непустой массив"
def test_parse_real_fixture_end_to_end() -> None:
"""Фикстура HTML → _extract_preloaded_state → parse_houses_state → HouseInfo.ext_id."""
html = (FIXTURES_DIR / "avito_houses_sample.html").read_text(encoding="utf-8")
state = _extract_preloaded_state(html)
assert state is not None
e = parse_houses_state(state, "/catalog/houses/ekaterinburg/ul_yumasheva_6/115188")
assert isinstance(e, HouseCatalogEnrichment)
# housePage распарсился — ext_id (avitoId) присутствует и совпадает с URL
assert e.house.ext_id == 115188
assert e.house.ext_id_hash # base64 internal id присутствует
# expandParams распарсились (год постройки реальной фикстуры — 2016)
assert e.house.year_built == 2016
assert e.house.total_floors == 17
assert e.house.house_type == "monolith"
assert e.house.lat is not None and e.house.lon is not None
def test_extract_preloaded_state_jsparse_fallback() -> None:
"""Старая форма JSON.parse("...") всё ещё декодируется (fallback-ветка)."""
inner = '{\\"data\\":{\\"data\\":{\\"page\\":{\\"placeholders\\":[]}}}}'
html = f'<script>window.__preloadedState__ = JSON.parse("{inner}");</script>'
state = _extract_preloaded_state(html)
assert state is not None
assert state["data"]["data"]["page"]["placeholders"] == []
def test_extract_preloaded_state_missing_returns_none() -> None:
"""Нет маркера → None (а не исключение)."""
assert _extract_preloaded_state("<html><body>no state here</body></html>") is None
Reference in a new issue View git blame Copy permalink