719 lines
32 KiB
Python
719 lines
32 KiB
Python
"""Offline smoke for avito_houses parser.
|
||
|
||
Uses minimal JSON fixture mimicking Avito __preloadedState__ structure.
|
||
Не требует сети, БД или curl_cffi — тестирует только чистый парсер.
|
||
"""
|
||
|
||
from datetime import date
|
||
from pathlib import Path
|
||
|
||
import pytest
|
||
|
||
from app.services.scrapers.avito_houses import (
|
||
HouseCatalogEnrichment,
|
||
HouseInfo,
|
||
HouseReview,
|
||
MiniSerpListing,
|
||
PlacementHistoryItem,
|
||
RecommendationItem,
|
||
SellerInfo,
|
||
_extract_preloaded_state,
|
||
_parse_ru_date,
|
||
parse_houses_state,
|
||
)
|
||
|
||
FIXTURES_DIR = Path(__file__).parent / "fixtures"
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Minimal __preloadedState__ fixture (Python dict — не raw HTML)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
MINIMAL_STATE: dict = {
|
||
"data": {
|
||
"data": {
|
||
"page": {
|
||
"placeholders": [
|
||
{"type": "breadcrumbs"},
|
||
{"type": "title"},
|
||
{"type": "gallery", "props": {}},
|
||
{
|
||
"type": "housePage",
|
||
"props": {
|
||
"developmentData": {
|
||
"avitoId": 3171365,
|
||
"id": "MTE3LjM3MDg5MA",
|
||
"title": "Akademika Postovskogo 17a",
|
||
"address": "ул. Постовского, 17а",
|
||
"fullAddress": (
|
||
"Свердловская обл., Екатеринбург, ул. Постовского, 17а"
|
||
),
|
||
"coords": {"lat": 56.790699, "lng": 60.580191},
|
||
"aboutDevelopment": {
|
||
"expandParams": {
|
||
"items": [
|
||
{
|
||
"title": "Параметры дома",
|
||
"params": [
|
||
{"type": "Год постройки", "value": "2019"},
|
||
{"type": "Этажей", "value": "25"},
|
||
{"type": "Тип дома", "value": "Монолитный"},
|
||
{"type": "Пассажирский лифт", "value": "2"},
|
||
{"type": "Консьерж", "value": "Да"},
|
||
{"type": "Закрытый двор", "value": "Да"},
|
||
{"type": "Детская площадка", "value": "Да"},
|
||
],
|
||
},
|
||
]
|
||
}
|
||
},
|
||
"developer": {"name": "Атомстройкомплекс", "key": "atomstroy"},
|
||
"mapPreview": {
|
||
"distance": "в 5 минутах ходьбы",
|
||
"objects": "школа, парк, аптека",
|
||
"pins": [{"lat": 56.79, "lng": 60.58, "type": "school"}],
|
||
},
|
||
"ratingBadge": {"info": {"score": 4.666, "scoreString": "4,7"}},
|
||
"ratingSummaryStat": {
|
||
"ratingStat": [
|
||
{"score": 5, "count": 4},
|
||
{"score": 4, "count": 2},
|
||
],
|
||
"reviewCount": 6,
|
||
},
|
||
}
|
||
},
|
||
},
|
||
{
|
||
"type": "reviews",
|
||
"props": {
|
||
"entries": [
|
||
{
|
||
"type": "rating",
|
||
"value": {
|
||
"id": 12345,
|
||
"author": {"title": "Иван"},
|
||
"reviewTitle": "Хорошо",
|
||
"score": 5,
|
||
"modelExperience": "Живу в своей квартире",
|
||
"rated": "9 октября 2025",
|
||
"textSections": [
|
||
{"title": "", "text": "Общий отзыв"},
|
||
{"title": "Преимущества", "text": "Парк рядом"},
|
||
{"title": "Недостатки", "text": "Парковки мало"},
|
||
],
|
||
},
|
||
},
|
||
{"type": "pages", "value": {"nextPageUrl": "/reviews?page=2"}},
|
||
]
|
||
},
|
||
},
|
||
{
|
||
"type": "miniSerp",
|
||
"props": {
|
||
"items": [
|
||
{
|
||
"id": 7986882804,
|
||
"title": "3-к. квартира",
|
||
"description": "Описание",
|
||
"url": "/ekaterinburg/kvartiry/test_7986882804",
|
||
"price": "11 990 000 ₽",
|
||
"geo": {"content": "Чкаловская, 2,7 км", "colors": ["#cf2734"]},
|
||
"seller": {
|
||
"name": "DOMRF66",
|
||
"type": "Компания",
|
||
"from": "На Авито с июля 2012",
|
||
"url": "/brands/domrf66",
|
||
"logos": [],
|
||
},
|
||
},
|
||
]
|
||
},
|
||
},
|
||
{
|
||
"type": "housePlacementHistory",
|
||
"props": {
|
||
"items": [
|
||
{
|
||
"id": 7000000001,
|
||
"title": "1-к. квартира",
|
||
"startPrice": 5000000,
|
||
"startPriceDate": 1697000000,
|
||
"lastPrice": 4900000,
|
||
"lastPriceDate": 1715000000,
|
||
"exposure": 210,
|
||
"itemImage": {},
|
||
},
|
||
]
|
||
},
|
||
},
|
||
{"type": "uxFeedback"},
|
||
{
|
||
"type": "recommendations",
|
||
"props": {
|
||
"items": [
|
||
{
|
||
"title": "Похожая",
|
||
"priceRange": "от 6 млн",
|
||
"additionalInfo": ["Адрес 1"],
|
||
"url": "/rec1",
|
||
"images": [{"208x156": "https://avito.ru/img1.jpg"}],
|
||
},
|
||
]
|
||
},
|
||
},
|
||
{"type": "padding"},
|
||
]
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Helpers
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_ru_date_parse() -> None:
|
||
assert _parse_ru_date("9 октября 2025") == date(2025, 10, 9)
|
||
assert _parse_ru_date("31 декабря 2020") == date(2020, 12, 31)
|
||
assert _parse_ru_date("1 января 2000") == date(2000, 1, 1)
|
||
assert _parse_ru_date(None) is None
|
||
assert _parse_ru_date("bogus") is None
|
||
assert _parse_ru_date("") is None
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Main parse test
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_parse_minimal_state() -> None:
|
||
e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
|
||
|
||
assert isinstance(e, HouseCatalogEnrichment)
|
||
assert e.house_url == "/catalog/houses/ekaterinburg/test/3171365"
|
||
|
||
|
||
def test_house_info() -> None:
|
||
e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
|
||
h = e.house
|
||
|
||
assert isinstance(h, HouseInfo)
|
||
assert h.ext_id == 3171365
|
||
assert h.ext_id_hash == "MTE3LjM3MDg5MA"
|
||
assert h.title == "Akademika Postovskogo 17a"
|
||
assert h.short_address == "ул. Постовского, 17а"
|
||
assert h.full_address == "Свердловская обл., Екатеринбург, ул. Постовского, 17а"
|
||
assert h.lat == 56.790699
|
||
assert h.lon == 60.580191
|
||
|
||
|
||
def test_house_expand_params() -> None:
|
||
e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
|
||
h = e.house
|
||
|
||
assert h.year_built == 2019
|
||
assert h.total_floors == 25
|
||
assert h.house_type == "monolith"
|
||
assert h.passenger_elevators == 2
|
||
assert h.has_concierge is True
|
||
assert h.closed_yard is True
|
||
assert h.has_playground is True
|
||
|
||
|
||
def test_house_developer_and_rating() -> None:
|
||
e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
|
||
h = e.house
|
||
|
||
assert h.developer_name == "Атомстройкомплекс"
|
||
assert h.developer_key == "atomstroy"
|
||
assert h.rating_score == 4.666
|
||
assert h.rating_string == "4,7"
|
||
assert h.reviews_count == 6
|
||
assert len(h.rating_distribution) == 2
|
||
|
||
|
||
def test_house_map_and_infra() -> None:
|
||
e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
|
||
h = e.house
|
||
|
||
assert h.infrastructure_walk_distance == "в 5 минутах ходьбы"
|
||
assert h.infrastructure_summary == "школа, парк, аптека"
|
||
assert len(h.map_pins) == 1
|
||
assert h.map_pins[0]["type"] == "school"
|
||
|
||
|
||
def test_house_raw_characteristics() -> None:
|
||
e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
|
||
# raw_characteristics = полный expandParams items (1 категория)
|
||
assert len(e.house.raw_characteristics) == 1
|
||
assert e.house.raw_characteristics[0]["title"] == "Параметры дома"
|
||
|
||
|
||
def test_reviews() -> None:
|
||
e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
|
||
|
||
assert len(e.reviews) == 1
|
||
assert e.reviews_next_page_url == "/reviews?page=2"
|
||
|
||
r = e.reviews[0]
|
||
assert isinstance(r, HouseReview)
|
||
assert r.ext_review_id == 12345
|
||
assert r.author_name == "Иван"
|
||
assert r.review_title == "Хорошо"
|
||
assert r.score == 5
|
||
assert r.model_experience == "Живу в своей квартире"
|
||
assert r.rated_date == date(2025, 10, 9)
|
||
assert r.text_main == "Общий отзыв"
|
||
assert r.text_pros == "Парк рядом"
|
||
assert r.text_cons == "Парковки мало"
|
||
assert r.raw_payload is not None
|
||
|
||
|
||
def test_mini_serp() -> None:
|
||
e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
|
||
|
||
assert len(e.mini_serp) == 1
|
||
m = e.mini_serp[0]
|
||
assert isinstance(m, MiniSerpListing)
|
||
assert m.ext_item_id == 7986882804
|
||
assert m.price_rub == 11990000
|
||
assert m.metro_text == "Чкаловская, 2,7 км"
|
||
assert m.metro_color == "#cf2734"
|
||
assert m.description == "Описание"
|
||
|
||
|
||
def test_seller_extraction() -> None:
|
||
e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
|
||
m = e.mini_serp[0]
|
||
|
||
assert m.seller is not None
|
||
s = m.seller
|
||
assert isinstance(s, SellerInfo)
|
||
assert s.ext_seller_id == "domrf66"
|
||
assert s.name == "DOMRF66"
|
||
assert s.seller_type == "Компания"
|
||
assert s.on_platform_since == "На Авито с июля 2012"
|
||
assert s.profile_url == "/brands/domrf66"
|
||
|
||
|
||
def test_placement_history() -> None:
|
||
e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
|
||
|
||
assert len(e.placement_history) == 1
|
||
p = e.placement_history[0]
|
||
assert isinstance(p, PlacementHistoryItem)
|
||
assert p.ext_item_id == "7000000001"
|
||
assert p.title == "1-к. квартира"
|
||
assert p.start_price == 5000000
|
||
assert p.last_price == 4900000
|
||
assert p.exposure_days == 210
|
||
# removed_date ВСЕГДА None — виджет не отдаёт!
|
||
assert p.removed_date is None
|
||
# raw_payload не содержит itemImage (скипается)
|
||
assert p.raw_payload is not None
|
||
assert "itemImage" not in p.raw_payload
|
||
# start_price_date и last_price_date — из unix timestamp
|
||
assert isinstance(p.start_price_date, date)
|
||
assert isinstance(p.last_price_date, date)
|
||
|
||
|
||
def test_recommendations() -> None:
|
||
e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
|
||
|
||
assert len(e.recommendations) == 1
|
||
rec = e.recommendations[0]
|
||
assert isinstance(rec, RecommendationItem)
|
||
assert rec.title == "Похожая"
|
||
assert rec.price_text == "от 6 млн"
|
||
assert rec.address == "Адрес 1"
|
||
assert rec.url == "/rec1"
|
||
assert rec.image_url == "https://avito.ru/img1.jpg"
|
||
|
||
|
||
def test_missing_optional_widgets() -> None:
|
||
"""Парсер не падает если опциональные виджеты отсутствуют."""
|
||
state_minimal: dict = {
|
||
"data": {
|
||
"data": {
|
||
"page": {
|
||
"placeholders": [
|
||
{
|
||
"type": "housePage",
|
||
"props": {
|
||
"developmentData": {
|
||
"avitoId": 9999,
|
||
"coords": {"lat": 56.0, "lng": 60.0},
|
||
"aboutDevelopment": {"expandParams": {"items": []}},
|
||
"developer": {},
|
||
"mapPreview": {},
|
||
"ratingBadge": {"info": {}},
|
||
"ratingSummaryStat": {},
|
||
}
|
||
},
|
||
},
|
||
]
|
||
}
|
||
}
|
||
}
|
||
}
|
||
e = parse_houses_state(state_minimal, "/catalog/houses/test/9999")
|
||
assert e.house.ext_id == 9999
|
||
assert e.reviews == []
|
||
assert e.mini_serp == []
|
||
assert e.placement_history == []
|
||
assert e.recommendations == []
|
||
assert e.reviews_next_page_url is None
|
||
|
||
|
||
def test_invalid_state_raises() -> None:
|
||
"""ValueError если placeholders не найден."""
|
||
with pytest.raises(ValueError, match="placeholders"):
|
||
parse_houses_state({}, "/catalog/houses/test/1")
|
||
|
||
|
||
def test_missing_house_page_raises() -> None:
|
||
"""ValueError если housePage виджет отсутствует (вторичка — /catalog/houses/)."""
|
||
state: dict = {
|
||
"data": {
|
||
"data": {
|
||
"page": {
|
||
"placeholders": [
|
||
{"type": "breadcrumbs"},
|
||
]
|
||
}
|
||
}
|
||
}
|
||
}
|
||
with pytest.raises(ValueError, match="housePage"):
|
||
parse_houses_state(state, "/catalog/houses/test/1")
|
||
|
||
|
||
def test_novostroyka_minimal_development_page() -> None:
|
||
"""Новостройка: виджет 'developmentPage' парсится в partial HouseInfo без ValueError."""
|
||
state: dict = {
|
||
"data": {
|
||
"data": {
|
||
"page": {
|
||
"placeholders": [
|
||
{"type": "reNavigation"},
|
||
{
|
||
"type": "developmentPage",
|
||
"props": {
|
||
"developmentPage": {
|
||
"developmentData": {
|
||
"avitoId": 12879380,
|
||
"id": "MGDKHN70PQ",
|
||
"title": "ЖК Береговой",
|
||
"address": "ул. Щербакова, жилые дома",
|
||
"fullAddress": "",
|
||
"coords": {"lat": 56.77005, "lng": 60.67244},
|
||
"developer": {"name": "Талан", "key": "talan-1"},
|
||
"mapPreview": {
|
||
"distance": "в 15 минутах ходьбы",
|
||
"objects": "школа, детский сад",
|
||
"pins": [{"lat": 56.77, "lng": 60.67}],
|
||
},
|
||
"aboutDevelopment": {
|
||
"tabs": [
|
||
{
|
||
"title": "Параметры",
|
||
"expandParams": {
|
||
"items": [
|
||
{
|
||
"params": [
|
||
{
|
||
"type": "Тип дома",
|
||
"value": "Монолит",
|
||
},
|
||
{
|
||
"type": "Этажность",
|
||
"value": "от 4 до 24",
|
||
},
|
||
{
|
||
"type": (
|
||
"Пассажирских лифтов"
|
||
),
|
||
"value": "42",
|
||
},
|
||
{
|
||
"type": "Грузовых лифтов",
|
||
"value": "60",
|
||
},
|
||
]
|
||
}
|
||
]
|
||
},
|
||
}
|
||
]
|
||
},
|
||
}
|
||
}
|
||
},
|
||
},
|
||
{"type": "uxFeedback"},
|
||
]
|
||
}
|
||
}
|
||
}
|
||
}
|
||
url = "/catalog/novostroyki/ekaterinburg/zhk_beregovoy/12879380"
|
||
e = parse_houses_state(state, url)
|
||
|
||
assert isinstance(e, HouseCatalogEnrichment)
|
||
assert e.house_url == url
|
||
h = e.house
|
||
assert h.ext_id == 12879380
|
||
assert h.ext_id_hash == "MGDKHN70PQ"
|
||
assert h.title == "ЖК Береговой"
|
||
assert h.short_address == "ул. Щербакова, жилые дома"
|
||
assert h.lat == 56.77005
|
||
assert h.lon == 60.67244
|
||
assert h.house_type == "monolith" # "Монолит" → monolith
|
||
assert h.passenger_elevators == 42
|
||
assert h.cargo_elevators == 60
|
||
# Этажность = диапазон "от 4 до 24" → int-каст None (только в raw_characteristics)
|
||
assert h.total_floors is None
|
||
assert h.year_built is None # дом строится, года постройки нет
|
||
assert h.developer_name == "Талан"
|
||
assert h.developer_key == "talan-1"
|
||
assert h.infrastructure_summary == "школа, детский сад"
|
||
assert h.infrastructure_walk_distance == "в 15 минутах ходьбы"
|
||
assert len(h.map_pins) == 1
|
||
# Нет рейтинга/отзывов/истории у новостройки
|
||
assert h.rating_score is None
|
||
assert e.reviews == []
|
||
assert e.placement_history == []
|
||
assert e.recommendations == []
|
||
|
||
|
||
def test_novostroyka_no_development_page_graceful_skip() -> None:
|
||
"""Новостройка без developmentPage/housePage → graceful skip (НЕ ValueError)."""
|
||
state: dict = {
|
||
"data": {
|
||
"data": {
|
||
"page": {
|
||
"placeholders": [
|
||
{"type": "reNavigation"},
|
||
{"type": "uxFeedback"},
|
||
]
|
||
}
|
||
}
|
||
}
|
||
}
|
||
url = "/catalog/novostroyki/ekaterinburg/zhk_unknown/55667788"
|
||
e = parse_houses_state(state, url)
|
||
|
||
assert isinstance(e, HouseCatalogEnrichment)
|
||
assert e.house_url == url
|
||
# ext_id извлечён из хвоста URL
|
||
assert e.house.ext_id == 55667788
|
||
assert e.reviews == []
|
||
assert e.placement_history == []
|
||
|
||
|
||
def test_novostroyka_real_fixture_end_to_end() -> None:
|
||
"""Реальная фикстура новостройки: parse не кидает ValueError, отдаёт partial-обогащение."""
|
||
html = (FIXTURES_DIR / "avito_novostroyka_12879380.html").read_text(encoding="utf-8")
|
||
state = _extract_preloaded_state(html)
|
||
assert state is not None, "URL-encoded __preloadedState__ должен распарситься"
|
||
|
||
url = "/catalog/novostroyki/ekaterinburg/semeynyy_kvartal_beregovoy/12879380"
|
||
e = parse_houses_state(state, url)
|
||
|
||
assert isinstance(e, HouseCatalogEnrichment)
|
||
h = e.house
|
||
assert h.ext_id == 12879380
|
||
assert h.ext_id_hash # base64 internal id
|
||
assert h.house_type == "monolith"
|
||
assert h.passenger_elevators is not None and h.passenger_elevators > 0
|
||
assert h.lat is not None and h.lon is not None
|
||
assert h.developer_name # застройщик присутствует
|
||
# Новостройка: нет отзывов/истории/года постройки
|
||
assert h.year_built is None
|
||
assert e.reviews == []
|
||
assert e.placement_history == []
|
||
|
||
|
||
def test_house_type_normalization() -> None:
|
||
"""Все варианты типа дома нормализуются корректно."""
|
||
from app.services.scrapers.avito_houses import _normalize_house_type
|
||
|
||
assert _normalize_house_type("Монолитный") == "monolith"
|
||
assert _normalize_house_type("Панельный") == "panel"
|
||
assert _normalize_house_type("Кирпичный") == "brick"
|
||
assert _normalize_house_type("Монолитно-кирпичный") == "monolith_brick"
|
||
assert _normalize_house_type("Блочный") == "block"
|
||
assert _normalize_house_type("Деревянный") == "wood"
|
||
assert _normalize_house_type("Неизвестный") == "other"
|
||
assert _normalize_house_type(None) is None
|
||
|
||
|
||
def test_strip_price() -> None:
|
||
"""_strip_price корректно извлекает числа из строк Avito."""
|
||
from app.services.scrapers.avito_houses import _strip_price
|
||
|
||
assert _strip_price("11 990 000 ₽") == 11990000
|
||
assert _strip_price("5 000 000 ₽") == 5000000
|
||
assert _strip_price(None) is None
|
||
assert _strip_price("") is None
|
||
|
||
|
||
def test_mini_serp_no_seller() -> None:
|
||
"""Объявление без продавца не вызывает ошибку."""
|
||
state: dict = {
|
||
"data": {
|
||
"data": {
|
||
"page": {
|
||
"placeholders": [
|
||
{
|
||
"type": "housePage",
|
||
"props": {
|
||
"developmentData": {
|
||
"avitoId": 111,
|
||
"coords": {"lat": 56.0, "lng": 60.0},
|
||
"aboutDevelopment": {"expandParams": {"items": []}},
|
||
"developer": {},
|
||
"mapPreview": {},
|
||
"ratingBadge": {"info": {}},
|
||
"ratingSummaryStat": {},
|
||
}
|
||
},
|
||
},
|
||
{
|
||
"type": "miniSerp",
|
||
"props": {
|
||
"items": [
|
||
{
|
||
"id": 12345,
|
||
"title": "1-к.",
|
||
"url": "/test",
|
||
"price": "3 000 000 ₽",
|
||
"geo": {"content": "Центр", "colors": []},
|
||
},
|
||
]
|
||
},
|
||
},
|
||
]
|
||
}
|
||
}
|
||
}
|
||
}
|
||
e = parse_houses_state(state, "/catalog/houses/test/111")
|
||
assert len(e.mini_serp) == 1
|
||
assert e.mini_serp[0].seller is None
|
||
assert e.mini_serp[0].price_rub == 3000000
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Real-HTML fixture (regression for issue #566 — state-blob encoding/shape change)
|
||
# ---------------------------------------------------------------------------
|
||
#
|
||
# Fixture: реальная страница /catalog/houses/ekaterinburg/ul_yumasheva_6/115188,
|
||
# захвачена curl_cffi chrome120 (2026-05-28). Avito сменил формат __preloadedState__
|
||
# на URL-encoded литерал + обернул виджеты в placeholders[0]['widgets'] +
|
||
# вложил developmentData под props['housePage']. Этот тест ловит регрессию декода/shape.
|
||
|
||
|
||
def test_extract_preloaded_state_from_real_html() -> None:
|
||
"""_extract_preloaded_state декодирует текущий URL-encoded __preloadedState__."""
|
||
html = (FIXTURES_DIR / "avito_houses_sample.html").read_text(encoding="utf-8")
|
||
|
||
state = _extract_preloaded_state(html)
|
||
|
||
assert state is not None, "URL-encoded __preloadedState__ должен распарситься"
|
||
# Avito SSR обёртка — успешный ответ
|
||
assert state["data"]["status"] == 200
|
||
# Inner path до placeholders сохранился
|
||
placeholders = state["data"]["data"]["page"]["placeholders"]
|
||
assert isinstance(placeholders, list) and placeholders, "placeholders непустой массив"
|
||
|
||
|
||
def test_parse_real_fixture_end_to_end() -> None:
|
||
"""Фикстура HTML → _extract_preloaded_state → parse_houses_state → HouseInfo.ext_id."""
|
||
html = (FIXTURES_DIR / "avito_houses_sample.html").read_text(encoding="utf-8")
|
||
state = _extract_preloaded_state(html)
|
||
assert state is not None
|
||
|
||
e = parse_houses_state(state, "/catalog/houses/ekaterinburg/ul_yumasheva_6/115188")
|
||
|
||
assert isinstance(e, HouseCatalogEnrichment)
|
||
# housePage распарсился — ext_id (avitoId) присутствует и совпадает с URL
|
||
assert e.house.ext_id == 115188
|
||
assert e.house.ext_id_hash # base64 internal id присутствует
|
||
# expandParams распарсились (год постройки реальной фикстуры — 2016)
|
||
assert e.house.year_built == 2016
|
||
assert e.house.total_floors == 17
|
||
assert e.house.house_type == "monolith"
|
||
assert e.house.lat is not None and e.house.lon is not None
|
||
|
||
|
||
def test_extract_preloaded_state_jsparse_fallback() -> None:
|
||
"""Старая форма JSON.parse("...") всё ещё декодируется (fallback-ветка)."""
|
||
inner = '{\\"data\\":{\\"data\\":{\\"page\\":{\\"placeholders\\":[]}}}}'
|
||
html = f'<script>window.__preloadedState__ = JSON.parse("{inner}");</script>'
|
||
|
||
state = _extract_preloaded_state(html)
|
||
|
||
assert state is not None
|
||
assert state["data"]["data"]["page"]["placeholders"] == []
|
||
|
||
|
||
def test_extract_preloaded_state_missing_returns_none() -> None:
|
||
"""Нет маркера → None (а не исключение)."""
|
||
assert _extract_preloaded_state("<html><body>no state here</body></html>") is None
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Real-HTML fixture (regression for issue #1789 — MFE props nesting)
|
||
# ---------------------------------------------------------------------------
|
||
#
|
||
# Fixture: реальная страница /catalog/houses/ekaterinburg/ul_soni_morozovoy_190/572538,
|
||
# захвачена браузер-сервисом (2026-06-19). Avito перенёс данные виджетов на уровень
|
||
# глубже — под ключ с именем виджета:
|
||
# reviews: props['reviews']['entries'] (было props['entries'])
|
||
# miniSerp: props['miniSerp']['items'] (было props['items'])
|
||
# housePlacementHistory: props['housePlacementHistory']['items'] (было props['items'])
|
||
# recommendations: props['recommendations']['items'] (было props['items'])
|
||
# Из-за этого reviews/placement/seller-обогащение НЕ долетало в БД (0 строк).
|
||
# Этот тест ловит регрессию шейпа props.
|
||
|
||
|
||
def test_parse_mfe_fixture_widgets_reach_db() -> None:
|
||
"""Фикстура MFE: reviews/placement/miniSerp/house_type/year_built распарсены непусто."""
|
||
html = (FIXTURES_DIR / "avito_house_572538.html").read_text(encoding="utf-8")
|
||
state = _extract_preloaded_state(html)
|
||
assert state is not None, "URL-encoded __preloadedState__ должен распарситься"
|
||
|
||
e = parse_houses_state(state, "/catalog/houses/ekaterinburg/ul_soni_morozovoy_190/572538")
|
||
|
||
# housePage — тип дома и год постройки долетают
|
||
assert e.house.ext_id == 572538
|
||
assert e.house.house_type is not None, "house_type должен парситься из expandParams"
|
||
assert e.house.house_type == "brick"
|
||
assert e.house.year_built == 1994
|
||
assert e.house.reviews_count == 3
|
||
|
||
# reviews — >=1 отзыв с текстом, автором, оценкой
|
||
assert len(e.reviews) >= 1, "reviews должны распарситься из props['reviews']['entries']"
|
||
r = e.reviews[0]
|
||
assert r.ext_review_id, "у отзыва должен быть id"
|
||
assert r.author_name, "у отзыва должен быть автор"
|
||
assert r.score is not None, "у отзыва должна быть оценка"
|
||
assert r.text_main, "у отзыва должен быть основной текст (textSections)"
|
||
|
||
# placement — >=1 item из props['housePlacementHistory']['items']
|
||
assert len(e.placement_history) >= 1, "placement должен распарситься"
|
||
p = e.placement_history[0]
|
||
assert p.ext_item_id, "у placement item должен быть id"
|
||
assert p.start_price is not None
|
||
assert p.last_price is not None
|
||
assert isinstance(p.last_price_date, date)
|
||
|
||
# miniSerp — >=1 объявление из props['miniSerp']['items']
|
||
assert len(e.mini_serp) >= 1, "miniSerp должен распарситься"
|
||
m = e.mini_serp[0]
|
||
assert m.ext_item_id
|
||
assert m.price_rub is not None
|