397 lines
15 KiB
Python
397 lines
15 KiB
Python
"""Offline smoke for avito_houses parser.
|
||
|
||
Uses minimal JSON fixture mimicking Avito __preloadedState__ structure.
|
||
Не требует сети, БД или curl_cffi — тестирует только чистый парсер.
|
||
"""
|
||
|
||
from datetime import date
|
||
|
||
import pytest
|
||
|
||
from app.services.scrapers.avito_houses import (
|
||
HouseCatalogEnrichment,
|
||
HouseInfo,
|
||
HouseReview,
|
||
MiniSerpListing,
|
||
PlacementHistoryItem,
|
||
RecommendationItem,
|
||
SellerInfo,
|
||
_parse_ru_date,
|
||
parse_houses_state,
|
||
)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Minimal __preloadedState__ fixture (Python dict — не raw HTML)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
MINIMAL_STATE: dict = {
|
||
"data": {"data": {"page": {"placeholders": [
|
||
{"type": "breadcrumbs"},
|
||
{"type": "title"},
|
||
{"type": "gallery", "props": {}},
|
||
{
|
||
"type": "housePage",
|
||
"props": {"developmentData": {
|
||
"avitoId": 3171365,
|
||
"id": "MTE3LjM3MDg5MA",
|
||
"title": "Akademika Postovskogo 17a",
|
||
"address": "ул. Постовского, 17а",
|
||
"fullAddress": "Свердловская обл., Екатеринбург, ул. Постовского, 17а",
|
||
"coords": {"lat": 56.790699, "lng": 60.580191},
|
||
"aboutDevelopment": {"expandParams": {"items": [
|
||
{"title": "Параметры дома", "params": [
|
||
{"type": "Год постройки", "value": "2019"},
|
||
{"type": "Этажей", "value": "25"},
|
||
{"type": "Тип дома", "value": "Монолитный"},
|
||
{"type": "Пассажирский лифт", "value": "2"},
|
||
{"type": "Консьерж", "value": "Да"},
|
||
{"type": "Закрытый двор", "value": "Да"},
|
||
{"type": "Детская площадка", "value": "Да"},
|
||
]},
|
||
]}},
|
||
"developer": {"name": "Атомстройкомплекс", "key": "atomstroy"},
|
||
"mapPreview": {
|
||
"distance": "в 5 минутах ходьбы",
|
||
"objects": "школа, парк, аптека",
|
||
"pins": [{"lat": 56.79, "lng": 60.58, "type": "school"}],
|
||
},
|
||
"ratingBadge": {"info": {"score": 4.666, "scoreString": "4,7"}},
|
||
"ratingSummaryStat": {
|
||
"ratingStat": [{"score": 5, "count": 4}, {"score": 4, "count": 2}],
|
||
"reviewCount": 6,
|
||
},
|
||
}},
|
||
},
|
||
{
|
||
"type": "reviews",
|
||
"props": {"entries": [
|
||
{"type": "rating", "value": {
|
||
"id": 12345,
|
||
"author": {"title": "Иван"},
|
||
"reviewTitle": "Хорошо",
|
||
"score": 5,
|
||
"modelExperience": "Живу в своей квартире",
|
||
"rated": "9 октября 2025",
|
||
"textSections": [
|
||
{"title": "", "text": "Общий отзыв"},
|
||
{"title": "Преимущества", "text": "Парк рядом"},
|
||
{"title": "Недостатки", "text": "Парковки мало"},
|
||
],
|
||
}},
|
||
{"type": "pages", "value": {"nextPageUrl": "/reviews?page=2"}},
|
||
]},
|
||
},
|
||
{
|
||
"type": "miniSerp",
|
||
"props": {"items": [
|
||
{
|
||
"id": 7986882804,
|
||
"title": "3-к. квартира",
|
||
"description": "Описание",
|
||
"url": "/ekaterinburg/kvartiry/test_7986882804",
|
||
"price": "11 990 000 ₽",
|
||
"geo": {"content": "Чкаловская, 2,7 км", "colors": ["#cf2734"]},
|
||
"seller": {
|
||
"name": "DOMRF66",
|
||
"type": "Компания",
|
||
"from": "На Авито с июля 2012",
|
||
"url": "/brands/domrf66",
|
||
"logos": [],
|
||
},
|
||
},
|
||
]},
|
||
},
|
||
{
|
||
"type": "housePlacementHistory",
|
||
"props": {"items": [
|
||
{
|
||
"id": 7000000001,
|
||
"title": "1-к. квартира",
|
||
"startPrice": 5000000,
|
||
"startPriceDate": 1697000000,
|
||
"lastPrice": 4900000,
|
||
"lastPriceDate": 1715000000,
|
||
"exposure": 210,
|
||
"itemImage": {},
|
||
},
|
||
]},
|
||
},
|
||
{"type": "uxFeedback"},
|
||
{
|
||
"type": "recommendations",
|
||
"props": {"items": [
|
||
{
|
||
"title": "Похожая",
|
||
"priceRange": "от 6 млн",
|
||
"additionalInfo": ["Адрес 1"],
|
||
"url": "/rec1",
|
||
"images": [{"208x156": "https://avito.ru/img1.jpg"}],
|
||
},
|
||
]},
|
||
},
|
||
{"type": "padding"},
|
||
]}}}
|
||
}
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Helpers
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_ru_date_parse() -> None:
|
||
assert _parse_ru_date("9 октября 2025") == date(2025, 10, 9)
|
||
assert _parse_ru_date("31 декабря 2020") == date(2020, 12, 31)
|
||
assert _parse_ru_date("1 января 2000") == date(2000, 1, 1)
|
||
assert _parse_ru_date(None) is None
|
||
assert _parse_ru_date("bogus") is None
|
||
assert _parse_ru_date("") is None
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Main parse test
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_parse_minimal_state() -> None:
|
||
e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
|
||
|
||
assert isinstance(e, HouseCatalogEnrichment)
|
||
assert e.house_url == "/catalog/houses/ekaterinburg/test/3171365"
|
||
|
||
|
||
def test_house_info() -> None:
|
||
e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
|
||
h = e.house
|
||
|
||
assert isinstance(h, HouseInfo)
|
||
assert h.ext_id == 3171365
|
||
assert h.ext_id_hash == "MTE3LjM3MDg5MA"
|
||
assert h.title == "Akademika Postovskogo 17a"
|
||
assert h.short_address == "ул. Постовского, 17а"
|
||
assert h.full_address == "Свердловская обл., Екатеринбург, ул. Постовского, 17а"
|
||
assert h.lat == 56.790699
|
||
assert h.lon == 60.580191
|
||
|
||
|
||
def test_house_expand_params() -> None:
|
||
e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
|
||
h = e.house
|
||
|
||
assert h.year_built == 2019
|
||
assert h.total_floors == 25
|
||
assert h.house_type == "monolith"
|
||
assert h.passenger_elevators == 2
|
||
assert h.has_concierge is True
|
||
assert h.closed_yard is True
|
||
assert h.has_playground is True
|
||
|
||
|
||
def test_house_developer_and_rating() -> None:
|
||
e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
|
||
h = e.house
|
||
|
||
assert h.developer_name == "Атомстройкомплекс"
|
||
assert h.developer_key == "atomstroy"
|
||
assert h.rating_score == 4.666
|
||
assert h.rating_string == "4,7"
|
||
assert h.reviews_count == 6
|
||
assert len(h.rating_distribution) == 2
|
||
|
||
|
||
def test_house_map_and_infra() -> None:
|
||
e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
|
||
h = e.house
|
||
|
||
assert h.infrastructure_walk_distance == "в 5 минутах ходьбы"
|
||
assert h.infrastructure_summary == "школа, парк, аптека"
|
||
assert len(h.map_pins) == 1
|
||
assert h.map_pins[0]["type"] == "school"
|
||
|
||
|
||
def test_house_raw_characteristics() -> None:
|
||
e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
|
||
# raw_characteristics = полный expandParams items (1 категория)
|
||
assert len(e.house.raw_characteristics) == 1
|
||
assert e.house.raw_characteristics[0]["title"] == "Параметры дома"
|
||
|
||
|
||
def test_reviews() -> None:
|
||
e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
|
||
|
||
assert len(e.reviews) == 1
|
||
assert e.reviews_next_page_url == "/reviews?page=2"
|
||
|
||
r = e.reviews[0]
|
||
assert isinstance(r, HouseReview)
|
||
assert r.ext_review_id == 12345
|
||
assert r.author_name == "Иван"
|
||
assert r.review_title == "Хорошо"
|
||
assert r.score == 5
|
||
assert r.model_experience == "Живу в своей квартире"
|
||
assert r.rated_date == date(2025, 10, 9)
|
||
assert r.text_main == "Общий отзыв"
|
||
assert r.text_pros == "Парк рядом"
|
||
assert r.text_cons == "Парковки мало"
|
||
assert r.raw_payload is not None
|
||
|
||
|
||
def test_mini_serp() -> None:
|
||
e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
|
||
|
||
assert len(e.mini_serp) == 1
|
||
m = e.mini_serp[0]
|
||
assert isinstance(m, MiniSerpListing)
|
||
assert m.ext_item_id == 7986882804
|
||
assert m.price_rub == 11990000
|
||
assert m.metro_text == "Чкаловская, 2,7 км"
|
||
assert m.metro_color == "#cf2734"
|
||
assert m.description == "Описание"
|
||
|
||
|
||
def test_seller_extraction() -> None:
|
||
e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
|
||
m = e.mini_serp[0]
|
||
|
||
assert m.seller is not None
|
||
s = m.seller
|
||
assert isinstance(s, SellerInfo)
|
||
assert s.ext_seller_id == "domrf66"
|
||
assert s.name == "DOMRF66"
|
||
assert s.seller_type == "Компания"
|
||
assert s.on_platform_since == "На Авито с июля 2012"
|
||
assert s.profile_url == "/brands/domrf66"
|
||
|
||
|
||
def test_placement_history() -> None:
|
||
e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
|
||
|
||
assert len(e.placement_history) == 1
|
||
p = e.placement_history[0]
|
||
assert isinstance(p, PlacementHistoryItem)
|
||
assert p.ext_item_id == "7000000001"
|
||
assert p.title == "1-к. квартира"
|
||
assert p.start_price == 5000000
|
||
assert p.last_price == 4900000
|
||
assert p.exposure_days == 210
|
||
# removed_date ВСЕГДА None — виджет не отдаёт!
|
||
assert p.removed_date is None
|
||
# raw_payload не содержит itemImage (скипается)
|
||
assert p.raw_payload is not None
|
||
assert "itemImage" not in p.raw_payload
|
||
# start_price_date и last_price_date — из unix timestamp
|
||
assert isinstance(p.start_price_date, date)
|
||
assert isinstance(p.last_price_date, date)
|
||
|
||
|
||
def test_recommendations() -> None:
|
||
e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
|
||
|
||
assert len(e.recommendations) == 1
|
||
rec = e.recommendations[0]
|
||
assert isinstance(rec, RecommendationItem)
|
||
assert rec.title == "Похожая"
|
||
assert rec.price_text == "от 6 млн"
|
||
assert rec.address == "Адрес 1"
|
||
assert rec.url == "/rec1"
|
||
assert rec.image_url == "https://avito.ru/img1.jpg"
|
||
|
||
|
||
def test_missing_optional_widgets() -> None:
|
||
"""Парсер не падает если опциональные виджеты отсутствуют."""
|
||
state_minimal: dict = {
|
||
"data": {"data": {"page": {"placeholders": [
|
||
{
|
||
"type": "housePage",
|
||
"props": {"developmentData": {
|
||
"avitoId": 9999,
|
||
"coords": {"lat": 56.0, "lng": 60.0},
|
||
"aboutDevelopment": {"expandParams": {"items": []}},
|
||
"developer": {},
|
||
"mapPreview": {},
|
||
"ratingBadge": {"info": {}},
|
||
"ratingSummaryStat": {},
|
||
}},
|
||
},
|
||
]}}}
|
||
}
|
||
e = parse_houses_state(state_minimal, "/catalog/houses/test/9999")
|
||
assert e.house.ext_id == 9999
|
||
assert e.reviews == []
|
||
assert e.mini_serp == []
|
||
assert e.placement_history == []
|
||
assert e.recommendations == []
|
||
assert e.reviews_next_page_url is None
|
||
|
||
|
||
def test_invalid_state_raises() -> None:
|
||
"""ValueError если placeholders не найден."""
|
||
with pytest.raises(ValueError, match="placeholders"):
|
||
parse_houses_state({}, "/catalog/houses/test/1")
|
||
|
||
|
||
def test_missing_house_page_raises() -> None:
|
||
"""ValueError если housePage виджет отсутствует."""
|
||
state: dict = {
|
||
"data": {"data": {"page": {"placeholders": [
|
||
{"type": "breadcrumbs"},
|
||
]}}}
|
||
}
|
||
with pytest.raises(ValueError, match="housePage"):
|
||
parse_houses_state(state, "/catalog/houses/test/1")
|
||
|
||
|
||
def test_house_type_normalization() -> None:
|
||
"""Все варианты типа дома нормализуются корректно."""
|
||
from app.services.scrapers.avito_houses import _normalize_house_type
|
||
|
||
assert _normalize_house_type("Монолитный") == "monolith"
|
||
assert _normalize_house_type("Панельный") == "panel"
|
||
assert _normalize_house_type("Кирпичный") == "brick"
|
||
assert _normalize_house_type("Монолитно-кирпичный") == "monolith_brick"
|
||
assert _normalize_house_type("Блочный") == "block"
|
||
assert _normalize_house_type("Деревянный") == "wood"
|
||
assert _normalize_house_type("Неизвестный") == "other"
|
||
assert _normalize_house_type(None) is None
|
||
|
||
|
||
def test_strip_price() -> None:
|
||
"""_strip_price корректно извлекает числа из строк Avito."""
|
||
from app.services.scrapers.avito_houses import _strip_price
|
||
|
||
assert _strip_price("11 990 000 ₽") == 11990000
|
||
assert _strip_price("5 000 000 ₽") == 5000000
|
||
assert _strip_price(None) is None
|
||
assert _strip_price("") is None
|
||
|
||
|
||
def test_mini_serp_no_seller() -> None:
|
||
"""Объявление без продавца не вызывает ошибку."""
|
||
state: dict = {
|
||
"data": {"data": {"page": {"placeholders": [
|
||
{
|
||
"type": "housePage",
|
||
"props": {"developmentData": {
|
||
"avitoId": 111,
|
||
"coords": {"lat": 56.0, "lng": 60.0},
|
||
"aboutDevelopment": {"expandParams": {"items": []}},
|
||
"developer": {},
|
||
"mapPreview": {},
|
||
"ratingBadge": {"info": {}},
|
||
"ratingSummaryStat": {},
|
||
}},
|
||
},
|
||
{
|
||
"type": "miniSerp",
|
||
"props": {"items": [
|
||
{"id": 12345, "title": "1-к.", "url": "/test", "price": "3 000 000 ₽",
|
||
"geo": {"content": "Центр", "colors": []}},
|
||
]},
|
||
},
|
||
]}}}
|
||
}
|
||
e = parse_houses_state(state, "/catalog/houses/test/111")
|
||
assert len(e.mini_serp) == 1
|
||
assert e.mini_serp[0].seller is None
|
||
assert e.mini_serp[0].price_rub == 3000000
|