"""Offline smoke for avito_houses parser. Uses minimal JSON fixture mimicking Avito __preloadedState__ structure. Не требует сети, БД или curl_cffi — тестирует только чистый парсер. """ from datetime import date from pathlib import Path import pytest from app.services.scrapers.avito_houses import ( HouseCatalogEnrichment, HouseInfo, HouseReview, MiniSerpListing, PlacementHistoryItem, RecommendationItem, SellerInfo, _extract_preloaded_state, _parse_ru_date, parse_houses_state, ) FIXTURES_DIR = Path(__file__).parent / "fixtures" # --------------------------------------------------------------------------- # Minimal __preloadedState__ fixture (Python dict — не raw HTML) # --------------------------------------------------------------------------- MINIMAL_STATE: dict = { "data": {"data": {"page": {"placeholders": [ {"type": "breadcrumbs"}, {"type": "title"}, {"type": "gallery", "props": {}}, { "type": "housePage", "props": {"developmentData": { "avitoId": 3171365, "id": "MTE3LjM3MDg5MA", "title": "Akademika Postovskogo 17a", "address": "ул. Постовского, 17а", "fullAddress": "Свердловская обл., Екатеринбург, ул. Постовского, 17а", "coords": {"lat": 56.790699, "lng": 60.580191}, "aboutDevelopment": {"expandParams": {"items": [ {"title": "Параметры дома", "params": [ {"type": "Год постройки", "value": "2019"}, {"type": "Этажей", "value": "25"}, {"type": "Тип дома", "value": "Монолитный"}, {"type": "Пассажирский лифт", "value": "2"}, {"type": "Консьерж", "value": "Да"}, {"type": "Закрытый двор", "value": "Да"}, {"type": "Детская площадка", "value": "Да"}, ]}, ]}}, "developer": {"name": "Атомстройкомплекс", "key": "atomstroy"}, "mapPreview": { "distance": "в 5 минутах ходьбы", "objects": "школа, парк, аптека", "pins": [{"lat": 56.79, "lng": 60.58, "type": "school"}], }, "ratingBadge": {"info": {"score": 4.666, "scoreString": "4,7"}}, "ratingSummaryStat": { "ratingStat": [{"score": 5, "count": 4}, {"score": 4, "count": 2}], "reviewCount": 6, }, }}, }, { "type": "reviews", "props": {"entries": [ {"type": "rating", "value": { "id": 12345, "author": {"title": "Иван"}, "reviewTitle": "Хорошо", "score": 5, "modelExperience": "Живу в своей квартире", "rated": "9 октября 2025", "textSections": [ {"title": "", "text": "Общий отзыв"}, {"title": "Преимущества", "text": "Парк рядом"}, {"title": "Недостатки", "text": "Парковки мало"}, ], }}, {"type": "pages", "value": {"nextPageUrl": "/reviews?page=2"}}, ]}, }, { "type": "miniSerp", "props": {"items": [ { "id": 7986882804, "title": "3-к. квартира", "description": "Описание", "url": "/ekaterinburg/kvartiry/test_7986882804", "price": "11 990 000 ₽", "geo": {"content": "Чкаловская, 2,7 км", "colors": ["#cf2734"]}, "seller": { "name": "DOMRF66", "type": "Компания", "from": "На Авито с июля 2012", "url": "/brands/domrf66", "logos": [], }, }, ]}, }, { "type": "housePlacementHistory", "props": {"items": [ { "id": 7000000001, "title": "1-к. квартира", "startPrice": 5000000, "startPriceDate": 1697000000, "lastPrice": 4900000, "lastPriceDate": 1715000000, "exposure": 210, "itemImage": {}, }, ]}, }, {"type": "uxFeedback"}, { "type": "recommendations", "props": {"items": [ { "title": "Похожая", "priceRange": "от 6 млн", "additionalInfo": ["Адрес 1"], "url": "/rec1", "images": [{"208x156": "https://avito.ru/img1.jpg"}], }, ]}, }, {"type": "padding"}, ]}}} } # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- def test_ru_date_parse() -> None: assert _parse_ru_date("9 октября 2025") == date(2025, 10, 9) assert _parse_ru_date("31 декабря 2020") == date(2020, 12, 31) assert _parse_ru_date("1 января 2000") == date(2000, 1, 1) assert _parse_ru_date(None) is None assert _parse_ru_date("bogus") is None assert _parse_ru_date("") is None # --------------------------------------------------------------------------- # Main parse test # --------------------------------------------------------------------------- def test_parse_minimal_state() -> None: e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365") assert isinstance(e, HouseCatalogEnrichment) assert e.house_url == "/catalog/houses/ekaterinburg/test/3171365" def test_house_info() -> None: e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365") h = e.house assert isinstance(h, HouseInfo) assert h.ext_id == 3171365 assert h.ext_id_hash == "MTE3LjM3MDg5MA" assert h.title == "Akademika Postovskogo 17a" assert h.short_address == "ул. Постовского, 17а" assert h.full_address == "Свердловская обл., Екатеринбург, ул. Постовского, 17а" assert h.lat == 56.790699 assert h.lon == 60.580191 def test_house_expand_params() -> None: e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365") h = e.house assert h.year_built == 2019 assert h.total_floors == 25 assert h.house_type == "monolith" assert h.passenger_elevators == 2 assert h.has_concierge is True assert h.closed_yard is True assert h.has_playground is True def test_house_developer_and_rating() -> None: e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365") h = e.house assert h.developer_name == "Атомстройкомплекс" assert h.developer_key == "atomstroy" assert h.rating_score == 4.666 assert h.rating_string == "4,7" assert h.reviews_count == 6 assert len(h.rating_distribution) == 2 def test_house_map_and_infra() -> None: e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365") h = e.house assert h.infrastructure_walk_distance == "в 5 минутах ходьбы" assert h.infrastructure_summary == "школа, парк, аптека" assert len(h.map_pins) == 1 assert h.map_pins[0]["type"] == "school" def test_house_raw_characteristics() -> None: e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365") # raw_characteristics = полный expandParams items (1 категория) assert len(e.house.raw_characteristics) == 1 assert e.house.raw_characteristics[0]["title"] == "Параметры дома" def test_reviews() -> None: e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365") assert len(e.reviews) == 1 assert e.reviews_next_page_url == "/reviews?page=2" r = e.reviews[0] assert isinstance(r, HouseReview) assert r.ext_review_id == 12345 assert r.author_name == "Иван" assert r.review_title == "Хорошо" assert r.score == 5 assert r.model_experience == "Живу в своей квартире" assert r.rated_date == date(2025, 10, 9) assert r.text_main == "Общий отзыв" assert r.text_pros == "Парк рядом" assert r.text_cons == "Парковки мало" assert r.raw_payload is not None def test_mini_serp() -> None: e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365") assert len(e.mini_serp) == 1 m = e.mini_serp[0] assert isinstance(m, MiniSerpListing) assert m.ext_item_id == 7986882804 assert m.price_rub == 11990000 assert m.metro_text == "Чкаловская, 2,7 км" assert m.metro_color == "#cf2734" assert m.description == "Описание" def test_seller_extraction() -> None: e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365") m = e.mini_serp[0] assert m.seller is not None s = m.seller assert isinstance(s, SellerInfo) assert s.ext_seller_id == "domrf66" assert s.name == "DOMRF66" assert s.seller_type == "Компания" assert s.on_platform_since == "На Авито с июля 2012" assert s.profile_url == "/brands/domrf66" def test_placement_history() -> None: e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365") assert len(e.placement_history) == 1 p = e.placement_history[0] assert isinstance(p, PlacementHistoryItem) assert p.ext_item_id == "7000000001" assert p.title == "1-к. квартира" assert p.start_price == 5000000 assert p.last_price == 4900000 assert p.exposure_days == 210 # removed_date ВСЕГДА None — виджет не отдаёт! assert p.removed_date is None # raw_payload не содержит itemImage (скипается) assert p.raw_payload is not None assert "itemImage" not in p.raw_payload # start_price_date и last_price_date — из unix timestamp assert isinstance(p.start_price_date, date) assert isinstance(p.last_price_date, date) def test_recommendations() -> None: e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365") assert len(e.recommendations) == 1 rec = e.recommendations[0] assert isinstance(rec, RecommendationItem) assert rec.title == "Похожая" assert rec.price_text == "от 6 млн" assert rec.address == "Адрес 1" assert rec.url == "/rec1" assert rec.image_url == "https://avito.ru/img1.jpg" def test_missing_optional_widgets() -> None: """Парсер не падает если опциональные виджеты отсутствуют.""" state_minimal: dict = { "data": {"data": {"page": {"placeholders": [ { "type": "housePage", "props": {"developmentData": { "avitoId": 9999, "coords": {"lat": 56.0, "lng": 60.0}, "aboutDevelopment": {"expandParams": {"items": []}}, "developer": {}, "mapPreview": {}, "ratingBadge": {"info": {}}, "ratingSummaryStat": {}, }}, }, ]}}} } e = parse_houses_state(state_minimal, "/catalog/houses/test/9999") assert e.house.ext_id == 9999 assert e.reviews == [] assert e.mini_serp == [] assert e.placement_history == [] assert e.recommendations == [] assert e.reviews_next_page_url is None def test_invalid_state_raises() -> None: """ValueError если placeholders не найден.""" with pytest.raises(ValueError, match="placeholders"): parse_houses_state({}, "/catalog/houses/test/1") def test_missing_house_page_raises() -> None: """ValueError если housePage виджет отсутствует.""" state: dict = { "data": {"data": {"page": {"placeholders": [ {"type": "breadcrumbs"}, ]}}} } with pytest.raises(ValueError, match="housePage"): parse_houses_state(state, "/catalog/houses/test/1") def test_house_type_normalization() -> None: """Все варианты типа дома нормализуются корректно.""" from app.services.scrapers.avito_houses import _normalize_house_type assert _normalize_house_type("Монолитный") == "monolith" assert _normalize_house_type("Панельный") == "panel" assert _normalize_house_type("Кирпичный") == "brick" assert _normalize_house_type("Монолитно-кирпичный") == "monolith_brick" assert _normalize_house_type("Блочный") == "block" assert _normalize_house_type("Деревянный") == "wood" assert _normalize_house_type("Неизвестный") == "other" assert _normalize_house_type(None) is None def test_strip_price() -> None: """_strip_price корректно извлекает числа из строк Avito.""" from app.services.scrapers.avito_houses import _strip_price assert _strip_price("11 990 000 ₽") == 11990000 assert _strip_price("5 000 000 ₽") == 5000000 assert _strip_price(None) is None assert _strip_price("") is None def test_mini_serp_no_seller() -> None: """Объявление без продавца не вызывает ошибку.""" state: dict = { "data": {"data": {"page": {"placeholders": [ { "type": "housePage", "props": {"developmentData": { "avitoId": 111, "coords": {"lat": 56.0, "lng": 60.0}, "aboutDevelopment": {"expandParams": {"items": []}}, "developer": {}, "mapPreview": {}, "ratingBadge": {"info": {}}, "ratingSummaryStat": {}, }}, }, { "type": "miniSerp", "props": {"items": [ {"id": 12345, "title": "1-к.", "url": "/test", "price": "3 000 000 ₽", "geo": {"content": "Центр", "colors": []}}, ]}, }, ]}}} } e = parse_houses_state(state, "/catalog/houses/test/111") assert len(e.mini_serp) == 1 assert e.mini_serp[0].seller is None assert e.mini_serp[0].price_rub == 3000000 # --------------------------------------------------------------------------- # Real-HTML fixture (regression for issue #566 — state-blob encoding/shape change) # --------------------------------------------------------------------------- # # Fixture: реальная страница /catalog/houses/ekaterinburg/ul_yumasheva_6/115188, # захвачена curl_cffi chrome120 (2026-05-28). Avito сменил формат __preloadedState__ # на URL-encoded литерал + обернул виджеты в placeholders[0]['widgets'] + # вложил developmentData под props['housePage']. Этот тест ловит регрессию декода/shape. def test_extract_preloaded_state_from_real_html() -> None: """_extract_preloaded_state декодирует текущий URL-encoded __preloadedState__.""" html = (FIXTURES_DIR / "avito_houses_sample.html").read_text(encoding="utf-8") state = _extract_preloaded_state(html) assert state is not None, "URL-encoded __preloadedState__ должен распарситься" # Avito SSR обёртка — успешный ответ assert state["data"]["status"] == 200 # Inner path до placeholders сохранился placeholders = state["data"]["data"]["page"]["placeholders"] assert isinstance(placeholders, list) and placeholders, "placeholders непустой массив" def test_parse_real_fixture_end_to_end() -> None: """Фикстура HTML → _extract_preloaded_state → parse_houses_state → HouseInfo.ext_id.""" html = (FIXTURES_DIR / "avito_houses_sample.html").read_text(encoding="utf-8") state = _extract_preloaded_state(html) assert state is not None e = parse_houses_state(state, "/catalog/houses/ekaterinburg/ul_yumasheva_6/115188") assert isinstance(e, HouseCatalogEnrichment) # housePage распарсился — ext_id (avitoId) присутствует и совпадает с URL assert e.house.ext_id == 115188 assert e.house.ext_id_hash # base64 internal id присутствует # expandParams распарсились (год постройки реальной фикстуры — 2016) assert e.house.year_built == 2016 assert e.house.total_floors == 17 assert e.house.house_type == "monolith" assert e.house.lat is not None and e.house.lon is not None def test_extract_preloaded_state_jsparse_fallback() -> None: """Старая форма JSON.parse("...") всё ещё декодируется (fallback-ветка).""" inner = '{\\"data\\":{\\"data\\":{\\"page\\":{\\"placeholders\\":[]}}}}' html = f'' state = _extract_preloaded_state(html) assert state is not None assert state["data"]["data"]["page"]["placeholders"] == [] def test_extract_preloaded_state_missing_returns_none() -> None: """Нет маркера → None (а не исключение).""" assert _extract_preloaded_state("no state here") is None