feat(tradein): avito_houses.py — Houses Catalog parser (state + reviews + history + recs)

Stage 2c of AvitoScraper_v2.

- fetch_house_catalog(house_url) — async HTTP via curl_cffi (chrome120)
- parse_houses_state(state, house_url) — pure parser (testable без сети)
- 10-widget JSON state structure (housePage, reviews, miniSerp, housePlacementHistory, recommendations)
- Dataclasses:
  * HouseInfo (28 fields: identity, geo, expandParams flatten, developer, rating, map_pins, raw_characteristics)
  * HouseReview (textSections split: main/pros/cons; ru date parse)
  * PlacementHistoryItem (removed_date ALWAYS None — widget не отдаёт; точная дата только в IMV API)
  * SellerInfo (ext_seller_id из URL last segment, без logos)
  * MiniSerpListing (active listings в доме + metro + seller)
  * RecommendationItem (lightweight: title/price_text/address/url/image)
  * HouseCatalogEnrichment (orchestrator dataclass)
- Save flow (5 ops):
  * _upsert_house_with_url → houses (UNIQUE source='avito', ext_house_id) + raw_characteristics jsonb
  * save_house_reviews → house_reviews (UNIQUE source, ext_review_id), без images
  * save_sellers → sellers (UNIQUE source, ext_seller_id), без logos
  * link_listings_to_seller → UPDATE listings.seller_id_fk by source_id match
  * save_placement_history → source='avito_widget', removed_date=NULL
- pytest offline smoke: 18 tests, 5 widget types, 25+ field assertions
- Recommendations save skipped — Stage 3 решит

tradein-mvp/backend/tests/test_avito_houses_parse.py

@@ -0,0 +1,397 @@
+"""Offline smoke for avito_houses parser.
+
+Uses minimal JSON fixture mimicking Avito __preloadedState__ structure.
+Не требует сети, БД или curl_cffi — тестирует только чистый парсер.
+"""
+
+from datetime import date
+
+import pytest
+
+from app.services.scrapers.avito_houses import (
+    HouseCatalogEnrichment,
+    HouseInfo,
+    HouseReview,
+    MiniSerpListing,
+    PlacementHistoryItem,
+    RecommendationItem,
+    SellerInfo,
+    _parse_ru_date,
+    parse_houses_state,
+)
+
+
+# ---------------------------------------------------------------------------
+# Minimal __preloadedState__ fixture (Python dict — не raw HTML)
+# ---------------------------------------------------------------------------
+
+MINIMAL_STATE: dict = {
+    "data": {"data": {"page": {"placeholders": [
+        {"type": "breadcrumbs"},
+        {"type": "title"},
+        {"type": "gallery", "props": {}},
+        {
+            "type": "housePage",
+            "props": {"developmentData": {
+                "avitoId": 3171365,
+                "id": "MTE3LjM3MDg5MA",
+                "title": "Akademika Postovskogo 17a",
+                "address": "ул. Постовского, 17а",
+                "fullAddress": "Свердловская обл., Екатеринбург, ул. Постовского, 17а",
+                "coords": {"lat": 56.790699, "lng": 60.580191},
+                "aboutDevelopment": {"expandParams": {"items": [
+                    {"title": "Параметры дома", "params": [
+                        {"type": "Год постройки", "value": "2019"},
+                        {"type": "Этажей", "value": "25"},
+                        {"type": "Тип дома", "value": "Монолитный"},
+                        {"type": "Пассажирский лифт", "value": "2"},
+                        {"type": "Консьерж", "value": "Да"},
+                        {"type": "Закрытый двор", "value": "Да"},
+                        {"type": "Детская площадка", "value": "Да"},
+                    ]},
+                ]}},
+                "developer": {"name": "Атомстройкомплекс", "key": "atomstroy"},
+                "mapPreview": {
+                    "distance": "в 5 минутах ходьбы",
+                    "objects": "школа, парк, аптека",
+                    "pins": [{"lat": 56.79, "lng": 60.58, "type": "school"}],
+                },
+                "ratingBadge": {"info": {"score": 4.666, "scoreString": "4,7"}},
+                "ratingSummaryStat": {
+                    "ratingStat": [{"score": 5, "count": 4}, {"score": 4, "count": 2}],
+                    "reviewCount": 6,
+                },
+            }},
+        },
+        {
+            "type": "reviews",
+            "props": {"entries": [
+                {"type": "rating", "value": {
+                    "id": 12345,
+                    "author": {"title": "Иван"},
+                    "reviewTitle": "Хорошо",
+                    "score": 5,
+                    "modelExperience": "Живу в своей квартире",
+                    "rated": "9 октября 2025",
+                    "textSections": [
+                        {"title": "", "text": "Общий отзыв"},
+                        {"title": "Преимущества", "text": "Парк рядом"},
+                        {"title": "Недостатки", "text": "Парковки мало"},
+                    ],
+                }},
+                {"type": "pages", "value": {"nextPageUrl": "/reviews?page=2"}},
+            ]},
+        },
+        {
+            "type": "miniSerp",
+            "props": {"items": [
+                {
+                    "id": 7986882804,
+                    "title": "3-к. квартира",
+                    "description": "Описание",
+                    "url": "/ekaterinburg/kvartiry/test_7986882804",
+                    "price": "11 990 000 ₽",
+                    "geo": {"content": "Чкаловская, 2,7 км", "colors": ["#cf2734"]},
+                    "seller": {
+                        "name": "DOMRF66",
+                        "type": "Компания",
+                        "from": "На Авито с июля 2012",
+                        "url": "/brands/domrf66",
+                        "logos": [],
+                    },
+                },
+            ]},
+        },
+        {
+            "type": "housePlacementHistory",
+            "props": {"items": [
+                {
+                    "id": 7000000001,
+                    "title": "1-к. квартира",
+                    "startPrice": 5000000,
+                    "startPriceDate": 1697000000,
+                    "lastPrice": 4900000,
+                    "lastPriceDate": 1715000000,
+                    "exposure": 210,
+                    "itemImage": {},
+                },
+            ]},
+        },
+        {"type": "uxFeedback"},
+        {
+            "type": "recommendations",
+            "props": {"items": [
+                {
+                    "title": "Похожая",
+                    "priceRange": "от 6 млн",
+                    "additionalInfo": ["Адрес 1"],
+                    "url": "/rec1",
+                    "images": [{"208x156": "https://avito.ru/img1.jpg"}],
+                },
+            ]},
+        },
+        {"type": "padding"},
+    ]}}}
+}
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+
+def test_ru_date_parse() -> None:
+    assert _parse_ru_date("9 октября 2025") == date(2025, 10, 9)
+    assert _parse_ru_date("31 декабря 2020") == date(2020, 12, 31)
+    assert _parse_ru_date("1 января 2000") == date(2000, 1, 1)
+    assert _parse_ru_date(None) is None
+    assert _parse_ru_date("bogus") is None
+    assert _parse_ru_date("") is None
+
+
+# ---------------------------------------------------------------------------
+# Main parse test
+# ---------------------------------------------------------------------------
+
+
+def test_parse_minimal_state() -> None:
+    e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
+
+    assert isinstance(e, HouseCatalogEnrichment)
+    assert e.house_url == "/catalog/houses/ekaterinburg/test/3171365"
+
+
+def test_house_info() -> None:
+    e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
+    h = e.house
+
+    assert isinstance(h, HouseInfo)
+    assert h.ext_id == 3171365
+    assert h.ext_id_hash == "MTE3LjM3MDg5MA"
+    assert h.title == "Akademika Postovskogo 17a"
+    assert h.short_address == "ул. Постовского, 17а"
+    assert h.full_address == "Свердловская обл., Екатеринбург, ул. Постовского, 17а"
+    assert h.lat == 56.790699
+    assert h.lon == 60.580191
+
+
+def test_house_expand_params() -> None:
+    e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
+    h = e.house
+
+    assert h.year_built == 2019
+    assert h.total_floors == 25
+    assert h.house_type == "monolith"
+    assert h.passenger_elevators == 2
+    assert h.has_concierge is True
+    assert h.closed_yard is True
+    assert h.has_playground is True
+
+
+def test_house_developer_and_rating() -> None:
+    e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
+    h = e.house
+
+    assert h.developer_name == "Атомстройкомплекс"
+    assert h.developer_key == "atomstroy"
+    assert h.rating_score == 4.666
+    assert h.rating_string == "4,7"
+    assert h.reviews_count == 6
+    assert len(h.rating_distribution) == 2
+
+
+def test_house_map_and_infra() -> None:
+    e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
+    h = e.house
+
+    assert h.infrastructure_walk_distance == "в 5 минутах ходьбы"
+    assert h.infrastructure_summary == "школа, парк, аптека"
+    assert len(h.map_pins) == 1
+    assert h.map_pins[0]["type"] == "school"
+
+
+def test_house_raw_characteristics() -> None:
+    e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
+    # raw_characteristics = полный expandParams items (1 категория)
+    assert len(e.house.raw_characteristics) == 1
+    assert e.house.raw_characteristics[0]["title"] == "Параметры дома"
+
+
+def test_reviews() -> None:
+    e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
+
+    assert len(e.reviews) == 1
+    assert e.reviews_next_page_url == "/reviews?page=2"
+
+    r = e.reviews[0]
+    assert isinstance(r, HouseReview)
+    assert r.ext_review_id == 12345
+    assert r.author_name == "Иван"
+    assert r.review_title == "Хорошо"
+    assert r.score == 5
+    assert r.model_experience == "Живу в своей квартире"
+    assert r.rated_date == date(2025, 10, 9)
+    assert r.text_main == "Общий отзыв"
+    assert r.text_pros == "Парк рядом"
+    assert r.text_cons == "Парковки мало"
+    assert r.raw_payload is not None
+
+
+def test_mini_serp() -> None:
+    e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
+
+    assert len(e.mini_serp) == 1
+    m = e.mini_serp[0]
+    assert isinstance(m, MiniSerpListing)
+    assert m.ext_item_id == 7986882804
+    assert m.price_rub == 11990000
+    assert m.metro_text == "Чкаловская, 2,7 км"
+    assert m.metro_color == "#cf2734"
+    assert m.description == "Описание"
+
+
+def test_seller_extraction() -> None:
+    e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
+    m = e.mini_serp[0]
+
+    assert m.seller is not None
+    s = m.seller
+    assert isinstance(s, SellerInfo)
+    assert s.ext_seller_id == "domrf66"
+    assert s.name == "DOMRF66"
+    assert s.seller_type == "Компания"
+    assert s.on_platform_since == "На Авито с июля 2012"
+    assert s.profile_url == "/brands/domrf66"
+
+
+def test_placement_history() -> None:
+    e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
+
+    assert len(e.placement_history) == 1
+    p = e.placement_history[0]
+    assert isinstance(p, PlacementHistoryItem)
+    assert p.ext_item_id == "7000000001"
+    assert p.title == "1-к. квартира"
+    assert p.start_price == 5000000
+    assert p.last_price == 4900000
+    assert p.exposure_days == 210
+    # removed_date ВСЕГДА None — виджет не отдаёт!
+    assert p.removed_date is None
+    # raw_payload не содержит itemImage (скипается)
+    assert p.raw_payload is not None
+    assert "itemImage" not in p.raw_payload
+    # start_price_date и last_price_date — из unix timestamp
+    assert isinstance(p.start_price_date, date)
+    assert isinstance(p.last_price_date, date)
+
+
+def test_recommendations() -> None:
+    e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
+
+    assert len(e.recommendations) == 1
+    rec = e.recommendations[0]
+    assert isinstance(rec, RecommendationItem)
+    assert rec.title == "Похожая"
+    assert rec.price_text == "от 6 млн"
+    assert rec.address == "Адрес 1"
+    assert rec.url == "/rec1"
+    assert rec.image_url == "https://avito.ru/img1.jpg"
+
+
+def test_missing_optional_widgets() -> None:
+    """Парсер не падает если опциональные виджеты отсутствуют."""
+    state_minimal: dict = {
+        "data": {"data": {"page": {"placeholders": [
+            {
+                "type": "housePage",
+                "props": {"developmentData": {
+                    "avitoId": 9999,
+                    "coords": {"lat": 56.0, "lng": 60.0},
+                    "aboutDevelopment": {"expandParams": {"items": []}},
+                    "developer": {},
+                    "mapPreview": {},
+                    "ratingBadge": {"info": {}},
+                    "ratingSummaryStat": {},
+                }},
+            },
+        ]}}}
+    }
+    e = parse_houses_state(state_minimal, "/catalog/houses/test/9999")
+    assert e.house.ext_id == 9999
+    assert e.reviews == []
+    assert e.mini_serp == []
+    assert e.placement_history == []
+    assert e.recommendations == []
+    assert e.reviews_next_page_url is None
+
+
+def test_invalid_state_raises() -> None:
+    """ValueError если placeholders не найден."""
+    with pytest.raises(ValueError, match="placeholders"):
+        parse_houses_state({}, "/catalog/houses/test/1")
+
+
+def test_missing_house_page_raises() -> None:
+    """ValueError если housePage виджет отсутствует."""
+    state: dict = {
+        "data": {"data": {"page": {"placeholders": [
+            {"type": "breadcrumbs"},
+        ]}}}
+    }
+    with pytest.raises(ValueError, match="housePage"):
+        parse_houses_state(state, "/catalog/houses/test/1")
+
+
+def test_house_type_normalization() -> None:
+    """Все варианты типа дома нормализуются корректно."""
+    from app.services.scrapers.avito_houses import _normalize_house_type
+
+    assert _normalize_house_type("Монолитный") == "monolith"
+    assert _normalize_house_type("Панельный") == "panel"
+    assert _normalize_house_type("Кирпичный") == "brick"
+    assert _normalize_house_type("Монолитно-кирпичный") == "monolith_brick"
+    assert _normalize_house_type("Блочный") == "block"
+    assert _normalize_house_type("Деревянный") == "wood"
+    assert _normalize_house_type("Неизвестный") == "other"
+    assert _normalize_house_type(None) is None
+
+
+def test_strip_price() -> None:
+    """_strip_price корректно извлекает числа из строк Avito."""
+    from app.services.scrapers.avito_houses import _strip_price
+
+    assert _strip_price("11 990 000 ₽") == 11990000
+    assert _strip_price("5 000 000 ₽") == 5000000
+    assert _strip_price(None) is None
+    assert _strip_price("") is None
+
+
+def test_mini_serp_no_seller() -> None:
+    """Объявление без продавца не вызывает ошибку."""
+    state: dict = {
+        "data": {"data": {"page": {"placeholders": [
+            {
+                "type": "housePage",
+                "props": {"developmentData": {
+                    "avitoId": 111,
+                    "coords": {"lat": 56.0, "lng": 60.0},
+                    "aboutDevelopment": {"expandParams": {"items": []}},
+                    "developer": {},
+                    "mapPreview": {},
+                    "ratingBadge": {"info": {}},
+                    "ratingSummaryStat": {},
+                }},
+            },
+            {
+                "type": "miniSerp",
+                "props": {"items": [
+                    {"id": 12345, "title": "1-к.", "url": "/test", "price": "3 000 000 ₽",
+                     "geo": {"content": "Центр", "colors": []}},
+                ]},
+            },
+        ]}}}
+    }
+    e = parse_houses_state(state, "/catalog/houses/test/111")
+    assert len(e.mini_serp) == 1
+    assert e.mini_serp[0].seller is None
+    assert e.mini_serp[0].price_rub == 3000000