tradein-mvp/backend/app/services/scrapers/base.py

@@ -93,6 +93,21 @@ class ScrapedLot(BaseModel):
     photo_urls: list[str] = Field(default_factory=list)
     raw_payload: dict[str, Any] | None = None
 
+    # Cian-specific extensions (Stage 2 of CianScraper v1)
+    living_area_m2: float | None = None
+    bedrooms_count: int | None = None
+    balconies_count: int | None = None
+    loggias_count: int | None = None
+    description_minhash: str | None = None
+    cadastral_number: str | None = None
+    building_cadastral_number: str | None = None
+    phones: list[dict] = Field(default_factory=list)
+    is_homeowner: bool | None = None
+    is_pro_seller: bool | None = None
+    bargain_allowed: bool | None = None
+    sale_type: str | None = None
+    metro_stations: list[dict] = Field(default_factory=list)
+
     def compute_dedup_hash(self) -> str:
         """SHA256(source + (source_id или source_url) + price_rub) — стабильный uniqueness key.
 

tradein-mvp/backend/app/services/scrapers/cian_state_parser.py

@@ -0,0 +1,93 @@
+"""Shared utility для извлечения Cian Redux state из _cianConfig.
+
+Pattern: window._cianConfig['<mfe>'].push({key:..., value:..., priority:..., filter:...})
+Used by SERP / detail / newbuilding / valuation scrapers.
+"""
+import json
+import logging
+import re
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+# Match both patterns Cian uses:
+#   1. window._cianConfig['mfe'].push({key:..., value:..., priority:..., filter:...})
+#   2. window._cianConfig['mfe'] = window._cianConfig['mfe'] || [];
+#      window._cianConfig['mfe'].push({...})
+_RE_CIAN_PUSH = re.compile(
+    r"window\._cianConfig\['(?P<mfe>[\w-]+)'\]"
+    r"(?:\.push\(\s*|"
+    r"\s*=\s*window\._cianConfig\['[\w-]+'\]\s*\|\|\s*\[\];\s*"
+    r"window\._cianConfig\['[\w-]+'\]\.push\(\s*)"
+    r"\{\s*key:\s*['\"](?P<key>\w+)['\"]\s*,\s*"
+    r"value:\s*(?P<value>.+?)\s*,\s*"
+    r"priority:",
+    re.DOTALL,
+)
+
+
+def extract_state(html: str, mfe: str, key: str = "initialState") -> dict[str, Any] | None:
+    """Extract state.value where _cianConfig[<mfe>] entry.key matches.
+
+    Returns parsed JSON dict, or None if not found / parse failed.
+    Logs warning at debug level on parse failure for diagnostics.
+    """
+    for match in _RE_CIAN_PUSH.finditer(html):
+        if match.group("mfe") != mfe or match.group("key") != key:
+            continue
+
+        value_str = match.group("value").strip()
+
+        # Strategy 1: direct JSON parse (value уже JSON-like)
+        try:
+            return json.loads(value_str)
+        except json.JSONDecodeError:
+            pass
+
+        # Strategy 2: JSON.parse("escaped string")
+        m_jp = re.match(r'JSON\.parse\(\s*"(.+?)"\s*\)\s*$', value_str, re.DOTALL)
+        if m_jp:
+            escaped = m_jp.group(1)
+            unescaped = (
+                escaped
+                .replace('\\"', '"')
+                .replace('\\\\', '\\')
+                .replace('\\n', '\n')
+                .replace('\\/', '/')
+            )
+            try:
+                return json.loads(unescaped)
+            except json.JSONDecodeError as exc:
+                logger.debug(
+                    "JSON.parse() unescape failed for mfe=%s key=%s: %s", mfe, key, exc
+                )
+                continue
+
+        # Strategy 3: raw JS object literal — try demjson3 (optional dep)
+        try:
+            import demjson3  # type: ignore[import-untyped]
+
+            return demjson3.decode(value_str)
+        except ImportError:
+            logger.debug(
+                "demjson3 not installed; cannot parse raw JS object for mfe=%s key=%s", mfe, key
+            )
+        except Exception as exc:
+            logger.debug("demjson3 parse failed for mfe=%s key=%s: %s", mfe, key, exc)
+
+    return None
+
+
+def extract_all_states(html: str) -> dict[str, dict[str, Any]]:
+    """Returns {mfe_name: {key: parsed_value}} for all push() entries.
+
+    Useful for discovery / debugging during scraper development.
+    """
+    result: dict[str, dict[str, Any]] = {}
+    for match in _RE_CIAN_PUSH.finditer(html):
+        mfe = match.group("mfe")
+        key = match.group("key")
+        parsed = extract_state(html, mfe, key)
+        if parsed is not None:
+            result.setdefault(mfe, {})[key] = parsed
+    return result

tradein-mvp/backend/tests/test_avito_houses_parse.py

@@ -0,0 +1,397 @@
+"""Offline smoke for avito_houses parser.
+
+Uses minimal JSON fixture mimicking Avito __preloadedState__ structure.
+Не требует сети, БД или curl_cffi — тестирует только чистый парсер.
+"""
+
+from datetime import date
+
+import pytest
+
+from app.services.scrapers.avito_houses import (
+    HouseCatalogEnrichment,
+    HouseInfo,
+    HouseReview,
+    MiniSerpListing,
+    PlacementHistoryItem,
+    RecommendationItem,
+    SellerInfo,
+    _parse_ru_date,
+    parse_houses_state,
+)
+
+
+# ---------------------------------------------------------------------------
+# Minimal __preloadedState__ fixture (Python dict — не raw HTML)
+# ---------------------------------------------------------------------------
+
+MINIMAL_STATE: dict = {
+    "data": {"data": {"page": {"placeholders": [
+        {"type": "breadcrumbs"},
+        {"type": "title"},
+        {"type": "gallery", "props": {}},
+        {
+            "type": "housePage",
+            "props": {"developmentData": {
+                "avitoId": 3171365,
+                "id": "MTE3LjM3MDg5MA",
+                "title": "Akademika Postovskogo 17a",
+                "address": "ул. Постовского, 17а",
+                "fullAddress": "Свердловская обл., Екатеринбург, ул. Постовского, 17а",
+                "coords": {"lat": 56.790699, "lng": 60.580191},
+                "aboutDevelopment": {"expandParams": {"items": [
+                    {"title": "Параметры дома", "params": [
+                        {"type": "Год постройки", "value": "2019"},
+                        {"type": "Этажей", "value": "25"},
+                        {"type": "Тип дома", "value": "Монолитный"},
+                        {"type": "Пассажирский лифт", "value": "2"},
+                        {"type": "Консьерж", "value": "Да"},
+                        {"type": "Закрытый двор", "value": "Да"},
+                        {"type": "Детская площадка", "value": "Да"},
+                    ]},
+                ]}},
+                "developer": {"name": "Атомстройкомплекс", "key": "atomstroy"},
+                "mapPreview": {
+                    "distance": "в 5 минутах ходьбы",
+                    "objects": "школа, парк, аптека",
+                    "pins": [{"lat": 56.79, "lng": 60.58, "type": "school"}],
+                },
+                "ratingBadge": {"info": {"score": 4.666, "scoreString": "4,7"}},
+                "ratingSummaryStat": {
+                    "ratingStat": [{"score": 5, "count": 4}, {"score": 4, "count": 2}],
+                    "reviewCount": 6,
+                },
+            }},
+        },
+        {
+            "type": "reviews",
+            "props": {"entries": [
+                {"type": "rating", "value": {
+                    "id": 12345,
+                    "author": {"title": "Иван"},
+                    "reviewTitle": "Хорошо",
+                    "score": 5,
+                    "modelExperience": "Живу в своей квартире",
+                    "rated": "9 октября 2025",
+                    "textSections": [
+                        {"title": "", "text": "Общий отзыв"},
+                        {"title": "Преимущества", "text": "Парк рядом"},
+                        {"title": "Недостатки", "text": "Парковки мало"},
+                    ],
+                }},
+                {"type": "pages", "value": {"nextPageUrl": "/reviews?page=2"}},
+            ]},
+        },
+        {
+            "type": "miniSerp",
+            "props": {"items": [
+                {
+                    "id": 7986882804,
+                    "title": "3-к. квартира",
+                    "description": "Описание",
+                    "url": "/ekaterinburg/kvartiry/test_7986882804",
+                    "price": "11 990 000 ₽",
+                    "geo": {"content": "Чкаловская, 2,7 км", "colors": ["#cf2734"]},
+                    "seller": {
+                        "name": "DOMRF66",
+                        "type": "Компания",
+                        "from": "На Авито с июля 2012",
+                        "url": "/brands/domrf66",
+                        "logos": [],
+                    },
+                },
+            ]},
+        },
+        {
+            "type": "housePlacementHistory",
+            "props": {"items": [
+                {
+                    "id": 7000000001,
+                    "title": "1-к. квартира",
+                    "startPrice": 5000000,
+                    "startPriceDate": 1697000000,
+                    "lastPrice": 4900000,
+                    "lastPriceDate": 1715000000,
+                    "exposure": 210,
+                    "itemImage": {},
+                },
+            ]},
+        },
+        {"type": "uxFeedback"},
+        {
+            "type": "recommendations",
+            "props": {"items": [
+                {
+                    "title": "Похожая",
+                    "priceRange": "от 6 млн",
+                    "additionalInfo": ["Адрес 1"],
+                    "url": "/rec1",
+                    "images": [{"208x156": "https://avito.ru/img1.jpg"}],
+                },
+            ]},
+        },
+        {"type": "padding"},
+    ]}}}
+}
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+
+def test_ru_date_parse() -> None:
+    assert _parse_ru_date("9 октября 2025") == date(2025, 10, 9)
+    assert _parse_ru_date("31 декабря 2020") == date(2020, 12, 31)
+    assert _parse_ru_date("1 января 2000") == date(2000, 1, 1)
+    assert _parse_ru_date(None) is None
+    assert _parse_ru_date("bogus") is None
+    assert _parse_ru_date("") is None
+
+
+# ---------------------------------------------------------------------------
+# Main parse test
+# ---------------------------------------------------------------------------
+
+
+def test_parse_minimal_state() -> None:
+    e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
+
+    assert isinstance(e, HouseCatalogEnrichment)
+    assert e.house_url == "/catalog/houses/ekaterinburg/test/3171365"
+
+
+def test_house_info() -> None:
+    e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
+    h = e.house
+
+    assert isinstance(h, HouseInfo)
+    assert h.ext_id == 3171365
+    assert h.ext_id_hash == "MTE3LjM3MDg5MA"
+    assert h.title == "Akademika Postovskogo 17a"
+    assert h.short_address == "ул. Постовского, 17а"
+    assert h.full_address == "Свердловская обл., Екатеринбург, ул. Постовского, 17а"
+    assert h.lat == 56.790699
+    assert h.lon == 60.580191
+
+
+def test_house_expand_params() -> None:
+    e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
+    h = e.house
+
+    assert h.year_built == 2019
+    assert h.total_floors == 25
+    assert h.house_type == "monolith"
+    assert h.passenger_elevators == 2
+    assert h.has_concierge is True
+    assert h.closed_yard is True
+    assert h.has_playground is True
+
+
+def test_house_developer_and_rating() -> None:
+    e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
+    h = e.house
+
+    assert h.developer_name == "Атомстройкомплекс"
+    assert h.developer_key == "atomstroy"
+    assert h.rating_score == 4.666
+    assert h.rating_string == "4,7"
+    assert h.reviews_count == 6
+    assert len(h.rating_distribution) == 2
+
+
+def test_house_map_and_infra() -> None:
+    e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
+    h = e.house
+
+    assert h.infrastructure_walk_distance == "в 5 минутах ходьбы"
+    assert h.infrastructure_summary == "школа, парк, аптека"
+    assert len(h.map_pins) == 1
+    assert h.map_pins[0]["type"] == "school"
+
+
+def test_house_raw_characteristics() -> None:
+    e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
+    # raw_characteristics = полный expandParams items (1 категория)
+    assert len(e.house.raw_characteristics) == 1
+    assert e.house.raw_characteristics[0]["title"] == "Параметры дома"
+
+
+def test_reviews() -> None:
+    e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
+
+    assert len(e.reviews) == 1
+    assert e.reviews_next_page_url == "/reviews?page=2"
+
+    r = e.reviews[0]
+    assert isinstance(r, HouseReview)
+    assert r.ext_review_id == 12345
+    assert r.author_name == "Иван"
+    assert r.review_title == "Хорошо"
+    assert r.score == 5
+    assert r.model_experience == "Живу в своей квартире"
+    assert r.rated_date == date(2025, 10, 9)
+    assert r.text_main == "Общий отзыв"
+    assert r.text_pros == "Парк рядом"
+    assert r.text_cons == "Парковки мало"
+    assert r.raw_payload is not None
+
+
+def test_mini_serp() -> None:
+    e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
+
+    assert len(e.mini_serp) == 1
+    m = e.mini_serp[0]
+    assert isinstance(m, MiniSerpListing)
+    assert m.ext_item_id == 7986882804
+    assert m.price_rub == 11990000
+    assert m.metro_text == "Чкаловская, 2,7 км"
+    assert m.metro_color == "#cf2734"
+    assert m.description == "Описание"
+
+
+def test_seller_extraction() -> None:
+    e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
+    m = e.mini_serp[0]
+
+    assert m.seller is not None
+    s = m.seller
+    assert isinstance(s, SellerInfo)
+    assert s.ext_seller_id == "domrf66"
+    assert s.name == "DOMRF66"
+    assert s.seller_type == "Компания"
+    assert s.on_platform_since == "На Авито с июля 2012"
+    assert s.profile_url == "/brands/domrf66"
+
+
+def test_placement_history() -> None:
+    e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
+
+    assert len(e.placement_history) == 1
+    p = e.placement_history[0]
+    assert isinstance(p, PlacementHistoryItem)
+    assert p.ext_item_id == "7000000001"
+    assert p.title == "1-к. квартира"
+    assert p.start_price == 5000000
+    assert p.last_price == 4900000
+    assert p.exposure_days == 210
+    # removed_date ВСЕГДА None — виджет не отдаёт!
+    assert p.removed_date is None
+    # raw_payload не содержит itemImage (скипается)
+    assert p.raw_payload is not None
+    assert "itemImage" not in p.raw_payload
+    # start_price_date и last_price_date — из unix timestamp
+    assert isinstance(p.start_price_date, date)
+    assert isinstance(p.last_price_date, date)
+
+
+def test_recommendations() -> None:
+    e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
+
+    assert len(e.recommendations) == 1
+    rec = e.recommendations[0]
+    assert isinstance(rec, RecommendationItem)
+    assert rec.title == "Похожая"
+    assert rec.price_text == "от 6 млн"
+    assert rec.address == "Адрес 1"
+    assert rec.url == "/rec1"
+    assert rec.image_url == "https://avito.ru/img1.jpg"
+
+
+def test_missing_optional_widgets() -> None:
+    """Парсер не падает если опциональные виджеты отсутствуют."""
+    state_minimal: dict = {
+        "data": {"data": {"page": {"placeholders": [
+            {
+                "type": "housePage",
+                "props": {"developmentData": {
+                    "avitoId": 9999,
+                    "coords": {"lat": 56.0, "lng": 60.0},
+                    "aboutDevelopment": {"expandParams": {"items": []}},
+                    "developer": {},
+                    "mapPreview": {},
+                    "ratingBadge": {"info": {}},
+                    "ratingSummaryStat": {},
+                }},
+            },
+        ]}}}
+    }
+    e = parse_houses_state(state_minimal, "/catalog/houses/test/9999")
+    assert e.house.ext_id == 9999
+    assert e.reviews == []
+    assert e.mini_serp == []
+    assert e.placement_history == []
+    assert e.recommendations == []
+    assert e.reviews_next_page_url is None
+
+
+def test_invalid_state_raises() -> None:
+    """ValueError если placeholders не найден."""
+    with pytest.raises(ValueError, match="placeholders"):
+        parse_houses_state({}, "/catalog/houses/test/1")
+
+
+def test_missing_house_page_raises() -> None:
+    """ValueError если housePage виджет отсутствует."""
+    state: dict = {
+        "data": {"data": {"page": {"placeholders": [
+            {"type": "breadcrumbs"},
+        ]}}}
+    }
+    with pytest.raises(ValueError, match="housePage"):
+        parse_houses_state(state, "/catalog/houses/test/1")
+
+
+def test_house_type_normalization() -> None:
+    """Все варианты типа дома нормализуются корректно."""
+    from app.services.scrapers.avito_houses import _normalize_house_type
+
+    assert _normalize_house_type("Монолитный") == "monolith"
+    assert _normalize_house_type("Панельный") == "panel"
+    assert _normalize_house_type("Кирпичный") == "brick"
+    assert _normalize_house_type("Монолитно-кирпичный") == "monolith_brick"
+    assert _normalize_house_type("Блочный") == "block"
+    assert _normalize_house_type("Деревянный") == "wood"
+    assert _normalize_house_type("Неизвестный") == "other"
+    assert _normalize_house_type(None) is None
+
+
+def test_strip_price() -> None:
+    """_strip_price корректно извлекает числа из строк Avito."""
+    from app.services.scrapers.avito_houses import _strip_price
+
+    assert _strip_price("11 990 000 ₽") == 11990000
+    assert _strip_price("5 000 000 ₽") == 5000000
+    assert _strip_price(None) is None
+    assert _strip_price("") is None
+
+
+def test_mini_serp_no_seller() -> None:
+    """Объявление без продавца не вызывает ошибку."""
+    state: dict = {
+        "data": {"data": {"page": {"placeholders": [
+            {
+                "type": "housePage",
+                "props": {"developmentData": {
+                    "avitoId": 111,
+                    "coords": {"lat": 56.0, "lng": 60.0},
+                    "aboutDevelopment": {"expandParams": {"items": []}},
+                    "developer": {},
+                    "mapPreview": {},
+                    "ratingBadge": {"info": {}},
+                    "ratingSummaryStat": {},
+                }},
+            },
+            {
+                "type": "miniSerp",
+                "props": {"items": [
+                    {"id": 12345, "title": "1-к.", "url": "/test", "price": "3 000 000 ₽",
+                     "geo": {"content": "Центр", "colors": []}},
+                ]},
+            },
+        ]}}}
+    }
+    e = parse_houses_state(state, "/catalog/houses/test/111")
+    assert len(e.mini_serp) == 1
+    assert e.mini_serp[0].seller is None
+    assert e.mini_serp[0].price_rub == 3000000

tradein-mvp/backend/tests/test_cian_state_parser.py

@@ -0,0 +1,46 @@
+"""Tests for cian_state_parser."""
+import pytest
+from app.services.scrapers.cian_state_parser import extract_state
+
+
+def test_extract_state_simple_json():
+    html = '''
+    <html><body>
+    <script>
+    window._cianConfig['frontend-serp'] = window._cianConfig['frontend-serp'] || [];
+    window._cianConfig['frontend-serp'].push({
+        key: 'initialState',
+        value: {"results": {"offers": [{"id": 1}]}},
+        priority: 1
+    });
+    </script>
+    </body></html>
+    '''
+    state = extract_state(html, mfe='frontend-serp', key='initialState')
+    assert state is not None
+    assert state['results']['offers'][0]['id'] == 1
+
+
+def test_extract_state_json_parse_form():
+    html = r'''
+    window._cianConfig['frontend-offer-card'].push({
+        key: 'defaultState',
+        value: JSON.parse("{\"offerData\":{\"offer\":{\"cianId\":123}}}"),
+        priority: 1
+    });
+    '''
+    state = extract_state(html, mfe='frontend-offer-card', key='defaultState')
+    assert state is not None
+    assert state['offerData']['offer']['cianId'] == 123
+
+
+def test_extract_state_missing_returns_none():
+    html = "<html></html>"
+    assert extract_state(html, mfe='frontend-serp') is None
+
+
+def test_extract_state_wrong_mfe_returns_none():
+    html = '''
+    window._cianConfig['other-mfe'].push({key: 'initialState', value: {}, priority: 1});
+    '''
+    assert extract_state(html, mfe='frontend-serp', key='initialState') is None