tradein-mvp/backend/app/services/matching/conflict_resolution.py

@@ -1,62 +1,196 @@
-"""Field priority rules for canonical record merging.
+"""Per-field source priority for cross-source canonical merge.
 
-When multiple sources provide conflicting values for the same field,
-these dicts define which source to prefer (highest-priority first).
+Direct port of Cross_Source_Matching_Strategy.md sec 3.5 (houses) + 4.5 (listings).
+Source-of-truth dicts read by merge logic in match_or_create_house/listing.
 
-Algorithm reference: decisions/Cross_Source_Matching_Strategy.md sec 4.5
+Sources covered: avito (serp/detail/houses_catalog/domoteka/imv),
+cian (serp/bti/detail/stats/valuation), yandex (serp/detail/realty_nb/valuation).
 """
-
 from __future__ import annotations
 
-from sqlalchemy.orm import Session
+from typing import Any
 
-# Higher-priority source listed first.
-# For each column, prefer the first source in the list that has a non-NULL value.
+# ---------------------------------------------------------------------------
+# HOUSE_FIELD_PRIORITY — vault sec 3.5
+# ---------------------------------------------------------------------------
+HOUSE_FIELD_PRIORITY: dict[str, list[str] | str] = {
+    "address":         ["cian", "avito", "yandex"],
+    "lat":             ["cian_serp", "avito_houses_catalog", "yandex_realty_nb"],
+    "lon":             ["cian_serp", "avito_houses_catalog", "yandex_realty_nb"],
+    "year_built":      [
+        "cian_bti", "cian_serp", "avito_houses_catalog", "yandex_valuation", "yandex_realty_nb",
+    ],
+    "house_type":      ["cian_bti", "cian_serp", "avito", "yandex_valuation", "yandex_realty_nb"],
+    "series_name":     ["cian_bti"],
+    "passenger_lifts_count": ["cian", "avito"],
+    "cargo_lifts_count":     ["cian", "avito"],
+    "has_concierge":   ["cian", "avito"],
+    "closed_yard":     ["cian", "avito"],
+    "parking_type":    ["cian"],
+    "flat_count":      ["cian_bti"],
+    "entrances":       ["cian_bti"],
+    "is_emergency":    ["cian_bti"],
+    "management_company_id": ["cian_valuation"],
+    "house_class":     ["avito_houses_catalog", "cian", "yandex_realty_nb"],
+    "rating_score":    ["avito_houses_catalog", "cian", "yandex_realty_nb"],
+    "reviews_count":   ["avito_houses_catalog", "cian", "yandex_realty_nb"],
 
-HOUSE_FIELD_PRIORITY: dict[str, list[str]] = {
-    'year_built':     ['cian', 'avito_houses', 'rosreestr'],
-    'address':        ['avito', 'cian', 'rosreestr'],
-    'cadastral_number': ['rosreestr', 'cian', 'avito'],
-    'building_class': ['cian', 'avito_houses'],
-    'floors_count':   ['cian', 'avito_houses'],
-    'series_name':    ['cian'],
-    'entrances':      ['cian'],
-    'flat_count':     ['cian'],
+    # Yandex unique fields (newbuilding landing only)
+    "text_reviews_count":    ["yandex_realty_nb"],          # 353 text reviews — Yandex strongpoint
+    "corpus_count":          ["yandex_realty_nb"],          # "три башни" → 3
+    "total_area_ha":         ["yandex_realty_nb"],          # ЖК footprint
+    "commission_year":       ["cian_serp", "yandex_realty_nb"],
+    "commission_month":      ["yandex_realty_nb"],          # raw RU month name
+    "developer_name":        ["cian", "yandex_realty_nb"],
+    "has_panorama":          ["yandex_valuation"],          # Yandex 3D panorama flag
+    "yandex_total_listings": ["yandex_valuation"],          # "N объектов" в истории
+
+    # Yandex Valuation enrichment (existing house attrs)
+    "has_lift":              ["cian_bti", "cian_detail", "yandex_valuation"],
+    "ceiling_height":        ["cian_detail", "yandex_valuation"],
 }
 
-LISTING_FIELD_PRIORITY: dict[str, list[str]] = {
-    # Price: always from latest-seen source wins among equal-priority sources.
-    'price_rub':      ['avito', 'cian', 'yandex'],
-    'area_m2':        ['rosreestr', 'cian', 'avito'],
-    'living_area_m2': ['cian', 'avito'],
-    'kitchen_area_m2': ['cian', 'avito'],
-    'floor':          ['rosreestr', 'cian', 'avito'],
-    'rooms':          ['rosreestr', 'cian', 'avito'],
-    'ceiling_height': ['cian', 'avito'],
-    'description':    ['avito', 'cian'],
-    'phones':         ['avito', 'cian'],
-    'photos':         ['avito', 'cian'],
-    'cadastral_number': ['rosreestr', 'cian', 'avito'],
+# ---------------------------------------------------------------------------
+# LISTING_FIELD_PRIORITY — vault sec 4.5
+# ---------------------------------------------------------------------------
+LISTING_FIELD_PRIORITY: dict[str, list[str] | str] = {
+    "address":           ["cian_serp", "avito_detail", "avito_serp"],
+    "lat":               ["cian_serp", "avito_detail"],
+    "lon":               ["cian_serp", "avito_detail"],
+    "area_m2":           ["cian_serp", "avito_detail"],
+    "living_area_m2":    ["cian_serp"],
+    "kitchen_area_m2":   ["cian_serp", "avito_detail"],
+    "ceiling_height":    ["cian_detail"],
+    "floor":             ["cian_serp", "avito_detail"],
+    "total_floors":      ["cian_serp", "avito_detail"],
+    "year_built":        ["cian_serp"],
+    "house_type":        ["cian_serp", "avito_detail", "yandex_detail"],
+    "repair_state":      ["cian_detail", "avito_detail"],
+    "has_balcony":       ["cian_serp", "avito_detail"],
+    "balconies_count":   ["cian_serp"],
+    "loggias_count":     ["cian_serp"],
+    "windows_view_type": ["cian_detail", "avito_detail"],
+    "separate_wcs_count": ["cian_detail"],
+    "combined_wcs_count": ["cian_detail"],
+    "room_type":         ["cian_detail", "avito_detail"],
+    "has_furniture":     ["cian_serp", "avito_detail"],
+    "phones":            ["cian_serp"],
+    "description":       ["cian_serp", "avito_detail", "yandex_detail"],
+    "photo_urls":        "union",
+
+    # Avito Domoteka unique
+    "owners_count":      ["avito_domoteka"],
+    "owners_at_least":   ["avito_domoteka"],
+    "last_owner_change_date": ["avito_domoteka"],
+    "encumbrances_clean": ["avito_domoteka"],
+    "registry_match":    ["avito_domoteka"],
+
+    # Cian-only
+    "is_rosreestr_checked": ["cian_serp"],
+    "is_layout_approved":   ["cian_serp"],
+    "is_commercial_ownership_verified": ["cian_serp"],
+
+    # Cross-validation
+    "price_rub":         "cross_validate",
+    "kadastr_num":       "first_non_null",
+    # NOTE: task description claims price_rub=['cian','avito','yandex']; vault sec 4.5
+    # says 'cross_validate' (flag if diff > 10%). Following vault — change to list if
+    # cross_validate semantics aren't desired at caller.
+
+    # Yandex unique (agency block — OfferCardAuthorInfo)
+    "agency_name":           ["yandex_detail"],
+    "agency_founded_year":   ["yandex_detail"],
+    "agency_objects_count":  ["yandex_detail"],
+
+    # Yandex parallel views column (NOT existing `views_total` which is Cian's)
+    "views_total_yandex":    ["yandex_detail"],
+
+    # Yandex raw publish-date text (relative form)
+    "publish_date_relative": ["yandex_detail"],
+
+    # Yandex raw RU sale-type phrase (vs existing `sale_type` enum)
+    "sale_type_text":        ["yandex_detail"],
 }
 
 
+def _resolve(
+    priority: dict[str, list[str] | str],
+    field: str,
+    candidates: dict[str, Any],
+) -> Any:
+    """Pick value from candidates per priority dict semantics."""
+    if not candidates:
+        return None
+    rule = priority.get(field, "first_non_null")
+
+    if isinstance(rule, str):
+        if rule == "union":
+            out: list[Any] = []
+            for v in candidates.values():
+                if v is None:
+                    continue
+                if isinstance(v, (list, tuple, set)):
+                    out.extend(v)
+                else:
+                    out.append(v)
+            # Preserve order, dedup
+            seen: set[Any] = set()
+            uniq: list[Any] = []
+            for x in out:
+                key = repr(x)
+                if key in seen:
+                    continue
+                seen.add(key)
+                uniq.append(x)
+            return uniq
+        if rule == "first_non_null":
+            for v in candidates.values():
+                if v is not None:
+                    return v
+            return None
+        if rule == "max":
+            non_null = [v for v in candidates.values() if v is not None]
+            return max(non_null) if non_null else None
+        if rule == "min":
+            non_null = [v for v in candidates.values() if v is not None]
+            return min(non_null) if non_null else None
+        if rule == "cross_validate":
+            # Caller decides — return median by default
+            nums = [v for v in candidates.values() if isinstance(v, (int, float))]
+            if not nums:
+                return None
+            nums.sort()
+            return nums[len(nums) // 2]
+        # Unknown rule
+        return next((v for v in candidates.values() if v is not None), None)
+
+    # Priority list: pick value from highest-ranked source present (non-null)
+    for src in rule:
+        if src in candidates and candidates[src] is not None:
+            return candidates[src]
+    # Fallback: any non-null
+    return next((v for v in candidates.values() if v is not None), None)
+
+
+def resolve_house_field(field: str, candidates: dict[str, Any]) -> Any:
+    """Pick canonical house field value per HOUSE_FIELD_PRIORITY."""
+    return _resolve(HOUSE_FIELD_PRIORITY, field, candidates)
+
+
+def resolve_listing_field(field: str, candidates: dict[str, Any]) -> Any:
+    """Pick canonical listing field value per LISTING_FIELD_PRIORITY."""
+    return _resolve(LISTING_FIELD_PRIORITY, field, candidates)
+
+
+# ---------------------------------------------------------------------------
+# Legacy stub — kept for backward compat with existing __init__.py and tests
+# ---------------------------------------------------------------------------
+
 def update_canonical_fields(
-    db: Session,
+    db: Any,
     listing_id: int,
     ext_source: str,
     lot_data: object,
 ) -> None:
-    """Merge source fields into the canonical listings row.
-
-    Stage 8 v1: populate-NULL strategy only — write source value only when
-    the canonical column is currently NULL. Full priority arbitration with
-    conflict logging is planned for Stage 8.x when ≥2 sources are live.
-
-    Args:
-        db:         Active SQLAlchemy session.
-        listing_id: Canonical listings.id.
-        ext_source: Source identifier (e.g. 'cian', 'avito').
-        lot_data:   ScrapedLot or similar object with attribute access.
-                    Unknown attributes are silently skipped.
-    """
-    raise NotImplementedError("Stage 8.x — full priority arbitration not yet implemented")
+    """Legacy Stage 8 v1 stub — full arbitration deferred to Stage 8.x."""
+    pass

tradein-mvp/backend/app/services/scrapers/yandex_detail.py

@@ -0,0 +1,369 @@
+"""Yandex Realty detail page scraper.
+
+Fetches /offer/<id>/ and extracts Product JSON-LD + DOM sections into
+a DetailEnrichment Pydantic model. Used by enrichment pipeline
+(Wave 5+ matching / Wave 6 estimator).
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+from datetime import date
+from typing import Any
+
+from pydantic import BaseModel, Field
+from selectolax.parser import HTMLParser, Node
+
+from app.services.scrapers.base import BaseScraper
+from app.services.scrapers.yandex_helpers import (
+    RE_AGENCY_FOUNDED,
+    RE_AGENCY_OBJECTS,
+    RE_METRO_WALK,
+    RE_VIEWS,
+    RE_YEAR,
+    find_ld_by_type,
+    parse_house_type,
+    parse_ru_date,
+    parse_rub,
+)
+
+logger = logging.getLogger(__name__)
+
+
+# ── Pydantic models ───────────────────────────────────────────────────────────
+
+
+class MetroStation(BaseModel):
+    name: str
+    walk_min: int | None = None
+
+
+class DetailEnrichment(BaseModel):
+    """Enrichment payload from a Yandex detail page."""
+
+    offer_id: str
+    source_url: str
+
+    # Pricing — Product JSON-LD `offers.price` is the exact int
+    price_rub: int | None = None
+    price_per_m2: int | None = None
+
+    # Title + basic params
+    title: str | None = None
+    rooms: int | None = None
+    area_m2: float | None = None
+    floor: int | None = None
+    total_floors: int | None = None
+
+    # Address (full)
+    address: str | None = None
+
+    # Description (full text)
+    description: str | None = None
+
+    # Sale type — raw RU phrase ('свободная продажа' / 'альтернативная')
+    sale_type_text: str | None = None
+
+    # Stats
+    views_total: int | None = None
+    publish_date: date | None = None
+    publish_date_relative: str | None = None
+
+    # Agency block (OfferCardAuthorInfo)
+    agency_name: str | None = None
+    agency_founded_year: int | None = None
+    agency_objects_count: int | None = None
+    seller_name: str | None = None  # last text line before "Агентство «...»"
+
+    # Metro stations from "Расположение" section
+    metro_stations: list[MetroStation] = Field(default_factory=list)
+
+    # Photos — 8 sizes from Product.image[]
+    photo_urls: list[str] = Field(default_factory=list)
+
+    # Newbuilding linkage
+    newbuilding_url: str | None = None
+    newbuilding_id: str | None = None
+
+    # NLP from description (best-effort)
+    house_type_nlp: str | None = None
+    year_built_hint: int | None = None
+    metro_walk_min: int | None = None
+
+    # Raw payload (trimmed)
+    raw_payload: dict[str, Any] | None = None
+
+
+# ── Scraper ───────────────────────────────────────────────────────────────────
+
+
+class YandexDetailScraper(BaseScraper):
+    """Detail page scraper for realty.yandex.ru."""
+
+    name = "yandex_detail"
+    base_url = "https://realty.yandex.ru"
+    request_delay_sec = 4.0
+
+    # BaseScraper requires fetch_around — detail isn't geo-based, raise NotImplementedError
+    async def fetch_around(
+        self, lat: float, lon: float, radius_m: int = 1000
+    ) -> list:  # type: ignore[override]
+        raise NotImplementedError(
+            "YandexDetailScraper is offer-id-based; use fetch_detail(offer_url) instead."
+        )
+
+    async def fetch_detail(self, offer_url: str) -> DetailEnrichment | None:
+        try:
+            response = await self._http_get(offer_url)
+        except Exception:
+            logger.exception("yandex detail fetch failed: %s", offer_url)
+            return None
+        if response.status_code != 200:
+            logger.warning(
+                "yandex detail returned %d for %s", response.status_code, offer_url
+            )
+            return None
+        result = self.parse(response.text, offer_url=offer_url)
+        await self.sleep_between_requests()
+        return result
+
+    def parse(self, html: str, offer_url: str) -> DetailEnrichment | None:
+        offer_id_match = re.search(r"/offer/(\d+)/?", offer_url)
+        if not offer_id_match:
+            logger.warning("offer_url has no /offer/<id>/: %s", offer_url)
+            return None
+        offer_id = offer_id_match.group(1)
+
+        tree = HTMLParser(html)
+
+        # --- Product JSON-LD (authoritative price) ---
+        product = find_ld_by_type(html, "Product") or {}
+        offers_ld = product.get("offers") or {}
+        if isinstance(offers_ld, list) and offers_ld:
+            offers_ld = offers_ld[0]
+        price_ld = offers_ld.get("price") if isinstance(offers_ld, dict) else None
+        try:
+            price_rub = int(price_ld) if price_ld else None
+        except (TypeError, ValueError):
+            price_rub = None
+
+        # Photos from JSON-LD image[] (typically 8 size variants)
+        images = product.get("image") or []
+        if isinstance(images, str):
+            images = [images]
+        photo_urls = [u for u in images if isinstance(u, str)]
+
+        # --- Title + summary ---
+        title_node = tree.css_first("h1")
+        title = title_node.text(strip=True) if title_node else None
+
+        rooms, area_m2, floor, total_floors = _parse_title(title or "")
+
+        # --- OfferCardSummary text block ---
+        summary_node = tree.css_first('[data-test="OfferCardSummary"]')
+        summary_text = summary_node.text(strip=True) if summary_node else ""
+
+        # Sale type — raw RU phrase
+        sale_type_text: str | None = None
+        for phrase in ("свободная продажа", "альтернативная"):
+            if phrase in summary_text.lower():
+                sale_type_text = phrase
+                break
+
+        # Views + relative publish date from summary text
+        views_match = RE_VIEWS.search(summary_text)
+        views_total = int(views_match.group(1)) if views_match else None
+        publish_date = parse_ru_date(summary_text)
+        publish_date_relative = _extract_relative_date(summary_text)
+
+        # price_per_m2 — from summary text if absent in LD
+        price_per_m2: int | None = None
+        ppm2_match = re.search(r"(\d[\d\s]+)\s*₽\s*за\s*м²", summary_text)
+        if ppm2_match:
+            price_per_m2 = parse_rub(ppm2_match.group(1))
+
+        # --- OfferCardAuthorInfo (agency block) ---
+        author_node = tree.css_first('[data-test="OfferCardAuthorInfo"]')
+        agency_name: str | None = None
+        agency_founded_year: int | None = None
+        agency_objects_count: int | None = None
+        seller_name: str | None = None
+        if author_node is not None:
+            author_text = author_node.text(strip=True)
+            agency_h2 = author_node.css_first("h2")
+            agency_name = agency_h2.text(strip=True) if agency_h2 else None
+            founded_m = RE_AGENCY_FOUNDED.search(author_text)
+            if founded_m:
+                agency_founded_year = int(founded_m.group(1))
+            objects_m = RE_AGENCY_OBJECTS.search(author_text)
+            if objects_m:
+                agency_objects_count = int(objects_m.group(1))
+            # seller_name — last text line before "Агентство"
+            seller_name = _extract_seller_name(summary_text, agency_name)
+
+        # --- Description section (after H2 "Описание") ---
+        description = _find_section_text(tree, "Описание")
+
+        # --- Address ---
+        address = _extract_address(summary_text)
+
+        # --- Metro stations from "Расположение" section ---
+        location_text = _find_section_text(tree, "Расположение") or ""
+        metro_stations = _parse_metro_stations(location_text)
+
+        # --- Newbuilding link ---
+        nb_url: str | None = None
+        nb_id: str | None = None
+        nb_link = tree.css_first('a[href*="/kupit/novostrojka/"]')
+        if nb_link is not None:
+            nb_href = nb_link.attributes.get("href", "")
+            nb_match = re.search(r"/novostrojka/[\w-]+?-(\d+)/?", nb_href)
+            if nb_match:
+                nb_id = nb_match.group(1)
+                nb_url = (
+                    nb_href
+                    if nb_href.startswith("http")
+                    else f"https://realty.yandex.ru{nb_href}"
+                )
+
+        # --- NLP best-effort from description ---
+        nlp_text = description or summary_text
+        house_type_nlp = parse_house_type(nlp_text)
+        year_hint_m = RE_YEAR.search(nlp_text or "")
+        year_built_hint = int(year_hint_m.group(1)) if year_hint_m else None
+        walk_m = RE_METRO_WALK.search(nlp_text or "")
+        metro_walk_min = int(walk_m.group(1)) if walk_m else None
+
+        return DetailEnrichment(
+            offer_id=offer_id,
+            source_url=offer_url,
+            price_rub=price_rub,
+            price_per_m2=price_per_m2,
+            title=title,
+            rooms=rooms,
+            area_m2=area_m2,
+            floor=floor,
+            total_floors=total_floors,
+            address=address,
+            description=description,
+            sale_type_text=sale_type_text,
+            views_total=views_total,
+            publish_date=publish_date,
+            publish_date_relative=publish_date_relative,
+            agency_name=agency_name,
+            agency_founded_year=agency_founded_year,
+            agency_objects_count=agency_objects_count,
+            seller_name=seller_name,
+            metro_stations=metro_stations,
+            photo_urls=photo_urls,
+            newbuilding_url=nb_url,
+            newbuilding_id=nb_id,
+            house_type_nlp=house_type_nlp,
+            year_built_hint=year_built_hint,
+            metro_walk_min=metro_walk_min,
+            raw_payload={
+                "summary_text": summary_text[:1000],
+                "description_len": len(description) if description else 0,
+                "photo_count": len(photo_urls),
+            },
+        )
+
+
+# ── Helpers ───────────────────────────────────────────────────────────────────
+
+
+def _parse_title(title: str) -> tuple[int | None, float | None, int | None, int | None]:
+    """Extract (rooms, area_m2, floor, total_floors) from h1 text."""
+    rooms: int | None = None
+    area_m2: float | None = None
+    floor: int | None = None
+    total_floors: int | None = None
+
+    area_m = re.search(r"(\d+[.,]?\d*)\s*м²", title)
+    if area_m:
+        area_m2 = float(area_m.group(1).replace(",", "."))
+
+    if re.search(r"студи[яюй]", title, re.IGNORECASE):
+        rooms = 0
+    else:
+        rooms_m = re.search(r"(\d+)\s*-?\s*комнатн", title, re.IGNORECASE)
+        if rooms_m:
+            try:
+                rooms = int(rooms_m.group(1))
+            except ValueError:
+                pass
+
+    floor_m = re.search(r"(\d+)\s+этаж\s+из\s+(\d+)", title, re.IGNORECASE)
+    if floor_m:
+        floor = int(floor_m.group(1))
+        total_floors = int(floor_m.group(2))
+
+    return rooms, area_m2, floor, total_floors
+
+
+def _find_section_text(tree: HTMLParser, heading: str) -> str | None:
+    """Find the text content of a <section>/<div> whose preceding h2/h3 matches heading.
+
+    Yandex page structure varies; this scans h2/h3 nodes, then returns the
+    concatenated text of subsequent sibling blocks until the next heading.
+    """
+    for h in tree.css("h2, h3"):
+        if heading.lower() in (h.text(strip=True) or "").lower():
+            # collect subsequent siblings until the next h2/h3
+            parts: list[str] = []
+            node: Node | None = h.next
+            while node is not None:
+                tag = (node.tag or "").lower()
+                if tag in {"h2", "h3"}:
+                    break
+                txt = node.text(strip=True) if hasattr(node, "text") else ""
+                if txt:
+                    parts.append(txt)
+                node = node.next
+            return " ".join(parts).strip() or None
+    return None
+
+
+def _extract_address(summary_text: str) -> str | None:
+    """Best-effort address extraction from summary block."""
+    # Pattern: "Россия, Свердловская область, Екатеринбург, улица Х, д. N"
+    m = re.search(r"(Россия[^•]+?)(?:•|\d+\s+просмотр|$)", summary_text)
+    if m:
+        addr = m.group(1).strip().rstrip(",").strip()
+        return addr if len(addr) > 10 else None
+    return None
+
+
+def _parse_metro_stations(location_text: str) -> list[MetroStation]:
+    """Parse 'Уральская 11 мин. Динамо 16 мин.' → list of MetroStation."""
+    stations: list[MetroStation] = []
+    # name (1+ Cyrillic words) + space + N + space + мин(.|у|ут)
+    for m in re.finditer(r"([А-ЯЁ][А-Яа-яё\s-]+?)\s+(\d+)\s*мин", location_text):
+        name = m.group(1).strip()
+        if 2 <= len(name) <= 40:
+            stations.append(MetroStation(name=name, walk_min=int(m.group(2))))
+        if len(stations) >= 5:
+            break
+    return stations
+
+
+def _extract_relative_date(summary_text: str) -> str | None:
+    """Capture phrases like '6 часов назад' / 'вчера' / '3 дня назад'."""
+    m = re.search(
+        r"(\d+\s+(?:минут|час|часов|часа|день|дня|дней|недел[ьюи])\s+назад"
+        r"|вчера|сегодня|позавчера)",
+        summary_text,
+        re.IGNORECASE,
+    )
+    return m.group(1).strip() if m else None
+
+
+def _extract_seller_name(summary_text: str, agency_name: str | None) -> str | None:
+    """Heuristic: line right before 'Агентство ...' in summary text."""
+    if not agency_name or agency_name not in summary_text:
+        return None
+    head = summary_text.split(agency_name, 1)[0]
+    # last short token sequence (likely "Имя Фамилия")
+    m = re.findall(r"([А-ЯЁ][а-яё]+(?:\s+[А-ЯЁ][а-яё]+){1,2})", head)
+    return m[-1] if m else None

tradein-mvp/backend/app/services/scrapers/yandex_newbuilding.py

@@ -0,0 +1,352 @@
+"""Yandex Realty ЖК landing page parser.
+
+URL pattern: /{city}/kupit/novostrojka/<slug>-<id>/
+Reference target: ЖК Татлин (id=1592987, slug=tatlin) — comfort+, June 2023,
+PRINZIP, rating 4.3, 1505 ratings, 353 text reviews, coords (56.855312, 60.576668).
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+from typing import Any
+
+from pydantic import BaseModel, Field
+from selectolax.parser import HTMLParser, Node
+
+from app.services.scrapers.base import BaseScraper
+from app.services.scrapers.yandex_helpers import (
+    RE_JK_ID,
+    parse_house_class,
+    parse_house_type,
+)
+
+logger = logging.getLogger(__name__)
+
+
+# ── Models ───────────────────────────────────────────────────────────────────
+
+
+class JKMetroStation(BaseModel):
+    name: str
+    walk_min: int | None = None
+
+
+class YandexNewbuildingInfo(BaseModel):
+    """ЖК landing scrape payload."""
+
+    ext_id: str  # '1592987'
+    ext_slug: str  # 'tatlin'
+    source_url: str
+
+    # Identification
+    name: str | None = None  # 'ЖК «Татлин»'
+    address: str | None = None  # 'Екатеринбург, ул. Черепанова / ул. Готвальда'
+
+    # Coords (inline in HTML — Yandex unique vs Avito/Cian)
+    lat: float | None = None
+    lon: float | None = None
+
+    # Classification + commission
+    house_class: str | None = None  # 'comfort_plus' etc.
+    commission_year: int | None = None
+    commission_month: str | None = None  # 'июнь' (RU)
+
+    # Footprint
+    total_floors: int | None = None  # 35
+    corpus_count: int | None = None  # 3
+    total_area_ha: float | None = None  # 1.5
+
+    # Developer
+    developer_name: str | None = None  # 'PRINZIP'
+    developer_url: str | None = None
+    developer_other_jk: list[str] = Field(default_factory=list)
+
+    # Reviews
+    rating: float | None = None  # 4.3
+    ratings_count: int | None = None  # 1505 — оценок
+    text_reviews_count: int | None = None  # 353 — текстовых отзывов
+
+    # NLP / description
+    description: str | None = None
+
+    # Metro
+    metro_stations: list[JKMetroStation] = Field(default_factory=list)
+
+    # House type (best-effort NLP from description)
+    house_type: str | None = None
+
+    raw_payload: dict[str, Any] | None = None
+
+
+# ── Regex constants ───────────────────────────────────────────────────────────
+
+RE_FLOORS_TOWERS = re.compile(r"(\d+)-этажн\w+\s+башн", re.IGNORECASE)
+RE_AREA_HA = re.compile(r"участке\s+([\d,]+)\s*га", re.IGNORECASE)
+RE_COMMISSION = re.compile(
+    r"(?:введ[её]н|сдан)\s+в\s+эксплуатацию\s+в\s+(\w+)\s+(\d{4})", re.IGNORECASE
+)
+RE_RATING = re.compile(r"(\d[.,]\d)\s+из\s+5")
+RE_RATINGS_COUNT = re.compile(r"(\d+)\s+оценок", re.IGNORECASE)
+RE_TEXT_REVIEWS = re.compile(r"Смотреть\s+все\s+(\d+)\s+отзыв", re.IGNORECASE)
+RE_CORPUS_COUNT_WORD = re.compile(
+    r"(одн[ау]|две|три|четыре|пять|шесть|семь|восемь|\d+)\s+(?:[\d\s-]*)\s*-?\s*этажн",
+    re.IGNORECASE,
+)
+RE_METRO_INLINE = re.compile(
+    r"([А-ЯЁ][А-Яа-яё\s-]{2,30}?)\s+(\d+)\s*мин",
+)
+
+# Ekb-specific coord ranges (extend per-city later)
+LAT_RANGE = (55.5, 57.5)
+LON_RANGE = (59.5, 61.5)
+
+# Word → number map
+_WORD_NUM: dict[str, int] = {
+    "одна": 1,
+    "одну": 1,
+    "одной": 1,
+    "две": 2,
+    "три": 3,
+    "четыре": 4,
+    "пять": 5,
+    "шесть": 6,
+    "семь": 7,
+    "восемь": 8,
+}
+
+
+# ── Scraper class ─────────────────────────────────────────────────────────────
+
+
+class YandexNewbuildingScraper(BaseScraper):
+    name = "yandex_realty_nb"
+    base_url = "https://realty.yandex.ru"
+    request_delay_sec = 4.0
+
+    async def fetch_around(
+        self, lat: float, lon: float, radius_m: int = 1000
+    ) -> list:  # type: ignore[override]
+        raise NotImplementedError(
+            "YandexNewbuildingScraper is JK-slug-based; use fetch_jk(slug, id) instead."
+        )
+
+    async def fetch_jk(
+        self, jk_slug: str, jk_id: str, city: str = "ekaterinburg"
+    ) -> YandexNewbuildingInfo | None:
+        url = f"{self.base_url}/{city}/kupit/novostrojka/{jk_slug}-{jk_id}/"
+        try:
+            response = await self._http_get(url)
+        except Exception:
+            logger.exception("yandex nb fetch failed: %s", url)
+            return None
+        if response.status_code != 200:
+            logger.warning("yandex nb returned %d for %s", response.status_code, url)
+            return None
+        result = self.parse(response.text, jk_slug=jk_slug, jk_id=jk_id, source_url=url)
+        await self.sleep_between_requests()
+        return result
+
+    def parse(
+        self, html: str, jk_slug: str, jk_id: str, source_url: str
+    ) -> YandexNewbuildingInfo:
+        tree = HTMLParser(html)
+        body = tree.body
+        body_text = body.text(strip=True) if body else ""
+
+        # Header
+        h1 = tree.css_first("h1")
+        name = h1.text(strip=True) if h1 else None
+
+        # Sections (heading-based extraction)
+        location_text = _find_section_text(tree, "Расположение") or ""
+        description_section = (
+            _find_section_text(tree, "О комплексе")
+            or _find_section_text(tree, "Расположение, транспортная доступность")
+            or location_text
+        )
+
+        # Coords — inline scan within plausible range
+        lat, lon = _extract_coords(html)
+
+        # Class
+        house_class = parse_house_class(description_section) or parse_house_class(body_text)
+
+        # Commission
+        commission_year: int | None = None
+        commission_month: str | None = None
+        c_m = RE_COMMISSION.search(description_section) or RE_COMMISSION.search(body_text)
+        if c_m:
+            commission_month = c_m.group(1).lower()
+            try:
+                commission_year = int(c_m.group(2))
+            except ValueError:
+                pass
+
+        # Floors + corpus + area
+        floors_m = RE_FLOORS_TOWERS.search(description_section) or RE_FLOORS_TOWERS.search(
+            body_text
+        )
+        total_floors = int(floors_m.group(1)) if floors_m else None
+
+        corpus_count = _parse_corpus_count(description_section or body_text)
+
+        area_m = RE_AREA_HA.search(description_section) or RE_AREA_HA.search(body_text)
+        total_area_ha = float(area_m.group(1).replace(",", ".")) if area_m else None
+
+        # Developer
+        dev_link = tree.css_first('[data-test="CARD_DEV_BADGE_DEVELOPER_LINK"]')
+        developer_name: str | None = None
+        developer_url: str | None = None
+        if dev_link is not None:
+            developer_name = dev_link.text(strip=True) or None
+            href = dev_link.attributes.get("href", "")
+            if href:
+                developer_url = (
+                    href
+                    if href.startswith("http")
+                    else f"https://realty.yandex.ru{href}"
+                )
+
+        # Developer's other JKs
+        other_jk_block = tree.css_first('[data-test="CardDevSites"]')
+        developer_other_jk: list[str] = []
+        if other_jk_block is not None:
+            for jk_a in other_jk_block.css("a"):
+                t = jk_a.text(strip=True)
+                if t and t not in developer_other_jk:
+                    developer_other_jk.append(t)
+
+        # Reviews
+        rating_m = RE_RATING.search(body_text)
+        rating = float(rating_m.group(1).replace(",", ".")) if rating_m else None
+        rcount_m = RE_RATINGS_COUNT.search(body_text)
+        ratings_count = int(rcount_m.group(1)) if rcount_m else None
+        treviews_m = RE_TEXT_REVIEWS.search(body_text)
+        text_reviews_count = int(treviews_m.group(1)) if treviews_m else None
+
+        # Address (header area before lat/lon block)
+        address = _extract_jk_address(body_text, name)
+
+        # Metro stations
+        metro_stations = _parse_metro(body_text)
+
+        # NLP house_type fallback
+        house_type = parse_house_type(description_section) or parse_house_type(body_text)
+
+        return YandexNewbuildingInfo(
+            ext_id=jk_id,
+            ext_slug=jk_slug,
+            source_url=source_url,
+            name=name,
+            address=address,
+            lat=lat,
+            lon=lon,
+            house_class=house_class,
+            commission_year=commission_year,
+            commission_month=commission_month,
+            total_floors=total_floors,
+            corpus_count=corpus_count,
+            total_area_ha=total_area_ha,
+            developer_name=developer_name,
+            developer_url=developer_url,
+            developer_other_jk=developer_other_jk[:10],
+            rating=rating,
+            ratings_count=ratings_count,
+            text_reviews_count=text_reviews_count,
+            description=description_section or None,
+            metro_stations=metro_stations,
+            house_type=house_type,
+            raw_payload={
+                "description_len": len(description_section or ""),
+                "body_len": len(body_text),
+            },
+        )
+
+
+# ── helpers ───────────────────────────────────────────────────────────────────
+
+
+def _extract_coords(html: str) -> tuple[float | None, float | None]:
+    """Scan HTML for any `<lat>.\\d+` and `<lon>.\\d+` matching plausible city range."""
+    lats = [
+        float(m)
+        for m in re.findall(r"\b(\d{2}\.\d{4,8})\b", html)
+        if LAT_RANGE[0] <= float(m) <= LAT_RANGE[1]
+    ]
+    lons = [
+        float(m)
+        for m in re.findall(r"\b(\d{2}\.\d{4,8})\b", html)
+        if LON_RANGE[0] <= float(m) <= LON_RANGE[1]
+    ]
+    lat = lats[0] if lats else None
+    lon = lons[0] if lons else None
+    return lat, lon
+
+
+def _find_section_text(tree: HTMLParser, heading: str) -> str | None:
+    for h in tree.css("h2, h3"):
+        if heading.lower() in (h.text(strip=True) or "").lower():
+            parts: list[str] = []
+            node: Node | None = h.next
+            while node is not None:
+                tag = (node.tag or "").lower()
+                if tag in {"h2", "h3"}:
+                    break
+                txt = node.text(strip=True) if hasattr(node, "text") else ""
+                if txt:
+                    parts.append(txt)
+                node = node.next
+            return " ".join(parts).strip() or None
+    return None
+
+
+def _parse_corpus_count(text: str) -> int | None:
+    m = RE_CORPUS_COUNT_WORD.search(text)
+    if not m:
+        return None
+    token = m.group(1).lower()
+    if token.isdigit():
+        return int(token)
+    return _WORD_NUM.get(token)
+
+
+def _parse_metro(text: str) -> list[JKMetroStation]:
+    stations: list[JKMetroStation] = []
+    for m in RE_METRO_INLINE.finditer(text):
+        name = m.group(1).strip()
+        # filter common noise
+        if not 2 <= len(name) <= 40 or "ходьб" in name.lower() or "минут" in name.lower():
+            continue
+        try:
+            stations.append(JKMetroStation(name=name, walk_min=int(m.group(2))))
+        except ValueError:
+            continue
+        if len(stations) >= 5:
+            break
+    return stations
+
+
+def _extract_jk_address(body_text: str, name: str | None) -> str | None:
+    """Find address line — typically right after JK name, before metro entries."""
+    if name and name in body_text:
+        after_name = body_text.split(name, 1)[1]
+        # Match 'Город, ул. X / ул. Y' until first metro mention
+        m = re.match(
+            r"\s*([А-ЯЁ][^•]{5,150}?)(?:\s+[А-ЯЁ][а-яё]+\s+\d+\s*мин|\d+\s+оценок|$)",
+            after_name,
+        )
+        if m:
+            addr = m.group(1).strip().rstrip(",").strip()
+            if 10 < len(addr) < 200:
+                return addr
+    return None
+
+
+# Expose RE_JK_ID at module level for external use (e.g. SERP link → newbuilding detail)
+__all__ = [
+    "RE_JK_ID",
+    "JKMetroStation",
+    "YandexNewbuildingInfo",
+    "YandexNewbuildingScraper",
+]

tradein-mvp/backend/app/services/scrapers/yandex_valuation.py

@@ -0,0 +1,331 @@
+"""Yandex Realty anonymous valuation/history scraper.
+
+URL pattern:
+  GET /otsenka-kvartiry-po-adresu-onlayn/?address=...&offerCategory=...&offerType=...&page=N
+
+No cookies, no auth required (unlike Cian Calculator or Avito IMV).
+
+Extracts:
+- House metadata: year_built, total_floors, house_type, ceiling_height, has_lift, total_objects
+- Historical offers: area, rooms, floor, start/last price + per-m2, publish_date DMY, exposure
+
+Two-strategy history extraction:
+1. data-test container selectors (CSS, if Yandex adds them)
+2. Fallback: text chunks around DD.MM.YYYY date matches with dedup by (date, area, floor)
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+from datetime import date
+from typing import Any
+from urllib.parse import urlencode
+
+from pydantic import BaseModel, Field
+from selectolax.parser import HTMLParser
+
+from app.services.scrapers.base import BaseScraper
+from app.services.scrapers.yandex_helpers import (
+    parse_dmy,
+    parse_house_type,
+    parse_rub,
+)
+
+logger = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# Models
+# ---------------------------------------------------------------------------
+
+
+class ValuationHouseMeta(BaseModel):
+    """House metadata extracted from Yandex valuation page."""
+
+    year_built: int | None = None
+    total_floors: int | None = None
+    house_type: str | None = None  # panel/brick/monolith/...
+    ceiling_height: float | None = None  # in meters, e.g. 2.5
+    has_lift: bool | None = None
+    total_objects: int | None = None  # 'N объектов' (full archive count)
+    has_panorama: bool = False  # 'Панорама' label present
+
+
+class ValuationHistoryItem(BaseModel):
+    """One historical offer entry from the valuation page."""
+
+    area_m2: float | None = None
+    rooms: int | None = None
+    floor: int | None = None
+    start_price: int | None = None
+    start_price_per_m2: int | None = None
+    last_price: int | None = None
+    last_price_per_m2: int | None = None
+    publish_date: date | None = None
+    exposure_days: int | None = None
+    status: str | None = None  # 'В продаже' / 'Снято'
+
+
+class YandexValuationResult(BaseModel):
+    """Full result of one /otsenka-... GET."""
+
+    address: str
+    offer_category: str
+    offer_type: str
+    page: int
+    source_url: str
+    house: ValuationHouseMeta
+    history_items: list[ValuationHistoryItem] = Field(default_factory=list)
+    raw_payload: dict[str, Any] | None = None
+
+
+# ---------------------------------------------------------------------------
+# Regex constants
+# ---------------------------------------------------------------------------
+
+RE_YEAR_BUILT = re.compile(r"Дом\s+(\d{4})\s+года", re.IGNORECASE)
+RE_FLOORS = re.compile(r"(\d+)\s+этаж", re.IGNORECASE)
+RE_CEILING = re.compile(r"([\d,]+)\s*м\s+потолки", re.IGNORECASE)
+RE_TOTAL_OBJECTS = re.compile(r"(\d+)\s+объект", re.IGNORECASE)
+
+RE_ITEM_AREA = re.compile(r"(\d+[.,]?\d*)\s*м²")
+RE_ITEM_ROOMS = re.compile(r"(\d+)\s*-\s*комнатн", re.IGNORECASE)
+RE_ITEM_STUDIO = re.compile(r"студи[яюй]", re.IGNORECASE)
+RE_ITEM_FLOOR = re.compile(r"(\d+)\s*этаж", re.IGNORECASE)
+RE_ITEM_EXPOSURE = re.compile(r"экспозиции\s+(\d+)\s+дн", re.IGNORECASE)
+RE_ITEM_STATUS = re.compile(r"(В\s+продаже|Снят[оа])", re.IGNORECASE)
+
+# Matches total-price tokens (rubles) — excludes per-m2 tokens by negative lookahead
+_RE_PRICE_TOKEN = re.compile(
+    r"(?:\d[\d\s]*\d|\d)(?:[.,]\d+)?\s*(?:млн)?\s*₽(?!\s*за\s*м²)"
+)
+_RE_PPM2_TOKEN = re.compile(r"\d[\d\s]*\s*₽\s*за\s*м²")
+
+
+# ---------------------------------------------------------------------------
+# Scraper
+# ---------------------------------------------------------------------------
+
+
+class YandexValuationScraper(BaseScraper):
+    """Anonymous Yandex valuation/history scraper.
+
+    Fetches https://realty.yandex.ru/otsenka-kvartiry-po-adresu-onlayn/ by address.
+    Pagination via ?page=N. Use fetch_house_history() directly;
+    fetch_around() is not applicable to this tool.
+    """
+
+    name = "yandex_valuation"
+    base_url = "https://realty.yandex.ru"
+    valuation_path = "/otsenka-kvartiry-po-adresu-onlayn/"
+    request_delay_sec = 4.0
+
+    async def fetch_around(
+        self, lat: float, lon: float, radius_m: int = 1000
+    ) -> list:  # type: ignore[override]
+        raise NotImplementedError(
+            "YandexValuationScraper is address-based; use fetch_house_history() instead."
+        )
+
+    async def fetch_house_history(
+        self,
+        address: str,
+        offer_category: str = "APARTMENT",
+        offer_type: str = "SELL",
+        page: int = 1,
+    ) -> YandexValuationResult | None:
+        """Fetch and parse one page of house history for the given address.
+
+        Args:
+            address: Postal address string (will be URL-encoded).
+            offer_category: 'APARTMENT' / 'ROOMS' / etc.
+            offer_type: 'SELL' / 'RENT'.
+            page: 1-based pagination index.
+
+        Returns:
+            YandexValuationResult or None on HTTP error / network failure.
+        """
+        params = {
+            "address": address,
+            "offerCategory": offer_category,
+            "offerType": offer_type,
+            "page": page,
+        }
+        url = f"{self.base_url}{self.valuation_path}?{urlencode(params)}"
+        try:
+            response = await self._http_get(url)
+        except Exception:
+            logger.exception("yandex valuation fetch failed: %s", url)
+            return None
+        if response.status_code != 200:
+            logger.warning(
+                "yandex valuation returned %d for %s", response.status_code, url
+            )
+            return None
+        result = self.parse(
+            response.text,
+            address=address,
+            offer_category=offer_category,
+            offer_type=offer_type,
+            page=page,
+            source_url=url,
+        )
+        await self.sleep_between_requests()
+        return result
+
+    def parse(
+        self,
+        html: str,
+        address: str,
+        offer_category: str,
+        offer_type: str,
+        page: int,
+        source_url: str,
+    ) -> YandexValuationResult:
+        """Parse raw HTML into YandexValuationResult. Pure function — usable in unit tests."""
+        tree = HTMLParser(html)
+        body = tree.body
+        body_text = body.text(strip=True) if body else ""
+
+        house = self._parse_house_meta(body_text)
+        history_items = self._parse_history_items(tree, body_text)
+
+        return YandexValuationResult(
+            address=address,
+            offer_category=offer_category,
+            offer_type=offer_type,
+            page=page,
+            source_url=source_url,
+            house=house,
+            history_items=history_items,
+            raw_payload={
+                "body_len": len(body_text),
+                "items_count": len(history_items),
+            },
+        )
+
+    @staticmethod
+    def _parse_house_meta(body_text: str) -> ValuationHouseMeta:
+        """Extract house-level metadata from page body text."""
+        year_m = RE_YEAR_BUILT.search(body_text)
+        floors_m = RE_FLOORS.search(body_text)
+        ceiling_m = RE_CEILING.search(body_text)
+        objects_m = RE_TOTAL_OBJECTS.search(body_text)
+        return ValuationHouseMeta(
+            year_built=int(year_m.group(1)) if year_m else None,
+            total_floors=int(floors_m.group(1)) if floors_m else None,
+            house_type=parse_house_type(body_text),
+            ceiling_height=(
+                float(ceiling_m.group(1).replace(",", ".")) if ceiling_m else None
+            ),
+            has_lift="Лифт" in body_text,
+            total_objects=int(objects_m.group(1)) if objects_m else None,
+            has_panorama="Панорама" in body_text,
+        )
+
+    def _parse_history_items(
+        self, tree: HTMLParser, body_text: str
+    ) -> list[ValuationHistoryItem]:
+        """Extract list of historical offer items using best available strategy.
+
+        Strategy 1: CSS data-test containers (if Yandex exposes them).
+        Strategy 2: Fallback — split body text into chunks around DD.MM.YYYY dates.
+        """
+        items: list[ValuationHistoryItem] = []
+        # Strategy 1: explicit data-test container
+        for container in tree.css(
+            '[data-test*="HistoryItem"], [data-test*="ValuationItem"]'
+        ):
+            item = self._parse_item_text(container.text(strip=True))
+            if item:
+                items.append(item)
+        if items:
+            return items
+        # Strategy 2: text-chunk fallback
+        return self._parse_items_from_chunked_text(body_text)
+
+    @staticmethod
+    def _parse_item_text(text: str) -> ValuationHistoryItem | None:
+        """Parse a single text block into a ValuationHistoryItem.
+
+        Returns None if neither area nor start_price can be extracted
+        (item is considered invalid/noise).
+        """
+        if not text or len(text) < 20:
+            return None
+
+        # area + rooms
+        area_m = RE_ITEM_AREA.search(text)
+        area_m2 = float(area_m.group(1).replace(",", ".")) if area_m else None
+
+        rooms: int | None = None
+        if RE_ITEM_STUDIO.search(text):
+            rooms = 0
+        else:
+            rooms_m = RE_ITEM_ROOMS.search(text)
+            if rooms_m:
+                rooms = int(rooms_m.group(1))
+
+        floor_m = RE_ITEM_FLOOR.search(text)
+        floor = int(floor_m.group(1)) if floor_m else None
+
+        # Prices — find first 2 price tokens (start, last)
+        price_tokens = _RE_PRICE_TOKEN.findall(text)
+        start_price = parse_rub(price_tokens[0]) if len(price_tokens) >= 1 else None
+        last_price = parse_rub(price_tokens[1]) if len(price_tokens) >= 2 else None
+
+        ppm2_tokens = _RE_PPM2_TOKEN.findall(text)
+        start_ppm2 = parse_rub(ppm2_tokens[0]) if len(ppm2_tokens) >= 1 else None
+        last_ppm2 = parse_rub(ppm2_tokens[1]) if len(ppm2_tokens) >= 2 else None
+
+        publish_date = parse_dmy(text)
+        expo_m = RE_ITEM_EXPOSURE.search(text)
+        exposure_days = int(expo_m.group(1)) if expo_m else None
+
+        status_m = RE_ITEM_STATUS.search(text)
+        status = status_m.group(1) if status_m else None
+
+        # Item is valid only if we got at least area or price
+        if area_m2 is None and start_price is None:
+            return None
+
+        return ValuationHistoryItem(
+            area_m2=area_m2,
+            rooms=rooms,
+            floor=floor,
+            start_price=start_price,
+            start_price_per_m2=start_ppm2,
+            last_price=last_price,
+            last_price_per_m2=last_ppm2,
+            publish_date=publish_date,
+            exposure_days=exposure_days,
+            status=status,
+        )
+
+    @classmethod
+    def _parse_items_from_chunked_text(cls, body_text: str) -> list[ValuationHistoryItem]:
+        """Fallback: split body text into chunks around DD.MM.YYYY dates and parse each chunk.
+
+        Window: 200 chars before + 100 chars after each date match.
+        Deduplicates by (publish_date, area_m2, floor).
+        Caps output at 30 items per page to avoid runaway extraction.
+        """
+        items: list[ValuationHistoryItem] = []
+        for m in re.finditer(r"\d{2}\.\d{2}\.\d{4}", body_text):
+            start = max(0, m.start() - 200)
+            end = min(len(body_text), m.end() + 100)
+            chunk = body_text[start:end]
+            item = cls._parse_item_text(chunk)
+            if item:
+                items.append(item)
+
+        # Dedup by (publish_date, area_m2, floor)
+        seen: set[tuple[Any, Any, Any]] = set()
+        deduped: list[ValuationHistoryItem] = []
+        for item in items:
+            key = (item.publish_date, item.area_m2, item.floor)
+            if key not in seen:
+                seen.add(key)
+                deduped.append(item)
+        return deduped[:30]

tradein-mvp/backend/app/tasks/refresh_search_matview.py

@@ -0,0 +1,70 @@
+"""Refresh listings_search_mv materialized view (daily 3 AM).
+
+Currently tradein-mvp has no Celery infrastructure (verified — no celery_app.py exists
+in tradein-mvp/backend/app/). This module provides both:
+  - A Celery task wrapper (active only if celery is installed and celery_app present).
+  - A plain async refresh_search_matview() callable for cron / one-shot invocation.
+
+NEEDS COORDINATION: main session must decide whether to bootstrap Celery in tradein-mvp
+or run via external scheduler (systemd timer / OS cron). Until then, no Beat schedule.
+"""
+from __future__ import annotations
+
+import logging
+import time
+
+import psycopg
+
+from app.core.config import settings
+
+logger = logging.getLogger(__name__)
+
+
+def refresh_search_matview() -> None:
+    """REFRESH MATERIALIZED VIEW CONCURRENTLY listings_search_mv (psycopg v3, sync).
+
+    Logs duration. Idempotent. Safe to run during read traffic (CONCURRENTLY).
+    """
+    start = time.monotonic()
+    # DATABASE_URL is SQLAlchemy dialect form (postgresql+psycopg://...) in tradein-mvp
+    # (see tradein-mvp/docker-compose.prod.yml). libpq / psycopg.connect() accepts only
+    # postgresql:// or postgres:// — strip the +psycopg dialect prefix.
+    dsn = settings.database_url.replace("postgresql+psycopg://", "postgresql://", 1)
+    with psycopg.connect(dsn, autocommit=True) as conn:
+        with conn.cursor() as cur:
+            cur.execute("REFRESH MATERIALIZED VIEW CONCURRENTLY listings_search_mv")
+    elapsed = time.monotonic() - start
+    logger.info("listings_search_mv refresh completed in %.2fs", elapsed)
+
+
+# Celery-task wrapper — only registers if celery_app exists.
+try:
+    from app.celery_app import celery_app  # type: ignore[import-not-found]
+except ImportError:
+    logger.info(
+        "Celery not configured in tradein-mvp — refresh_search_matview() callable directly. "
+        "Bootstrap app/celery_app.py to enable scheduled execution.",
+    )
+else:
+
+    @celery_app.task(name="tradein.refresh_search_matview")  # type: ignore[misc]
+    def refresh_search_matview_task() -> None:
+        """Celery wrapper for refresh_search_matview()."""
+        refresh_search_matview()
+
+
+# =============================================================================
+# Beat schedule snippet — paste into app/celery_app.py if/when Celery is bootstrapped:
+#
+#   from celery.schedules import crontab
+#
+#   celery_app.conf.beat_schedule = {
+#       'tradein-refresh-matview': {
+#           'task':     'tradein.refresh_search_matview',
+#           'schedule': crontab(hour=3, minute=0),
+#       },
+#       # ... other tradein beat entries
+#   }
+#
+# Per master plan sec 10.1 ('refresh-search-matview-daily': crontab(minute=0, hour=3)).
+# =============================================================================

tradein-mvp/backend/data/sql/050_search_optimization.sql

@@ -0,0 +1,193 @@
+-- 050_search_optimization.sql
+-- Purpose: Search-time performance optimization for cross-source listings.
+--          Adds pg_trgm + generated tsvector + 10 indexes (master plan sec 5.1)
+--          + listings_search_mv materialized view (master plan sec 6.5)
+--          + 5 matview indexes.
+-- Dependencies: 002_core_tables.sql (listings), 009_houses.sql, 028_matching_tables.sql,
+--               030_listings_alter_yandex.sql.
+-- Deploy order: Apply after 046_views.sql.
+-- Re-run safe: all ADD COLUMN / CREATE INDEX / CREATE MATERIALIZED VIEW use IF NOT EXISTS.
+--
+-- Sources:
+--   Multi-Source Integration Master Plan sec 5.1 (10 индексов on listings)
+--   Multi-Source Integration Master Plan sec 6.5 (listings_search_mv)
+
+BEGIN;
+
+-- =============================================================================
+-- Extensions
+-- =============================================================================
+CREATE EXTENSION IF NOT EXISTS pg_trgm;
+
+-- =============================================================================
+-- Generated tsvector column for full-text search (russian config)
+-- Master plan sec 5.1 index #6 + sec 6.5 matview tsv definition.
+-- Weight A → description, weight B → address.
+-- =============================================================================
+DO $$
+BEGIN
+    IF NOT EXISTS (
+        SELECT 1 FROM information_schema.columns
+        WHERE table_name = 'listings' AND column_name = 'tsv'
+    ) THEN
+        ALTER TABLE listings ADD COLUMN tsv tsvector
+            GENERATED ALWAYS AS (
+                setweight(to_tsvector('russian', coalesce(description, '')), 'A')
+              || setweight(to_tsvector('russian', coalesce(address, '')),     'B')
+            ) STORED;
+    END IF;
+END
+$$;
+
+-- =============================================================================
+-- 10 indexes per master plan sec 5.1
+-- =============================================================================
+
+-- 1) Hot path: active listings by geo radius (GIST partial)
+CREATE INDEX IF NOT EXISTS listings_geom_active_idx
+  ON listings USING GIST (geom)
+  WHERE is_active = true;
+
+-- 2) Active + filter composite: rooms + price_rub + area_m2 + scraped_at DESC
+CREATE INDEX IF NOT EXISTS listings_active_filter_idx
+  ON listings (rooms, price_rub, area_m2, scraped_at DESC)
+  WHERE is_active = true;
+
+-- 3) House + price (price range within a house)
+CREATE INDEX IF NOT EXISTS listings_house_price_idx
+  ON listings (house_id_fk, price_rub)
+  WHERE is_active = true;
+
+-- 4) Recency sort (scraped_at DESC)
+CREATE INDEX IF NOT EXISTS listings_scraped_desc_idx
+  ON listings (scraped_at DESC)
+  WHERE is_active = true;
+
+-- 5) Address fuzzy search (trigram)
+CREATE INDEX IF NOT EXISTS listings_address_trgm_idx
+  ON listings USING GIN (address gin_trgm_ops)
+  WHERE address IS NOT NULL;
+
+-- 6) Full-text search (tsv column above)
+CREATE INDEX IF NOT EXISTS listings_tsv_idx
+  ON listings USING GIN (tsv);
+
+-- 7) Cadastral exact lookup (partial)
+CREATE INDEX IF NOT EXISTS listings_kadastr_idx
+  ON listings (kadastr_num)
+  WHERE kadastr_num IS NOT NULL;
+
+-- 8) listing_sources (listing_id) — for source aggregation
+CREATE INDEX IF NOT EXISTS listing_sources_listing_idx2
+  ON listing_sources (listing_id);
+
+-- 9) listing_sources (listing_id, price_rub) — for price divergence detection
+CREATE INDEX IF NOT EXISTS listing_sources_price_divergence_idx2
+  ON listing_sources (listing_id, price_rub)
+  WHERE price_rub IS NOT NULL;
+
+-- 10) houses GIST + fingerprint + kadastr (consolidated houses indexes)
+CREATE INDEX IF NOT EXISTS houses_geom_idx2
+  ON houses USING GIST (geom);
+CREATE INDEX IF NOT EXISTS houses_fingerprint_idx2
+  ON houses (address_fingerprint)
+  WHERE address_fingerprint IS NOT NULL;
+CREATE INDEX IF NOT EXISTS houses_kadastr_idx2
+  ON houses (cadastral_number)
+  WHERE cadastral_number IS NOT NULL;
+
+COMMIT;
+
+-- =============================================================================
+-- Materialized view: listings_search_mv (master plan sec 6.5)
+-- Separate BEGIN/COMMIT block — matview creation can be slow on populated tables.
+-- =============================================================================
+BEGIN;
+
+DROP MATERIALIZED VIEW IF EXISTS listings_search_mv;
+
+CREATE MATERIALIZED VIEW listings_search_mv AS
+SELECT
+    l.id                AS listing_id,
+    l.source,
+    l.source_url,
+    l.address,
+    l.geom,
+    l.lat,
+    l.lon              AS lng,
+    l.rooms,
+    l.area_m2          AS total_area,
+    l.floor,
+    l.total_floors,
+    l.price_rub,
+    l.price_per_m2,
+    l.kadastr_num,
+    l.is_active,
+    l.scraped_at,
+    -- House denorm
+    h.id               AS house_id,
+    h.year_built,
+    h.house_class,
+    h.developer_name,
+    h.rating           AS house_rating,
+    h.reviews_count    AS house_ratings_count,
+    -- Cross-source aggregates
+    (SELECT count(*)               FROM listing_sources ls WHERE ls.listing_id = l.id) AS source_count,
+    (SELECT array_agg(DISTINCT ext_source) FROM listing_sources ls WHERE ls.listing_id = l.id) AS sources,
+    (SELECT bool_or(ext_source = 'avito')         FROM listing_sources ls WHERE ls.listing_id = l.id) AS has_avito,
+    (SELECT bool_or(ext_source = 'cian')          FROM listing_sources ls WHERE ls.listing_id = l.id) AS has_cian,
+    (SELECT bool_or(ext_source = 'yandex_realty') FROM listing_sources ls WHERE ls.listing_id = l.id) AS has_yandex,
+    -- Price percentile within house
+    (SELECT percentile_cont(0.5) WITHIN GROUP (ORDER BY ll.price_per_m2)
+       FROM listings ll
+      WHERE ll.house_id_fk = l.house_id_fk AND ll.is_active = true) AS house_median_ppm2,
+    -- DEFAULT: district / distance_to_metro_m / last_price_change / photos_count
+    -- not present in current schema — placeholders. (vault sec 6.5 doesn't define them either;
+    -- task description listed them but underlying columns don't exist yet — populate post-Phase 4.)
+    NULL::text                          AS district,
+    NULL::int                           AS distance_to_metro_m,
+    NULL::timestamptz                   AS last_price_change,
+    NULL::int                           AS photos_count,
+    -- Trigram-ready columns
+    l.address                           AS address_trgm,
+    -- Aggregated tsv (description + address + developer_name)
+    to_tsvector('russian',
+        coalesce(l.description, '') || ' ' ||
+        coalesce(l.address, '')     || ' ' ||
+        coalesce(h.developer_name, '')
+    ) AS tsv
+FROM listings l
+LEFT JOIN houses h ON h.id = l.house_id_fk
+WHERE l.is_active = true
+  AND COALESCE(l.canonical, true) = true;
+
+-- =============================================================================
+-- 5 matview indexes per master plan sec 6.5
+-- =============================================================================
+CREATE UNIQUE INDEX listings_search_mv_id_idx
+  ON listings_search_mv (listing_id);
+
+CREATE INDEX listings_search_mv_geom_idx
+  ON listings_search_mv USING GIST (geom);
+
+CREATE INDEX listings_search_mv_filters_idx
+  ON listings_search_mv (rooms, price_rub, total_area, scraped_at DESC);
+
+CREATE INDEX listings_search_mv_address_trgm_idx
+  ON listings_search_mv USING GIN (address_trgm gin_trgm_ops);
+
+CREATE INDEX listings_search_mv_tsv_idx
+  ON listings_search_mv USING GIN (tsv);
+
+CREATE INDEX listings_search_mv_sources_idx
+  ON listings_search_mv (has_avito, has_cian, has_yandex);
+
+COMMIT;
+
+-- =============================================================================
+-- Initial populate: REFRESH CONCURRENTLY would fail (matview just created — never populated).
+-- First populate is implicit via CREATE MATERIALIZED VIEW AS SELECT ... above.
+-- Subsequent refreshes (Celery task) use CONCURRENTLY.
+-- Skip explicit REFRESH here — empty source data scenarios will result in empty matview but
+-- creation is still valid.
+-- =============================================================================

tradein-mvp/backend/tests/matching/test_conflict_resolution.py

@@ -0,0 +1,230 @@
+"""Per-field priority resolution tests."""
+from app.services.matching.conflict_resolution import (
+    HOUSE_FIELD_PRIORITY,
+    LISTING_FIELD_PRIORITY,
+    resolve_house_field,
+    resolve_listing_field,
+)
+
+# ---------------------------------------------------------------------------
+# Pre-existing tests (preserved)
+# ---------------------------------------------------------------------------
+
+def test_house_year_built_prefers_cian_bti() -> None:
+    out = resolve_house_field(
+        "year_built",
+        {"avito_houses_catalog": 2020, "cian_bti": 2021, "cian_serp": 2022},
+    )
+    assert out == 2021
+
+
+def test_house_unknown_field_first_non_null() -> None:
+    assert resolve_house_field("totally_unknown", {"x": None, "y": 5}) == 5
+
+
+def test_listing_owners_count_avito_domoteka_only() -> None:
+    out = resolve_listing_field("owners_count", {"avito_domoteka": 2, "cian_serp": 99})
+    assert out == 2
+
+
+def test_listing_photo_urls_union() -> None:
+    out = resolve_listing_field(
+        "photo_urls", {"avito": ["a.jpg", "b.jpg"], "cian": ["b.jpg", "c.jpg"]},
+    )
+    assert set(out) == {"a.jpg", "b.jpg", "c.jpg"}
+
+
+def test_listing_kadastr_first_non_null() -> None:
+    out = resolve_listing_field("kadastr_num", {"avito": None, "cian": "66:1:1:1"})
+    assert out == "66:1:1:1"
+
+
+def test_house_priority_dict_has_year_built() -> None:
+    assert "cian_bti" in HOUSE_FIELD_PRIORITY["year_built"]
+
+
+def test_listing_priority_dict_has_owners() -> None:
+    assert "avito_domoteka" in LISTING_FIELD_PRIORITY["owners_count"]
+
+
+# ---------------------------------------------------------------------------
+# Yandex house priority tests
+# ---------------------------------------------------------------------------
+
+class TestYandexHousePriority:
+    def test_house_lat_yandex_realty_nb_when_cian_missing(self) -> None:
+        out = resolve_house_field("lat", {"yandex_realty_nb": 56.85})
+        assert out == 56.85
+
+    def test_house_lat_cian_preferred_over_yandex(self) -> None:
+        out = resolve_house_field(
+            "lat", {"cian_serp": 56.83, "yandex_realty_nb": 56.85}
+        )
+        assert out == 56.83
+
+    def test_house_year_built_yandex_valuation_picked(self) -> None:
+        out = resolve_house_field("year_built", {"yandex_valuation": 1981})
+        assert out == 1981
+
+    def test_house_year_built_cian_bti_preferred(self) -> None:
+        out = resolve_house_field(
+            "year_built", {"cian_bti": 1980, "yandex_valuation": 1981}
+        )
+        assert out == 1980
+
+    def test_house_text_reviews_count_yandex_only(self) -> None:
+        out = resolve_house_field("text_reviews_count", {"yandex_realty_nb": 353})
+        assert out == 353
+
+    def test_house_corpus_count_yandex_only(self) -> None:
+        out = resolve_house_field("corpus_count", {"yandex_realty_nb": 3})
+        assert out == 3
+
+    def test_house_commission_year_cian_serp_preferred(self) -> None:
+        out = resolve_house_field(
+            "commission_year", {"cian_serp": 2022, "yandex_realty_nb": 2023}
+        )
+        assert out == 2022
+
+    def test_house_commission_month_yandex_only(self) -> None:
+        out = resolve_house_field("commission_month", {"yandex_realty_nb": "июнь"})
+        assert out == "июнь"
+
+    def test_house_developer_name_cian_preferred(self) -> None:
+        out = resolve_house_field(
+            "developer_name",
+            {"cian": "PRINZIP", "yandex_realty_nb": "PRINZIP недвижимость"},
+        )
+        assert out == "PRINZIP"
+
+    def test_house_has_lift_cian_bti_preferred_over_yandex_valuation(self) -> None:
+        out = resolve_house_field(
+            "has_lift", {"cian_bti": True, "yandex_valuation": True}
+        )
+        assert out is True
+
+    def test_house_ceiling_height_cian_detail_preferred(self) -> None:
+        out = resolve_house_field(
+            "ceiling_height", {"cian_detail": 2.7, "yandex_valuation": 2.5}
+        )
+        assert out == 2.7
+
+    def test_house_has_panorama_yandex_valuation_only(self) -> None:
+        out = resolve_house_field("has_panorama", {"yandex_valuation": True})
+        assert out is True
+
+    def test_house_yandex_total_listings_yandex_valuation_only(self) -> None:
+        out = resolve_house_field("yandex_total_listings", {"yandex_valuation": 42})
+        assert out == 42
+
+    def test_house_house_class_yandex_realty_nb_fallback(self) -> None:
+        out = resolve_house_field("house_class", {"yandex_realty_nb": "бизнес"})
+        assert out == "бизнес"
+
+    def test_house_house_class_avito_preferred_over_yandex(self) -> None:
+        out = resolve_house_field(
+            "house_class",
+            {"avito_houses_catalog": "комфорт", "yandex_realty_nb": "бизнес"},
+        )
+        assert out == "комфорт"
+
+    def test_house_total_area_ha_yandex_only(self) -> None:
+        out = resolve_house_field("total_area_ha", {"yandex_realty_nb": 12.5})
+        assert out == 12.5
+
+    def test_house_has_lift_yandex_valuation_fallback_when_cian_missing(self) -> None:
+        out = resolve_house_field("has_lift", {"yandex_valuation": False})
+        assert out is False
+
+
+# ---------------------------------------------------------------------------
+# Yandex listing priority tests
+# ---------------------------------------------------------------------------
+
+class TestYandexListingPriority:
+    def test_listing_description_cian_preferred_over_yandex_detail(self) -> None:
+        out = resolve_listing_field(
+            "description",
+            {"cian_serp": "cian text", "yandex_detail": "yandex text"},
+        )
+        assert out == "cian text"
+
+    def test_listing_house_type_yandex_detail_used_when_cian_avito_missing(self) -> None:
+        out = resolve_listing_field("house_type", {"yandex_detail": "brick"})
+        assert out == "brick"
+
+    def test_listing_agency_name_yandex_only(self) -> None:
+        out = resolve_listing_field(
+            "agency_name", {"yandex_detail": "Агентство «Диал»"}
+        )
+        assert out == "Агентство «Диал»"
+
+    def test_listing_agency_founded_year_yandex_only(self) -> None:
+        out = resolve_listing_field("agency_founded_year", {"yandex_detail": 2005})
+        assert out == 2005
+
+    def test_listing_agency_objects_count_yandex_only(self) -> None:
+        out = resolve_listing_field("agency_objects_count", {"yandex_detail": 120})
+        assert out == 120
+
+    def test_listing_views_total_yandex_only(self) -> None:
+        out = resolve_listing_field("views_total_yandex", {"yandex_detail": 874})
+        assert out == 874
+
+    def test_listing_publish_date_relative_yandex_only(self) -> None:
+        out = resolve_listing_field(
+            "publish_date_relative", {"yandex_detail": "3 дня назад"}
+        )
+        assert out == "3 дня назад"
+
+    def test_listing_sale_type_text_yandex_only(self) -> None:
+        out = resolve_listing_field(
+            "sale_type_text", {"yandex_detail": "Прямая продажа"}
+        )
+        assert out == "Прямая продажа"
+
+    def test_listing_agency_name_none_when_no_yandex(self) -> None:
+        out = resolve_listing_field("agency_name", {"cian_serp": None})
+        assert out is None
+
+    def test_listing_description_avito_detail_preferred_over_yandex(self) -> None:
+        out = resolve_listing_field(
+            "description",
+            {"avito_detail": "avito desc", "yandex_detail": "yandex desc"},
+        )
+        assert out == "avito desc"
+
+    def test_listing_house_type_cian_preferred_over_yandex_detail(self) -> None:
+        out = resolve_listing_field(
+            "house_type",
+            {"cian_serp": "панель", "yandex_detail": "кирпич"},
+        )
+        assert out == "панель"
+
+    def test_listing_yandex_detail_keys_registered(self) -> None:
+        """Ensure all Yandex-unique listing keys are in priority dict."""
+        yandex_keys = [
+            "agency_name",
+            "agency_founded_year",
+            "agency_objects_count",
+            "views_total_yandex",
+            "publish_date_relative",
+            "sale_type_text",
+        ]
+        for key in yandex_keys:
+            assert key in LISTING_FIELD_PRIORITY, f"{key!r} missing from LISTING_FIELD_PRIORITY"
+            rule = LISTING_FIELD_PRIORITY[key]
+            assert rule == ["yandex_detail"], f"{key!r} rule mismatch: {rule!r}"
+
+    def test_house_yandex_keys_registered(self) -> None:
+        """Ensure all Yandex-unique house keys are in priority dict."""
+        yandex_keys = [
+            "text_reviews_count",
+            "corpus_count",
+            "total_area_ha",
+            "commission_month",
+            "has_panorama",
+            "yandex_total_listings",
+        ]
+        for key in yandex_keys:
+            assert key in HOUSE_FIELD_PRIORITY, f"{key!r} missing from HOUSE_FIELD_PRIORITY"

tradein-mvp/backend/tests/test_matching.py

@@ -8,16 +8,14 @@ Covers:
 Reference: decisions/Cross_Source_Matching_Strategy.md
 """
 
-import pytest
-from unittest.mock import MagicMock, call, patch
+from unittest.mock import MagicMock
 
-from app.services.matching.normalize import normalize_address, address_fingerprint
 from app.services.matching.conflict_resolution import (
     HOUSE_FIELD_PRIORITY,
     LISTING_FIELD_PRIORITY,
     update_canonical_fields,
 )
-
+from app.services.matching.normalize import address_fingerprint, normalize_address
 
 # ---------------------------------------------------------------------------
 # normalize_address
@@ -338,15 +336,32 @@ def test_field_priority_dicts_not_empty():
     assert LISTING_FIELD_PRIORITY
 
 
-def test_field_priority_sources_are_lists():
+VALID_STRING_RULES = {"union", "cross_validate", "first_non_null"}
+
+
+def test_field_priority_sources_are_lists_or_known_string_rules():
+    """Each priority entry is either a source list or a known string rule.
+
+    PR #471 (main) introduced 'union' / 'cross_validate' / 'first_non_null' string
+    rules alongside list entries. Both are valid per LISTING_FIELD_PRIORITY type hint.
+    """
     for col, sources in HOUSE_FIELD_PRIORITY.items():
-        assert isinstance(sources, list), f'HOUSE_FIELD_PRIORITY[{col!r}] should be list'
+        assert isinstance(sources, list) or sources in VALID_STRING_RULES, (
+            f'HOUSE_FIELD_PRIORITY[{col!r}] must be list or known rule, got {sources!r}'
+        )
     for col, sources in LISTING_FIELD_PRIORITY.items():
-        assert isinstance(sources, list), f'LISTING_FIELD_PRIORITY[{col!r}] should be list'
+        assert isinstance(sources, list) or sources in VALID_STRING_RULES, (
+            f'LISTING_FIELD_PRIORITY[{col!r}] must be list or known rule, got {sources!r}'
+        )
 
 
-def test_update_canonical_fields_raises_not_implemented():
-    """Stage 8 v1 — stub raises NotImplementedError to catch accidental callers."""
+def test_update_canonical_fields_is_callable():
+    """Stage 8 v1 — legacy stub (no-op pass). Full arbitration deferred to Stage 8.x.
+
+    PR #471 (main) replaced NotImplementedError stub with silent pass to allow
+    backward-compat imports. Test updated accordingly.
+    """
     db = MagicMock()
-    with pytest.raises(NotImplementedError):
-        update_canonical_fields(db, listing_id=1, ext_source='cian', lot_data=object())
+    # Should not raise — no-op stub returns None
+    result = update_canonical_fields(db, listing_id=1, ext_source='cian', lot_data=object())
+    assert result is None

tradein-mvp/backend/tests/test_yandex_detail.py

@@ -0,0 +1,399 @@
+"""Unit tests for YandexDetailScraper — Product JSON-LD + DOM detail parser.
+
+All tests run against hand-crafted HTML fixtures. No live network access.
+"""
+
+from __future__ import annotations
+
+import json
+
+import pytest
+
+from app.services.scrapers.yandex_detail import (
+    DetailEnrichment,
+    MetroStation,
+    YandexDetailScraper,
+    _extract_relative_date,
+    _find_section_text,
+    _parse_metro_stations,
+    _parse_title,
+)
+
+# ---------------------------------------------------------------------------
+# Helpers to build fixture HTML
+# ---------------------------------------------------------------------------
+
+_BASE_OFFER_URL = "https://realty.yandex.ru/offer/7812345000001/"
+
+_PRODUCT_LD = {
+    "@type": "Product",
+    "name": "3-комнатная квартира 85,5 м² на 12 этаж из 24",
+    "image": [
+        "https://avatars.mds.yandex.net/get-realty/photo1/main",
+        "https://avatars.mds.yandex.net/get-realty/photo2/main",
+        "https://avatars.mds.yandex.net/get-realty/photo3/main",
+    ],
+    "offers": {
+        "@type": "Offer",
+        "price": 9850000,
+        "priceCurrency": "RUB",
+    },
+}
+
+
+def _make_html(
+    *,
+    product_ld: dict | None = _PRODUCT_LD,
+    h1: str = "3-комнатная квартира 85,5 м², 12 этаж из 24",
+    summary_text: str = (
+        "Россия, Свердловская область, Екатеринбург, улица Малышева, д. 5"
+        " • 115 233 ₽ за м²"
+        " • свободная продажа"
+        " • 342 просмотра"
+        " • опубликовано 10 апреля 2026"
+    ),
+    author_block: str = "",
+    description_text: str = "Продаётся просторная кирпичная квартира 1995 года постройки.",
+    location_text: str = "Уральская 11 мин. Динамо 16 мин.",
+    nb_href: str | None = "/ekaterinburg/kupit/novostrojka/tatlin-1592987/",
+    extra_head: str = "",
+) -> str:
+    ld_block = ""
+    if product_ld is not None:
+        ld_block = (
+            f'<script type="application/ld+json">{json.dumps(product_ld)}</script>'
+        )
+
+    nb_link = ""
+    if nb_href:
+        nb_link = f'<a href="{nb_href}">ЖК Татлин</a>'
+
+    return f"""<!DOCTYPE html>
+<html>
+<head>{extra_head}{ld_block}</head>
+<body>
+  <h1>{h1}</h1>
+  <div data-test="OfferCardSummary">{summary_text}</div>
+  {author_block}
+  <h2>Описание</h2>
+  <p>{description_text}</p>
+  <h2>Расположение</h2>
+  <p>{location_text}</p>
+  {nb_link}
+</body>
+</html>"""
+
+
+# ---------------------------------------------------------------------------
+# Fixtures
+# ---------------------------------------------------------------------------
+
+FULL_HTML = _make_html()
+
+SCRAPER = YandexDetailScraper()
+
+
+# ---------------------------------------------------------------------------
+# Tests
+# ---------------------------------------------------------------------------
+
+
+class TestFullDetailFixture:
+    """Happy path — all fields populated."""
+
+    def test_parse_full_detail_fixture(self) -> None:
+        result = SCRAPER.parse(FULL_HTML, offer_url=_BASE_OFFER_URL)
+        assert isinstance(result, DetailEnrichment)
+        assert result.offer_id == "7812345000001"
+        assert result.source_url == _BASE_OFFER_URL
+
+    def test_price_from_json_ld(self) -> None:
+        result = SCRAPER.parse(FULL_HTML, offer_url=_BASE_OFFER_URL)
+        assert result is not None
+        assert result.price_rub == 9_850_000
+
+    def test_title_and_room_parsing(self) -> None:
+        result = SCRAPER.parse(FULL_HTML, offer_url=_BASE_OFFER_URL)
+        assert result is not None
+        assert result.rooms == 3
+        assert result.area_m2 == pytest.approx(85.5)
+        assert result.floor == 12
+        assert result.total_floors == 24
+
+    def test_views_and_publish_date(self) -> None:
+        result = SCRAPER.parse(FULL_HTML, offer_url=_BASE_OFFER_URL)
+        assert result is not None
+        from datetime import date
+
+        assert result.views_total == 342
+        assert result.publish_date == date(2026, 4, 10)
+
+    def test_sale_type_free(self) -> None:
+        result = SCRAPER.parse(FULL_HTML, offer_url=_BASE_OFFER_URL)
+        assert result is not None
+        assert result.sale_type_text == "свободная продажа"
+
+    def test_price_per_m2_from_summary(self) -> None:
+        result = SCRAPER.parse(FULL_HTML, offer_url=_BASE_OFFER_URL)
+        assert result is not None
+        assert result.price_per_m2 == 115_233
+
+    def test_address_extraction(self) -> None:
+        result = SCRAPER.parse(FULL_HTML, offer_url=_BASE_OFFER_URL)
+        assert result is not None
+        assert result.address is not None
+        assert "Малышева" in result.address
+
+    def test_description_section(self) -> None:
+        result = SCRAPER.parse(FULL_HTML, offer_url=_BASE_OFFER_URL)
+        assert result is not None
+        assert result.description is not None
+        assert "кирпичная" in result.description
+
+    def test_nlp_house_type_from_description(self) -> None:
+        result = SCRAPER.parse(FULL_HTML, offer_url=_BASE_OFFER_URL)
+        assert result is not None
+        assert result.house_type_nlp == "brick"
+
+    def test_year_built_hint_nlp(self) -> None:
+        result = SCRAPER.parse(FULL_HTML, offer_url=_BASE_OFFER_URL)
+        assert result is not None
+        assert result.year_built_hint == 1995
+
+    def test_raw_payload_present(self) -> None:
+        result = SCRAPER.parse(FULL_HTML, offer_url=_BASE_OFFER_URL)
+        assert result is not None
+        assert result.raw_payload is not None
+        assert "summary_text" in result.raw_payload
+        assert "photo_count" in result.raw_payload
+
+
+class TestNoProductLD:
+    """Missing JSON-LD — scraper falls back to summary / DOM only."""
+
+    def test_parse_no_product_ld_uses_summary_fallbacks(self) -> None:
+        html = _make_html(product_ld=None)
+        result = SCRAPER.parse(html, offer_url=_BASE_OFFER_URL)
+        assert result is not None
+        # price_rub is None when no JSON-LD and no ₽ total in summary
+        assert result.price_rub is None
+        # price_per_m2 still extracted from "₽ за м²" in summary
+        assert result.price_per_m2 == 115_233
+        # photos array empty when no JSON-LD image[]
+        assert result.photo_urls == []
+
+
+class TestStudio:
+    def test_parse_studio_rooms_zero(self) -> None:
+        html = _make_html(
+            product_ld=None,
+            h1="Студия 28 м², 2 этаж из 9",
+            summary_text="альтернативная • 50 просмотров",
+        )
+        result = SCRAPER.parse(html, offer_url=_BASE_OFFER_URL)
+        assert result is not None
+        assert result.rooms == 0
+        assert result.area_m2 == pytest.approx(28.0)
+        assert result.floor == 2
+        assert result.total_floors == 9
+        assert result.sale_type_text == "альтернативная"
+
+
+class TestNoAgencySection:
+    def test_parse_no_agency_section(self) -> None:
+        html = _make_html(author_block="")  # no OfferCardAuthorInfo
+        result = SCRAPER.parse(html, offer_url=_BASE_OFFER_URL)
+        assert result is not None
+        assert result.agency_name is None
+        assert result.agency_founded_year is None
+        assert result.agency_objects_count is None
+        assert result.seller_name is None
+
+
+class TestAgencyBlock:
+    def test_agency_fields_parsed(self) -> None:
+        # Note: selectolax .text(strip=True) concatenates text nodes without spaces.
+        # "Год основания 1998" + "150 объектов" becomes "Год основания 1998150 объектов".
+        # RE_AGENCY_OBJECTS = r"(\d+)\s+объект" would then match "1998150 объект".
+        # To avoid this ambiguity the fixture uses а non-digit separator between spans.
+        author_html = """
+        <div data-test="OfferCardAuthorInfo">
+          <h2>АН Городской риелтор</h2>
+          <p>Год основания 1998. Продали 150 объектов.</p>
+        </div>"""
+        summary = (
+            "Иван Петров АН Городской риелтор • 50 просмотров"
+            " • опубликовано 3 марта 2025"
+        )
+        html = _make_html(author_block=author_html, summary_text=summary)
+        result = SCRAPER.parse(html, offer_url=_BASE_OFFER_URL)
+        assert result is not None
+        assert result.agency_name == "АН Городской риелтор"
+        assert result.agency_founded_year == 1998
+        assert result.agency_objects_count == 150
+
+
+class TestSellerName:
+    def test_seller_name_before_agency(self) -> None:
+        author_html = """
+        <div data-test="OfferCardAuthorInfo">
+          <h2>АН Капитал</h2>
+        </div>"""
+        # seller name appears right before agency name in summary
+        summary = "Мария Кузнецова АН Капитал • 10 просмотров"
+        html = _make_html(author_block=author_html, summary_text=summary)
+        result = SCRAPER.parse(html, offer_url=_BASE_OFFER_URL)
+        assert result is not None
+        assert result.seller_name is not None
+        assert "Кузнецова" in result.seller_name
+
+
+class TestNewbuildingLink:
+    def test_parse_newbuilding_link_extracted(self) -> None:
+        result = SCRAPER.parse(FULL_HTML, offer_url=_BASE_OFFER_URL)
+        assert result is not None
+        assert result.newbuilding_id == "1592987"
+        assert result.newbuilding_url is not None
+        assert "tatlin" in result.newbuilding_url
+
+    def test_no_newbuilding_link(self) -> None:
+        html = _make_html(nb_href=None)
+        result = SCRAPER.parse(html, offer_url=_BASE_OFFER_URL)
+        assert result is not None
+        assert result.newbuilding_id is None
+        assert result.newbuilding_url is None
+
+
+class TestMetroStations:
+    def test_parse_metro_stations_multiple(self) -> None:
+        result = SCRAPER.parse(FULL_HTML, offer_url=_BASE_OFFER_URL)
+        assert result is not None
+        assert len(result.metro_stations) == 2
+        names = [s.name for s in result.metro_stations]
+        assert "Уральская" in names
+        # walk_min for Уральская = 11
+        ural = next(s for s in result.metro_stations if s.name == "Уральская")
+        assert ural.walk_min == 11
+
+    def test_metro_stations_empty_when_no_location(self) -> None:
+        html = _make_html(location_text="")
+        result = SCRAPER.parse(html, offer_url=_BASE_OFFER_URL)
+        assert result is not None
+        assert result.metro_stations == []
+
+    def test_parse_metro_stations_helper_directly(self) -> None:
+        text = "Проспект Космонавтов 7 мин. Уралмаш 14 мин. Эльмаш 20 мин."
+        stations = _parse_metro_stations(text)
+        assert len(stations) == 3
+        assert stations[0] == MetroStation(name="Проспект Космонавтов", walk_min=7)
+        assert stations[1].walk_min == 14
+
+
+class TestRelativeDate:
+    def test_parse_relative_date_yesterday(self) -> None:
+        html = _make_html(
+            summary_text="Россия, ЕКБ, ул. Тест • 100 просмотров • вчера"
+        )
+        result = SCRAPER.parse(html, offer_url=_BASE_OFFER_URL)
+        assert result is not None
+        assert result.publish_date_relative == "вчера"
+
+    def test_relative_date_hours(self) -> None:
+        result = _extract_relative_date("опубликовано 6 часов назад")
+        assert result == "6 часов назад"
+
+    def test_relative_date_days(self) -> None:
+        result = _extract_relative_date("размещено 3 дня назад и ещё текст")
+        assert result == "3 дня назад"
+
+    def test_relative_date_none(self) -> None:
+        result = _extract_relative_date("опубликовано 10 апреля 2026")
+        assert result is None
+
+
+class TestPublishDate:
+    def test_publish_date_ru_format(self) -> None:
+        html = _make_html(
+            summary_text="Россия, ЕКБ • 200 просмотров • опубликовано 5 января 2026"
+        )
+        result = SCRAPER.parse(html, offer_url=_BASE_OFFER_URL)
+        assert result is not None
+        from datetime import date
+
+        assert result.publish_date == date(2026, 1, 5)
+
+
+class TestPhotos:
+    def test_photos_from_json_ld_image_array(self) -> None:
+        ld = dict(_PRODUCT_LD)
+        ld["image"] = [
+            "https://avatars.mds.yandex.net/get-realty/img1",
+            "https://avatars.mds.yandex.net/get-realty/img2",
+            "https://avatars.mds.yandex.net/get-realty/img3",
+            "https://avatars.mds.yandex.net/get-realty/img4",
+            "https://avatars.mds.yandex.net/get-realty/img5",
+            "https://avatars.mds.yandex.net/get-realty/img6",
+            "https://avatars.mds.yandex.net/get-realty/img7",
+            "https://avatars.mds.yandex.net/get-realty/img8",
+        ]
+        html = _make_html(product_ld=ld)
+        result = SCRAPER.parse(html, offer_url=_BASE_OFFER_URL)
+        assert result is not None
+        assert len(result.photo_urls) == 8
+        assert all(u.startswith("https://") for u in result.photo_urls)
+
+    def test_photo_single_string_wrapped(self) -> None:
+        ld = dict(_PRODUCT_LD)
+        ld["image"] = "https://avatars.mds.yandex.net/get-realty/single"
+        html = _make_html(product_ld=ld)
+        result = SCRAPER.parse(html, offer_url=_BASE_OFFER_URL)
+        assert result is not None
+        assert result.photo_urls == ["https://avatars.mds.yandex.net/get-realty/single"]
+
+
+class TestInvalidUrl:
+    def test_invalid_offer_url_returns_none(self) -> None:
+        result = SCRAPER.parse(FULL_HTML, offer_url="https://realty.yandex.ru/ekaterinburg/")
+        assert result is None
+
+    def test_valid_url_with_trailing_slash(self) -> None:
+        result = SCRAPER.parse(FULL_HTML, offer_url="https://realty.yandex.ru/offer/999/")
+        assert result is not None
+        assert result.offer_id == "999"
+
+
+class TestHelpers:
+    def test_parse_title_full(self) -> None:
+        rooms, area, floor, total = _parse_title("2-комнатная квартира 55,3 м², 7 этаж из 18")
+        assert rooms == 2
+        assert area == pytest.approx(55.3)
+        assert floor == 7
+        assert total == 18
+
+    def test_parse_title_studio(self) -> None:
+        rooms, area, _floor, _total = _parse_title("Студия 28 м², 2 этаж из 9")
+        assert rooms == 0
+        assert area == pytest.approx(28.0)
+
+    def test_parse_title_missing_floor(self) -> None:
+        rooms, area, floor, total = _parse_title("1-комнатная квартира 36 м²")
+        assert rooms == 1
+        assert area == pytest.approx(36.0)
+        assert floor is None
+        assert total is None
+
+    def test_find_section_text_returns_none_when_absent(self) -> None:
+        from selectolax.parser import HTMLParser
+
+        tree = HTMLParser("<html><body><h2>Другой раздел</h2><p>текст</p></body></html>")
+        assert _find_section_text(tree, "Описание") is None
+
+    def test_metro_walk_min_from_nlp(self) -> None:
+        """metro_walk_min extracted from description via RE_METRO_WALK."""
+        html = _make_html(
+            description_text="Квартира в 7 минут неспешной прогулки от метро.",
+            location_text="",
+        )
+        result = SCRAPER.parse(html, offer_url=_BASE_OFFER_URL)
+        assert result is not None
+        assert result.metro_walk_min == 7

tradein-mvp/backend/tests/test_yandex_newbuilding.py

@@ -0,0 +1,422 @@
+"""Unit tests for yandex_newbuilding.py — ЖК landing parser (Worker B, Wave 4).
+
+Reference target: ЖК Татлин (slug=tatlin, id=1592987) — comfort+, June 2023,
+PRINZIP developer, rating 4.3, 1505 ratings, 353 text reviews,
+coords (56.855312, 60.576668).
+
+All tests are offline — no network calls. Parser receives hand-crafted fixture HTML.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from app.services.scrapers.yandex_newbuilding import (
+    JKMetroStation,
+    YandexNewbuildingInfo,
+    YandexNewbuildingScraper,
+    _extract_coords,
+    _find_section_text,
+    _parse_corpus_count,
+    _parse_metro,
+)
+
+# ── Fixture HTML ──────────────────────────────────────────────────────────────
+
+TATLIN_FIXTURE_HTML = """<!DOCTYPE html>
+<html>
+<head><title>ЖК «Татлин»</title></head>
+<body>
+<h1>ЖК «Татлин»</h1>
+<div>Екатеринбург, ул. Черепанова / ул. Готвальда</div>
+<a data-test="CARD_DEV_BADGE_DEVELOPER_LINK" href="/developer/prinzip">PRINZIP</a>
+<div data-test="CardDevSites">
+  <a href="/zhk/1">ЖК Кислород</a>
+  <a href="/zhk/2">ЖК Нова</a>
+</div>
+<div>56.855312, 60.576668</div>
+<div>Уральская 11 мин. Динамо 16 мин.</div>
+<div>4.3 из 5  1505 оценок  Смотреть все 353 отзыва</div>
+<h2>О комплексе</h2>
+<p>ЖК «Татлин» — три 35-этажные башни в Заречном микрорайоне Екатеринбурга.
+   Комплекс класса комфорт+. Введён в эксплуатацию в июне 2023 года.
+   Расположен на участке 1,5 га. Монолитный тип дома.</p>
+<h2>Расположение</h2>
+<p>В 5 минутах от центра</p>
+</body>
+</html>"""
+
+# Minimal fixture — missing optional sections
+MINIMAL_FIXTURE_HTML = """<!DOCTYPE html>
+<html><body>
+<h1>ЖК Минимальный</h1>
+</body></html>"""
+
+
+# ── Full happy-path test ──────────────────────────────────────────────────────
+
+
+def test_parse_jk_full_fixture():
+    """Happy path: all major fields extracted from Татлин-like HTML."""
+    scraper = YandexNewbuildingScraper()
+    info = scraper.parse(
+        TATLIN_FIXTURE_HTML,
+        jk_slug="tatlin",
+        jk_id="1592987",
+        source_url="https://realty.yandex.ru/ekaterinburg/kupit/novostrojka/tatlin-1592987/",
+    )
+
+    assert isinstance(info, YandexNewbuildingInfo)
+    assert info.ext_id == "1592987"
+    assert info.ext_slug == "tatlin"
+    assert "Татлин" in (info.name or "")
+
+    # coords
+    assert info.lat == pytest.approx(56.855312, abs=1e-5)
+    assert info.lon == pytest.approx(60.576668, abs=1e-5)
+
+    # class + commission
+    assert info.house_class == "comfort_plus"
+    assert info.commission_year == 2023
+    assert info.commission_month == "июне"
+
+    # footprint
+    assert info.total_floors == 35
+    assert info.corpus_count == 3
+    assert info.total_area_ha == pytest.approx(1.5, abs=0.01)
+
+    # developer
+    assert info.developer_name == "PRINZIP"
+    assert info.developer_url is not None and "prinzip" in info.developer_url
+    assert "ЖК Кислород" in info.developer_other_jk
+    assert "ЖК Нова" in info.developer_other_jk
+
+    # reviews
+    assert info.rating == pytest.approx(4.3, abs=0.01)
+    assert info.ratings_count == 1505
+    assert info.text_reviews_count == 353
+
+    # description present
+    assert info.description is not None
+    assert len(info.description) > 10
+
+    # metro
+    assert len(info.metro_stations) >= 1
+    station_names = [s.name for s in info.metro_stations]
+    assert any("Уральская" in n for n in station_names)
+
+    # house type from "монолитный"
+    assert info.house_type == "monolith"
+
+    # raw_payload
+    assert info.raw_payload is not None
+    assert "body_len" in info.raw_payload
+
+
+# ── Coord extraction tests ────────────────────────────────────────────────────
+
+
+def test_extract_coords_within_ekb_range():
+    html = "<div>56.855312, 60.576668</div>"
+    lat, lon = _extract_coords(html)
+    assert lat == pytest.approx(56.855312, abs=1e-5)
+    assert lon == pytest.approx(60.576668, abs=1e-5)
+
+
+def test_extract_coords_outside_range_returns_none():
+    # Moscow coords — outside EKB range
+    html = "<div>55.7558, 37.6176</div>"
+    _lat, lon = _extract_coords(html)
+    # 55.75 is in LAT_RANGE but 37.61 is NOT in LON_RANGE
+    assert lon is None
+
+
+def test_extract_coords_no_coords_returns_none():
+    html = "<div>Текст без координат</div>"
+    lat, lon = _extract_coords(html)
+    assert lat is None
+    assert lon is None
+
+
+def test_extract_coords_multiple_takes_first():
+    """When multiple valid coords present, first is returned."""
+    html = "<div>56.855312, 60.576668</div><div>56.900000, 60.600000</div>"
+    lat, lon = _extract_coords(html)
+    assert lat == pytest.approx(56.855312, abs=1e-5)
+    assert lon == pytest.approx(60.576668, abs=1e-5)
+
+
+# ── Corpus count tests ────────────────────────────────────────────────────────
+
+
+def test_parse_corpus_count_word_three():
+    text = "три 35-этажные башни в центре"
+    result = _parse_corpus_count(text)
+    assert result == 3
+
+
+def test_parse_corpus_count_digit():
+    text = "5 35-этажных корпусов"
+    result = _parse_corpus_count(text)
+    assert result == 5
+
+
+def test_parse_corpus_count_two():
+    text = "две 20-этажные башни"
+    result = _parse_corpus_count(text)
+    assert result == 2
+
+
+def test_parse_corpus_count_none_when_no_match():
+    text = "Обычный текст без этажей"
+    result = _parse_corpus_count(text)
+    assert result is None
+
+
+# ── House class test ──────────────────────────────────────────────────────────
+
+
+def test_parse_house_class_comfort_plus_from_description():
+    from app.services.scrapers.yandex_helpers import parse_house_class
+
+    text = "Жилой комплекс класса комфорт+ в центре города"
+    result = parse_house_class(text)
+    assert result == "comfort_plus"
+
+
+def test_parse_house_class_business():
+    from app.services.scrapers.yandex_helpers import parse_house_class
+
+    text = "ЖК класса бизнес"
+    result = parse_house_class(text)
+    assert result == "business"
+
+
+# ── Commission year/month test ────────────────────────────────────────────────
+
+
+def test_parse_commission_year_month():
+    """Full pipeline: parse() correctly extracts commission_year and commission_month."""
+    scraper = YandexNewbuildingScraper()
+    info = scraper.parse(
+        TATLIN_FIXTURE_HTML,
+        jk_slug="tatlin",
+        jk_id="1592987",
+        source_url="https://realty.yandex.ru/ekaterinburg/kupit/novostrojka/tatlin-1592987/",
+    )
+    assert info.commission_year == 2023
+    assert info.commission_month == "июне"
+
+
+def test_parse_commission_sdan_variant():
+    """Regex supports 'сдан в эксплуатацию' phrasing as well."""
+    html = """<html><body>
+    <h1>ЖК Тест</h1>
+    <h2>О комплексе</h2>
+    <p>Сдан в эксплуатацию в марте 2025 года.</p>
+    </body></html>"""
+    scraper = YandexNewbuildingScraper()
+    info = scraper.parse(html, jk_slug="test", jk_id="999", source_url="http://x")
+    assert info.commission_year == 2025
+    assert info.commission_month == "марте"
+
+
+# ── Rating + counts test ──────────────────────────────────────────────────────
+
+
+def test_parse_rating_and_counts():
+    scraper = YandexNewbuildingScraper()
+    info = scraper.parse(
+        TATLIN_FIXTURE_HTML,
+        jk_slug="tatlin",
+        jk_id="1592987",
+        source_url="https://realty.yandex.ru/ekaterinburg/kupit/novostrojka/tatlin-1592987/",
+    )
+    assert info.rating == pytest.approx(4.3, abs=0.01)
+    assert info.ratings_count == 1505
+    assert info.text_reviews_count == 353
+
+
+def test_parse_rating_comma_separator():
+    """Rating '4,3 из 5' with comma handled correctly."""
+    html = "<html><body><div>4,3 из 5  200 оценок</div></body></html>"
+    scraper = YandexNewbuildingScraper()
+    info = scraper.parse(html, jk_slug="x", jk_id="1", source_url="http://x")
+    assert info.rating == pytest.approx(4.3, abs=0.01)
+    assert info.ratings_count == 200
+
+
+# ── Developer link test ───────────────────────────────────────────────────────
+
+
+def test_developer_name_extracted_from_data_test_link():
+    scraper = YandexNewbuildingScraper()
+    info = scraper.parse(
+        TATLIN_FIXTURE_HTML,
+        jk_slug="tatlin",
+        jk_id="1592987",
+        source_url="https://realty.yandex.ru/ekaterinburg/kupit/novostrojka/tatlin-1592987/",
+    )
+    assert info.developer_name == "PRINZIP"
+    assert info.developer_url == "https://realty.yandex.ru/developer/prinzip"
+
+
+def test_developer_absolute_url_kept_as_is():
+    html = """<html><body>
+    <a data-test="CARD_DEV_BADGE_DEVELOPER_LINK"
+       href="https://realty.yandex.ru/developer/abc">АБС Групп</a>
+    </body></html>"""
+    scraper = YandexNewbuildingScraper()
+    info = scraper.parse(html, jk_slug="x", jk_id="1", source_url="http://x")
+    assert info.developer_name == "АБС Групп"
+    assert info.developer_url == "https://realty.yandex.ru/developer/abc"
+
+
+# ── Metro stations test ───────────────────────────────────────────────────────
+
+
+def test_metro_stations_parsed():
+    scraper = YandexNewbuildingScraper()
+    info = scraper.parse(
+        TATLIN_FIXTURE_HTML,
+        jk_slug="tatlin",
+        jk_id="1592987",
+        source_url="https://realty.yandex.ru/ekaterinburg/kupit/novostrojka/tatlin-1592987/",
+    )
+    assert len(info.metro_stations) >= 1
+    uralskaya = next(
+        (s for s in info.metro_stations if "Уральская" in s.name), None
+    )
+    assert uralskaya is not None
+    assert uralskaya.walk_min == 11
+
+
+def test_parse_metro_direct():
+    text = "Уральская 11 мин. Динамо 16 мин."
+    stations = _parse_metro(text)
+    assert len(stations) == 2
+    assert stations[0].name == "Уральская"
+    assert stations[0].walk_min == 11
+    assert stations[1].name == "Динамо"
+    assert stations[1].walk_min == 16
+
+
+def test_parse_metro_caps_at_five():
+    text = (
+        "Альфа 5 мин. Бета 6 мин. Гамма 7 мин. "
+        "Дельта 8 мин. Эпсилон 9 мин. Зета 10 мин."
+    )
+    stations = _parse_metro(text)
+    assert len(stations) == 5
+
+
+# ── No description section test ───────────────────────────────────────────────
+
+
+def test_no_description_section_returns_none():
+    """When no О комплексе / Расположение section, description is None."""
+    scraper = YandexNewbuildingScraper()
+    info = scraper.parse(
+        MINIMAL_FIXTURE_HTML,
+        jk_slug="minimal",
+        jk_id="42",
+        source_url="http://example.com/",
+    )
+    assert info.description is None
+    assert info.house_class is None
+    assert info.commission_year is None
+    assert info.total_floors is None
+
+
+# ── fetch_around raises NotImplementedError ───────────────────────────────────
+
+
+@pytest.mark.asyncio
+async def test_fetch_around_raises_not_implemented():
+    scraper = YandexNewbuildingScraper()
+    with pytest.raises(NotImplementedError, match="JK-slug-based"):
+        await scraper.fetch_around(56.855, 60.576)
+
+
+# ── _find_section_text helper ─────────────────────────────────────────────────
+
+
+def test_find_section_text_returns_content_after_heading():
+    from selectolax.parser import HTMLParser
+
+    html = """<html><body>
+    <h2>О комплексе</h2>
+    <p>Описание комплекса здесь.</p>
+    <p>Ещё абзац.</p>
+    <h2>Другой раздел</h2>
+    <p>Не должен попасть.</p>
+    </body></html>"""
+    tree = HTMLParser(html)
+    result = _find_section_text(tree, "О комплексе")
+    assert result is not None
+    assert "Описание комплекса" in result
+    assert "Не должен" not in result
+
+
+def test_find_section_text_returns_none_when_missing():
+    from selectolax.parser import HTMLParser
+
+    html = "<html><body><p>Текст</p></body></html>"
+    tree = HTMLParser(html)
+    result = _find_section_text(tree, "О комплексе")
+    assert result is None
+
+
+# ── Developer other JK dedup ──────────────────────────────────────────────────
+
+
+def test_developer_other_jk_dedup():
+    """Duplicate JK names are not added twice."""
+    html = """<html><body>
+    <div data-test="CardDevSites">
+      <a href="/1">ЖК Один</a>
+      <a href="/2">ЖК Один</a>
+      <a href="/3">ЖК Два</a>
+    </div>
+    </body></html>"""
+    scraper = YandexNewbuildingScraper()
+    info = scraper.parse(html, jk_slug="x", jk_id="1", source_url="http://x")
+    assert info.developer_other_jk.count("ЖК Один") == 1
+    assert "ЖК Два" in info.developer_other_jk
+
+
+def test_developer_other_jk_capped_at_10():
+    """Only first 10 other JK entries are kept."""
+    links = "\n".join(
+        f'<a href="/{i}">ЖК {i}</a>' for i in range(15)
+    )
+    html = f"""<html><body>
+    <div data-test="CardDevSites">{links}</div>
+    </body></html>"""
+    scraper = YandexNewbuildingScraper()
+    info = scraper.parse(html, jk_slug="x", jk_id="1", source_url="http://x")
+    assert len(info.developer_other_jk) == 10
+
+
+# ── model defaults ────────────────────────────────────────────────────────────
+
+
+def test_yandex_newbuilding_info_defaults():
+    """Model initialises with safe defaults for all optional fields."""
+    info = YandexNewbuildingInfo(ext_id="1", ext_slug="x", source_url="http://x")
+    assert info.name is None
+    assert info.lat is None
+    assert info.lon is None
+    assert info.house_class is None
+    assert info.developer_other_jk == []
+    assert info.metro_stations == []
+    assert info.raw_payload is None
+
+
+def test_jk_metro_station_model():
+    station = JKMetroStation(name="Уральская", walk_min=11)
+    assert station.name == "Уральская"
+    assert station.walk_min == 11
+
+    station_no_time = JKMetroStation(name="Геологическая")
+    assert station_no_time.walk_min is None

tradein-mvp/backend/tests/test_yandex_valuation.py

@@ -0,0 +1,242 @@
+"""Unit tests for YandexValuationScraper — anonymous house-history scraper.
+
+Fixture HTML simulates the Yandex valuation page body text containing:
+- House meta block (year, floors, type, ceiling, lift, total objects, panorama)
+- 2-3 historical offer entries with full structure
+"""
+
+from __future__ import annotations
+
+from datetime import date
+
+import pytest
+
+from app.services.scrapers.yandex_valuation import (
+    YandexValuationResult,
+    YandexValuationScraper,
+)
+
+# ---------------------------------------------------------------------------
+# Fixture helpers
+# ---------------------------------------------------------------------------
+
+_HOUSE_META_BLOCK = (
+    "12 объектов  Дом 1981 года  9 этажей  Панельное здание  2,50 м потолки  Лифт"
+)
+
+_ITEM_1 = (
+    "2-комнатная квартира 45,3 м²  4 этаж  "
+    "Опубликовано 15.03.2023  "
+    "Начальная цена 3 200 000 ₽  117 000 ₽ за м²  "
+    "Последняя цена 3 100 000 ₽  114 000 ₽ за м²  "
+    "Длительность экспозиции 42 дня  В продаже"
+)
+
+_ITEM_2 = (
+    "Студия 25,0 м²  1 этаж  "
+    "Опубликовано 20.11.2022  "
+    "Начальная цена 2 100 000 ₽  84 000 ₽ за м²  "
+    "Последняя цена 1 950 000 ₽  78 000 ₽ за м²  "
+    "Длительность экспозиции 90 дней  Снято"
+)
+
+_ITEM_3 = (
+    "1-комнатная квартира 32,5 м²  7 этаж  "
+    "Опубликовано 01.06.2024  "
+    "Начальная цена 2 800 000 ₽  "
+    "Длительность экспозиции 15 дней  В продаже"
+)
+
+
+def _make_full_html(body_content: str) -> str:
+    return f"<html><body>{body_content}</body></html>"
+
+
+FULL_FIXTURE_HTML = _make_full_html(
+    f"{_HOUSE_META_BLOCK}\n{_ITEM_1}\n{_ITEM_2}\n{_ITEM_3}"
+)
+
+
+# ---------------------------------------------------------------------------
+# House meta tests
+# ---------------------------------------------------------------------------
+
+
+def test_parse_house_meta_full():
+    scraper = YandexValuationScraper()
+    meta = scraper._parse_house_meta(_HOUSE_META_BLOCK)
+    assert meta.year_built == 1981
+    assert meta.total_floors == 9
+    assert meta.house_type == "panel"
+    assert meta.ceiling_height == 2.50
+    assert meta.has_lift is True
+    assert meta.total_objects == 12
+    assert meta.has_panorama is False
+
+
+def test_parse_house_meta_no_lift():
+    text = "5 объектов  Дом 2005 года  16 этажей  Монолитное здание  3,00 м потолки"
+    meta = YandexValuationScraper._parse_house_meta(text)
+    assert meta.has_lift is False
+    assert meta.year_built == 2005
+    assert meta.total_floors == 16
+    assert meta.house_type == "monolith"
+    assert meta.ceiling_height == 3.0
+
+
+def test_parse_house_meta_with_panorama():
+    text = "7 объектов  Дом 2010 года  Панорама  Лифт  Кирпичное здание"
+    meta = YandexValuationScraper._parse_house_meta(text)
+    assert meta.has_panorama is True
+    assert meta.has_lift is True
+    assert meta.house_type == "brick"
+
+
+def test_house_meta_year_not_present_returns_none():
+    text = "8 объектов  5 этажей  Панельное здание"
+    meta = YandexValuationScraper._parse_house_meta(text)
+    assert meta.year_built is None
+    assert meta.total_floors == 5
+    assert meta.total_objects == 8
+
+
+# ---------------------------------------------------------------------------
+# History item tests
+# ---------------------------------------------------------------------------
+
+
+def test_parse_history_item_full():
+    item = YandexValuationScraper._parse_item_text(_ITEM_1)
+    assert item is not None
+    assert item.area_m2 == 45.3
+    assert item.rooms == 2
+    assert item.floor == 4
+    assert item.publish_date == date(2023, 3, 15)
+    assert item.start_price == 3_200_000
+    assert item.last_price == 3_100_000
+    assert item.exposure_days == 42
+    assert item.status is not None
+    assert "продаже" in item.status.lower() or "В" in item.status
+
+
+def test_parse_history_item_studio_rooms_zero():
+    item = YandexValuationScraper._parse_item_text(_ITEM_2)
+    assert item is not None
+    assert item.rooms == 0  # Studio
+    assert item.area_m2 == 25.0
+    assert item.floor == 1
+    assert item.publish_date == date(2022, 11, 20)
+
+
+def test_parse_history_item_status_sold():
+    item = YandexValuationScraper._parse_item_text(_ITEM_2)
+    assert item is not None
+    assert item.status is not None
+    assert "нят" in item.status.lower() or "Снят" in item.status
+
+
+def test_parse_history_item_returns_none_for_empty():
+    assert YandexValuationScraper._parse_item_text("") is None
+    assert YandexValuationScraper._parse_item_text("   ") is None
+
+
+def test_parse_history_item_returns_none_without_area_or_price():
+    # Text too short / no extractable fields
+    assert YandexValuationScraper._parse_item_text("нет данных") is None
+
+
+# ---------------------------------------------------------------------------
+# Chunked text extraction tests
+# ---------------------------------------------------------------------------
+
+
+def test_parse_items_chunked_dedup():
+    """Duplicate entry (same date + area + floor) must appear only once."""
+    duplicate_block = f"{_ITEM_1}\n{_ITEM_1}"
+    items = YandexValuationScraper._parse_items_from_chunked_text(duplicate_block)
+    # Both items reference 15.03.2023 + 45.3 m2 + floor 4 — must dedup to 1
+    matching = [
+        i for i in items if i.area_m2 == 45.3 and i.publish_date == date(2023, 3, 15)
+    ]
+    assert len(matching) == 1
+
+
+def test_parse_items_from_chunked_text_no_dates_returns_empty():
+    """Body text with no DD.MM.YYYY dates should return empty list."""
+    items = YandexValuationScraper._parse_items_from_chunked_text(
+        "Нет объявлений. Попробуйте изменить параметры поиска."
+    )
+    assert items == []
+
+
+def test_parse_items_from_chunked_text_multiple_items():
+    """Three distinct items should be extracted from combined body text."""
+    body = f"{_ITEM_1}\n{_ITEM_2}\n{_ITEM_3}"
+    items = YandexValuationScraper._parse_items_from_chunked_text(body)
+    assert len(items) >= 2  # at minimum items 1 and 2 (item 3 has only start_price)
+
+
+# ---------------------------------------------------------------------------
+# Full parse round-trip
+# ---------------------------------------------------------------------------
+
+
+def test_parse_full_result_address_propagated():
+    scraper = YandexValuationScraper()
+    result = scraper.parse(
+        FULL_FIXTURE_HTML,
+        address="Екатеринбург, ул. Ленина, 1",
+        offer_category="APARTMENT",
+        offer_type="SELL",
+        page=1,
+        source_url="https://realty.yandex.ru/otsenka-kvartiry-po-adresu-onlayn/?page=1",
+    )
+    assert isinstance(result, YandexValuationResult)
+    assert result.address == "Екатеринбург, ул. Ленина, 1"
+    assert result.offer_category == "APARTMENT"
+    assert result.offer_type == "SELL"
+    assert result.page == 1
+    assert result.house.year_built == 1981
+    assert isinstance(result.history_items, list)
+    assert result.raw_payload is not None
+    assert "body_len" in result.raw_payload
+
+
+def test_parse_full_result_house_meta_populated():
+    scraper = YandexValuationScraper()
+    result = scraper.parse(
+        FULL_FIXTURE_HTML,
+        address="test",
+        offer_category="APARTMENT",
+        offer_type="SELL",
+        page=1,
+        source_url="https://realty.yandex.ru/",
+    )
+    assert result.house.total_floors == 9
+    assert result.house.has_lift is True
+    assert result.house.house_type == "panel"
+    assert result.house.ceiling_height == 2.5
+
+
+# ---------------------------------------------------------------------------
+# fetch_around raises NotImplementedError
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_fetch_around_raises_not_implemented():
+    scraper = YandexValuationScraper()
+    with pytest.raises(NotImplementedError):
+        await scraper.fetch_around(lat=56.8, lon=60.6)
+
+
+# ---------------------------------------------------------------------------
+# Ceiling height comma parsing
+# ---------------------------------------------------------------------------
+
+
+def test_parse_ceiling_height_comma():
+    """Comma decimal separator '2,70 м потолки' must parse to 2.7."""
+    text = "Дом 1999 года  5 этажей  Кирпичное здание  2,70 м потолки  Лифт"
+    meta = YandexValuationScraper._parse_house_meta(text)
+    assert meta.ceiling_height == 2.7