tradein-mvp/backend/app/services/scrapers/base.py

@@ -101,7 +101,9 @@ class ScrapedLot(BaseModel):
 
     # Метаданные
     listing_date: date | None = None
+    publish_date: date | None = None
     days_on_market: int | None = None
+    description: str | None = None
     photo_urls: list[str] = Field(default_factory=list)
     raw_payload: dict[str, Any] | None = None
 
@@ -119,6 +121,15 @@ class ScrapedLot(BaseModel):
     bargain_allowed: bool | None = None
     sale_type: str | None = None
     metro_stations: list[dict] = Field(default_factory=list)
+    agency_name: str | None = None
+
+    # Yandex gate-API rich fields
+    yandex_offer_id: str | None = None  # numeric string offerId (kept as text)
+    predicted_price_rub: int | None = None  # Yandex per-offer valuation (predictedPrice.value)
+    predicted_price_min: int | None = None
+    predicted_price_max: int | None = None
+    price_trend: str | None = None  # INCREASED / DECREASED / UNCHANGED / ...
+    price_previous_rub: int | None = None  # price.previous
 
     def compute_dedup_hash(self) -> str:
         """SHA256(source + stable_key) — стабильный uniqueness key.
@@ -252,11 +263,15 @@ def save_listings(
                     house_type, repair_state, has_balcony,
                     house_source, house_ext_id, house_url, listing_segment,
                     price_rub, price_per_m2,
-                    listing_date, days_on_market, photo_urls, raw_payload,
+                    listing_date, publish_date, days_on_market, description,
+                    photo_urls, raw_payload,
                     living_area_m2, bedrooms_count, balconies_count, loggias_count,
                     description_minhash, cadastral_number, building_cadastral_number,
                     phones, is_homeowner, is_pro_seller,
-                    bargain_allowed, sale_type, metro_stations,
+                    bargain_allowed, sale_type, metro_stations, agency_name,
+                    yandex_offer_id, predicted_price_rub,
+                    predicted_price_min, predicted_price_max,
+                    price_trend, price_previous_rub,
                     geo_precision,
                     scraped_at, last_seen_at
                 ) VALUES (
@@ -266,13 +281,16 @@ def save_listings(
                     :house_type, :repair_state, :has_balcony,
                     :house_source, :house_ext_id, :house_url, :listing_segment,
                     :price_rub, :ppm2,
-                    :listing_date, :days_on_market,
+                    :listing_date, :publish_date, :days_on_market, :description,
                     CAST(:photos AS jsonb),
                     CAST(:raw AS jsonb),
                     :living_area_m2, :bedrooms_count, :balconies_count, :loggias_count,
                     :description_minhash, :cadastral_number, :building_cadastral_number,
                     CAST(:phones AS jsonb), :is_homeowner, :is_pro_seller,
-                    :bargain_allowed, :sale_type, CAST(:metro_stations AS jsonb),
+                    :bargain_allowed, :sale_type, CAST(:metro_stations AS jsonb), :agency_name,
+                    :yandex_offer_id, :predicted_price_rub,
+                    :predicted_price_min, :predicted_price_max,
+                    :price_trend, :price_previous_rub,
                     :geo_precision,
                     NOW(), NOW()
                 )
@@ -297,7 +315,32 @@ def save_listings(
                         sale_type = EXCLUDED.sale_type,
                         metro_stations = EXCLUDED.metro_stations,
                         listing_date = COALESCE(EXCLUDED.listing_date, listings.listing_date),
-                        area_m2 = COALESCE(EXCLUDED.area_m2, listings.area_m2)
+                        area_m2 = COALESCE(EXCLUDED.area_m2, listings.area_m2),
+                        -- Yandex rich fields (+ shared description/agency/publish_date):
+                        -- COALESCE so a source that does not provide them never wipes
+                        -- a value previously written by another source / scrape.
+                        publish_date = COALESCE(EXCLUDED.publish_date, listings.publish_date),
+                        days_on_market = COALESCE(
+                            EXCLUDED.days_on_market, listings.days_on_market
+                        ),
+                        description = COALESCE(EXCLUDED.description, listings.description),
+                        agency_name = COALESCE(EXCLUDED.agency_name, listings.agency_name),
+                        yandex_offer_id = COALESCE(
+                            EXCLUDED.yandex_offer_id, listings.yandex_offer_id
+                        ),
+                        predicted_price_rub = COALESCE(
+                            EXCLUDED.predicted_price_rub, listings.predicted_price_rub
+                        ),
+                        predicted_price_min = COALESCE(
+                            EXCLUDED.predicted_price_min, listings.predicted_price_min
+                        ),
+                        predicted_price_max = COALESCE(
+                            EXCLUDED.predicted_price_max, listings.predicted_price_max
+                        ),
+                        price_trend = COALESCE(EXCLUDED.price_trend, listings.price_trend),
+                        price_previous_rub = COALESCE(
+                            EXCLUDED.price_previous_rub, listings.price_previous_rub
+                        )
                 RETURNING id, (xmax = 0) AS inserted
                 """
             ),
@@ -324,7 +367,9 @@ def save_listings(
                 "price_rub": lot.price_rub,
                 "ppm2": ppm2,
                 "listing_date": lot.listing_date,
+                "publish_date": lot.publish_date,
                 "days_on_market": lot.days_on_market,
+                "description": lot.description,
                 "photos": _to_json(lot.photo_urls),
                 "raw": _to_json(lot.raw_payload) if lot.raw_payload else None,
                 # Cian-specific columns — None for non-Cian scrapers (→ SQL NULL)
@@ -341,6 +386,13 @@ def save_listings(
                 "bargain_allowed": lot.bargain_allowed,
                 "sale_type": lot.sale_type,
                 "metro_stations": _to_json(lot.metro_stations) if lot.metro_stations else None,
+                "agency_name": lot.agency_name,
+                "yandex_offer_id": lot.yandex_offer_id,
+                "predicted_price_rub": lot.predicted_price_rub,
+                "predicted_price_min": lot.predicted_price_min,
+                "predicted_price_max": lot.predicted_price_max,
+                "price_trend": lot.price_trend,
+                "price_previous_rub": lot.price_previous_rub,
                 "geo_precision": lot.geo_precision,
             },
         ).fetchone()

tradein-mvp/backend/app/services/scrapers/yandex_realty.py

@@ -32,6 +32,7 @@ import logging
 import math
 import re
 from dataclasses import dataclass
+from datetime import UTC, date, datetime
 from typing import Any
 from urllib.parse import urlencode
 
@@ -134,6 +135,91 @@ def _segment_for(new_flat: str) -> str:
     return "novostroyki" if new_flat == "YES" else "vtorichka"
 
 
+def _parse_creation_date(raw: Any) -> tuple[date | None, int | None]:
+    """Parse gate-API creationDate ('2026-05-24T15:10:27Z') → (date, days_on_market).
+
+    days_on_market = floor((now - creationDate).days), clamped to >= 0.
+    Returns (None, None) when raw is absent or unparseable (never raises).
+    """
+    if not raw or not isinstance(raw, str):
+        return None, None
+    txt = raw.strip()
+    # datetime.fromisoformat handles 'Z' only from py3.11+, but normalise defensively.
+    if txt.endswith("Z"):
+        txt = txt[:-1] + "+00:00"
+    try:
+        dt = datetime.fromisoformat(txt)
+    except ValueError:
+        return None, None
+    if dt.tzinfo is None:
+        dt = dt.replace(tzinfo=UTC)
+    listing_date = dt.date()
+    delta_days = (datetime.now(UTC) - dt).days
+    days_on_market = max(delta_days, 0)
+    return listing_date, days_on_market
+
+
+def _author_fields(author: dict[str, Any]) -> tuple[bool | None, bool | None, str | None]:
+    """Map entity.author → (is_homeowner, is_pro_seller, agency_name).
+
+    author.category ∈ {OWNER, AGENCY, AGENT, ...}.
+      OWNER          → is_homeowner=True
+      AGENCY / AGENT → is_pro_seller=True, agency_name = organization or agentName
+    Returns (None, None, None) when author/category absent.
+    """
+    category = (author.get("category") or "").upper()
+    if not category:
+        return None, None, None
+    if category == "OWNER":
+        return True, None, None
+    if category in {"AGENCY", "AGENT"}:
+        agency_name = author.get("organization") or author.get("agentName") or None
+        return None, True, agency_name
+    return None, None, None
+
+
+def _to_bigint(raw: Any) -> int | None:
+    """Coerce a gate-API numeric value (str or number) to int. None when unparseable."""
+    if raw is None:
+        return None
+    try:
+        return int(float(raw))
+    except (TypeError, ValueError):
+        return None
+
+
+def _predicted_prices(entity: dict[str, Any]) -> tuple[int | None, int | None, int | None]:
+    """Extract (value, min, max) from entity.predictedPrice.predictedPrice.
+
+    Yandex's own per-offer valuation. Fields are strings (e.g. "7554000").
+    Returns (None, None, None) when the nested path is absent.
+    """
+    outer = entity.get("predictedPrice") or {}
+    inner = outer.get("predictedPrice") or {}
+    if not isinstance(inner, dict):
+        return None, None, None
+    return (
+        _to_bigint(inner.get("value")),
+        _to_bigint(inner.get("min")),
+        _to_bigint(inner.get("max")),
+    )
+
+
+def _metro_stations(location: dict[str, Any]) -> list[dict[str, Any]]:
+    """Build metro_stations [{name, min}] from location.metroList[] (+ location.metro)."""
+    stations: list[dict[str, Any]] = []
+    raw_list = location.get("metroList")
+    if isinstance(raw_list, list):
+        for m in raw_list:
+            if isinstance(m, dict) and m.get("name"):
+                stations.append({"name": m.get("name"), "min": m.get("timeToMetro")})
+    if not stations:
+        single = location.get("metro")
+        if isinstance(single, dict) and single.get("name"):
+            stations.append({"name": single.get("name"), "min": single.get("timeToMetro")})
+    return stations
+
+
 def _parse_gate_json(
     payload: dict[str, Any], page_param: int = 1, new_flat: str = "NO"
 ) -> list[ScrapedLot]:
@@ -158,6 +244,13 @@ def _parse_gate_json(
       mainImages[]                    -> photo_urls (prepend "https:", up to 5)
       building.siteId                 -> house_ext_id (JK linkage)
       listing_segment                 -> "novostroyki" (newFlat=YES) | "vtorichka" (newFlat=NO)
+      creationDate (ISO Z)            -> listing_date / publish_date + days_on_market
+      description                     -> description
+      author.category/organization   -> is_homeowner / is_pro_seller / agency_name
+      predictedPrice.predictedPrice.* -> predicted_price_rub / _min / _max
+      price.trend / price.previous    -> price_trend / price_previous_rub
+      location.metroList[]            -> metro_stations [{name, min}]
+      offerId                         -> yandex_offer_id
     """
     result = _extract_gate_data(payload)
     if result is None:
@@ -245,6 +338,20 @@ def _entity_to_lot(
 
         address = location.get("geocoderAddress") or location.get("streetAddress") or None
 
+        # ── Rich gate-API fields ───────────────────────────────────────────
+        listing_date, days_on_market = _parse_creation_date(entity.get("creationDate"))
+        description = entity.get("description") or None
+
+        author = entity.get("author") or {}
+        is_homeowner, is_pro_seller, agency_name = _author_fields(author)
+
+        predicted_value, predicted_min, predicted_max = _predicted_prices(entity)
+
+        price_trend = price_dict.get("trend") or None
+        price_previous_rub = _to_bigint(price_dict.get("previous"))
+
+        metro_stations = _metro_stations(location)
+
         photo_urls: list[str] = []
         for img in entity.get("mainImages") or []:
             if img:
@@ -273,6 +380,20 @@ def _entity_to_lot(
             house_ext_id=house_ext_id,
             house_url=None,
             listing_segment=_segment_for(new_flat),
+            listing_date=listing_date,
+            publish_date=listing_date,
+            days_on_market=days_on_market,
+            description=description,
+            is_homeowner=is_homeowner,
+            is_pro_seller=is_pro_seller,
+            agency_name=agency_name,
+            metro_stations=metro_stations,
+            yandex_offer_id=offer_id,
+            predicted_price_rub=predicted_value,
+            predicted_price_min=predicted_min,
+            predicted_price_max=predicted_max,
+            price_trend=price_trend,
+            price_previous_rub=price_previous_rub,
             raw_payload={
                 "page_param": page_param,
                 "ceiling_height": ceiling_height,

tradein-mvp/backend/data/sql/121_yandex_rich_fields.sql

@@ -0,0 +1,30 @@
+-- Migration 121: Yandex gate-API rich entity fields the parser previously dropped.
+--
+-- Источник: realty.yandex.ru/gate/react-page/get/ entity object содержит ~90 полей,
+-- из которых маппилось ~11. Эта миграция добавляет колонки под high-value поля:
+--
+--   predictedPrice.predictedPrice.{value,min,max} → собственная оценка Яндекса
+--       per-offer (рыночная цена) — headline-выигрыш для estimator/анализа bargain.
+--   price.trend / price.previous → динамика цены объявления.
+--
+-- description / listing_date / publish_date / days_on_market / metro_stations /
+-- is_homeowner / is_pro_seller / agency_name / yandex_offer_id — уже существуют
+-- в listings (миграции 011 / 033 и др.); эта миграция их НЕ трогает, парсер просто
+-- начинает их наполнять (см. base.py save_listings + yandex_realty._entity_to_lot).
+--
+-- Idempotent: ADD COLUMN IF NOT EXISTS. Авто-применяется на деплое.
+
+BEGIN;
+
+ALTER TABLE listings ADD COLUMN IF NOT EXISTS predicted_price_rub bigint;
+ALTER TABLE listings ADD COLUMN IF NOT EXISTS predicted_price_min bigint;
+ALTER TABLE listings ADD COLUMN IF NOT EXISTS predicted_price_max bigint;
+ALTER TABLE listings ADD COLUMN IF NOT EXISTS price_trend       text;
+ALTER TABLE listings ADD COLUMN IF NOT EXISTS price_previous_rub bigint;
+
+COMMENT ON COLUMN listings.predicted_price_rub IS
+  'Yandex per-offer valuation (predictedPrice.predictedPrice.value), RUB';
+COMMENT ON COLUMN listings.price_trend IS
+  'Yandex price.trend: INCREASED / DECREASED / UNCHANGED / ...';
+
+COMMIT;

tradein-mvp/backend/tests/test_base_save_listings.py

@@ -594,3 +594,104 @@ def test_save_listings_null_area_and_date_pass_none_in_params():
     params = _get_execute_params(db)
     assert params["area_m2"] is None
     assert params["listing_date"] is None
+
+
+# ── Tests 18-20: Yandex rich fields (predictedPrice, trend, description, agency) ──
+
+
+def _yandex_rich_lot(**overrides) -> ScrapedLot:
+    """ScrapedLot с заполненными Yandex rich-полями."""
+    import datetime
+
+    base = {
+        "source": "yandex",
+        "source_url": "https://realty.yandex.ru/offer/5500000000000000001/",
+        "source_id": "5500000000000000001",
+        "price_rub": 7_200_000,
+        "address": "Екатеринбург, улица Мира, 10",
+        "rooms": 2,
+        "area_m2": 48.0,
+        "listing_date": datetime.date(2026, 5, 24),
+        "publish_date": datetime.date(2026, 5, 24),
+        "days_on_market": 24,
+        "description": "Светлая квартира.",
+        "is_homeowner": True,
+        "agency_name": None,
+        "metro_stations": [{"name": "Динамо", "min": 7}],
+        "yandex_offer_id": "5500000000000000001",
+        "predicted_price_rub": 7_554_000,
+        "predicted_price_min": 7_100_000,
+        "predicted_price_max": 8_000_000,
+        "price_trend": "DECREASED",
+        "price_previous_rub": 7_500_000,
+    }
+    base.update(overrides)
+    return ScrapedLot(**base)
+
+
+def test_save_listings_persists_yandex_rich_fields():
+    """Yandex rich-поля попадают в params INSERT."""
+    db = _mock_db(inserted=True)
+    lot = _yandex_rich_lot()
+
+    save_listings(db, [lot])
+
+    params = _get_execute_params(db)
+    assert params["description"] == "Светлая квартира."
+    assert params["days_on_market"] == 24
+    assert params["yandex_offer_id"] == "5500000000000000001"
+    assert params["predicted_price_rub"] == 7_554_000
+    assert params["predicted_price_min"] == 7_100_000
+    assert params["predicted_price_max"] == 8_000_000
+    assert params["price_trend"] == "DECREASED"
+    assert params["price_previous_rub"] == 7_500_000
+    assert params["agency_name"] is None
+    metro = json.loads(params["metro_stations"])
+    assert metro == [{"name": "Динамо", "min": 7}]
+
+
+def test_save_listings_yandex_rich_cols_in_sql():
+    """INSERT + ON CONFLICT SET содержат новые Yandex rich колонки."""
+    db = _mock_db(inserted=False)
+    lot = _yandex_rich_lot(agency_name="Агентство «Диал»", is_pro_seller=True, is_homeowner=None)
+
+    save_listings(db, [lot])
+
+    sql_text = _get_listings_insert_sql(db)
+    for col in (
+        "description",
+        "publish_date",
+        "agency_name",
+        "yandex_offer_id",
+        "predicted_price_rub",
+        "predicted_price_min",
+        "predicted_price_max",
+        "price_trend",
+        "price_previous_rub",
+    ):
+        assert col in sql_text, f"Column '{col}' missing from INSERT SQL"
+
+    # COALESCE backfill in DO UPDATE SET (never wipe a prior value)
+    assert "description = COALESCE(EXCLUDED.description, listings.description)" in sql_text
+    assert "agency_name = COALESCE(EXCLUDED.agency_name, listings.agency_name)" in sql_text
+    assert "predicted_price_rub = COALESCE(" in sql_text
+    assert "price_trend = COALESCE(EXCLUDED.price_trend, listings.price_trend)" in sql_text
+
+
+def test_save_listings_non_yandex_rich_cols_are_null():
+    """Avito lot не имеет Yandex rich-полей → params = None (backward compat)."""
+    db = _mock_db(inserted=True)
+    lot = _base_lot(source="avito", source_id="987654")
+
+    save_listings(db, [lot])
+
+    params = _get_execute_params(db)
+    assert params["description"] is None
+    assert params["publish_date"] is None
+    assert params["agency_name"] is None
+    assert params["yandex_offer_id"] is None
+    assert params["predicted_price_rub"] is None
+    assert params["predicted_price_min"] is None
+    assert params["predicted_price_max"] is None
+    assert params["price_trend"] is None
+    assert params["price_previous_rub"] is None

tradein-mvp/backend/tests/test_yandex_realty_serp.py

@@ -100,6 +100,58 @@ _ENTITY_NO_OFFER_ID: dict = {
 }
 
 
+# Entity carrying the rich gate-API fields (creationDate, description, author,
+# predictedPrice, price.trend/previous, location.metroList) — verified live 2026-06-17.
+_ENTITY_RICH_OWNER: dict = {
+    "offerId": 5500000000000000001,
+    "url": "//realty.yandex.ru/offer/5500000000000000001",
+    "creationDate": "2026-05-24T15:10:27Z",
+    "description": "Продаётся светлая квартира с видом на парк.",
+    "author": {"category": "OWNER", "agentName": "Иван", "organization": None},
+    "price": {
+        "value": 7200000,
+        "valuePerPart": 150000,
+        "trend": "DECREASED",
+        "previous": 7500000,
+    },
+    "predictedPrice": {"predictedPrice": {"value": "7554000", "min": "7100000", "max": "8000000"}},
+    "area": {"value": 48.0},
+    "roomsTotal": 2,
+    "floorsOffered": [5],
+    "floorsTotal": 12,
+    "building": {"builtYear": 2018, "buildingType": "MONOLIT"},
+    "location": {
+        "geocoderAddress": "Ekaterinburg, ulitsa Mira, 10",
+        "point": {"latitude": 56.84, "longitude": 60.61},
+        "metroList": [
+            {"name": "Ploshchad 1905 goda", "timeToMetro": 7},
+            {"name": "Geologicheskaya", "timeToMetro": 12},
+        ],
+    },
+    "mainImages": [],
+}
+
+_ENTITY_RICH_AGENCY: dict = {
+    "offerId": 5500000000000000002,
+    "url": "//realty.yandex.ru/offer/5500000000000000002",
+    "creationDate": "2026-06-01T09:00:00Z",
+    "description": "Агентская продажа.",
+    "author": {"category": "AGENCY", "agentName": "Петров", "organization": "Агентство «Диал»"},
+    "price": {"value": 9900000, "trend": "UNCHANGED"},
+    "area": {"value": 60.0},
+    "roomsTotal": 3,
+    "floorsOffered": [3],
+    "floorsTotal": 9,
+    "building": {"builtYear": 2012, "buildingType": "BRICK"},
+    "location": {
+        "geocoderAddress": "Ekaterinburg, ulitsa Lenina, 1",
+        "point": {"latitude": 56.83, "longitude": 60.6},
+        "metro": {"name": "Dinamo", "timeToMetro": 4},
+    },
+    "mainImages": [],
+}
+
+
 def _make_gate_payload(entities: list, pager: dict | None = None) -> dict:
     """Wrap entities + pager into gate-API response shape."""
     if pager is None:
@@ -271,6 +323,88 @@ def test_entity_to_lot_page_param_in_raw():
     assert lot.raw_payload["page_param"] == 3
 
 
+# ---------------------------------------------------------------------------
+# PART B2: rich gate-API fields (predictedPrice, seller, dates, trend, metro)
+# ---------------------------------------------------------------------------
+
+
+def test_entity_to_lot_rich_owner_fields():
+    lot = _entity_to_lot(_ENTITY_RICH_OWNER)
+    assert lot is not None
+    # creationDate → listing_date / publish_date / days_on_market
+    assert lot.listing_date is not None
+    assert lot.listing_date.isoformat() == "2026-05-24"
+    assert lot.publish_date == lot.listing_date
+    assert lot.days_on_market is not None
+    assert lot.days_on_market >= 0
+    # description
+    assert lot.description == "Продаётся светлая квартира с видом на парк."
+    # author OWNER
+    assert lot.is_homeowner is True
+    assert lot.is_pro_seller is None
+    assert lot.agency_name is None
+    # predictedPrice
+    assert lot.predicted_price_rub == 7_554_000
+    assert lot.predicted_price_min == 7_100_000
+    assert lot.predicted_price_max == 8_000_000
+    # price trend / previous
+    assert lot.price_trend == "DECREASED"
+    assert lot.price_previous_rub == 7_500_000
+    # metro_stations from metroList
+    assert lot.metro_stations == [
+        {"name": "Ploshchad 1905 goda", "min": 7},
+        {"name": "Geologicheskaya", "min": 12},
+    ]
+    # yandex_offer_id
+    assert lot.yandex_offer_id == "5500000000000000001"
+
+
+def test_entity_to_lot_rich_agency_fields():
+    lot = _entity_to_lot(_ENTITY_RICH_AGENCY)
+    assert lot is not None
+    # AGENCY → pro seller + agency_name from organization
+    assert lot.is_pro_seller is True
+    assert lot.is_homeowner is None
+    assert lot.agency_name == "Агентство «Диал»"
+    # price.trend present, no previous → price_previous_rub None
+    assert lot.price_trend == "UNCHANGED"
+    assert lot.price_previous_rub is None
+    # no predictedPrice path → all None
+    assert lot.predicted_price_rub is None
+    assert lot.predicted_price_min is None
+    assert lot.predicted_price_max is None
+    # single location.metro fallback
+    assert lot.metro_stations == [{"name": "Dinamo", "min": 4}]
+
+
+def test_entity_to_lot_no_rich_fields_are_none():
+    """Base fixture lacks rich fields → all new fields default to None/[] (no crash)."""
+    lot = _entity_to_lot(_ENTITY_FULL)
+    assert lot is not None
+    assert lot.listing_date is None
+    assert lot.days_on_market is None
+    assert lot.description is None
+    assert lot.is_homeowner is None
+    assert lot.is_pro_seller is None
+    assert lot.agency_name is None
+    assert lot.predicted_price_rub is None
+    assert lot.price_trend is None
+    assert lot.price_previous_rub is None
+    assert lot.metro_stations == []
+    # yandex_offer_id is always populated from offerId
+    assert lot.yandex_offer_id == "4740475460451078271"
+
+
+def test_entity_to_lot_bad_creation_date_is_safe():
+    """Unparseable creationDate → listing_date/days_on_market None, no exception."""
+    entity = dict(_ENTITY_RICH_OWNER)
+    entity["creationDate"] = "not-a-date"
+    lot = _entity_to_lot(entity)
+    assert lot is not None
+    assert lot.listing_date is None
+    assert lot.days_on_market is None
+
+
 # ---------------------------------------------------------------------------
 # PART C: _parse_gate_json (list-level)
 # ---------------------------------------------------------------------------