From 1fe670ea6a1f7e3805225014b97a5bb07fdf8da6 Mon Sep 17 00:00:00 2001 From: bot-backend Date: Wed, 17 Jun 2026 22:58:16 +0300 Subject: [PATCH] feat(yandex): map rich entity fields (predictedPrice, seller, listing_date, trend, metro) --- .../backend/app/services/scrapers/base.py | 62 +++++++- .../app/services/scrapers/yandex_realty.py | 121 ++++++++++++++++ .../data/sql/121_yandex_rich_fields.sql | 30 ++++ .../backend/tests/test_base_save_listings.py | 101 +++++++++++++ .../backend/tests/test_yandex_realty_serp.py | 134 ++++++++++++++++++ 5 files changed, 443 insertions(+), 5 deletions(-) create mode 100644 tradein-mvp/backend/data/sql/121_yandex_rich_fields.sql diff --git a/tradein-mvp/backend/app/services/scrapers/base.py b/tradein-mvp/backend/app/services/scrapers/base.py index 14c5310d..9fe05390 100644 --- a/tradein-mvp/backend/app/services/scrapers/base.py +++ b/tradein-mvp/backend/app/services/scrapers/base.py @@ -101,7 +101,9 @@ class ScrapedLot(BaseModel): # Метаданные listing_date: date | None = None + publish_date: date | None = None days_on_market: int | None = None + description: str | None = None photo_urls: list[str] = Field(default_factory=list) raw_payload: dict[str, Any] | None = None @@ -119,6 +121,15 @@ class ScrapedLot(BaseModel): bargain_allowed: bool | None = None sale_type: str | None = None metro_stations: list[dict] = Field(default_factory=list) + agency_name: str | None = None + + # Yandex gate-API rich fields + yandex_offer_id: str | None = None # numeric string offerId (kept as text) + predicted_price_rub: int | None = None # Yandex per-offer valuation (predictedPrice.value) + predicted_price_min: int | None = None + predicted_price_max: int | None = None + price_trend: str | None = None # INCREASED / DECREASED / UNCHANGED / ... + price_previous_rub: int | None = None # price.previous def compute_dedup_hash(self) -> str: """SHA256(source + stable_key) — стабильный uniqueness key. @@ -252,11 +263,15 @@ def save_listings( house_type, repair_state, has_balcony, house_source, house_ext_id, house_url, listing_segment, price_rub, price_per_m2, - listing_date, days_on_market, photo_urls, raw_payload, + listing_date, publish_date, days_on_market, description, + photo_urls, raw_payload, living_area_m2, bedrooms_count, balconies_count, loggias_count, description_minhash, cadastral_number, building_cadastral_number, phones, is_homeowner, is_pro_seller, - bargain_allowed, sale_type, metro_stations, + bargain_allowed, sale_type, metro_stations, agency_name, + yandex_offer_id, predicted_price_rub, + predicted_price_min, predicted_price_max, + price_trend, price_previous_rub, geo_precision, scraped_at, last_seen_at ) VALUES ( @@ -266,13 +281,16 @@ def save_listings( :house_type, :repair_state, :has_balcony, :house_source, :house_ext_id, :house_url, :listing_segment, :price_rub, :ppm2, - :listing_date, :days_on_market, + :listing_date, :publish_date, :days_on_market, :description, CAST(:photos AS jsonb), CAST(:raw AS jsonb), :living_area_m2, :bedrooms_count, :balconies_count, :loggias_count, :description_minhash, :cadastral_number, :building_cadastral_number, CAST(:phones AS jsonb), :is_homeowner, :is_pro_seller, - :bargain_allowed, :sale_type, CAST(:metro_stations AS jsonb), + :bargain_allowed, :sale_type, CAST(:metro_stations AS jsonb), :agency_name, + :yandex_offer_id, :predicted_price_rub, + :predicted_price_min, :predicted_price_max, + :price_trend, :price_previous_rub, :geo_precision, NOW(), NOW() ) @@ -297,7 +315,32 @@ def save_listings( sale_type = EXCLUDED.sale_type, metro_stations = EXCLUDED.metro_stations, listing_date = COALESCE(EXCLUDED.listing_date, listings.listing_date), - area_m2 = COALESCE(EXCLUDED.area_m2, listings.area_m2) + area_m2 = COALESCE(EXCLUDED.area_m2, listings.area_m2), + -- Yandex rich fields (+ shared description/agency/publish_date): + -- COALESCE so a source that does not provide them never wipes + -- a value previously written by another source / scrape. + publish_date = COALESCE(EXCLUDED.publish_date, listings.publish_date), + days_on_market = COALESCE( + EXCLUDED.days_on_market, listings.days_on_market + ), + description = COALESCE(EXCLUDED.description, listings.description), + agency_name = COALESCE(EXCLUDED.agency_name, listings.agency_name), + yandex_offer_id = COALESCE( + EXCLUDED.yandex_offer_id, listings.yandex_offer_id + ), + predicted_price_rub = COALESCE( + EXCLUDED.predicted_price_rub, listings.predicted_price_rub + ), + predicted_price_min = COALESCE( + EXCLUDED.predicted_price_min, listings.predicted_price_min + ), + predicted_price_max = COALESCE( + EXCLUDED.predicted_price_max, listings.predicted_price_max + ), + price_trend = COALESCE(EXCLUDED.price_trend, listings.price_trend), + price_previous_rub = COALESCE( + EXCLUDED.price_previous_rub, listings.price_previous_rub + ) RETURNING id, (xmax = 0) AS inserted """ ), @@ -324,7 +367,9 @@ def save_listings( "price_rub": lot.price_rub, "ppm2": ppm2, "listing_date": lot.listing_date, + "publish_date": lot.publish_date, "days_on_market": lot.days_on_market, + "description": lot.description, "photos": _to_json(lot.photo_urls), "raw": _to_json(lot.raw_payload) if lot.raw_payload else None, # Cian-specific columns — None for non-Cian scrapers (→ SQL NULL) @@ -341,6 +386,13 @@ def save_listings( "bargain_allowed": lot.bargain_allowed, "sale_type": lot.sale_type, "metro_stations": _to_json(lot.metro_stations) if lot.metro_stations else None, + "agency_name": lot.agency_name, + "yandex_offer_id": lot.yandex_offer_id, + "predicted_price_rub": lot.predicted_price_rub, + "predicted_price_min": lot.predicted_price_min, + "predicted_price_max": lot.predicted_price_max, + "price_trend": lot.price_trend, + "price_previous_rub": lot.price_previous_rub, "geo_precision": lot.geo_precision, }, ).fetchone() diff --git a/tradein-mvp/backend/app/services/scrapers/yandex_realty.py b/tradein-mvp/backend/app/services/scrapers/yandex_realty.py index a9310ce8..64792c94 100644 --- a/tradein-mvp/backend/app/services/scrapers/yandex_realty.py +++ b/tradein-mvp/backend/app/services/scrapers/yandex_realty.py @@ -32,6 +32,7 @@ import logging import math import re from dataclasses import dataclass +from datetime import UTC, date, datetime from typing import Any from urllib.parse import urlencode @@ -134,6 +135,91 @@ def _segment_for(new_flat: str) -> str: return "novostroyki" if new_flat == "YES" else "vtorichka" +def _parse_creation_date(raw: Any) -> tuple[date | None, int | None]: + """Parse gate-API creationDate ('2026-05-24T15:10:27Z') → (date, days_on_market). + + days_on_market = floor((now - creationDate).days), clamped to >= 0. + Returns (None, None) when raw is absent or unparseable (never raises). + """ + if not raw or not isinstance(raw, str): + return None, None + txt = raw.strip() + # datetime.fromisoformat handles 'Z' only from py3.11+, but normalise defensively. + if txt.endswith("Z"): + txt = txt[:-1] + "+00:00" + try: + dt = datetime.fromisoformat(txt) + except ValueError: + return None, None + if dt.tzinfo is None: + dt = dt.replace(tzinfo=UTC) + listing_date = dt.date() + delta_days = (datetime.now(UTC) - dt).days + days_on_market = max(delta_days, 0) + return listing_date, days_on_market + + +def _author_fields(author: dict[str, Any]) -> tuple[bool | None, bool | None, str | None]: + """Map entity.author → (is_homeowner, is_pro_seller, agency_name). + + author.category ∈ {OWNER, AGENCY, AGENT, ...}. + OWNER → is_homeowner=True + AGENCY / AGENT → is_pro_seller=True, agency_name = organization or agentName + Returns (None, None, None) when author/category absent. + """ + category = (author.get("category") or "").upper() + if not category: + return None, None, None + if category == "OWNER": + return True, None, None + if category in {"AGENCY", "AGENT"}: + agency_name = author.get("organization") or author.get("agentName") or None + return None, True, agency_name + return None, None, None + + +def _to_bigint(raw: Any) -> int | None: + """Coerce a gate-API numeric value (str or number) to int. None when unparseable.""" + if raw is None: + return None + try: + return int(float(raw)) + except (TypeError, ValueError): + return None + + +def _predicted_prices(entity: dict[str, Any]) -> tuple[int | None, int | None, int | None]: + """Extract (value, min, max) from entity.predictedPrice.predictedPrice. + + Yandex's own per-offer valuation. Fields are strings (e.g. "7554000"). + Returns (None, None, None) when the nested path is absent. + """ + outer = entity.get("predictedPrice") or {} + inner = outer.get("predictedPrice") or {} + if not isinstance(inner, dict): + return None, None, None + return ( + _to_bigint(inner.get("value")), + _to_bigint(inner.get("min")), + _to_bigint(inner.get("max")), + ) + + +def _metro_stations(location: dict[str, Any]) -> list[dict[str, Any]]: + """Build metro_stations [{name, min}] from location.metroList[] (+ location.metro).""" + stations: list[dict[str, Any]] = [] + raw_list = location.get("metroList") + if isinstance(raw_list, list): + for m in raw_list: + if isinstance(m, dict) and m.get("name"): + stations.append({"name": m.get("name"), "min": m.get("timeToMetro")}) + if not stations: + single = location.get("metro") + if isinstance(single, dict) and single.get("name"): + stations.append({"name": single.get("name"), "min": single.get("timeToMetro")}) + return stations + + def _parse_gate_json( payload: dict[str, Any], page_param: int = 1, new_flat: str = "NO" ) -> list[ScrapedLot]: @@ -158,6 +244,13 @@ def _parse_gate_json( mainImages[] -> photo_urls (prepend "https:", up to 5) building.siteId -> house_ext_id (JK linkage) listing_segment -> "novostroyki" (newFlat=YES) | "vtorichka" (newFlat=NO) + creationDate (ISO Z) -> listing_date / publish_date + days_on_market + description -> description + author.category/organization -> is_homeowner / is_pro_seller / agency_name + predictedPrice.predictedPrice.* -> predicted_price_rub / _min / _max + price.trend / price.previous -> price_trend / price_previous_rub + location.metroList[] -> metro_stations [{name, min}] + offerId -> yandex_offer_id """ result = _extract_gate_data(payload) if result is None: @@ -245,6 +338,20 @@ def _entity_to_lot( address = location.get("geocoderAddress") or location.get("streetAddress") or None + # ── Rich gate-API fields ─────────────────────────────────────────── + listing_date, days_on_market = _parse_creation_date(entity.get("creationDate")) + description = entity.get("description") or None + + author = entity.get("author") or {} + is_homeowner, is_pro_seller, agency_name = _author_fields(author) + + predicted_value, predicted_min, predicted_max = _predicted_prices(entity) + + price_trend = price_dict.get("trend") or None + price_previous_rub = _to_bigint(price_dict.get("previous")) + + metro_stations = _metro_stations(location) + photo_urls: list[str] = [] for img in entity.get("mainImages") or []: if img: @@ -273,6 +380,20 @@ def _entity_to_lot( house_ext_id=house_ext_id, house_url=None, listing_segment=_segment_for(new_flat), + listing_date=listing_date, + publish_date=listing_date, + days_on_market=days_on_market, + description=description, + is_homeowner=is_homeowner, + is_pro_seller=is_pro_seller, + agency_name=agency_name, + metro_stations=metro_stations, + yandex_offer_id=offer_id, + predicted_price_rub=predicted_value, + predicted_price_min=predicted_min, + predicted_price_max=predicted_max, + price_trend=price_trend, + price_previous_rub=price_previous_rub, raw_payload={ "page_param": page_param, "ceiling_height": ceiling_height, diff --git a/tradein-mvp/backend/data/sql/121_yandex_rich_fields.sql b/tradein-mvp/backend/data/sql/121_yandex_rich_fields.sql new file mode 100644 index 00000000..112a5e95 --- /dev/null +++ b/tradein-mvp/backend/data/sql/121_yandex_rich_fields.sql @@ -0,0 +1,30 @@ +-- Migration 121: Yandex gate-API rich entity fields the parser previously dropped. +-- +-- Источник: realty.yandex.ru/gate/react-page/get/ entity object содержит ~90 полей, +-- из которых маппилось ~11. Эта миграция добавляет колонки под high-value поля: +-- +-- predictedPrice.predictedPrice.{value,min,max} → собственная оценка Яндекса +-- per-offer (рыночная цена) — headline-выигрыш для estimator/анализа bargain. +-- price.trend / price.previous → динамика цены объявления. +-- +-- description / listing_date / publish_date / days_on_market / metro_stations / +-- is_homeowner / is_pro_seller / agency_name / yandex_offer_id — уже существуют +-- в listings (миграции 011 / 033 и др.); эта миграция их НЕ трогает, парсер просто +-- начинает их наполнять (см. base.py save_listings + yandex_realty._entity_to_lot). +-- +-- Idempotent: ADD COLUMN IF NOT EXISTS. Авто-применяется на деплое. + +BEGIN; + +ALTER TABLE listings ADD COLUMN IF NOT EXISTS predicted_price_rub bigint; +ALTER TABLE listings ADD COLUMN IF NOT EXISTS predicted_price_min bigint; +ALTER TABLE listings ADD COLUMN IF NOT EXISTS predicted_price_max bigint; +ALTER TABLE listings ADD COLUMN IF NOT EXISTS price_trend text; +ALTER TABLE listings ADD COLUMN IF NOT EXISTS price_previous_rub bigint; + +COMMENT ON COLUMN listings.predicted_price_rub IS + 'Yandex per-offer valuation (predictedPrice.predictedPrice.value), RUB'; +COMMENT ON COLUMN listings.price_trend IS + 'Yandex price.trend: INCREASED / DECREASED / UNCHANGED / ...'; + +COMMIT; diff --git a/tradein-mvp/backend/tests/test_base_save_listings.py b/tradein-mvp/backend/tests/test_base_save_listings.py index 9d2d8a26..1a38b7e8 100644 --- a/tradein-mvp/backend/tests/test_base_save_listings.py +++ b/tradein-mvp/backend/tests/test_base_save_listings.py @@ -594,3 +594,104 @@ def test_save_listings_null_area_and_date_pass_none_in_params(): params = _get_execute_params(db) assert params["area_m2"] is None assert params["listing_date"] is None + + +# ── Tests 18-20: Yandex rich fields (predictedPrice, trend, description, agency) ── + + +def _yandex_rich_lot(**overrides) -> ScrapedLot: + """ScrapedLot с заполненными Yandex rich-полями.""" + import datetime + + base = { + "source": "yandex", + "source_url": "https://realty.yandex.ru/offer/5500000000000000001/", + "source_id": "5500000000000000001", + "price_rub": 7_200_000, + "address": "Екатеринбург, улица Мира, 10", + "rooms": 2, + "area_m2": 48.0, + "listing_date": datetime.date(2026, 5, 24), + "publish_date": datetime.date(2026, 5, 24), + "days_on_market": 24, + "description": "Светлая квартира.", + "is_homeowner": True, + "agency_name": None, + "metro_stations": [{"name": "Динамо", "min": 7}], + "yandex_offer_id": "5500000000000000001", + "predicted_price_rub": 7_554_000, + "predicted_price_min": 7_100_000, + "predicted_price_max": 8_000_000, + "price_trend": "DECREASED", + "price_previous_rub": 7_500_000, + } + base.update(overrides) + return ScrapedLot(**base) + + +def test_save_listings_persists_yandex_rich_fields(): + """Yandex rich-поля попадают в params INSERT.""" + db = _mock_db(inserted=True) + lot = _yandex_rich_lot() + + save_listings(db, [lot]) + + params = _get_execute_params(db) + assert params["description"] == "Светлая квартира." + assert params["days_on_market"] == 24 + assert params["yandex_offer_id"] == "5500000000000000001" + assert params["predicted_price_rub"] == 7_554_000 + assert params["predicted_price_min"] == 7_100_000 + assert params["predicted_price_max"] == 8_000_000 + assert params["price_trend"] == "DECREASED" + assert params["price_previous_rub"] == 7_500_000 + assert params["agency_name"] is None + metro = json.loads(params["metro_stations"]) + assert metro == [{"name": "Динамо", "min": 7}] + + +def test_save_listings_yandex_rich_cols_in_sql(): + """INSERT + ON CONFLICT SET содержат новые Yandex rich колонки.""" + db = _mock_db(inserted=False) + lot = _yandex_rich_lot(agency_name="Агентство «Диал»", is_pro_seller=True, is_homeowner=None) + + save_listings(db, [lot]) + + sql_text = _get_listings_insert_sql(db) + for col in ( + "description", + "publish_date", + "agency_name", + "yandex_offer_id", + "predicted_price_rub", + "predicted_price_min", + "predicted_price_max", + "price_trend", + "price_previous_rub", + ): + assert col in sql_text, f"Column '{col}' missing from INSERT SQL" + + # COALESCE backfill in DO UPDATE SET (never wipe a prior value) + assert "description = COALESCE(EXCLUDED.description, listings.description)" in sql_text + assert "agency_name = COALESCE(EXCLUDED.agency_name, listings.agency_name)" in sql_text + assert "predicted_price_rub = COALESCE(" in sql_text + assert "price_trend = COALESCE(EXCLUDED.price_trend, listings.price_trend)" in sql_text + + +def test_save_listings_non_yandex_rich_cols_are_null(): + """Avito lot не имеет Yandex rich-полей → params = None (backward compat).""" + db = _mock_db(inserted=True) + lot = _base_lot(source="avito", source_id="987654") + + save_listings(db, [lot]) + + params = _get_execute_params(db) + assert params["description"] is None + assert params["publish_date"] is None + assert params["agency_name"] is None + assert params["yandex_offer_id"] is None + assert params["predicted_price_rub"] is None + assert params["predicted_price_min"] is None + assert params["predicted_price_max"] is None + assert params["price_trend"] is None + assert params["price_previous_rub"] is None diff --git a/tradein-mvp/backend/tests/test_yandex_realty_serp.py b/tradein-mvp/backend/tests/test_yandex_realty_serp.py index 2561b0c5..17206a30 100644 --- a/tradein-mvp/backend/tests/test_yandex_realty_serp.py +++ b/tradein-mvp/backend/tests/test_yandex_realty_serp.py @@ -100,6 +100,58 @@ _ENTITY_NO_OFFER_ID: dict = { } +# Entity carrying the rich gate-API fields (creationDate, description, author, +# predictedPrice, price.trend/previous, location.metroList) — verified live 2026-06-17. +_ENTITY_RICH_OWNER: dict = { + "offerId": 5500000000000000001, + "url": "//realty.yandex.ru/offer/5500000000000000001", + "creationDate": "2026-05-24T15:10:27Z", + "description": "Продаётся светлая квартира с видом на парк.", + "author": {"category": "OWNER", "agentName": "Иван", "organization": None}, + "price": { + "value": 7200000, + "valuePerPart": 150000, + "trend": "DECREASED", + "previous": 7500000, + }, + "predictedPrice": {"predictedPrice": {"value": "7554000", "min": "7100000", "max": "8000000"}}, + "area": {"value": 48.0}, + "roomsTotal": 2, + "floorsOffered": [5], + "floorsTotal": 12, + "building": {"builtYear": 2018, "buildingType": "MONOLIT"}, + "location": { + "geocoderAddress": "Ekaterinburg, ulitsa Mira, 10", + "point": {"latitude": 56.84, "longitude": 60.61}, + "metroList": [ + {"name": "Ploshchad 1905 goda", "timeToMetro": 7}, + {"name": "Geologicheskaya", "timeToMetro": 12}, + ], + }, + "mainImages": [], +} + +_ENTITY_RICH_AGENCY: dict = { + "offerId": 5500000000000000002, + "url": "//realty.yandex.ru/offer/5500000000000000002", + "creationDate": "2026-06-01T09:00:00Z", + "description": "Агентская продажа.", + "author": {"category": "AGENCY", "agentName": "Петров", "organization": "Агентство «Диал»"}, + "price": {"value": 9900000, "trend": "UNCHANGED"}, + "area": {"value": 60.0}, + "roomsTotal": 3, + "floorsOffered": [3], + "floorsTotal": 9, + "building": {"builtYear": 2012, "buildingType": "BRICK"}, + "location": { + "geocoderAddress": "Ekaterinburg, ulitsa Lenina, 1", + "point": {"latitude": 56.83, "longitude": 60.6}, + "metro": {"name": "Dinamo", "timeToMetro": 4}, + }, + "mainImages": [], +} + + def _make_gate_payload(entities: list, pager: dict | None = None) -> dict: """Wrap entities + pager into gate-API response shape.""" if pager is None: @@ -271,6 +323,88 @@ def test_entity_to_lot_page_param_in_raw(): assert lot.raw_payload["page_param"] == 3 +# --------------------------------------------------------------------------- +# PART B2: rich gate-API fields (predictedPrice, seller, dates, trend, metro) +# --------------------------------------------------------------------------- + + +def test_entity_to_lot_rich_owner_fields(): + lot = _entity_to_lot(_ENTITY_RICH_OWNER) + assert lot is not None + # creationDate → listing_date / publish_date / days_on_market + assert lot.listing_date is not None + assert lot.listing_date.isoformat() == "2026-05-24" + assert lot.publish_date == lot.listing_date + assert lot.days_on_market is not None + assert lot.days_on_market >= 0 + # description + assert lot.description == "Продаётся светлая квартира с видом на парк." + # author OWNER + assert lot.is_homeowner is True + assert lot.is_pro_seller is None + assert lot.agency_name is None + # predictedPrice + assert lot.predicted_price_rub == 7_554_000 + assert lot.predicted_price_min == 7_100_000 + assert lot.predicted_price_max == 8_000_000 + # price trend / previous + assert lot.price_trend == "DECREASED" + assert lot.price_previous_rub == 7_500_000 + # metro_stations from metroList + assert lot.metro_stations == [ + {"name": "Ploshchad 1905 goda", "min": 7}, + {"name": "Geologicheskaya", "min": 12}, + ] + # yandex_offer_id + assert lot.yandex_offer_id == "5500000000000000001" + + +def test_entity_to_lot_rich_agency_fields(): + lot = _entity_to_lot(_ENTITY_RICH_AGENCY) + assert lot is not None + # AGENCY → pro seller + agency_name from organization + assert lot.is_pro_seller is True + assert lot.is_homeowner is None + assert lot.agency_name == "Агентство «Диал»" + # price.trend present, no previous → price_previous_rub None + assert lot.price_trend == "UNCHANGED" + assert lot.price_previous_rub is None + # no predictedPrice path → all None + assert lot.predicted_price_rub is None + assert lot.predicted_price_min is None + assert lot.predicted_price_max is None + # single location.metro fallback + assert lot.metro_stations == [{"name": "Dinamo", "min": 4}] + + +def test_entity_to_lot_no_rich_fields_are_none(): + """Base fixture lacks rich fields → all new fields default to None/[] (no crash).""" + lot = _entity_to_lot(_ENTITY_FULL) + assert lot is not None + assert lot.listing_date is None + assert lot.days_on_market is None + assert lot.description is None + assert lot.is_homeowner is None + assert lot.is_pro_seller is None + assert lot.agency_name is None + assert lot.predicted_price_rub is None + assert lot.price_trend is None + assert lot.price_previous_rub is None + assert lot.metro_stations == [] + # yandex_offer_id is always populated from offerId + assert lot.yandex_offer_id == "4740475460451078271" + + +def test_entity_to_lot_bad_creation_date_is_safe(): + """Unparseable creationDate → listing_date/days_on_market None, no exception.""" + entity = dict(_ENTITY_RICH_OWNER) + entity["creationDate"] = "not-a-date" + lot = _entity_to_lot(entity) + assert lot is not None + assert lot.listing_date is None + assert lot.days_on_market is None + + # --------------------------------------------------------------------------- # PART C: _parse_gate_json (list-level) # --------------------------------------------------------------------------- -- 2.45.3