feat(yandex): map rich entity fields (predictedPrice, seller, listing_date, trend, metro)
This commit is contained in:
parent
4ebc271a6c
commit
1fe670ea6a
5 changed files with 443 additions and 5 deletions
|
|
@ -101,7 +101,9 @@ class ScrapedLot(BaseModel):
|
||||||
|
|
||||||
# Метаданные
|
# Метаданные
|
||||||
listing_date: date | None = None
|
listing_date: date | None = None
|
||||||
|
publish_date: date | None = None
|
||||||
days_on_market: int | None = None
|
days_on_market: int | None = None
|
||||||
|
description: str | None = None
|
||||||
photo_urls: list[str] = Field(default_factory=list)
|
photo_urls: list[str] = Field(default_factory=list)
|
||||||
raw_payload: dict[str, Any] | None = None
|
raw_payload: dict[str, Any] | None = None
|
||||||
|
|
||||||
|
|
@ -119,6 +121,15 @@ class ScrapedLot(BaseModel):
|
||||||
bargain_allowed: bool | None = None
|
bargain_allowed: bool | None = None
|
||||||
sale_type: str | None = None
|
sale_type: str | None = None
|
||||||
metro_stations: list[dict] = Field(default_factory=list)
|
metro_stations: list[dict] = Field(default_factory=list)
|
||||||
|
agency_name: str | None = None
|
||||||
|
|
||||||
|
# Yandex gate-API rich fields
|
||||||
|
yandex_offer_id: str | None = None # numeric string offerId (kept as text)
|
||||||
|
predicted_price_rub: int | None = None # Yandex per-offer valuation (predictedPrice.value)
|
||||||
|
predicted_price_min: int | None = None
|
||||||
|
predicted_price_max: int | None = None
|
||||||
|
price_trend: str | None = None # INCREASED / DECREASED / UNCHANGED / ...
|
||||||
|
price_previous_rub: int | None = None # price.previous
|
||||||
|
|
||||||
def compute_dedup_hash(self) -> str:
|
def compute_dedup_hash(self) -> str:
|
||||||
"""SHA256(source + stable_key) — стабильный uniqueness key.
|
"""SHA256(source + stable_key) — стабильный uniqueness key.
|
||||||
|
|
@ -252,11 +263,15 @@ def save_listings(
|
||||||
house_type, repair_state, has_balcony,
|
house_type, repair_state, has_balcony,
|
||||||
house_source, house_ext_id, house_url, listing_segment,
|
house_source, house_ext_id, house_url, listing_segment,
|
||||||
price_rub, price_per_m2,
|
price_rub, price_per_m2,
|
||||||
listing_date, days_on_market, photo_urls, raw_payload,
|
listing_date, publish_date, days_on_market, description,
|
||||||
|
photo_urls, raw_payload,
|
||||||
living_area_m2, bedrooms_count, balconies_count, loggias_count,
|
living_area_m2, bedrooms_count, balconies_count, loggias_count,
|
||||||
description_minhash, cadastral_number, building_cadastral_number,
|
description_minhash, cadastral_number, building_cadastral_number,
|
||||||
phones, is_homeowner, is_pro_seller,
|
phones, is_homeowner, is_pro_seller,
|
||||||
bargain_allowed, sale_type, metro_stations,
|
bargain_allowed, sale_type, metro_stations, agency_name,
|
||||||
|
yandex_offer_id, predicted_price_rub,
|
||||||
|
predicted_price_min, predicted_price_max,
|
||||||
|
price_trend, price_previous_rub,
|
||||||
geo_precision,
|
geo_precision,
|
||||||
scraped_at, last_seen_at
|
scraped_at, last_seen_at
|
||||||
) VALUES (
|
) VALUES (
|
||||||
|
|
@ -266,13 +281,16 @@ def save_listings(
|
||||||
:house_type, :repair_state, :has_balcony,
|
:house_type, :repair_state, :has_balcony,
|
||||||
:house_source, :house_ext_id, :house_url, :listing_segment,
|
:house_source, :house_ext_id, :house_url, :listing_segment,
|
||||||
:price_rub, :ppm2,
|
:price_rub, :ppm2,
|
||||||
:listing_date, :days_on_market,
|
:listing_date, :publish_date, :days_on_market, :description,
|
||||||
CAST(:photos AS jsonb),
|
CAST(:photos AS jsonb),
|
||||||
CAST(:raw AS jsonb),
|
CAST(:raw AS jsonb),
|
||||||
:living_area_m2, :bedrooms_count, :balconies_count, :loggias_count,
|
:living_area_m2, :bedrooms_count, :balconies_count, :loggias_count,
|
||||||
:description_minhash, :cadastral_number, :building_cadastral_number,
|
:description_minhash, :cadastral_number, :building_cadastral_number,
|
||||||
CAST(:phones AS jsonb), :is_homeowner, :is_pro_seller,
|
CAST(:phones AS jsonb), :is_homeowner, :is_pro_seller,
|
||||||
:bargain_allowed, :sale_type, CAST(:metro_stations AS jsonb),
|
:bargain_allowed, :sale_type, CAST(:metro_stations AS jsonb), :agency_name,
|
||||||
|
:yandex_offer_id, :predicted_price_rub,
|
||||||
|
:predicted_price_min, :predicted_price_max,
|
||||||
|
:price_trend, :price_previous_rub,
|
||||||
:geo_precision,
|
:geo_precision,
|
||||||
NOW(), NOW()
|
NOW(), NOW()
|
||||||
)
|
)
|
||||||
|
|
@ -297,7 +315,32 @@ def save_listings(
|
||||||
sale_type = EXCLUDED.sale_type,
|
sale_type = EXCLUDED.sale_type,
|
||||||
metro_stations = EXCLUDED.metro_stations,
|
metro_stations = EXCLUDED.metro_stations,
|
||||||
listing_date = COALESCE(EXCLUDED.listing_date, listings.listing_date),
|
listing_date = COALESCE(EXCLUDED.listing_date, listings.listing_date),
|
||||||
area_m2 = COALESCE(EXCLUDED.area_m2, listings.area_m2)
|
area_m2 = COALESCE(EXCLUDED.area_m2, listings.area_m2),
|
||||||
|
-- Yandex rich fields (+ shared description/agency/publish_date):
|
||||||
|
-- COALESCE so a source that does not provide them never wipes
|
||||||
|
-- a value previously written by another source / scrape.
|
||||||
|
publish_date = COALESCE(EXCLUDED.publish_date, listings.publish_date),
|
||||||
|
days_on_market = COALESCE(
|
||||||
|
EXCLUDED.days_on_market, listings.days_on_market
|
||||||
|
),
|
||||||
|
description = COALESCE(EXCLUDED.description, listings.description),
|
||||||
|
agency_name = COALESCE(EXCLUDED.agency_name, listings.agency_name),
|
||||||
|
yandex_offer_id = COALESCE(
|
||||||
|
EXCLUDED.yandex_offer_id, listings.yandex_offer_id
|
||||||
|
),
|
||||||
|
predicted_price_rub = COALESCE(
|
||||||
|
EXCLUDED.predicted_price_rub, listings.predicted_price_rub
|
||||||
|
),
|
||||||
|
predicted_price_min = COALESCE(
|
||||||
|
EXCLUDED.predicted_price_min, listings.predicted_price_min
|
||||||
|
),
|
||||||
|
predicted_price_max = COALESCE(
|
||||||
|
EXCLUDED.predicted_price_max, listings.predicted_price_max
|
||||||
|
),
|
||||||
|
price_trend = COALESCE(EXCLUDED.price_trend, listings.price_trend),
|
||||||
|
price_previous_rub = COALESCE(
|
||||||
|
EXCLUDED.price_previous_rub, listings.price_previous_rub
|
||||||
|
)
|
||||||
RETURNING id, (xmax = 0) AS inserted
|
RETURNING id, (xmax = 0) AS inserted
|
||||||
"""
|
"""
|
||||||
),
|
),
|
||||||
|
|
@ -324,7 +367,9 @@ def save_listings(
|
||||||
"price_rub": lot.price_rub,
|
"price_rub": lot.price_rub,
|
||||||
"ppm2": ppm2,
|
"ppm2": ppm2,
|
||||||
"listing_date": lot.listing_date,
|
"listing_date": lot.listing_date,
|
||||||
|
"publish_date": lot.publish_date,
|
||||||
"days_on_market": lot.days_on_market,
|
"days_on_market": lot.days_on_market,
|
||||||
|
"description": lot.description,
|
||||||
"photos": _to_json(lot.photo_urls),
|
"photos": _to_json(lot.photo_urls),
|
||||||
"raw": _to_json(lot.raw_payload) if lot.raw_payload else None,
|
"raw": _to_json(lot.raw_payload) if lot.raw_payload else None,
|
||||||
# Cian-specific columns — None for non-Cian scrapers (→ SQL NULL)
|
# Cian-specific columns — None for non-Cian scrapers (→ SQL NULL)
|
||||||
|
|
@ -341,6 +386,13 @@ def save_listings(
|
||||||
"bargain_allowed": lot.bargain_allowed,
|
"bargain_allowed": lot.bargain_allowed,
|
||||||
"sale_type": lot.sale_type,
|
"sale_type": lot.sale_type,
|
||||||
"metro_stations": _to_json(lot.metro_stations) if lot.metro_stations else None,
|
"metro_stations": _to_json(lot.metro_stations) if lot.metro_stations else None,
|
||||||
|
"agency_name": lot.agency_name,
|
||||||
|
"yandex_offer_id": lot.yandex_offer_id,
|
||||||
|
"predicted_price_rub": lot.predicted_price_rub,
|
||||||
|
"predicted_price_min": lot.predicted_price_min,
|
||||||
|
"predicted_price_max": lot.predicted_price_max,
|
||||||
|
"price_trend": lot.price_trend,
|
||||||
|
"price_previous_rub": lot.price_previous_rub,
|
||||||
"geo_precision": lot.geo_precision,
|
"geo_precision": lot.geo_precision,
|
||||||
},
|
},
|
||||||
).fetchone()
|
).fetchone()
|
||||||
|
|
|
||||||
|
|
@ -32,6 +32,7 @@ import logging
|
||||||
import math
|
import math
|
||||||
import re
|
import re
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
|
from datetime import UTC, date, datetime
|
||||||
from typing import Any
|
from typing import Any
|
||||||
from urllib.parse import urlencode
|
from urllib.parse import urlencode
|
||||||
|
|
||||||
|
|
@ -134,6 +135,91 @@ def _segment_for(new_flat: str) -> str:
|
||||||
return "novostroyki" if new_flat == "YES" else "vtorichka"
|
return "novostroyki" if new_flat == "YES" else "vtorichka"
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_creation_date(raw: Any) -> tuple[date | None, int | None]:
|
||||||
|
"""Parse gate-API creationDate ('2026-05-24T15:10:27Z') → (date, days_on_market).
|
||||||
|
|
||||||
|
days_on_market = floor((now - creationDate).days), clamped to >= 0.
|
||||||
|
Returns (None, None) when raw is absent or unparseable (never raises).
|
||||||
|
"""
|
||||||
|
if not raw or not isinstance(raw, str):
|
||||||
|
return None, None
|
||||||
|
txt = raw.strip()
|
||||||
|
# datetime.fromisoformat handles 'Z' only from py3.11+, but normalise defensively.
|
||||||
|
if txt.endswith("Z"):
|
||||||
|
txt = txt[:-1] + "+00:00"
|
||||||
|
try:
|
||||||
|
dt = datetime.fromisoformat(txt)
|
||||||
|
except ValueError:
|
||||||
|
return None, None
|
||||||
|
if dt.tzinfo is None:
|
||||||
|
dt = dt.replace(tzinfo=UTC)
|
||||||
|
listing_date = dt.date()
|
||||||
|
delta_days = (datetime.now(UTC) - dt).days
|
||||||
|
days_on_market = max(delta_days, 0)
|
||||||
|
return listing_date, days_on_market
|
||||||
|
|
||||||
|
|
||||||
|
def _author_fields(author: dict[str, Any]) -> tuple[bool | None, bool | None, str | None]:
|
||||||
|
"""Map entity.author → (is_homeowner, is_pro_seller, agency_name).
|
||||||
|
|
||||||
|
author.category ∈ {OWNER, AGENCY, AGENT, ...}.
|
||||||
|
OWNER → is_homeowner=True
|
||||||
|
AGENCY / AGENT → is_pro_seller=True, agency_name = organization or agentName
|
||||||
|
Returns (None, None, None) when author/category absent.
|
||||||
|
"""
|
||||||
|
category = (author.get("category") or "").upper()
|
||||||
|
if not category:
|
||||||
|
return None, None, None
|
||||||
|
if category == "OWNER":
|
||||||
|
return True, None, None
|
||||||
|
if category in {"AGENCY", "AGENT"}:
|
||||||
|
agency_name = author.get("organization") or author.get("agentName") or None
|
||||||
|
return None, True, agency_name
|
||||||
|
return None, None, None
|
||||||
|
|
||||||
|
|
||||||
|
def _to_bigint(raw: Any) -> int | None:
|
||||||
|
"""Coerce a gate-API numeric value (str or number) to int. None when unparseable."""
|
||||||
|
if raw is None:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
return int(float(raw))
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _predicted_prices(entity: dict[str, Any]) -> tuple[int | None, int | None, int | None]:
|
||||||
|
"""Extract (value, min, max) from entity.predictedPrice.predictedPrice.
|
||||||
|
|
||||||
|
Yandex's own per-offer valuation. Fields are strings (e.g. "7554000").
|
||||||
|
Returns (None, None, None) when the nested path is absent.
|
||||||
|
"""
|
||||||
|
outer = entity.get("predictedPrice") or {}
|
||||||
|
inner = outer.get("predictedPrice") or {}
|
||||||
|
if not isinstance(inner, dict):
|
||||||
|
return None, None, None
|
||||||
|
return (
|
||||||
|
_to_bigint(inner.get("value")),
|
||||||
|
_to_bigint(inner.get("min")),
|
||||||
|
_to_bigint(inner.get("max")),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _metro_stations(location: dict[str, Any]) -> list[dict[str, Any]]:
|
||||||
|
"""Build metro_stations [{name, min}] from location.metroList[] (+ location.metro)."""
|
||||||
|
stations: list[dict[str, Any]] = []
|
||||||
|
raw_list = location.get("metroList")
|
||||||
|
if isinstance(raw_list, list):
|
||||||
|
for m in raw_list:
|
||||||
|
if isinstance(m, dict) and m.get("name"):
|
||||||
|
stations.append({"name": m.get("name"), "min": m.get("timeToMetro")})
|
||||||
|
if not stations:
|
||||||
|
single = location.get("metro")
|
||||||
|
if isinstance(single, dict) and single.get("name"):
|
||||||
|
stations.append({"name": single.get("name"), "min": single.get("timeToMetro")})
|
||||||
|
return stations
|
||||||
|
|
||||||
|
|
||||||
def _parse_gate_json(
|
def _parse_gate_json(
|
||||||
payload: dict[str, Any], page_param: int = 1, new_flat: str = "NO"
|
payload: dict[str, Any], page_param: int = 1, new_flat: str = "NO"
|
||||||
) -> list[ScrapedLot]:
|
) -> list[ScrapedLot]:
|
||||||
|
|
@ -158,6 +244,13 @@ def _parse_gate_json(
|
||||||
mainImages[] -> photo_urls (prepend "https:", up to 5)
|
mainImages[] -> photo_urls (prepend "https:", up to 5)
|
||||||
building.siteId -> house_ext_id (JK linkage)
|
building.siteId -> house_ext_id (JK linkage)
|
||||||
listing_segment -> "novostroyki" (newFlat=YES) | "vtorichka" (newFlat=NO)
|
listing_segment -> "novostroyki" (newFlat=YES) | "vtorichka" (newFlat=NO)
|
||||||
|
creationDate (ISO Z) -> listing_date / publish_date + days_on_market
|
||||||
|
description -> description
|
||||||
|
author.category/organization -> is_homeowner / is_pro_seller / agency_name
|
||||||
|
predictedPrice.predictedPrice.* -> predicted_price_rub / _min / _max
|
||||||
|
price.trend / price.previous -> price_trend / price_previous_rub
|
||||||
|
location.metroList[] -> metro_stations [{name, min}]
|
||||||
|
offerId -> yandex_offer_id
|
||||||
"""
|
"""
|
||||||
result = _extract_gate_data(payload)
|
result = _extract_gate_data(payload)
|
||||||
if result is None:
|
if result is None:
|
||||||
|
|
@ -245,6 +338,20 @@ def _entity_to_lot(
|
||||||
|
|
||||||
address = location.get("geocoderAddress") or location.get("streetAddress") or None
|
address = location.get("geocoderAddress") or location.get("streetAddress") or None
|
||||||
|
|
||||||
|
# ── Rich gate-API fields ───────────────────────────────────────────
|
||||||
|
listing_date, days_on_market = _parse_creation_date(entity.get("creationDate"))
|
||||||
|
description = entity.get("description") or None
|
||||||
|
|
||||||
|
author = entity.get("author") or {}
|
||||||
|
is_homeowner, is_pro_seller, agency_name = _author_fields(author)
|
||||||
|
|
||||||
|
predicted_value, predicted_min, predicted_max = _predicted_prices(entity)
|
||||||
|
|
||||||
|
price_trend = price_dict.get("trend") or None
|
||||||
|
price_previous_rub = _to_bigint(price_dict.get("previous"))
|
||||||
|
|
||||||
|
metro_stations = _metro_stations(location)
|
||||||
|
|
||||||
photo_urls: list[str] = []
|
photo_urls: list[str] = []
|
||||||
for img in entity.get("mainImages") or []:
|
for img in entity.get("mainImages") or []:
|
||||||
if img:
|
if img:
|
||||||
|
|
@ -273,6 +380,20 @@ def _entity_to_lot(
|
||||||
house_ext_id=house_ext_id,
|
house_ext_id=house_ext_id,
|
||||||
house_url=None,
|
house_url=None,
|
||||||
listing_segment=_segment_for(new_flat),
|
listing_segment=_segment_for(new_flat),
|
||||||
|
listing_date=listing_date,
|
||||||
|
publish_date=listing_date,
|
||||||
|
days_on_market=days_on_market,
|
||||||
|
description=description,
|
||||||
|
is_homeowner=is_homeowner,
|
||||||
|
is_pro_seller=is_pro_seller,
|
||||||
|
agency_name=agency_name,
|
||||||
|
metro_stations=metro_stations,
|
||||||
|
yandex_offer_id=offer_id,
|
||||||
|
predicted_price_rub=predicted_value,
|
||||||
|
predicted_price_min=predicted_min,
|
||||||
|
predicted_price_max=predicted_max,
|
||||||
|
price_trend=price_trend,
|
||||||
|
price_previous_rub=price_previous_rub,
|
||||||
raw_payload={
|
raw_payload={
|
||||||
"page_param": page_param,
|
"page_param": page_param,
|
||||||
"ceiling_height": ceiling_height,
|
"ceiling_height": ceiling_height,
|
||||||
|
|
|
||||||
30
tradein-mvp/backend/data/sql/121_yandex_rich_fields.sql
Normal file
30
tradein-mvp/backend/data/sql/121_yandex_rich_fields.sql
Normal file
|
|
@ -0,0 +1,30 @@
|
||||||
|
-- Migration 121: Yandex gate-API rich entity fields the parser previously dropped.
|
||||||
|
--
|
||||||
|
-- Источник: realty.yandex.ru/gate/react-page/get/ entity object содержит ~90 полей,
|
||||||
|
-- из которых маппилось ~11. Эта миграция добавляет колонки под high-value поля:
|
||||||
|
--
|
||||||
|
-- predictedPrice.predictedPrice.{value,min,max} → собственная оценка Яндекса
|
||||||
|
-- per-offer (рыночная цена) — headline-выигрыш для estimator/анализа bargain.
|
||||||
|
-- price.trend / price.previous → динамика цены объявления.
|
||||||
|
--
|
||||||
|
-- description / listing_date / publish_date / days_on_market / metro_stations /
|
||||||
|
-- is_homeowner / is_pro_seller / agency_name / yandex_offer_id — уже существуют
|
||||||
|
-- в listings (миграции 011 / 033 и др.); эта миграция их НЕ трогает, парсер просто
|
||||||
|
-- начинает их наполнять (см. base.py save_listings + yandex_realty._entity_to_lot).
|
||||||
|
--
|
||||||
|
-- Idempotent: ADD COLUMN IF NOT EXISTS. Авто-применяется на деплое.
|
||||||
|
|
||||||
|
BEGIN;
|
||||||
|
|
||||||
|
ALTER TABLE listings ADD COLUMN IF NOT EXISTS predicted_price_rub bigint;
|
||||||
|
ALTER TABLE listings ADD COLUMN IF NOT EXISTS predicted_price_min bigint;
|
||||||
|
ALTER TABLE listings ADD COLUMN IF NOT EXISTS predicted_price_max bigint;
|
||||||
|
ALTER TABLE listings ADD COLUMN IF NOT EXISTS price_trend text;
|
||||||
|
ALTER TABLE listings ADD COLUMN IF NOT EXISTS price_previous_rub bigint;
|
||||||
|
|
||||||
|
COMMENT ON COLUMN listings.predicted_price_rub IS
|
||||||
|
'Yandex per-offer valuation (predictedPrice.predictedPrice.value), RUB';
|
||||||
|
COMMENT ON COLUMN listings.price_trend IS
|
||||||
|
'Yandex price.trend: INCREASED / DECREASED / UNCHANGED / ...';
|
||||||
|
|
||||||
|
COMMIT;
|
||||||
|
|
@ -594,3 +594,104 @@ def test_save_listings_null_area_and_date_pass_none_in_params():
|
||||||
params = _get_execute_params(db)
|
params = _get_execute_params(db)
|
||||||
assert params["area_m2"] is None
|
assert params["area_m2"] is None
|
||||||
assert params["listing_date"] is None
|
assert params["listing_date"] is None
|
||||||
|
|
||||||
|
|
||||||
|
# ── Tests 18-20: Yandex rich fields (predictedPrice, trend, description, agency) ──
|
||||||
|
|
||||||
|
|
||||||
|
def _yandex_rich_lot(**overrides) -> ScrapedLot:
|
||||||
|
"""ScrapedLot с заполненными Yandex rich-полями."""
|
||||||
|
import datetime
|
||||||
|
|
||||||
|
base = {
|
||||||
|
"source": "yandex",
|
||||||
|
"source_url": "https://realty.yandex.ru/offer/5500000000000000001/",
|
||||||
|
"source_id": "5500000000000000001",
|
||||||
|
"price_rub": 7_200_000,
|
||||||
|
"address": "Екатеринбург, улица Мира, 10",
|
||||||
|
"rooms": 2,
|
||||||
|
"area_m2": 48.0,
|
||||||
|
"listing_date": datetime.date(2026, 5, 24),
|
||||||
|
"publish_date": datetime.date(2026, 5, 24),
|
||||||
|
"days_on_market": 24,
|
||||||
|
"description": "Светлая квартира.",
|
||||||
|
"is_homeowner": True,
|
||||||
|
"agency_name": None,
|
||||||
|
"metro_stations": [{"name": "Динамо", "min": 7}],
|
||||||
|
"yandex_offer_id": "5500000000000000001",
|
||||||
|
"predicted_price_rub": 7_554_000,
|
||||||
|
"predicted_price_min": 7_100_000,
|
||||||
|
"predicted_price_max": 8_000_000,
|
||||||
|
"price_trend": "DECREASED",
|
||||||
|
"price_previous_rub": 7_500_000,
|
||||||
|
}
|
||||||
|
base.update(overrides)
|
||||||
|
return ScrapedLot(**base)
|
||||||
|
|
||||||
|
|
||||||
|
def test_save_listings_persists_yandex_rich_fields():
|
||||||
|
"""Yandex rich-поля попадают в params INSERT."""
|
||||||
|
db = _mock_db(inserted=True)
|
||||||
|
lot = _yandex_rich_lot()
|
||||||
|
|
||||||
|
save_listings(db, [lot])
|
||||||
|
|
||||||
|
params = _get_execute_params(db)
|
||||||
|
assert params["description"] == "Светлая квартира."
|
||||||
|
assert params["days_on_market"] == 24
|
||||||
|
assert params["yandex_offer_id"] == "5500000000000000001"
|
||||||
|
assert params["predicted_price_rub"] == 7_554_000
|
||||||
|
assert params["predicted_price_min"] == 7_100_000
|
||||||
|
assert params["predicted_price_max"] == 8_000_000
|
||||||
|
assert params["price_trend"] == "DECREASED"
|
||||||
|
assert params["price_previous_rub"] == 7_500_000
|
||||||
|
assert params["agency_name"] is None
|
||||||
|
metro = json.loads(params["metro_stations"])
|
||||||
|
assert metro == [{"name": "Динамо", "min": 7}]
|
||||||
|
|
||||||
|
|
||||||
|
def test_save_listings_yandex_rich_cols_in_sql():
|
||||||
|
"""INSERT + ON CONFLICT SET содержат новые Yandex rich колонки."""
|
||||||
|
db = _mock_db(inserted=False)
|
||||||
|
lot = _yandex_rich_lot(agency_name="Агентство «Диал»", is_pro_seller=True, is_homeowner=None)
|
||||||
|
|
||||||
|
save_listings(db, [lot])
|
||||||
|
|
||||||
|
sql_text = _get_listings_insert_sql(db)
|
||||||
|
for col in (
|
||||||
|
"description",
|
||||||
|
"publish_date",
|
||||||
|
"agency_name",
|
||||||
|
"yandex_offer_id",
|
||||||
|
"predicted_price_rub",
|
||||||
|
"predicted_price_min",
|
||||||
|
"predicted_price_max",
|
||||||
|
"price_trend",
|
||||||
|
"price_previous_rub",
|
||||||
|
):
|
||||||
|
assert col in sql_text, f"Column '{col}' missing from INSERT SQL"
|
||||||
|
|
||||||
|
# COALESCE backfill in DO UPDATE SET (never wipe a prior value)
|
||||||
|
assert "description = COALESCE(EXCLUDED.description, listings.description)" in sql_text
|
||||||
|
assert "agency_name = COALESCE(EXCLUDED.agency_name, listings.agency_name)" in sql_text
|
||||||
|
assert "predicted_price_rub = COALESCE(" in sql_text
|
||||||
|
assert "price_trend = COALESCE(EXCLUDED.price_trend, listings.price_trend)" in sql_text
|
||||||
|
|
||||||
|
|
||||||
|
def test_save_listings_non_yandex_rich_cols_are_null():
|
||||||
|
"""Avito lot не имеет Yandex rich-полей → params = None (backward compat)."""
|
||||||
|
db = _mock_db(inserted=True)
|
||||||
|
lot = _base_lot(source="avito", source_id="987654")
|
||||||
|
|
||||||
|
save_listings(db, [lot])
|
||||||
|
|
||||||
|
params = _get_execute_params(db)
|
||||||
|
assert params["description"] is None
|
||||||
|
assert params["publish_date"] is None
|
||||||
|
assert params["agency_name"] is None
|
||||||
|
assert params["yandex_offer_id"] is None
|
||||||
|
assert params["predicted_price_rub"] is None
|
||||||
|
assert params["predicted_price_min"] is None
|
||||||
|
assert params["predicted_price_max"] is None
|
||||||
|
assert params["price_trend"] is None
|
||||||
|
assert params["price_previous_rub"] is None
|
||||||
|
|
|
||||||
|
|
@ -100,6 +100,58 @@ _ENTITY_NO_OFFER_ID: dict = {
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# Entity carrying the rich gate-API fields (creationDate, description, author,
|
||||||
|
# predictedPrice, price.trend/previous, location.metroList) — verified live 2026-06-17.
|
||||||
|
_ENTITY_RICH_OWNER: dict = {
|
||||||
|
"offerId": 5500000000000000001,
|
||||||
|
"url": "//realty.yandex.ru/offer/5500000000000000001",
|
||||||
|
"creationDate": "2026-05-24T15:10:27Z",
|
||||||
|
"description": "Продаётся светлая квартира с видом на парк.",
|
||||||
|
"author": {"category": "OWNER", "agentName": "Иван", "organization": None},
|
||||||
|
"price": {
|
||||||
|
"value": 7200000,
|
||||||
|
"valuePerPart": 150000,
|
||||||
|
"trend": "DECREASED",
|
||||||
|
"previous": 7500000,
|
||||||
|
},
|
||||||
|
"predictedPrice": {"predictedPrice": {"value": "7554000", "min": "7100000", "max": "8000000"}},
|
||||||
|
"area": {"value": 48.0},
|
||||||
|
"roomsTotal": 2,
|
||||||
|
"floorsOffered": [5],
|
||||||
|
"floorsTotal": 12,
|
||||||
|
"building": {"builtYear": 2018, "buildingType": "MONOLIT"},
|
||||||
|
"location": {
|
||||||
|
"geocoderAddress": "Ekaterinburg, ulitsa Mira, 10",
|
||||||
|
"point": {"latitude": 56.84, "longitude": 60.61},
|
||||||
|
"metroList": [
|
||||||
|
{"name": "Ploshchad 1905 goda", "timeToMetro": 7},
|
||||||
|
{"name": "Geologicheskaya", "timeToMetro": 12},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
"mainImages": [],
|
||||||
|
}
|
||||||
|
|
||||||
|
_ENTITY_RICH_AGENCY: dict = {
|
||||||
|
"offerId": 5500000000000000002,
|
||||||
|
"url": "//realty.yandex.ru/offer/5500000000000000002",
|
||||||
|
"creationDate": "2026-06-01T09:00:00Z",
|
||||||
|
"description": "Агентская продажа.",
|
||||||
|
"author": {"category": "AGENCY", "agentName": "Петров", "organization": "Агентство «Диал»"},
|
||||||
|
"price": {"value": 9900000, "trend": "UNCHANGED"},
|
||||||
|
"area": {"value": 60.0},
|
||||||
|
"roomsTotal": 3,
|
||||||
|
"floorsOffered": [3],
|
||||||
|
"floorsTotal": 9,
|
||||||
|
"building": {"builtYear": 2012, "buildingType": "BRICK"},
|
||||||
|
"location": {
|
||||||
|
"geocoderAddress": "Ekaterinburg, ulitsa Lenina, 1",
|
||||||
|
"point": {"latitude": 56.83, "longitude": 60.6},
|
||||||
|
"metro": {"name": "Dinamo", "timeToMetro": 4},
|
||||||
|
},
|
||||||
|
"mainImages": [],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def _make_gate_payload(entities: list, pager: dict | None = None) -> dict:
|
def _make_gate_payload(entities: list, pager: dict | None = None) -> dict:
|
||||||
"""Wrap entities + pager into gate-API response shape."""
|
"""Wrap entities + pager into gate-API response shape."""
|
||||||
if pager is None:
|
if pager is None:
|
||||||
|
|
@ -271,6 +323,88 @@ def test_entity_to_lot_page_param_in_raw():
|
||||||
assert lot.raw_payload["page_param"] == 3
|
assert lot.raw_payload["page_param"] == 3
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# PART B2: rich gate-API fields (predictedPrice, seller, dates, trend, metro)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_entity_to_lot_rich_owner_fields():
|
||||||
|
lot = _entity_to_lot(_ENTITY_RICH_OWNER)
|
||||||
|
assert lot is not None
|
||||||
|
# creationDate → listing_date / publish_date / days_on_market
|
||||||
|
assert lot.listing_date is not None
|
||||||
|
assert lot.listing_date.isoformat() == "2026-05-24"
|
||||||
|
assert lot.publish_date == lot.listing_date
|
||||||
|
assert lot.days_on_market is not None
|
||||||
|
assert lot.days_on_market >= 0
|
||||||
|
# description
|
||||||
|
assert lot.description == "Продаётся светлая квартира с видом на парк."
|
||||||
|
# author OWNER
|
||||||
|
assert lot.is_homeowner is True
|
||||||
|
assert lot.is_pro_seller is None
|
||||||
|
assert lot.agency_name is None
|
||||||
|
# predictedPrice
|
||||||
|
assert lot.predicted_price_rub == 7_554_000
|
||||||
|
assert lot.predicted_price_min == 7_100_000
|
||||||
|
assert lot.predicted_price_max == 8_000_000
|
||||||
|
# price trend / previous
|
||||||
|
assert lot.price_trend == "DECREASED"
|
||||||
|
assert lot.price_previous_rub == 7_500_000
|
||||||
|
# metro_stations from metroList
|
||||||
|
assert lot.metro_stations == [
|
||||||
|
{"name": "Ploshchad 1905 goda", "min": 7},
|
||||||
|
{"name": "Geologicheskaya", "min": 12},
|
||||||
|
]
|
||||||
|
# yandex_offer_id
|
||||||
|
assert lot.yandex_offer_id == "5500000000000000001"
|
||||||
|
|
||||||
|
|
||||||
|
def test_entity_to_lot_rich_agency_fields():
|
||||||
|
lot = _entity_to_lot(_ENTITY_RICH_AGENCY)
|
||||||
|
assert lot is not None
|
||||||
|
# AGENCY → pro seller + agency_name from organization
|
||||||
|
assert lot.is_pro_seller is True
|
||||||
|
assert lot.is_homeowner is None
|
||||||
|
assert lot.agency_name == "Агентство «Диал»"
|
||||||
|
# price.trend present, no previous → price_previous_rub None
|
||||||
|
assert lot.price_trend == "UNCHANGED"
|
||||||
|
assert lot.price_previous_rub is None
|
||||||
|
# no predictedPrice path → all None
|
||||||
|
assert lot.predicted_price_rub is None
|
||||||
|
assert lot.predicted_price_min is None
|
||||||
|
assert lot.predicted_price_max is None
|
||||||
|
# single location.metro fallback
|
||||||
|
assert lot.metro_stations == [{"name": "Dinamo", "min": 4}]
|
||||||
|
|
||||||
|
|
||||||
|
def test_entity_to_lot_no_rich_fields_are_none():
|
||||||
|
"""Base fixture lacks rich fields → all new fields default to None/[] (no crash)."""
|
||||||
|
lot = _entity_to_lot(_ENTITY_FULL)
|
||||||
|
assert lot is not None
|
||||||
|
assert lot.listing_date is None
|
||||||
|
assert lot.days_on_market is None
|
||||||
|
assert lot.description is None
|
||||||
|
assert lot.is_homeowner is None
|
||||||
|
assert lot.is_pro_seller is None
|
||||||
|
assert lot.agency_name is None
|
||||||
|
assert lot.predicted_price_rub is None
|
||||||
|
assert lot.price_trend is None
|
||||||
|
assert lot.price_previous_rub is None
|
||||||
|
assert lot.metro_stations == []
|
||||||
|
# yandex_offer_id is always populated from offerId
|
||||||
|
assert lot.yandex_offer_id == "4740475460451078271"
|
||||||
|
|
||||||
|
|
||||||
|
def test_entity_to_lot_bad_creation_date_is_safe():
|
||||||
|
"""Unparseable creationDate → listing_date/days_on_market None, no exception."""
|
||||||
|
entity = dict(_ENTITY_RICH_OWNER)
|
||||||
|
entity["creationDate"] = "not-a-date"
|
||||||
|
lot = _entity_to_lot(entity)
|
||||||
|
assert lot is not None
|
||||||
|
assert lot.listing_date is None
|
||||||
|
assert lot.days_on_market is None
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# PART C: _parse_gate_json (list-level)
|
# PART C: _parse_gate_json (list-level)
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue