From 3485b977f1c538dd42d3ad0862e8c0476503ca9d Mon Sep 17 00:00:00 2001 From: bot-backend Date: Sun, 31 May 2026 21:33:36 +0300 Subject: [PATCH] =?UTF-8?q?feat(tradein):=20=D0=A1=D0=B1=D0=B5=D1=80=D0=98?= =?UTF-8?q?=D0=BD=D0=B4=D0=B5=D0=BA=D1=81=20time-adjust=20for=20frozen=20R?= =?UTF-8?q?osreestr=20=D0=94=D0=9A=D0=9F=20+=20per-dashboard=20filter=20fi?= =?UTF-8?q?x=20(#794)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tradein-mvp/backend/app/services/estimator.py | 93 +++++++++++- .../backend/app/services/sber_index.py | 143 +++++++++++------- .../fixtures/sber_real_estate_deals_66.json | 130 ++++++++++++++++ .../test_estimator_sber_time_adjust.py | 80 ++++++++++ .../backend/tests/services/test_sber_index.py | 113 ++++++++++++++ 5 files changed, 500 insertions(+), 59 deletions(-) create mode 100644 tradein-mvp/backend/tests/fixtures/sber_real_estate_deals_66.json create mode 100644 tradein-mvp/backend/tests/services/test_estimator_sber_time_adjust.py create mode 100644 tradein-mvp/backend/tests/services/test_sber_index.py diff --git a/tradein-mvp/backend/app/services/estimator.py b/tradein-mvp/backend/app/services/estimator.py index 289415f0..b12dcc8b 100644 --- a/tradein-mvp/backend/app/services/estimator.py +++ b/tradein-mvp/backend/app/services/estimator.py @@ -26,7 +26,7 @@ import logging import math import re import time -from datetime import UTC, datetime, timedelta +from datetime import UTC, date, datetime, timedelta from typing import Any from uuid import uuid4 @@ -78,6 +78,15 @@ MIN_ANALOGS_PER_SOURCE = 5 # гарантированный минимум на LISTINGS_FRESH_DAYS = 14 # объявления не старше 14 дней DEALS_PERIOD_MONTHS = 12 # сделки за последний год +# #794: СберИндекс time-adjustment of frozen Rosreestr ДКП deals. +# Rosreestr deals freeze ~2026-01; the sber monthly index re-bases a stale deal's ppm² +# to the latest available month. Region fixed to Свердловская обл. (tradein MVP = ЕКБ). +SBER_TIME_ADJUST_REGION = "Свердловская область" +# Coefficient series preference: hedonic (quality-adjusted, cleanest) → deals (fallback). +SBER_COEFF_DASHBOARDS = ("residential_real_estate_prices", "real_estate_deals") +SBER_TIME_FACTOR_MIN = 0.7 # clamp guards against bad/sparse index months +SBER_TIME_FACTOR_MAX = 1.6 + # #699: санитизация ДКП-выбросов (Росреестр `deals`). В сырых сделках встречаются # нерыночные/битые записи — доли, сделки с обременением, опечатки этажа/площади — # которые шумят actual_deals (display) и dkp_corridor/expected_sold. Абсолютные @@ -934,6 +943,60 @@ def _apply_quarter_index( return adjusted_ppm2, adjusted_median_price, adjusted_range_low, adjusted_range_high, factor +def _load_sber_index_series(db: Session, *, region: str) -> dict[date, float]: + """#794: monthly {period_month: index_value} for region from sber_price_index. + + Tries SBER_COEFF_DASHBOARDS in order; returns first non-empty series. {} on any error. + """ + for dash in SBER_COEFF_DASHBOARDS: + try: + rows = ( + db.execute( + text(""" + SELECT period_month, index_value_rub_m2 + FROM sber_price_index + WHERE city = CAST(:region AS text) + AND dashboard = CAST(:dash AS text) + ORDER BY period_month + """), + {"region": region, "dash": dash}, + ) + .mappings() + .all() + ) + except Exception as exc: + logger.warning("sber_price_index lookup failed (graceful, no-op): %s", exc) + return {} + if rows: + return {r["period_month"]: float(r["index_value_rub_m2"]) for r in rows} + return {} + + +def _sber_time_factor(series: dict[date, float], deal_month: date) -> float: + """#794: factor = idx[latest] / idx[deal_month], clamped. 1.0 when no data / recent deal. + + series: {first-of-month date -> index value}. deal_month: first-of-month of the deal. + If deal_month absent, use the nearest available month <= deal_month; if none, nearest overall. + If deal_month >= latest available month -> 1.0 (no extrapolation of recent deals). + """ + if not series: + return 1.0 + latest = max(series) + if deal_month >= latest: + return 1.0 + base = series.get(deal_month) + if base is None: + earlier = [m for m in series if m <= deal_month] + if earlier: + base = series[max(earlier)] + else: + base = series[min(series)] # deal older than series start → use earliest + if not base or base <= 0: + return 1.0 + factor = series[latest] / base + return max(SBER_TIME_FACTOR_MIN, min(SBER_TIME_FACTOR_MAX, factor)) + + def _fetch_dkp_corridor( db: Session, *, @@ -963,7 +1026,7 @@ def _fetch_dkp_corridor( db.execute( text( """ - SELECT price_per_m2 + SELECT price_per_m2, deal_date FROM deals WHERE source = 'rosreestr' AND address ILIKE :street_pattern @@ -995,9 +1058,33 @@ def _fetch_dkp_corridor( logger.warning("dkp_corridor lookup failed (graceful): %s", exc) return None - ppm2_values = sorted(float(r["price_per_m2"]) for r in rows if r["price_per_m2"]) + # #794: apply СберИндекс time-adjustment to re-base stale Rosreestr ДКП ppm² + # to the latest available index month. Graceful: factor=1.0 when table is empty. + series = _load_sber_index_series(db, region=SBER_TIME_ADJUST_REGION) + adjusted: list[float] = [] + factors_applied: list[float] = [] + for r in rows: + ppm2 = r["price_per_m2"] + if not ppm2: + continue + dd = r.get("deal_date") + factor = 1.0 + if series and dd is not None: + deal_month = date(dd.year, dd.month, 1) + factor = _sber_time_factor(series, deal_month) + adjusted.append(float(ppm2) * factor) + factors_applied.append(factor) + ppm2_values = sorted(adjusted) if not ppm2_values: return None + if series and factors_applied: + logger.info( + "dkp_corridor #794 time-adjust: n=%d factor min=%.3f max=%.3f region=%s", + len(factors_applied), + min(factors_applied), + max(factors_applied), + SBER_TIME_ADJUST_REGION, + ) return { "count": len(ppm2_values), "low_ppm2": int(ppm2_values[0]), diff --git a/tradein-mvp/backend/app/services/sber_index.py b/tradein-mvp/backend/app/services/sber_index.py index 813a1df8..7cbe48ba 100644 --- a/tradein-mvp/backend/app/services/sber_index.py +++ b/tradein-mvp/backend/app/services/sber_index.py @@ -1,4 +1,4 @@ -"""СберИндекс city-level secondary-housing price index pull (#887). +"""СберИндекс city-level secondary-housing price index pull (#887 + #794). Fetches monthly time-series data from sberindex.ru/api/sowa (reverse-engineered in vault research note 2026-05-31-0845-resolver-794) across three dashboards: @@ -10,19 +10,15 @@ API summary: POST https://sberindex.ru/api/sowa Body: {"SOWA": {"method": "GET", "route": "", "data": {"type": "object", "value": []}}} route = base64( /dataset/v1/?filter=&limit=1000&offset=0 ) - filter = {"REF_AREA": "", "FREQ": "M", "SOURCE": "SI", "REALTY": "2"} + filter = per-dashboard (see SBER_DASHBOARDS below); common keys: REF_AREA + FREQ:M Required headers: content-type, rquid (random 32-hex), x-language, user-agent. -Response fields (each cell is __string__ or __number__): - indicator_id, kpi_id, period, value, obs_status, source, ref_area, realty, - freq, unit_measure, unit_mult +Response cells: __string__ or __number__. Field names read from 'fields' entry. -Data-quality benchmark: after upsert of the dinamika-tsen-obyavlenii (asking) series, -the latest asking index per city is logged at INFO level as a data-quality reference. -Full asking-vs-our-median reconciliation is a separate follow-up (#887 step 5+). - -NOTE — estimator time-adjust wire (step 4) is OUT OF SCOPE for this PR. -It is a client-visible change that requires a backtest gate. Defer to a separate issue. +#794 fix: build_sber_route was hardcoding SOURCE:SI + REALTY:2 for ALL dashboards. +Live-verified 2026-05-31: residential_real_estate_prices needs REAL_ESTATE_NOVELTY, +dinamika-tsen-obyavlenii needs SOURCE:DK + PRICE_TYPE. Filter is now per-dashboard via +SberDashboard dataclass. REF_AREA codes (sberindex internal region IDs, not ОКАТО/ISO): 643 = Россия — confirmed live, vault research note 2026-05-31-0845. @@ -41,6 +37,7 @@ import json import logging import uuid from collections.abc import AsyncIterator +from dataclasses import dataclass from datetime import UTC, date, datetime from urllib.parse import quote @@ -63,11 +60,37 @@ SBER_REF_AREAS: dict[str, str] = { "77": "Москва", } -# Dashboards to pull: slug → human label (used as 'segment' placeholder) -SBER_DASHBOARDS: list[str] = [ - "real_estate_deals", - "residential_real_estate_prices", - "dinamika-tsen-obyavlenii", + +# --------------------------------------------------------------------------- +# Per-dashboard config (#794 fix — each dashboard needs different filter params) +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class SberDashboard: + slug: str + extra_filter: dict[str, str] # beyond REF_AREA + FREQ:"M" — verified live 2026-05-31 + segment_field: str # response field carrying the segment label + + +# All three series are the secondary-market (вторичка) ряд; +# one segment per dashboard keeps the (city,period_month,dashboard) PK valid. +SBER_DASHBOARDS: list[SberDashboard] = [ + SberDashboard( + "real_estate_deals", + {"SOURCE": "SI", "REALTY": "2"}, + "realty", + ), + SberDashboard( + "residential_real_estate_prices", + {"SOURCE": "SI", "REAL_ESTATE_TYPE": "1", "REAL_ESTATE_NOVELTY": "3"}, + "real_estate_novelty", + ), + SberDashboard( + "dinamika-tsen-obyavlenii", + {"SOURCE": "DK", "PRICE_TYPE": "1"}, + "price_type", + ), ] SBER_API_URL = "https://sberindex.ru/api/sowa" @@ -83,20 +106,16 @@ _CHROME_UA = ( # --------------------------------------------------------------------------- -def build_sber_route(dashboard: str, ref_area: str) -> str: +def build_sber_route(dashboard: str, ref_area: str, extra_filter: dict[str, str]) -> str: """Build the base64-encoded route for the /api/sowa payload. Encodes /dataset/v1/?filter=&limit=1000&offset=0 - where filter = {"REF_AREA": ref_area, "FREQ": "M", "SOURCE": "SI", "REALTY": "2"}. + where filter = {"REF_AREA": ref_area, "FREQ": "M", **extra_filter}. + extra_filter is per-dashboard (see SberDashboard.extra_filter); verified live 2026-05-31. All filter values are strings per the live-captured payload (vault research note). """ - filter_dict = { - "REF_AREA": str(ref_area), - "FREQ": "M", - "SOURCE": "SI", - "REALTY": "2", - } + filter_dict = {"REF_AREA": str(ref_area), "FREQ": "M", **extra_filter} # urlencode the JSON, then build the path filter_str = quote(json.dumps(filter_dict, separators=(",", ":")), safe="") path = f"/dataset/v1/{dashboard}?filter={filter_str}&limit=1000&offset=0" @@ -107,21 +126,6 @@ def build_sber_route(dashboard: str, ref_area: str) -> str: # Response decoder # --------------------------------------------------------------------------- -# Field order returned by the API (confirmed from live session) -_FIELD_ORDER = [ - "indicator_id", - "kpi_id", - "period", - "value", - "obs_status", - "source", - "ref_area", - "realty", - "freq", - "unit_measure", - "unit_mult", -] - def _decode_cell(cell: str) -> str | float: """Decode a single response cell. @@ -212,19 +216,20 @@ def _parse_period_month(period_str: str) -> date: async def fetch_sber_index( client: httpx.AsyncClient, *, - dashboard: str, + dashboard: SberDashboard, ref_area: str, ) -> AsyncIterator[tuple[str, date, str, str, float]]: """Fetch one dashboard × REF_AREA combination from sberindex.ru/api/sowa. Yields tuples of: - (city_label, period_month, segment_label, dashboard, index_value_rub_m2) + (city_label, period_month, segment_label, dashboard_slug, index_value_rub_m2) - city_label and segment_label come from the API response fields 'ref_area' and 'realty'. + city_label comes from the API response field 'ref_area'. + segment_label comes from dashboard.segment_field (per-dashboard, verified 2026-05-31). Raises httpx.HTTPStatusError on non-2xx response (caller handles per-series). """ - route = build_sber_route(dashboard, ref_area) + route = build_sber_route(dashboard.slug, ref_area, dashboard.extra_filter) rquid = uuid.uuid4().hex # 32-hex random, required header payload = { @@ -251,7 +256,7 @@ async def fetch_sber_index( period_raw = row.get("period", "") value_raw = row.get("value") city_label = str(row.get("ref_area", ref_area)) - segment_label = str(row.get("realty", "")) + segment_label = str(row.get(dashboard.segment_field, "")) if not period_raw or value_raw is None: continue @@ -262,13 +267,13 @@ async def fetch_sber_index( except (ValueError, TypeError) as exc: logger.warning( "sber_index: skip row — parse error for dashboard=%s ref_area=%s: %s", - dashboard, + dashboard.slug, ref_area, exc, ) continue - yield (city_label, period_month, segment_label, dashboard, index_value) + yield (city_label, period_month, segment_label, dashboard.slug, index_value) # --------------------------------------------------------------------------- @@ -280,20 +285,20 @@ async def pull_sber_indices( db: Session, *, cities: dict[str, str] | None = None, - dashboards: list[str] | None = None, + dashboards: list[SberDashboard] | None = None, ) -> dict[str, int]: """Fetch all configured dashboards × cities and upsert into sber_price_index. Args: db: SQLAlchemy Session (tradein DB). cities: REF_AREA code → city label mapping. Defaults to SBER_REF_AREAS. - dashboards: list of dashboard slugs to pull. Defaults to SBER_DASHBOARDS. + dashboards: list of SberDashboard configs to pull. Defaults to SBER_DASHBOARDS. Returns dict of counters: {upserted: int, skipped: int, errors: int}. Per-series try/except: one failing series logs + continues (does not abort others). After upserting 'dinamika-tsen-obyavlenii', logs the latest asking index per city - as a data-quality benchmark reference. + as a data-quality benchmark reference and compares to our scrape-median asking ppm². """ if cities is None: cities = SBER_REF_AREAS @@ -315,7 +320,7 @@ async def pull_sber_indices( if not rows_to_upsert: logger.info( "sber_index: no rows returned for dashboard=%s ref_area=%s", - dashboard, + dashboard.slug, ref_area, ) counters["skipped"] += 1 @@ -359,35 +364,61 @@ async def pull_sber_indices( logger.info( "sber_index: upserted %d rows for dashboard=%s ref_area=%s", len(rows_to_upsert), - dashboard, + dashboard.slug, ref_area, ) # Data-quality benchmark log for asking-price series - if dashboard == "dinamika-tsen-obyavlenii" and rows_to_upsert: + if dashboard.slug == "dinamika-tsen-obyavlenii" and rows_to_upsert: latest_row = max(rows_to_upsert, key=lambda r: r[1]) city_lbl, latest_month, _, _, latest_val = latest_row logger.info( "sber_index benchmark [asking]: city=%r latest_month=%s " "index_value_rub_m2=%.0f " - "(data-quality ref; full asking-vs-our-median is follow-up)", + "(data-quality ref; full asking-vs-our-median below)", city_lbl, latest_month.isoformat(), latest_val, ) + # Task B: best-effort reconciliation to our scrape-median asking ppm² + try: + our_median = db.execute( + text(""" + SELECT percentile_cont(0.5) + WITHIN GROUP (ORDER BY price_per_m2) AS m + FROM listings + WHERE is_active = true + AND price_per_m2 BETWEEN 50000 AND 800000 + """) + ).scalar() + if our_median is not None and latest_val: + sber_latest = latest_val + divergence_pct = (our_median - sber_latest) / sber_latest * 100 + logger.info( + "sber_index benchmark [asking vs our median]: " + "sber=%.0f our_median=%.0f divergence=%+.1f%% " + "(data-quality signal)", + sber_latest, + our_median, + divergence_pct, + ) + except Exception as exc: + logger.debug( + "sber benchmark reconciliation skipped (graceful): %s", exc + ) except httpx.HTTPStatusError as exc: logger.error( "sber_index: HTTP %d for dashboard=%s ref_area=%s — skip series", exc.response.status_code, - dashboard, + dashboard.slug, ref_area, ) counters["errors"] += 1 except httpx.RequestError as exc: logger.error( "sber_index: network error for dashboard=%s ref_area=%s: %s — skip", - dashboard, + dashboard.slug, ref_area, exc, ) @@ -395,7 +426,7 @@ async def pull_sber_indices( except Exception: logger.exception( "sber_index: unexpected error for dashboard=%s ref_area=%s — skip", - dashboard, + dashboard.slug, ref_area, ) counters["errors"] += 1 diff --git a/tradein-mvp/backend/tests/fixtures/sber_real_estate_deals_66.json b/tradein-mvp/backend/tests/fixtures/sber_real_estate_deals_66.json new file mode 100644 index 00000000..d0a2fb9d --- /dev/null +++ b/tradein-mvp/backend/tests/fixtures/sber_real_estate_deals_66.json @@ -0,0 +1,130 @@ +{ + "SOWA": { + "method": "GET", + "route": "L2RhdGFzZXQvdjEvcmVhbF9lc3RhdGVfZGVhbHM/ZmlsdGVyPSU3QiUyMlJFRl9BUkVBJTIyJTNBJTIyNjYlMjIlMkMlMjJGUkVRJTIyJTNBJTIyTSUyMiUyQyUyMlNPVVJDRSUyMiUzQSUyMlNJJTIyJTJDJTIyUkVBTFRZJTIyJTNBJTIyMiUyMiU3RCZsaW1pdD00Jm9mZnNldD0w", + "data": { + "type": "object", + "value": [ + { + "key": "fields", + "type": "object", + "value": [ + "__string__aW5kaWNhdG9yX2lk", + "__string__a3BpX2lk", + "__string__cGVyaW9k", + "__string__dmFsdWU=", + "__string__b2JzX3N0YXR1cw==", + "__string__c291cmNl", + "__string__cmVmX2FyZWE=", + "__string__cmVhbHR5", + "__string__ZnJlcQ==", + "__string__dW5pdF9tZWFzdXJl", + "__string__dW5pdF9tdWx0" + ] + }, + { + "key": "data", + "type": "object", + "value": [ + [ + "__string__ODM0NTUxNTktMjYyOC00ZmNlLWIzNDktZGZjODU3MTU1ZjEx", + "__string__NjQzMDMyMDAx", + "__string__MjAxNy0wMS0zMFQyMTowMDowMC4wMDBa", + "__number__51046", + "__string__QQ==", + "__string__0JTQsNC90L3Ri9C1INCh0LHQtdGA0JjQvdC00LXQutGB0LA=", + "__string__0KHQstC10YDQtNC70L7QstGB0LrQsNGPINC+0LHQu9Cw0YHRgtGM", + "__string__0JLRgtC+0YDQuNGH0L3Ri9C5", + "__string__0JzQtdGB0Y/Rhg==", + "__string__0YDRg9Cx", + "__string__MA==" + ], + [ + "__string__ODM0NTUxNTktMjYyOC00ZmNlLWIzNDktZGZjODU3MTU1ZjEx", + "__string__NjQzMDMyMDAx", + "__string__MjAxNy0wMi0yN1QyMTowMDowMC4wMDBa", + "__number__50762", + "__string__QQ==", + "__string__0JTQsNC90L3Ri9C1INCh0LHQtdGA0JjQvdC00LXQutGB0LA=", + "__string__0KHQstC10YDQtNC70L7QstGB0LrQsNGPINC+0LHQu9Cw0YHRgtGM", + "__string__0JLRgtC+0YDQuNGH0L3Ri9C5", + "__string__0JzQtdGB0Y/Rhg==", + "__string__0YDRg9Cx", + "__string__MA==" + ], + [ + "__string__ODM0NTUxNTktMjYyOC00ZmNlLWIzNDktZGZjODU3MTU1ZjEx", + "__string__NjQzMDMyMDAx", + "__string__MjAxNy0wMy0zMFQyMTowMDowMC4wMDBa", + "__number__50902", + "__string__QQ==", + "__string__0JTQsNC90L3Ri9C1INCh0LHQtdGA0JjQvdC00LXQutGB0LA=", + "__string__0KHQstC10YDQtNC70L7QstGB0LrQsNGPINC+0LHQu9Cw0YHRgtGM", + "__string__0JLRgtC+0YDQuNGH0L3Ri9C5", + "__string__0JzQtdGB0Y/Rhg==", + "__string__0YDRg9Cx", + "__string__MA==" + ], + [ + "__string__ODM0NTUxNTktMjYyOC00ZmNlLWIzNDktZGZjODU3MTU1ZjEx", + "__string__NjQzMDMyMDAx", + "__string__MjAxNy0wNC0yOVQyMTowMDowMC4wMDBa", + "__number__50958", + "__string__QQ==", + "__string__0JTQsNC90L3Ri9C1INCh0LHQtdGA0JjQvdC00LXQutGB0LA=", + "__string__0KHQstC10YDQtNC70L7QstGB0LrQsNGPINC+0LHQu9Cw0YHRgtGM", + "__string__0JLRgtC+0YDQuNGH0L3Ri9C5", + "__string__0JzQtdGB0Y/Rhg==", + "__string__0YDRg9Cx", + "__string__MA==" + ] + ] + }, + { + "key": "pagination", + "type": "object", + "value": { + "type": "object", + "value": [ + { + "key": "total_records", + "type": "number", + "value": "__number__112" + }, + { + "key": "current_page", + "type": "number", + "value": "__number__1" + }, + { + "key": "total_pages", + "type": "number", + "value": "__number__28" + }, + { + "key": "next_page", + "type": "number", + "value": "__number__2" + }, + { + "key": "prev_page", + "type": "null", + "value": "__null__null" + } + ] + } + }, + { + "key": "limit", + "type": "number", + "value": "__number__4" + }, + { + "key": "offset", + "type": "number", + "value": "__number__0" + } + ] + } + } +} diff --git a/tradein-mvp/backend/tests/services/test_estimator_sber_time_adjust.py b/tradein-mvp/backend/tests/services/test_estimator_sber_time_adjust.py new file mode 100644 index 00000000..990566ca --- /dev/null +++ b/tradein-mvp/backend/tests/services/test_estimator_sber_time_adjust.py @@ -0,0 +1,80 @@ +"""Tests for #794 СберИндекс time-adjustment helper — pure, no DB.""" + +from __future__ import annotations + +import os +import sys +from datetime import date +from unittest.mock import MagicMock + +# DATABASE_URL required by pydantic Settings before any app import. +os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test") + +# WeasyPrint stub — not installed in CI без GTK. +sys.modules.setdefault("weasyprint", MagicMock()) + +from app.services.estimator import ( # noqa: E402 + SBER_TIME_FACTOR_MAX, + SBER_TIME_FACTOR_MIN, + _sber_time_factor, +) + + +def test_empty_series_returns_one() -> None: + assert _sber_time_factor({}, date(2026, 1, 1)) == 1.0 + + +def test_deal_month_equals_latest_returns_one() -> None: + series = {date(2026, 1, 1): 95_000.0, date(2026, 4, 1): 100_000.0} + assert _sber_time_factor(series, date(2026, 4, 1)) == 1.0 + + +def test_deal_month_after_latest_returns_one() -> None: + series = {date(2026, 1, 1): 95_000.0, date(2026, 4, 1): 100_000.0} + assert _sber_time_factor(series, date(2026, 6, 1)) == 1.0 + + +def test_normal_factor_calculation() -> None: + series = {date(2026, 1, 1): 95_000.0, date(2026, 4, 1): 100_000.0} + factor = _sber_time_factor(series, date(2026, 1, 1)) + expected = 100_000.0 / 95_000.0 # ≈ 1.0526 + assert abs(factor - expected) < 0.0001 + assert SBER_TIME_FACTOR_MIN <= factor <= SBER_TIME_FACTOR_MAX + + +def test_clamp_high() -> None: + """Very old deal → factor would be >>1.6 → clamp to SBER_TIME_FACTOR_MAX.""" + series = {date(2020, 1, 1): 10_000.0, date(2026, 4, 1): 100_000.0} + factor = _sber_time_factor(series, date(2020, 1, 1)) + assert factor == SBER_TIME_FACTOR_MAX + + +def test_clamp_low() -> None: + """Series shows price drop → factor <<0.7 → clamp to SBER_TIME_FACTOR_MIN.""" + series = {date(2020, 1, 1): 200_000.0, date(2026, 4, 1): 50_000.0} + factor = _sber_time_factor(series, date(2020, 1, 1)) + assert factor == SBER_TIME_FACTOR_MIN + + +def test_missing_month_uses_nearest_earlier() -> None: + """deal_month=2026-02 not in series; nearest earlier is 2026-01.""" + series = {date(2026, 1, 1): 95_000.0, date(2026, 4, 1): 100_000.0} + # Using 2026-01 as base: factor = 100000/95000 + factor_with_missing = _sber_time_factor(series, date(2026, 2, 1)) + factor_explicit = _sber_time_factor(series, date(2026, 1, 1)) + assert abs(factor_with_missing - factor_explicit) < 0.0001 + + +def test_deal_older_than_series_uses_earliest() -> None: + """Deal month before all series data → use earliest available month.""" + series = {date(2023, 1, 1): 80_000.0, date(2026, 4, 1): 100_000.0} + # deal at 2020-01 is before series start → base = series[2023-01] = 80000 + factor = _sber_time_factor(series, date(2020, 1, 1)) + expected = min(100_000.0 / 80_000.0, SBER_TIME_FACTOR_MAX) + assert abs(factor - expected) < 0.0001 + + +def test_zero_base_returns_one() -> None: + """Zero index value in series → guard → return 1.0.""" + series = {date(2026, 1, 1): 0.0, date(2026, 4, 1): 100_000.0} + assert _sber_time_factor(series, date(2026, 1, 1)) == 1.0 diff --git a/tradein-mvp/backend/tests/services/test_sber_index.py b/tradein-mvp/backend/tests/services/test_sber_index.py new file mode 100644 index 00000000..804f92b3 --- /dev/null +++ b/tradein-mvp/backend/tests/services/test_sber_index.py @@ -0,0 +1,113 @@ +"""Tests for app/services/sber_index.py — per-dashboard filter fix (#794).""" + +from __future__ import annotations + +import base64 +import json +from datetime import date +from pathlib import Path +from urllib.parse import unquote + +import pytest + +from app.services.sber_index import ( + SBER_DASHBOARDS, + SberDashboard, + _parse_period_month, + build_sber_route, + decode_sber_response, +) + +# --------------------------------------------------------------------------- +# Fixture path +# --------------------------------------------------------------------------- + +FIXTURE_PATH = Path(__file__).parent.parent / "fixtures" / "sber_real_estate_deals_66.json" + + +# --------------------------------------------------------------------------- +# test_build_sber_route_per_dashboard +# LOCKS the bug fix: each dashboard must have the correct filter keys. +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("dashboard", SBER_DASHBOARDS) +def test_build_sber_route_per_dashboard(dashboard: SberDashboard) -> None: + route_b64 = build_sber_route(dashboard.slug, "66", dashboard.extra_filter) + path = base64.b64decode(route_b64).decode("utf-8") + + # Extract the filter JSON from the path: ?filter=&limit=... + filter_part = path.split("filter=")[1].split("&")[0] + filter_dict = json.loads(unquote(filter_part)) + + # All routes must have REF_AREA + FREQ=M + assert filter_dict.get("REF_AREA") == "66" + assert filter_dict.get("FREQ") == "M" + + if dashboard.slug == "real_estate_deals": + assert filter_dict.get("SOURCE") == "SI" + assert filter_dict.get("REALTY") == "2" + + elif dashboard.slug == "residential_real_estate_prices": + assert filter_dict.get("SOURCE") == "SI" + assert filter_dict.get("REAL_ESTATE_NOVELTY") == "3" + + elif dashboard.slug == "dinamika-tsen-obyavlenii": + assert filter_dict.get("SOURCE") == "DK" + assert filter_dict.get("PRICE_TYPE") == "1" + + else: + pytest.fail(f"Unknown dashboard slug: {dashboard.slug!r}") + + +# --------------------------------------------------------------------------- +# test_decode_sber_response_real_fixture +# --------------------------------------------------------------------------- + + +def test_decode_sber_response_real_fixture() -> None: + with FIXTURE_PATH.open(encoding="utf-8") as f: + raw = json.load(f) + + rows = decode_sber_response(raw) + + assert len(rows) >= 1, "Expected at least 1 decoded row from real fixture" + + # All rows should have the expected fields present + first = rows[0] + assert "ref_area" in first + assert "realty" in first + assert "value" in first + assert "period" in first + + # Values must be typed correctly + assert isinstance(first["value"], float) + assert isinstance(first["period"], str) and first["period"] + + # Verify region label decoded correctly + regions = {r["ref_area"] for r in rows} + assert ( + "Свердловская область" in regions + ), f"Expected 'Свердловская область' in ref_area values, got: {regions}" + + # Verify secondary-market segment present + realty_vals = {r["realty"] for r in rows} + assert any( + "тори" in str(v) or "Вторичн" in str(v) for v in realty_vals + ), f"Expected secondary-market label in realty field, got: {realty_vals}" + + +# --------------------------------------------------------------------------- +# test_parse_period_month +# --------------------------------------------------------------------------- + + +def test_parse_period_month() -> None: + result = _parse_period_month("2026-04-29T21:00:00.000Z") + assert result == date(2026, 4, 1) + + +def test_parse_period_month_z_suffix() -> None: + """Original format from live API.""" + result = _parse_period_month("2017-01-30T21:00:00Z") + assert result == date(2017, 1, 1)