feat(tradein): СберИндекс time-adjust for frozen Rosreestr ДКП + per-dashboard filter fix (#794) #919

Merged
bot-reviewer merged 2 commits from feat/794-sber-time-adjust into main 2026-05-31 18:48:23 +00:00
5 changed files with 503 additions and 59 deletions

View file

@ -26,7 +26,7 @@ import logging
import math
import re
import time
from datetime import UTC, datetime, timedelta
from datetime import UTC, date, datetime, timedelta
from typing import Any
from uuid import uuid4
@ -78,6 +78,15 @@ MIN_ANALOGS_PER_SOURCE = 5 # гарантированный минимум на
LISTINGS_FRESH_DAYS = 14 # объявления не старше 14 дней
DEALS_PERIOD_MONTHS = 12 # сделки за последний год
# #794: СберИндекс time-adjustment of frozen Rosreestr ДКП deals.
# Rosreestr deals freeze ~2026-01; the sber monthly index re-bases a stale deal's ppm²
# to the latest available month. Region fixed to Свердловская обл. (tradein MVP = ЕКБ).
SBER_TIME_ADJUST_REGION = "Свердловская область"
# Coefficient series preference: hedonic (quality-adjusted, cleanest) → deals (fallback).
SBER_COEFF_DASHBOARDS = ("residential_real_estate_prices", "real_estate_deals")
SBER_TIME_FACTOR_MIN = 0.7 # clamp guards against bad/sparse index months
SBER_TIME_FACTOR_MAX = 1.6
# #699: санитизация ДКП-выбросов (Росреестр `deals`). В сырых сделках встречаются
# нерыночные/битые записи — доли, сделки с обременением, опечатки этажа/площади —
# которые шумят actual_deals (display) и dkp_corridor/expected_sold. Абсолютные
@ -934,6 +943,60 @@ def _apply_quarter_index(
return adjusted_ppm2, adjusted_median_price, adjusted_range_low, adjusted_range_high, factor
def _load_sber_index_series(db: Session, *, region: str) -> dict[date, float]:
"""#794: monthly {period_month: index_value} for region from sber_price_index.
Tries SBER_COEFF_DASHBOARDS in order; returns first non-empty series. {} on any error.
"""
for dash in SBER_COEFF_DASHBOARDS:
try:
rows = (
db.execute(
text("""
SELECT period_month, index_value_rub_m2
FROM sber_price_index
WHERE city = CAST(:region AS text)
AND dashboard = CAST(:dash AS text)
ORDER BY period_month
"""),
{"region": region, "dash": dash},
)
.mappings()
.all()
)
except Exception as exc:
logger.warning("sber_price_index lookup failed for %s (graceful): %s", dash, exc)
continue
if rows:
return {r["period_month"]: float(r["index_value_rub_m2"]) for r in rows}
return {}
def _sber_time_factor(series: dict[date, float], deal_month: date) -> float:
"""#794: factor = idx[latest] / idx[deal_month], clamped. 1.0 when no data / recent deal.
series: {first-of-month date -> index value}. deal_month: first-of-month of the deal.
If deal_month absent, use the nearest available month <= deal_month; if none, nearest overall.
If deal_month >= latest available month -> 1.0 (no extrapolation of recent deals).
"""
if not series:
return 1.0
latest = max(series)
if deal_month >= latest:
return 1.0
base = series.get(deal_month)
if base is None:
earlier = [m for m in series if m <= deal_month]
if earlier:
base = series[max(earlier)]
else:
base = series[min(series)] # deal older than series start → use earliest
if not base or base <= 0:
return 1.0
factor = series[latest] / base
return max(SBER_TIME_FACTOR_MIN, min(SBER_TIME_FACTOR_MAX, factor))
def _fetch_dkp_corridor(
db: Session,
*,
@ -963,7 +1026,7 @@ def _fetch_dkp_corridor(
db.execute(
text(
"""
SELECT price_per_m2
SELECT price_per_m2, deal_date
FROM deals
WHERE source = 'rosreestr'
AND address ILIKE :street_pattern
@ -995,9 +1058,33 @@ def _fetch_dkp_corridor(
logger.warning("dkp_corridor lookup failed (graceful): %s", exc)
return None
ppm2_values = sorted(float(r["price_per_m2"]) for r in rows if r["price_per_m2"])
# #794: apply СберИндекс time-adjustment to re-base stale Rosreestr ДКП ppm²
# to the latest available index month. Graceful: factor=1.0 when table is empty.
series = _load_sber_index_series(db, region=SBER_TIME_ADJUST_REGION)
adjusted: list[float] = []
factors_applied: list[float] = []
for r in rows:
ppm2 = r["price_per_m2"]
if not ppm2:
continue
dd = r.get("deal_date")
factor = 1.0
if series and dd is not None:
deal_month = date(dd.year, dd.month, 1)
factor = _sber_time_factor(series, deal_month)
adjusted.append(float(ppm2) * factor)
factors_applied.append(factor)
ppm2_values = sorted(adjusted)
if not ppm2_values:
return None
if series and factors_applied:
logger.info(
"dkp_corridor #794 time-adjust: n=%d factor min=%.3f max=%.3f region=%s",
len(factors_applied),
min(factors_applied),
max(factors_applied),
SBER_TIME_ADJUST_REGION,
)
return {
"count": len(ppm2_values),
"low_ppm2": int(ppm2_values[0]),

View file

@ -1,4 +1,4 @@
"""СберИндекс city-level secondary-housing price index pull (#887).
"""СберИндекс city-level secondary-housing price index pull (#887 + #794).
Fetches monthly time-series data from sberindex.ru/api/sowa (reverse-engineered
in vault research note 2026-05-31-0845-resolver-794) across three dashboards:
@ -10,19 +10,15 @@ API summary:
POST https://sberindex.ru/api/sowa
Body: {"SOWA": {"method": "GET", "route": "<base64>", "data": {"type": "object", "value": []}}}
route = base64( /dataset/v1/<dashboard>?filter=<urlencode(json)>&limit=1000&offset=0 )
filter = {"REF_AREA": "<code>", "FREQ": "M", "SOURCE": "SI", "REALTY": "2"}
filter = per-dashboard (see SBER_DASHBOARDS below); common keys: REF_AREA + FREQ:M
Required headers: content-type, rquid (random 32-hex), x-language, user-agent.
Response fields (each cell is __string__<b64> or __number__<raw>):
indicator_id, kpi_id, period, value, obs_status, source, ref_area, realty,
freq, unit_measure, unit_mult
Response cells: __string__<b64> or __number__<raw>. Field names read from 'fields' entry.
Data-quality benchmark: after upsert of the dinamika-tsen-obyavlenii (asking) series,
the latest asking index per city is logged at INFO level as a data-quality reference.
Full asking-vs-our-median reconciliation is a separate follow-up (#887 step 5+).
NOTE estimator time-adjust wire (step 4) is OUT OF SCOPE for this PR.
It is a client-visible change that requires a backtest gate. Defer to a separate issue.
#794 fix: build_sber_route was hardcoding SOURCE:SI + REALTY:2 for ALL dashboards.
Live-verified 2026-05-31: residential_real_estate_prices needs REAL_ESTATE_NOVELTY,
dinamika-tsen-obyavlenii needs SOURCE:DK + PRICE_TYPE. Filter is now per-dashboard via
SberDashboard dataclass.
REF_AREA codes (sberindex internal region IDs, not ОКАТО/ISO):
643 = Россия confirmed live, vault research note 2026-05-31-0845.
@ -41,6 +37,7 @@ import json
import logging
import uuid
from collections.abc import AsyncIterator
from dataclasses import dataclass
from datetime import UTC, date, datetime
from urllib.parse import quote
@ -63,11 +60,37 @@ SBER_REF_AREAS: dict[str, str] = {
"77": "Москва",
}
# Dashboards to pull: slug → human label (used as 'segment' placeholder)
SBER_DASHBOARDS: list[str] = [
"real_estate_deals",
"residential_real_estate_prices",
"dinamika-tsen-obyavlenii",
# ---------------------------------------------------------------------------
# Per-dashboard config (#794 fix — each dashboard needs different filter params)
# ---------------------------------------------------------------------------
@dataclass(frozen=True)
class SberDashboard:
slug: str
extra_filter: dict[str, str] # beyond REF_AREA + FREQ:"M" — verified live 2026-05-31
segment_field: str # response field carrying the segment label
# All three series are the secondary-market (вторичка) ряд;
# one segment per dashboard keeps the (city,period_month,dashboard) PK valid.
SBER_DASHBOARDS: list[SberDashboard] = [
SberDashboard(
"real_estate_deals",
{"SOURCE": "SI", "REALTY": "2"},
"realty",
),
SberDashboard(
"residential_real_estate_prices",
{"SOURCE": "SI", "REAL_ESTATE_TYPE": "1", "REAL_ESTATE_NOVELTY": "3"},
"real_estate_novelty",
),
SberDashboard(
"dinamika-tsen-obyavlenii",
{"SOURCE": "DK", "PRICE_TYPE": "1"},
"price_type",
),
]
SBER_API_URL = "https://sberindex.ru/api/sowa"
@ -83,20 +106,16 @@ _CHROME_UA = (
# ---------------------------------------------------------------------------
def build_sber_route(dashboard: str, ref_area: str) -> str:
def build_sber_route(dashboard: str, ref_area: str, extra_filter: dict[str, str]) -> str:
"""Build the base64-encoded route for the /api/sowa payload.
Encodes /dataset/v1/<dashboard>?filter=<urlencode(json)>&limit=1000&offset=0
where filter = {"REF_AREA": ref_area, "FREQ": "M", "SOURCE": "SI", "REALTY": "2"}.
where filter = {"REF_AREA": ref_area, "FREQ": "M", **extra_filter}.
extra_filter is per-dashboard (see SberDashboard.extra_filter); verified live 2026-05-31.
All filter values are strings per the live-captured payload (vault research note).
"""
filter_dict = {
"REF_AREA": str(ref_area),
"FREQ": "M",
"SOURCE": "SI",
"REALTY": "2",
}
filter_dict = {"REF_AREA": str(ref_area), "FREQ": "M", **extra_filter}
# urlencode the JSON, then build the path
filter_str = quote(json.dumps(filter_dict, separators=(",", ":")), safe="")
path = f"/dataset/v1/{dashboard}?filter={filter_str}&limit=1000&offset=0"
@ -107,21 +126,6 @@ def build_sber_route(dashboard: str, ref_area: str) -> str:
# Response decoder
# ---------------------------------------------------------------------------
# Field order returned by the API (confirmed from live session)
_FIELD_ORDER = [
"indicator_id",
"kpi_id",
"period",
"value",
"obs_status",
"source",
"ref_area",
"realty",
"freq",
"unit_measure",
"unit_mult",
]
def _decode_cell(cell: str) -> str | float:
"""Decode a single response cell.
@ -212,19 +216,20 @@ def _parse_period_month(period_str: str) -> date:
async def fetch_sber_index(
client: httpx.AsyncClient,
*,
dashboard: str,
dashboard: SberDashboard,
ref_area: str,
) -> AsyncIterator[tuple[str, date, str, str, float]]:
"""Fetch one dashboard × REF_AREA combination from sberindex.ru/api/sowa.
Yields tuples of:
(city_label, period_month, segment_label, dashboard, index_value_rub_m2)
(city_label, period_month, segment_label, dashboard_slug, index_value_rub_m2)
city_label and segment_label come from the API response fields 'ref_area' and 'realty'.
city_label comes from the API response field 'ref_area'.
segment_label comes from dashboard.segment_field (per-dashboard, verified 2026-05-31).
Raises httpx.HTTPStatusError on non-2xx response (caller handles per-series).
"""
route = build_sber_route(dashboard, ref_area)
route = build_sber_route(dashboard.slug, ref_area, dashboard.extra_filter)
rquid = uuid.uuid4().hex # 32-hex random, required header
payload = {
@ -251,7 +256,7 @@ async def fetch_sber_index(
period_raw = row.get("period", "")
value_raw = row.get("value")
city_label = str(row.get("ref_area", ref_area))
segment_label = str(row.get("realty", ""))
segment_label = str(row.get(dashboard.segment_field, ""))
if not period_raw or value_raw is None:
continue
@ -262,13 +267,13 @@ async def fetch_sber_index(
except (ValueError, TypeError) as exc:
logger.warning(
"sber_index: skip row — parse error for dashboard=%s ref_area=%s: %s",
dashboard,
dashboard.slug,
ref_area,
exc,
)
continue
yield (city_label, period_month, segment_label, dashboard, index_value)
yield (city_label, period_month, segment_label, dashboard.slug, index_value)
# ---------------------------------------------------------------------------
@ -280,20 +285,20 @@ async def pull_sber_indices(
db: Session,
*,
cities: dict[str, str] | None = None,
dashboards: list[str] | None = None,
dashboards: list[SberDashboard] | None = None,
) -> dict[str, int]:
"""Fetch all configured dashboards × cities and upsert into sber_price_index.
Args:
db: SQLAlchemy Session (tradein DB).
cities: REF_AREA code city label mapping. Defaults to SBER_REF_AREAS.
dashboards: list of dashboard slugs to pull. Defaults to SBER_DASHBOARDS.
dashboards: list of SberDashboard configs to pull. Defaults to SBER_DASHBOARDS.
Returns dict of counters: {upserted: int, skipped: int, errors: int}.
Per-series try/except: one failing series logs + continues (does not abort others).
After upserting 'dinamika-tsen-obyavlenii', logs the latest asking index per city
as a data-quality benchmark reference.
as a data-quality benchmark reference and compares to our scrape-median asking ppm².
"""
if cities is None:
cities = SBER_REF_AREAS
@ -315,7 +320,7 @@ async def pull_sber_indices(
if not rows_to_upsert:
logger.info(
"sber_index: no rows returned for dashboard=%s ref_area=%s",
dashboard,
dashboard.slug,
ref_area,
)
counters["skipped"] += 1
@ -359,35 +364,64 @@ async def pull_sber_indices(
logger.info(
"sber_index: upserted %d rows for dashboard=%s ref_area=%s",
len(rows_to_upsert),
dashboard,
dashboard.slug,
ref_area,
)
# Data-quality benchmark log for asking-price series
if dashboard == "dinamika-tsen-obyavlenii" and rows_to_upsert:
if dashboard.slug == "dinamika-tsen-obyavlenii" and rows_to_upsert:
latest_row = max(rows_to_upsert, key=lambda r: r[1])
city_lbl, latest_month, _, _, latest_val = latest_row
logger.info(
"sber_index benchmark [asking]: city=%r latest_month=%s "
"index_value_rub_m2=%.0f "
"(data-quality ref; full asking-vs-our-median is follow-up)",
"(data-quality ref; full asking-vs-our-median below)",
city_lbl,
latest_month.isoformat(),
latest_val,
)
# our listings median is ЕКБ-only →
# compare only against Свердловская обл. (REF_AREA 66)
if ref_area == "66":
# Task B: best-effort reconciliation to our scrape-median asking ppm²
try:
our_median = db.execute(
text("""
SELECT percentile_cont(0.5)
WITHIN GROUP (ORDER BY price_per_m2) AS m
FROM listings
WHERE is_active = true
AND price_per_m2 BETWEEN 50000 AND 800000
""")
).scalar()
if our_median is not None and latest_val:
sber_latest = latest_val
divergence_pct = (our_median - sber_latest) / sber_latest * 100
logger.info(
"sber_index benchmark [asking vs our median]: "
"sber=%.0f our_median=%.0f divergence=%+.1f%% "
"(data-quality signal)",
sber_latest,
our_median,
divergence_pct,
)
except Exception as exc:
logger.debug(
"sber benchmark reconciliation skipped (graceful): %s", exc
)
except httpx.HTTPStatusError as exc:
logger.error(
"sber_index: HTTP %d for dashboard=%s ref_area=%s — skip series",
exc.response.status_code,
dashboard,
dashboard.slug,
ref_area,
)
counters["errors"] += 1
except httpx.RequestError as exc:
logger.error(
"sber_index: network error for dashboard=%s ref_area=%s: %s — skip",
dashboard,
dashboard.slug,
ref_area,
exc,
)
@ -395,7 +429,7 @@ async def pull_sber_indices(
except Exception:
logger.exception(
"sber_index: unexpected error for dashboard=%s ref_area=%s — skip",
dashboard,
dashboard.slug,
ref_area,
)
counters["errors"] += 1

View file

@ -0,0 +1,130 @@
{
"SOWA": {
"method": "GET",
"route": "L2RhdGFzZXQvdjEvcmVhbF9lc3RhdGVfZGVhbHM/ZmlsdGVyPSU3QiUyMlJFRl9BUkVBJTIyJTNBJTIyNjYlMjIlMkMlMjJGUkVRJTIyJTNBJTIyTSUyMiUyQyUyMlNPVVJDRSUyMiUzQSUyMlNJJTIyJTJDJTIyUkVBTFRZJTIyJTNBJTIyMiUyMiU3RCZsaW1pdD00Jm9mZnNldD0w",
"data": {
"type": "object",
"value": [
{
"key": "fields",
"type": "object",
"value": [
"__string__aW5kaWNhdG9yX2lk",
"__string__a3BpX2lk",
"__string__cGVyaW9k",
"__string__dmFsdWU=",
"__string__b2JzX3N0YXR1cw==",
"__string__c291cmNl",
"__string__cmVmX2FyZWE=",
"__string__cmVhbHR5",
"__string__ZnJlcQ==",
"__string__dW5pdF9tZWFzdXJl",
"__string__dW5pdF9tdWx0"
]
},
{
"key": "data",
"type": "object",
"value": [
[
"__string__ODM0NTUxNTktMjYyOC00ZmNlLWIzNDktZGZjODU3MTU1ZjEx",
"__string__NjQzMDMyMDAx",
"__string__MjAxNy0wMS0zMFQyMTowMDowMC4wMDBa",
"__number__51046",
"__string__QQ==",
"__string__0JTQsNC90L3Ri9C1INCh0LHQtdGA0JjQvdC00LXQutGB0LA=",
"__string__0KHQstC10YDQtNC70L7QstGB0LrQsNGPINC+0LHQu9Cw0YHRgtGM",
"__string__0JLRgtC+0YDQuNGH0L3Ri9C5",
"__string__0JzQtdGB0Y/Rhg==",
"__string__0YDRg9Cx",
"__string__MA=="
],
[
"__string__ODM0NTUxNTktMjYyOC00ZmNlLWIzNDktZGZjODU3MTU1ZjEx",
"__string__NjQzMDMyMDAx",
"__string__MjAxNy0wMi0yN1QyMTowMDowMC4wMDBa",
"__number__50762",
"__string__QQ==",
"__string__0JTQsNC90L3Ri9C1INCh0LHQtdGA0JjQvdC00LXQutGB0LA=",
"__string__0KHQstC10YDQtNC70L7QstGB0LrQsNGPINC+0LHQu9Cw0YHRgtGM",
"__string__0JLRgtC+0YDQuNGH0L3Ri9C5",
"__string__0JzQtdGB0Y/Rhg==",
"__string__0YDRg9Cx",
"__string__MA=="
],
[
"__string__ODM0NTUxNTktMjYyOC00ZmNlLWIzNDktZGZjODU3MTU1ZjEx",
"__string__NjQzMDMyMDAx",
"__string__MjAxNy0wMy0zMFQyMTowMDowMC4wMDBa",
"__number__50902",
"__string__QQ==",
"__string__0JTQsNC90L3Ri9C1INCh0LHQtdGA0JjQvdC00LXQutGB0LA=",
"__string__0KHQstC10YDQtNC70L7QstGB0LrQsNGPINC+0LHQu9Cw0YHRgtGM",
"__string__0JLRgtC+0YDQuNGH0L3Ri9C5",
"__string__0JzQtdGB0Y/Rhg==",
"__string__0YDRg9Cx",
"__string__MA=="
],
[
"__string__ODM0NTUxNTktMjYyOC00ZmNlLWIzNDktZGZjODU3MTU1ZjEx",
"__string__NjQzMDMyMDAx",
"__string__MjAxNy0wNC0yOVQyMTowMDowMC4wMDBa",
"__number__50958",
"__string__QQ==",
"__string__0JTQsNC90L3Ri9C1INCh0LHQtdGA0JjQvdC00LXQutGB0LA=",
"__string__0KHQstC10YDQtNC70L7QstGB0LrQsNGPINC+0LHQu9Cw0YHRgtGM",
"__string__0JLRgtC+0YDQuNGH0L3Ri9C5",
"__string__0JzQtdGB0Y/Rhg==",
"__string__0YDRg9Cx",
"__string__MA=="
]
]
},
{
"key": "pagination",
"type": "object",
"value": {
"type": "object",
"value": [
{
"key": "total_records",
"type": "number",
"value": "__number__112"
},
{
"key": "current_page",
"type": "number",
"value": "__number__1"
},
{
"key": "total_pages",
"type": "number",
"value": "__number__28"
},
{
"key": "next_page",
"type": "number",
"value": "__number__2"
},
{
"key": "prev_page",
"type": "null",
"value": "__null__null"
}
]
}
},
{
"key": "limit",
"type": "number",
"value": "__number__4"
},
{
"key": "offset",
"type": "number",
"value": "__number__0"
}
]
}
}
}

View file

@ -0,0 +1,80 @@
"""Tests for #794 СберИндекс time-adjustment helper — pure, no DB."""
from __future__ import annotations
import os
import sys
from datetime import date
from unittest.mock import MagicMock
# DATABASE_URL required by pydantic Settings before any app import.
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
# WeasyPrint stub — not installed in CI без GTK.
sys.modules.setdefault("weasyprint", MagicMock())
from app.services.estimator import ( # noqa: E402
SBER_TIME_FACTOR_MAX,
SBER_TIME_FACTOR_MIN,
_sber_time_factor,
)
def test_empty_series_returns_one() -> None:
assert _sber_time_factor({}, date(2026, 1, 1)) == 1.0
def test_deal_month_equals_latest_returns_one() -> None:
series = {date(2026, 1, 1): 95_000.0, date(2026, 4, 1): 100_000.0}
assert _sber_time_factor(series, date(2026, 4, 1)) == 1.0
def test_deal_month_after_latest_returns_one() -> None:
series = {date(2026, 1, 1): 95_000.0, date(2026, 4, 1): 100_000.0}
assert _sber_time_factor(series, date(2026, 6, 1)) == 1.0
def test_normal_factor_calculation() -> None:
series = {date(2026, 1, 1): 95_000.0, date(2026, 4, 1): 100_000.0}
factor = _sber_time_factor(series, date(2026, 1, 1))
expected = 100_000.0 / 95_000.0 # ≈ 1.0526
assert abs(factor - expected) < 0.0001
assert SBER_TIME_FACTOR_MIN <= factor <= SBER_TIME_FACTOR_MAX
def test_clamp_high() -> None:
"""Very old deal → factor would be >>1.6 → clamp to SBER_TIME_FACTOR_MAX."""
series = {date(2020, 1, 1): 10_000.0, date(2026, 4, 1): 100_000.0}
factor = _sber_time_factor(series, date(2020, 1, 1))
assert factor == SBER_TIME_FACTOR_MAX
def test_clamp_low() -> None:
"""Series shows price drop → factor <<0.7 → clamp to SBER_TIME_FACTOR_MIN."""
series = {date(2020, 1, 1): 200_000.0, date(2026, 4, 1): 50_000.0}
factor = _sber_time_factor(series, date(2020, 1, 1))
assert factor == SBER_TIME_FACTOR_MIN
def test_missing_month_uses_nearest_earlier() -> None:
"""deal_month=2026-02 not in series; nearest earlier is 2026-01."""
series = {date(2026, 1, 1): 95_000.0, date(2026, 4, 1): 100_000.0}
# Using 2026-01 as base: factor = 100000/95000
factor_with_missing = _sber_time_factor(series, date(2026, 2, 1))
factor_explicit = _sber_time_factor(series, date(2026, 1, 1))
assert abs(factor_with_missing - factor_explicit) < 0.0001
def test_deal_older_than_series_uses_earliest() -> None:
"""Deal month before all series data → use earliest available month."""
series = {date(2023, 1, 1): 80_000.0, date(2026, 4, 1): 100_000.0}
# deal at 2020-01 is before series start → base = series[2023-01] = 80000
factor = _sber_time_factor(series, date(2020, 1, 1))
expected = min(100_000.0 / 80_000.0, SBER_TIME_FACTOR_MAX)
assert abs(factor - expected) < 0.0001
def test_zero_base_returns_one() -> None:
"""Zero index value in series → guard → return 1.0."""
series = {date(2026, 1, 1): 0.0, date(2026, 4, 1): 100_000.0}
assert _sber_time_factor(series, date(2026, 1, 1)) == 1.0

View file

@ -0,0 +1,113 @@
"""Tests for app/services/sber_index.py — per-dashboard filter fix (#794)."""
from __future__ import annotations
import base64
import json
from datetime import date
from pathlib import Path
from urllib.parse import unquote
import pytest
from app.services.sber_index import (
SBER_DASHBOARDS,
SberDashboard,
_parse_period_month,
build_sber_route,
decode_sber_response,
)
# ---------------------------------------------------------------------------
# Fixture path
# ---------------------------------------------------------------------------
FIXTURE_PATH = Path(__file__).parent.parent / "fixtures" / "sber_real_estate_deals_66.json"
# ---------------------------------------------------------------------------
# test_build_sber_route_per_dashboard
# LOCKS the bug fix: each dashboard must have the correct filter keys.
# ---------------------------------------------------------------------------
@pytest.mark.parametrize("dashboard", SBER_DASHBOARDS)
def test_build_sber_route_per_dashboard(dashboard: SberDashboard) -> None:
route_b64 = build_sber_route(dashboard.slug, "66", dashboard.extra_filter)
path = base64.b64decode(route_b64).decode("utf-8")
# Extract the filter JSON from the path: ?filter=<urlencoded>&limit=...
filter_part = path.split("filter=")[1].split("&")[0]
filter_dict = json.loads(unquote(filter_part))
# All routes must have REF_AREA + FREQ=M
assert filter_dict.get("REF_AREA") == "66"
assert filter_dict.get("FREQ") == "M"
if dashboard.slug == "real_estate_deals":
assert filter_dict.get("SOURCE") == "SI"
assert filter_dict.get("REALTY") == "2"
elif dashboard.slug == "residential_real_estate_prices":
assert filter_dict.get("SOURCE") == "SI"
assert filter_dict.get("REAL_ESTATE_NOVELTY") == "3"
elif dashboard.slug == "dinamika-tsen-obyavlenii":
assert filter_dict.get("SOURCE") == "DK"
assert filter_dict.get("PRICE_TYPE") == "1"
else:
pytest.fail(f"Unknown dashboard slug: {dashboard.slug!r}")
# ---------------------------------------------------------------------------
# test_decode_sber_response_real_fixture
# ---------------------------------------------------------------------------
def test_decode_sber_response_real_fixture() -> None:
with FIXTURE_PATH.open(encoding="utf-8") as f:
raw = json.load(f)
rows = decode_sber_response(raw)
assert len(rows) >= 1, "Expected at least 1 decoded row from real fixture"
# All rows should have the expected fields present
first = rows[0]
assert "ref_area" in first
assert "realty" in first
assert "value" in first
assert "period" in first
# Values must be typed correctly
assert isinstance(first["value"], float)
assert isinstance(first["period"], str) and first["period"]
# Verify region label decoded correctly
regions = {r["ref_area"] for r in rows}
assert (
"Свердловская область" in regions
), f"Expected 'Свердловская область' in ref_area values, got: {regions}"
# Verify secondary-market segment present
realty_vals = {r["realty"] for r in rows}
assert any(
"тори" in str(v) or "Вторичн" in str(v) for v in realty_vals
), f"Expected secondary-market label in realty field, got: {realty_vals}"
# ---------------------------------------------------------------------------
# test_parse_period_month
# ---------------------------------------------------------------------------
def test_parse_period_month() -> None:
result = _parse_period_month("2026-04-29T21:00:00.000Z")
assert result == date(2026, 4, 1)
def test_parse_period_month_z_suffix() -> None:
"""Original format from live API."""
result = _parse_period_month("2017-01-30T21:00:00Z")
assert result == date(2017, 1, 1)