feat(tradein): СберИндекс time-adjust for frozen Rosreestr ДКП + per-dashboard filter fix (#794) #919

Merged
bot-reviewer merged 2 commits from feat/794-sber-time-adjust into main 2026-05-31 18:48:23 +00:00
5 changed files with 503 additions and 59 deletions

View file

@ -26,7 +26,7 @@ import logging
import math import math
import re import re
import time import time
from datetime import UTC, datetime, timedelta from datetime import UTC, date, datetime, timedelta
from typing import Any from typing import Any
from uuid import uuid4 from uuid import uuid4
@ -78,6 +78,15 @@ MIN_ANALOGS_PER_SOURCE = 5 # гарантированный минимум на
LISTINGS_FRESH_DAYS = 14 # объявления не старше 14 дней LISTINGS_FRESH_DAYS = 14 # объявления не старше 14 дней
DEALS_PERIOD_MONTHS = 12 # сделки за последний год DEALS_PERIOD_MONTHS = 12 # сделки за последний год
# #794: СберИндекс time-adjustment of frozen Rosreestr ДКП deals.
# Rosreestr deals freeze ~2026-01; the sber monthly index re-bases a stale deal's ppm²
# to the latest available month. Region fixed to Свердловская обл. (tradein MVP = ЕКБ).
SBER_TIME_ADJUST_REGION = "Свердловская область"
# Coefficient series preference: hedonic (quality-adjusted, cleanest) → deals (fallback).
SBER_COEFF_DASHBOARDS = ("residential_real_estate_prices", "real_estate_deals")
SBER_TIME_FACTOR_MIN = 0.7 # clamp guards against bad/sparse index months
SBER_TIME_FACTOR_MAX = 1.6
# #699: санитизация ДКП-выбросов (Росреестр `deals`). В сырых сделках встречаются # #699: санитизация ДКП-выбросов (Росреестр `deals`). В сырых сделках встречаются
# нерыночные/битые записи — доли, сделки с обременением, опечатки этажа/площади — # нерыночные/битые записи — доли, сделки с обременением, опечатки этажа/площади —
# которые шумят actual_deals (display) и dkp_corridor/expected_sold. Абсолютные # которые шумят actual_deals (display) и dkp_corridor/expected_sold. Абсолютные
@ -934,6 +943,60 @@ def _apply_quarter_index(
return adjusted_ppm2, adjusted_median_price, adjusted_range_low, adjusted_range_high, factor return adjusted_ppm2, adjusted_median_price, adjusted_range_low, adjusted_range_high, factor
def _load_sber_index_series(db: Session, *, region: str) -> dict[date, float]:
"""#794: monthly {period_month: index_value} for region from sber_price_index.
Tries SBER_COEFF_DASHBOARDS in order; returns first non-empty series. {} on any error.
"""
for dash in SBER_COEFF_DASHBOARDS:
try:
rows = (
db.execute(
text("""
SELECT period_month, index_value_rub_m2
FROM sber_price_index
WHERE city = CAST(:region AS text)
AND dashboard = CAST(:dash AS text)
ORDER BY period_month
"""),
{"region": region, "dash": dash},
)
.mappings()
.all()
)
except Exception as exc:
logger.warning("sber_price_index lookup failed for %s (graceful): %s", dash, exc)
continue
if rows:
return {r["period_month"]: float(r["index_value_rub_m2"]) for r in rows}
return {}
def _sber_time_factor(series: dict[date, float], deal_month: date) -> float:
"""#794: factor = idx[latest] / idx[deal_month], clamped. 1.0 when no data / recent deal.
series: {first-of-month date -> index value}. deal_month: first-of-month of the deal.
If deal_month absent, use the nearest available month <= deal_month; if none, nearest overall.
If deal_month >= latest available month -> 1.0 (no extrapolation of recent deals).
"""
if not series:
return 1.0
latest = max(series)
if deal_month >= latest:
return 1.0
base = series.get(deal_month)
if base is None:
earlier = [m for m in series if m <= deal_month]
if earlier:
base = series[max(earlier)]
else:
base = series[min(series)] # deal older than series start → use earliest
if not base or base <= 0:
return 1.0
factor = series[latest] / base
return max(SBER_TIME_FACTOR_MIN, min(SBER_TIME_FACTOR_MAX, factor))
def _fetch_dkp_corridor( def _fetch_dkp_corridor(
db: Session, db: Session,
*, *,
@ -963,7 +1026,7 @@ def _fetch_dkp_corridor(
db.execute( db.execute(
text( text(
""" """
SELECT price_per_m2 SELECT price_per_m2, deal_date
FROM deals FROM deals
WHERE source = 'rosreestr' WHERE source = 'rosreestr'
AND address ILIKE :street_pattern AND address ILIKE :street_pattern
@ -995,9 +1058,33 @@ def _fetch_dkp_corridor(
logger.warning("dkp_corridor lookup failed (graceful): %s", exc) logger.warning("dkp_corridor lookup failed (graceful): %s", exc)
return None return None
ppm2_values = sorted(float(r["price_per_m2"]) for r in rows if r["price_per_m2"]) # #794: apply СберИндекс time-adjustment to re-base stale Rosreestr ДКП ppm²
# to the latest available index month. Graceful: factor=1.0 when table is empty.
series = _load_sber_index_series(db, region=SBER_TIME_ADJUST_REGION)
adjusted: list[float] = []
factors_applied: list[float] = []
for r in rows:
ppm2 = r["price_per_m2"]
if not ppm2:
continue
dd = r.get("deal_date")
factor = 1.0
if series and dd is not None:
deal_month = date(dd.year, dd.month, 1)
factor = _sber_time_factor(series, deal_month)
adjusted.append(float(ppm2) * factor)
factors_applied.append(factor)
ppm2_values = sorted(adjusted)
if not ppm2_values: if not ppm2_values:
return None return None
if series and factors_applied:
logger.info(
"dkp_corridor #794 time-adjust: n=%d factor min=%.3f max=%.3f region=%s",
len(factors_applied),
min(factors_applied),
max(factors_applied),
SBER_TIME_ADJUST_REGION,
)
return { return {
"count": len(ppm2_values), "count": len(ppm2_values),
"low_ppm2": int(ppm2_values[0]), "low_ppm2": int(ppm2_values[0]),

View file

@ -1,4 +1,4 @@
"""СберИндекс city-level secondary-housing price index pull (#887). """СберИндекс city-level secondary-housing price index pull (#887 + #794).
Fetches monthly time-series data from sberindex.ru/api/sowa (reverse-engineered Fetches monthly time-series data from sberindex.ru/api/sowa (reverse-engineered
in vault research note 2026-05-31-0845-resolver-794) across three dashboards: in vault research note 2026-05-31-0845-resolver-794) across three dashboards:
@ -10,19 +10,15 @@ API summary:
POST https://sberindex.ru/api/sowa POST https://sberindex.ru/api/sowa
Body: {"SOWA": {"method": "GET", "route": "<base64>", "data": {"type": "object", "value": []}}} Body: {"SOWA": {"method": "GET", "route": "<base64>", "data": {"type": "object", "value": []}}}
route = base64( /dataset/v1/<dashboard>?filter=<urlencode(json)>&limit=1000&offset=0 ) route = base64( /dataset/v1/<dashboard>?filter=<urlencode(json)>&limit=1000&offset=0 )
filter = {"REF_AREA": "<code>", "FREQ": "M", "SOURCE": "SI", "REALTY": "2"} filter = per-dashboard (see SBER_DASHBOARDS below); common keys: REF_AREA + FREQ:M
Required headers: content-type, rquid (random 32-hex), x-language, user-agent. Required headers: content-type, rquid (random 32-hex), x-language, user-agent.
Response fields (each cell is __string__<b64> or __number__<raw>): Response cells: __string__<b64> or __number__<raw>. Field names read from 'fields' entry.
indicator_id, kpi_id, period, value, obs_status, source, ref_area, realty,
freq, unit_measure, unit_mult
Data-quality benchmark: after upsert of the dinamika-tsen-obyavlenii (asking) series, #794 fix: build_sber_route was hardcoding SOURCE:SI + REALTY:2 for ALL dashboards.
the latest asking index per city is logged at INFO level as a data-quality reference. Live-verified 2026-05-31: residential_real_estate_prices needs REAL_ESTATE_NOVELTY,
Full asking-vs-our-median reconciliation is a separate follow-up (#887 step 5+). dinamika-tsen-obyavlenii needs SOURCE:DK + PRICE_TYPE. Filter is now per-dashboard via
SberDashboard dataclass.
NOTE estimator time-adjust wire (step 4) is OUT OF SCOPE for this PR.
It is a client-visible change that requires a backtest gate. Defer to a separate issue.
REF_AREA codes (sberindex internal region IDs, not ОКАТО/ISO): REF_AREA codes (sberindex internal region IDs, not ОКАТО/ISO):
643 = Россия confirmed live, vault research note 2026-05-31-0845. 643 = Россия confirmed live, vault research note 2026-05-31-0845.
@ -41,6 +37,7 @@ import json
import logging import logging
import uuid import uuid
from collections.abc import AsyncIterator from collections.abc import AsyncIterator
from dataclasses import dataclass
from datetime import UTC, date, datetime from datetime import UTC, date, datetime
from urllib.parse import quote from urllib.parse import quote
@ -63,11 +60,37 @@ SBER_REF_AREAS: dict[str, str] = {
"77": "Москва", "77": "Москва",
} }
# Dashboards to pull: slug → human label (used as 'segment' placeholder)
SBER_DASHBOARDS: list[str] = [ # ---------------------------------------------------------------------------
"real_estate_deals", # Per-dashboard config (#794 fix — each dashboard needs different filter params)
"residential_real_estate_prices", # ---------------------------------------------------------------------------
"dinamika-tsen-obyavlenii",
@dataclass(frozen=True)
class SberDashboard:
slug: str
extra_filter: dict[str, str] # beyond REF_AREA + FREQ:"M" — verified live 2026-05-31
segment_field: str # response field carrying the segment label
# All three series are the secondary-market (вторичка) ряд;
# one segment per dashboard keeps the (city,period_month,dashboard) PK valid.
SBER_DASHBOARDS: list[SberDashboard] = [
SberDashboard(
"real_estate_deals",
{"SOURCE": "SI", "REALTY": "2"},
"realty",
),
SberDashboard(
"residential_real_estate_prices",
{"SOURCE": "SI", "REAL_ESTATE_TYPE": "1", "REAL_ESTATE_NOVELTY": "3"},
"real_estate_novelty",
),
SberDashboard(
"dinamika-tsen-obyavlenii",
{"SOURCE": "DK", "PRICE_TYPE": "1"},
"price_type",
),
] ]
SBER_API_URL = "https://sberindex.ru/api/sowa" SBER_API_URL = "https://sberindex.ru/api/sowa"
@ -83,20 +106,16 @@ _CHROME_UA = (
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
def build_sber_route(dashboard: str, ref_area: str) -> str: def build_sber_route(dashboard: str, ref_area: str, extra_filter: dict[str, str]) -> str:
"""Build the base64-encoded route for the /api/sowa payload. """Build the base64-encoded route for the /api/sowa payload.
Encodes /dataset/v1/<dashboard>?filter=<urlencode(json)>&limit=1000&offset=0 Encodes /dataset/v1/<dashboard>?filter=<urlencode(json)>&limit=1000&offset=0
where filter = {"REF_AREA": ref_area, "FREQ": "M", "SOURCE": "SI", "REALTY": "2"}. where filter = {"REF_AREA": ref_area, "FREQ": "M", **extra_filter}.
extra_filter is per-dashboard (see SberDashboard.extra_filter); verified live 2026-05-31.
All filter values are strings per the live-captured payload (vault research note). All filter values are strings per the live-captured payload (vault research note).
""" """
filter_dict = { filter_dict = {"REF_AREA": str(ref_area), "FREQ": "M", **extra_filter}
"REF_AREA": str(ref_area),
"FREQ": "M",
"SOURCE": "SI",
"REALTY": "2",
}
# urlencode the JSON, then build the path # urlencode the JSON, then build the path
filter_str = quote(json.dumps(filter_dict, separators=(",", ":")), safe="") filter_str = quote(json.dumps(filter_dict, separators=(",", ":")), safe="")
path = f"/dataset/v1/{dashboard}?filter={filter_str}&limit=1000&offset=0" path = f"/dataset/v1/{dashboard}?filter={filter_str}&limit=1000&offset=0"
@ -107,21 +126,6 @@ def build_sber_route(dashboard: str, ref_area: str) -> str:
# Response decoder # Response decoder
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
# Field order returned by the API (confirmed from live session)
_FIELD_ORDER = [
"indicator_id",
"kpi_id",
"period",
"value",
"obs_status",
"source",
"ref_area",
"realty",
"freq",
"unit_measure",
"unit_mult",
]
def _decode_cell(cell: str) -> str | float: def _decode_cell(cell: str) -> str | float:
"""Decode a single response cell. """Decode a single response cell.
@ -212,19 +216,20 @@ def _parse_period_month(period_str: str) -> date:
async def fetch_sber_index( async def fetch_sber_index(
client: httpx.AsyncClient, client: httpx.AsyncClient,
*, *,
dashboard: str, dashboard: SberDashboard,
ref_area: str, ref_area: str,
) -> AsyncIterator[tuple[str, date, str, str, float]]: ) -> AsyncIterator[tuple[str, date, str, str, float]]:
"""Fetch one dashboard × REF_AREA combination from sberindex.ru/api/sowa. """Fetch one dashboard × REF_AREA combination from sberindex.ru/api/sowa.
Yields tuples of: Yields tuples of:
(city_label, period_month, segment_label, dashboard, index_value_rub_m2) (city_label, period_month, segment_label, dashboard_slug, index_value_rub_m2)
city_label and segment_label come from the API response fields 'ref_area' and 'realty'. city_label comes from the API response field 'ref_area'.
segment_label comes from dashboard.segment_field (per-dashboard, verified 2026-05-31).
Raises httpx.HTTPStatusError on non-2xx response (caller handles per-series). Raises httpx.HTTPStatusError on non-2xx response (caller handles per-series).
""" """
route = build_sber_route(dashboard, ref_area) route = build_sber_route(dashboard.slug, ref_area, dashboard.extra_filter)
rquid = uuid.uuid4().hex # 32-hex random, required header rquid = uuid.uuid4().hex # 32-hex random, required header
payload = { payload = {
@ -251,7 +256,7 @@ async def fetch_sber_index(
period_raw = row.get("period", "") period_raw = row.get("period", "")
value_raw = row.get("value") value_raw = row.get("value")
city_label = str(row.get("ref_area", ref_area)) city_label = str(row.get("ref_area", ref_area))
segment_label = str(row.get("realty", "")) segment_label = str(row.get(dashboard.segment_field, ""))
if not period_raw or value_raw is None: if not period_raw or value_raw is None:
continue continue
@ -262,13 +267,13 @@ async def fetch_sber_index(
except (ValueError, TypeError) as exc: except (ValueError, TypeError) as exc:
logger.warning( logger.warning(
"sber_index: skip row — parse error for dashboard=%s ref_area=%s: %s", "sber_index: skip row — parse error for dashboard=%s ref_area=%s: %s",
dashboard, dashboard.slug,
ref_area, ref_area,
exc, exc,
) )
continue continue
yield (city_label, period_month, segment_label, dashboard, index_value) yield (city_label, period_month, segment_label, dashboard.slug, index_value)
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
@ -280,20 +285,20 @@ async def pull_sber_indices(
db: Session, db: Session,
*, *,
cities: dict[str, str] | None = None, cities: dict[str, str] | None = None,
dashboards: list[str] | None = None, dashboards: list[SberDashboard] | None = None,
) -> dict[str, int]: ) -> dict[str, int]:
"""Fetch all configured dashboards × cities and upsert into sber_price_index. """Fetch all configured dashboards × cities and upsert into sber_price_index.
Args: Args:
db: SQLAlchemy Session (tradein DB). db: SQLAlchemy Session (tradein DB).
cities: REF_AREA code city label mapping. Defaults to SBER_REF_AREAS. cities: REF_AREA code city label mapping. Defaults to SBER_REF_AREAS.
dashboards: list of dashboard slugs to pull. Defaults to SBER_DASHBOARDS. dashboards: list of SberDashboard configs to pull. Defaults to SBER_DASHBOARDS.
Returns dict of counters: {upserted: int, skipped: int, errors: int}. Returns dict of counters: {upserted: int, skipped: int, errors: int}.
Per-series try/except: one failing series logs + continues (does not abort others). Per-series try/except: one failing series logs + continues (does not abort others).
After upserting 'dinamika-tsen-obyavlenii', logs the latest asking index per city After upserting 'dinamika-tsen-obyavlenii', logs the latest asking index per city
as a data-quality benchmark reference. as a data-quality benchmark reference and compares to our scrape-median asking ppm².
""" """
if cities is None: if cities is None:
cities = SBER_REF_AREAS cities = SBER_REF_AREAS
@ -315,7 +320,7 @@ async def pull_sber_indices(
if not rows_to_upsert: if not rows_to_upsert:
logger.info( logger.info(
"sber_index: no rows returned for dashboard=%s ref_area=%s", "sber_index: no rows returned for dashboard=%s ref_area=%s",
dashboard, dashboard.slug,
ref_area, ref_area,
) )
counters["skipped"] += 1 counters["skipped"] += 1
@ -359,35 +364,64 @@ async def pull_sber_indices(
logger.info( logger.info(
"sber_index: upserted %d rows for dashboard=%s ref_area=%s", "sber_index: upserted %d rows for dashboard=%s ref_area=%s",
len(rows_to_upsert), len(rows_to_upsert),
dashboard, dashboard.slug,
ref_area, ref_area,
) )
# Data-quality benchmark log for asking-price series # Data-quality benchmark log for asking-price series
if dashboard == "dinamika-tsen-obyavlenii" and rows_to_upsert: if dashboard.slug == "dinamika-tsen-obyavlenii" and rows_to_upsert:
latest_row = max(rows_to_upsert, key=lambda r: r[1]) latest_row = max(rows_to_upsert, key=lambda r: r[1])
city_lbl, latest_month, _, _, latest_val = latest_row city_lbl, latest_month, _, _, latest_val = latest_row
logger.info( logger.info(
"sber_index benchmark [asking]: city=%r latest_month=%s " "sber_index benchmark [asking]: city=%r latest_month=%s "
"index_value_rub_m2=%.0f " "index_value_rub_m2=%.0f "
"(data-quality ref; full asking-vs-our-median is follow-up)", "(data-quality ref; full asking-vs-our-median below)",
city_lbl, city_lbl,
latest_month.isoformat(), latest_month.isoformat(),
latest_val, latest_val,
) )
# our listings median is ЕКБ-only →
# compare only against Свердловская обл. (REF_AREA 66)
if ref_area == "66":
# Task B: best-effort reconciliation to our scrape-median asking ppm²
try:
our_median = db.execute(
text("""
SELECT percentile_cont(0.5)
WITHIN GROUP (ORDER BY price_per_m2) AS m
FROM listings
WHERE is_active = true
AND price_per_m2 BETWEEN 50000 AND 800000
""")
).scalar()
if our_median is not None and latest_val:
sber_latest = latest_val
divergence_pct = (our_median - sber_latest) / sber_latest * 100
logger.info(
"sber_index benchmark [asking vs our median]: "
"sber=%.0f our_median=%.0f divergence=%+.1f%% "
"(data-quality signal)",
sber_latest,
our_median,
divergence_pct,
)
except Exception as exc:
logger.debug(
"sber benchmark reconciliation skipped (graceful): %s", exc
)
except httpx.HTTPStatusError as exc: except httpx.HTTPStatusError as exc:
logger.error( logger.error(
"sber_index: HTTP %d for dashboard=%s ref_area=%s — skip series", "sber_index: HTTP %d for dashboard=%s ref_area=%s — skip series",
exc.response.status_code, exc.response.status_code,
dashboard, dashboard.slug,
ref_area, ref_area,
) )
counters["errors"] += 1 counters["errors"] += 1
except httpx.RequestError as exc: except httpx.RequestError as exc:
logger.error( logger.error(
"sber_index: network error for dashboard=%s ref_area=%s: %s — skip", "sber_index: network error for dashboard=%s ref_area=%s: %s — skip",
dashboard, dashboard.slug,
ref_area, ref_area,
exc, exc,
) )
@ -395,7 +429,7 @@ async def pull_sber_indices(
except Exception: except Exception:
logger.exception( logger.exception(
"sber_index: unexpected error for dashboard=%s ref_area=%s — skip", "sber_index: unexpected error for dashboard=%s ref_area=%s — skip",
dashboard, dashboard.slug,
ref_area, ref_area,
) )
counters["errors"] += 1 counters["errors"] += 1

View file

@ -0,0 +1,130 @@
{
"SOWA": {
"method": "GET",
"route": "L2RhdGFzZXQvdjEvcmVhbF9lc3RhdGVfZGVhbHM/ZmlsdGVyPSU3QiUyMlJFRl9BUkVBJTIyJTNBJTIyNjYlMjIlMkMlMjJGUkVRJTIyJTNBJTIyTSUyMiUyQyUyMlNPVVJDRSUyMiUzQSUyMlNJJTIyJTJDJTIyUkVBTFRZJTIyJTNBJTIyMiUyMiU3RCZsaW1pdD00Jm9mZnNldD0w",
"data": {
"type": "object",
"value": [
{
"key": "fields",
"type": "object",
"value": [
"__string__aW5kaWNhdG9yX2lk",
"__string__a3BpX2lk",
"__string__cGVyaW9k",
"__string__dmFsdWU=",
"__string__b2JzX3N0YXR1cw==",
"__string__c291cmNl",
"__string__cmVmX2FyZWE=",
"__string__cmVhbHR5",
"__string__ZnJlcQ==",
"__string__dW5pdF9tZWFzdXJl",
"__string__dW5pdF9tdWx0"
]
},
{
"key": "data",
"type": "object",
"value": [
[
"__string__ODM0NTUxNTktMjYyOC00ZmNlLWIzNDktZGZjODU3MTU1ZjEx",
"__string__NjQzMDMyMDAx",
"__string__MjAxNy0wMS0zMFQyMTowMDowMC4wMDBa",
"__number__51046",
"__string__QQ==",
"__string__0JTQsNC90L3Ri9C1INCh0LHQtdGA0JjQvdC00LXQutGB0LA=",
"__string__0KHQstC10YDQtNC70L7QstGB0LrQsNGPINC+0LHQu9Cw0YHRgtGM",
"__string__0JLRgtC+0YDQuNGH0L3Ri9C5",
"__string__0JzQtdGB0Y/Rhg==",
"__string__0YDRg9Cx",
"__string__MA=="
],
[
"__string__ODM0NTUxNTktMjYyOC00ZmNlLWIzNDktZGZjODU3MTU1ZjEx",
"__string__NjQzMDMyMDAx",
"__string__MjAxNy0wMi0yN1QyMTowMDowMC4wMDBa",
"__number__50762",
"__string__QQ==",
"__string__0JTQsNC90L3Ri9C1INCh0LHQtdGA0JjQvdC00LXQutGB0LA=",
"__string__0KHQstC10YDQtNC70L7QstGB0LrQsNGPINC+0LHQu9Cw0YHRgtGM",
"__string__0JLRgtC+0YDQuNGH0L3Ri9C5",
"__string__0JzQtdGB0Y/Rhg==",
"__string__0YDRg9Cx",
"__string__MA=="
],
[
"__string__ODM0NTUxNTktMjYyOC00ZmNlLWIzNDktZGZjODU3MTU1ZjEx",
"__string__NjQzMDMyMDAx",
"__string__MjAxNy0wMy0zMFQyMTowMDowMC4wMDBa",
"__number__50902",
"__string__QQ==",
"__string__0JTQsNC90L3Ri9C1INCh0LHQtdGA0JjQvdC00LXQutGB0LA=",
"__string__0KHQstC10YDQtNC70L7QstGB0LrQsNGPINC+0LHQu9Cw0YHRgtGM",
"__string__0JLRgtC+0YDQuNGH0L3Ri9C5",
"__string__0JzQtdGB0Y/Rhg==",
"__string__0YDRg9Cx",
"__string__MA=="
],
[
"__string__ODM0NTUxNTktMjYyOC00ZmNlLWIzNDktZGZjODU3MTU1ZjEx",
"__string__NjQzMDMyMDAx",
"__string__MjAxNy0wNC0yOVQyMTowMDowMC4wMDBa",
"__number__50958",
"__string__QQ==",
"__string__0JTQsNC90L3Ri9C1INCh0LHQtdGA0JjQvdC00LXQutGB0LA=",
"__string__0KHQstC10YDQtNC70L7QstGB0LrQsNGPINC+0LHQu9Cw0YHRgtGM",
"__string__0JLRgtC+0YDQuNGH0L3Ri9C5",
"__string__0JzQtdGB0Y/Rhg==",
"__string__0YDRg9Cx",
"__string__MA=="
]
]
},
{
"key": "pagination",
"type": "object",
"value": {
"type": "object",
"value": [
{
"key": "total_records",
"type": "number",
"value": "__number__112"
},
{
"key": "current_page",
"type": "number",
"value": "__number__1"
},
{
"key": "total_pages",
"type": "number",
"value": "__number__28"
},
{
"key": "next_page",
"type": "number",
"value": "__number__2"
},
{
"key": "prev_page",
"type": "null",
"value": "__null__null"
}
]
}
},
{
"key": "limit",
"type": "number",
"value": "__number__4"
},
{
"key": "offset",
"type": "number",
"value": "__number__0"
}
]
}
}
}

View file

@ -0,0 +1,80 @@
"""Tests for #794 СберИндекс time-adjustment helper — pure, no DB."""
from __future__ import annotations
import os
import sys
from datetime import date
from unittest.mock import MagicMock
# DATABASE_URL required by pydantic Settings before any app import.
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
# WeasyPrint stub — not installed in CI без GTK.
sys.modules.setdefault("weasyprint", MagicMock())
from app.services.estimator import ( # noqa: E402
SBER_TIME_FACTOR_MAX,
SBER_TIME_FACTOR_MIN,
_sber_time_factor,
)
def test_empty_series_returns_one() -> None:
assert _sber_time_factor({}, date(2026, 1, 1)) == 1.0
def test_deal_month_equals_latest_returns_one() -> None:
series = {date(2026, 1, 1): 95_000.0, date(2026, 4, 1): 100_000.0}
assert _sber_time_factor(series, date(2026, 4, 1)) == 1.0
def test_deal_month_after_latest_returns_one() -> None:
series = {date(2026, 1, 1): 95_000.0, date(2026, 4, 1): 100_000.0}
assert _sber_time_factor(series, date(2026, 6, 1)) == 1.0
def test_normal_factor_calculation() -> None:
series = {date(2026, 1, 1): 95_000.0, date(2026, 4, 1): 100_000.0}
factor = _sber_time_factor(series, date(2026, 1, 1))
expected = 100_000.0 / 95_000.0 # ≈ 1.0526
assert abs(factor - expected) < 0.0001
assert SBER_TIME_FACTOR_MIN <= factor <= SBER_TIME_FACTOR_MAX
def test_clamp_high() -> None:
"""Very old deal → factor would be >>1.6 → clamp to SBER_TIME_FACTOR_MAX."""
series = {date(2020, 1, 1): 10_000.0, date(2026, 4, 1): 100_000.0}
factor = _sber_time_factor(series, date(2020, 1, 1))
assert factor == SBER_TIME_FACTOR_MAX
def test_clamp_low() -> None:
"""Series shows price drop → factor <<0.7 → clamp to SBER_TIME_FACTOR_MIN."""
series = {date(2020, 1, 1): 200_000.0, date(2026, 4, 1): 50_000.0}
factor = _sber_time_factor(series, date(2020, 1, 1))
assert factor == SBER_TIME_FACTOR_MIN
def test_missing_month_uses_nearest_earlier() -> None:
"""deal_month=2026-02 not in series; nearest earlier is 2026-01."""
series = {date(2026, 1, 1): 95_000.0, date(2026, 4, 1): 100_000.0}
# Using 2026-01 as base: factor = 100000/95000
factor_with_missing = _sber_time_factor(series, date(2026, 2, 1))
factor_explicit = _sber_time_factor(series, date(2026, 1, 1))
assert abs(factor_with_missing - factor_explicit) < 0.0001
def test_deal_older_than_series_uses_earliest() -> None:
"""Deal month before all series data → use earliest available month."""
series = {date(2023, 1, 1): 80_000.0, date(2026, 4, 1): 100_000.0}
# deal at 2020-01 is before series start → base = series[2023-01] = 80000
factor = _sber_time_factor(series, date(2020, 1, 1))
expected = min(100_000.0 / 80_000.0, SBER_TIME_FACTOR_MAX)
assert abs(factor - expected) < 0.0001
def test_zero_base_returns_one() -> None:
"""Zero index value in series → guard → return 1.0."""
series = {date(2026, 1, 1): 0.0, date(2026, 4, 1): 100_000.0}
assert _sber_time_factor(series, date(2026, 1, 1)) == 1.0

View file

@ -0,0 +1,113 @@
"""Tests for app/services/sber_index.py — per-dashboard filter fix (#794)."""
from __future__ import annotations
import base64
import json
from datetime import date
from pathlib import Path
from urllib.parse import unquote
import pytest
from app.services.sber_index import (
SBER_DASHBOARDS,
SberDashboard,
_parse_period_month,
build_sber_route,
decode_sber_response,
)
# ---------------------------------------------------------------------------
# Fixture path
# ---------------------------------------------------------------------------
FIXTURE_PATH = Path(__file__).parent.parent / "fixtures" / "sber_real_estate_deals_66.json"
# ---------------------------------------------------------------------------
# test_build_sber_route_per_dashboard
# LOCKS the bug fix: each dashboard must have the correct filter keys.
# ---------------------------------------------------------------------------
@pytest.mark.parametrize("dashboard", SBER_DASHBOARDS)
def test_build_sber_route_per_dashboard(dashboard: SberDashboard) -> None:
route_b64 = build_sber_route(dashboard.slug, "66", dashboard.extra_filter)
path = base64.b64decode(route_b64).decode("utf-8")
# Extract the filter JSON from the path: ?filter=<urlencoded>&limit=...
filter_part = path.split("filter=")[1].split("&")[0]
filter_dict = json.loads(unquote(filter_part))
# All routes must have REF_AREA + FREQ=M
assert filter_dict.get("REF_AREA") == "66"
assert filter_dict.get("FREQ") == "M"
if dashboard.slug == "real_estate_deals":
assert filter_dict.get("SOURCE") == "SI"
assert filter_dict.get("REALTY") == "2"
elif dashboard.slug == "residential_real_estate_prices":
assert filter_dict.get("SOURCE") == "SI"
assert filter_dict.get("REAL_ESTATE_NOVELTY") == "3"
elif dashboard.slug == "dinamika-tsen-obyavlenii":
assert filter_dict.get("SOURCE") == "DK"
assert filter_dict.get("PRICE_TYPE") == "1"
else:
pytest.fail(f"Unknown dashboard slug: {dashboard.slug!r}")
# ---------------------------------------------------------------------------
# test_decode_sber_response_real_fixture
# ---------------------------------------------------------------------------
def test_decode_sber_response_real_fixture() -> None:
with FIXTURE_PATH.open(encoding="utf-8") as f:
raw = json.load(f)
rows = decode_sber_response(raw)
assert len(rows) >= 1, "Expected at least 1 decoded row from real fixture"
# All rows should have the expected fields present
first = rows[0]
assert "ref_area" in first
assert "realty" in first
assert "value" in first
assert "period" in first
# Values must be typed correctly
assert isinstance(first["value"], float)
assert isinstance(first["period"], str) and first["period"]
# Verify region label decoded correctly
regions = {r["ref_area"] for r in rows}
assert (
"Свердловская область" in regions
), f"Expected 'Свердловская область' in ref_area values, got: {regions}"
# Verify secondary-market segment present
realty_vals = {r["realty"] for r in rows}
assert any(
"тори" in str(v) or "Вторичн" in str(v) for v in realty_vals
), f"Expected secondary-market label in realty field, got: {realty_vals}"
# ---------------------------------------------------------------------------
# test_parse_period_month
# ---------------------------------------------------------------------------
def test_parse_period_month() -> None:
result = _parse_period_month("2026-04-29T21:00:00.000Z")
assert result == date(2026, 4, 1)
def test_parse_period_month_z_suffix() -> None:
"""Original format from live API."""
result = _parse_period_month("2017-01-30T21:00:00Z")
assert result == date(2017, 1, 1)