feat(tradein): СберИндекс time-adjust for frozen Rosreestr ДКП + per-dashboard filter fix (#794) #919
5 changed files with 503 additions and 59 deletions
|
|
@ -26,7 +26,7 @@ import logging
|
|||
import math
|
||||
import re
|
||||
import time
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from datetime import UTC, date, datetime, timedelta
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
|
|
@ -78,6 +78,15 @@ MIN_ANALOGS_PER_SOURCE = 5 # гарантированный минимум на
|
|||
LISTINGS_FRESH_DAYS = 14 # объявления не старше 14 дней
|
||||
DEALS_PERIOD_MONTHS = 12 # сделки за последний год
|
||||
|
||||
# #794: СберИндекс time-adjustment of frozen Rosreestr ДКП deals.
|
||||
# Rosreestr deals freeze ~2026-01; the sber monthly index re-bases a stale deal's ppm²
|
||||
# to the latest available month. Region fixed to Свердловская обл. (tradein MVP = ЕКБ).
|
||||
SBER_TIME_ADJUST_REGION = "Свердловская область"
|
||||
# Coefficient series preference: hedonic (quality-adjusted, cleanest) → deals (fallback).
|
||||
SBER_COEFF_DASHBOARDS = ("residential_real_estate_prices", "real_estate_deals")
|
||||
SBER_TIME_FACTOR_MIN = 0.7 # clamp guards against bad/sparse index months
|
||||
SBER_TIME_FACTOR_MAX = 1.6
|
||||
|
||||
# #699: санитизация ДКП-выбросов (Росреестр `deals`). В сырых сделках встречаются
|
||||
# нерыночные/битые записи — доли, сделки с обременением, опечатки этажа/площади —
|
||||
# которые шумят actual_deals (display) и dkp_corridor/expected_sold. Абсолютные
|
||||
|
|
@ -934,6 +943,60 @@ def _apply_quarter_index(
|
|||
return adjusted_ppm2, adjusted_median_price, adjusted_range_low, adjusted_range_high, factor
|
||||
|
||||
|
||||
def _load_sber_index_series(db: Session, *, region: str) -> dict[date, float]:
|
||||
"""#794: monthly {period_month: index_value} for region from sber_price_index.
|
||||
|
||||
Tries SBER_COEFF_DASHBOARDS in order; returns first non-empty series. {} on any error.
|
||||
"""
|
||||
for dash in SBER_COEFF_DASHBOARDS:
|
||||
try:
|
||||
rows = (
|
||||
db.execute(
|
||||
text("""
|
||||
SELECT period_month, index_value_rub_m2
|
||||
FROM sber_price_index
|
||||
WHERE city = CAST(:region AS text)
|
||||
AND dashboard = CAST(:dash AS text)
|
||||
ORDER BY period_month
|
||||
"""),
|
||||
{"region": region, "dash": dash},
|
||||
)
|
||||
.mappings()
|
||||
.all()
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning("sber_price_index lookup failed for %s (graceful): %s", dash, exc)
|
||||
continue
|
||||
if rows:
|
||||
return {r["period_month"]: float(r["index_value_rub_m2"]) for r in rows}
|
||||
return {}
|
||||
|
||||
|
||||
def _sber_time_factor(series: dict[date, float], deal_month: date) -> float:
|
||||
"""#794: factor = idx[latest] / idx[deal_month], clamped. 1.0 when no data / recent deal.
|
||||
|
||||
series: {first-of-month date -> index value}. deal_month: first-of-month of the deal.
|
||||
If deal_month absent, use the nearest available month <= deal_month; if none, nearest overall.
|
||||
If deal_month >= latest available month -> 1.0 (no extrapolation of recent deals).
|
||||
"""
|
||||
if not series:
|
||||
return 1.0
|
||||
latest = max(series)
|
||||
if deal_month >= latest:
|
||||
return 1.0
|
||||
base = series.get(deal_month)
|
||||
if base is None:
|
||||
earlier = [m for m in series if m <= deal_month]
|
||||
if earlier:
|
||||
base = series[max(earlier)]
|
||||
else:
|
||||
base = series[min(series)] # deal older than series start → use earliest
|
||||
if not base or base <= 0:
|
||||
return 1.0
|
||||
factor = series[latest] / base
|
||||
return max(SBER_TIME_FACTOR_MIN, min(SBER_TIME_FACTOR_MAX, factor))
|
||||
|
||||
|
||||
def _fetch_dkp_corridor(
|
||||
db: Session,
|
||||
*,
|
||||
|
|
@ -963,7 +1026,7 @@ def _fetch_dkp_corridor(
|
|||
db.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT price_per_m2
|
||||
SELECT price_per_m2, deal_date
|
||||
FROM deals
|
||||
WHERE source = 'rosreestr'
|
||||
AND address ILIKE :street_pattern
|
||||
|
|
@ -995,9 +1058,33 @@ def _fetch_dkp_corridor(
|
|||
logger.warning("dkp_corridor lookup failed (graceful): %s", exc)
|
||||
return None
|
||||
|
||||
ppm2_values = sorted(float(r["price_per_m2"]) for r in rows if r["price_per_m2"])
|
||||
# #794: apply СберИндекс time-adjustment to re-base stale Rosreestr ДКП ppm²
|
||||
# to the latest available index month. Graceful: factor=1.0 when table is empty.
|
||||
series = _load_sber_index_series(db, region=SBER_TIME_ADJUST_REGION)
|
||||
adjusted: list[float] = []
|
||||
factors_applied: list[float] = []
|
||||
for r in rows:
|
||||
ppm2 = r["price_per_m2"]
|
||||
if not ppm2:
|
||||
continue
|
||||
dd = r.get("deal_date")
|
||||
factor = 1.0
|
||||
if series and dd is not None:
|
||||
deal_month = date(dd.year, dd.month, 1)
|
||||
factor = _sber_time_factor(series, deal_month)
|
||||
adjusted.append(float(ppm2) * factor)
|
||||
factors_applied.append(factor)
|
||||
ppm2_values = sorted(adjusted)
|
||||
if not ppm2_values:
|
||||
return None
|
||||
if series and factors_applied:
|
||||
logger.info(
|
||||
"dkp_corridor #794 time-adjust: n=%d factor min=%.3f max=%.3f region=%s",
|
||||
len(factors_applied),
|
||||
min(factors_applied),
|
||||
max(factors_applied),
|
||||
SBER_TIME_ADJUST_REGION,
|
||||
)
|
||||
return {
|
||||
"count": len(ppm2_values),
|
||||
"low_ppm2": int(ppm2_values[0]),
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
"""СберИндекс city-level secondary-housing price index pull (#887).
|
||||
"""СберИндекс city-level secondary-housing price index pull (#887 + #794).
|
||||
|
||||
Fetches monthly time-series data from sberindex.ru/api/sowa (reverse-engineered
|
||||
in vault research note 2026-05-31-0845-resolver-794) across three dashboards:
|
||||
|
|
@ -10,19 +10,15 @@ API summary:
|
|||
POST https://sberindex.ru/api/sowa
|
||||
Body: {"SOWA": {"method": "GET", "route": "<base64>", "data": {"type": "object", "value": []}}}
|
||||
route = base64( /dataset/v1/<dashboard>?filter=<urlencode(json)>&limit=1000&offset=0 )
|
||||
filter = {"REF_AREA": "<code>", "FREQ": "M", "SOURCE": "SI", "REALTY": "2"}
|
||||
filter = per-dashboard (see SBER_DASHBOARDS below); common keys: REF_AREA + FREQ:M
|
||||
Required headers: content-type, rquid (random 32-hex), x-language, user-agent.
|
||||
|
||||
Response fields (each cell is __string__<b64> or __number__<raw>):
|
||||
indicator_id, kpi_id, period, value, obs_status, source, ref_area, realty,
|
||||
freq, unit_measure, unit_mult
|
||||
Response cells: __string__<b64> or __number__<raw>. Field names read from 'fields' entry.
|
||||
|
||||
Data-quality benchmark: after upsert of the dinamika-tsen-obyavlenii (asking) series,
|
||||
the latest asking index per city is logged at INFO level as a data-quality reference.
|
||||
Full asking-vs-our-median reconciliation is a separate follow-up (#887 step 5+).
|
||||
|
||||
NOTE — estimator time-adjust wire (step 4) is OUT OF SCOPE for this PR.
|
||||
It is a client-visible change that requires a backtest gate. Defer to a separate issue.
|
||||
#794 fix: build_sber_route was hardcoding SOURCE:SI + REALTY:2 for ALL dashboards.
|
||||
Live-verified 2026-05-31: residential_real_estate_prices needs REAL_ESTATE_NOVELTY,
|
||||
dinamika-tsen-obyavlenii needs SOURCE:DK + PRICE_TYPE. Filter is now per-dashboard via
|
||||
SberDashboard dataclass.
|
||||
|
||||
REF_AREA codes (sberindex internal region IDs, not ОКАТО/ISO):
|
||||
643 = Россия — confirmed live, vault research note 2026-05-31-0845.
|
||||
|
|
@ -41,6 +37,7 @@ import json
|
|||
import logging
|
||||
import uuid
|
||||
from collections.abc import AsyncIterator
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, date, datetime
|
||||
from urllib.parse import quote
|
||||
|
||||
|
|
@ -63,11 +60,37 @@ SBER_REF_AREAS: dict[str, str] = {
|
|||
"77": "Москва",
|
||||
}
|
||||
|
||||
# Dashboards to pull: slug → human label (used as 'segment' placeholder)
|
||||
SBER_DASHBOARDS: list[str] = [
|
||||
"real_estate_deals",
|
||||
"residential_real_estate_prices",
|
||||
"dinamika-tsen-obyavlenii",
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Per-dashboard config (#794 fix — each dashboard needs different filter params)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SberDashboard:
|
||||
slug: str
|
||||
extra_filter: dict[str, str] # beyond REF_AREA + FREQ:"M" — verified live 2026-05-31
|
||||
segment_field: str # response field carrying the segment label
|
||||
|
||||
|
||||
# All three series are the secondary-market (вторичка) ряд;
|
||||
# one segment per dashboard keeps the (city,period_month,dashboard) PK valid.
|
||||
SBER_DASHBOARDS: list[SberDashboard] = [
|
||||
SberDashboard(
|
||||
"real_estate_deals",
|
||||
{"SOURCE": "SI", "REALTY": "2"},
|
||||
"realty",
|
||||
),
|
||||
SberDashboard(
|
||||
"residential_real_estate_prices",
|
||||
{"SOURCE": "SI", "REAL_ESTATE_TYPE": "1", "REAL_ESTATE_NOVELTY": "3"},
|
||||
"real_estate_novelty",
|
||||
),
|
||||
SberDashboard(
|
||||
"dinamika-tsen-obyavlenii",
|
||||
{"SOURCE": "DK", "PRICE_TYPE": "1"},
|
||||
"price_type",
|
||||
),
|
||||
]
|
||||
|
||||
SBER_API_URL = "https://sberindex.ru/api/sowa"
|
||||
|
|
@ -83,20 +106,16 @@ _CHROME_UA = (
|
|||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def build_sber_route(dashboard: str, ref_area: str) -> str:
|
||||
def build_sber_route(dashboard: str, ref_area: str, extra_filter: dict[str, str]) -> str:
|
||||
"""Build the base64-encoded route for the /api/sowa payload.
|
||||
|
||||
Encodes /dataset/v1/<dashboard>?filter=<urlencode(json)>&limit=1000&offset=0
|
||||
where filter = {"REF_AREA": ref_area, "FREQ": "M", "SOURCE": "SI", "REALTY": "2"}.
|
||||
where filter = {"REF_AREA": ref_area, "FREQ": "M", **extra_filter}.
|
||||
|
||||
extra_filter is per-dashboard (see SberDashboard.extra_filter); verified live 2026-05-31.
|
||||
All filter values are strings per the live-captured payload (vault research note).
|
||||
"""
|
||||
filter_dict = {
|
||||
"REF_AREA": str(ref_area),
|
||||
"FREQ": "M",
|
||||
"SOURCE": "SI",
|
||||
"REALTY": "2",
|
||||
}
|
||||
filter_dict = {"REF_AREA": str(ref_area), "FREQ": "M", **extra_filter}
|
||||
# urlencode the JSON, then build the path
|
||||
filter_str = quote(json.dumps(filter_dict, separators=(",", ":")), safe="")
|
||||
path = f"/dataset/v1/{dashboard}?filter={filter_str}&limit=1000&offset=0"
|
||||
|
|
@ -107,21 +126,6 @@ def build_sber_route(dashboard: str, ref_area: str) -> str:
|
|||
# Response decoder
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Field order returned by the API (confirmed from live session)
|
||||
_FIELD_ORDER = [
|
||||
"indicator_id",
|
||||
"kpi_id",
|
||||
"period",
|
||||
"value",
|
||||
"obs_status",
|
||||
"source",
|
||||
"ref_area",
|
||||
"realty",
|
||||
"freq",
|
||||
"unit_measure",
|
||||
"unit_mult",
|
||||
]
|
||||
|
||||
|
||||
def _decode_cell(cell: str) -> str | float:
|
||||
"""Decode a single response cell.
|
||||
|
|
@ -212,19 +216,20 @@ def _parse_period_month(period_str: str) -> date:
|
|||
async def fetch_sber_index(
|
||||
client: httpx.AsyncClient,
|
||||
*,
|
||||
dashboard: str,
|
||||
dashboard: SberDashboard,
|
||||
ref_area: str,
|
||||
) -> AsyncIterator[tuple[str, date, str, str, float]]:
|
||||
"""Fetch one dashboard × REF_AREA combination from sberindex.ru/api/sowa.
|
||||
|
||||
Yields tuples of:
|
||||
(city_label, period_month, segment_label, dashboard, index_value_rub_m2)
|
||||
(city_label, period_month, segment_label, dashboard_slug, index_value_rub_m2)
|
||||
|
||||
city_label and segment_label come from the API response fields 'ref_area' and 'realty'.
|
||||
city_label comes from the API response field 'ref_area'.
|
||||
segment_label comes from dashboard.segment_field (per-dashboard, verified 2026-05-31).
|
||||
|
||||
Raises httpx.HTTPStatusError on non-2xx response (caller handles per-series).
|
||||
"""
|
||||
route = build_sber_route(dashboard, ref_area)
|
||||
route = build_sber_route(dashboard.slug, ref_area, dashboard.extra_filter)
|
||||
rquid = uuid.uuid4().hex # 32-hex random, required header
|
||||
|
||||
payload = {
|
||||
|
|
@ -251,7 +256,7 @@ async def fetch_sber_index(
|
|||
period_raw = row.get("period", "")
|
||||
value_raw = row.get("value")
|
||||
city_label = str(row.get("ref_area", ref_area))
|
||||
segment_label = str(row.get("realty", ""))
|
||||
segment_label = str(row.get(dashboard.segment_field, ""))
|
||||
|
||||
if not period_raw or value_raw is None:
|
||||
continue
|
||||
|
|
@ -262,13 +267,13 @@ async def fetch_sber_index(
|
|||
except (ValueError, TypeError) as exc:
|
||||
logger.warning(
|
||||
"sber_index: skip row — parse error for dashboard=%s ref_area=%s: %s",
|
||||
dashboard,
|
||||
dashboard.slug,
|
||||
ref_area,
|
||||
exc,
|
||||
)
|
||||
continue
|
||||
|
||||
yield (city_label, period_month, segment_label, dashboard, index_value)
|
||||
yield (city_label, period_month, segment_label, dashboard.slug, index_value)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
@ -280,20 +285,20 @@ async def pull_sber_indices(
|
|||
db: Session,
|
||||
*,
|
||||
cities: dict[str, str] | None = None,
|
||||
dashboards: list[str] | None = None,
|
||||
dashboards: list[SberDashboard] | None = None,
|
||||
) -> dict[str, int]:
|
||||
"""Fetch all configured dashboards × cities and upsert into sber_price_index.
|
||||
|
||||
Args:
|
||||
db: SQLAlchemy Session (tradein DB).
|
||||
cities: REF_AREA code → city label mapping. Defaults to SBER_REF_AREAS.
|
||||
dashboards: list of dashboard slugs to pull. Defaults to SBER_DASHBOARDS.
|
||||
dashboards: list of SberDashboard configs to pull. Defaults to SBER_DASHBOARDS.
|
||||
|
||||
Returns dict of counters: {upserted: int, skipped: int, errors: int}.
|
||||
|
||||
Per-series try/except: one failing series logs + continues (does not abort others).
|
||||
After upserting 'dinamika-tsen-obyavlenii', logs the latest asking index per city
|
||||
as a data-quality benchmark reference.
|
||||
as a data-quality benchmark reference and compares to our scrape-median asking ppm².
|
||||
"""
|
||||
if cities is None:
|
||||
cities = SBER_REF_AREAS
|
||||
|
|
@ -315,7 +320,7 @@ async def pull_sber_indices(
|
|||
if not rows_to_upsert:
|
||||
logger.info(
|
||||
"sber_index: no rows returned for dashboard=%s ref_area=%s",
|
||||
dashboard,
|
||||
dashboard.slug,
|
||||
ref_area,
|
||||
)
|
||||
counters["skipped"] += 1
|
||||
|
|
@ -359,35 +364,64 @@ async def pull_sber_indices(
|
|||
logger.info(
|
||||
"sber_index: upserted %d rows for dashboard=%s ref_area=%s",
|
||||
len(rows_to_upsert),
|
||||
dashboard,
|
||||
dashboard.slug,
|
||||
ref_area,
|
||||
)
|
||||
|
||||
# Data-quality benchmark log for asking-price series
|
||||
if dashboard == "dinamika-tsen-obyavlenii" and rows_to_upsert:
|
||||
if dashboard.slug == "dinamika-tsen-obyavlenii" and rows_to_upsert:
|
||||
latest_row = max(rows_to_upsert, key=lambda r: r[1])
|
||||
city_lbl, latest_month, _, _, latest_val = latest_row
|
||||
logger.info(
|
||||
"sber_index benchmark [asking]: city=%r latest_month=%s "
|
||||
"index_value_rub_m2=%.0f "
|
||||
"(data-quality ref; full asking-vs-our-median is follow-up)",
|
||||
"(data-quality ref; full asking-vs-our-median below)",
|
||||
city_lbl,
|
||||
latest_month.isoformat(),
|
||||
latest_val,
|
||||
)
|
||||
# our listings median is ЕКБ-only →
|
||||
# compare only against Свердловская обл. (REF_AREA 66)
|
||||
if ref_area == "66":
|
||||
# Task B: best-effort reconciliation to our scrape-median asking ppm²
|
||||
try:
|
||||
our_median = db.execute(
|
||||
text("""
|
||||
SELECT percentile_cont(0.5)
|
||||
WITHIN GROUP (ORDER BY price_per_m2) AS m
|
||||
FROM listings
|
||||
WHERE is_active = true
|
||||
AND price_per_m2 BETWEEN 50000 AND 800000
|
||||
""")
|
||||
).scalar()
|
||||
if our_median is not None and latest_val:
|
||||
sber_latest = latest_val
|
||||
divergence_pct = (our_median - sber_latest) / sber_latest * 100
|
||||
logger.info(
|
||||
"sber_index benchmark [asking vs our median]: "
|
||||
"sber=%.0f our_median=%.0f divergence=%+.1f%% "
|
||||
"(data-quality signal)",
|
||||
sber_latest,
|
||||
our_median,
|
||||
divergence_pct,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.debug(
|
||||
"sber benchmark reconciliation skipped (graceful): %s", exc
|
||||
)
|
||||
|
||||
except httpx.HTTPStatusError as exc:
|
||||
logger.error(
|
||||
"sber_index: HTTP %d for dashboard=%s ref_area=%s — skip series",
|
||||
exc.response.status_code,
|
||||
dashboard,
|
||||
dashboard.slug,
|
||||
ref_area,
|
||||
)
|
||||
counters["errors"] += 1
|
||||
except httpx.RequestError as exc:
|
||||
logger.error(
|
||||
"sber_index: network error for dashboard=%s ref_area=%s: %s — skip",
|
||||
dashboard,
|
||||
dashboard.slug,
|
||||
ref_area,
|
||||
exc,
|
||||
)
|
||||
|
|
@ -395,7 +429,7 @@ async def pull_sber_indices(
|
|||
except Exception:
|
||||
logger.exception(
|
||||
"sber_index: unexpected error for dashboard=%s ref_area=%s — skip",
|
||||
dashboard,
|
||||
dashboard.slug,
|
||||
ref_area,
|
||||
)
|
||||
counters["errors"] += 1
|
||||
|
|
|
|||
130
tradein-mvp/backend/tests/fixtures/sber_real_estate_deals_66.json
vendored
Normal file
130
tradein-mvp/backend/tests/fixtures/sber_real_estate_deals_66.json
vendored
Normal file
|
|
@ -0,0 +1,130 @@
|
|||
{
|
||||
"SOWA": {
|
||||
"method": "GET",
|
||||
"route": "L2RhdGFzZXQvdjEvcmVhbF9lc3RhdGVfZGVhbHM/ZmlsdGVyPSU3QiUyMlJFRl9BUkVBJTIyJTNBJTIyNjYlMjIlMkMlMjJGUkVRJTIyJTNBJTIyTSUyMiUyQyUyMlNPVVJDRSUyMiUzQSUyMlNJJTIyJTJDJTIyUkVBTFRZJTIyJTNBJTIyMiUyMiU3RCZsaW1pdD00Jm9mZnNldD0w",
|
||||
"data": {
|
||||
"type": "object",
|
||||
"value": [
|
||||
{
|
||||
"key": "fields",
|
||||
"type": "object",
|
||||
"value": [
|
||||
"__string__aW5kaWNhdG9yX2lk",
|
||||
"__string__a3BpX2lk",
|
||||
"__string__cGVyaW9k",
|
||||
"__string__dmFsdWU=",
|
||||
"__string__b2JzX3N0YXR1cw==",
|
||||
"__string__c291cmNl",
|
||||
"__string__cmVmX2FyZWE=",
|
||||
"__string__cmVhbHR5",
|
||||
"__string__ZnJlcQ==",
|
||||
"__string__dW5pdF9tZWFzdXJl",
|
||||
"__string__dW5pdF9tdWx0"
|
||||
]
|
||||
},
|
||||
{
|
||||
"key": "data",
|
||||
"type": "object",
|
||||
"value": [
|
||||
[
|
||||
"__string__ODM0NTUxNTktMjYyOC00ZmNlLWIzNDktZGZjODU3MTU1ZjEx",
|
||||
"__string__NjQzMDMyMDAx",
|
||||
"__string__MjAxNy0wMS0zMFQyMTowMDowMC4wMDBa",
|
||||
"__number__51046",
|
||||
"__string__QQ==",
|
||||
"__string__0JTQsNC90L3Ri9C1INCh0LHQtdGA0JjQvdC00LXQutGB0LA=",
|
||||
"__string__0KHQstC10YDQtNC70L7QstGB0LrQsNGPINC+0LHQu9Cw0YHRgtGM",
|
||||
"__string__0JLRgtC+0YDQuNGH0L3Ri9C5",
|
||||
"__string__0JzQtdGB0Y/Rhg==",
|
||||
"__string__0YDRg9Cx",
|
||||
"__string__MA=="
|
||||
],
|
||||
[
|
||||
"__string__ODM0NTUxNTktMjYyOC00ZmNlLWIzNDktZGZjODU3MTU1ZjEx",
|
||||
"__string__NjQzMDMyMDAx",
|
||||
"__string__MjAxNy0wMi0yN1QyMTowMDowMC4wMDBa",
|
||||
"__number__50762",
|
||||
"__string__QQ==",
|
||||
"__string__0JTQsNC90L3Ri9C1INCh0LHQtdGA0JjQvdC00LXQutGB0LA=",
|
||||
"__string__0KHQstC10YDQtNC70L7QstGB0LrQsNGPINC+0LHQu9Cw0YHRgtGM",
|
||||
"__string__0JLRgtC+0YDQuNGH0L3Ri9C5",
|
||||
"__string__0JzQtdGB0Y/Rhg==",
|
||||
"__string__0YDRg9Cx",
|
||||
"__string__MA=="
|
||||
],
|
||||
[
|
||||
"__string__ODM0NTUxNTktMjYyOC00ZmNlLWIzNDktZGZjODU3MTU1ZjEx",
|
||||
"__string__NjQzMDMyMDAx",
|
||||
"__string__MjAxNy0wMy0zMFQyMTowMDowMC4wMDBa",
|
||||
"__number__50902",
|
||||
"__string__QQ==",
|
||||
"__string__0JTQsNC90L3Ri9C1INCh0LHQtdGA0JjQvdC00LXQutGB0LA=",
|
||||
"__string__0KHQstC10YDQtNC70L7QstGB0LrQsNGPINC+0LHQu9Cw0YHRgtGM",
|
||||
"__string__0JLRgtC+0YDQuNGH0L3Ri9C5",
|
||||
"__string__0JzQtdGB0Y/Rhg==",
|
||||
"__string__0YDRg9Cx",
|
||||
"__string__MA=="
|
||||
],
|
||||
[
|
||||
"__string__ODM0NTUxNTktMjYyOC00ZmNlLWIzNDktZGZjODU3MTU1ZjEx",
|
||||
"__string__NjQzMDMyMDAx",
|
||||
"__string__MjAxNy0wNC0yOVQyMTowMDowMC4wMDBa",
|
||||
"__number__50958",
|
||||
"__string__QQ==",
|
||||
"__string__0JTQsNC90L3Ri9C1INCh0LHQtdGA0JjQvdC00LXQutGB0LA=",
|
||||
"__string__0KHQstC10YDQtNC70L7QstGB0LrQsNGPINC+0LHQu9Cw0YHRgtGM",
|
||||
"__string__0JLRgtC+0YDQuNGH0L3Ri9C5",
|
||||
"__string__0JzQtdGB0Y/Rhg==",
|
||||
"__string__0YDRg9Cx",
|
||||
"__string__MA=="
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"key": "pagination",
|
||||
"type": "object",
|
||||
"value": {
|
||||
"type": "object",
|
||||
"value": [
|
||||
{
|
||||
"key": "total_records",
|
||||
"type": "number",
|
||||
"value": "__number__112"
|
||||
},
|
||||
{
|
||||
"key": "current_page",
|
||||
"type": "number",
|
||||
"value": "__number__1"
|
||||
},
|
||||
{
|
||||
"key": "total_pages",
|
||||
"type": "number",
|
||||
"value": "__number__28"
|
||||
},
|
||||
{
|
||||
"key": "next_page",
|
||||
"type": "number",
|
||||
"value": "__number__2"
|
||||
},
|
||||
{
|
||||
"key": "prev_page",
|
||||
"type": "null",
|
||||
"value": "__null__null"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "limit",
|
||||
"type": "number",
|
||||
"value": "__number__4"
|
||||
},
|
||||
{
|
||||
"key": "offset",
|
||||
"type": "number",
|
||||
"value": "__number__0"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,80 @@
|
|||
"""Tests for #794 СберИндекс time-adjustment helper — pure, no DB."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
from datetime import date
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
# DATABASE_URL required by pydantic Settings before any app import.
|
||||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
|
||||
|
||||
# WeasyPrint stub — not installed in CI без GTK.
|
||||
sys.modules.setdefault("weasyprint", MagicMock())
|
||||
|
||||
from app.services.estimator import ( # noqa: E402
|
||||
SBER_TIME_FACTOR_MAX,
|
||||
SBER_TIME_FACTOR_MIN,
|
||||
_sber_time_factor,
|
||||
)
|
||||
|
||||
|
||||
def test_empty_series_returns_one() -> None:
|
||||
assert _sber_time_factor({}, date(2026, 1, 1)) == 1.0
|
||||
|
||||
|
||||
def test_deal_month_equals_latest_returns_one() -> None:
|
||||
series = {date(2026, 1, 1): 95_000.0, date(2026, 4, 1): 100_000.0}
|
||||
assert _sber_time_factor(series, date(2026, 4, 1)) == 1.0
|
||||
|
||||
|
||||
def test_deal_month_after_latest_returns_one() -> None:
|
||||
series = {date(2026, 1, 1): 95_000.0, date(2026, 4, 1): 100_000.0}
|
||||
assert _sber_time_factor(series, date(2026, 6, 1)) == 1.0
|
||||
|
||||
|
||||
def test_normal_factor_calculation() -> None:
|
||||
series = {date(2026, 1, 1): 95_000.0, date(2026, 4, 1): 100_000.0}
|
||||
factor = _sber_time_factor(series, date(2026, 1, 1))
|
||||
expected = 100_000.0 / 95_000.0 # ≈ 1.0526
|
||||
assert abs(factor - expected) < 0.0001
|
||||
assert SBER_TIME_FACTOR_MIN <= factor <= SBER_TIME_FACTOR_MAX
|
||||
|
||||
|
||||
def test_clamp_high() -> None:
|
||||
"""Very old deal → factor would be >>1.6 → clamp to SBER_TIME_FACTOR_MAX."""
|
||||
series = {date(2020, 1, 1): 10_000.0, date(2026, 4, 1): 100_000.0}
|
||||
factor = _sber_time_factor(series, date(2020, 1, 1))
|
||||
assert factor == SBER_TIME_FACTOR_MAX
|
||||
|
||||
|
||||
def test_clamp_low() -> None:
|
||||
"""Series shows price drop → factor <<0.7 → clamp to SBER_TIME_FACTOR_MIN."""
|
||||
series = {date(2020, 1, 1): 200_000.0, date(2026, 4, 1): 50_000.0}
|
||||
factor = _sber_time_factor(series, date(2020, 1, 1))
|
||||
assert factor == SBER_TIME_FACTOR_MIN
|
||||
|
||||
|
||||
def test_missing_month_uses_nearest_earlier() -> None:
|
||||
"""deal_month=2026-02 not in series; nearest earlier is 2026-01."""
|
||||
series = {date(2026, 1, 1): 95_000.0, date(2026, 4, 1): 100_000.0}
|
||||
# Using 2026-01 as base: factor = 100000/95000
|
||||
factor_with_missing = _sber_time_factor(series, date(2026, 2, 1))
|
||||
factor_explicit = _sber_time_factor(series, date(2026, 1, 1))
|
||||
assert abs(factor_with_missing - factor_explicit) < 0.0001
|
||||
|
||||
|
||||
def test_deal_older_than_series_uses_earliest() -> None:
|
||||
"""Deal month before all series data → use earliest available month."""
|
||||
series = {date(2023, 1, 1): 80_000.0, date(2026, 4, 1): 100_000.0}
|
||||
# deal at 2020-01 is before series start → base = series[2023-01] = 80000
|
||||
factor = _sber_time_factor(series, date(2020, 1, 1))
|
||||
expected = min(100_000.0 / 80_000.0, SBER_TIME_FACTOR_MAX)
|
||||
assert abs(factor - expected) < 0.0001
|
||||
|
||||
|
||||
def test_zero_base_returns_one() -> None:
|
||||
"""Zero index value in series → guard → return 1.0."""
|
||||
series = {date(2026, 1, 1): 0.0, date(2026, 4, 1): 100_000.0}
|
||||
assert _sber_time_factor(series, date(2026, 1, 1)) == 1.0
|
||||
113
tradein-mvp/backend/tests/services/test_sber_index.py
Normal file
113
tradein-mvp/backend/tests/services/test_sber_index.py
Normal file
|
|
@ -0,0 +1,113 @@
|
|||
"""Tests for app/services/sber_index.py — per-dashboard filter fix (#794)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import json
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
from urllib.parse import unquote
|
||||
|
||||
import pytest
|
||||
|
||||
from app.services.sber_index import (
|
||||
SBER_DASHBOARDS,
|
||||
SberDashboard,
|
||||
_parse_period_month,
|
||||
build_sber_route,
|
||||
decode_sber_response,
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixture path
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
FIXTURE_PATH = Path(__file__).parent.parent / "fixtures" / "sber_real_estate_deals_66.json"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# test_build_sber_route_per_dashboard
|
||||
# LOCKS the bug fix: each dashboard must have the correct filter keys.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dashboard", SBER_DASHBOARDS)
|
||||
def test_build_sber_route_per_dashboard(dashboard: SberDashboard) -> None:
|
||||
route_b64 = build_sber_route(dashboard.slug, "66", dashboard.extra_filter)
|
||||
path = base64.b64decode(route_b64).decode("utf-8")
|
||||
|
||||
# Extract the filter JSON from the path: ?filter=<urlencoded>&limit=...
|
||||
filter_part = path.split("filter=")[1].split("&")[0]
|
||||
filter_dict = json.loads(unquote(filter_part))
|
||||
|
||||
# All routes must have REF_AREA + FREQ=M
|
||||
assert filter_dict.get("REF_AREA") == "66"
|
||||
assert filter_dict.get("FREQ") == "M"
|
||||
|
||||
if dashboard.slug == "real_estate_deals":
|
||||
assert filter_dict.get("SOURCE") == "SI"
|
||||
assert filter_dict.get("REALTY") == "2"
|
||||
|
||||
elif dashboard.slug == "residential_real_estate_prices":
|
||||
assert filter_dict.get("SOURCE") == "SI"
|
||||
assert filter_dict.get("REAL_ESTATE_NOVELTY") == "3"
|
||||
|
||||
elif dashboard.slug == "dinamika-tsen-obyavlenii":
|
||||
assert filter_dict.get("SOURCE") == "DK"
|
||||
assert filter_dict.get("PRICE_TYPE") == "1"
|
||||
|
||||
else:
|
||||
pytest.fail(f"Unknown dashboard slug: {dashboard.slug!r}")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# test_decode_sber_response_real_fixture
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_decode_sber_response_real_fixture() -> None:
|
||||
with FIXTURE_PATH.open(encoding="utf-8") as f:
|
||||
raw = json.load(f)
|
||||
|
||||
rows = decode_sber_response(raw)
|
||||
|
||||
assert len(rows) >= 1, "Expected at least 1 decoded row from real fixture"
|
||||
|
||||
# All rows should have the expected fields present
|
||||
first = rows[0]
|
||||
assert "ref_area" in first
|
||||
assert "realty" in first
|
||||
assert "value" in first
|
||||
assert "period" in first
|
||||
|
||||
# Values must be typed correctly
|
||||
assert isinstance(first["value"], float)
|
||||
assert isinstance(first["period"], str) and first["period"]
|
||||
|
||||
# Verify region label decoded correctly
|
||||
regions = {r["ref_area"] for r in rows}
|
||||
assert (
|
||||
"Свердловская область" in regions
|
||||
), f"Expected 'Свердловская область' in ref_area values, got: {regions}"
|
||||
|
||||
# Verify secondary-market segment present
|
||||
realty_vals = {r["realty"] for r in rows}
|
||||
assert any(
|
||||
"тори" in str(v) or "Вторичн" in str(v) for v in realty_vals
|
||||
), f"Expected secondary-market label in realty field, got: {realty_vals}"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# test_parse_period_month
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_parse_period_month() -> None:
|
||||
result = _parse_period_month("2026-04-29T21:00:00.000Z")
|
||||
assert result == date(2026, 4, 1)
|
||||
|
||||
|
||||
def test_parse_period_month_z_suffix() -> None:
|
||||
"""Original format from live API."""
|
||||
result = _parse_period_month("2017-01-30T21:00:00Z")
|
||||
assert result == date(2017, 1, 1)
|
||||
Loading…
Add table
Reference in a new issue