fix(tradein): correct SberIndex hedonic/asking dataset-paths + 404 warning (#902)
This commit is contained in:
parent
13db14c8c9
commit
5ca987fcd3
2 changed files with 157 additions and 15 deletions
|
|
@ -20,6 +20,18 @@ Live-verified 2026-05-31: residential_real_estate_prices needs REAL_ESTATE_NOVEL
|
|||
dinamika-tsen-obyavlenii needs SOURCE:DK + PRICE_TYPE. Filter is now per-dashboard via
|
||||
SberDashboard dataclass.
|
||||
|
||||
#902 fix: residential_real_estate_prices + dinamika-tsen-obyavlenii returned 404 — the
|
||||
#794 filter dims were wrong. Re-captured live 2026-06-17 via Playwright route-interception
|
||||
on the sber dashboards (200 OK):
|
||||
- data path = /dataset/v1/<slug>?filter=<urlencode(json)>&limit=1000&offset=0
|
||||
(NOT /dataset/v1/data/<slug>)
|
||||
- residential_real_estate_prices: SOURCE=SI, FREQ=M, REAL_ESTATE_NOVELTY=1, and
|
||||
REAL_ESTATE_TYPE ∈ {1=новостройка, 2=вторичка}. The #794 code sent NOVELTY=3 (no such
|
||||
code) + TYPE=1 → 404. We pull TYPE=2 (вторичка) — the estimator runs вторичка-only and
|
||||
sber_price_index PK is (city, period_month, dashboard) = one series per slug.
|
||||
- dinamika-tsen-obyavlenii: SOURCE=DK, FREQ=M, PRICE_TYPE ∈ {1, 2} (both 200). We keep
|
||||
PRICE_TYPE=1; the slug+SOURCE=DK is the verified-live combo.
|
||||
|
||||
REF_AREA codes (sberindex internal region IDs, not ОКАТО/ISO):
|
||||
643 = Россия — confirmed live, vault research note 2026-05-31-0845.
|
||||
66 = Свердловская область — verified 2026-05-31 brute-force against /api/sowa
|
||||
|
|
@ -83,11 +95,15 @@ SBER_DASHBOARDS: list[SberDashboard] = [
|
|||
"realty",
|
||||
),
|
||||
SberDashboard(
|
||||
# #902: вторичка hedonic. REAL_ESTATE_TYPE=2 (вторичка), NOVELTY=1.
|
||||
# Old #794 dims (TYPE=1, NOVELTY=3) → 404; re-captured live 2026-06-17.
|
||||
"residential_real_estate_prices",
|
||||
{"SOURCE": "SI", "REAL_ESTATE_TYPE": "1", "REAL_ESTATE_NOVELTY": "3"},
|
||||
"real_estate_novelty",
|
||||
{"SOURCE": "SI", "REAL_ESTATE_TYPE": "2", "REAL_ESTATE_NOVELTY": "1"},
|
||||
"real_estate_type",
|
||||
),
|
||||
SberDashboard(
|
||||
# #902: asking-price benchmark (ДомКлик). SOURCE=DK, PRICE_TYPE=1 —
|
||||
# verified-live 200 combo 2026-06-17.
|
||||
"dinamika-tsen-obyavlenii",
|
||||
{"SOURCE": "DK", "PRICE_TYPE": "1"},
|
||||
"price_type",
|
||||
|
|
@ -282,9 +298,7 @@ async def fetch_sber_index(
|
|||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _upsert_rows_sync(
|
||||
db: Session, rows_to_upsert: list[tuple[str, date, str, str, float]]
|
||||
) -> None:
|
||||
def _upsert_rows_sync(db: Session, rows_to_upsert: list[tuple[str, date, str, str, float]]) -> None:
|
||||
"""Synchronous per-row UPSERT into sber_price_index + commit.
|
||||
|
||||
#1348: blocking psycopg work — must run via asyncio.to_thread, never directly
|
||||
|
|
@ -428,9 +442,7 @@ async def pull_sber_indices(
|
|||
# Task B: best-effort reconciliation to our scrape-median asking ppm²
|
||||
try:
|
||||
# #1348: blocking benchmark SELECT off the event loop
|
||||
our_median = await asyncio.to_thread(
|
||||
_query_our_median_sync, db
|
||||
)
|
||||
our_median = await asyncio.to_thread(_query_our_median_sync, db)
|
||||
if our_median is not None and latest_val:
|
||||
sber_latest = latest_val
|
||||
divergence_pct = (our_median - sber_latest) / sber_latest * 100
|
||||
|
|
@ -448,12 +460,26 @@ async def pull_sber_indices(
|
|||
)
|
||||
|
||||
except httpx.HTTPStatusError as exc:
|
||||
logger.error(
|
||||
"sber_index: HTTP %d for dashboard=%s ref_area=%s — skip series",
|
||||
exc.response.status_code,
|
||||
dashboard.slug,
|
||||
ref_area,
|
||||
)
|
||||
# #902: a 404 on /api/sowa almost always means the /dataset/v1/<slug>
|
||||
# path or its filter dims are stale (sber renames slugs / changes
|
||||
# dimension codes). Surface it loudly with the slug + filter so the
|
||||
# next breakage is diagnosable instead of a silent error-counter bump.
|
||||
if exc.response.status_code == 404:
|
||||
logger.warning(
|
||||
"sber_index: 404 for dashboard=%s ref_area=%s filter=%s — "
|
||||
"dataset-path invalid? slug renamed or filter dims stale "
|
||||
"(re-capture /dataset/v1/<slug> via dashboard route-interception)",
|
||||
dashboard.slug,
|
||||
ref_area,
|
||||
{"REF_AREA": ref_area, "FREQ": "M", **dashboard.extra_filter},
|
||||
)
|
||||
else:
|
||||
logger.error(
|
||||
"sber_index: HTTP %d for dashboard=%s ref_area=%s — skip series",
|
||||
exc.response.status_code,
|
||||
dashboard.slug,
|
||||
ref_area,
|
||||
)
|
||||
counters["errors"] += 1
|
||||
except httpx.RequestError as exc:
|
||||
logger.error(
|
||||
|
|
|
|||
|
|
@ -118,8 +118,10 @@ def test_build_sber_route_per_dashboard(dashboard: SberDashboard) -> None:
|
|||
assert filter_dict.get("REALTY") == "2"
|
||||
|
||||
elif dashboard.slug == "residential_real_estate_prices":
|
||||
# #902: вторичка hedonic — re-captured live 2026-06-17 (was TYPE=1/NOVELTY=3 → 404).
|
||||
assert filter_dict.get("SOURCE") == "SI"
|
||||
assert filter_dict.get("REAL_ESTATE_NOVELTY") == "3"
|
||||
assert filter_dict.get("REAL_ESTATE_TYPE") == "2"
|
||||
assert filter_dict.get("REAL_ESTATE_NOVELTY") == "1"
|
||||
|
||||
elif dashboard.slug == "dinamika-tsen-obyavlenii":
|
||||
assert filter_dict.get("SOURCE") == "DK"
|
||||
|
|
@ -129,6 +131,83 @@ def test_build_sber_route_per_dashboard(dashboard: SberDashboard) -> None:
|
|||
pytest.fail(f"Unknown dashboard slug: {dashboard.slug!r}")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# (a2) #902 — exact base64 lock for the live-captured /dataset/v1/<slug> routes
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# REF_AREA=643 (РФ) routes captured live 2026-06-17 via Playwright route-interception
|
||||
# on the sber dashboards (all 200 OK). Each entry: (slug, filter_dict, expected_base64).
|
||||
# The expected base64 is base64(/dataset/v1/<slug>?filter=<urlencode(json)>&limit=1000&offset=0)
|
||||
# with json.dumps(separators=(",", ":")) field order = REF_AREA, FREQ, SOURCE, <dims...>.
|
||||
_CAPTURED_ROUTES_643: list[tuple[str, dict[str, str], str]] = [
|
||||
(
|
||||
"residential_real_estate_prices",
|
||||
{"SOURCE": "SI", "REAL_ESTATE_TYPE": "1", "REAL_ESTATE_NOVELTY": "1"},
|
||||
"L2RhdGFzZXQvdjEvcmVzaWRlbnRpYWxfcmVhbF9lc3RhdGVfcHJpY2VzP2ZpbHRlcj0lN0IlMjJSRUZf"
|
||||
"QVJFQSUyMiUzQSUyMjY0MyUyMiUyQyUyMkZSRVElMjIlM0ElMjJNJTIyJTJDJTIyU09VUkNFJTIyJTNB"
|
||||
"JTIyU0klMjIlMkMlMjJSRUFMX0VTVEFURV9UWVBFJTIyJTNBJTIyMSUyMiUyQyUyMlJFQUxfRVNUQVRF"
|
||||
"X05PVkVMVFklMjIlM0ElMjIxJTIyJTdEJmxpbWl0PTEwMDAmb2Zmc2V0PTA=",
|
||||
),
|
||||
(
|
||||
"residential_real_estate_prices",
|
||||
{"SOURCE": "SI", "REAL_ESTATE_TYPE": "2", "REAL_ESTATE_NOVELTY": "1"},
|
||||
"L2RhdGFzZXQvdjEvcmVzaWRlbnRpYWxfcmVhbF9lc3RhdGVfcHJpY2VzP2ZpbHRlcj0lN0IlMjJSRUZf"
|
||||
"QVJFQSUyMiUzQSUyMjY0MyUyMiUyQyUyMkZSRVElMjIlM0ElMjJNJTIyJTJDJTIyU09VUkNFJTIyJTNB"
|
||||
"JTIyU0klMjIlMkMlMjJSRUFMX0VTVEFURV9UWVBFJTIyJTNBJTIyMiUyMiUyQyUyMlJFQUxfRVNUQVRF"
|
||||
"X05PVkVMVFklMjIlM0ElMjIxJTIyJTdEJmxpbWl0PTEwMDAmb2Zmc2V0PTA=",
|
||||
),
|
||||
(
|
||||
"dinamika-tsen-obyavlenii",
|
||||
{"SOURCE": "DK", "PRICE_TYPE": "1"},
|
||||
"L2RhdGFzZXQvdjEvZGluYW1pa2EtdHNlbi1vYnlhdmxlbmlpP2ZpbHRlcj0lN0IlMjJSRUZfQVJFQSUy"
|
||||
"MiUzQSUyMjY0MyUyMiUyQyUyMkZSRVElMjIlM0ElMjJNJTIyJTJDJTIyU09VUkNFJTIyJTNBJTIyREsl"
|
||||
"MjIlMkMlMjJQUklDRV9UWVBFJTIyJTNBJTIyMSUyMiU3RCZsaW1pdD0xMDAwJm9mZnNldD0w",
|
||||
),
|
||||
(
|
||||
"dinamika-tsen-obyavlenii",
|
||||
{"SOURCE": "DK", "PRICE_TYPE": "2"},
|
||||
"L2RhdGFzZXQvdjEvZGluYW1pa2EtdHNlbi1vYnlhdmxlbmlpP2ZpbHRlcj0lN0IlMjJSRUZfQVJFQSUy"
|
||||
"MiUzQSUyMjY0MyUyMiUyQyUyMkZSRVElMjIlM0ElMjJNJTIyJTJDJTIyU09VUkNFJTIyJTNBJTIyREsl"
|
||||
"MjIlMkMlMjJQUklDRV9UWVBFJTIyJTNBJTIyMiUyMiU3RCZsaW1pdD0xMDAwJm9mZnNldD0w",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("slug", "extra_filter", "expected_b64"), _CAPTURED_ROUTES_643)
|
||||
def test_build_sber_route_matches_captured_b64(
|
||||
slug: str, extra_filter: dict[str, str], expected_b64: str
|
||||
) -> None:
|
||||
"""#902: build_sber_route must reproduce the live-captured base64 route byte-for-byte.
|
||||
|
||||
Locks the /dataset/v1/<slug> path (NOT /dataset/v1/data/<slug>) and the exact filter
|
||||
dimension set + JSON key order. Guards against silent dataset-path regressions.
|
||||
"""
|
||||
encoded = build_sber_route(slug, "643", extra_filter)
|
||||
assert encoded == expected_b64
|
||||
|
||||
# And the decoded path must use the bare /dataset/v1/<slug> form (not /data/).
|
||||
decoded = base64.b64decode(encoded).decode("utf-8")
|
||||
assert decoded.startswith(f"/dataset/v1/{slug}?filter=")
|
||||
assert "/dataset/v1/data/" not in decoded
|
||||
|
||||
|
||||
def test_active_dashboards_match_captured_secondary_series() -> None:
|
||||
"""#902: the two previously-404'ing dashboards now use a captured-200 filter combo.
|
||||
|
||||
residential_real_estate_prices → вторичка hedonic (TYPE=2, NOVELTY=1).
|
||||
dinamika-tsen-obyavlenii → ДомКлик asking (PRICE_TYPE=1).
|
||||
Each active SBER_DASHBOARDS entry's base64 must equal one of the captured 643 routes.
|
||||
"""
|
||||
captured_b64 = {b for _, _, b in _CAPTURED_ROUTES_643}
|
||||
for slug in ("residential_real_estate_prices", "dinamika-tsen-obyavlenii"):
|
||||
d = _DASH[slug]
|
||||
encoded = build_sber_route(d.slug, "643", d.extra_filter)
|
||||
assert encoded in captured_b64, (
|
||||
f"active dashboard {slug!r} route not in captured-live set — "
|
||||
f"filter={d.extra_filter}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# (b) Response decoder
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
@ -389,6 +468,43 @@ async def test_pull_sber_indices_error_per_series_continues() -> None:
|
|||
assert result["upserted"] == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pull_sber_indices_404_logs_warning_with_path_hint(
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
) -> None:
|
||||
"""#902: a 404 logs a WARNING naming the slug + 'dataset-path invalid?' hint."""
|
||||
import logging
|
||||
|
||||
import httpx
|
||||
|
||||
from app.services.sber_index import pull_sber_indices
|
||||
|
||||
fake_db = _ExecuteCapture()
|
||||
|
||||
async def _mock_fetch_404( # type: ignore[override]
|
||||
client: object, *, dashboard: SberDashboard, ref_area: str
|
||||
):
|
||||
request = httpx.Request("POST", "https://sberindex.ru/api/sowa")
|
||||
response = httpx.Response(404, request=request)
|
||||
raise httpx.HTTPStatusError("Not Found", request=request, response=response)
|
||||
yield # pragma: no cover — makes this an async generator
|
||||
|
||||
with patch("app.services.sber_index.fetch_sber_index", side_effect=_mock_fetch_404):
|
||||
with caplog.at_level(logging.WARNING, logger="app.services.sber_index"):
|
||||
result = await pull_sber_indices(
|
||||
fake_db, # type: ignore[arg-type]
|
||||
cities={"643": "Россия"},
|
||||
dashboards=[_DASH["residential_real_estate_prices"]],
|
||||
)
|
||||
|
||||
assert result["errors"] == 1
|
||||
warnings = [r for r in caplog.records if r.levelno == logging.WARNING]
|
||||
assert any(
|
||||
"residential_real_estate_prices" in r.message and "dataset-path invalid" in r.message
|
||||
for r in warnings
|
||||
), f"Expected a 404 dataset-path warning, got: {[r.message for r in warnings]}"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pull_sber_indices_asking_benchmark_logged(caplog: pytest.LogCaptureFixture) -> None:
|
||||
"""After upserting dinamika-tsen-obyavlenii, a benchmark INFO line is logged."""
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue