fix(tradein): correct SberIndex hedonic/asking dataset-paths + 404 warning (#902)

tradein-mvp/backend/app/services/sber_index.py

@@ -20,6 +20,18 @@ Live-verified 2026-05-31: residential_real_estate_prices needs REAL_ESTATE_NOVEL
 dinamika-tsen-obyavlenii needs SOURCE:DK + PRICE_TYPE. Filter is now per-dashboard via
 SberDashboard dataclass.
 
+#902 fix: residential_real_estate_prices + dinamika-tsen-obyavlenii returned 404 — the
+#794 filter dims were wrong. Re-captured live 2026-06-17 via Playwright route-interception
+on the sber dashboards (200 OK):
+  - data path  = /dataset/v1/<slug>?filter=<urlencode(json)>&limit=1000&offset=0
+                 (NOT /dataset/v1/data/<slug>)
+  - residential_real_estate_prices: SOURCE=SI, FREQ=M, REAL_ESTATE_NOVELTY=1, and
+    REAL_ESTATE_TYPE ∈ {1=новостройка, 2=вторичка}. The #794 code sent NOVELTY=3 (no such
+    code) + TYPE=1 → 404. We pull TYPE=2 (вторичка) — the estimator runs вторичка-only and
+    sber_price_index PK is (city, period_month, dashboard) = one series per slug.
+  - dinamika-tsen-obyavlenii: SOURCE=DK, FREQ=M, PRICE_TYPE ∈ {1, 2} (both 200). We keep
+    PRICE_TYPE=1; the slug+SOURCE=DK is the verified-live combo.
+
 REF_AREA codes (sberindex internal region IDs, not ОКАТО/ISO):
   643 = Россия               — confirmed live, vault research note 2026-05-31-0845.
   66  = Свердловская область — verified 2026-05-31 brute-force against /api/sowa
@@ -83,11 +95,15 @@ SBER_DASHBOARDS: list[SberDashboard] = [
         "realty",
     ),
     SberDashboard(
+        # #902: вторичка hedonic. REAL_ESTATE_TYPE=2 (вторичка), NOVELTY=1.
+        # Old #794 dims (TYPE=1, NOVELTY=3) → 404; re-captured live 2026-06-17.
         "residential_real_estate_prices",
-        {"SOURCE": "SI", "REAL_ESTATE_TYPE": "1", "REAL_ESTATE_NOVELTY": "3"},
+        {"SOURCE": "SI", "REAL_ESTATE_TYPE": "2", "REAL_ESTATE_NOVELTY": "1"},
-        "real_estate_novelty",
+        "real_estate_type",
     ),
     SberDashboard(
+        # #902: asking-price benchmark (ДомКлик). SOURCE=DK, PRICE_TYPE=1 —
+        # verified-live 200 combo 2026-06-17.
         "dinamika-tsen-obyavlenii",
         {"SOURCE": "DK", "PRICE_TYPE": "1"},
         "price_type",
@@ -282,9 +298,7 @@ async def fetch_sber_index(
 # ---------------------------------------------------------------------------
 
 
-def _upsert_rows_sync(
+def _upsert_rows_sync(db: Session, rows_to_upsert: list[tuple[str, date, str, str, float]]) -> None:
-    db: Session, rows_to_upsert: list[tuple[str, date, str, str, float]]
-) -> None:
     """Synchronous per-row UPSERT into sber_price_index + commit.
 
     #1348: blocking psycopg work — must run via asyncio.to_thread, never directly
@@ -428,9 +442,7 @@ async def pull_sber_indices(
                             # Task B: best-effort reconciliation to our scrape-median asking ppm²
                             try:
                                 # #1348: blocking benchmark SELECT off the event loop
-                                our_median = await asyncio.to_thread(
+                                our_median = await asyncio.to_thread(_query_our_median_sync, db)
-                                    _query_our_median_sync, db
-                                )
                                 if our_median is not None and latest_val:
                                     sber_latest = latest_val
                                     divergence_pct = (our_median - sber_latest) / sber_latest * 100
@@ -448,12 +460,26 @@ async def pull_sber_indices(
                                 )
 
                 except httpx.HTTPStatusError as exc:
-                    logger.error(
+                    # #902: a 404 on /api/sowa almost always means the /dataset/v1/<slug>
-                        "sber_index: HTTP %d for dashboard=%s ref_area=%s — skip series",
+                    # path or its filter dims are stale (sber renames slugs / changes
-                        exc.response.status_code,
+                    # dimension codes). Surface it loudly with the slug + filter so the
-                        dashboard.slug,
+                    # next breakage is diagnosable instead of a silent error-counter bump.
-                        ref_area,
+                    if exc.response.status_code == 404:
-                    )
+                        logger.warning(
+                            "sber_index: 404 for dashboard=%s ref_area=%s filter=%s — "
+                            "dataset-path invalid? slug renamed or filter dims stale "
+                            "(re-capture /dataset/v1/<slug> via dashboard route-interception)",
+                            dashboard.slug,
+                            ref_area,
+                            {"REF_AREA": ref_area, "FREQ": "M", **dashboard.extra_filter},
+                        )
+                    else:
+                        logger.error(
+                            "sber_index: HTTP %d for dashboard=%s ref_area=%s — skip series",
+                            exc.response.status_code,
+                            dashboard.slug,
+                            ref_area,
+                        )
                     counters["errors"] += 1
                 except httpx.RequestError as exc:
                     logger.error(

tradein-mvp/backend/tests/test_sber_index.py

@@ -118,8 +118,10 @@ def test_build_sber_route_per_dashboard(dashboard: SberDashboard) -> None:
         assert filter_dict.get("REALTY") == "2"
 
     elif dashboard.slug == "residential_real_estate_prices":
+        # #902: вторичка hedonic — re-captured live 2026-06-17 (was TYPE=1/NOVELTY=3 → 404).
         assert filter_dict.get("SOURCE") == "SI"
-        assert filter_dict.get("REAL_ESTATE_NOVELTY") == "3"
+        assert filter_dict.get("REAL_ESTATE_TYPE") == "2"
+        assert filter_dict.get("REAL_ESTATE_NOVELTY") == "1"
 
     elif dashboard.slug == "dinamika-tsen-obyavlenii":
         assert filter_dict.get("SOURCE") == "DK"
@@ -129,6 +131,83 @@ def test_build_sber_route_per_dashboard(dashboard: SberDashboard) -> None:
         pytest.fail(f"Unknown dashboard slug: {dashboard.slug!r}")
 
 
+# ---------------------------------------------------------------------------
+# (a2) #902 — exact base64 lock for the live-captured /dataset/v1/<slug> routes
+# ---------------------------------------------------------------------------
+
+# REF_AREA=643 (РФ) routes captured live 2026-06-17 via Playwright route-interception
+# on the sber dashboards (all 200 OK). Each entry: (slug, filter_dict, expected_base64).
+# The expected base64 is base64(/dataset/v1/<slug>?filter=<urlencode(json)>&limit=1000&offset=0)
+# with json.dumps(separators=(",", ":")) field order = REF_AREA, FREQ, SOURCE, <dims...>.
+_CAPTURED_ROUTES_643: list[tuple[str, dict[str, str], str]] = [
+    (
+        "residential_real_estate_prices",
+        {"SOURCE": "SI", "REAL_ESTATE_TYPE": "1", "REAL_ESTATE_NOVELTY": "1"},
+        "L2RhdGFzZXQvdjEvcmVzaWRlbnRpYWxfcmVhbF9lc3RhdGVfcHJpY2VzP2ZpbHRlcj0lN0IlMjJSRUZf"
+        "QVJFQSUyMiUzQSUyMjY0MyUyMiUyQyUyMkZSRVElMjIlM0ElMjJNJTIyJTJDJTIyU09VUkNFJTIyJTNB"
+        "JTIyU0klMjIlMkMlMjJSRUFMX0VTVEFURV9UWVBFJTIyJTNBJTIyMSUyMiUyQyUyMlJFQUxfRVNUQVRF"
+        "X05PVkVMVFklMjIlM0ElMjIxJTIyJTdEJmxpbWl0PTEwMDAmb2Zmc2V0PTA=",
+    ),
+    (
+        "residential_real_estate_prices",
+        {"SOURCE": "SI", "REAL_ESTATE_TYPE": "2", "REAL_ESTATE_NOVELTY": "1"},
+        "L2RhdGFzZXQvdjEvcmVzaWRlbnRpYWxfcmVhbF9lc3RhdGVfcHJpY2VzP2ZpbHRlcj0lN0IlMjJSRUZf"
+        "QVJFQSUyMiUzQSUyMjY0MyUyMiUyQyUyMkZSRVElMjIlM0ElMjJNJTIyJTJDJTIyU09VUkNFJTIyJTNB"
+        "JTIyU0klMjIlMkMlMjJSRUFMX0VTVEFURV9UWVBFJTIyJTNBJTIyMiUyMiUyQyUyMlJFQUxfRVNUQVRF"
+        "X05PVkVMVFklMjIlM0ElMjIxJTIyJTdEJmxpbWl0PTEwMDAmb2Zmc2V0PTA=",
+    ),
+    (
+        "dinamika-tsen-obyavlenii",
+        {"SOURCE": "DK", "PRICE_TYPE": "1"},
+        "L2RhdGFzZXQvdjEvZGluYW1pa2EtdHNlbi1vYnlhdmxlbmlpP2ZpbHRlcj0lN0IlMjJSRUZfQVJFQSUy"
+        "MiUzQSUyMjY0MyUyMiUyQyUyMkZSRVElMjIlM0ElMjJNJTIyJTJDJTIyU09VUkNFJTIyJTNBJTIyREsl"
+        "MjIlMkMlMjJQUklDRV9UWVBFJTIyJTNBJTIyMSUyMiU3RCZsaW1pdD0xMDAwJm9mZnNldD0w",
+    ),
+    (
+        "dinamika-tsen-obyavlenii",
+        {"SOURCE": "DK", "PRICE_TYPE": "2"},
+        "L2RhdGFzZXQvdjEvZGluYW1pa2EtdHNlbi1vYnlhdmxlbmlpP2ZpbHRlcj0lN0IlMjJSRUZfQVJFQSUy"
+        "MiUzQSUyMjY0MyUyMiUyQyUyMkZSRVElMjIlM0ElMjJNJTIyJTJDJTIyU09VUkNFJTIyJTNBJTIyREsl"
+        "MjIlMkMlMjJQUklDRV9UWVBFJTIyJTNBJTIyMiUyMiU3RCZsaW1pdD0xMDAwJm9mZnNldD0w",
+    ),
+]
+
+
+@pytest.mark.parametrize(("slug", "extra_filter", "expected_b64"), _CAPTURED_ROUTES_643)
+def test_build_sber_route_matches_captured_b64(
+    slug: str, extra_filter: dict[str, str], expected_b64: str
+) -> None:
+    """#902: build_sber_route must reproduce the live-captured base64 route byte-for-byte.
+
+    Locks the /dataset/v1/<slug> path (NOT /dataset/v1/data/<slug>) and the exact filter
+    dimension set + JSON key order. Guards against silent dataset-path regressions.
+    """
+    encoded = build_sber_route(slug, "643", extra_filter)
+    assert encoded == expected_b64
+
+    # And the decoded path must use the bare /dataset/v1/<slug> form (not /data/).
+    decoded = base64.b64decode(encoded).decode("utf-8")
+    assert decoded.startswith(f"/dataset/v1/{slug}?filter=")
+    assert "/dataset/v1/data/" not in decoded
+
+
+def test_active_dashboards_match_captured_secondary_series() -> None:
+    """#902: the two previously-404'ing dashboards now use a captured-200 filter combo.
+
+    residential_real_estate_prices → вторичка hedonic (TYPE=2, NOVELTY=1).
+    dinamika-tsen-obyavlenii       → ДомКлик asking (PRICE_TYPE=1).
+    Each active SBER_DASHBOARDS entry's base64 must equal one of the captured 643 routes.
+    """
+    captured_b64 = {b for _, _, b in _CAPTURED_ROUTES_643}
+    for slug in ("residential_real_estate_prices", "dinamika-tsen-obyavlenii"):
+        d = _DASH[slug]
+        encoded = build_sber_route(d.slug, "643", d.extra_filter)
+        assert encoded in captured_b64, (
+            f"active dashboard {slug!r} route not in captured-live set — "
+            f"filter={d.extra_filter}"
+        )
+
+
 # ---------------------------------------------------------------------------
 # (b) Response decoder
 # ---------------------------------------------------------------------------
@@ -389,6 +468,43 @@ async def test_pull_sber_indices_error_per_series_continues() -> None:
     assert result["upserted"] == 1
 
 
+@pytest.mark.asyncio
+async def test_pull_sber_indices_404_logs_warning_with_path_hint(
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    """#902: a 404 logs a WARNING naming the slug + 'dataset-path invalid?' hint."""
+    import logging
+
+    import httpx
+
+    from app.services.sber_index import pull_sber_indices
+
+    fake_db = _ExecuteCapture()
+
+    async def _mock_fetch_404(  # type: ignore[override]
+        client: object, *, dashboard: SberDashboard, ref_area: str
+    ):
+        request = httpx.Request("POST", "https://sberindex.ru/api/sowa")
+        response = httpx.Response(404, request=request)
+        raise httpx.HTTPStatusError("Not Found", request=request, response=response)
+        yield  # pragma: no cover — makes this an async generator
+
+    with patch("app.services.sber_index.fetch_sber_index", side_effect=_mock_fetch_404):
+        with caplog.at_level(logging.WARNING, logger="app.services.sber_index"):
+            result = await pull_sber_indices(
+                fake_db,  # type: ignore[arg-type]
+                cities={"643": "Россия"},
+                dashboards=[_DASH["residential_real_estate_prices"]],
+            )
+
+    assert result["errors"] == 1
+    warnings = [r for r in caplog.records if r.levelno == logging.WARNING]
+    assert any(
+        "residential_real_estate_prices" in r.message and "dataset-path invalid" in r.message
+        for r in warnings
+    ), f"Expected a 404 dataset-path warning, got: {[r.message for r in warnings]}"
+
+
 @pytest.mark.asyncio
 async def test_pull_sber_indices_asking_benchmark_logged(caplog: pytest.LogCaptureFixture) -> None:
     """After upserting dinamika-tsen-obyavlenii, a benchmark INFO line is logged."""