From 5ca987fcd32bc38f3d037a7e5d9ddd36f80607a7 Mon Sep 17 00:00:00 2001 From: bot-backend Date: Wed, 17 Jun 2026 22:50:52 +0300 Subject: [PATCH] fix(tradein): correct SberIndex hedonic/asking dataset-paths + 404 warning (#902) --- .../backend/app/services/sber_index.py | 54 +++++--- tradein-mvp/backend/tests/test_sber_index.py | 118 +++++++++++++++++- 2 files changed, 157 insertions(+), 15 deletions(-) diff --git a/tradein-mvp/backend/app/services/sber_index.py b/tradein-mvp/backend/app/services/sber_index.py index e0b0942b..36b7bc95 100644 --- a/tradein-mvp/backend/app/services/sber_index.py +++ b/tradein-mvp/backend/app/services/sber_index.py @@ -20,6 +20,18 @@ Live-verified 2026-05-31: residential_real_estate_prices needs REAL_ESTATE_NOVEL dinamika-tsen-obyavlenii needs SOURCE:DK + PRICE_TYPE. Filter is now per-dashboard via SberDashboard dataclass. +#902 fix: residential_real_estate_prices + dinamika-tsen-obyavlenii returned 404 — the +#794 filter dims were wrong. Re-captured live 2026-06-17 via Playwright route-interception +on the sber dashboards (200 OK): + - data path = /dataset/v1/?filter=&limit=1000&offset=0 + (NOT /dataset/v1/data/) + - residential_real_estate_prices: SOURCE=SI, FREQ=M, REAL_ESTATE_NOVELTY=1, and + REAL_ESTATE_TYPE ∈ {1=новостройка, 2=вторичка}. The #794 code sent NOVELTY=3 (no such + code) + TYPE=1 → 404. We pull TYPE=2 (вторичка) — the estimator runs вторичка-only and + sber_price_index PK is (city, period_month, dashboard) = one series per slug. + - dinamika-tsen-obyavlenii: SOURCE=DK, FREQ=M, PRICE_TYPE ∈ {1, 2} (both 200). We keep + PRICE_TYPE=1; the slug+SOURCE=DK is the verified-live combo. + REF_AREA codes (sberindex internal region IDs, not ОКАТО/ISO): 643 = Россия — confirmed live, vault research note 2026-05-31-0845. 66 = Свердловская область — verified 2026-05-31 brute-force against /api/sowa @@ -83,11 +95,15 @@ SBER_DASHBOARDS: list[SberDashboard] = [ "realty", ), SberDashboard( + # #902: вторичка hedonic. REAL_ESTATE_TYPE=2 (вторичка), NOVELTY=1. + # Old #794 dims (TYPE=1, NOVELTY=3) → 404; re-captured live 2026-06-17. "residential_real_estate_prices", - {"SOURCE": "SI", "REAL_ESTATE_TYPE": "1", "REAL_ESTATE_NOVELTY": "3"}, - "real_estate_novelty", + {"SOURCE": "SI", "REAL_ESTATE_TYPE": "2", "REAL_ESTATE_NOVELTY": "1"}, + "real_estate_type", ), SberDashboard( + # #902: asking-price benchmark (ДомКлик). SOURCE=DK, PRICE_TYPE=1 — + # verified-live 200 combo 2026-06-17. "dinamika-tsen-obyavlenii", {"SOURCE": "DK", "PRICE_TYPE": "1"}, "price_type", @@ -282,9 +298,7 @@ async def fetch_sber_index( # --------------------------------------------------------------------------- -def _upsert_rows_sync( - db: Session, rows_to_upsert: list[tuple[str, date, str, str, float]] -) -> None: +def _upsert_rows_sync(db: Session, rows_to_upsert: list[tuple[str, date, str, str, float]]) -> None: """Synchronous per-row UPSERT into sber_price_index + commit. #1348: blocking psycopg work — must run via asyncio.to_thread, never directly @@ -428,9 +442,7 @@ async def pull_sber_indices( # Task B: best-effort reconciliation to our scrape-median asking ppm² try: # #1348: blocking benchmark SELECT off the event loop - our_median = await asyncio.to_thread( - _query_our_median_sync, db - ) + our_median = await asyncio.to_thread(_query_our_median_sync, db) if our_median is not None and latest_val: sber_latest = latest_val divergence_pct = (our_median - sber_latest) / sber_latest * 100 @@ -448,12 +460,26 @@ async def pull_sber_indices( ) except httpx.HTTPStatusError as exc: - logger.error( - "sber_index: HTTP %d for dashboard=%s ref_area=%s — skip series", - exc.response.status_code, - dashboard.slug, - ref_area, - ) + # #902: a 404 on /api/sowa almost always means the /dataset/v1/ + # path or its filter dims are stale (sber renames slugs / changes + # dimension codes). Surface it loudly with the slug + filter so the + # next breakage is diagnosable instead of a silent error-counter bump. + if exc.response.status_code == 404: + logger.warning( + "sber_index: 404 for dashboard=%s ref_area=%s filter=%s — " + "dataset-path invalid? slug renamed or filter dims stale " + "(re-capture /dataset/v1/ via dashboard route-interception)", + dashboard.slug, + ref_area, + {"REF_AREA": ref_area, "FREQ": "M", **dashboard.extra_filter}, + ) + else: + logger.error( + "sber_index: HTTP %d for dashboard=%s ref_area=%s — skip series", + exc.response.status_code, + dashboard.slug, + ref_area, + ) counters["errors"] += 1 except httpx.RequestError as exc: logger.error( diff --git a/tradein-mvp/backend/tests/test_sber_index.py b/tradein-mvp/backend/tests/test_sber_index.py index 728868f5..ed31e11c 100644 --- a/tradein-mvp/backend/tests/test_sber_index.py +++ b/tradein-mvp/backend/tests/test_sber_index.py @@ -118,8 +118,10 @@ def test_build_sber_route_per_dashboard(dashboard: SberDashboard) -> None: assert filter_dict.get("REALTY") == "2" elif dashboard.slug == "residential_real_estate_prices": + # #902: вторичка hedonic — re-captured live 2026-06-17 (was TYPE=1/NOVELTY=3 → 404). assert filter_dict.get("SOURCE") == "SI" - assert filter_dict.get("REAL_ESTATE_NOVELTY") == "3" + assert filter_dict.get("REAL_ESTATE_TYPE") == "2" + assert filter_dict.get("REAL_ESTATE_NOVELTY") == "1" elif dashboard.slug == "dinamika-tsen-obyavlenii": assert filter_dict.get("SOURCE") == "DK" @@ -129,6 +131,83 @@ def test_build_sber_route_per_dashboard(dashboard: SberDashboard) -> None: pytest.fail(f"Unknown dashboard slug: {dashboard.slug!r}") +# --------------------------------------------------------------------------- +# (a2) #902 — exact base64 lock for the live-captured /dataset/v1/ routes +# --------------------------------------------------------------------------- + +# REF_AREA=643 (РФ) routes captured live 2026-06-17 via Playwright route-interception +# on the sber dashboards (all 200 OK). Each entry: (slug, filter_dict, expected_base64). +# The expected base64 is base64(/dataset/v1/?filter=&limit=1000&offset=0) +# with json.dumps(separators=(",", ":")) field order = REF_AREA, FREQ, SOURCE, . +_CAPTURED_ROUTES_643: list[tuple[str, dict[str, str], str]] = [ + ( + "residential_real_estate_prices", + {"SOURCE": "SI", "REAL_ESTATE_TYPE": "1", "REAL_ESTATE_NOVELTY": "1"}, + "L2RhdGFzZXQvdjEvcmVzaWRlbnRpYWxfcmVhbF9lc3RhdGVfcHJpY2VzP2ZpbHRlcj0lN0IlMjJSRUZf" + "QVJFQSUyMiUzQSUyMjY0MyUyMiUyQyUyMkZSRVElMjIlM0ElMjJNJTIyJTJDJTIyU09VUkNFJTIyJTNB" + "JTIyU0klMjIlMkMlMjJSRUFMX0VTVEFURV9UWVBFJTIyJTNBJTIyMSUyMiUyQyUyMlJFQUxfRVNUQVRF" + "X05PVkVMVFklMjIlM0ElMjIxJTIyJTdEJmxpbWl0PTEwMDAmb2Zmc2V0PTA=", + ), + ( + "residential_real_estate_prices", + {"SOURCE": "SI", "REAL_ESTATE_TYPE": "2", "REAL_ESTATE_NOVELTY": "1"}, + "L2RhdGFzZXQvdjEvcmVzaWRlbnRpYWxfcmVhbF9lc3RhdGVfcHJpY2VzP2ZpbHRlcj0lN0IlMjJSRUZf" + "QVJFQSUyMiUzQSUyMjY0MyUyMiUyQyUyMkZSRVElMjIlM0ElMjJNJTIyJTJDJTIyU09VUkNFJTIyJTNB" + "JTIyU0klMjIlMkMlMjJSRUFMX0VTVEFURV9UWVBFJTIyJTNBJTIyMiUyMiUyQyUyMlJFQUxfRVNUQVRF" + "X05PVkVMVFklMjIlM0ElMjIxJTIyJTdEJmxpbWl0PTEwMDAmb2Zmc2V0PTA=", + ), + ( + "dinamika-tsen-obyavlenii", + {"SOURCE": "DK", "PRICE_TYPE": "1"}, + "L2RhdGFzZXQvdjEvZGluYW1pa2EtdHNlbi1vYnlhdmxlbmlpP2ZpbHRlcj0lN0IlMjJSRUZfQVJFQSUy" + "MiUzQSUyMjY0MyUyMiUyQyUyMkZSRVElMjIlM0ElMjJNJTIyJTJDJTIyU09VUkNFJTIyJTNBJTIyREsl" + "MjIlMkMlMjJQUklDRV9UWVBFJTIyJTNBJTIyMSUyMiU3RCZsaW1pdD0xMDAwJm9mZnNldD0w", + ), + ( + "dinamika-tsen-obyavlenii", + {"SOURCE": "DK", "PRICE_TYPE": "2"}, + "L2RhdGFzZXQvdjEvZGluYW1pa2EtdHNlbi1vYnlhdmxlbmlpP2ZpbHRlcj0lN0IlMjJSRUZfQVJFQSUy" + "MiUzQSUyMjY0MyUyMiUyQyUyMkZSRVElMjIlM0ElMjJNJTIyJTJDJTIyU09VUkNFJTIyJTNBJTIyREsl" + "MjIlMkMlMjJQUklDRV9UWVBFJTIyJTNBJTIyMiUyMiU3RCZsaW1pdD0xMDAwJm9mZnNldD0w", + ), +] + + +@pytest.mark.parametrize(("slug", "extra_filter", "expected_b64"), _CAPTURED_ROUTES_643) +def test_build_sber_route_matches_captured_b64( + slug: str, extra_filter: dict[str, str], expected_b64: str +) -> None: + """#902: build_sber_route must reproduce the live-captured base64 route byte-for-byte. + + Locks the /dataset/v1/ path (NOT /dataset/v1/data/) and the exact filter + dimension set + JSON key order. Guards against silent dataset-path regressions. + """ + encoded = build_sber_route(slug, "643", extra_filter) + assert encoded == expected_b64 + + # And the decoded path must use the bare /dataset/v1/ form (not /data/). + decoded = base64.b64decode(encoded).decode("utf-8") + assert decoded.startswith(f"/dataset/v1/{slug}?filter=") + assert "/dataset/v1/data/" not in decoded + + +def test_active_dashboards_match_captured_secondary_series() -> None: + """#902: the two previously-404'ing dashboards now use a captured-200 filter combo. + + residential_real_estate_prices → вторичка hedonic (TYPE=2, NOVELTY=1). + dinamika-tsen-obyavlenii → ДомКлик asking (PRICE_TYPE=1). + Each active SBER_DASHBOARDS entry's base64 must equal one of the captured 643 routes. + """ + captured_b64 = {b for _, _, b in _CAPTURED_ROUTES_643} + for slug in ("residential_real_estate_prices", "dinamika-tsen-obyavlenii"): + d = _DASH[slug] + encoded = build_sber_route(d.slug, "643", d.extra_filter) + assert encoded in captured_b64, ( + f"active dashboard {slug!r} route not in captured-live set — " + f"filter={d.extra_filter}" + ) + + # --------------------------------------------------------------------------- # (b) Response decoder # --------------------------------------------------------------------------- @@ -389,6 +468,43 @@ async def test_pull_sber_indices_error_per_series_continues() -> None: assert result["upserted"] == 1 +@pytest.mark.asyncio +async def test_pull_sber_indices_404_logs_warning_with_path_hint( + caplog: pytest.LogCaptureFixture, +) -> None: + """#902: a 404 logs a WARNING naming the slug + 'dataset-path invalid?' hint.""" + import logging + + import httpx + + from app.services.sber_index import pull_sber_indices + + fake_db = _ExecuteCapture() + + async def _mock_fetch_404( # type: ignore[override] + client: object, *, dashboard: SberDashboard, ref_area: str + ): + request = httpx.Request("POST", "https://sberindex.ru/api/sowa") + response = httpx.Response(404, request=request) + raise httpx.HTTPStatusError("Not Found", request=request, response=response) + yield # pragma: no cover — makes this an async generator + + with patch("app.services.sber_index.fetch_sber_index", side_effect=_mock_fetch_404): + with caplog.at_level(logging.WARNING, logger="app.services.sber_index"): + result = await pull_sber_indices( + fake_db, # type: ignore[arg-type] + cities={"643": "Россия"}, + dashboards=[_DASH["residential_real_estate_prices"]], + ) + + assert result["errors"] == 1 + warnings = [r for r in caplog.records if r.levelno == logging.WARNING] + assert any( + "residential_real_estate_prices" in r.message and "dataset-path invalid" in r.message + for r in warnings + ), f"Expected a 404 dataset-path warning, got: {[r.message for r in warnings]}" + + @pytest.mark.asyncio async def test_pull_sber_indices_asking_benchmark_logged(caplog: pytest.LogCaptureFixture) -> None: """After upserting dinamika-tsen-obyavlenii, a benchmark INFO line is logged.""" -- 2.45.3