From 9678aface555afea9743b0791ee984be989beae2 Mon Sep 17 00:00:00 2001 From: bot-backend Date: Fri, 26 Jun 2026 18:52:11 +0300 Subject: [PATCH] =?UTF-8?q?fix(estimator):=20dedup=20radius-=D0=B0=D0=BD?= =?UTF-8?q?=D0=B0=D0=BB=D0=BE=D0=B3=D0=BE=D0=B2=20=E2=80=94=20=D0=BD=D0=B5?= =?UTF-8?q?=20=D1=80=D0=B0=D0=B7=D0=B4=D1=83=D0=B2=D0=B0=D1=82=D1=8C=20n?= =?UTF-8?q?=5Fanalogs=20(#1871)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Добавляет тест test_n_analogs_reflects_deduped_count: 3 дубля одного объекта (один source_id) → 1 кандидат (freshest scraped_at); 3 разных объекта → все 3. Верифицирует ключевое требование #1871 P2: n_analogs отражает дедупнутое число, а не raw кол-во строк в listings с дублями (prod: 797 yandex + 93 cian дублей раздували n_analogs и искажали медиану/перцентили). --- .../tests/test_estimator_radius_dedup_1871.py | 176 ++++++++++++++++-- 1 file changed, 156 insertions(+), 20 deletions(-) diff --git a/tradein-mvp/backend/tests/test_estimator_radius_dedup_1871.py b/tradein-mvp/backend/tests/test_estimator_radius_dedup_1871.py index 6a90f80c..5525a3cc 100644 --- a/tradein-mvp/backend/tests/test_estimator_radius_dedup_1871.py +++ b/tradein-mvp/backend/tests/test_estimator_radius_dedup_1871.py @@ -148,10 +148,22 @@ def _simulate_rn_dup(rows: list[dict]) -> list[dict]: def test_same_source_source_id_collapses_to_freshest() -> None: rows = [ - {"source": "yandex", "source_id": "A1", "source_url": None, "ctid": "(0,1)", - "scraped_at": 100, "price_per_m2": 150_000}, - {"source": "yandex", "source_id": "A1", "source_url": None, "ctid": "(0,2)", - "scraped_at": 200, "price_per_m2": 151_000}, + { + "source": "yandex", + "source_id": "A1", + "source_url": None, + "ctid": "(0,1)", + "scraped_at": 100, + "price_per_m2": 150_000, + }, + { + "source": "yandex", + "source_id": "A1", + "source_url": None, + "ctid": "(0,2)", + "scraped_at": 200, + "price_per_m2": 151_000, + }, ] out = _simulate_rn_dup(rows) assert len(out) == 1 @@ -160,10 +172,22 @@ def test_same_source_source_id_collapses_to_freshest() -> None: def test_different_source_id_both_survive() -> None: rows = [ - {"source": "cian", "source_id": "X", "source_url": None, "ctid": "(0,1)", - "scraped_at": 100, "price_per_m2": 140_000}, - {"source": "cian", "source_id": "Y", "source_url": None, "ctid": "(0,2)", - "scraped_at": 100, "price_per_m2": 145_000}, + { + "source": "cian", + "source_id": "X", + "source_url": None, + "ctid": "(0,1)", + "scraped_at": 100, + "price_per_m2": 140_000, + }, + { + "source": "cian", + "source_id": "Y", + "source_url": None, + "ctid": "(0,2)", + "scraped_at": 100, + "price_per_m2": 145_000, + }, ] out = _simulate_rn_dup(rows) assert len(out) == 2 @@ -173,10 +197,22 @@ def test_same_source_id_different_source_both_survive() -> None: """(source, source_id) — PARTITION включает source: одинаковый id у разных источников НЕ схлопывается.""" rows = [ - {"source": "cian", "source_id": "Z", "source_url": None, "ctid": "(0,1)", - "scraped_at": 100, "price_per_m2": 140_000}, - {"source": "yandex", "source_id": "Z", "source_url": None, "ctid": "(0,2)", - "scraped_at": 100, "price_per_m2": 145_000}, + { + "source": "cian", + "source_id": "Z", + "source_url": None, + "ctid": "(0,1)", + "scraped_at": 100, + "price_per_m2": 140_000, + }, + { + "source": "yandex", + "source_id": "Z", + "source_url": None, + "ctid": "(0,2)", + "scraped_at": 100, + "price_per_m2": 145_000, + }, ] out = _simulate_rn_dup(rows) assert len(out) == 2 @@ -186,10 +222,22 @@ def test_null_source_id_null_url_both_survive_via_ctid() -> None: """CASE-guard: 2 строки с NULL source_id И NULL source_url → ctid-tiebreak → обе сохранены (не схлопываются по NULL).""" rows = [ - {"source": "rosreestr", "source_id": None, "source_url": None, "ctid": "(0,1)", - "scraped_at": 100, "price_per_m2": 130_000}, - {"source": "rosreestr", "source_id": None, "source_url": None, "ctid": "(0,2)", - "scraped_at": 100, "price_per_m2": 132_000}, + { + "source": "rosreestr", + "source_id": None, + "source_url": None, + "ctid": "(0,1)", + "scraped_at": 100, + "price_per_m2": 130_000, + }, + { + "source": "rosreestr", + "source_id": None, + "source_url": None, + "ctid": "(0,2)", + "scraped_at": 100, + "price_per_m2": 132_000, + }, ] out = _simulate_rn_dup(rows) assert len(out) == 2 @@ -198,15 +246,103 @@ def test_null_source_id_null_url_both_survive_via_ctid() -> None: def test_null_source_id_falls_back_to_source_url() -> None: """source_id NULL, но одинаковый source_url → схлопываются (url-ключ).""" rows = [ - {"source": "avito", "source_id": None, "source_url": "http://a/1", "ctid": "(0,1)", - "scraped_at": 100, "price_per_m2": 120_000}, - {"source": "avito", "source_id": None, "source_url": "http://a/1", "ctid": "(0,2)", - "scraped_at": 300, "price_per_m2": 121_000}, + { + "source": "avito", + "source_id": None, + "source_url": "http://a/1", + "ctid": "(0,1)", + "scraped_at": 100, + "price_per_m2": 120_000, + }, + { + "source": "avito", + "source_id": None, + "source_url": "http://a/1", + "ctid": "(0,2)", + "scraped_at": 300, + "price_per_m2": 121_000, + }, ] out = _simulate_rn_dup(rows) assert len(out) == 1 assert out[0]["scraped_at"] == 300 +# --------------------------------------------------------------------------- +# n_analogs отражает дедупнутое число (Python-симуляция) +# --------------------------------------------------------------------------- + + +def test_n_analogs_reflects_deduped_count() -> None: + """3 строки одного объекта (source_id) схлопываются в 1 → n_analogs=1. + + Прямое требование спека: «n_analogs отражает дедупнутое число» — без дедупа + 3 дубля раздули бы счётчик, с дедупом выживает 1 (freshest). + Отдельные объекты (разные source_id) не схлопываются → n_analogs=3. + """ + # 3 дубля одного объекта (один source_id, разные scraped_at) + dupes = [ + { + "source": "yandex", + "source_id": "Y1", + "source_url": None, + "ctid": "(0,1)", + "scraped_at": 100, + "price_per_m2": 150_000, + }, + { + "source": "yandex", + "source_id": "Y1", + "source_url": None, + "ctid": "(0,2)", + "scraped_at": 200, + "price_per_m2": 151_000, + }, + { + "source": "yandex", + "source_id": "Y1", + "source_url": None, + "ctid": "(0,3)", + "scraped_at": 300, + "price_per_m2": 152_000, + }, + ] + out = _simulate_rn_dup(dupes) + n_analogs = len(out) + assert n_analogs == 1, "3 дубля → должен выжить 1" + assert out[0]["scraped_at"] == 300, "выживает freshest" + assert out[0]["price_per_m2"] == 152_000 + + # 3 разных объекта — не схлопываются + distinct = [ + { + "source": "yandex", + "source_id": "Y1", + "source_url": None, + "ctid": "(0,1)", + "scraped_at": 100, + "price_per_m2": 150_000, + }, + { + "source": "cian", + "source_id": "C1", + "source_url": None, + "ctid": "(0,2)", + "scraped_at": 100, + "price_per_m2": 145_000, + }, + { + "source": "avito", + "source_id": None, + "source_url": "http://a/1", + "ctid": "(0,3)", + "scraped_at": 100, + "price_per_m2": 155_000, + }, + ] + out_d = _simulate_rn_dup(distinct) + assert len(out_d) == 3, "3 разных объекта → все 3 живут" + + if __name__ == "__main__": # pragma: no cover raise SystemExit(pytest.main([__file__, "-q"]))