From f814f222a1e7c2405e71661af9ba39ad2124cfb0 Mon Sep 17 00:00:00 2001 From: bot-backend Date: Sat, 4 Jul 2026 13:30:41 +0300 Subject: [PATCH] =?UTF-8?q?fix(tradein/scraper-kit):=20=D0=BA=D0=BB=D0=B0?= =?UTF-8?q?=D0=BC=D0=BF=D0=B8=D1=82=D1=8C=20diff=5Fpercent=20=D0=BF=D0=B5?= =?UTF-8?q?=D1=80=D0=B5=D0=B4=20INSERT=20offer=5Fprice=5Fhistory=20(#2277)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Прямые app-level INSERT-пути (domclick/cian kit-провайдеры + legacy cian_detail) читают diff_percent из скрапленного HTML/JSON и не проходят через БД-триггер из миграции 131 — гарбадж/экстремальное значение с редизайна source-страницы переполняет NUMERIC(8,2) и спамит "numeric field overflow" в проде. --- .../app/services/scrapers/cian_detail.py | 20 +++- .../tests/scrapers/test_domclick_detail.py | 92 +++++++++++++++++++ tradein-mvp/backend/tests/test_cian_detail.py | 65 +++++++++++++ .../src/scraper_kit/offer_price_history.py | 30 ++++++ .../src/scraper_kit/providers/cian/detail.py | 3 +- .../scraper_kit/providers/domclick/detail.py | 3 +- 6 files changed, 210 insertions(+), 3 deletions(-) create mode 100644 tradein-mvp/packages/scraper-kit/src/scraper_kit/offer_price_history.py diff --git a/tradein-mvp/backend/app/services/scrapers/cian_detail.py b/tradein-mvp/backend/app/services/scrapers/cian_detail.py index 6279c816..d3b03742 100644 --- a/tradein-mvp/backend/app/services/scrapers/cian_detail.py +++ b/tradein-mvp/backend/app/services/scrapers/cian_detail.py @@ -34,6 +34,24 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) +# offer_price_history.diff_percent — NUMERIC(8,2) (миграция +# 131_fix_diff_percent_overflow.sql). Прямой INSERT ниже читает diff_percent из +# скрапленного Cian JSON и не проходит через БД-триггер record_listing_price_change() +# (тот срабатывает только на UPDATE listings) — гарбадж/экстремальное значение +# с редизайна source-страницы иначе переполняет колонку. +_DIFF_PERCENT_BOUND = 999999.99 + + +def _clamp_diff_percent(value: float | int | None) -> float | None: + """Клампит diff_percent в границы NUMERIC(8,2) (зеркало migration 131 trigger-clamp).""" + if value is None or isinstance(value, bool): + return None + try: + numeric = float(value) + except (TypeError, ValueError): + return None + return min(max(numeric, -_DIFF_PERCENT_BOUND), _DIFF_PERCENT_BOUND) + @dataclass class DetailEnrichment: @@ -390,7 +408,7 @@ def save_detail_enrichment(db: Session, listing_id: int, enrichment: DetailEnric "lid": listing_id, "ct": change["change_time"], "price": change["price_rub"], - "diff": change.get("diff_percent"), + "diff": _clamp_diff_percent(change.get("diff_percent")), }, ) diff --git a/tradein-mvp/backend/tests/scrapers/test_domclick_detail.py b/tradein-mvp/backend/tests/scrapers/test_domclick_detail.py index 12e05ec5..cd629987 100644 --- a/tradein-mvp/backend/tests/scrapers/test_domclick_detail.py +++ b/tradein-mvp/backend/tests/scrapers/test_domclick_detail.py @@ -37,6 +37,7 @@ from unittest.mock import AsyncMock, MagicMock import pytest from scraper_kit.domclick_exceptions import DomClickBlockedError, DomClickParseError +from scraper_kit.offer_price_history import clamp_diff_percent from scraper_kit.providers.domclick.detail import ( DomClickDetailEnrichment, _extract_ssr_state, @@ -409,3 +410,94 @@ def test_save_detail_enrichment_none_passthrough() -> None: assert params["living_area_m2"] is None # пустой raw_extra → "{}" (merge no-op, не затирает) assert params["raw_extra"] == "{}" + + +# ── save_detail_enrichment — diff_percent клампится перед INSERT (#2277) ────── +# offer_price_history.diff_percent — NUMERIC(8,2) (миграция 131). Прямой INSERT +# здесь не проходит через БД-триггер (тот срабатывает только на UPDATE listings) — +# garbage/экстремальный diff_percent из скрапленного HTML иначе переполняет колонку. + + +def _insert_diff_param(db: MagicMock) -> float | None: + insert_calls = [c for c in db.execute.call_args_list if "offer_price_history" in str(c[0][0])] + assert len(insert_calls) == 1 + return insert_calls[0][0][1]["diff"] + + +def test_save_detail_enrichment_clamps_extreme_positive_diff() -> None: + db = MagicMock() + db.execute.return_value.rowcount = 1 + ct = datetime(2026, 4, 1, tzinfo=UTC) + e = DomClickDetailEnrichment( + item_id="x", + source_url=_CARD_URL, + price_changes=[{"change_time": ct, "price_rub": 5000000, "diff_percent": 5000000}], + ) + save_detail_enrichment(db, 1, e) + assert _insert_diff_param(db) == 999999.99 + + +def test_save_detail_enrichment_clamps_extreme_negative_diff() -> None: + db = MagicMock() + db.execute.return_value.rowcount = 1 + ct = datetime(2026, 4, 1, tzinfo=UTC) + e = DomClickDetailEnrichment( + item_id="x", + source_url=_CARD_URL, + price_changes=[{"change_time": ct, "price_rub": 5000000, "diff_percent": -5000000}], + ) + save_detail_enrichment(db, 1, e) + assert _insert_diff_param(db) == -999999.99 + + +def test_save_detail_enrichment_normal_diff_unchanged() -> None: + db = MagicMock() + db.execute.return_value.rowcount = 1 + ct = datetime(2026, 4, 1, tzinfo=UTC) + e = DomClickDetailEnrichment( + item_id="x", + source_url=_CARD_URL, + price_changes=[{"change_time": ct, "price_rub": 5000000, "diff_percent": -2.5}], + ) + save_detail_enrichment(db, 1, e) + assert _insert_diff_param(db) == -2.5 + + +def test_save_detail_enrichment_none_diff_unchanged() -> None: + db = MagicMock() + db.execute.return_value.rowcount = 1 + ct = datetime(2026, 4, 1, tzinfo=UTC) + e = DomClickDetailEnrichment( + item_id="x", + source_url=_CARD_URL, + price_changes=[{"change_time": ct, "price_rub": 5000000, "diff_percent": None}], + ) + save_detail_enrichment(db, 1, e) + assert _insert_diff_param(db) is None + + +# ── clamp_diff_percent (scraper_kit.offer_price_history) — standalone unit ──── +# Нет отдельной tests/ директории для scraper-kit-пакета (см. audit перед фиксом) — +# прямые unit-тесты хелпера живут здесь, рядом с save_detail_enrichment-тестами, +# которые его же и используют. + + +def test_clamp_diff_percent_extreme_positive() -> None: + assert clamp_diff_percent(5000000) == 999999.99 + + +def test_clamp_diff_percent_extreme_negative() -> None: + assert clamp_diff_percent(-5000000) == -999999.99 + + +def test_clamp_diff_percent_in_range_unchanged() -> None: + assert clamp_diff_percent(-2.5) == -2.5 + + +def test_clamp_diff_percent_none() -> None: + assert clamp_diff_percent(None) is None + + +def test_clamp_diff_percent_bool_treated_as_none() -> None: + assert clamp_diff_percent(True) is None + assert clamp_diff_percent(False) is None diff --git a/tradein-mvp/backend/tests/test_cian_detail.py b/tradein-mvp/backend/tests/test_cian_detail.py index 1eea78eb..e23323f5 100644 --- a/tradein-mvp/backend/tests/test_cian_detail.py +++ b/tradein-mvp/backend/tests/test_cian_detail.py @@ -608,6 +608,71 @@ def test_save_detail_enrichment_stamps_detail_enriched_at(): assert "detail_enriched_at = NOW()" in update_sql +# ── save_detail_enrichment — diff_percent клампится перед INSERT (#2277) ────── +# offer_price_history.diff_percent — NUMERIC(8,2) (миграция 131). Этот прямой +# INSERT не проходит через БД-триггер record_listing_price_change() (тот +# срабатывает только на UPDATE listings) — garbage/экстремальный diff_percent +# из скрапленного Cian JSON иначе переполняет колонку. + + +def _insert_diff_param(db: MagicMock) -> float | None: + insert_calls = [c for c in db.execute.call_args_list if "offer_price_history" in str(c[0][0])] + assert len(insert_calls) == 1 + return insert_calls[0][0][1]["diff"] + + +def test_save_detail_enrichment_clamps_extreme_positive_diff(): + db = MagicMock() + db.execute.return_value.fetchone.return_value = None + enrichment = DetailEnrichment( + price_changes=[ + {"change_time": "2026-04-01T00:00:00Z", "price_rub": 5000000, "diff_percent": 5000000} + ] + ) + save_detail_enrichment(db, listing_id=2001, enrichment=enrichment) + assert _insert_diff_param(db) == 999999.99 + + +def test_save_detail_enrichment_clamps_extreme_negative_diff(): + db = MagicMock() + db.execute.return_value.fetchone.return_value = None + enrichment = DetailEnrichment( + price_changes=[ + { + "change_time": "2026-04-01T00:00:00Z", + "price_rub": 5000000, + "diff_percent": -5000000, + } + ] + ) + save_detail_enrichment(db, listing_id=2002, enrichment=enrichment) + assert _insert_diff_param(db) == -999999.99 + + +def test_save_detail_enrichment_normal_diff_unchanged(): + db = MagicMock() + db.execute.return_value.fetchone.return_value = None + enrichment = DetailEnrichment( + price_changes=[ + {"change_time": "2026-04-01T00:00:00Z", "price_rub": 5000000, "diff_percent": -2.5} + ] + ) + save_detail_enrichment(db, listing_id=2003, enrichment=enrichment) + assert _insert_diff_param(db) == -2.5 + + +def test_save_detail_enrichment_none_diff_unchanged(): + db = MagicMock() + db.execute.return_value.fetchone.return_value = None + enrichment = DetailEnrichment( + price_changes=[ + {"change_time": "2026-04-01T00:00:00Z", "price_rub": 5000000, "diff_percent": None} + ] + ) + save_detail_enrichment(db, listing_id=2004, enrichment=enrichment) + assert _insert_diff_param(db) is None + + # ── fetch_detail browser_fetcher param ─────────────────────────────────────── diff --git a/tradein-mvp/packages/scraper-kit/src/scraper_kit/offer_price_history.py b/tradein-mvp/packages/scraper-kit/src/scraper_kit/offer_price_history.py new file mode 100644 index 00000000..52396975 --- /dev/null +++ b/tradein-mvp/packages/scraper-kit/src/scraper_kit/offer_price_history.py @@ -0,0 +1,30 @@ +"""Клампинг diff_percent перед прямым INSERT в offer_price_history. + +offer_price_history.diff_percent — NUMERIC(8,2) (±999999.99, см. миграцию +131_fix_diff_percent_overflow.sql). Та миграция клампит diff_percent, который +СЧИТАЕТ БД-триггер record_listing_price_change() на UPDATE listings, но триггер +не срабатывает на прямые INSERT из provider-модулей (domclick/cian detail), +которые пишут diff_percent, взятый напрямую из скрапленного HTML/JSON внешнего +источника. Garbage/экстремальное значение с редизайна source-страницы иначе +переполняет колонку и спамит "numeric field overflow" в проде. +""" + +from __future__ import annotations + +_DIFF_PERCENT_BOUND = 999999.99 + + +def clamp_diff_percent(value: float | int | None) -> float | None: + """Clamp raw scraped diff_percent into offer_price_history's NUMERIC(8,2) bound. + + Mirrors the DB-trigger clamp in migration 131_fix_diff_percent_overflow.sql + (LEAST(GREATEST(x, -999999.99), 999999.99)) so direct application-level INSERTs + (bypassing that trigger) cannot overflow the column either. + """ + if value is None or isinstance(value, bool): + return None + try: + numeric = float(value) + except (TypeError, ValueError): + return None + return min(max(numeric, -_DIFF_PERCENT_BOUND), _DIFF_PERCENT_BOUND) diff --git a/tradein-mvp/packages/scraper-kit/src/scraper_kit/providers/cian/detail.py b/tradein-mvp/packages/scraper-kit/src/scraper_kit/providers/cian/detail.py index e27e667f..3657f3d7 100644 --- a/tradein-mvp/packages/scraper-kit/src/scraper_kit/providers/cian/detail.py +++ b/tradein-mvp/packages/scraper-kit/src/scraper_kit/providers/cian/detail.py @@ -21,6 +21,7 @@ from sqlalchemy import text from sqlalchemy.orm import Session from scraper_kit.cian_state_parser import extract_all_states, extract_state +from scraper_kit.offer_price_history import clamp_diff_percent from scraper_kit.providers._base import build_curl_cffi_session from scraper_kit.providers._proxy import curl_proxy_url from scraper_kit.repair_state_normalizer import ( @@ -400,7 +401,7 @@ def save_detail_enrichment(db: Session, listing_id: int, enrichment: DetailEnric "lid": listing_id, "ct": change["change_time"], "price": change["price_rub"], - "diff": change.get("diff_percent"), + "diff": clamp_diff_percent(change.get("diff_percent")), }, ) diff --git a/tradein-mvp/packages/scraper-kit/src/scraper_kit/providers/domclick/detail.py b/tradein-mvp/packages/scraper-kit/src/scraper_kit/providers/domclick/detail.py index aabe9c72..a41803df 100644 --- a/tradein-mvp/packages/scraper-kit/src/scraper_kit/providers/domclick/detail.py +++ b/tradein-mvp/packages/scraper-kit/src/scraper_kit/providers/domclick/detail.py @@ -56,6 +56,7 @@ from sqlalchemy import text from sqlalchemy.orm import Session from scraper_kit.domclick_exceptions import DomClickBlockedError, DomClickParseError +from scraper_kit.offer_price_history import clamp_diff_percent from scraper_kit.repair_state_normalizer import ( infer_repair_state_from_text, normalize_repair_state, @@ -572,7 +573,7 @@ def save_detail_enrichment( "lid": listing_id, "ct": ct, "price": price, - "diff": change.get("diff_percent"), + "diff": clamp_diff_percent(change.get("diff_percent")), }, ) except Exception as exc: -- 2.45.3