fix(tradein/scrapers): clamp implausible listing/publish dates (LOW audit R2 #6) (#2507)

tradein-mvp/backend/data/sql/181_clamp_bad_listing_dates.sql

@@ -0,0 +1,16 @@
+-- 181_clamp_bad_listing_dates.sql
+-- LOW audit R2 (#6): null out implausible listing_date/publish_date left by scrapers
+-- before the clamp guard landed (Avito 1970-epoch: ~698 rows; near-future: ~79 rows).
+-- Idempotent: re-running is a no-op once values are already NULL / in-window.
+
+BEGIN;
+
+UPDATE listings SET listing_date = NULL
+ WHERE listing_date IS NOT NULL
+   AND (listing_date < DATE '2010-01-01' OR listing_date > CURRENT_DATE + 2);
+
+UPDATE listings SET publish_date = NULL
+ WHERE publish_date IS NOT NULL
+   AND (publish_date < DATE '2010-01-01' OR publish_date > CURRENT_DATE + 2);
+
+COMMIT;

tradein-mvp/backend/tests/scrapers/test_base_clamp_date.py

@@ -0,0 +1,60 @@
+"""LOW audit R2 (#6) — `_clamp_plausible_date` в `scraper_kit.base`.
+
+Choke point для `listing_date`/`publish_date` в `save_listings()` (единый для
+avito/cian/yandex/domclick). Проверено в БД: avito пишет 1970-01-01
+(epoch-sentinel, ~698 строк) и даты до +11 дней в будущем (~79 строк); у
+yandex — единичные аномально старые годы. Правдоподобный диапазон —
+[2010-01-01, today+2d]; всё остальное → None (колонка nullable).
+
+Офлайн: без сети/curl_cffi/БД, только импорт + чистая функция.
+"""
+
+from __future__ import annotations
+
+import os
+from datetime import date, timedelta
+
+os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
+
+from scraper_kit.base import _clamp_plausible_date
+
+
+def test_avito_epoch_sentinel_rejected() -> None:
+    """1970-01-01 — классический Avito 0-epoch sentinel."""
+    assert _clamp_plausible_date(date(1970, 1, 1)) is None
+
+
+def test_just_before_min_plausible_rejected() -> None:
+    assert _clamp_plausible_date(date(2009, 12, 31)) is None
+
+
+def test_far_future_rejected() -> None:
+    assert _clamp_plausible_date(date(2099, 1, 1)) is None
+
+
+def test_none_passes_through_as_none() -> None:
+    assert _clamp_plausible_date(None) is None
+
+
+def test_normal_recent_date_passes_through() -> None:
+    d = date(2024, 6, 1)
+    assert _clamp_plausible_date(d) == d
+
+
+def test_today_passes() -> None:
+    today = date.today()
+    assert _clamp_plausible_date(today) == today
+
+
+def test_today_plus_one_day_passes() -> None:
+    d = date.today() + timedelta(days=1)
+    assert _clamp_plausible_date(d) == d
+
+
+def test_today_plus_three_days_rejected() -> None:
+    assert _clamp_plausible_date(date.today() + timedelta(days=3)) is None
+
+
+def test_min_plausible_boundary_passes() -> None:
+    """2010-01-01 сам по себе — граница включительно (в окне)."""
+    assert _clamp_plausible_date(date(2010, 1, 1)) == date(2010, 1, 1)

tradein-mvp/packages/scraper-kit/src/scraper_kit/base.py

@@ -27,7 +27,7 @@ import hashlib
 import logging
 import random
 from abc import ABC, abstractmethod
-from datetime import date, datetime
+from datetime import date, datetime, timedelta
 from typing import TYPE_CHECKING, Any
 from zoneinfo import ZoneInfo
 
@@ -69,6 +69,26 @@ def random_user_agent() -> str:
     return random.choice(_USER_AGENTS)
 
 
+# ── Правдоподобность listing_date/publish_date ──────────────────────────────
+
+_MIN_PLAUSIBLE_LISTING_DATE = date(2010, 1, 1)
+
+
+def _clamp_plausible_date(d: date | None) -> date | None:
+    """Reject implausible listing/publish dates (epoch-sentinel, far-future, ancient).
+
+    Providers occasionally emit 1970-01-01 (Avito 0-epoch), near-future timestamps,
+    or garbled years. Anything outside [2010-01-01, today+2d] → None (column is
+    nullable). Single choke point so every provider (avito/cian/yandex/domclick)
+    is covered by one guard. LOW audit R2 (#6).
+    """
+    if d is None:
+        return None
+    if d < _MIN_PLAUSIBLE_LISTING_DATE or d > date.today() + timedelta(days=2):
+        return None
+    return d
+
+
 # ── Унифицированная схема результата ────────────────────────────────────────
 class ScrapedLot(BaseModel):
     """Результат парсинга одного объявления.
@@ -383,8 +403,9 @@ def save_listings(
             "newbuilding_url": lot.newbuilding_url,
             "price_rub": lot.price_rub,
             "ppm2": ppm2,
-            "listing_date": lot.listing_date,
+            # Clamp implausible dates (Avito 1970-epoch, near-future garbage) — LOW audit R2 (#6)
-            "publish_date": lot.publish_date,
+            "listing_date": _clamp_plausible_date(lot.listing_date),
+            "publish_date": _clamp_plausible_date(lot.publish_date),
             "days_on_market": lot.days_on_market,
             "description": lot.description,
             "photos": _to_json(lot.photo_urls),