From 0e0911dd28f3b0f133f0df031f6e618071c0c7c6 Mon Sep 17 00:00:00 2001 From: bot-backend Date: Mon, 13 Jul 2026 01:03:24 +0300 Subject: [PATCH] fix(tradein/scrapers): clamp implausible listing/publish dates at the save choke point Avito emits 1970-epoch and near-future listing_date; a single guard in save_listings() nulls anything outside [2010-01-01, today+2d], covering all providers. + migration 181 nulls existing bad rows. LOW audit R2 (#6). --- .../data/sql/181_clamp_bad_listing_dates.sql | 16 +++++ .../tests/scrapers/test_base_clamp_date.py | 60 +++++++++++++++++++ .../scraper-kit/src/scraper_kit/base.py | 27 ++++++++- 3 files changed, 100 insertions(+), 3 deletions(-) create mode 100644 tradein-mvp/backend/data/sql/181_clamp_bad_listing_dates.sql create mode 100644 tradein-mvp/backend/tests/scrapers/test_base_clamp_date.py diff --git a/tradein-mvp/backend/data/sql/181_clamp_bad_listing_dates.sql b/tradein-mvp/backend/data/sql/181_clamp_bad_listing_dates.sql new file mode 100644 index 00000000..5015fdb9 --- /dev/null +++ b/tradein-mvp/backend/data/sql/181_clamp_bad_listing_dates.sql @@ -0,0 +1,16 @@ +-- 181_clamp_bad_listing_dates.sql +-- LOW audit R2 (#6): null out implausible listing_date/publish_date left by scrapers +-- before the clamp guard landed (Avito 1970-epoch: ~698 rows; near-future: ~79 rows). +-- Idempotent: re-running is a no-op once values are already NULL / in-window. + +BEGIN; + +UPDATE listings SET listing_date = NULL + WHERE listing_date IS NOT NULL + AND (listing_date < DATE '2010-01-01' OR listing_date > CURRENT_DATE + 2); + +UPDATE listings SET publish_date = NULL + WHERE publish_date IS NOT NULL + AND (publish_date < DATE '2010-01-01' OR publish_date > CURRENT_DATE + 2); + +COMMIT; diff --git a/tradein-mvp/backend/tests/scrapers/test_base_clamp_date.py b/tradein-mvp/backend/tests/scrapers/test_base_clamp_date.py new file mode 100644 index 00000000..98e3840b --- /dev/null +++ b/tradein-mvp/backend/tests/scrapers/test_base_clamp_date.py @@ -0,0 +1,60 @@ +"""LOW audit R2 (#6) — `_clamp_plausible_date` в `scraper_kit.base`. + +Choke point для `listing_date`/`publish_date` в `save_listings()` (единый для +avito/cian/yandex/domclick). Проверено в БД: avito пишет 1970-01-01 +(epoch-sentinel, ~698 строк) и даты до +11 дней в будущем (~79 строк); у +yandex — единичные аномально старые годы. Правдоподобный диапазон — +[2010-01-01, today+2d]; всё остальное → None (колонка nullable). + +Офлайн: без сети/curl_cffi/БД, только импорт + чистая функция. +""" + +from __future__ import annotations + +import os +from datetime import date, timedelta + +os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db") + +from scraper_kit.base import _clamp_plausible_date + + +def test_avito_epoch_sentinel_rejected() -> None: + """1970-01-01 — классический Avito 0-epoch sentinel.""" + assert _clamp_plausible_date(date(1970, 1, 1)) is None + + +def test_just_before_min_plausible_rejected() -> None: + assert _clamp_plausible_date(date(2009, 12, 31)) is None + + +def test_far_future_rejected() -> None: + assert _clamp_plausible_date(date(2099, 1, 1)) is None + + +def test_none_passes_through_as_none() -> None: + assert _clamp_plausible_date(None) is None + + +def test_normal_recent_date_passes_through() -> None: + d = date(2024, 6, 1) + assert _clamp_plausible_date(d) == d + + +def test_today_passes() -> None: + today = date.today() + assert _clamp_plausible_date(today) == today + + +def test_today_plus_one_day_passes() -> None: + d = date.today() + timedelta(days=1) + assert _clamp_plausible_date(d) == d + + +def test_today_plus_three_days_rejected() -> None: + assert _clamp_plausible_date(date.today() + timedelta(days=3)) is None + + +def test_min_plausible_boundary_passes() -> None: + """2010-01-01 сам по себе — граница включительно (в окне).""" + assert _clamp_plausible_date(date(2010, 1, 1)) == date(2010, 1, 1) diff --git a/tradein-mvp/packages/scraper-kit/src/scraper_kit/base.py b/tradein-mvp/packages/scraper-kit/src/scraper_kit/base.py index ec155c5d..d3919be1 100644 --- a/tradein-mvp/packages/scraper-kit/src/scraper_kit/base.py +++ b/tradein-mvp/packages/scraper-kit/src/scraper_kit/base.py @@ -27,7 +27,7 @@ import hashlib import logging import random from abc import ABC, abstractmethod -from datetime import date, datetime +from datetime import date, datetime, timedelta from typing import TYPE_CHECKING, Any from zoneinfo import ZoneInfo @@ -69,6 +69,26 @@ def random_user_agent() -> str: return random.choice(_USER_AGENTS) +# ── Правдоподобность listing_date/publish_date ────────────────────────────── + +_MIN_PLAUSIBLE_LISTING_DATE = date(2010, 1, 1) + + +def _clamp_plausible_date(d: date | None) -> date | None: + """Reject implausible listing/publish dates (epoch-sentinel, far-future, ancient). + + Providers occasionally emit 1970-01-01 (Avito 0-epoch), near-future timestamps, + or garbled years. Anything outside [2010-01-01, today+2d] → None (column is + nullable). Single choke point so every provider (avito/cian/yandex/domclick) + is covered by one guard. LOW audit R2 (#6). + """ + if d is None: + return None + if d < _MIN_PLAUSIBLE_LISTING_DATE or d > date.today() + timedelta(days=2): + return None + return d + + # ── Унифицированная схема результата ──────────────────────────────────────── class ScrapedLot(BaseModel): """Результат парсинга одного объявления. @@ -383,8 +403,9 @@ def save_listings( "newbuilding_url": lot.newbuilding_url, "price_rub": lot.price_rub, "ppm2": ppm2, - "listing_date": lot.listing_date, - "publish_date": lot.publish_date, + # Clamp implausible dates (Avito 1970-epoch, near-future garbage) — LOW audit R2 (#6) + "listing_date": _clamp_plausible_date(lot.listing_date), + "publish_date": _clamp_plausible_date(lot.publish_date), "days_on_market": lot.days_on_market, "description": lot.description, "photos": _to_json(lot.photo_urls), -- 2.45.3