From d4cbd4bb9f8c230a2fe561cb8f87bed7e3ed12cb Mon Sep 17 00:00:00 2001 From: lekss361 Date: Sun, 24 May 2026 11:17:13 +0000 Subject: [PATCH] feat(tradein-scrapers): parse listing_date from Avito + Yandex (#508) --- .../backend/app/services/scrapers/avito.py | 50 +++++++++++++++++ .../app/services/scrapers/yandex_helpers.py | 54 ++++++++++++++++++- .../app/services/scrapers/yandex_realty.py | 4 +- .../backend/tests/test_avito_relative_date.py | 21 ++++++++ 4 files changed, 126 insertions(+), 3 deletions(-) create mode 100644 tradein-mvp/backend/tests/test_avito_relative_date.py diff --git a/tradein-mvp/backend/app/services/scrapers/avito.py b/tradein-mvp/backend/app/services/scrapers/avito.py index 6a52ee15..4d849e82 100644 --- a/tradein-mvp/backend/app/services/scrapers/avito.py +++ b/tradein-mvp/backend/app/services/scrapers/avito.py @@ -17,6 +17,7 @@ from __future__ import annotations import logging import re +from datetime import date, timedelta from typing import Any from urllib.parse import urlencode, urljoin @@ -29,6 +30,42 @@ from app.services.scrapers.base import BaseScraper, ScrapedLot logger = logging.getLogger(__name__) +# ── Relative date parsing ──────────────────────────────────────────────────── +_REL_DATE_RE = re.compile( + r"(?P\d+)\s+(?P" + r"секунд[ауы]?|" + r"минут[ауы]?|" + r"час(?:а|ов)?|" + r"день|дн(?:я|ей?)|" + r"недел[июяь]+|" + r"месяц(?:а|ев)?|" + r"год(?:а|ов)?" + r")\s+назад", + flags=re.I, +) +_UNIT_DAYS: dict[str, int] = { + "секунд": 0, "минут": 0, "час": 0, + "день": 1, "дн": 1, + "недел": 7, + "месяц": 30, + "год": 365, +} + + +def _parse_relative_date(s: str | None) -> date | None: + """Avito показывает '5 дней назад' / 'час назад'. Конвертирует в date.""" + if not s: + return None + m = _REL_DATE_RE.search(s) + if not m: + return None + n = int(m["n"]) + unit_raw = m["unit"].lower() + for prefix, days in _UNIT_DAYS.items(): + if unit_raw.startswith(prefix): + return date.today() - timedelta(days=n * days) + return None + class AvitoScraper(BaseScraper): """Avito vtorichka parser. Источник = 'avito'. @@ -216,6 +253,18 @@ class AvitoScraper(BaseScraper): if len(photo_urls) >= 5: break + # Дата публикации: data-marker="item-date" / data-marker="item-date-info" + listing_date: date | None = None + date_el = card.css_first('[data-marker="item-date"]') + if date_el is None: + date_el = card.css_first('[data-marker="item-date-info"]') + if date_el is not None: + listing_date = _parse_relative_date(date_el.text(strip=True)) + if listing_date is None: + # Fallback: ищем любой текст «N дней/недель/месяцев назад» в карточке + card_text = card.text(strip=True) + listing_date = _parse_relative_date(card_text) + # House link: # or /catalog/houses// house_source: str | None = None @@ -251,6 +300,7 @@ class AvitoScraper(BaseScraper): total_floors=total, price_rub=price, photo_urls=photo_urls, + listing_date=listing_date, raw_payload={"title": title, "address": address, "house_ext_id": house_ext_id}, house_source=house_source, house_ext_id=house_ext_id, diff --git a/tradein-mvp/backend/app/services/scrapers/yandex_helpers.py b/tradein-mvp/backend/app/services/scrapers/yandex_helpers.py index 8c1e0b77..8b945943 100644 --- a/tradein-mvp/backend/app/services/scrapers/yandex_helpers.py +++ b/tradein-mvp/backend/app/services/scrapers/yandex_helpers.py @@ -5,7 +5,7 @@ from __future__ import annotations import json import re -from datetime import date +from datetime import date, timedelta from typing import Any from selectolax.parser import HTMLParser @@ -31,6 +31,7 @@ __all__ = [ "parse_dmy", "parse_house_class", "parse_house_type", + "parse_listing_date", "parse_ru_date", "parse_rub", ] @@ -150,6 +151,57 @@ def parse_dmy(text: str | None) -> date | None: return None +# Relative date parsing (Yandex SERP shows "N дней назад" / "N часов назад") +_RE_REL_DATE = re.compile( + r"(?P\d+)\s+(?P" + r"секунд[ауы]?|" + r"минут[ауы]?|" + r"час(?:а|ов)?|" + r"день|дн(?:я|ей?)|" + r"недел[июяь]+|" + r"месяц(?:а|ев)?|" + r"год(?:а|ов)?" + r")\s+назад", + flags=re.I, +) +_REL_UNIT_DAYS: dict[str, int] = { + "секунд": 0, "минут": 0, "час": 0, + "день": 1, "дн": 1, + "недел": 7, + "месяц": 30, + "год": 365, +} + + +def _parse_relative_date_yandex(s: str | None) -> date | None: + """Parse 'N дней/недель/месяцев назад' → date. Returns None if not matched.""" + if not s: + return None + m = _RE_REL_DATE.search(s) + if not m: + return None + n = int(m["n"]) + unit_raw = m["unit"].lower() + for prefix, days in _REL_UNIT_DAYS.items(): + if unit_raw.startswith(prefix): + return date.today() - timedelta(days=n * days) + return None + + +def parse_listing_date(text: str | None) -> date | None: + """Parse listing date from Yandex card text. + + Tries in order: + 1. Absolute Russian date: '9 мая 2026' → date(2026, 5, 9) + 2. Relative date: 'N дней назад' → date.today() - N days + Returns None if neither matches. + """ + result = parse_ru_date(text) + if result is not None: + return result + return _parse_relative_date_yandex(text) + + # --------------------------------------------------------------------------- # Section 4 — House type / class NLP from description text # --------------------------------------------------------------------------- diff --git a/tradein-mvp/backend/app/services/scrapers/yandex_realty.py b/tradein-mvp/backend/app/services/scrapers/yandex_realty.py index cae46c71..f991dd81 100644 --- a/tradein-mvp/backend/app/services/scrapers/yandex_realty.py +++ b/tradein-mvp/backend/app/services/scrapers/yandex_realty.py @@ -28,7 +28,7 @@ from app.services.scrapers.yandex_helpers import ( RE_PRICE, RE_TITLE_AREA, RE_TITLE_ROOMS, - parse_ru_date, + parse_listing_date, parse_rub, ) @@ -156,7 +156,7 @@ class YandexRealtyScraper(BaseScraper): price_per_m2 = parse_rub(ppm2_m.group(1)) if ppm2_m else None bargain = "торг" in text.lower() - listing_date = parse_ru_date(text) + listing_date = parse_listing_date(text) # Address — street aggregator link street_link = card.css_first('a[href*="/kupit/kvartira/st-"]') diff --git a/tradein-mvp/backend/tests/test_avito_relative_date.py b/tradein-mvp/backend/tests/test_avito_relative_date.py new file mode 100644 index 00000000..00aca379 --- /dev/null +++ b/tradein-mvp/backend/tests/test_avito_relative_date.py @@ -0,0 +1,21 @@ +from datetime import date, timedelta + +from app.services.scrapers.avito import _parse_relative_date + + +def test_days_ago(): + assert _parse_relative_date("5 дней назад") == date.today() - timedelta(days=5) + + +def test_week_ago(): + assert _parse_relative_date("2 недели назад") == date.today() - timedelta(days=14) + + +def test_hour_ago_is_today(): + assert _parse_relative_date("3 часа назад") == date.today() + + +def test_none_passthrough(): + assert _parse_relative_date(None) is None + assert _parse_relative_date("") is None + assert _parse_relative_date("random text") is None