bot-backend
96027357b5
All checks were successful
Deploy Trade-In / changes (push) Successful in 5s
Deploy Trade-In / build-frontend (push) Has been skipped
Deploy Trade-In / test (push) Successful in 24s
Deploy Trade-In / deploy (push) Successful in 33s
Deploy Trade-In / build-backend (push) Successful in 39s
Co-authored-by: bot-backend <bot-backend@gendsgn.local> Co-committed-by: bot-backend <bot-backend@gendsgn.local>
62 lines
2.8 KiB
Python
62 lines
2.8 KiB
Python
"""#726 — avito SERP listing_date extraction (sortTimeStamp JSON + DOM fallback).
|
||
|
||
Offline: парсит фикстуру через AvitoScraper._parse_html, без сети/curl_cffi/БД.
|
||
Регрессия на дрейф разметки: дата у lazy-карточек (без DOM item-date) берётся из
|
||
per-item JSON sortTimeStamp; у верхних карточек — из DOM data-marker="item-date".
|
||
"""
|
||
|
||
import os
|
||
from datetime import date, datetime, timedelta, timezone
|
||
from pathlib import Path
|
||
|
||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
|
||
|
||
from app.services.scrapers.avito import (
|
||
AvitoScraper,
|
||
_build_sort_timestamp_map,
|
||
)
|
||
|
||
FIXTURE = Path(__file__).parent / "fixtures" / "avito_serp_sample.html"
|
||
|
||
# Из фикстуры (sortTimeStamp, epoch-ms).
|
||
_TS_LAZY = 1700000000000
|
||
_TS_TOP = 1700086400000
|
||
|
||
# #726: дата берётся в MSK (UTC+3), чтобы совпадать с тем, что Avito показывает на
|
||
# сайте (sortTimeStamp — UTC epoch, но отображается по Москве).
|
||
_MSK = timezone(timedelta(hours=3))
|
||
|
||
|
||
def _expected(ms: int) -> date:
|
||
return datetime.fromtimestamp(ms / 1000, tz=_MSK).date()
|
||
|
||
|
||
def test_sort_timestamp_map_built_from_json() -> None:
|
||
html = FIXTURE.read_text(encoding="utf-8")
|
||
ts_map = _build_sort_timestamp_map(html)
|
||
assert ts_map["8043936560"] == _expected(_TS_LAZY)
|
||
assert ts_map["8163084615"] == _expected(_TS_TOP)
|
||
|
||
|
||
def test_parse_html_sets_listing_date_for_all_cards() -> None:
|
||
"""Обе карточки получают listing_date != None (DoD #726)."""
|
||
html = FIXTURE.read_text(encoding="utf-8")
|
||
lots = AvitoScraper()._parse_html(html, "https://www.avito.ru/ekaterinburg/kvartiry")
|
||
assert len(lots) == 2
|
||
by_url = {lot.source_url: lot for lot in lots}
|
||
assert all(lot.listing_date is not None for lot in lots), "every card must carry a date"
|
||
|
||
# Lazy card (нет DOM item-date) → дата из JSON sortTimeStamp.
|
||
lazy = next(lot for lot in lots if "8043936560" in (lot.source_url or ""))
|
||
assert lazy.listing_date == _expected(_TS_LAZY)
|
||
assert by_url # touch to keep linter calm if structure changes
|
||
|
||
|
||
def test_lazy_card_date_comes_from_json_not_dom() -> None:
|
||
"""Карточка без data-marker='item-date' всё равно датируется (JSON primary)."""
|
||
html = FIXTURE.read_text(encoding="utf-8")
|
||
assert 'data-item-id="8043936560"' in html
|
||
# у этой карточки в фикстуре НЕТ DOM item-date — проверяем, что дата не потерялась
|
||
lots = AvitoScraper()._parse_html(html, "https://www.avito.ru/x")
|
||
lazy = next(lot for lot in lots if "8043936560" in (lot.source_url or ""))
|
||
assert lazy.listing_date == _expected(_TS_LAZY)
|