From 21c399733c93554baf6a8526fb2f8164b421c3c0 Mon Sep 17 00:00:00 2001 From: lekss361 Date: Sun, 31 May 2026 09:03:17 +0300 Subject: [PATCH] feat(tradein): avito citywide + byrooms sweep mode (T6) Add ROOM_SLUGS constant, _build_citywide_url/_build_rooms_url methods, and fetch_city_wide/fetch_by_rooms public methods to AvitoScraper. Citywide mode removes geoCoords/radius; byrooms mode splits by 7 room categories to bypass Avito 5000-result cap. Both reuse existing _fetch_serp_html anti-block pipeline (firewall detect, IP rotation, transient retries). fetch_around unchanged. --- .../backend/app/services/scrapers/avito.py | 210 ++++++++++++- .../backend/tests/test_avito_citywide.py | 288 ++++++++++++++++++ 2 files changed, 491 insertions(+), 7 deletions(-) create mode 100644 tradein-mvp/backend/tests/test_avito_citywide.py diff --git a/tradein-mvp/backend/app/services/scrapers/avito.py b/tradein-mvp/backend/app/services/scrapers/avito.py index 53743593..88343564 100644 --- a/tradein-mvp/backend/app/services/scrapers/avito.py +++ b/tradein-mvp/backend/app/services/scrapers/avito.py @@ -1,13 +1,21 @@ -"""Avito.ru scraper — парсинг вторички вокруг точки. +"""Avito.ru scraper — парсинг вторички вокруг точки + citywide/byrooms sweep. Стратегия: HTML scrape карточек объявлений через DOM. -URL pattern (EKB): - https://www.avito.ru/ekaterinburg/kvartiry/prodam-ASgBAgICAUSSA8YQ - ?geoCoords=56.838,60.605 - &radius=1 # в км - &s=104 # sort by date - &p=1 # страница +URL patterns (EKB): + geo-mode (default): + https://www.avito.ru/ekaterinburg/kvartiry/prodam-ASgBAgICAUSSA8YQ + ?geoCoords=56.838,60.605 + &radius=1 # в км + &s=104 # sort by date + &p=1 # страница + + citywide-mode (без geo, весь город, T6): + https://www.avito.ru/ekaterinburg/kvartiry/prodam-ASgBAgICAUSSA8YQ + ?s=104&p=1 + + byrooms-mode (citywide + фильтр комнатности, T6): + https://www.avito.ru/ekaterinburg/kvartiry/prodam/?s=104&p=1 ВАЖНО: Avito банит httpx (403/429) по TLS fingerprint от server IP. Используем curl_cffi с impersonate='chrome120' — настоящий Chrome TLS ClientHello. @@ -414,6 +422,181 @@ class AvitoScraper(BaseScraper): } return f"{self.base_url}/ekaterinburg/kvartiry/prodam-ASgBAgICAUSSA8YQ?{urlencode(params)}" + def _build_citywide_url(self, page: int = 1) -> str: + """T6: URL всего города ЕКБ без geo-фильтра, сортировка по дате. + + Avito отдаёт все объявления города (cap ~5000 = 100 страниц × 50 карточек). + """ + params = {"s": 104, "p": page} + return f"{self.base_url}/ekaterinburg/kvartiry/prodam-ASgBAgICAUSSA8YQ?{urlencode(params)}" + + def _build_rooms_url(self, room_slug: str, page: int = 1) -> str: + """T6: URL с фильтром по комнатности для всего ЕКБ (no geo). + + Дробление по комнатности позволяет обойти cap ~5000 на category. + room_slug — берётся из ROOM_SLUGS (кодирует ASgB-параметры выборки). + """ + params = {"s": 104, "p": page} + return f"{self.base_url}/ekaterinburg/kvartiry/prodam/{room_slug}?{urlencode(params)}" + + # ── T6: citywide / byrooms fetch methods ────────────────────────────────── + + async def fetch_city_wide( + self, + pages: int = 100, + *, + delay_override_sec: float | None = None, + ) -> list[ScrapedLot]: + """T6: Обход всего ЕКБ без geo-фильтра (citywide-mode), paginated. + + Возвращает дедуплицированный список лотов (по source_id/source_url). + Break-on-empty: останавливается когда страница отдаёт 0 карточек. + Сохраняет весь anti-block pipeline (_fetch_serp_html: firewall-detect, + IP rotation, транзиентные ретраи). Не трогает fetch_around. + + Args: + pages: максимальное число страниц (default 100 = Avito cap ~5000 лотов). + delay_override_sec: если задан — переопределяет request_delay_sec для + этого вызова. + + Returns: + Список ScrapedLot (все страницы, дедуп по source_id). + """ + if delay_override_sec is not None: + self.request_delay_sec = delay_override_sec + + all_lots: list[ScrapedLot] = [] + seen_ids: set[str] = set() + + for page in range(1, pages + 1): + url = self._build_citywide_url(page) + try: + html = await self._fetch_serp_html(url, page) + except (AvitoBlockedError, AvitoRateLimitedError): + raise + except Exception: + logger.exception("avito citywide page=%d fetch failed url=%s", page, url) + break + if html is None: + logger.info("avito citywide page=%d: non-200 — end of pagination", page) + break + + lots = self._parse_html(html, source_url_base=url) + if not lots: + logger.info("avito citywide page=%d: 0 lots — end of pagination", page) + break + + new_lots = [lot for lot in lots if lot.source_id not in seen_ids] + for lot in new_lots: + if lot.source_id: + seen_ids.add(lot.source_id) + all_lots.extend(new_lots) + logger.info( + "avito citywide page=%d: %d lots (%d new, %d dedup'd, total=%d)", + page, + len(lots), + len(new_lots), + len(lots) - len(new_lots), + len(all_lots), + ) + if page < pages: + await self.sleep_between_requests() + + logger.info("avito fetch_city_wide pages=%d total_lots=%d", pages, len(all_lots)) + return all_lots + + async def fetch_by_rooms( + self, + pages: int = 100, + *, + delay_override_sec: float | None = None, + room_slugs: list[tuple[str, str]] | None = None, + ) -> list[ScrapedLot]: + """T6: Обход ЕКБ по категориям комнатности (byrooms-mode). + + Дробление по 7 категориям комнатности позволяет обойти Avito cap ~5000 + результатов на поиск: каждый сегмент < 5000. Итого до 7 × 5000 = 35k лотов. + Break-on-empty на каждой категории. Глобальный dedup по source_id. + + Args: + pages: страниц на категорию (default 100). + delay_override_sec: переопределяет request_delay_sec. + room_slugs: список (name, slug) для обхода; по умолчанию ROOM_SLUGS + (все 7 категорий). Передай подмножество для частичного обхода. + + Returns: + Список ScrapedLot (все категории, глобальный дедуп по source_id). + """ + if delay_override_sec is not None: + self.request_delay_sec = delay_override_sec + + slugs = room_slugs if room_slugs is not None else ROOM_SLUGS + all_lots: list[ScrapedLot] = [] + seen_ids: set[str] = set() + + for ci, (name, slug) in enumerate(slugs, 1): + logger.info( + "avito byrooms [%d/%d] category=%s slug_prefix=%.30s", + ci, + len(slugs), + name, + slug, + ) + cat_count = 0 + for page in range(1, pages + 1): + url = self._build_rooms_url(slug, page) + try: + html = await self._fetch_serp_html(url, page) + except (AvitoBlockedError, AvitoRateLimitedError): + raise + except Exception: + logger.exception( + "avito byrooms category=%s page=%d fetch failed url=%s", + name, + page, + url, + ) + break + if html is None: + logger.info("avito byrooms category=%s page=%d: non-200 — end", name, page) + break + + lots = self._parse_html(html, source_url_base=url) + if not lots: + logger.info( + "avito byrooms category=%s page=%d: 0 lots — end of category", + name, + page, + ) + break + + new_lots = [lot for lot in lots if lot.source_id not in seen_ids] + for lot in new_lots: + if lot.source_id: + seen_ids.add(lot.source_id) + all_lots.extend(new_lots) + cat_count += len(new_lots) + logger.info( + "avito byrooms category=%s page=%d: %d lots (%d new, total=%d)", + name, + page, + len(lots), + len(new_lots), + len(all_lots), + ) + if page < pages: + await self.sleep_between_requests() + + logger.info( + "avito byrooms category=%s done: cat_new=%d grand_total=%d", + name, + cat_count, + len(all_lots), + ) + + logger.info("avito fetch_by_rooms pages_per_cat=%d total_lots=%d", pages, len(all_lots)) + return all_lots + def _parse_html(self, html: str, source_url_base: str) -> list[ScrapedLot]: """Парсим карточки объявлений из HTML через DOM scrape. @@ -553,6 +736,19 @@ _RE_STUDIO = re.compile(r"студи[яиюей]", re.IGNORECASE) _RE_AREA = re.compile(r"(\d+[.,]?\d*)\s*м[²2]", re.IGNORECASE) _RE_FLOOR = re.compile(r"(\d+)\s*/\s*(\d+)\s*эт\.?", re.IGNORECASE) +# ── T6: citywide / byrooms sweep ────────────────────────────────────────────── +# Avito URL slugs для room-filter (извлечены из реальных search-страниц 2026-05-23). +# Каждый slug кодирует ASgB-параметры выборки комнатности. +ROOM_SLUGS: list[tuple[str, str]] = [ + ("студии", "studii-ASgBAgICAkSSA8YQygj~WA"), + ("1-комн.", "1-komnatnye-ASgBAgICAkSSA8YQygiAWQ"), + ("2-комн.", "2-komnatnye-ASgBAgICAkSSA8YQygiCWQ"), + ("3-комн.", "3-komnatnye-ASgBAgICAkSSA8YQygiEWQ"), + ("4-комн.", "4-komnatnye-ASgBAgICAkSSA8YQygiGWQ"), + ("5+комн.", "5-komnatnye-ASgBAgICAkSSA8YQygiIWQ"), + ("своб.планир.", "svobodnaya_planirovka-ASgBAgICAkSSA8YQygj8zzI"), +] + _CSS_NOISE_RE = re.compile(r"\.?css-[a-z0-9_-]+\s*\{[^}]*\}", flags=re.I) _NOT_ADDRESS_TAIL_RE = re.compile( r"\s*(Площадь \d|от \d+\s?мин\.|css-[a-z0-9_-]+)", diff --git a/tradein-mvp/backend/tests/test_avito_citywide.py b/tradein-mvp/backend/tests/test_avito_citywide.py new file mode 100644 index 00000000..4c58df39 --- /dev/null +++ b/tradein-mvp/backend/tests/test_avito_citywide.py @@ -0,0 +1,288 @@ +"""T6 — avito citywide + byrooms sweep mode. + +Тесты: +1. _build_citywide_url — без geoCoords/radius, sort=104, page param. +2. _build_rooms_url — room-slug в пути, sort=104, page param. +3. ROOM_SLUGS — 7 категорий, структура кортежей. +4. fetch_city_wide break-on-empty — останавливается когда _parse_html вернул []. +5. fetch_city_wide dedup — дублирующийся source_id не дублируется в результате. +6. fetch_by_rooms break-on-empty per-category — каждая категория прерывается. +7. fetch_by_rooms global dedup — cross-category дедуп по source_id. +8. fetch_around не затронут — сигнатура и URL-режим не изменились. + +Без сети, без БД, без curl_cffi. +""" + +from __future__ import annotations + +import os +from unittest.mock import AsyncMock, patch + +import pytest + +os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db") + +from app.services.scrapers.avito import ROOM_SLUGS, AvitoScraper +from app.services.scrapers.base import ScrapedLot + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_lot(source_id: str) -> ScrapedLot: + return ScrapedLot( + source="avito", + source_url=f"https://www.avito.ru/ekaterinburg/kvartiry/{source_id}", + source_id=source_id, + price_rub=5_000_000, + ) + + +# --------------------------------------------------------------------------- +# 1. _build_citywide_url — без geo, s=104 +# --------------------------------------------------------------------------- + + +def test_build_citywide_url_no_geocoords() -> None: + s = AvitoScraper() + url = s._build_citywide_url(page=1) + assert "geoCoords" not in url + assert "radius" not in url + assert "s=104" in url + assert "p=1" in url + assert "ekaterinburg/kvartiry/prodam-ASgBAgICAUSSA8YQ" in url + + +def test_build_citywide_url_page_param() -> None: + s = AvitoScraper() + assert "p=3" in s._build_citywide_url(page=3) + assert "p=10" in s._build_citywide_url(page=10) + + +def test_build_citywide_url_default_page() -> None: + s = AvitoScraper() + assert "p=1" in s._build_citywide_url() + + +# --------------------------------------------------------------------------- +# 2. _build_rooms_url — slug в пути, без geo +# --------------------------------------------------------------------------- + + +def test_build_rooms_url_contains_slug() -> None: + s = AvitoScraper() + slug = "1-komnatnye-ASgBAgICAkSSA8YQygiAWQ" + url = s._build_rooms_url(slug, page=1) + assert slug in url + assert "geoCoords" not in url + assert "radius" not in url + assert "s=104" in url + assert "p=1" in url + assert "ekaterinburg/kvartiry/prodam/" in url + + +def test_build_rooms_url_page_param() -> None: + s = AvitoScraper() + slug = "studii-ASgBAgICAkSSA8YQygj~WA" + assert "p=5" in s._build_rooms_url(slug, page=5) + + +# --------------------------------------------------------------------------- +# 3. ROOM_SLUGS structure +# --------------------------------------------------------------------------- + + +def test_room_slugs_count() -> None: + assert len(ROOM_SLUGS) == 7 + + +def test_room_slugs_are_pairs() -> None: + for name, slug in ROOM_SLUGS: + assert isinstance(name, str) and name + assert isinstance(slug, str) and slug + # Каждый slug содержит ASgB-кодировку + assert "ASgB" in slug + + +def test_room_slugs_names_unique() -> None: + names = [name for name, _ in ROOM_SLUGS] + assert len(names) == len(set(names)) + + +# --------------------------------------------------------------------------- +# 4. fetch_city_wide — break-on-empty +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_fetch_city_wide_break_on_empty() -> None: + """Если _parse_html вернул [] — цикл прерывается, не идёт на следующие страницы.""" + s = AvitoScraper() + # page=1 возвращает 2 лота, page=2 возвращает [] → стоп + page_lots: dict[int, list[ScrapedLot]] = { + 1: [_make_lot("AAA"), _make_lot("BBB")], + 2: [], + } + call_count = 0 + + async def mock_fetch_html(url: str, page: int) -> str: + return f"page{page}" + + def mock_parse(html: str, source_url_base: str) -> list[ScrapedLot]: + nonlocal call_count + call_count += 1 + # Derive page number from html string + for p, lots in page_lots.items(): + if f"page{p}" in html: + return lots + return [] + + with patch.object(s, "_fetch_serp_html", side_effect=mock_fetch_html): + with patch.object(s, "_parse_html", side_effect=mock_parse): + with patch.object(s, "sleep_between_requests", new=AsyncMock()): + result = await s.fetch_city_wide(pages=10, delay_override_sec=0) + + assert len(result) == 2 + assert call_count == 2 # page1 + page2(empty) → stop; page3..10 не вызваны + + +# --------------------------------------------------------------------------- +# 5. fetch_city_wide — dedup по source_id +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_fetch_city_wide_dedup_by_source_id() -> None: + """Лот с повторяющимся source_id добавляется только один раз.""" + s = AvitoScraper() + dup_lot = _make_lot("SAME_ID") + # pages 1 и 2 возвращают один и тот же lot + pages_data: list[list[ScrapedLot]] = [[dup_lot], [dup_lot], []] + + async def mock_html(url: str, page: int) -> str: + return f"{page}" + + call_n = 0 + + def mock_parse(html: str, source_url_base: str) -> list[ScrapedLot]: + nonlocal call_n + idx = call_n + call_n += 1 + return pages_data[idx] if idx < len(pages_data) else [] + + with patch.object(s, "_fetch_serp_html", side_effect=mock_html): + with patch.object(s, "_parse_html", side_effect=mock_parse): + with patch.object(s, "sleep_between_requests", new=AsyncMock()): + result = await s.fetch_city_wide(pages=10, delay_override_sec=0) + + assert len(result) == 1 + assert result[0].source_id == "SAME_ID" + + +# --------------------------------------------------------------------------- +# 6. fetch_by_rooms — break-on-empty per category +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_fetch_by_rooms_break_on_empty_per_category() -> None: + """Каждая категория прерывается на пустой странице, следующая категория продолжается.""" + s = AvitoScraper() + slugs = [("1-комн.", "slug-1k"), ("2-комн.", "slug-2k")] + + # category 1: page1=2 lots, page2=[] → стоп на этой категории + # category 2: page1=3 lots, page2=[] → стоп + parse_responses: list[list[ScrapedLot]] = [ + [_make_lot("C1P1a"), _make_lot("C1P1b")], # cat1 p1 + [], # cat1 p2 → break + [_make_lot("C2P1a"), _make_lot("C2P1b"), _make_lot("C2P1c")], # cat2 p1 + [], # cat2 p2 → break + ] + parse_call = 0 + + async def mock_html(url: str, page: int) -> str: + return f"{url}{page}" + + def mock_parse(html: str, source_url_base: str) -> list[ScrapedLot]: + nonlocal parse_call + idx = parse_call + parse_call += 1 + return parse_responses[idx] if idx < len(parse_responses) else [] + + with patch.object(s, "_fetch_serp_html", side_effect=mock_html): + with patch.object(s, "_parse_html", side_effect=mock_parse): + with patch.object(s, "sleep_between_requests", new=AsyncMock()): + result = await s.fetch_by_rooms(pages=10, delay_override_sec=0, room_slugs=slugs) + + assert len(result) == 5 # 2 + 3 (нет дублей) + assert parse_call == 4 # 2 pages per category × 2 categories + + +# --------------------------------------------------------------------------- +# 7. fetch_by_rooms — global dedup cross-category +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_fetch_by_rooms_global_dedup() -> None: + """Лот присутствующий в двух категориях добавляется только один раз.""" + s = AvitoScraper() + slugs = [("A", "slug-a"), ("B", "slug-b")] + + shared = _make_lot("SHARED") + unique_a = _make_lot("UNIQUE_A") + unique_b = _make_lot("UNIQUE_B") + + parse_data: list[list[ScrapedLot]] = [ + [shared, unique_a], # cat A p1 + [], # cat A p2 → break + [shared, unique_b], # cat B p1 (shared — дубль) + [], # cat B p2 → break + ] + call_idx = 0 + + async def mock_html(url: str, page: int) -> str: + return "" + + def mock_parse(html: str, source_url_base: str) -> list[ScrapedLot]: + nonlocal call_idx + i = call_idx + call_idx += 1 + return parse_data[i] if i < len(parse_data) else [] + + with patch.object(s, "_fetch_serp_html", side_effect=mock_html): + with patch.object(s, "_parse_html", side_effect=mock_parse): + with patch.object(s, "sleep_between_requests", new=AsyncMock()): + result = await s.fetch_by_rooms(pages=10, delay_override_sec=0, room_slugs=slugs) + + ids = [lot.source_id for lot in result] + assert ids.count("SHARED") == 1 + assert "UNIQUE_A" in ids + assert "UNIQUE_B" in ids + assert len(result) == 3 + + +# --------------------------------------------------------------------------- +# 8. fetch_around не затронут — URL содержит geoCoords +# --------------------------------------------------------------------------- + + +def test_build_web_url_still_has_geocoords() -> None: + """Существующий _build_web_url сохраняет geoCoords+radius — geo-режим не сломан.""" + s = AvitoScraper() + url = s._build_web_url(56.838, 60.605, radius_km=2, page=1) + assert "geoCoords=56.838%2C60.605" in url or "geoCoords=56.838,60.605" in url + assert "radius=2" in url + assert "s=104" in url + assert "p=1" in url + + +def test_citywide_url_differs_from_geo_url() -> None: + """citywide URL не содержит geoCoords — это принципиально разные URL.""" + s = AvitoScraper() + geo_url = s._build_web_url(56.838, 60.605, radius_km=1, page=1) + cw_url = s._build_citywide_url(page=1) + assert geo_url != cw_url + assert "geoCoords" not in cw_url + assert "geoCoords" in geo_url -- 2.45.3