diff --git a/tradein-mvp/backend/app/services/estimator.py b/tradein-mvp/backend/app/services/estimator.py index 9975723e..941db436 100644 --- a/tradein-mvp/backend/app/services/estimator.py +++ b/tradein-mvp/backend/app/services/estimator.py @@ -34,7 +34,7 @@ logger = logging.getLogger(__name__) # ── Constants ──────────────────────────────────────────────────────────────── -DEFAULT_RADIUS_M = 800 +DEFAULT_RADIUS_M = 1000 # ПО ВСТРЕЧЕ ПТИЦЫ: «локация не дальше 800-1000 м» FALLBACK_RADIUS_M = 2000 AREA_TOLERANCE = 0.15 # ±15% площади LISTINGS_FRESH_DAYS = 14 # объявления не старше 14 дней diff --git a/tradein-mvp/backend/app/services/scrapers/cian.py b/tradein-mvp/backend/app/services/scrapers/cian.py index 57c55fe4..86118522 100644 --- a/tradein-mvp/backend/app/services/scrapers/cian.py +++ b/tradein-mvp/backend/app/services/scrapers/cian.py @@ -65,17 +65,19 @@ class CianScraper(BaseScraper): await super().__aexit__(*args) async def fetch_around( - self, lat: float, lon: float, radius_m: int = 1000, rooms: tuple[int, ...] | None = None + self, lat: float, lon: float, radius_m: int = 1000, + rooms: tuple[int, ...] | None = None, page: int = 1, ) -> list[ScrapedLot]: """Найти объявления Циан вокруг (lat, lon) в radius_m метрах. rooms — список из 1,2,3,4 (Cian's room codes). Если None — все. + page — страница выдачи (Cian ~28 на страницу). """ self._anchor_lat = lat self._anchor_lon = lon self._anchor_radius_m = radius_m - url = self._build_url(lat, lon, radius_m, rooms) + url = self._build_url(lat, lon, radius_m, rooms, page) try: assert self._cffi is not None response = await self._cffi.get(url) @@ -88,28 +90,34 @@ class CianScraper(BaseScraper): lots = self._parse_html(response.text) logger.info( - "cian: %d lots around (%.4f, %.4f) rooms=%s", len(lots), lat, lon, rooms + "cian: %d lots around (%.4f, %.4f) rooms=%s page=%d", + len(lots), lat, lon, rooms, page, ) await self.sleep_between_requests() return lots async def fetch_around_multi_room( - self, lat: float, lon: float, radius_m: int = 1000 + self, lat: float, lon: float, radius_m: int = 1000, pages: int = 3, ) -> list[ScrapedLot]: - """Скрейп Циан отдельно по 1к/2к/3к/4+ — каждый запрос ~25 лотов, всего ~100. - Циан умеет несколько rooms[]=N сразу, но для расширения выборки делаем отдельно. + """Скрейп Циан по 1к/2к/3к/4+ × N страниц — расширяет выборку. + 4 комнаты × 3 страницы × ~28 = до ~330 лотов до дедупликации. """ seen: dict[str, ScrapedLot] = {} for rooms in ((1,), (2,), (3,), (4,)): - try: - lots = await self.fetch_around(lat, lon, radius_m, rooms=rooms) - except Exception: - logger.exception("cian multi-room fetch failed for rooms=%s", rooms) - continue - for lot in lots: - key = lot.source_id or lot.source_url - if key and key not in seen: - seen[key] = lot + for page in range(1, pages + 1): + try: + lots = await self.fetch_around(lat, lon, radius_m, rooms=rooms, page=page) + except Exception: + logger.exception( + "cian multi-room fetch failed rooms=%s page=%d", rooms, page + ) + continue + if not lots: + break # пустая страница → дальше нет смысла + for lot in lots: + key = lot.source_id or lot.source_url + if key and key not in seen: + seen[key] = lot logger.info( "cian multi-room: %d unique lots around (%.4f, %.4f)", len(seen), lat, lon ) @@ -121,6 +129,7 @@ class CianScraper(BaseScraper): lon: float, radius_m: int, rooms: tuple[int, ...] | None = None, + page: int = 1, ) -> str: """URL для Cian каталога с регионом ЕКБ. @@ -138,6 +147,8 @@ class CianScraper(BaseScraper): for r in rooms: # Cian rooms[]=1 для 1к, rooms[]=2 для 2к, и т.д. params.append((f"room{r}", "1")) + if page > 1: + params.append(("p", page)) return f"{self.base_url}/cat.php?{urlencode(params)}" def _parse_html(self, html: str) -> list[ScrapedLot]: diff --git a/tradein-mvp/backend/app/services/scrapers/n1.py b/tradein-mvp/backend/app/services/scrapers/n1.py index 09891ba9..02e59c4d 100644 --- a/tradein-mvp/backend/app/services/scrapers/n1.py +++ b/tradein-mvp/backend/app/services/scrapers/n1.py @@ -63,13 +63,14 @@ class N1Scraper(BaseScraper): await super().__aexit__(*args) async def fetch_around( - self, lat: float, lon: float, radius_m: int = 1000, rooms: int | None = None, + self, lat: float, lon: float, radius_m: int = 1000, + rooms: int | None = None, page: int = 1, ) -> list[ScrapedLot]: self._anchor_lat = lat self._anchor_lon = lon self._anchor_radius_m = radius_m - url = self._build_url(rooms) + url = self._build_url(rooms, page) try: assert self._cffi is not None response = await self._cffi.get(url) @@ -82,35 +83,42 @@ class N1Scraper(BaseScraper): lots = self._parse_html(response.text) logger.info( - "n1: %d lots around (%.4f, %.4f) rooms=%s", len(lots), lat, lon, rooms + "n1: %d lots around (%.4f, %.4f) rooms=%s page=%d", + len(lots), lat, lon, rooms, page, ) await self.sleep_between_requests() return lots async def fetch_around_multi_room( - self, lat: float, lon: float, radius_m: int = 1000 + self, lat: float, lon: float, radius_m: int = 1000, pages: int = 3, ) -> list[ScrapedLot]: - """Скрейп N1 по сегментам комнат 1/2/3/4 — расширяет выборку.""" + """Скрейп N1 по сегментам комнат 1/2/3/4 × N страниц — расширяет выборку.""" seen: dict[str, ScrapedLot] = {} for rooms in (1, 2, 3, 4): - try: - lots = await self.fetch_around(lat, lon, radius_m, rooms=rooms) - except Exception: - logger.exception("n1 multi-room fetch failed for rooms=%s", rooms) - continue - for lot in lots: - key = lot.source_id or lot.source_url - if key and key not in seen: - seen[key] = lot + for page in range(1, pages + 1): + try: + lots = await self.fetch_around(lat, lon, radius_m, rooms=rooms, page=page) + except Exception: + logger.exception( + "n1 multi-room fetch failed rooms=%s page=%d", rooms, page + ) + continue + if not lots: + break # пустая страница → дальше нет смысла + for lot in lots: + key = lot.source_id or lot.source_url + if key and key not in seen: + seen[key] = lot logger.info( "n1 multi-room: %d unique lots around (%.4f, %.4f)", len(seen), lat, lon ) return list(seen.values()) - def _build_url(self, rooms: int | None = None) -> str: + def _build_url(self, rooms: int | None = None, page: int = 1) -> str: """URL каталога вторички ЕКБ. N1 фильтр комнат: /kupit/kvartiry/vtorichka/1-komnatnye/ итд. + Пагинация: ?page=N. """ rooms_segment = { 1: "1-komnatnye/", @@ -118,7 +126,10 @@ class N1Scraper(BaseScraper): 3: "3-komnatnye/", 4: "4-komnatnye/", }.get(rooms or 0, "") - return f"{self.base_url}/kupit/kvartiry/vtorichka/{rooms_segment}" + url = f"{self.base_url}/kupit/kvartiry/vtorichka/{rooms_segment}" + if page > 1: + url += f"?page={page}" + return url def _parse_html(self, html: str) -> list[ScrapedLot]: tree = HTMLParser(html)