diff --git a/tradein-mvp/backend/app/services/scrape_pipeline.py b/tradein-mvp/backend/app/services/scrape_pipeline.py index ffc95615..6283bf4a 100644 --- a/tradein-mvp/backend/app/services/scrape_pipeline.py +++ b/tradein-mvp/backend/app/services/scrape_pipeline.py @@ -1644,6 +1644,7 @@ async def run_cian_full_load( await scraper.fetch_all_secondary( price_cap_per_bucket=price_cap_per_bucket, concurrency=concurrency, + secondary_only=True, on_bucket=_on_bucket, on_progress=_on_progress, skip_buckets=skip_set if skip_set else None, diff --git a/tradein-mvp/backend/app/services/scrapers/cian.py b/tradein-mvp/backend/app/services/scrapers/cian.py index c12e161c..8dc2d0e3 100644 --- a/tradein-mvp/backend/app/services/scrapers/cian.py +++ b/tradein-mvp/backend/app/services/scrapers/cian.py @@ -318,6 +318,7 @@ class CianScraper(BaseScraper): price_cap_per_bucket: int = 1400, max_pages_per_bucket: int = 54, concurrency: int = 5, + secondary_only: bool = True, on_bucket: Callable[..., Any] | None = None, on_progress: Callable[[int], None] | None = None, skip_buckets: set[str] | None = None, @@ -335,6 +336,10 @@ class CianScraper(BaseScraper): price_cap_per_bucket: максимум офферов в бакете перед делением (< 1500). max_pages_per_bucket: Cian hard cap ~54; не превышать. concurrency: максимум параллельных page-фетчей в leaf-бакете (default=5). + secondary_only: если True (default) — отбрасывает новостройки + (listing_segment=="novostroyki") после парсинга, до save/on_bucket. + object_type=1 SERP-параметр Cian ненадёжен (~5% выдачи), поэтому + фильтруем по authoritative listing_segment из offer.newbuilding.id. on_bucket: опциональный callback(bucket_key, list[ScrapedLot]) после каждого leaf-бакета. Может быть async или sync. Если кидает исключение — прерывает прогон. on_progress: опциональный callback(unique_count) для heartbeat (per room-bucket). @@ -350,10 +355,11 @@ class CianScraper(BaseScraper): for rooms in _buckets: room_label = f"room{'_'.join(str(r) for r in rooms)}" logger.info( - "cian exhaustive: starting %s (price_cap=%d concurrency=%d)", + "cian exhaustive: starting %s (price_cap=%d concurrency=%d secondary_only=%s)", room_label, price_cap_per_bucket, concurrency, + secondary_only, ) before = len(seen) await self._walk_price_range( @@ -364,6 +370,7 @@ class CianScraper(BaseScraper): price_cap_per_bucket=price_cap_per_bucket, max_pages_per_bucket=max_pages_per_bucket, concurrency=concurrency, + secondary_only=secondary_only, on_bucket=on_bucket, skip_buckets=skip_buckets, ) @@ -390,6 +397,7 @@ class CianScraper(BaseScraper): price_cap_per_bucket: int, max_pages_per_bucket: int, concurrency: int = 5, + secondary_only: bool = True, on_bucket: Callable[..., Any] | None = None, skip_buckets: set[str] | None = None, _depth: int = 0, @@ -406,6 +414,8 @@ class CianScraper(BaseScraper): bucket_key = "room_label:lo:hi" (checkpoint-ключ для resume). on_bucket может быть async или sync. Исключение в on_bucket прерывает прогон. skip_buckets: если bucket_key в skip_buckets — пагинацию и on_bucket пропускаем. + secondary_only: если True — новостройки (listing_segment=="novostroyki") отбрасываются + после сбора bucket_lots, до дедупа в seen и вызова on_bucket. """ effective_hi = hi @@ -488,6 +498,7 @@ class CianScraper(BaseScraper): price_cap_per_bucket=price_cap_per_bucket, max_pages_per_bucket=max_pages_per_bucket, concurrency=concurrency, + secondary_only=secondary_only, on_bucket=on_bucket, skip_buckets=skip_buckets, _depth=_depth + 1, @@ -501,6 +512,7 @@ class CianScraper(BaseScraper): price_cap_per_bucket=price_cap_per_bucket, max_pages_per_bucket=max_pages_per_bucket, concurrency=concurrency, + secondary_only=secondary_only, on_bucket=on_bucket, skip_buckets=skip_buckets, _depth=_depth + 1, @@ -566,6 +578,13 @@ class CianScraper(BaseScraper): collected_this_bucket = len(bucket_lots) + # ── Фильтр новостроек (secondary_only) ──────────────────────────────── + dropped_nb = 0 + if secondary_only: + filtered = [lot for lot in bucket_lots if lot.listing_segment != "novostroyki"] + dropped_nb = collected_this_bucket - len(filtered) + bucket_lots = filtered + # Дедуп в общий seen for lot in bucket_lots: key = lot.source_id or lot.source_url @@ -573,12 +592,13 @@ class CianScraper(BaseScraper): seen[key] = lot logger.info( - "cian: %s [%d, %d] paginated=%d pages collected=%d unique_total=%d", + "cian: %s [%d, %d] paginated=%d pages collected=%d dropped_nb=%d unique_total=%d", room_label, lo, hi, max_pages, collected_this_bucket, + dropped_nb, len(seen), ) diff --git a/tradein-mvp/backend/tests/scrapers/test_cian_exhaustive.py b/tradein-mvp/backend/tests/scrapers/test_cian_exhaustive.py index 79391ed5..3c2521db 100644 --- a/tradein-mvp/backend/tests/scrapers/test_cian_exhaustive.py +++ b/tradein-mvp/backend/tests/scrapers/test_cian_exhaustive.py @@ -433,3 +433,167 @@ async def test_on_bucket_receives_key_and_lots(scraper: CianScraper) -> None: # Ключ должен быть «room2:0:» assert key == f"room2:0:{_MAX_PRICE}", f"Неверный bucket_key: {key!r}" assert count == 5, f"Ожидалось 5 лотов, получено {count}" + + +# ── secondary_only фильтр ───────────────────────────────────────────────────── + + +@pytest.mark.asyncio +async def test_secondary_only_drops_novostroyki(scraper: CianScraper) -> None: + """secondary_only=True: новостройки (listing_segment=='novostroyki') не попадают + в on_bucket и в итоговый список; вторичка и None-сегмент — пропускаются. + """ + on_bucket_calls: list[list[ScrapedLot]] = [] + + def fake_on_bucket(bucket_key: str, lots: list[ScrapedLot]) -> None: + on_bucket_calls.append(list(lots)) + + # Микс лотов: 2 новостройки + 2 вторички + 1 None-сегмент + mixed_lots = [ + ScrapedLot( + source="cian", + source_url="https://ekb.cian.ru/sale/flat/nb_1/", + source_id="nb_1", + address="ЖК Новый", + price_rub=5_000_000, + listing_segment="novostroyki", + ), + ScrapedLot( + source="cian", + source_url="https://ekb.cian.ru/sale/flat/nb_2/", + source_id="nb_2", + address="ЖК Новый 2", + price_rub=6_000_000, + listing_segment="novostroyki", + ), + ScrapedLot( + source="cian", + source_url="https://ekb.cian.ru/sale/flat/vt_1/", + source_id="vt_1", + address="ул. Ленина", + price_rub=3_500_000, + listing_segment="vtorichka", + ), + ScrapedLot( + source="cian", + source_url="https://ekb.cian.ru/sale/flat/vt_2/", + source_id="vt_2", + address="ул. Мира", + price_rub=4_000_000, + listing_segment="vtorichka", + ), + ScrapedLot( + source="cian", + source_url="https://ekb.cian.ru/sale/flat/none_seg/", + source_id="none_seg", + address="ул. Пушкина", + price_rub=3_000_000, + listing_segment=None, + ), + ] + + page_call_count = 0 + + async def fake_fetch_page_html( + rooms: tuple, page: int, min_price: int | None, max_price: int | None + ) -> str: + nonlocal page_call_count + page_call_count += 1 + return f"page={page}" + + def fake_extract_total_offers(html: str) -> int | None: + return 5 # <= cap → leaf-бакет + + def fake_parse_serp_html(html: str) -> list[ScrapedLot]: + # Первая страница отдаёт весь микс; вторая — пусто + import re + + page_m = re.search(r"page=(\d+)", html) + page = int(page_m.group(1)) if page_m else 1 + return mixed_lots if page == 1 else [] + + with ( + patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html), + patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers), + patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html), + patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock), + patch.object(scraper, "_rotate_ip", return_value=False), + ): + lots = await scraper.fetch_all_secondary( + rooms_buckets=[(1,)], + price_cap_per_bucket=1400, + secondary_only=True, + on_bucket=fake_on_bucket, + ) + + # Итоговый список: только vtorichka + None-сегмент (3 лота), без новостроек + assert len(lots) == 3, f"Ожидалось 3 лота (без novostroyki), получено {len(lots)}" + lot_ids = {lot.source_id for lot in lots} + assert "nb_1" not in lot_ids, "nb_1 (novostroyki) не должен присутствовать" + assert "nb_2" not in lot_ids, "nb_2 (novostroyki) не должен присутствовать" + assert "vt_1" in lot_ids, "vt_1 (vtorichka) должен присутствовать" + assert "vt_2" in lot_ids, "vt_2 (vtorichka) должен присутствовать" + assert "none_seg" in lot_ids, "none_seg (listing_segment=None) должен присутствовать" + + # on_bucket тоже не должен получить новостройки + assert len(on_bucket_calls) == 1, f"Ожидался 1 вызов on_bucket, получено {len(on_bucket_calls)}" + bucket_ids = {lot.source_id for lot in on_bucket_calls[0]} + assert "nb_1" not in bucket_ids + assert "nb_2" not in bucket_ids + assert len(on_bucket_calls[0]) == 3 + + +@pytest.mark.asyncio +async def test_secondary_only_false_passes_all(scraper: CianScraper) -> None: + """secondary_only=False: новостройки НЕ отфильтровываются — все лоты проходят.""" + mixed_lots = [ + ScrapedLot( + source="cian", + source_url="https://ekb.cian.ru/sale/flat/nb_1/", + source_id="nb_1", + address="ЖК Новый", + price_rub=5_000_000, + listing_segment="novostroyki", + ), + ScrapedLot( + source="cian", + source_url="https://ekb.cian.ru/sale/flat/vt_1/", + source_id="vt_1", + address="ул. Ленина", + price_rub=3_500_000, + listing_segment="vtorichka", + ), + ] + + async def fake_fetch_page_html( + rooms: tuple, page: int, min_price: int | None, max_price: int | None + ) -> str: + return f"page={page}" + + def fake_extract_total_offers(html: str) -> int | None: + return 2 + + def fake_parse_serp_html(html: str) -> list[ScrapedLot]: + import re + + page_m = re.search(r"page=(\d+)", html) + page = int(page_m.group(1)) if page_m else 1 + return mixed_lots if page == 1 else [] + + with ( + patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html), + patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers), + patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html), + patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock), + patch.object(scraper, "_rotate_ip", return_value=False), + ): + lots = await scraper.fetch_all_secondary( + rooms_buckets=[(1,)], + price_cap_per_bucket=1400, + secondary_only=False, + ) + + assert len(lots) == 2, f"Ожидалось 2 лота (фильтр выключен), получено {len(lots)}" + lot_ids = {lot.source_id for lot in lots} + assert "nb_1" in lot_ids, "nb_1 должен присутствовать при secondary_only=False" + assert "vt_1" in lot_ids