feat(tradein): cian full-load secondary-only — drop novostroyki at parse (#938)

Co-authored-by: bot-backend <bot-backend@gendsgn.local>
Co-committed-by: bot-backend <bot-backend@gendsgn.local>

tradein-mvp/backend/app/services/scrape_pipeline.py

@@ -1644,6 +1644,7 @@ async def run_cian_full_load(
             await scraper.fetch_all_secondary(
                 price_cap_per_bucket=price_cap_per_bucket,
                 concurrency=concurrency,
+                secondary_only=True,
                 on_bucket=_on_bucket,
                 on_progress=_on_progress,
                 skip_buckets=skip_set if skip_set else None,

tradein-mvp/backend/app/services/scrapers/cian.py

@@ -318,6 +318,7 @@ class CianScraper(BaseScraper):
         price_cap_per_bucket: int = 1400,
         max_pages_per_bucket: int = 54,
         concurrency: int = 5,
+        secondary_only: bool = True,
         on_bucket: Callable[..., Any] | None = None,
         on_progress: Callable[[int], None] | None = None,
         skip_buckets: set[str] | None = None,
@@ -335,6 +336,10 @@ class CianScraper(BaseScraper):
           price_cap_per_bucket: максимум офферов в бакете перед делением (< 1500).
           max_pages_per_bucket: Cian hard cap ~54; не превышать.
           concurrency: максимум параллельных page-фетчей в leaf-бакете (default=5).
+          secondary_only: если True (default) — отбрасывает новостройки
+            (listing_segment=="novostroyki") после парсинга, до save/on_bucket.
+            object_type=1 SERP-параметр Cian ненадёжен (~5% выдачи), поэтому
+            фильтруем по authoritative listing_segment из offer.newbuilding.id.
           on_bucket: опциональный callback(bucket_key, list[ScrapedLot]) после каждого
             leaf-бакета. Может быть async или sync. Если кидает исключение — прерывает прогон.
           on_progress: опциональный callback(unique_count) для heartbeat (per room-bucket).
@@ -350,10 +355,11 @@ class CianScraper(BaseScraper):
         for rooms in _buckets:
             room_label = f"room{'_'.join(str(r) for r in rooms)}"
             logger.info(
-                "cian exhaustive: starting %s (price_cap=%d concurrency=%d)",
+                "cian exhaustive: starting %s (price_cap=%d concurrency=%d secondary_only=%s)",
                 room_label,
                 price_cap_per_bucket,
                 concurrency,
+                secondary_only,
             )
             before = len(seen)
             await self._walk_price_range(
@@ -364,6 +370,7 @@ class CianScraper(BaseScraper):
                 price_cap_per_bucket=price_cap_per_bucket,
                 max_pages_per_bucket=max_pages_per_bucket,
                 concurrency=concurrency,
+                secondary_only=secondary_only,
                 on_bucket=on_bucket,
                 skip_buckets=skip_buckets,
             )
@@ -390,6 +397,7 @@ class CianScraper(BaseScraper):
         price_cap_per_bucket: int,
         max_pages_per_bucket: int,
         concurrency: int = 5,
+        secondary_only: bool = True,
         on_bucket: Callable[..., Any] | None = None,
         skip_buckets: set[str] | None = None,
         _depth: int = 0,
@@ -406,6 +414,8 @@ class CianScraper(BaseScraper):
         bucket_key = "room_label:lo:hi" (checkpoint-ключ для resume).
         on_bucket может быть async или sync. Исключение в on_bucket прерывает прогон.
         skip_buckets: если bucket_key в skip_buckets — пагинацию и on_bucket пропускаем.
+        secondary_only: если True — новостройки (listing_segment=="novostroyki") отбрасываются
+          после сбора bucket_lots, до дедупа в seen и вызова on_bucket.
         """
         effective_hi = hi
 
@@ -488,6 +498,7 @@ class CianScraper(BaseScraper):
                 price_cap_per_bucket=price_cap_per_bucket,
                 max_pages_per_bucket=max_pages_per_bucket,
                 concurrency=concurrency,
+                secondary_only=secondary_only,
                 on_bucket=on_bucket,
                 skip_buckets=skip_buckets,
                 _depth=_depth + 1,
@@ -501,6 +512,7 @@ class CianScraper(BaseScraper):
                 price_cap_per_bucket=price_cap_per_bucket,
                 max_pages_per_bucket=max_pages_per_bucket,
                 concurrency=concurrency,
+                secondary_only=secondary_only,
                 on_bucket=on_bucket,
                 skip_buckets=skip_buckets,
                 _depth=_depth + 1,
@@ -566,6 +578,13 @@ class CianScraper(BaseScraper):
 
         collected_this_bucket = len(bucket_lots)
 
+        # ── Фильтр новостроек (secondary_only) ────────────────────────────────
+        dropped_nb = 0
+        if secondary_only:
+            filtered = [lot for lot in bucket_lots if lot.listing_segment != "novostroyki"]
+            dropped_nb = collected_this_bucket - len(filtered)
+            bucket_lots = filtered
+
         # Дедуп в общий seen
         for lot in bucket_lots:
             key = lot.source_id or lot.source_url
@@ -573,12 +592,13 @@ class CianScraper(BaseScraper):
                 seen[key] = lot
 
         logger.info(
-            "cian: %s [%d, %d] paginated=%d pages collected=%d unique_total=%d",
+            "cian: %s [%d, %d] paginated=%d pages collected=%d dropped_nb=%d unique_total=%d",
             room_label,
             lo,
             hi,
             max_pages,
             collected_this_bucket,
+            dropped_nb,
             len(seen),
         )
 

tradein-mvp/backend/tests/scrapers/test_cian_exhaustive.py

@@ -433,3 +433,167 @@ async def test_on_bucket_receives_key_and_lots(scraper: CianScraper) -> None:
     # Ключ должен быть «room2:0:<MAX_PRICE>»
     assert key == f"room2:0:{_MAX_PRICE}", f"Неверный bucket_key: {key!r}"
     assert count == 5, f"Ожидалось 5 лотов, получено {count}"
+
+
+# ── secondary_only фильтр ─────────────────────────────────────────────────────
+
+
+@pytest.mark.asyncio
+async def test_secondary_only_drops_novostroyki(scraper: CianScraper) -> None:
+    """secondary_only=True: новостройки (listing_segment=='novostroyki') не попадают
+    в on_bucket и в итоговый список; вторичка и None-сегмент — пропускаются.
+    """
+    on_bucket_calls: list[list[ScrapedLot]] = []
+
+    def fake_on_bucket(bucket_key: str, lots: list[ScrapedLot]) -> None:
+        on_bucket_calls.append(list(lots))
+
+    # Микс лотов: 2 новостройки + 2 вторички + 1 None-сегмент
+    mixed_lots = [
+        ScrapedLot(
+            source="cian",
+            source_url="https://ekb.cian.ru/sale/flat/nb_1/",
+            source_id="nb_1",
+            address="ЖК Новый",
+            price_rub=5_000_000,
+            listing_segment="novostroyki",
+        ),
+        ScrapedLot(
+            source="cian",
+            source_url="https://ekb.cian.ru/sale/flat/nb_2/",
+            source_id="nb_2",
+            address="ЖК Новый 2",
+            price_rub=6_000_000,
+            listing_segment="novostroyki",
+        ),
+        ScrapedLot(
+            source="cian",
+            source_url="https://ekb.cian.ru/sale/flat/vt_1/",
+            source_id="vt_1",
+            address="ул. Ленина",
+            price_rub=3_500_000,
+            listing_segment="vtorichka",
+        ),
+        ScrapedLot(
+            source="cian",
+            source_url="https://ekb.cian.ru/sale/flat/vt_2/",
+            source_id="vt_2",
+            address="ул. Мира",
+            price_rub=4_000_000,
+            listing_segment="vtorichka",
+        ),
+        ScrapedLot(
+            source="cian",
+            source_url="https://ekb.cian.ru/sale/flat/none_seg/",
+            source_id="none_seg",
+            address="ул. Пушкина",
+            price_rub=3_000_000,
+            listing_segment=None,
+        ),
+    ]
+
+    page_call_count = 0
+
+    async def fake_fetch_page_html(
+        rooms: tuple, page: int, min_price: int | None, max_price: int | None
+    ) -> str:
+        nonlocal page_call_count
+        page_call_count += 1
+        return f"<html>page={page}</html>"
+
+    def fake_extract_total_offers(html: str) -> int | None:
+        return 5  # <= cap → leaf-бакет
+
+    def fake_parse_serp_html(html: str) -> list[ScrapedLot]:
+        # Первая страница отдаёт весь микс; вторая — пусто
+        import re
+
+        page_m = re.search(r"page=(\d+)", html)
+        page = int(page_m.group(1)) if page_m else 1
+        return mixed_lots if page == 1 else []
+
+    with (
+        patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html),
+        patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers),
+        patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html),
+        patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock),
+        patch.object(scraper, "_rotate_ip", return_value=False),
+    ):
+        lots = await scraper.fetch_all_secondary(
+            rooms_buckets=[(1,)],
+            price_cap_per_bucket=1400,
+            secondary_only=True,
+            on_bucket=fake_on_bucket,
+        )
+
+    # Итоговый список: только vtorichka + None-сегмент (3 лота), без новостроек
+    assert len(lots) == 3, f"Ожидалось 3 лота (без novostroyki), получено {len(lots)}"
+    lot_ids = {lot.source_id for lot in lots}
+    assert "nb_1" not in lot_ids, "nb_1 (novostroyki) не должен присутствовать"
+    assert "nb_2" not in lot_ids, "nb_2 (novostroyki) не должен присутствовать"
+    assert "vt_1" in lot_ids, "vt_1 (vtorichka) должен присутствовать"
+    assert "vt_2" in lot_ids, "vt_2 (vtorichka) должен присутствовать"
+    assert "none_seg" in lot_ids, "none_seg (listing_segment=None) должен присутствовать"
+
+    # on_bucket тоже не должен получить новостройки
+    assert len(on_bucket_calls) == 1, f"Ожидался 1 вызов on_bucket, получено {len(on_bucket_calls)}"
+    bucket_ids = {lot.source_id for lot in on_bucket_calls[0]}
+    assert "nb_1" not in bucket_ids
+    assert "nb_2" not in bucket_ids
+    assert len(on_bucket_calls[0]) == 3
+
+
+@pytest.mark.asyncio
+async def test_secondary_only_false_passes_all(scraper: CianScraper) -> None:
+    """secondary_only=False: новостройки НЕ отфильтровываются — все лоты проходят."""
+    mixed_lots = [
+        ScrapedLot(
+            source="cian",
+            source_url="https://ekb.cian.ru/sale/flat/nb_1/",
+            source_id="nb_1",
+            address="ЖК Новый",
+            price_rub=5_000_000,
+            listing_segment="novostroyki",
+        ),
+        ScrapedLot(
+            source="cian",
+            source_url="https://ekb.cian.ru/sale/flat/vt_1/",
+            source_id="vt_1",
+            address="ул. Ленина",
+            price_rub=3_500_000,
+            listing_segment="vtorichka",
+        ),
+    ]
+
+    async def fake_fetch_page_html(
+        rooms: tuple, page: int, min_price: int | None, max_price: int | None
+    ) -> str:
+        return f"<html>page={page}</html>"
+
+    def fake_extract_total_offers(html: str) -> int | None:
+        return 2
+
+    def fake_parse_serp_html(html: str) -> list[ScrapedLot]:
+        import re
+
+        page_m = re.search(r"page=(\d+)", html)
+        page = int(page_m.group(1)) if page_m else 1
+        return mixed_lots if page == 1 else []
+
+    with (
+        patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html),
+        patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers),
+        patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html),
+        patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock),
+        patch.object(scraper, "_rotate_ip", return_value=False),
+    ):
+        lots = await scraper.fetch_all_secondary(
+            rooms_buckets=[(1,)],
+            price_cap_per_bucket=1400,
+            secondary_only=False,
+        )
+
+    assert len(lots) == 2, f"Ожидалось 2 лота (фильтр выключен), получено {len(lots)}"
+    lot_ids = {lot.source_id for lot in lots}
+    assert "nb_1" in lot_ids, "nb_1 должен присутствовать при secondary_only=False"
+    assert "vt_1" in lot_ids