tradein-mvp/scripts/sweep-yandex-valuation-ekb.py

@@ -137,17 +137,23 @@ async def _sweep_one_address(
 ) -> tuple[int, int]:
     """Fetch+save up to max_pages for one address.
 
+    Pagination stop conditions (whichever fires first):
+      1. Page returns 0 items (empty / past last page)
+      2. Cumulative fetched >= total_objects from page-1 header (when known)
+      3. max_pages reached
+
     Returns (items_fetched_total, items_saved_total).
     """
     total_fetched = 0
     total_saved = 0
+    total_objects_expected: int | None = None
+    seen_keys: set[tuple] = set()  # dedup within sweep (page overlap protection)
+
     for page in range(1, max_pages + 1):
         db = SessionLocal()
         try:
-            # _get_or_fetch_yandex_valuation_cached signature accepts address+offer_*;
-            # page is not in its signature — for paginated walk we must call the
-            # scraper directly. Use the cached helper for page=1 only, then call
-            # YandexValuationScraper.fetch_house_history for pages >= 2.
+            # _get_or_fetch_yandex_valuation_cached only fetches page 1 (no `page`
+            # arg); cache key includes address only. Pages >= 2 go direct to scraper.
             if page == 1:
                 result = await _get_or_fetch_yandex_valuation_cached(
                     db,
@@ -166,15 +172,32 @@ async def _sweep_one_address(
             if result is None or not result.history_items:
                 break
 
+            # Capture total_objects on first response (subsequent pages may not carry it)
+            if total_objects_expected is None and result.house.total_objects:
+                total_objects_expected = result.house.total_objects
+
+            # Dedup against prior pages (Yandex sometimes repeats items at page boundaries)
+            new_items = []
+            for it in result.history_items:
+                key = (it.publish_date, it.area_m2, it.floor, it.start_price)
+                if key not in seen_keys:
+                    seen_keys.add(key)
+                    new_items.append(it)
+
             saved = _save_yandex_history_items(db, result)
-            total_fetched += len(result.history_items)
+            total_fetched += len(new_items)
             total_saved += saved
+            exp_str = f"/{total_objects_expected}" if total_objects_expected is not None else ""
             print(
-                f"    page={page}/{max_pages}: items={len(result.history_items)} saved={saved}",
+                f"    page={page}/{max_pages}: items={len(result.history_items)} "
+                f"new={len(new_items)} saved={saved} cum={total_fetched}{exp_str}",
                 flush=True,
             )
-            # Stop early if Yandex doesn't paginate further (less than full page returned)
-            if len(result.history_items) < 13:
+            # Stop if we've collected at least what the header promised
+            if total_objects_expected is not None and total_fetched >= total_objects_expected:
+                break
+            # Or if this page yielded zero NEW items (pure duplicates → end of feed)
+            if not new_items:
                 break
         except Exception as e:
             print(