diff --git a/tradein-mvp/scripts/sweep-yandex-valuation-ekb.py b/tradein-mvp/scripts/sweep-yandex-valuation-ekb.py index e2fea0bc..f96c8303 100644 --- a/tradein-mvp/scripts/sweep-yandex-valuation-ekb.py +++ b/tradein-mvp/scripts/sweep-yandex-valuation-ekb.py @@ -137,17 +137,23 @@ async def _sweep_one_address( ) -> tuple[int, int]: """Fetch+save up to max_pages for one address. + Pagination stop conditions (whichever fires first): + 1. Page returns 0 items (empty / past last page) + 2. Cumulative fetched >= total_objects from page-1 header (when known) + 3. max_pages reached + Returns (items_fetched_total, items_saved_total). """ total_fetched = 0 total_saved = 0 + total_objects_expected: int | None = None + seen_keys: set[tuple] = set() # dedup within sweep (page overlap protection) + for page in range(1, max_pages + 1): db = SessionLocal() try: - # _get_or_fetch_yandex_valuation_cached signature accepts address+offer_*; - # page is not in its signature — for paginated walk we must call the - # scraper directly. Use the cached helper for page=1 only, then call - # YandexValuationScraper.fetch_house_history for pages >= 2. + # _get_or_fetch_yandex_valuation_cached only fetches page 1 (no `page` + # arg); cache key includes address only. Pages >= 2 go direct to scraper. if page == 1: result = await _get_or_fetch_yandex_valuation_cached( db, @@ -166,15 +172,32 @@ async def _sweep_one_address( if result is None or not result.history_items: break + # Capture total_objects on first response (subsequent pages may not carry it) + if total_objects_expected is None and result.house.total_objects: + total_objects_expected = result.house.total_objects + + # Dedup against prior pages (Yandex sometimes repeats items at page boundaries) + new_items = [] + for it in result.history_items: + key = (it.publish_date, it.area_m2, it.floor, it.start_price) + if key not in seen_keys: + seen_keys.add(key) + new_items.append(it) + saved = _save_yandex_history_items(db, result) - total_fetched += len(result.history_items) + total_fetched += len(new_items) total_saved += saved + exp_str = f"/{total_objects_expected}" if total_objects_expected is not None else "" print( - f" page={page}/{max_pages}: items={len(result.history_items)} saved={saved}", + f" page={page}/{max_pages}: items={len(result.history_items)} " + f"new={len(new_items)} saved={saved} cum={total_fetched}{exp_str}", flush=True, ) - # Stop early if Yandex doesn't paginate further (less than full page returned) - if len(result.history_items) < 13: + # Stop if we've collected at least what the header promised + if total_objects_expected is not None and total_fetched >= total_objects_expected: + break + # Or if this page yielded zero NEW items (pure duplicates → end of feed) + if not new_items: break except Exception as e: print(