From ab4bc229b2ad74cc7f24002e4dca5e602ba0cb78 Mon Sep 17 00:00:00 2001 From: bot-backend Date: Sat, 27 Jun 2026 14:13:46 +0300 Subject: [PATCH] fix(tradein/domclick): open-band recursion guard, count schema-guard, 5+ rooms, geo canary + tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - _walk_price_range open band: depth guard + progressing anchor (lo+_HIGH_ANCHOR) → forward progress, no infinite recursion / tail-loss on >cap open tail - COUNT: raise DomClickBlockedError on missing snippetsCount (schema regression) - _offer_to_lot: "5+" bucket fallback → rooms=5 (was None) - _parse_page: canary when raw>0 and all geo-dropped (ekb=0) - tests: closed-band bisect + real offset-cap (max offset 1980); studio force rooms=0 over present value; real save_listings-error sweep test - 138 migration header filename 132→138 --- .../backend/app/services/scrapers/domclick.py | 64 ++++++++- .../sql/138_update_domclick_sweep_params.sql | 2 +- .../backend/tests/scrapers/test_domclick.py | 125 +++++++++++++++++- .../backend/tests/test_domclick_sweep.py | 42 ++++-- 4 files changed, 212 insertions(+), 21 deletions(-) diff --git a/tradein-mvp/backend/app/services/scrapers/domclick.py b/tradein-mvp/backend/app/services/scrapers/domclick.py index ac333fdf..04196a3d 100644 --- a/tradein-mvp/backend/app/services/scrapers/domclick.py +++ b/tradein-mvp/backend/app/services/scrapers/domclick.py @@ -153,6 +153,9 @@ class DomClickScraper(BaseScraper): ) -> list[ScrapedLot]: """Citywide sweep: rooms × recursive price-bands → дедуплицированный список. + Кооперативный cancel проверяется только перед стартом sweep (как у citywide + проходов cian/yandex): mid-flight отмены нет, рантайм ограничен watchdog'ом. + Args: city_id: числовой ID города (4 = ЕКБ). Только ЕКБ поддерживается; для других значений логируем warning, продолжаем с EKB GUID. @@ -241,7 +244,24 @@ class DomClickScraper(BaseScraper): # ── 1. COUNT ───────────────────────────────────────────────────────── count_url = self._build_count_url(room_token, lo, hi) count_data = await self._fetch_json(count_url) # raises DomClickBlockedError on block - snippets: int = (count_data.get("result") or {}).get("snippetsCount", 0) + _count_result = count_data.get("result") or {} + if "snippetsCount" not in _count_result: + logger.error( + "domklik: count/v1 ответ без snippetsCount — возможна schema regression " + "url=%s payload=%r", + count_url, + str(count_data)[:300], + ) + try: + if settings.glitchtip_dsn: + sentry_sdk.capture_message( + "domklik count/v1: missing snippetsCount — possible schema regression", + level="error", + ) + except Exception: + pass + raise DomClickBlockedError("domclick count/v1 без snippetsCount (schema regression)") + snippets: int = _count_result.get("snippetsCount", 0) logger.info( "domklik: %s [%d, %s] snippetsCount=%d depth=%d", @@ -263,12 +283,31 @@ class DomClickScraper(BaseScraper): await self._paginate_leaf( room_token, lo, None, snippets, seen, pages, geo_dropped_ref ) + elif _depth >= _MAX_DEPTH: + # Не можем больше делить — пагинируем открытый хвост как есть (tail-loss). + tail_loss = max(0, snippets - self.OFFSET_CAP) + logger.warning( + "domklik: open band %s [%d, open] snippets=%d > cap=%d depth=%d " + "— paginating as-is (tail-loss ~%d)", + room_token, + lo, + snippets, + self.OFFSET_CAP, + _depth, + tail_loss, + ) + await self._paginate_leaf( + room_token, lo, None, snippets, seen, pages, geo_dropped_ref + ) else: - # Делим через HIGH_ANCHOR; хвост [HIGH_ANCHOR+1, None] обычно мал + # Progressing anchor: lo + _HIGH_ANCHOR строго > lo → остаточный + # открытый хвост [anchor+1, None] монотонно сдвигается вверх и + # за конечное число шагов (или depth-guard) станет ≤cap. + anchor = lo + _HIGH_ANCHOR await self._walk_price_range( room_token=room_token, lo=lo, - hi=_HIGH_ANCHOR, + hi=anchor, seen=seen, pages=pages, geo_dropped_ref=geo_dropped_ref, @@ -276,7 +315,7 @@ class DomClickScraper(BaseScraper): ) await self._walk_price_range( room_token=room_token, - lo=_HIGH_ANCHOR + 1, + lo=anchor + 1, hi=None, seen=seen, pages=pages, @@ -428,6 +467,21 @@ class DomClickScraper(BaseScraper): except Exception: pass # sentry_sdk не инициализирован в dev + if raw_count > 0 and ekb_count == 0: + logger.error( + "domklik: ВСЕ %d офферов geo-dropped (ekb=0) — возможен region/coord schema break", + raw_count, + ) + try: + if settings.glitchtip_dsn: + sentry_sdk.capture_message( + f"domklik: {raw_count} offers, all geo-dropped (ekb=0)" + " — possible region/coord schema regression", + level="error", + ) + except Exception: + pass + return result def _offer_to_lot(self, offer: dict[str, Any], room_token: str) -> ScrapedLot | None: @@ -503,6 +557,8 @@ class DomClickScraper(BaseScraper): rooms = int(rooms_raw) except (TypeError, ValueError): rooms = None + elif room_token == "5+": + rooms = 5 else: # fallback: числовой токен try: diff --git a/tradein-mvp/backend/data/sql/138_update_domclick_sweep_params.sql b/tradein-mvp/backend/data/sql/138_update_domclick_sweep_params.sql index f5c56c2e..bddc3705 100644 --- a/tradein-mvp/backend/data/sql/138_update_domclick_sweep_params.sql +++ b/tradein-mvp/backend/data/sql/138_update_domclick_sweep_params.sql @@ -1,4 +1,4 @@ --- 132_update_domclick_sweep_params.sql +-- 138_update_domclick_sweep_params.sql -- Обновление параметров DomClick citywide sweep после рефакторинга на JSON API. -- -- Контекст (2026-06-27): diff --git a/tradein-mvp/backend/tests/scrapers/test_domclick.py b/tradein-mvp/backend/tests/scrapers/test_domclick.py index 04813c2a..d9cceea9 100644 --- a/tradein-mvp/backend/tests/scrapers/test_domclick.py +++ b/tradein-mvp/backend/tests/scrapers/test_domclick.py @@ -67,7 +67,8 @@ _STUDIO_OFFER: dict = { "path": "/card/sale__flat__9001", "location": {"lat": 56.84, "lon": 60.60}, "address": {"displayName": "Екатеринбург, пр-т Ленина, 50"}, - "objectInfo": {"area": 24.5, "rooms": None, "floor": 3, "isApartment": False}, + # rooms=2 присутствует — bucket "st" обязан перебить его до 0 + "objectInfo": {"area": 24.5, "rooms": 2, "floor": 3, "isApartment": False}, "house": {"floors": 10, "buildYear": 2020}, "price": 3100000, "squarePrice": 126531, @@ -128,12 +129,13 @@ def test_offer_to_lot_basic_fields() -> None: def test_offer_to_lot_studio_rooms_zero() -> None: - """Studio bucket (room_token='st') → rooms=0 независимо от objectInfo.rooms.""" + """Studio bucket (room_token='st') → rooms=0 даже если objectInfo.rooms=2.""" scraper = _make_scraper() + assert _STUDIO_OFFER["objectInfo"]["rooms"] == 2 # фикстура несёт «лишнюю» комнатность lot = scraper._offer_to_lot(_STUDIO_OFFER, room_token="st") assert lot is not None - assert lot.rooms == 0 + assert lot.rooms == 0 # bucket "st" перебивает present rooms=2 assert lot.area_m2 == pytest.approx(24.5) assert lot.price_rub == 3_100_000 # publishedDate=None → None @@ -392,6 +394,123 @@ async def test_bucketing_split_on_overflow(monkeypatch: pytest.MonkeyPatch) -> N assert "101" in seen +def _make_ekb_offer(offer_id: str) -> dict: + """Минимальный валидный EKB-оффер для bucketing-тестов.""" + return { + "id": offer_id, + "path": f"/card/sale__flat__{offer_id}", + "location": {"lat": 56.83, "lon": 60.61}, + "address": {"displayName": "Екатеринбург, ул. Тест, 1"}, + "objectInfo": {"area": 45.0, "rooms": 2, "floor": 3, "isApartment": False}, + "house": {"floors": 9, "buildYear": 2000}, + "price": 5000000, + "squarePrice": 100000, + "flatComplex": None, + "isRosreestrApproved": False, + "publishedDate": None, + "updatedDate": None, + "lastPriceHistoryState": None, + "offerRegionName": "Екатеринбург", + } + + +async def test_bucketing_closed_band_bisect(monkeypatch: pytest.MonkeyPatch) -> None: + """Closed-band over cap → mid=(lo+hi)//2 bisect (real rooms=2 path). + + Open [0,None]=2001 → split в [0,30M]+[30M+1,None]. [0,30M]=4000 (>cap) → + бисектируется в [0,15M] и [15000001,30M], каждый ≤cap. Проверяем что хотя бы + один COUNT url несёт ОБЕ ценовые границы с mid-boundary 15M. + """ + call_log: list[str] = [] + + async def mock_fetch_json(self: DomClickScraper, url: str) -> dict: + call_log.append(url) + qs = parse_qs(urlparse(url).query) + + if "/count/v1" in url: + gte = int(qs["sale_price__gte"][0]) if "sale_price__gte" in qs else None + lte = int(qs["sale_price__lte"][0]) if "sale_price__lte" in qs else None + if gte is None and lte is None: + return {"result": {"snippetsCount": 2001}} # open [0,None] → split + if gte is None and lte == 30_000_000: + return {"result": {"snippetsCount": 4000}} # [0,30M] → must bisect + if gte is None and lte == 15_000_000: + return {"result": {"snippetsCount": 1500}} # [0,15M] leaf + if gte == 15_000_001 and lte == 30_000_000: + return {"result": {"snippetsCount": 1800}} # [15000001,30M] leaf + return {"result": {"snippetsCount": 0}} # хвост [30M+1,None] пуст + + if "/offers/v1" in url: + offset_val = int(qs.get("offset", ["0"])[0]) + if offset_val == 0: + return {"result": {"items": [_make_ekb_offer("101")]}} + return {"result": {"items": []}} + + return {"result": {}} + + import asyncio as _asyncio + + monkeypatch.setattr(DomClickScraper, "_fetch_json", mock_fetch_json) + monkeypatch.setattr(DomClickScraper, "sleep_between_requests", lambda self: _asyncio.sleep(0)) + + scraper = _make_scraper() + seen: dict = {} + geo_ref = [0] + await scraper._walk_price_range( + room_token="2", lo=0, hi=None, seen=seen, pages=100, geo_dropped_ref=geo_ref + ) + + count_urls = [u for u in call_log if "/count/v1" in u] + # Closed-band бисект отработал: есть COUNT url с ОБЕИМИ ценовыми границами. + both_bounds = [ + u + for u in count_urls + if "sale_price__gte" in parse_qs(urlparse(u).query) + and "sale_price__lte" in parse_qs(urlparse(u).query) + ] + assert both_bounds, f"no closed-band (both-bounds) count url: {count_urls}" + # Mid boundary 15M присутствует (правая половина gte=15000001 или левая lte=15000000). + mid_seen = any( + parse_qs(urlparse(u).query).get("sale_price__gte") == ["15000001"] + or parse_qs(urlparse(u).query).get("sale_price__lte") == ["15000000"] + for u in count_urls + ) + assert mid_seen, f"mid boundary 15M not found in count urls: {count_urls}" + + +async def test_paginate_leaf_offset_capped(monkeypatch: pytest.MonkeyPatch) -> None: + """Over-cap leaf (snippets=2500) → offset монотонно растёт, max == 1980 (< 2000). + + Драйвим _paginate_leaf напрямую: LIST отдаёт 20 элементов/страницу (пагинация + не обрывается), max_offset=min(2500, OFFSET_CAP=2000, pages*20=2000)=2000 → + последний offset = 1980 (2000-20), 2000 уже не запрашивается. + """ + call_log: list[str] = [] + + async def mock_fetch_json(self: DomClickScraper, url: str) -> dict: + call_log.append(url) + qs = parse_qs(urlparse(url).query) + offset_val = int(qs.get("offset", ["0"])[0]) + items = [_make_ekb_offer(f"{offset_val}-{i}") for i in range(20)] + return {"result": {"items": items}} + + import asyncio as _asyncio + + monkeypatch.setattr(DomClickScraper, "_fetch_json", mock_fetch_json) + monkeypatch.setattr(DomClickScraper, "sleep_between_requests", lambda self: _asyncio.sleep(0)) + + scraper = _make_scraper() + seen: dict = {} + geo_ref = [0] + await scraper._paginate_leaf("2", 0, None, 2500, seen, 100, geo_ref) + + list_urls = [u for u in call_log if "/offers/v1" in u] + offsets = [int(parse_qs(urlparse(u).query).get("offset", ["0"])[0]) for u in list_urls] + assert offsets, "no list urls requested" + assert max(offsets) == 1980, f"max offset {max(offsets)} != 1980" + assert all(o < 2000 for o in offsets), f"offset >= OFFSET_CAP: {offsets}" + + # ── _parse_publish_date ─────────────────────────────────────────────────────── diff --git a/tradein-mvp/backend/tests/test_domclick_sweep.py b/tradein-mvp/backend/tests/test_domclick_sweep.py index fac43463..30a65b55 100644 --- a/tradein-mvp/backend/tests/test_domclick_sweep.py +++ b/tradein-mvp/backend/tests/test_domclick_sweep.py @@ -299,27 +299,39 @@ async def test_domclick_blocked_triggers_mark_failed( assert fake.done is None -async def test_lots_but_errors_still_mark_done(monkeypatch: pytest.MonkeyPatch) -> None: - """Если лоты собраны (>0), но были partial errors → всё равно mark_done.""" - # Этот случай не тривиален — errors могут прийти при save_listings, но - # lots_fetched > 0. В нашей логике: lots_fetched>0 → mark_done. +async def test_save_listings_error_after_fetch_records_error( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """save_listings бросает ПОСЛЕ успешного fetch → ошибка реально учитывается. - async def fake_fetch_partial( + Раньше этот тест был no-op дубликатом happy-path (errors никогда не возникали). + Теперь инжектим настоящую ошибку в save_listings и проверяем, что она + проходит через перехват SERP-фазы (errors_count++). + + NB: save_listings вызывается ВНУТРИ _domclick_phase под asyncio.wait_for, и + исключение ловится внутренним `except Exception` (scrape_pipeline.py ~3205), + БЕЗ re-raise. lots_fetched инкрементируется ДО save (~3187), поэтому + honest-status (lots_fetched==0 and errors_count>0) ложен и run финализируется + как mark_done. Реальный путь mark_failed+re-raise — только для исключений, + утекающих во ВНЕШНИЙ except (например fetch_city), что уже покрыто + test_fetch_city_exception_triggers_mark_failed. + """ + fetch_calls = [0] + + async def fake_fetch( self: DomClickScraper, city_id: int, rooms: list[int] | None = None, pages: int = 100, ) -> list[ScrapedLot]: + fetch_calls[0] += 1 return [_fake_lot("200")] - fake_save_calls = [0] + def fake_save_raises(_db: Any, lots: list, *, run_id: int | None = None) -> tuple[int, int]: + raise RuntimeError("save_listings DB error") - def fake_save_ok(_db: Any, lots: list, *, run_id: int | None = None) -> tuple[int, int]: - fake_save_calls[0] += 1 - return 1, 0 - - monkeypatch.setattr(DomClickScraper, "fetch_city", fake_fetch_partial) - monkeypatch.setattr(scrape_pipeline, "save_listings", fake_save_ok) + monkeypatch.setattr(DomClickScraper, "fetch_city", fake_fetch) + monkeypatch.setattr(scrape_pipeline, "save_listings", fake_save_raises) fake = _FakeRuns() _install_runs(monkeypatch, fake) @@ -328,7 +340,11 @@ async def test_lots_but_errors_still_mark_done(monkeypatch: pytest.MonkeyPatch) db=MagicMock(), run_id=6, request_delay_sec=0.0 ) + # fetch отдал лот, save упал — ошибка учтена внутренним перехватом фазы. + assert fetch_calls[0] == 1 assert counters.lots_fetched == 1 - # Успешно → mark_done + assert counters.errors_count == 1 + assert counters.lots_inserted == 0 + # lots_fetched>0 → honest-status → mark_done (save-ошибка не утекает в outer except). assert fake.done is not None assert fake.failed is None