diff --git a/tradein-mvp/backend/app/services/scrape_pipeline.py b/tradein-mvp/backend/app/services/scrape_pipeline.py index 9af1ec8b..48a2a015 100644 --- a/tradein-mvp/backend/app/services/scrape_pipeline.py +++ b/tradein-mvp/backend/app/services/scrape_pipeline.py @@ -705,41 +705,58 @@ async def run_yandex_city_sweep( *, run_id: int, anchors: list[tuple[float, float, str]] | None = None, - radius_m: int = 1500, + radius_m: int = 25000, pages_per_anchor: int = 2, request_delay_sec: float | None = None, enrich_address: bool = True, + rooms_list: list[str] | None = None, + price_ranges: list[tuple[int | None, int | None]] | None = None, ) -> YandexCitySweepCounters: - """Yandex.Недвижимость city sweep: SERP → save → address-enrich. + """Yandex.Недвижимость city sweep: rooms × price combos от центра ЕКБ → save → address-enrich. - Фазы per anchor: + Вместо 5 географических anchor'ов итерирует ВСЕ combinations (room × price_range) + из одной центральной точки ЕКБ (56.8400, 60.6050). radius_m=25000 охватывает весь + город; price/room segmentation обходит Yandex SERP ~575-card cap без geo-шрапнели. + + Фазы (единственный центральный якорь): 1. SERP: YandexRealtyScraper.fetch_around_multi_room(lat, lon, radius_m, - max_pages=pages_per_anchor) — rooms × price-range combos, dedup внутри. + max_pages=pages_per_anchor, rooms_list=..., price_ranges=...) — combos mode. 2. SAVE: save_listings(db, lots, run_id=run_id). - 3. ADDRESS-ENRICH (если enrich_address=True): для yandex-листингов этого anchor'а - без номера дома в address (address IS NOT NULL AND NOT (address ~ ',\\s*\\d+')) + 3. ADDRESS-ENRICH (если enrich_address=True): для yandex-листингов без номера + дома в address (address IS NOT NULL AND NOT (address ~ ',\\s*\\d+')) запрашиваем detail-страницу и извлекаем полный адрес из . - Переиспользует приватные функции yandex_address_backfill: - _eligible_listings_for_ids → _fetch_and_update_one. + + Параметр `anchors` сохранён для backward-compat тестов (anchors=[...]). В prod + (anchors=None) всегда используется один EKB_CENTER anchor. Anti-bot / cancel семантика: - - Cooperative cancel: is_cancelled(db, run_id) проверяется перед каждым anchor. + - Cooperative cancel: is_cancelled(db, run_id) проверяется перед фазой SERP. - YandexRealtyScraper не propagate'ит block/rate exceptions (глотает → []). - Поэтому per-anchor ловим broad Exception; при N подряд — mark_banned + abort. - - request_delay_sec: доп. asyncio.sleep МЕЖДУ anchor'ами (поверх per-page sleep - внутри scraper'а). Также используется как inter-request delay для address-enrich. + Ловим broad Exception; при N подряд — mark_banned + abort. + - request_delay_sec: inter-request delay для address-enrich (поверх per-page + sleep внутри scraper'а). - Возвращает YandexCitySweepCounters (агрегат по всем anchor'ам). + Возвращает YandexCitySweepCounters. mark_done вызывается ВСЕГДА (кроме cancel / ban / fatal). """ from sqlalchemy import text as _text + from app.services.scrapers.yandex_realty import DEFAULT_PRICE_RANGES, ROOM_PATH from app.services.yandex_address_backfill import ( _RE_HAS_HOUSE_NUMBER, _extract_address_from_title, ) - _anchors = anchors if anchors is not None else EKB_ANCHORS + # EKB center — единственный anchor для citywide combos sweep. + # anchors-параметр принимается для backward-compat тестов. + if anchors is not None: + _anchors: list[tuple[float, float, str]] = anchors + else: + _anchors = [(56.8400, 60.6050, "Центр (combos)")] + + _rooms_list = rooms_list or list(ROOM_PATH.keys()) + _price_ranges = price_ranges or DEFAULT_PRICE_RANGES + counters = YandexCitySweepCounters(anchors_total=len(_anchors)) inter_anchor_delay = request_delay_sec if request_delay_sec is not None else 7.0 enrich_delay = request_delay_sec if request_delay_sec is not None else 3.0 @@ -764,13 +781,17 @@ async def run_yandex_city_sweep( return counters logger.info( - "yandex-sweep run_id=%d anchor #%d/%d (%s, %.4f, %.4f)", + "yandex-sweep run_id=%d center-combos #%d/%d (%s, %.4f, %.4f) " + "radius_m=%d rooms=%d price_ranges=%d", run_id, idx, len(_anchors), name, lat, lon, + radius_m, + len(_rooms_list), + len(_price_ranges), ) # ── Per-anchor phases под watchdog-таймаутом ──────────────────── @@ -799,6 +820,8 @@ async def run_yandex_city_sweep( _a_lon, radius_m, max_pages=pages_per_anchor, + rooms_list=_rooms_list, + price_ranges=_price_ranges, ) counters.lots_fetched += len(anchor_lots) if anchor_lots: @@ -807,7 +830,7 @@ async def run_yandex_city_sweep( counters.lots_updated += updated consecutive_failures = 0 logger.info( - "yandex-sweep run_id=%d anchor %s: SERP fetched=%d ins=%d upd=%d", + "yandex-sweep run_id=%d center-combos %s: fetched=%d ins=%d upd=%d", run_id, _a_name, len(anchor_lots), diff --git a/tradein-mvp/backend/data/sql/119_yandex_city_sweep_center_combos.sql b/tradein-mvp/backend/data/sql/119_yandex_city_sweep_center_combos.sql new file mode 100644 index 00000000..4aedfb5b --- /dev/null +++ b/tradein-mvp/backend/data/sql/119_yandex_city_sweep_center_combos.sql @@ -0,0 +1,32 @@ +-- 119_yandex_city_sweep_center_combos.sql +-- Обновляет default_params для yandex_city_sweep: переход с 5-anchor × shallow-pages +-- на single-center × rooms × price combos (exhaustive citywide coverage). +-- +-- Что меняется: +-- radius_m: 1500 → 25000 (охват всего ЕКБ от центральной точки; Yandex path-based, +-- geo-bbox используется только как fallback фильтр — большой +-- радиус не увеличивает load, только снимает geo-ограничение) +-- pages_per_anchor: 2 → 3 (max_pages per combo; при 5 комнатностей × 6 price_ranges +-- = 30 combos × 3 pages × ~25 карточек = до ~2250 офферов; +-- реальная уникальность после dedup ≪ SERP-cap ~575) +-- request_delay_sec: 9 → 9 (без изменений — пауза между address-enrich requests) +-- +-- anchors-параметр исчезает из default_params: run_yandex_city_sweep теперь игнорирует +-- его при anchors=None и всегда использует EKB_CENTER (56.8400, 60.6050) из кода. +-- +-- rooms_list/price_ranges НЕ добавляются в params — код читает defaults из +-- ROOM_PATH.keys() + DEFAULT_PRICE_RANGES (все 5 комнатностей × 6 price-buckets). +-- Если в будущем потребуется ограничить набор — добавить в default_params вручную. +-- +-- Idempotent: DO UPDATE перезаписывает params. SAFE to run повторно. +-- ЗАВИСИМОСТИ: 078_scrape_schedules_seed_yandex_sweep.sql (INSERT строки). + +BEGIN; + +UPDATE scrape_schedules +SET + default_params = '{"pages_per_anchor": 3, "request_delay_sec": 9, "radius_m": 25000}'::jsonb, + updated_at = NOW() +WHERE source = 'yandex_city_sweep'; + +COMMIT; diff --git a/tradein-mvp/backend/tests/test_yandex_city_sweep.py b/tradein-mvp/backend/tests/test_yandex_city_sweep.py index a8102b71..fb6aa586 100644 --- a/tradein-mvp/backend/tests/test_yandex_city_sweep.py +++ b/tradein-mvp/backend/tests/test_yandex_city_sweep.py @@ -665,3 +665,135 @@ def test_yandex_list_runs_endpoint(app_with_admin) -> None: assert body[0]["run_id"] == 20 assert body[0]["source"] == "yandex_city_sweep" assert "2025-05-01" in body[0]["started_at"] + + +# ── Center-combos wiring: prod (anchors=None) uses EKB center + combos ────── + + +async def test_prod_mode_uses_center_anchor_with_combos( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """anchors=None (prod mode) → fetch_around_multi_room called with EKB center coords + AND rooms_list + price_ranges (combos mode), NOT with None/None (legacy mode). + anchors_total == 1 (single center sweep). + """ + from app.services.scrapers.yandex_realty import DEFAULT_PRICE_RANGES, ROOM_PATH + + calls: list[dict] = [] + + async def fake_fetch_multi( + self: YandexRealtyScraper, + lat: float, + lon: float, + radius_m: int = 1000, + max_pages: int = 2, + rooms_list: list | None = None, + price_ranges: list | None = None, + **_: Any, + ) -> list[ScrapedLot]: + calls.append( + { + "lat": lat, + "lon": lon, + "radius_m": radius_m, + "rooms_list": rooms_list, + "price_ranges": price_ranges, + "max_pages": max_pages, + } + ) + return [_fake_lot("center-1"), _fake_lot("center-2")] + + monkeypatch.setattr(YandexRealtyScraper, "fetch_around_multi_room", fake_fetch_multi) + monkeypatch.setattr(scrape_pipeline, "save_listings", _fake_save) + fake = _FakeRuns() + _install_runs(monkeypatch, fake) + + counters = await scrape_pipeline.run_yandex_city_sweep( + db=MagicMock(), + run_id=10, + # anchors=None → prod mode (EKB center) + pages_per_anchor=3, + enrich_address=False, + request_delay_sec=0.0, + ) + + # Single center anchor + assert counters.anchors_total == 1 + assert counters.anchors_done == 1 + + # fetch_around_multi_room called exactly once + assert len(calls) == 1 + call = calls[0] + + # EKB center coords + assert abs(call["lat"] - 56.8400) < 0.0001 + assert abs(call["lon"] - 60.6050) < 0.0001 + + # radius covers the whole city (25km default) + assert call["radius_m"] >= 10000 + + # Combos mode: rooms_list and price_ranges are NOT None + assert call["rooms_list"] is not None + assert call["price_ranges"] is not None + + # rooms_list covers all ROOM_PATH keys + assert set(call["rooms_list"]) == set(ROOM_PATH.keys()) + + # price_ranges matches DEFAULT_PRICE_RANGES + assert call["price_ranges"] == DEFAULT_PRICE_RANGES + + # max_pages forwarded from pages_per_anchor + assert call["max_pages"] == 3 + + # lots aggregated correctly + assert counters.lots_fetched == 2 + assert counters.lots_inserted == 2 + assert fake.done is not None + assert fake.failed is None + + +async def test_explicit_anchors_still_works( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """anchors=[...] (test/manual override) → fetch called for each anchor, combos mode.""" + calls: list[dict] = [] + + async def fake_fetch_multi( + self: YandexRealtyScraper, + lat: float, + lon: float, + radius_m: int = 1000, + max_pages: int = 2, + rooms_list: list | None = None, + price_ranges: list | None = None, + **_: Any, + ) -> list[ScrapedLot]: + calls.append({"lat": lat, "rooms_list": rooms_list, "price_ranges": price_ranges}) + return [_fake_lot(f"{lat}")] + + monkeypatch.setattr(YandexRealtyScraper, "fetch_around_multi_room", fake_fetch_multi) + monkeypatch.setattr(scrape_pipeline, "save_listings", _fake_save) + fake = _FakeRuns() + _install_runs(monkeypatch, fake) + + two_anchors = [(56.8400, 60.6050, "A"), (56.7950, 60.5300, "B")] + counters = await scrape_pipeline.run_yandex_city_sweep( + db=MagicMock(), + run_id=11, + anchors=two_anchors, + pages_per_anchor=2, + enrich_address=False, + request_delay_sec=0.0, + ) + + # Both anchors iterated + assert len(calls) == 2 + assert counters.anchors_total == 2 + assert counters.anchors_done == 2 + + # Each call uses combos (not None/None) + for c in calls: + assert c["rooms_list"] is not None + assert c["price_ranges"] is not None + + assert fake.done is not None