feat(tradein): yandex city sweep — center combos replaces 5-anchor shallow
All checks were successful
CI / changes (push) Successful in 8s
CI / openapi-codegen-check (push) Has been skipped
CI / changes (pull_request) Successful in 6s
CI / backend-tests (pull_request) Has been skipped
CI / backend-tests (push) Has been skipped
CI / frontend-tests (push) Has been skipped
CI / frontend-tests (pull_request) Has been skipped
CI / openapi-codegen-check (pull_request) Has been skipped

Switch run_yandex_city_sweep from iterating 5 geographic anchors x 2 shallow
pages to a single EKB center anchor (56.8400, 60.6050) with exhaustive
rooms x price-range combos (5 room types x 6 price buckets = 30 combos),
radius_m=25000 to cover the whole city.

- scrape_pipeline.py: new params rooms_list/price_ranges; anchors=None (prod)
  always uses EKB center; fetch_around_multi_room called in combos mode
- 119_yandex_city_sweep_center_combos.sql: updates default_params
  (radius_m: 1500 -> 25000, pages_per_anchor: 2 -> 3)
- tests: 2 new tests assert center coords + combos wiring (prod mode and
  explicit-anchors backward-compat); 1836 suite green
This commit is contained in:
bot-backend 2026-06-16 22:42:12 +03:00
parent b5829fa226
commit e551eda289
3 changed files with 203 additions and 16 deletions

View file

@ -705,41 +705,58 @@ async def run_yandex_city_sweep(
*,
run_id: int,
anchors: list[tuple[float, float, str]] | None = None,
radius_m: int = 1500,
radius_m: int = 25000,
pages_per_anchor: int = 2,
request_delay_sec: float | None = None,
enrich_address: bool = True,
rooms_list: list[str] | None = None,
price_ranges: list[tuple[int | None, int | None]] | None = None,
) -> YandexCitySweepCounters:
"""Yandex.Недвижимость city sweep: SERP → save → address-enrich.
"""Yandex.Недвижимость city sweep: rooms × price combos от центра ЕКБ → save → address-enrich.
Фазы per anchor:
Вместо 5 географических anchor'ов итерирует ВСЕ combinations (room × price_range)
из одной центральной точки ЕКБ (56.8400, 60.6050). radius_m=25000 охватывает весь
город; price/room segmentation обходит Yandex SERP ~575-card cap без geo-шрапнели.
Фазы (единственный центральный якорь):
1. SERP: YandexRealtyScraper.fetch_around_multi_room(lat, lon, radius_m,
max_pages=pages_per_anchor) rooms × price-range combos, dedup внутри.
max_pages=pages_per_anchor, rooms_list=..., price_ranges=...) combos mode.
2. SAVE: save_listings(db, lots, run_id=run_id).
3. ADDRESS-ENRICH (если enrich_address=True): для yandex-листингов этого anchor'а
без номера дома в address (address IS NOT NULL AND NOT (address ~ ',\\s*\\d+'))
3. ADDRESS-ENRICH (если enrich_address=True): для yandex-листингов без номера
дома в address (address IS NOT NULL AND NOT (address ~ ',\\s*\\d+'))
запрашиваем detail-страницу и извлекаем полный адрес из <title>.
Переиспользует приватные функции yandex_address_backfill:
_eligible_listings_for_ids _fetch_and_update_one.
Параметр `anchors` сохранён для backward-compat тестов (anchors=[...]). В prod
(anchors=None) всегда используется один EKB_CENTER anchor.
Anti-bot / cancel семантика:
- Cooperative cancel: is_cancelled(db, run_id) проверяется перед каждым anchor.
- Cooperative cancel: is_cancelled(db, run_id) проверяется перед фазой SERP.
- YandexRealtyScraper не propagate'ит block/rate exceptions (глотает → []).
Поэтому per-anchor ловим broad Exception; при N подряд mark_banned + abort.
- request_delay_sec: доп. asyncio.sleep МЕЖДУ anchor'ами (поверх per-page sleep
внутри scraper'а). Также используется как inter-request delay для address-enrich.
Ловим broad Exception; при N подряд mark_banned + abort.
- request_delay_sec: inter-request delay для address-enrich (поверх per-page
sleep внутри scraper'а).
Возвращает YandexCitySweepCounters (агрегат по всем anchor'ам).
Возвращает YandexCitySweepCounters.
mark_done вызывается ВСЕГДА (кроме cancel / ban / fatal).
"""
from sqlalchemy import text as _text
from app.services.scrapers.yandex_realty import DEFAULT_PRICE_RANGES, ROOM_PATH
from app.services.yandex_address_backfill import (
_RE_HAS_HOUSE_NUMBER,
_extract_address_from_title,
)
_anchors = anchors if anchors is not None else EKB_ANCHORS
# EKB center — единственный anchor для citywide combos sweep.
# anchors-параметр принимается для backward-compat тестов.
if anchors is not None:
_anchors: list[tuple[float, float, str]] = anchors
else:
_anchors = [(56.8400, 60.6050, "Центр (combos)")]
_rooms_list = rooms_list or list(ROOM_PATH.keys())
_price_ranges = price_ranges or DEFAULT_PRICE_RANGES
counters = YandexCitySweepCounters(anchors_total=len(_anchors))
inter_anchor_delay = request_delay_sec if request_delay_sec is not None else 7.0
enrich_delay = request_delay_sec if request_delay_sec is not None else 3.0
@ -764,13 +781,17 @@ async def run_yandex_city_sweep(
return counters
logger.info(
"yandex-sweep run_id=%d anchor #%d/%d (%s, %.4f, %.4f)",
"yandex-sweep run_id=%d center-combos #%d/%d (%s, %.4f, %.4f) "
"radius_m=%d rooms=%d price_ranges=%d",
run_id,
idx,
len(_anchors),
name,
lat,
lon,
radius_m,
len(_rooms_list),
len(_price_ranges),
)
# ── Per-anchor phases под watchdog-таймаутом ────────────────────
@ -799,6 +820,8 @@ async def run_yandex_city_sweep(
_a_lon,
radius_m,
max_pages=pages_per_anchor,
rooms_list=_rooms_list,
price_ranges=_price_ranges,
)
counters.lots_fetched += len(anchor_lots)
if anchor_lots:
@ -807,7 +830,7 @@ async def run_yandex_city_sweep(
counters.lots_updated += updated
consecutive_failures = 0
logger.info(
"yandex-sweep run_id=%d anchor %s: SERP fetched=%d ins=%d upd=%d",
"yandex-sweep run_id=%d center-combos %s: fetched=%d ins=%d upd=%d",
run_id,
_a_name,
len(anchor_lots),

View file

@ -0,0 +1,32 @@
-- 119_yandex_city_sweep_center_combos.sql
-- Обновляет default_params для yandex_city_sweep: переход с 5-anchor × shallow-pages
-- на single-center × rooms × price combos (exhaustive citywide coverage).
--
-- Что меняется:
-- radius_m: 1500 → 25000 (охват всего ЕКБ от центральной точки; Yandex path-based,
-- geo-bbox используется только как fallback фильтр — большой
-- радиус не увеличивает load, только снимает geo-ограничение)
-- pages_per_anchor: 2 → 3 (max_pages per combo; при 5 комнатностей × 6 price_ranges
-- = 30 combos × 3 pages × ~25 карточек = до ~2250 офферов;
-- реальная уникальность после dedup ≪ SERP-cap ~575)
-- request_delay_sec: 9 → 9 (без изменений — пауза между address-enrich requests)
--
-- anchors-параметр исчезает из default_params: run_yandex_city_sweep теперь игнорирует
-- его при anchors=None и всегда использует EKB_CENTER (56.8400, 60.6050) из кода.
--
-- rooms_list/price_ranges НЕ добавляются в params — код читает defaults из
-- ROOM_PATH.keys() + DEFAULT_PRICE_RANGES (все 5 комнатностей × 6 price-buckets).
-- Если в будущем потребуется ограничить набор — добавить в default_params вручную.
--
-- Idempotent: DO UPDATE перезаписывает params. SAFE to run повторно.
-- ЗАВИСИМОСТИ: 078_scrape_schedules_seed_yandex_sweep.sql (INSERT строки).
BEGIN;
UPDATE scrape_schedules
SET
default_params = '{"pages_per_anchor": 3, "request_delay_sec": 9, "radius_m": 25000}'::jsonb,
updated_at = NOW()
WHERE source = 'yandex_city_sweep';
COMMIT;

View file

@ -665,3 +665,135 @@ def test_yandex_list_runs_endpoint(app_with_admin) -> None:
assert body[0]["run_id"] == 20
assert body[0]["source"] == "yandex_city_sweep"
assert "2025-05-01" in body[0]["started_at"]
# ── Center-combos wiring: prod (anchors=None) uses EKB center + combos ──────
async def test_prod_mode_uses_center_anchor_with_combos(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""anchors=None (prod mode) → fetch_around_multi_room called with EKB center coords
AND rooms_list + price_ranges (combos mode), NOT with None/None (legacy mode).
anchors_total == 1 (single center sweep).
"""
from app.services.scrapers.yandex_realty import DEFAULT_PRICE_RANGES, ROOM_PATH
calls: list[dict] = []
async def fake_fetch_multi(
self: YandexRealtyScraper,
lat: float,
lon: float,
radius_m: int = 1000,
max_pages: int = 2,
rooms_list: list | None = None,
price_ranges: list | None = None,
**_: Any,
) -> list[ScrapedLot]:
calls.append(
{
"lat": lat,
"lon": lon,
"radius_m": radius_m,
"rooms_list": rooms_list,
"price_ranges": price_ranges,
"max_pages": max_pages,
}
)
return [_fake_lot("center-1"), _fake_lot("center-2")]
monkeypatch.setattr(YandexRealtyScraper, "fetch_around_multi_room", fake_fetch_multi)
monkeypatch.setattr(scrape_pipeline, "save_listings", _fake_save)
fake = _FakeRuns()
_install_runs(monkeypatch, fake)
counters = await scrape_pipeline.run_yandex_city_sweep(
db=MagicMock(),
run_id=10,
# anchors=None → prod mode (EKB center)
pages_per_anchor=3,
enrich_address=False,
request_delay_sec=0.0,
)
# Single center anchor
assert counters.anchors_total == 1
assert counters.anchors_done == 1
# fetch_around_multi_room called exactly once
assert len(calls) == 1
call = calls[0]
# EKB center coords
assert abs(call["lat"] - 56.8400) < 0.0001
assert abs(call["lon"] - 60.6050) < 0.0001
# radius covers the whole city (25km default)
assert call["radius_m"] >= 10000
# Combos mode: rooms_list and price_ranges are NOT None
assert call["rooms_list"] is not None
assert call["price_ranges"] is not None
# rooms_list covers all ROOM_PATH keys
assert set(call["rooms_list"]) == set(ROOM_PATH.keys())
# price_ranges matches DEFAULT_PRICE_RANGES
assert call["price_ranges"] == DEFAULT_PRICE_RANGES
# max_pages forwarded from pages_per_anchor
assert call["max_pages"] == 3
# lots aggregated correctly
assert counters.lots_fetched == 2
assert counters.lots_inserted == 2
assert fake.done is not None
assert fake.failed is None
async def test_explicit_anchors_still_works(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""anchors=[...] (test/manual override) → fetch called for each anchor, combos mode."""
calls: list[dict] = []
async def fake_fetch_multi(
self: YandexRealtyScraper,
lat: float,
lon: float,
radius_m: int = 1000,
max_pages: int = 2,
rooms_list: list | None = None,
price_ranges: list | None = None,
**_: Any,
) -> list[ScrapedLot]:
calls.append({"lat": lat, "rooms_list": rooms_list, "price_ranges": price_ranges})
return [_fake_lot(f"{lat}")]
monkeypatch.setattr(YandexRealtyScraper, "fetch_around_multi_room", fake_fetch_multi)
monkeypatch.setattr(scrape_pipeline, "save_listings", _fake_save)
fake = _FakeRuns()
_install_runs(monkeypatch, fake)
two_anchors = [(56.8400, 60.6050, "A"), (56.7950, 60.5300, "B")]
counters = await scrape_pipeline.run_yandex_city_sweep(
db=MagicMock(),
run_id=11,
anchors=two_anchors,
pages_per_anchor=2,
enrich_address=False,
request_delay_sec=0.0,
)
# Both anchors iterated
assert len(calls) == 2
assert counters.anchors_total == 2
assert counters.anchors_done == 2
# Each call uses combos (not None/None)
for c in calls:
assert c["rooms_list"] is not None
assert c["price_ranges"] is not None
assert fake.done is not None