diff --git a/tradein-mvp/backend/app/api/v1/admin.py b/tradein-mvp/backend/app/api/v1/admin.py
index 450dd0a7..6512c09e 100644
--- a/tradein-mvp/backend/app/api/v1/admin.py
+++ b/tradein-mvp/backend/app/api/v1/admin.py
@@ -31,14 +31,15 @@ from pydantic import BaseModel, Field, field_validator
#
# scraper_kit-миграция (#2397 slice A, эпик #2277): 5 debug city-sweep/full-load
# роутов (avito-city-sweep, cian-city-sweep, cian-full-load, yandex-city-sweep,
-# yandex-full-load) переключены с app.services.scrape_pipeline (legacy) на
-# scraper_kit.orchestration.pipeline — тот же DI-паттерн, что и app.scheduler_main
-# ._run_kit_scheduler / scraper_kit.orchestration.scheduler._job_*: config/matcher/
-# enrichment/proxy_provider инжектируются явно вместо module-level импортов app.*
-# внутри scrape_pipeline.py. shutdown_requested НЕ прокидывается (нет SIGTERM-drain
+# yandex-full-load) переключены с app.services.scrape_pipeline (legacy, удалён в
+# #2397 Part E1) на scraper_kit.orchestration.pipeline — тот же DI-паттерн, что и
+# app.scheduler_main._run_kit_scheduler / scraper_kit.orchestration.scheduler
+# ._job_*: config/matcher/enrichment/proxy_provider инжектируются явно вместо
+# module-level импортов app.*. shutdown_requested НЕ прокидывается (нет SIGTERM-drain
# семантики у admin BackgroundTasks — эквивалент дефолту lambda: False, поведение не
-# меняется). Боевой scheduler.py (production sweep-cron) остаётся на legacy — это
-# отдельный, ещё больший шаг (см. vault Event_Legacy_Scrapers_Partial_Deletion_Jul04).
+# меняется). Боевой scheduler-путь (production sweep-cron) уже на kit (#2397 Part C
+# убрал legacy app.services.scheduler.scheduler_loop fallback) — тот же orchestrator,
+# что и debug-роуты этого файла.
from scraper_kit.base import save_listings
from scraper_kit.browser_fetcher import BrowserFetcher
from scraper_kit.orchestration.pipeline import (
diff --git a/tradein-mvp/backend/app/services/house_imv_backfill.py b/tradein-mvp/backend/app/services/house_imv_backfill.py
index c564e508..38bdd005 100644
--- a/tradein-mvp/backend/app/services/house_imv_backfill.py
+++ b/tradein-mvp/backend/app/services/house_imv_backfill.py
@@ -13,9 +13,10 @@ For each house:
Resumable: re-running picks up from where the previous run stopped.
process_houses_imv_batch() IS wired into the scheduler: run_avito_city_sweep()
-(scrape_pipeline.py) calls it automatically as the final IMV-phase of every
-avito city sweep. backfill_house_imv() (batch/pending-status entrypoint) remains
-manual-only, triggered via admin API.
+(scraper_kit.orchestration.pipeline, via the injected EnrichmentJobs protocol)
+calls it automatically as the final IMV-phase of every avito city sweep.
+backfill_house_imv() (batch/pending-status entrypoint) remains manual-only,
+triggered via admin API.
"""
from __future__ import annotations
diff --git a/tradein-mvp/backend/app/services/scrape_pipeline.py b/tradein-mvp/backend/app/services/scrape_pipeline.py
deleted file mode 100644
index a46ed941..00000000
--- a/tradein-mvp/backend/app/services/scrape_pipeline.py
+++ /dev/null
@@ -1,3734 +0,0 @@
-"""scrape_pipeline.py — Avito full orchestrator (Stage 2e + city sweep).
-
-Координирует 5 шагов одного pipeline run:
- 1. SEARCH — AvitoScraper().fetch_around(lat, lon, radius_m, pages=N)
- 2. SAVE — save_listings(db, lots) → (inserted, updated)
- 3. GROUP — dedupe house_url из lots → unique set
- 4. ENRICH_HOUSES — fetch_house_catalog + save_house_catalog_enrichment per unique house
- 5. ENRICH_DETAIL — top-N listings (detail_enriched_at IS NULL) → fetch_detail + save
-
-City sweep: run_avito_city_sweep — итерирует EKB_ANCHORS × pages_per_anchor,
-с cooperative cancel через scrape_runs.status и heartbeat update каждый anchor.
-
-Anti-bot hardening (2023-05-23):
- - Shared AsyncSession на весь sweep (один TLS handshake)
- - asyncio.sleep с random +-20% jitter между detail requests
- - AvitoBlockedError/AvitoRateLimitedError propagation → early abort
- - После 3 consecutive blocks — abort sweep + mark_banned (status='banned')
-
-Graceful degradation на каждом step: exception в одной house/listing
-не валит весь pipeline (кроме blocked exceptions — они abort весь sweep).
-
-IMV-фаза (финальная, после всех anchor'ов): если enrich_imv=True —
-для домов, тронутых в этом sweep'е (touched house_id из houses-enrichment),
-вызывается house_imv_backfill.process_houses_imv_batch. Ошибки per-house
-логируются и считаются, но не валят sweep.
-"""
-
-from __future__ import annotations
-
-import asyncio
-import logging
-import random
-from contextlib import AsyncExitStack
-from dataclasses import dataclass, field, fields
-from datetime import date, timedelta
-from typing import Any
-from urllib.parse import urlparse
-
-from curl_cffi.requests import AsyncSession
-from sqlalchemy import text
-from sqlalchemy.orm import Session
-
-from app.core.config import settings
-from app.core.shutdown import shutdown_requested
-from app.services import proxy_pool, scrape_runs
-from app.services.scrapers.avito import AvitoScraper
-from app.services.scrapers.avito_detail import fetch_detail, save_detail_enrichment
-from app.services.scrapers.avito_exceptions import AvitoBlockedError, AvitoRateLimitedError
-from app.services.scrapers.avito_houses import fetch_house_catalog, save_house_catalog_enrichment
-from app.services.scrapers.base import ScrapedLot, save_listings
-from app.services.scrapers.browser_fetcher import BrowserFetcher
-from app.services.scrapers.yandex_realty import YandexRealtyScraper
-from app.services.yandex_price_history import record_yandex_price_history
-
-logger = logging.getLogger(__name__)
-
-
-async def _rotate_proxy_ip(*, reason: str, rotations_done: int, source: str = "avito") -> bool:
- """Вызвать changeip-ссылку mobileproxy и подождать settle (#1790/#1848).
-
- Provider-aware: выбирает rotate_url и max_rotations по параметру `source`
- (avito / cian / yandex). Fallback-цепочка для rotate_url:
- avito: avito_proxy_rotate_url
- cian: cian_proxy_rotate_url → avito_proxy_rotate_url
- yandex: yandex_proxy_rotate_url → avito_proxy_rotate_url
-
- Аналог AvitoScraper._rotate_ip, но на уровне pipeline — используется в
- enrichment-фазах где нет доступа к экземпляру scraper'а.
-
- Returns True при успешной смене IP, False если rotate_url не задан или ошибка.
- Логирует каждую ротацию (причина, порядковый номер, source, остаток лимита).
- """
- if source == "cian":
- rotate_url = settings.cian_proxy_rotate_url or settings.avito_proxy_rotate_url
- max_rot = settings.cian_proxy_max_rotations
- elif source == "yandex":
- rotate_url = settings.yandex_proxy_rotate_url or settings.avito_proxy_rotate_url
- max_rot = settings.yandex_proxy_max_rotations
- else:
- # avito (default) — backward compat
- rotate_url = settings.avito_proxy_rotate_url
- max_rot = settings.avito_proxy_max_rotations
- if not rotate_url:
- return False
- sep = "&" if "?" in rotate_url else "?"
- # #1950: retry changeip-GET короткими попытками вместо одношотного 30s-timeout.
- # Если changeip-сервер завис на 30s → весь прогон ждёт зря + ротация считается
- # успешной (False возвращался). Теперь: proxy_rotate_attempts попыток по
- # proxy_rotate_attempt_timeout_s каждая; возвращаем True при первом успехе.
- _attempts = settings.proxy_rotate_attempts
- _attempt_timeout = settings.proxy_rotate_attempt_timeout_s
- for attempt in range(_attempts):
- try:
- async with AsyncSession(timeout=_attempt_timeout) as rot:
- resp = await rot.get(f"{rotate_url}{sep}format=json")
- new_ip: str = ""
- try:
- data = resp.json()
- new_ip = str(data.get("new_ip", ""))
- except Exception:
- pass
- await asyncio.sleep(settings.avito_proxy_rotate_settle_s)
- logger.info(
- "pipeline: IP rotated via changeip — source=%s reason=%s "
- "rotation=#%d attempt=%d/%d remaining=%d new_ip=%s",
- source,
- reason,
- rotations_done + 1,
- attempt + 1,
- _attempts,
- max_rot - rotations_done - 1,
- new_ip or "unknown",
- )
- return True
- except Exception:
- logger.warning(
- "pipeline: IP rotation attempt %d/%d failed " "(source=%s reason=%s rotation=#%d)",
- attempt + 1,
- _attempts,
- source,
- reason,
- rotations_done + 1,
- exc_info=True,
- )
- if attempt < _attempts - 1:
- await asyncio.sleep(1.0)
- logger.error(
- "pipeline: all %d changeip attempts failed " "(source=%s reason=%s rotation=#%d)",
- _attempts,
- source,
- reason,
- rotations_done + 1,
- )
- return False
-
-
-# Per-anchor watchdog timeout (секунды). Если обработка одного anchor'а занимает
-# дольше этого значения (зависший HTTP-запрос, прокси hang), asyncio.wait_for
-# прерывает её с TimeoutError, sweep продолжается со следующим anchor'ом.
-# Диапазон 180-300 с; 240 — разумный середина (4 мин > любой нормальный anchor).
-ANCHOR_TIMEOUT_SEC: int = 240
-
-# Константы для расчёта watchdog-таймаута combos-режима Yandex sweep.
-# В combos-режиме единственный "anchor" делает num_combos × max_pages fetch'ей;
-# каждый fetch занимает request_delay_sec + сетевой overhead. ADDRESS_ENRICH_BUDGET_S
-# добавляет буфер на address-enrich фазу (per-listing HTTP + sleep × N листингов).
-# Пример: 30 combos × 3 pages × (9+12) + 300 ≈ 2190s (~37 мин) — надёжно укладывается
-# в окно 02:00-05:00 без риска перекрытия следующего sweep'а.
-_YANDEX_COMBOS_PER_FETCH_S: float = 12.0 # network + parse margin (секунды на страницу)
-_YANDEX_ADDRESS_ENRICH_BUDGET_S: float = 300.0 # бюджет на address-enrich фазу
-
-# Константы для расчёта watchdog-таймаута Avito city sweep.
-# Avito detail-fetch идёт через браузерный сервис с page.goto timeout 60s, и
-# anti-bot страницы нередко доходят до этого лимита. Фиксированный ANCHOR_TIMEOUT_SEC=240
-# guillotine'ит anchor ещё в detail-фазе → SERP-счётчики теряются, run показывает
-# lots_fetched=0 при реально сохранённых листингах. Таймаут масштабируется:
-# _avito_anchor_timeout = ANCHOR_TIMEOUT_SEC
-# + detail_top_n × _AVITO_PER_DETAIL_S
-# + (_AVITO_HOUSES_BUDGET_S if enrich_houses else 0)
-# Пример: detail_top_n=20 + houses → 240 + 1000 + 180 = 1420s/anchor (worst-case).
-# 5 anchor'ов × 1420s ≈ 2ч worst-case (нормальный прогон быстрее: не каждый detail
-# достигает 60s timeout). Окно лишь ограничивает старт следующего sweep'а — scheduler
-# ждёт финиша задачи и не запустит параллельный run.
-_AVITO_PER_DETAIL_S: float = 50.0 # секунд на один detail-fetch (incl. occasional 60s timeout)
-_AVITO_HOUSES_BUDGET_S: float = 180.0 # бюджет на house-enrich фазу
-# Fail-fast: если detail phase накапливает подряд N timeout/ошибок, прерываем её
-# (аналог consecutive_blocks для AvitoBlockedError в step 5). Срабатывает только при
-# явной деградации browser-сервиса, не мешает нормальным transient-ошибкам.
-_AVITO_DETAIL_CONSECUTIVE_TIMEOUT_ABORT: int = 5
-
-# #1820: порог ПОСЛЕДОВАТЕЛЬНЫХ блоков (AvitoBlockedError/AvitoRateLimitedError) внутри
-# enrichment-фаз run_avito_pipeline. Один заблокированный фетч (house/detail) больше НЕ
-# роняет весь прогон — блок ловится пер-item (как обычная ошибка), и лишь N подряд
-# прерывают enrichment-фазу. Listings из SEARCH+SAVE сохраняются всегда. Зеркалит
-# паттерн avito_detail_backfill (max_consecutive_blocks).
-_AVITO_PIPELINE_CONSECUTIVE_BLOCK_ABORT: int = 3
-
-# #2160: константы для расчёта watchdog-таймаута Cian city sweep. При
-# USE_PROXY_POOL_BROWSER=true каждый SERP-фетч идёт через camoufox с relaunch при смене
-# прокси (page.goto timeout 60s + overhead) = 13-45s/страница, а якорь cian = 4 room-buckets
-# × pages_per_anchor SERP-фетчей + detail_top_n detail-фетчей + houses-enrich. Фиксированный
-# ANCHOR_TIMEOUT_SEC=240 гильотинит якорь ещё в SERP-фазе → save_listings не успевает,
-# lots_fetched=0, после N подряд → mark_banned. Таймаут масштабируется:
-# _cian_anchor_timeout = max(
-# ANCHOR_TIMEOUT_SEC,
-# _CIAN_ROOM_BUCKETS × pages_per_anchor × (request_delay_sec + _CIAN_PER_SERP_FETCH_S)
-# + detail_top_n × _CIAN_PER_DETAIL_S
-# + (_CIAN_HOUSES_BUDGET_S if enrich_houses else 0))
-# Пример (дефолты): 4×3×(5+70) + 10×50 + 180 = 900 + 500 + 180 = 1580s/anchor (worst-case,
-# это watchdog от зависания, не ожидаемая длительность).
-_CIAN_ROOM_BUCKETS: int = 4 # fetch_around_multi_room итерирует ((1,),(2,),(3,),(4,)),
-# см. providers/cian/serp.py:215
-_CIAN_PER_SERP_FETCH_S: float = 70.0 # browser fetch через пул: page.goto 60s + camoufox relaunch
-_CIAN_PER_DETAIL_S: float = 50.0 # секунд на один detail-fetch (как avito)
-_CIAN_HOUSES_BUDGET_S: float = 180.0 # бюджет на house-enrich фазу (как avito)
-
-
-def _cian_anchor_timeout_s(
- pages_per_anchor: int,
- request_delay_sec: float,
- detail_top_n: int,
- enrich_houses: bool,
-) -> float:
- """Масштабируемый watchdog-таймаут одного Cian anchor'а (секунды).
-
- Чистая функция для тестируемости формулы (#2160). Никогда не опускается ниже
- ANCHOR_TIMEOUT_SEC (max-ветка): при малых параметрах фиксированный минимум остаётся.
- """
- return max(
- float(ANCHOR_TIMEOUT_SEC),
- _CIAN_ROOM_BUCKETS * pages_per_anchor * (request_delay_sec + _CIAN_PER_SERP_FETCH_S)
- + detail_top_n * _CIAN_PER_DETAIL_S
- + (_CIAN_HOUSES_BUDGET_S if enrich_houses else 0.0),
- )
-
-
-# Default anchors ЕКБ — 5 точек покрытия города
-EKB_ANCHORS: list[tuple[float, float, str]] = [
- (56.8400, 60.6050, "Центр"),
- (56.7950, 60.5300, "ЮЗ"),
- (56.8970, 60.6100, "Уралмаш"),
- (56.7700, 60.5500, "Академический"),
- (56.8650, 60.6200, "Пионерский"),
-]
-
-_CHROME_HEADERS = {
- "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
- "Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8",
- "Cache-Control": "max-age=0",
- "Sec-Fetch-Dest": "document",
- "Sec-Fetch-Mode": "navigate",
- "Sec-Fetch-Site": "none",
- "Sec-Fetch-User": "?1",
- "Upgrade-Insecure-Requests": "1",
-}
-
-
-def _avito_proxies() -> dict[str, str] | None:
- """#623/#806: mobile-proxy egress для shared-сессии sweep'а.
-
- КРИТИЧНО: sweep создаёт собственную shared AsyncSession и присваивает её
- `scraper._cffi`, перезатирая проксированную сессию из `AvitoScraper.__aenter__`.
- Через эту же сессию идёт и SERP, и detail-энричмент (fetch_detail). Без прокси
- detail-страницы летят с datacenter-IP → HTTP 429 → весь run mark_banned.
- Пусто (env не задан) → прямое подключение (dev/no-op).
-
- Читает settings.scraper_proxy_url (SCRAPER_PROXY_URL > AVITO_PROXY_URL fallback).
- """
- url = settings.scraper_proxy_url
- return {"http": url, "https": url} if url else None
-
-
-def _mask_proxy_url(url: str) -> str:
- """Скрыть пароль в proxy-url для логов (scheme://user:***@host)."""
- if "@" not in url or "//" not in url:
- return url
- scheme, rest = url.split("//", 1)
- creds, host = rest.split("@", 1)
- if ":" in creds:
- user, _pwd = creds.split(":", 1)
- creds = f"{user}:***"
- return f"{scheme}//{creds}@{host}"
-
-
-def _pool_proxy_dict(
- db: Session, provider: str, *, run_id: int | None = None
-) -> tuple[dict[str, str] | None, proxy_pool.ProxyLease | None]:
- """#2202: health-filtered lease из scrape_proxies вместо статического env-прокси.
-
- За флагом settings.use_proxy_pool_curl (ship-dark, default False — прод не
- меняется, пока флаг не включён явно). Флаг off ИЛИ пул пуст/ошибка acquire →
- fallback на статический _avito_proxies() (settings.scraper_proxy_url), поведение
- как раньше — сбор никогда не падает из-за проблем пула.
-
- run_id — id текущего scrape_runs-прогона (если он есть у вызывающего, см.
- run_avito_city_sweep/run_avito_newbuilding_sweep). Прокидывается в
- proxy_pool.acquire(), чтобы scrape_proxies.leased_by нёс реальный run_id для
- диагностики банов вместо NON_RUN_LEASE_MARKER. None (default) — прежнее
- поведение для вызывающих без run_id в scope (run_avito_pipeline standalone).
-
- Returns (proxies-dict для curl_cffi AsyncSession, lease | None). lease нужен
- вызывающему для release+mark_health в конце run (см. _release_pool_lease) и для
- последующей ротации на бан (см. _rotate_pool_proxy).
- """
- # getattr-defensive: некоторые тестовые/adapter settings-объекты (SimpleNamespace)
- # не несут этого поля — трактуем отсутствие как off (то же ship-dark поведение).
- if not getattr(settings, "use_proxy_pool_curl", False):
- return _avito_proxies(), None
- try:
- lease = proxy_pool.acquire(db, provider, run_id=run_id)
- except Exception:
- logger.warning(
- "pipeline: proxy_pool.acquire(%s) failed — fallback to env proxy",
- provider,
- exc_info=True,
- )
- lease = None
- if lease is None:
- logger.warning(
- "pipeline: proxy pool empty/unhealthy for provider=%s — fallback to env proxy",
- provider,
- )
- return _avito_proxies(), None
- logger.info(
- "pipeline: using pool proxy id=%d provider=%s url=%s for this run",
- lease.id,
- provider,
- _mask_proxy_url(lease.url),
- )
- return {"http": lease.url, "https": lease.url}, lease
-
-
-def _release_pool_lease(db: Session, lease: proxy_pool.ProxyLease | None, ok: bool) -> None:
- """Best-effort release+mark_health в конце run (#2202) — проблема пула не должна
- ронять уже собранные данные. lease=None (пул выключен/не выдал прокси) → no-op.
- """
- if lease is None:
- return
- try:
- proxy_pool.mark_health(db, lease.id, ok)
- except Exception:
- logger.warning("pipeline: mark_health failed for lease id=%d", lease.id, exc_info=True)
- try:
- proxy_pool.release(db, lease.id)
- except Exception:
- logger.warning("pipeline: release failed for lease id=%d", lease.id, exc_info=True)
-
-
-async def _rotate_pool_proxy(
- db: Session,
- provider: str,
- session: AsyncSession,
- current_lease: proxy_pool.ProxyLease | None,
-) -> proxy_pool.ProxyLease | None:
- """#2202: переключиться на ДРУГОЙ здоровый прокси из пула при бане (не changeip
- одного и того же IP). Помечает current_lease нездоровым (mark_health ok=False),
- затем acquire()-ит следующий свободный здоровый прокси, и ТОЛЬКО ПОСЛЕ этого
- освобождает current_lease (release). Порядок здесь принципиален (#2202 MAJOR-фикс,
- review-репро): release ДО acquire() убирает current_lease из leased_by-фильтра
- немедленно, и один бан поднимает consecutive_fails только до 1 (порог исключения —
- proxy_pool.MAX_CONSECUTIVE_FAILS=3) — только что забаненный прокси тут же снова
- проходит health-фильтр и acquire() может вернуть его же обратно (ORDER BY
- last_ok_at NULLS LAST, id — никак не гарантирует, что он не окажется первым).
- Пока current_lease НЕ освобождён, acquire()'s `leased_by IS NULL` условие
- гарантированно его исключает — никакой отдельный exclude_id-параметр не нужен.
-
- Успешная ротация мутирует session.proxies — curl_cffi AsyncSession читает
- self.proxies заново на каждый request (не кэширует на конструкторе), так что
- переприсвоение атрибута действует на следующий вызов без пересоздания сессии.
-
- Returns новый lease, или None если пул пуст/ошибка (вызывающий код в этом случае
- падает обратно на changeip-ротацию одного IP — см. _rotate_avito_proxy).
- current_lease освобождается в обоих случаях (успех/провал) — он подтверждённо
- нездоров, дальше эта функция за него не отвечает.
- """
- if current_lease is not None:
- try:
- proxy_pool.mark_health(db, current_lease.id, ok=False)
- except Exception:
- logger.warning(
- "pipeline: mark_health(False) failed for lease id=%d",
- current_lease.id,
- exc_info=True,
- )
-
- try:
- new_lease = proxy_pool.acquire(db, provider)
- except Exception:
- logger.warning(
- "pipeline: proxy_pool.acquire(%s) failed during rotation", provider, exc_info=True
- )
- new_lease = None
-
- if current_lease is not None:
- try:
- proxy_pool.release(db, current_lease.id)
- except Exception:
- logger.warning(
- "pipeline: release failed for lease id=%d", current_lease.id, exc_info=True
- )
-
- if new_lease is None:
- logger.warning(
- "pipeline: proxy pool exhausted for provider=%s during rotation — "
- "no healthy proxy to rotate to",
- provider,
- )
- return None
-
- session.proxies = {"http": new_lease.url, "https": new_lease.url}
- logger.info(
- "pipeline: ROTATED to pool proxy id=%d provider=%s url=%s (previous lease id=%s)",
- new_lease.id,
- provider,
- _mask_proxy_url(new_lease.url),
- current_lease.id if current_lease is not None else None,
- )
- return new_lease
-
-
-async def _rotate_avito_proxy(
- db: Session,
- session: AsyncSession | None,
- current_lease: proxy_pool.ProxyLease | None,
- *,
- reason: str,
- rotations_done: int,
-) -> tuple[bool, proxy_pool.ProxyLease | None]:
- """#2202: единая точка ротации на бан для Avito.
-
- Сначала пробует переключить прокси ПУЛА (health-фильтр, гарантированно другой
- узел) — только когда settings.use_proxy_pool_curl включён И есть curl-сессия
- (session is not None, т.е. не browser-mode). Если пул выключен, пуст, или это
- browser-mode (session=None) — падает на старый changeip (смена exit-IP ОДНОГО
- прокси-аккаунта, как раньше).
-
- NB: пока делит общий rotations_done/avito_proxy_max_rotations бюджет с
- changeip-путём — раздельные бюджеты (пул своё, changeip своё) можно завести
- отдельным follow-up, если пул-ротации на практике окажутся частым событием.
-
- Returns (rotated, lease для последующих вызовов — обновлён при успешной
- пул-ротации, иначе тот же current_lease).
- """
- if getattr(settings, "use_proxy_pool_curl", False) and session is not None:
- new_lease = await _rotate_pool_proxy(db, "avito", session, current_lease)
- if new_lease is not None:
- return True, new_lease
- logger.warning(
- "pipeline: pool rotation unavailable (reason=%s) — falling back to changeip", reason
- )
- rotated = await _rotate_proxy_ip(reason=reason, rotations_done=rotations_done, source="avito")
- return rotated, current_lease
-
-
-# ── Result dataclasses ──────────────────────────────────────────
-
-
-@dataclass
-class PipelineCounters:
- """Counters одного pipeline run для логов/админки."""
-
- lots_fetched: int = 0
- lots_inserted: int = 0
- lots_updated: int = 0
- unique_houses: int = 0
- houses_enriched: int = 0
- houses_failed: int = 0
- detail_attempted: int = 0
- detail_enriched: int = 0
- detail_failed: int = 0
- errors: list[str] = field(default_factory=list)
-
-
-@dataclass
-class PipelineResult:
- """Полный результат full Avito pipeline."""
-
- anchor_lat: float
- anchor_lon: float
- radius_m: int
- counters: PipelineCounters
- enrich_houses: bool
- enrich_detail_top_n: int
- touched_house_ids: set[int] = field(default_factory=set)
-
-
-# ── Main orchestrator ───────────────────────────────────────────
-
-
-async def run_avito_pipeline(
- db: Session,
- lat: float,
- lon: float,
- radius_m: int = 1500,
- *,
- enrich_houses: bool = True,
- enrich_detail_top_n: int = 10,
- pages: int = 1,
- request_delay_sec: float | None = None,
- shared_session: AsyncSession | None = None,
- shared_browser: BrowserFetcher | None = None,
-) -> PipelineResult:
- """Full Avito search → houses → detail enrichment pipeline.
-
- Steps:
- 1. SEARCH: AvitoScraper().fetch_around(lat, lon, radius_m, pages=pages)
- 2. SAVE: save_listings(db, lots) → (inserted, updated)
- 3. GROUP: dedupe house_url из новых lots → unique house path set
- 4. ENRICH_HOUSES: fetch_house_catalog + save per house (with sleep between)
- 5. ENRICH_DETAIL: top-N listings → fetch_detail + save (abort on 3 blocks)
-
- shared_session — если передан, используется без закрытия (lifecycle у вызывающего).
- shared_browser — если передан в browser-mode, используется без закрытия.
-
- Блокировки (AvitoBlockedError / AvitoRateLimitedError) в SEARCH-фазе (шаг 1)
- по-прежнему propagate наружу (без листингов прогон бессмыслен). В enrichment-фазах
- (houses шаг 4 + detail шаг 5 + house-detail шаг 5b) блок ловится ПЕР-ITEM как обычная
- ошибка (#1820): один заблокированный фетч НЕ роняет весь прогон. Лишь
- _AVITO_PIPELINE_CONSECUTIVE_BLOCK_ABORT блоков ПОДРЯД прерывают enrichment — но
- PipelineResult всё равно возвращается с уже собранными listings/обогащениями.
- """
- counters = PipelineCounters()
- lots: list[ScrapedLot] = []
- detail_delay = request_delay_sec if request_delay_sec is not None else 7.0
- # #1820: общий счётчик ПОСЛЕДОВАТЕЛЬНЫХ блоков на все enrichment-фазы (4/5/5b).
- # Сбрасывается на любом успешном fetch; при достижении порога — ротация IP (#1790)
- # или abort если бюджет ротаций исчерпан.
- enrich_consecutive_blocks = 0
- enrich_rotations_done = 0 # #1790: сколько раз уже сменили IP в этом run
- enrichment_aborted = False
-
- browser_mode = settings.scraper_fetch_mode == "browser"
- session: AsyncSession | None = None
- browser_fetcher: BrowserFetcher | None = None
- own_session = False
- own_browser = False
- # #2202: pool-lease текущей curl-сессии (None если browser-mode/pool off/shared
- # session переданный извне — тогда владелец проксей не мы, не трогаем).
- _lease: proxy_pool.ProxyLease | None = None
-
- scraper = AvitoScraper()
- if browser_mode:
- browser_fetcher = shared_browser
- if browser_fetcher is None:
- browser_fetcher = BrowserFetcher(source="avito")
- await browser_fetcher.__aenter__()
- own_browser = True
- scraper._browser = browser_fetcher
- else:
- own_session = shared_session is None
- if shared_session is not None:
- session = shared_session
- else:
- _proxies, _lease = _pool_proxy_dict(db, "avito")
- session = AsyncSession(
- impersonate="chrome120",
- timeout=25,
- headers=_CHROME_HEADERS,
- proxies=_proxies,
- )
- scraper._cffi = session
-
- try:
- # ── Step 1: search ──────────────────────────────────────
- try:
- lots = await scraper.fetch_around(
- lat,
- lon,
- radius_m,
- pages=pages,
- delay_override_sec=request_delay_sec,
- )
- counters.lots_fetched = len(lots)
- logger.info(
- "pipeline:search anchor=(%.4f,%.4f) r=%dm fetched=%d",
- lat,
- lon,
- radius_m,
- len(lots),
- )
- except (AvitoBlockedError, AvitoRateLimitedError):
- logger.error("pipeline:search BLOCKED by Avito at anchor=(%.4f,%.4f)", lat, lon)
- raise
- except Exception as e:
- logger.exception("pipeline:search failed")
- counters.errors.append(f"search: {e}")
- return PipelineResult(lat, lon, radius_m, counters, enrich_houses, enrich_detail_top_n)
-
- # ── Step 2: save listings ───────────────────────────────
- if lots:
- try:
- counters.lots_inserted, counters.lots_updated = save_listings(db, lots)
- except Exception as e:
- logger.exception("pipeline:save_listings failed")
- counters.errors.append(f"save_listings: {e}")
-
- # ── Step 3: group by house ──────────────────────────────
- unique_house_paths: set[str] = set()
- for lot in lots:
- if lot.house_url:
- try:
- parsed = urlparse(lot.house_url)
- path = parsed.path if parsed.path else lot.house_url
- unique_house_paths.add(path)
- except Exception:
- continue
-
- counters.unique_houses = len(unique_house_paths)
- logger.info("pipeline:group_by_house unique=%d", counters.unique_houses)
-
- # ── Step 4: enrich houses с shared session + sleep ──────
- touched_house_ids: set[int] = set()
- if enrich_houses and unique_house_paths and not enrichment_aborted:
- house_paths_list = list(unique_house_paths)
- for idx, house_path in enumerate(house_paths_list):
- try:
- enrichment = await fetch_house_catalog(
- house_path, cffi_session=session, browser_fetcher=browser_fetcher
- )
- house_counters = save_house_catalog_enrichment(db, enrichment)
- counters.houses_enriched += 1
- enrich_consecutive_blocks = 0
- hid = house_counters.get("house_id")
- if hid:
- touched_house_ids.add(int(hid))
- except (AvitoBlockedError, AvitoRateLimitedError) as e:
- # #1820: блок ловим пер-item как обычную ошибку — НЕ роняем прогон.
- enrich_consecutive_blocks += 1
- counters.houses_failed += 1
- counters.errors.append(f"BLOCKED_HOUSE {house_path}: {e}")
- logger.warning(
- "pipeline:house BLOCKED at %s (consecutive=%d)",
- house_path,
- enrich_consecutive_blocks,
- )
- if enrich_consecutive_blocks >= _AVITO_PIPELINE_CONSECUTIVE_BLOCK_ABORT:
- # #1790: Перед abort'ом пробуем сменить IP (changeip).
- # После ротации сбрасываем счётчик и продолжаем — НЕ abort.
- # Если ротации исчерпаны — abort enrichment (listings уже сохранены).
- if enrich_rotations_done < settings.avito_proxy_max_rotations:
- rotated, _lease = await _rotate_avito_proxy(
- db,
- session,
- _lease,
- reason=f"house consecutive={enrich_consecutive_blocks}",
- rotations_done=enrich_rotations_done,
- )
- enrich_rotations_done += 1
- if rotated:
- enrich_consecutive_blocks = 0
- logger.info(
- "pipeline:enrich ROTATE — proxy changed, "
- "reset consecutive blocks (rotation #%d/%d)",
- enrich_rotations_done,
- settings.avito_proxy_max_rotations,
- )
- continue
- logger.error(
- "pipeline:enrich ABORT — %d consecutive blocks (IP rate-limited), "
- "listings preserved",
- enrich_consecutive_blocks,
- )
- counters.errors.append(
- f"AVITO_BLOCKED — enrichment aborted after "
- f"{enrich_consecutive_blocks} consecutive blocks (house phase)"
- )
- enrichment_aborted = True
- break
- except Exception as e:
- logger.warning("pipeline:house_enrich failed for %s: %s", house_path, e)
- counters.houses_failed += 1
- counters.errors.append(f"house {house_path}: {e}")
- # #1368: save_house_catalog_enrichment не делает внутренний rollback,
- # поэтому упавший промежуточный statement оставляет общий Session в
- # InFailedSqlTransaction. Без сброса — каскад «current transaction is
- # aborted» на все оставшиеся дома/listings этого якоря.
- try:
- db.rollback()
- except Exception:
- pass
-
- if idx < len(house_paths_list) - 1:
- await asyncio.sleep(detail_delay)
-
- # ── Step 5: enrich detail для top-N priority listings ───
- if enrich_detail_top_n > 0 and not enrichment_aborted:
- priority_rows = (
- db.execute(
- text("""
- SELECT source_url
- FROM listings
- WHERE source = 'avito'
- AND source_url IS NOT NULL
- AND (
- (
- detail_enriched_at IS NULL
- AND price_rub > 0
- AND ST_DWithin(
- geom::geography,
- ST_MakePoint(:lon, :lat)::geography,
- :radius
- )
- )
- OR (
- detail_enriched_at IS NULL
- AND scraped_at > NOW() - INTERVAL '2 hours'
- )
- )
- ORDER BY scraped_at DESC NULLS LAST
- LIMIT :limit
- """),
- {
- "lat": lat,
- "lon": lon,
- "radius": radius_m * 2,
- "limit": enrich_detail_top_n,
- },
- )
- .mappings()
- .all()
- )
-
- detail_house_paths: set[str] = set()
- for idx, row in enumerate(priority_rows):
- source_url: str = row["source_url"]
- counters.detail_attempted += 1
- try:
- item_url = (
- urlparse(source_url).path if source_url.startswith("http") else source_url
- )
- enrichment_detail = await fetch_detail(
- item_url, cffi_session=session, browser_fetcher=browser_fetcher
- )
- if save_detail_enrichment(db, enrichment_detail):
- counters.detail_enriched += 1
- # #871: house-link есть в detail SSR (в SERP-карточке — JS-only).
- # Копим catalog-пути для house-enrich ниже (Step 5b).
- if enrichment_detail.house_catalog_url:
- hp = urlparse(enrichment_detail.house_catalog_url).path
- if hp:
- detail_house_paths.add(hp)
- enrich_consecutive_blocks = 0
- except (AvitoBlockedError, AvitoRateLimitedError) as e:
- # #1820: блок ловим пер-item, НЕ роняем прогон. Счётчик общий с house-фазой.
- enrich_consecutive_blocks += 1
- counters.detail_failed += 1
- counters.errors.append(f"BLOCKED_DETAIL #{idx}: {e}")
- logger.warning(
- "pipeline:detail BLOCKED #%d/%d (consecutive=%d): %s",
- idx + 1,
- len(priority_rows),
- enrich_consecutive_blocks,
- e,
- )
- if enrich_consecutive_blocks >= _AVITO_PIPELINE_CONSECUTIVE_BLOCK_ABORT:
- # #1790: Перед abort'ом пробуем сменить IP (changeip).
- # После ротации сбрасываем счётчик — продолжаем detail-обход.
- # Если ротации исчерпаны — abort detail-фазы (listings уже сохранены).
- if enrich_rotations_done < settings.avito_proxy_max_rotations:
- rotated, _lease = await _rotate_avito_proxy(
- db,
- session,
- _lease,
- reason=f"detail consecutive={enrich_consecutive_blocks}",
- rotations_done=enrich_rotations_done,
- )
- enrich_rotations_done += 1
- if rotated:
- enrich_consecutive_blocks = 0
- logger.info(
- "pipeline:enrich ROTATE — proxy changed, "
- "reset consecutive blocks (rotation #%d/%d)",
- enrich_rotations_done,
- settings.avito_proxy_max_rotations,
- )
- continue
- logger.error(
- "pipeline:detail ABORT — %d consecutive blocks (IP rate-limited), "
- "listings preserved",
- enrich_consecutive_blocks,
- )
- counters.errors.append(
- f"AVITO_BLOCKED — enrichment aborted after "
- f"{idx + 1}/{len(priority_rows)} details (consecutive blocks)"
- )
- # #1820: НЕ raise — прерываем только detail-фазу (и Step 5b),
- # PipelineResult возвращается с уже сохранёнными listings/houses.
- enrichment_aborted = True
- break
- except Exception as e:
- counters.detail_failed += 1
- counters.errors.append(f"detail {source_url}: {e}")
- logger.warning("pipeline:detail_enrich failed for %s: %s", source_url, e)
- # #1386: НЕ сбрасываем consecutive_blocks здесь. Транзиентная
- # сетевая/parse-ошибка (5xx, timeout, connection reset) — правдоподобный
- # побочный эффект активного IP-блока (Avito не отдаёт 403/429 единообразно).
- # Сброс маскировал бы продолжающийся блок (block, block, timeout→reset, ...)
- # и не давал бы сработать early-abort на 3 consecutive blocks.
- # Сброс на успехе (см. выше) остаётся.
- # #1368: save_detail_enrichment без внутреннего rollback мог оставить
- # общий Session в aborted-state — сбрасываем, иначе каскад на оставшиеся
- # detail-строки и Step 5b.
- try:
- db.rollback()
- except Exception:
- pass
-
- if idx < len(priority_rows) - 1:
- jitter = random.uniform(0.8, 1.2)
- await asyncio.sleep(detail_delay * jitter)
-
- # ── Step 5b: enrich houses, найденные через detail-страницы (#871) ──
- # SERP-карточки больше не несут house-link (JS-render) → дома берём из
- # detail SSR. Дедуп против обработанных в Step 4 (unique_house_paths).
- new_house_paths = detail_house_paths - unique_house_paths
- if enrich_houses and new_house_paths and not enrichment_aborted:
- counters.unique_houses += len(new_house_paths)
- nh_list = list(new_house_paths)
- for h_idx, house_path in enumerate(nh_list):
- try:
- enrichment = await fetch_house_catalog(
- house_path, cffi_session=session, browser_fetcher=browser_fetcher
- )
- house_counters = save_house_catalog_enrichment(db, enrichment)
- counters.houses_enriched += 1
- enrich_consecutive_blocks = 0
- hid = house_counters.get("house_id")
- if hid:
- touched_house_ids.add(int(hid))
- except (AvitoBlockedError, AvitoRateLimitedError) as e:
- # #1820: блок пер-item, общий счётчик; не роняем прогон.
- enrich_consecutive_blocks += 1
- counters.houses_failed += 1
- counters.errors.append(f"BLOCKED_HOUSE(detail) {house_path}: {e}")
- logger.warning(
- "pipeline:house(detail) BLOCKED at %s (consecutive=%d)",
- house_path,
- enrich_consecutive_blocks,
- )
- if enrich_consecutive_blocks >= _AVITO_PIPELINE_CONSECUTIVE_BLOCK_ABORT:
- # #1790: попытка ротации перед abort'ом (общий бюджет run).
- if enrich_rotations_done < settings.avito_proxy_max_rotations:
- rotated, _lease = await _rotate_avito_proxy(
- db,
- session,
- _lease,
- reason=f"house-detail consecutive={enrich_consecutive_blocks}",
- rotations_done=enrich_rotations_done,
- )
- enrich_rotations_done += 1
- if rotated:
- enrich_consecutive_blocks = 0
- logger.info(
- "pipeline:enrich ROTATE — proxy changed, "
- "reset consecutive blocks (rotation #%d/%d)",
- enrich_rotations_done,
- settings.avito_proxy_max_rotations,
- )
- continue
- logger.error(
- "pipeline:enrich ABORT — %d consecutive blocks "
- "(IP rate-limited), listings preserved",
- enrich_consecutive_blocks,
- )
- counters.errors.append(
- f"AVITO_BLOCKED — enrichment aborted after "
- f"{enrich_consecutive_blocks} consecutive blocks "
- f"(house-detail phase)"
- )
- enrichment_aborted = True
- break
- except Exception as e:
- logger.warning(
- "pipeline:house(detail) enrich failed for %s: %s", house_path, e
- )
- counters.houses_failed += 1
- counters.errors.append(f"house(detail) {house_path}: {e}")
- # #1368: см. Step 4 — сбрасываем aborted-state общего Session,
- # иначе каскад на оставшиеся дома detail-фазы.
- try:
- db.rollback()
- except Exception:
- pass
- if h_idx < len(nh_list) - 1:
- await asyncio.sleep(detail_delay)
-
- logger.info(
- "pipeline:done anchor=(%.4f,%.4f) lots=%d (ins=%d/upd=%d) "
- "houses=%d/%d detail=%d/%d touched_house_ids=%d errors=%d",
- lat,
- lon,
- counters.lots_fetched,
- counters.lots_inserted,
- counters.lots_updated,
- counters.houses_enriched,
- counters.unique_houses,
- counters.detail_enriched,
- counters.detail_attempted,
- len(touched_house_ids),
- len(counters.errors),
- )
- return PipelineResult(
- lat,
- lon,
- radius_m,
- counters,
- enrich_houses,
- enrich_detail_top_n,
- touched_house_ids=touched_house_ids,
- )
-
- finally:
- if own_session and session is not None:
- await session.close()
- if own_browser and browser_fetcher is not None:
- await browser_fetcher.__aexit__(None, None, None)
- if own_session:
- _release_pool_lease(db, _lease, ok=not enrichment_aborted)
-
-
-@dataclass
-class CitySweepCounters:
- """Aggregate counters для full city sweep run."""
-
- anchors_total: int = 0
- anchors_done: int = 0
- lots_fetched: int = 0
- lots_inserted: int = 0
- lots_updated: int = 0
- unique_houses: int = 0
- houses_enriched: int = 0
- houses_failed: int = 0
- detail_attempted: int = 0
- detail_enriched: int = 0
- detail_failed: int = 0
- imv_attempted: int = 0
- imv_enriched: int = 0
- imv_failed: int = 0
- errors_count: int = 0
-
- def to_dict(self) -> dict[str, int]:
- return {f.name: getattr(self, f.name) for f in fields(self)}
-
-
-async def run_avito_city_sweep(
- db: Session,
- *,
- run_id: int,
- anchors: list[tuple[float, float, str]] | None = None,
- radius_m: int = 1500,
- pages_per_anchor: int = 3,
- enrich_houses: bool = True,
- detail_top_n: int = 10,
- request_delay_sec: float = 7.0,
- enrich_imv: bool = True,
-) -> CitySweepCounters:
- """Full city sweep: iterate anchors × pages → save → enrich houses + detail → IMV.
-
- - Single shared AsyncSession на весь sweep (один TLS fingerprint)
- - AvitoBlockedError/AvitoRateLimitedError → ABORT sweep + mark_banned (status='banned')
- - Прочие errors per-anchor логируются, не валят весь sweep
- - После всех anchor'ов: если enrich_imv=True — IMV-оценка тронутых домов
- (cooperative cancel + per-house graceful error handling)
-
- SERP-счётчики (lots_fetched/inserted/updated) записываются в sweep-level counters
- НЕМЕДЛЕННО после save_listings — до начала detail/houses фазы. Даже если anchor
- превышает watchdog-таймаут в detail-фазе, SERP-результаты не теряются.
- """
- from app.services.house_imv_backfill import process_houses_imv_batch
-
- _anchors = anchors if anchors is not None else EKB_ANCHORS
- counters = CitySweepCounters(anchors_total=len(_anchors))
- all_touched_house_ids: set[int] = set()
-
- # Масштабируемый watchdog-таймаут для Avito anchor'а. В отличие от Yandex/Cian,
- # Avito detail-fetch идёт через browser service с page.goto timeout 60s. При
- # detail_top_n=20 detail-фаза может занять до N×60s — фиксированный ANCHOR_TIMEOUT_SEC
- # обрезает её раньше SERP, теряя счётчики. Масштабирование по detail_top_n + houses
- # гарантирует, что SERP+save всегда успевает до watchdog.
- _avito_anchor_timeout = (
- float(ANCHOR_TIMEOUT_SEC)
- + detail_top_n * _AVITO_PER_DETAIL_S
- + (_AVITO_HOUSES_BUDGET_S if enrich_houses else 0.0)
- )
- logger.info(
- "city-sweep run_id=%d: avito anchor_timeout=%.0fs "
- "(base=%d + detail_top_n=%d×%.0fs + houses=%.0fs)",
- run_id,
- _avito_anchor_timeout,
- ANCHOR_TIMEOUT_SEC,
- detail_top_n,
- _AVITO_PER_DETAIL_S,
- _AVITO_HOUSES_BUDGET_S if enrich_houses else 0.0,
- )
-
- browser_mode = settings.scraper_fetch_mode == "browser"
- # #1790: общий бюджет IP-ротаций на весь sweep (разделяется по всем anchor'ам).
- sweep_rotations_done = 0
- # #2202: pool-lease текущей curl-сессии (None в browser-mode/pool off).
- _lease: proxy_pool.ProxyLease | None = None
-
- async with AsyncExitStack() as stack:
- session: AsyncSession | None = None
- shared_bf: BrowserFetcher | None = None
- if browser_mode:
- shared_bf = await stack.enter_async_context(BrowserFetcher(source="avito"))
- else:
- _proxies, _lease = _pool_proxy_dict(db, "avito", run_id=run_id)
- session = await stack.enter_async_context(
- AsyncSession(
- impersonate="chrome120",
- timeout=25,
- headers=_CHROME_HEADERS,
- proxies=_proxies,
- )
- )
- try:
- for idx, (lat, lon, name) in enumerate(_anchors, start=1):
- if scrape_runs.is_cancelled(db, run_id):
- logger.info(
- "city-sweep run_id=%d: cancelled at anchor #%d/%d (%s)",
- run_id,
- idx,
- len(_anchors),
- name,
- )
- scrape_runs.update_heartbeat(db, run_id, counters.to_dict())
- return counters
- elif shutdown_requested():
- # Кооперативный SIGTERM-drain (#1182 Phase 3a): останавливаемся
- # на границе anchor'а и финализируем run как mark_done(partial) —
- # это НЕ user-cancel, поэтому НЕ mark_cancelled. Дальнейшие anchor'ы
- # и IMV-фаза не выполняются.
- logger.info(
- "city-sweep run_id=%d: SIGTERM-drain — stopping at anchor #%d/%d (%s)",
- run_id,
- idx,
- len(_anchors),
- name,
- )
- scrape_runs.update_heartbeat(db, run_id, counters.to_dict())
- scrape_runs.mark_done(db, run_id, counters.to_dict())
- return counters
-
- logger.info(
- "city-sweep run_id=%d anchor #%d/%d (%s, %.4f, %.4f)",
- run_id,
- idx,
- len(_anchors),
- name,
- lat,
- lon,
- )
-
- # Capture loop variables in default args (B023): prevents stale binding
- # if the coroutine is scheduled after the loop variable changes.
- _a_lat, _a_lon, _a_name = lat, lon, name
-
- async def _avito_anchor_phases(
- _lat: float = _a_lat,
- _lon: float = _a_lon,
- _name: str = _a_name,
- ) -> None:
- """Все фазы одного Avito anchor'а (SERP+save → houses → detail).
-
- Обновляет sweep-level counters и all_touched_house_ids через nonlocal
- НЕМЕДЛЕННО после SERP+save (до detail-фазы). Это гарантирует, что
- TimeoutError из wait_for теряет только detail-счётчики, но не SERP.
- """
- nonlocal all_touched_house_ids, sweep_rotations_done, _lease
-
- # ── Phase 1+2: SERP + save ─────────────────────────────────
- scraper = AvitoScraper()
- if browser_mode:
- scraper._browser = shared_bf
- # Shared-browser режим: _cffi=None → curl_cffi-fallback на
- # firewall пропускается (by design). Логируем для observability.
- logger.debug("avito pipeline-mode: shared browser only, no cffi fallback")
- else:
- scraper._cffi = session
-
- try:
- anchor_lots: list[ScrapedLot] = await scraper.fetch_around(
- _lat,
- _lon,
- radius_m,
- pages=pages_per_anchor,
- delay_override_sec=request_delay_sec,
- )
- except (AvitoBlockedError, AvitoRateLimitedError):
- logger.error(
- "city-sweep run_id=%d: SERP BLOCKED at anchor (%s, %.4f, %.4f)",
- run_id,
- _name,
- _lat,
- _lon,
- )
- raise
-
- counters.lots_fetched += len(anchor_lots)
- if anchor_lots:
- try:
- ins, upd = save_listings(db, anchor_lots)
- counters.lots_inserted += ins
- counters.lots_updated += upd
- except Exception as save_exc:
- logger.exception(
- "city-sweep run_id=%d: save_listings failed at anchor %s: %s",
- run_id,
- _name,
- save_exc,
- )
- counters.errors_count += 1
-
- logger.info(
- "city-sweep run_id=%d anchor %s: SERP fetched=%d ins=%d upd=%d",
- run_id,
- _name,
- len(anchor_lots),
- counters.lots_inserted,
- counters.lots_updated,
- )
- # SERP-счётчики зафиксированы в sweep-level counters.
- # Дальнейший TimeoutError из wait_for их не сотрёт.
-
- # ── Phase 3: group by house ────────────────────────────────
- unique_house_paths: set[str] = set()
- for lot in anchor_lots:
- if lot.house_url:
- try:
- parsed = urlparse(lot.house_url)
- path = parsed.path if parsed.path else lot.house_url
- unique_house_paths.add(path)
- except Exception:
- continue
- if unique_house_paths:
- counters.unique_houses += len(unique_house_paths)
-
- # ── Phase 4: enrich houses ─────────────────────────────────
- touched_house_ids: set[int] = set()
- sweep_consecutive_house_blocks = 0 # #1790: per-anchor счётчик
- if enrich_houses and unique_house_paths:
- house_paths_list = list(unique_house_paths)
- h_idx = 0
- while h_idx < len(house_paths_list):
- house_path = house_paths_list[h_idx]
- try:
- enrichment = await fetch_house_catalog(
- house_path,
- cffi_session=session,
- browser_fetcher=shared_bf,
- )
- hc = save_house_catalog_enrichment(db, enrichment)
- counters.houses_enriched += 1
- sweep_consecutive_house_blocks = 0
- hid = hc.get("house_id")
- if hid:
- touched_house_ids.add(int(hid))
- if h_idx < len(house_paths_list) - 1:
- await asyncio.sleep(request_delay_sec)
- h_idx += 1
- except (AvitoBlockedError, AvitoRateLimitedError) as be:
- # #1790: пробуем сменить IP перед re-raise.
- sweep_consecutive_house_blocks += 1
- counters.houses_failed += 1
- counters.errors_count += 1
- logger.warning(
- "city-sweep run_id=%d: house BLOCKED at %s "
- "(consecutive=%d) — %s",
- run_id,
- house_path,
- sweep_consecutive_house_blocks,
- be,
- )
- if (
- sweep_consecutive_house_blocks
- >= _AVITO_PIPELINE_CONSECUTIVE_BLOCK_ABORT
- and sweep_rotations_done < settings.avito_proxy_max_rotations
- ):
- rotated, _lease = await _rotate_avito_proxy(
- db,
- session,
- _lease,
- reason=(
- f"city-sweep house "
- f"consecutive={sweep_consecutive_house_blocks}"
- ),
- rotations_done=sweep_rotations_done,
- )
- sweep_rotations_done += 1
- if rotated:
- sweep_consecutive_house_blocks = 0
- logger.info(
- "city-sweep run_id=%d: house ROTATE — "
- "proxy changed, reset consecutive "
- "(rotation #%d/%d)",
- run_id,
- sweep_rotations_done,
- settings.avito_proxy_max_rotations,
- )
- # Не инкрементируем h_idx — retry текущего дома
- continue
- elif sweep_consecutive_house_blocks >= 3:
- logger.error(
- "city-sweep run_id=%d: house ABORT — "
- "%d consecutive blocks, propagating",
- run_id,
- sweep_consecutive_house_blocks,
- )
- raise
- h_idx += 1
- except Exception as he:
- logger.warning(
- "city-sweep run_id=%d: house_enrich failed %s: %s",
- run_id,
- house_path,
- he,
- )
- counters.houses_failed += 1
- counters.errors_count += 1
- try:
- db.rollback()
- except Exception:
- pass
- h_idx += 1
-
- # ── Phase 5: enrich detail (top-N) ────────────────────────
- if detail_top_n > 0 and anchor_lots:
- priority_rows = (
- db.execute(
- text("""
- SELECT source_url
- FROM listings
- WHERE source = 'avito'
- AND source_url IS NOT NULL
- AND (
- (
- detail_enriched_at IS NULL
- AND price_rub > 0
- AND ST_DWithin(
- geom::geography,
- ST_MakePoint(:lon, :lat)::geography,
- :radius
- )
- )
- OR (
- detail_enriched_at IS NULL
- AND scraped_at > NOW() - INTERVAL '2 hours'
- )
- )
- ORDER BY scraped_at DESC NULLS LAST
- LIMIT :limit
- """),
- {
- "lat": _lat,
- "lon": _lon,
- "radius": radius_m * 2,
- "limit": detail_top_n,
- },
- )
- .mappings()
- .all()
- )
-
- consecutive_blocks = 0
- consecutive_timeouts = 0
- detail_house_paths: set[str] = set()
- for d_idx, row in enumerate(priority_rows):
- source_url: str = row["source_url"]
- counters.detail_attempted += 1
- try:
- item_url = (
- urlparse(source_url).path
- if source_url.startswith("http")
- else source_url
- )
- enrichment_detail = await fetch_detail(
- item_url,
- cffi_session=session,
- browser_fetcher=shared_bf,
- )
- if save_detail_enrichment(db, enrichment_detail):
- counters.detail_enriched += 1
- if enrichment_detail.house_catalog_url:
- hp = urlparse(enrichment_detail.house_catalog_url).path
- if hp:
- detail_house_paths.add(hp)
- consecutive_blocks = 0
- consecutive_timeouts = 0
- except (AvitoBlockedError, AvitoRateLimitedError) as be:
- consecutive_blocks += 1
- counters.detail_failed += 1
- counters.errors_count += 1
- logger.warning(
- "city-sweep run_id=%d: detail BLOCKED #%d/%d "
- "(consecutive=%d): %s",
- run_id,
- d_idx + 1,
- len(priority_rows),
- consecutive_blocks,
- be,
- )
- if consecutive_blocks >= 3:
- # #1790: попытка ротации перед abort'ом.
- if sweep_rotations_done < settings.avito_proxy_max_rotations:
- rotated, _lease = await _rotate_avito_proxy(
- db,
- session,
- _lease,
- reason=(
- f"city-sweep detail "
- f"consecutive={consecutive_blocks}"
- ),
- rotations_done=sweep_rotations_done,
- )
- sweep_rotations_done += 1
- if rotated:
- consecutive_blocks = 0
- logger.info(
- "city-sweep run_id=%d: detail ROTATE — "
- "proxy changed, reset consecutive "
- "(rotation #%d/%d)",
- run_id,
- sweep_rotations_done,
- settings.avito_proxy_max_rotations,
- )
- continue
- logger.error(
- "city-sweep run_id=%d: detail ABORT — "
- "%d consecutive blocks (IP rate-limited)",
- run_id,
- consecutive_blocks,
- )
- raise
- except Exception as de:
- counters.detail_failed += 1
- counters.errors_count += 1
- consecutive_timeouts += 1
- logger.warning(
- "city-sweep run_id=%d: detail failed #%d/%d "
- "(consecutive_timeouts=%d): %s",
- run_id,
- d_idx + 1,
- len(priority_rows),
- consecutive_timeouts,
- de,
- )
- # (C) Fail-fast: браузер явно деградирует — прерываем
- # detail-фазу, не тратим время на оставшиеся запросы.
- if consecutive_timeouts >= _AVITO_DETAIL_CONSECUTIVE_TIMEOUT_ABORT:
- logger.error(
- "city-sweep run_id=%d: detail ABORT — "
- "%d consecutive timeouts/errors, browser degraded",
- run_id,
- consecutive_timeouts,
- )
- break
- try:
- db.rollback()
- except Exception:
- pass
-
- if d_idx < len(priority_rows) - 1:
- jitter = random.uniform(0.8, 1.2)
- await asyncio.sleep(request_delay_sec * jitter)
-
- # ── Phase 5b: houses найденные через detail-страницы ──
- new_house_paths = detail_house_paths - unique_house_paths
- if enrich_houses and new_house_paths:
- counters.unique_houses += len(new_house_paths)
- nh_list = list(new_house_paths)
- nh_idx = 0
- sweep_consecutive_detail_house_blocks = 0 # #1790
- while nh_idx < len(nh_list):
- house_path = nh_list[nh_idx]
- try:
- enrichment = await fetch_house_catalog(
- house_path,
- cffi_session=session,
- browser_fetcher=shared_bf,
- )
- hc = save_house_catalog_enrichment(db, enrichment)
- counters.houses_enriched += 1
- sweep_consecutive_detail_house_blocks = 0
- hid = hc.get("house_id")
- if hid:
- touched_house_ids.add(int(hid))
- if nh_idx < len(nh_list) - 1:
- await asyncio.sleep(request_delay_sec)
- nh_idx += 1
- except (AvitoBlockedError, AvitoRateLimitedError) as beh:
- # #1790: ротация перед re-raise.
- sweep_consecutive_detail_house_blocks += 1
- counters.houses_failed += 1
- counters.errors_count += 1
- logger.warning(
- "city-sweep run_id=%d: house(detail) BLOCKED %s"
- " (consecutive=%d) — %s",
- run_id,
- house_path,
- sweep_consecutive_detail_house_blocks,
- beh,
- )
- if (
- sweep_consecutive_detail_house_blocks >= 3
- and sweep_rotations_done
- < settings.avito_proxy_max_rotations
- ):
- rotated, _lease = await _rotate_avito_proxy(
- db,
- session,
- _lease,
- reason=(
- "city-sweep house(detail) "
- f"consecutive="
- f"{sweep_consecutive_detail_house_blocks}"
- ),
- rotations_done=sweep_rotations_done,
- )
- sweep_rotations_done += 1
- if rotated:
- sweep_consecutive_detail_house_blocks = 0
- logger.info(
- "city-sweep run_id=%d: house(detail) "
- "ROTATE — proxy changed (rotation #%d/%d)",
- run_id,
- sweep_rotations_done,
- settings.avito_proxy_max_rotations,
- )
- continue
- elif sweep_consecutive_detail_house_blocks >= 3:
- logger.error(
- "city-sweep run_id=%d: house(detail) ABORT"
- " — %d consecutive blocks, propagating",
- run_id,
- sweep_consecutive_detail_house_blocks,
- )
- raise
- nh_idx += 1
- except Exception as he:
- logger.warning(
- "city-sweep run_id=%d: house(detail) failed %s: %s",
- run_id,
- house_path,
- he,
- )
- counters.houses_failed += 1
- counters.errors_count += 1
- try:
- db.rollback()
- except Exception:
- pass
- nh_idx += 1
-
- all_touched_house_ids.update(touched_house_ids)
- logger.info(
- "city-sweep run_id=%d anchor %s done: "
- "lots=%d houses=%d/%d detail=%d/%d touched=%d errors=%d",
- run_id,
- _name,
- len(anchor_lots),
- counters.houses_enriched,
- counters.unique_houses,
- counters.detail_enriched,
- counters.detail_attempted,
- len(touched_house_ids),
- counters.errors_count,
- )
-
- try:
- await asyncio.wait_for(_avito_anchor_phases(), timeout=_avito_anchor_timeout)
- except TimeoutError:
- logger.warning(
- "city-sweep run_id=%d: anchor #%d/%d (%s, %.4f, %.4f) "
- "timed out after %.0fs — SERP counters preserved, "
- "detail phase incomplete",
- run_id,
- idx,
- len(_anchors),
- name,
- lat,
- lon,
- _avito_anchor_timeout,
- )
- counters.errors_count += 1
- except (AvitoBlockedError, AvitoRateLimitedError) as e:
- logger.error(
- "city-sweep run_id=%d ABORT at anchor #%d/%d (%s) — blocked: %s",
- run_id,
- idx,
- len(_anchors),
- name,
- e,
- )
- counters.errors_count += 1
- counters.anchors_done = idx
- scrape_runs.update_heartbeat(db, run_id, counters.to_dict())
- # #1950: если SERP уже собрал лоты и заблокировало только detail/houses,
- # ставим 'done' (не 'banned') — partial intake сохранён.
- # За флагом avito_serp_ok_not_banned (default True).
- _serp_ok = (counters.lots_inserted + counters.lots_updated) > 0
- if settings.avito_serp_ok_not_banned and _serp_ok:
- _note = (
- f"detail enrichment aborted ({e}) но SERP intake завершён: "
- f"{counters.lots_inserted} ins {counters.lots_updated} upd"
- )
- logger.warning(
- "city-sweep run_id=%d: SERP OK (ins=%d upd=%d), "
- "detail blocked — DONE not banned. %s",
- run_id,
- counters.lots_inserted,
- counters.lots_updated,
- _note,
- )
- scrape_runs.mark_done(
- db,
- run_id,
- {**counters.to_dict(), "enrichment_abort_note": _note}, # type: ignore[arg-type]
- )
- else:
- scrape_runs.mark_banned(db, run_id, str(e), counters.to_dict())
- return counters
- except Exception:
- logger.exception("city-sweep run_id=%d: anchor %s failed", run_id, name)
- counters.errors_count += 1
-
- counters.anchors_done = idx
- scrape_runs.update_heartbeat(db, run_id, counters.to_dict())
-
- # ── IMV-фаза: финальный обход тронутых домов ──────────
- if enrich_imv and all_touched_house_ids:
- if scrape_runs.is_cancelled(db, run_id):
- logger.info(
- "city-sweep run_id=%d: cancelled before IMV phase (%d houses skipped)",
- run_id,
- len(all_touched_house_ids),
- )
- scrape_runs.mark_done(db, run_id, counters.to_dict())
- return counters
- elif shutdown_requested():
- # SIGTERM-drain до IMV-фазы: финализируем без дорогой IMV-оценки.
- logger.info(
- "city-sweep run_id=%d: SIGTERM-drain — skipping IMV phase "
- "(%d houses skipped)",
- run_id,
- len(all_touched_house_ids),
- )
- scrape_runs.mark_done(db, run_id, counters.to_dict())
- return counters
-
- logger.info(
- "city-sweep run_id=%d: IMV phase — %d touched houses",
- run_id,
- len(all_touched_house_ids),
- )
- try:
- # Периодический heartbeat внутри длинной IMV-фазы (#1363):
- # без него десятки/сотни домов × request_delay_sec проходят
- # без update_heartbeat, и reap_zombies помечает живой run
- # 'zombie' → последующий mark_done становится no-op (дубль-sweep).
- def _imv_heartbeat() -> None:
- scrape_runs.update_heartbeat(db, run_id, counters.to_dict())
-
- imv_result = await process_houses_imv_batch(
- db,
- all_touched_house_ids,
- request_delay_sec=request_delay_sec,
- heartbeat=_imv_heartbeat,
- )
- counters.imv_attempted += imv_result.checked
- counters.imv_enriched += imv_result.saved
- counters.imv_failed += imv_result.errors
- counters.errors_count += imv_result.errors
- scrape_runs.update_heartbeat(db, run_id, counters.to_dict())
- logger.info(
- "city-sweep run_id=%d: IMV phase done — attempted=%d enriched=%d failed=%d",
- run_id,
- imv_result.checked,
- imv_result.saved,
- imv_result.errors,
- )
- except Exception as exc:
- logger.error(
- "city-sweep run_id=%d: IMV phase crashed — %r (sweep still done)",
- run_id,
- exc,
- )
- counters.errors_count += 1
- # #1521: IMV-фаза могла оставить Session в aborted-state
- # (необёрнутый eligibility SELECT / _mark_status на отравленной
- # сессии). update_heartbeat и mark_done — единственные финализаторы
- # БЕЗ defensive rollback, поэтому сбрасываем txn-state сами. Иначе
- # PendingRollbackError пролетает мимо внутреннего try во внешний
- # except (mark_failed) → успешный sweep ложно помечается 'failed'.
- try:
- db.rollback()
- except Exception:
- pass
- scrape_runs.update_heartbeat(db, run_id, counters.to_dict())
-
- scrape_runs.mark_done(db, run_id, counters.to_dict())
- logger.info(
- "city-sweep run_id=%d done: anchors=%d/%d lots=%d (ins=%d/upd=%d) "
- "houses=%d/%d detail=%d/%d imv=%d/%d errors=%d",
- run_id,
- counters.anchors_done,
- counters.anchors_total,
- counters.lots_fetched,
- counters.lots_inserted,
- counters.lots_updated,
- counters.houses_enriched,
- counters.unique_houses,
- counters.detail_enriched,
- counters.detail_attempted,
- counters.imv_enriched,
- counters.imv_attempted,
- counters.errors_count,
- )
- return counters
-
- except Exception as exc:
- logger.exception("city-sweep run_id=%d: fatal error", run_id)
- scrape_runs.mark_failed(db, run_id, str(exc), counters.to_dict())
- raise
- finally:
- # #2202: best-effort release+mark_health — не должен ронять уже
- # собранный sweep. ok=False только если весь run завершился с ошибками.
- _release_pool_lease(db, _lease, ok=(counters.errors_count == 0))
-
-
-# ── Avito newbuilding (novostroyka) citywide sweep ────────────────
-
-
-@dataclass
-class NewbuildingSweepCounters:
- """Aggregate counters для Avito novostroyka citywide sweep run."""
-
- lots_fetched: int = 0
- lots_inserted: int = 0
- lots_updated: int = 0
- errors_count: int = 0
-
- def to_dict(self) -> dict[str, int]:
- return {f.name: getattr(self, f.name) for f in fields(self)}
-
-
-async def run_avito_newbuilding_sweep(
- db: Session,
- *,
- run_id: int,
- pages: int = 20,
- request_delay_sec: float = 7.0,
-) -> NewbuildingSweepCounters:
- """Citywide-обход ЕКБ-выборки только новостроек (novostroyka-filter) → save.
-
- В отличие от run_avito_city_sweep (anchor-based fetch_around + houses/detail/IMV),
- это простой citywide paginated-обход через AvitoScraper.fetch_newbuildings:
- dedicated novostroyka-SERP отдаёт 100% new-build карточки (listing_segment=
- "novostroyki", newbuilding_id/newbuilding_url заполнены парсером). Сохраняем через
- стандартный save_listings, ведём SERP-счётчики через scrape_runs.
-
- - Single shared AsyncSession/BrowserFetcher на весь sweep (один TLS fingerprint)
- - AvitoBlockedError/AvitoRateLimitedError → mark_banned (status='banned')
- - is_cancelled-чек до старта SERP-фазы
- """
- counters = NewbuildingSweepCounters()
-
- browser_mode = settings.scraper_fetch_mode == "browser"
- # #2202: pool-lease текущей curl-сессии (None в browser-mode/pool off).
- _lease: proxy_pool.ProxyLease | None = None
- async with AsyncExitStack() as stack:
- session: AsyncSession | None = None
- shared_bf: BrowserFetcher | None = None
- if browser_mode:
- shared_bf = await stack.enter_async_context(BrowserFetcher(source="avito"))
- else:
- _proxies, _lease = _pool_proxy_dict(db, "avito", run_id=run_id)
- session = await stack.enter_async_context(
- AsyncSession(
- impersonate="chrome120",
- timeout=25,
- headers=_CHROME_HEADERS,
- proxies=_proxies,
- )
- )
- try:
- if scrape_runs.is_cancelled(db, run_id):
- logger.info("nb-sweep run_id=%d: cancelled before SERP phase", run_id)
- scrape_runs.mark_done(db, run_id, counters.to_dict())
- return counters
- elif shutdown_requested():
- logger.info(
- "nb-sweep run_id=%d: SIGTERM-drain — stopping before SERP phase", run_id
- )
- scrape_runs.update_heartbeat(db, run_id, counters.to_dict())
- scrape_runs.mark_done(db, run_id, counters.to_dict())
- return counters
-
- scraper = AvitoScraper()
- if browser_mode:
- scraper._browser = shared_bf
- # Shared-browser режим: _cffi=None → curl_cffi-fallback на firewall
- # пропускается (by design). Логируем для observability.
- logger.debug("avito pipeline-mode: shared browser only, no cffi fallback")
- else:
- scraper._cffi = session
-
- try:
- lots: list[ScrapedLot] = await scraper.fetch_newbuildings(
- pages=pages,
- delay_override_sec=request_delay_sec,
- )
- except (AvitoBlockedError, AvitoRateLimitedError) as e:
- logger.error("nb-sweep run_id=%d: SERP BLOCKED — %s", run_id, e)
- scrape_runs.mark_banned(db, run_id, str(e), counters.to_dict())
- return counters
-
- counters.lots_fetched += len(lots)
- if lots:
- try:
- ins, upd = save_listings(db, lots)
- counters.lots_inserted += ins
- counters.lots_updated += upd
- except Exception as save_exc:
- logger.exception(
- "nb-sweep run_id=%d: save_listings failed: %s", run_id, save_exc
- )
- counters.errors_count += 1
- try:
- db.rollback()
- except Exception:
- pass
-
- scrape_runs.update_heartbeat(db, run_id, counters.to_dict())
- scrape_runs.mark_done(db, run_id, counters.to_dict())
- logger.info(
- "nb-sweep run_id=%d done: lots=%d (ins=%d/upd=%d) errors=%d",
- run_id,
- counters.lots_fetched,
- counters.lots_inserted,
- counters.lots_updated,
- counters.errors_count,
- )
- return counters
-
- except Exception as exc:
- logger.exception("nb-sweep run_id=%d: fatal error", run_id)
- scrape_runs.mark_failed(db, run_id, str(exc), counters.to_dict())
- raise
- finally:
- _release_pool_lease(db, _lease, ok=(counters.errors_count == 0))
-
-
-# ── Yandex city sweep ───────────────────────────────────────────
-
-
-@dataclass
-class YandexCitySweepCounters:
- """Aggregate counters для Yandex.Недвижимость city sweep run.
-
- Фазы: SERP (fetch_around_multi_room) → save_listings инкрементально → address-enrich.
- address_* — результат T10-обогащения (detail → полный адрес с номером дома).
- combos_skipped — сколько (room×price) combo пропущено из-за таймаутов gate-API.
- """
-
- anchors_total: int = 0
- anchors_done: int = 0
- lots_fetched: int = 0
- lots_inserted: int = 0
- lots_updated: int = 0
- price_history_rows: int = 0
- combos_skipped: int = 0
- address_attempted: int = 0
- address_enriched: int = 0
- address_failed: int = 0
- errors_count: int = 0
-
- def to_dict(self) -> dict[str, int]:
- return {f.name: getattr(self, f.name) for f in fields(self)}
-
-
-# Defensive abort threshold: у YandexRealtyScraper нет block/rate-exception классов
-# (fetch_around глотает ошибки и возвращает []). Чтобы не молотить Yandex впустую при
-# системном блоке/бане IP, прерываем sweep после N подряд anchor'ов, упавших с
-# исключением. NB: пустой результат (0 lots) — это НЕ ошибка, счётчик не растёт.
-YANDEX_SWEEP_MAX_CONSECUTIVE_FAILURES = 3
-
-
-async def run_yandex_city_sweep(
- db: Session,
- *,
- run_id: int,
- anchors: list[tuple[float, float, str]] | None = None,
- radius_m: int = 25000,
- pages_per_anchor: int = 2,
- request_delay_sec: float | None = None,
- enrich_address: bool = True,
- rooms_list: list[str] | None = None,
- price_ranges: list[tuple[int | None, int | None]] | None = None,
- segments: list[str] | None = None,
-) -> YandexCitySweepCounters:
- """Yandex.Недвижимость city sweep: rooms × price combos от центра ЕКБ → save → address-enrich.
-
- Вместо 5 географических anchor'ов итерирует ВСЕ combinations (room × price_range)
- из одной центральной точки ЕКБ (56.8400, 60.6050). radius_m=25000 охватывает весь
- город; price/room segmentation обходит Yandex SERP ~575-card cap без geo-шрапнели.
-
- Фазы (единственный центральный якорь):
- 1. SERP: YandexRealtyScraper.fetch_around_multi_room(lat, lon, radius_m,
- max_pages=pages_per_anchor, rooms_list=..., price_ranges=...) — combos mode.
- 2. SAVE: save_listings(db, lots, run_id=run_id).
- 3. ADDRESS-ENRICH (если enrich_address=True): для yandex-листингов без номера
- дома в address (address IS NOT NULL AND NOT (address ~ ',\\s*\\d+'))
- запрашиваем detail-страницу и извлекаем полный адрес из <title>.
-
- Параметр `anchors` сохранён для backward-compat тестов (anchors=[...]). В prod
- (anchors=None) всегда используется один EKB_CENTER anchor.
-
- Anti-bot / cancel семантика:
- - Cooperative cancel: is_cancelled(db, run_id) проверяется перед фазой SERP.
- - YandexRealtyScraper не propagate'ит block/rate exceptions (глотает → []).
- Ловим broad Exception; при N подряд — mark_banned + abort.
- - request_delay_sec: inter-request delay для address-enrich (поверх per-page
- sleep внутри scraper'а).
-
- Возвращает YandexCitySweepCounters.
- mark_done вызывается ВСЕГДА (кроме cancel / ban / fatal).
- """
- from sqlalchemy import text as _text
-
- from app.services.scrapers.yandex_realty import DEFAULT_PRICE_RANGES, ROOM_PATH
- from app.services.yandex_address_backfill import (
- _RE_HAS_HOUSE_NUMBER,
- _extract_address_from_title,
- )
-
- # EKB center — единственный anchor для citywide combos sweep.
- # anchors-параметр принимается для backward-compat тестов.
- if anchors is not None:
- _anchors: list[tuple[float, float, str]] = anchors
- else:
- _anchors = [(56.8400, 60.6050, "Центр (combos)")]
-
- _rooms_list = rooms_list or list(ROOM_PATH.keys())
- _price_ranges = price_ranges or DEFAULT_PRICE_RANGES
- # newFlat-сегменты: оба прохода (vtorichka + novostroyki) по одним combos.
- # Дефолт ["NO", "YES"] — оба; dedup по source_id span'ит оба прохода (один seen
- # внутри fetch_around_multi_room). Counters (lots_fetched/inserted/updated)
- # агрегируют оба прохода через len(anchor_lots).
- _segments = segments or ["NO", "YES"]
-
- counters = YandexCitySweepCounters(anchors_total=len(_anchors))
- inter_anchor_delay = request_delay_sec if request_delay_sec is not None else 7.0
- enrich_delay = request_delay_sec if request_delay_sec is not None else 3.0
- _resolved_delay = request_delay_sec if request_delay_sec is not None else 9.0
- consecutive_failures = 0
- yandex_rotations_done = 0 # #1848: бюджет IP-ротаций на весь sweep
-
- # Вычисляем watchdog-таймаут для combos-режима (центр, anchors=None).
- # В combos-режиме один "anchor" выполняет num_segments × num_combos × max_pages
- # fetch'ей — каждый занимает примерно _resolved_delay + _YANDEX_COMBOS_PER_FETCH_S сек.
- # Для 2 segments × 30 combos × 3 pages × (9+12) + 300 ≈ 4080s (~68 мин).
- # Множитель len(_segments) масштабирует watchdog пропорционально числу проходов
- # (vtorichka + novostroyki ≈ ×2 wall-clock) — иначе второй проход гильотинится.
- # В explicit-anchor (тест/override) режиме оставляем ANCHOR_TIMEOUT_SEC (240s) —
- # там каждый anchor небольшой и watchdog работает как старый защитный барьер.
- _num_combos = len(_rooms_list) * len(_price_ranges)
- if anchors is None and _num_combos > 0:
- _sweep_timeout = max(
- ANCHOR_TIMEOUT_SEC,
- len(_segments)
- * _num_combos
- * pages_per_anchor
- * (_resolved_delay + _YANDEX_COMBOS_PER_FETCH_S)
- + _YANDEX_ADDRESS_ENRICH_BUDGET_S,
- )
- else:
- # Explicit anchors (backward-compat тесты или ручной override) — legacy timeout.
- _sweep_timeout = ANCHOR_TIMEOUT_SEC
-
- # Прокси для address-enrich curl_cffi сессии (зеркало yandex_address_backfill).
- _proxy_url = settings.scraper_proxy_url
- _proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None
-
- try:
- for idx, (lat, lon, name) in enumerate(_anchors, start=1):
- # ── Cooperative cancel ───────────────────────────────────────────
- if scrape_runs.is_cancelled(db, run_id):
- logger.info(
- "yandex-sweep run_id=%d: cancelled at anchor #%d/%d (%s)",
- run_id,
- idx,
- len(_anchors),
- name,
- )
- scrape_runs.update_heartbeat(db, run_id, counters.to_dict())
- return counters
- elif shutdown_requested():
- # SIGTERM-drain: чистый стоп на границе anchor'а → mark_done(partial).
- logger.info(
- "yandex-sweep run_id=%d: SIGTERM-drain — stopping at anchor #%d/%d (%s)",
- run_id,
- idx,
- len(_anchors),
- name,
- )
- scrape_runs.update_heartbeat(db, run_id, counters.to_dict())
- scrape_runs.mark_done(db, run_id, counters.to_dict())
- return counters
-
- logger.info(
- "yandex-sweep run_id=%d center-combos #%d/%d (%s, %.4f, %.4f) "
- "radius_m=%d rooms=%d price_ranges=%d",
- run_id,
- idx,
- len(_anchors),
- name,
- lat,
- lon,
- radius_m,
- len(_rooms_list),
- len(_price_ranges),
- )
-
- # ── Per-anchor phases под watchdog-таймаутом ────────────────────
- # Весь SERP + address-enrich одного anchor'а оборачиваем в wait_for.
- # TimeoutError → log + errors_count++ + continue (sweep не прерывается).
-
- anchor_lots: list[ScrapedLot] = []
- _anchor_timed_out = False
- # Capture loop variables in default args to satisfy B023 (flake8/ruff):
- # без этого inner async def захватывает изменяемые переменные цикла
- # по ссылке → неправильные значения при asyncio scheduling.
- _lat, _lon, _name = lat, lon, name
-
- # _combo_lots_accumulator: per-anchor мутируемый контейнер для on_combo.
- # Захватывается по ссылке и в _yandex_anchor_phases и в _on_combo —
- # избегаем B023 (loop-variable capture) через явный bind в default-arg.
- _combo_lots_accumulator: list[ScrapedLot] = anchor_lots
-
- async def _yandex_anchor_phases(
- _a_lat: float = _lat,
- _a_lon: float = _lon,
- _a_name: str = _name,
- _al: list[ScrapedLot] = _combo_lots_accumulator,
- ) -> None:
- """Все фазы одного anchor'а (SERP + address-enrich).
-
- FIX: инкрементальный save per-combo — on_combo вызывается scraper'ом после
- каждого (room×price) combo, сохраняет порцию сразу в БД + обновляет heartbeat.
- anchor_lots накапливается (через _al) для address-enrich и price-history.
- """
- nonlocal consecutive_failures, yandex_rotations_done
-
- # ── Phase 1+2: SERP + инкрементальный save per-combo ──────
- def _on_combo(
- combo_label: str,
- new_lots: list[ScrapedLot],
- _accumulator: list[ScrapedLot] = _al,
- ) -> None:
- """Callback: вызывается scraper'ом после каждого combo с новыми лотами.
-
- Сохраняет порцию новых лотов немедленно (анти-зомби: данные в БД
- до конца anchor'а) и обновляет heartbeat для мониторинга прогресса.
- """
- _accumulator.extend(new_lots)
- counters.lots_fetched += len(new_lots)
- try:
- ins, upd = save_listings(db, new_lots, run_id=run_id)
- counters.lots_inserted += ins
- counters.lots_updated += upd
- except Exception as save_exc:
- counters.errors_count += 1
- logger.warning(
- "yandex-sweep run_id=%d combo %s: save_listings failed: %s",
- run_id,
- combo_label,
- save_exc,
- )
- try:
- db.rollback()
- except Exception:
- pass
- scrape_runs.update_heartbeat(db, run_id, counters.to_dict())
- logger.debug(
- "yandex-sweep run_id=%d combo %s: saved %d lots "
- "(ins=%d upd=%d total_fetched=%d)",
- run_id,
- combo_label,
- len(new_lots),
- counters.lots_inserted,
- counters.lots_updated,
- counters.lots_fetched,
- )
-
- async with YandexRealtyScraper() as scraper:
- await scraper.fetch_around_multi_room(
- _a_lat,
- _a_lon,
- radius_m,
- max_pages=pages_per_anchor,
- rooms_list=_rooms_list,
- price_ranges=_price_ranges,
- segments=_segments,
- on_combo=_on_combo,
- )
- # _al накоплен on_combo; save уже вызван per-combo.
-
- # Price-history из gate price.previous/trend (graceful — провал
- # истории НЕ должен ронять сейв листингов).
- if _al:
- try:
- counters.price_history_rows += record_yandex_price_history(db, _al)
- except Exception as ph_exc:
- logger.warning(
- "yandex-sweep run_id=%d: price-history failed: %s",
- run_id,
- ph_exc,
- )
- try:
- db.rollback()
- except Exception:
- pass
-
- consecutive_failures = 0
- logger.info(
- "yandex-sweep run_id=%d center-combos %s: fetched=%d ins=%d upd=%d",
- run_id,
- _a_name,
- counters.lots_fetched,
- counters.lots_inserted,
- counters.lots_updated,
- )
-
- # ── Phase 3: address-enrich ────────────────────────────────
- if not (enrich_address and _al):
- return
- source_ids = [lot.source_id for lot in _al if lot.source_id]
- if not source_ids:
- return
- id_rows = (
- db.execute(
- _text("""
- SELECT id, source_id, address, source_url
- FROM listings
- WHERE source = 'yandex'
- AND source_id = ANY(CAST(:ids AS text[]))
- AND address IS NOT NULL
- AND NOT (address ~ ',\\s*\\d+')
- AND source_url IS NOT NULL
- """),
- {"ids": source_ids},
- )
- .mappings()
- .all()
- )
- items = [dict(r) for r in id_rows]
- counters.address_attempted += len(items)
-
- if items:
- from curl_cffi.requests import AsyncSession as _AsyncSession
-
- # #1848: rotation tracking для Yandex address-enrich (stuck-proxy)
- _yandex_enrich_consec_fail = 0
- _yandex_enrich_abort = 3 # abort address-enrich после N подряд
-
- async with _AsyncSession(
- impersonate="chrome120",
- timeout=30.0,
- proxies=_proxies,
- headers={"Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8"},
- ) as enrich_session:
- eidx = 0
- while eidx < len(items):
- item = items[eidx]
- lid: int = item["id"]
- old_addr: str = item["address"]
- url: str = item["source_url"]
- try:
- resp = await enrich_session.get(url, allow_redirects=True)
- if resp.status_code not in {200}:
- logger.warning(
- "yandex-sweep run_id=%d address-enrich:"
- " HTTP %d listing_id=%d",
- run_id,
- resp.status_code,
- lid,
- )
- counters.address_failed += 1
- # 503/429 = прокси заблокирован → считаем как failure
- if resp.status_code in {429, 503}:
- _yandex_enrich_consec_fail += 1
- else:
- _yandex_enrich_consec_fail = 0
- else:
- new_addr = _extract_address_from_title(resp.text)
- if (
- new_addr is None
- or new_addr == old_addr
- or not _RE_HAS_HOUSE_NUMBER.search(new_addr)
- ):
- pass
- else:
- try:
- db.execute(
- _text("""
- UPDATE listings
- SET address = :addr,
- geocode_tried_at = NULL
- WHERE id = CAST(:id AS bigint)
- """),
- {"addr": new_addr, "id": lid},
- )
- db.commit()
- counters.address_enriched += 1
- logger.info(
- "yandex-sweep run_id=%d: "
- "address updated listing_id=%d "
- "%r -> %r",
- run_id,
- lid,
- old_addr,
- new_addr,
- )
- except Exception as upd_exc:
- counters.address_failed += 1
- counters.errors_count += 1
- logger.warning(
- "yandex-sweep run_id=%d: "
- "address DB update failed "
- "listing_id=%d: %s",
- run_id,
- lid,
- upd_exc,
- )
- try:
- db.rollback()
- except Exception:
- pass
- _yandex_enrich_consec_fail = 0
- except Exception as fetch_exc:
- counters.address_failed += 1
- _yandex_enrich_consec_fail += 1
- logger.warning(
- "yandex-sweep run_id=%d: address-enrich "
- "fetch failed listing_id=%d "
- "(consecutive=%d): %s",
- run_id,
- lid,
- _yandex_enrich_consec_fail,
- fetch_exc,
- )
- # Ротация Yandex-прокси при stuck/ban (#1848)
- if _yandex_enrich_consec_fail >= _yandex_enrich_abort:
- if yandex_rotations_done < settings.yandex_proxy_max_rotations:
- rotated = await _rotate_proxy_ip(
- reason=(
- f"yandex enrich consecutive="
- f"{_yandex_enrich_consec_fail}"
- ),
- rotations_done=yandex_rotations_done,
- source="yandex",
- )
- yandex_rotations_done += 1
- if rotated:
- _yandex_enrich_consec_fail = 0
- logger.info(
- "yandex-sweep run_id=%d: enrich ROTATE — "
- "IP changed, reset consecutive "
- "(rotation #%d/%d)",
- run_id,
- yandex_rotations_done,
- settings.yandex_proxy_max_rotations,
- )
- # retry текущего item после ротации
- continue
- logger.error(
- "yandex-sweep run_id=%d: address-enrich ABORT — "
- "%d consecutive failures (proxy likely stuck/banned)",
- run_id,
- _yandex_enrich_consec_fail,
- )
- break
- if eidx < len(items) - 1:
- await asyncio.sleep(enrich_delay)
- eidx += 1
-
- logger.info(
- "yandex-sweep run_id=%d anchor %s: address enrich=%d/%d failed=%d",
- run_id,
- _a_name,
- counters.address_enriched,
- counters.address_attempted,
- counters.address_failed,
- )
-
- try:
- await asyncio.wait_for(_yandex_anchor_phases(), timeout=_sweep_timeout)
- except TimeoutError:
- logger.warning(
- "yandex-sweep run_id=%d: anchor #%d/%d (%s, %.4f, %.4f) "
- "timed out after %.0fs — skipping",
- run_id,
- idx,
- len(_anchors),
- name,
- lat,
- lon,
- _sweep_timeout,
- )
- counters.errors_count += 1
- consecutive_failures += 1
- _anchor_timed_out = True
- if consecutive_failures >= YANDEX_SWEEP_MAX_CONSECUTIVE_FAILURES:
- logger.error(
- "yandex-sweep run_id=%d ABORT at anchor #%d/%d (%s) — "
- "%d consecutive failures (last: timeout): "
- "IP likely blocked or proxy hung",
- run_id,
- idx,
- len(_anchors),
- name,
- consecutive_failures,
- )
- counters.anchors_done = idx
- scrape_runs.update_heartbeat(db, run_id, counters.to_dict())
- scrape_runs.mark_banned(
- db,
- run_id,
- f"yandex sweep aborted: {consecutive_failures} consecutive "
- f"anchor failures (last: anchor timeout {_sweep_timeout:.0f}s)",
- counters.to_dict(),
- )
- return counters
- counters.anchors_done = idx
- scrape_runs.update_heartbeat(db, run_id, counters.to_dict())
- if idx < len(_anchors):
- await asyncio.sleep(inter_anchor_delay)
- continue
- except Exception as e:
- # YandexRealtyScraper не propagate'ит block/rate exceptions — ловим broad.
- logger.exception("yandex-sweep run_id=%d: anchor %s SERP failed", run_id, name)
- counters.errors_count += 1
- consecutive_failures += 1
- if consecutive_failures >= YANDEX_SWEEP_MAX_CONSECUTIVE_FAILURES:
- logger.error(
- "yandex-sweep run_id=%d ABORT at anchor #%d/%d (%s) — "
- "%d consecutive failures (IP likely blocked): %s",
- run_id,
- idx,
- len(_anchors),
- name,
- consecutive_failures,
- e,
- )
- counters.anchors_done = idx
- scrape_runs.update_heartbeat(db, run_id, counters.to_dict())
- scrape_runs.mark_banned(
- db,
- run_id,
- f"yandex sweep aborted: {consecutive_failures} consecutive "
- f"anchor failures (last: {e})",
- counters.to_dict(),
- )
- return counters
- counters.anchors_done = idx
- scrape_runs.update_heartbeat(db, run_id, counters.to_dict())
- # Пауза перед следующим anchor'ом даже при ошибке.
- if idx < len(_anchors):
- await asyncio.sleep(inter_anchor_delay)
- continue
-
- counters.anchors_done = idx
- scrape_runs.update_heartbeat(db, run_id, counters.to_dict())
-
- # Доп. пауза между anchor'ами (поверх per-page sleep внутри scraper'а).
- if idx < len(_anchors):
- await asyncio.sleep(inter_anchor_delay)
-
- scrape_runs.mark_done(db, run_id, counters.to_dict())
- logger.info(
- "yandex-sweep run_id=%d done: anchors=%d/%d lots=%d (ins=%d/upd=%d) "
- "address=%d/%d failed=%d errors=%d",
- run_id,
- counters.anchors_done,
- counters.anchors_total,
- counters.lots_fetched,
- counters.lots_inserted,
- counters.lots_updated,
- counters.address_enriched,
- counters.address_attempted,
- counters.address_failed,
- counters.errors_count,
- )
- return counters
-
- except Exception as exc:
- logger.exception("yandex-sweep run_id=%d: fatal error", run_id)
- scrape_runs.mark_failed(db, run_id, str(exc), counters.to_dict())
- raise
-
-
-# ── Cian city sweep (#860) ──────────────────────────────────────────────────
-
-
-@dataclass
-class CianCitySweepCounters:
- """Aggregate counters для Cian city sweep run.
-
- Поля симметричны CitySweepCounters, но без imv_* (у Cian нет IMV-фазы).
- detail_* — обогащение через cian_detail (вкл. price-history).
- houses_* — обогащение newbuilding/ЖК через cian_newbuilding.
- """
-
- anchors_total: int = 0
- anchors_done: int = 0
- lots_fetched: int = 0
- lots_dropped_secondary: int = 0 # вторичка, отброшенная при newbuilding_only=True
- lots_inserted: int = 0
- lots_updated: int = 0
- detail_attempted: int = 0
- detail_enriched: int = 0
- detail_failed: int = 0
- houses_attempted: int = 0
- houses_enriched: int = 0
- houses_failed: int = 0
- errors_count: int = 0
-
- def to_dict(self) -> dict[str, int]:
- return {f.name: getattr(self, f.name) for f in fields(self)}
-
-
-# Defensive abort threshold: если N подряд anchor'ов упали с исключением
-# (не просто вернули пустой результат) — прерываем sweep.
-CIAN_SWEEP_MAX_CONSECUTIVE_FAILURES = 3
-
-
-def _mark_cian_sweep_abort(
- db: Session,
- run_id: int,
- counters: CianCitySweepCounters,
- reason: str,
-) -> None:
- """#1949: честный финальный статус при defensive abort run_cian_city_sweep.
-
- Зеркалит avito_serp_ok_not_banned (#1950): если SERP-фаза уже сохранила лоты
- (lots_inserted + lots_updated > 0) до момента abort'а, основной сбор состоялся —
- 'banned' лишний шум в мониторинге. Ставим 'done' с заметкой о причине abort'а.
- Если лотов нет (SERP сам заблокирован/завис на первом же anchor'е) — 'banned'
- как раньше. За флагом settings.cian_sweep_ok_not_banned (default True).
- """
- _serp_ok = (counters.lots_inserted + counters.lots_updated) > 0
- if settings.cian_sweep_ok_not_banned and _serp_ok:
- logger.warning(
- "cian-sweep run_id=%d: SERP OK (ins=%d upd=%d) before abort — " "DONE not banned. %s",
- run_id,
- counters.lots_inserted,
- counters.lots_updated,
- reason,
- )
- scrape_runs.mark_done(
- db,
- run_id,
- {**counters.to_dict(), "abort_note": reason}, # type: ignore[arg-type]
- )
- else:
- scrape_runs.mark_banned(db, run_id, reason, counters.to_dict())
-
-
-async def run_cian_city_sweep(
- db: Session,
- *,
- run_id: int,
- anchors: list[tuple[float, float, str]] | None = None,
- radius_m: int = 1500,
- pages_per_anchor: int = 3,
- request_delay_sec: float = 5.0,
- detail_top_n: int = 10,
- enrich_houses: bool = True,
- newbuilding_only: bool = True,
-) -> CianCitySweepCounters:
- """Cian newbuilding city sweep: SERP → detail(+price-history) → newbuilding/houses.
-
- Симметрично run_avito_city_sweep. Использует EKB_ANCHORS (общий список).
-
- newbuilding_only (default True): в SERP-фазе сохраняются только лоты с
- listing_segment == "novostroyki" — вторичку отбрасываем, ею авторитетно владеет
- run_cian_full_load (exhaustive региональный сбор). Зеркало secondary_only в
- fetch_all_secondary, которое фильтрует обратное (!= "novostroyki"). DETAIL-фаза
- (обогащение cian-листингов без detail_enriched_at) и HOUSES-фаза (newbuilding-enrich)
- не затронуты фильтром.
-
- Фазы per anchor:
- 1. SERP: CianScraper.fetch_around_multi_room(lat, lon, radius_m, pages=pages_per_anchor)
- 2. FILTER (newbuilding_only): отбросить вторичку (listing_segment != "novostroyki")
- 3. SAVE: save_listings(db, lots, run_id=run_id)
- 4. DETAIL: top-N свежих cian-листингов без detail_enriched_at →
- fetch_detail(source_url) + save_detail_enrichment(db, listing_id, enrichment)
- (save_detail_enrichment уже пишет price_changes → offer_price_history)
- 5. HOUSES (если enrich_houses=True): для домов из этого anchor'а с cian_zhk_url →
- fetch_newbuilding(zhk_url) + save_newbuilding_enrichment(db, house_id, enr)
-
- Anti-bot:
- - CianScraper управляет своей curl_cffi-сессией (прокси уже внутри).
- - asyncio.sleep(request_delay_sec) между detail-запросами (jitter ±20%).
- - Defensive abort при CIAN_SWEEP_MAX_CONSECUTIVE_FAILURES SERP-ошибок подряд →
- _mark_cian_sweep_abort (#1949): 'done' если SERP уже сохранил лоты, иначе 'banned'.
-
- Cooperative cancel: scrape_runs.is_cancelled(db, run_id) перед каждым anchor.
- mark_done вызывается ВСЕГДА (finally outer).
- """
- from app.services.scrapers.cian import CianScraper
- from app.services.scrapers.cian_detail import fetch_detail, save_detail_enrichment
- from app.services.scrapers.cian_newbuilding import (
- fetch_newbuilding,
- save_newbuilding_enrichment,
- )
-
- _anchors = anchors if anchors is not None else EKB_ANCHORS
- counters = CianCitySweepCounters(anchors_total=len(_anchors))
- consecutive_failures = 0
- cian_rotations_done = 0 # #1848: бюджет IP-ротаций на весь sweep
-
- # #2160: масштабируемый watchdog-таймаут для cian anchor'а. При proxy-pool browser каждый
- # SERP-фетч 13-45s (camoufox relaunch), фиксированный ANCHOR_TIMEOUT_SEC=240 гильотинит
- # якорь в SERP-фазе. См. _cian_anchor_timeout_s + константы выше.
- _cian_anchor_timeout = _cian_anchor_timeout_s(
- pages_per_anchor, request_delay_sec, detail_top_n, enrich_houses
- )
- logger.info(
- "cian-sweep run_id=%d: anchor_timeout=%.0fs "
- "(base=%d, serp=%d×%d×(%.0f+%.0f)s + detail_top_n=%d×%.0fs + houses=%.0fs)",
- run_id,
- _cian_anchor_timeout,
- ANCHOR_TIMEOUT_SEC,
- _CIAN_ROOM_BUCKETS,
- pages_per_anchor,
- request_delay_sec,
- _CIAN_PER_SERP_FETCH_S,
- detail_top_n,
- _CIAN_PER_DETAIL_S,
- _CIAN_HOUSES_BUDGET_S if enrich_houses else 0.0,
- )
-
- try:
- for idx, (lat, lon, name) in enumerate(_anchors, start=1):
- # Cooperative cancel перед каждым anchor
- if scrape_runs.is_cancelled(db, run_id):
- logger.info(
- "cian-sweep run_id=%d: cancelled at anchor #%d/%d (%s)",
- run_id,
- idx,
- len(_anchors),
- name,
- )
- scrape_runs.update_heartbeat(db, run_id, counters.to_dict())
- return counters
- elif shutdown_requested():
- # SIGTERM-drain: чистый стоп на границе anchor'а → mark_done(partial).
- logger.info(
- "cian-sweep run_id=%d: SIGTERM-drain — stopping at anchor #%d/%d (%s)",
- run_id,
- idx,
- len(_anchors),
- name,
- )
- scrape_runs.update_heartbeat(db, run_id, counters.to_dict())
- scrape_runs.mark_done(db, run_id, counters.to_dict())
- return counters
-
- logger.info(
- "cian-sweep run_id=%d anchor #%d/%d (%s, %.4f, %.4f)",
- run_id,
- idx,
- len(_anchors),
- name,
- lat,
- lon,
- )
-
- # ── Per-anchor phases под watchdog-таймаутом ────────────────────
- # Весь SERP + detail + houses одного anchor'а оборачиваем в wait_for.
- # TimeoutError → log + errors_count++ + continue (sweep не прерывается).
-
- anchor_lots: list[ScrapedLot] = []
- # Capture loop variables in default args (B023): prevents stale binding
- # if the coroutine is scheduled after the loop variable changes.
- _c_lat, _c_lon, _c_name = lat, lon, name
-
- async def _cian_anchor_phases(
- _a_lat: float = _c_lat,
- _a_lon: float = _c_lon,
- _a_name: str = _c_name,
- ) -> None:
- """Все фазы одного cian anchor'а (SERP + detail + houses)."""
- nonlocal anchor_lots, consecutive_failures, cian_rotations_done
- from sqlalchemy import text as _text
-
- # ── Phase 1+2: SERP + save ─────────────────────────────────
- async with CianScraper() as scraper:
- anchor_lots = await scraper.fetch_around_multi_room(
- _a_lat,
- _a_lon,
- radius_m,
- pages=pages_per_anchor,
- )
- counters.lots_fetched += len(anchor_lots)
- # ── Фильтр вторички (newbuilding_only) ─────────────────────
- # Зеркало secondary_only в fetch_all_secondary: тут оставляем
- # только новостройки — вторичкой авторитетно владеет run_cian_full_load.
- if newbuilding_only:
- _before = len(anchor_lots)
- anchor_lots = [
- lot for lot in anchor_lots if lot.listing_segment == "novostroyki"
- ]
- counters.lots_dropped_secondary += _before - len(anchor_lots)
- if anchor_lots:
- inserted, updated = save_listings(db, anchor_lots, run_id=run_id)
- counters.lots_inserted += inserted
- counters.lots_updated += updated
- consecutive_failures = 0
- logger.info(
- "cian-sweep run_id=%d anchor %s: SERP fetched=%d nb_kept=%d "
- "dropped_secondary=%d ins=%d upd=%d",
- run_id,
- _a_name,
- counters.lots_fetched,
- len(anchor_lots),
- counters.lots_dropped_secondary,
- counters.lots_inserted,
- counters.lots_updated,
- )
-
- # ── Phase 3: detail enrichment (top-N) ────────────────────
- if detail_top_n > 0:
- priority_rows = (
- db.execute(
- _text("""
- SELECT id, source_url
- FROM listings
- WHERE source = 'cian'
- AND source_url IS NOT NULL
- AND detail_enriched_at IS NULL
- AND scraped_at > NOW() - INTERVAL '2 hours'
- ORDER BY scraped_at DESC NULLS LAST
- LIMIT :lim
- """),
- {"lim": detail_top_n},
- )
- .mappings()
- .all()
- )
- # #1848: Cian bans/stuck-proxy tracking + rotation (зеркало avito)
- _cian_detail_consec_failures = 0
- _cian_detail_abort = 3 # abort detail-фазы после N подряд
- didx = 0
- while didx < len(priority_rows):
- row = priority_rows[didx]
- listing_id: int = row["id"]
- source_url: str = row["source_url"]
- counters.detail_attempted += 1
- try:
- enrichment = await fetch_detail(source_url)
- if enrichment is not None:
- save_detail_enrichment(db, listing_id, enrichment)
- counters.detail_enriched += 1
- else:
- counters.detail_failed += 1
- logger.warning(
- "cian-sweep run_id=%d: detail returned None for "
- "listing_id=%d url=%s",
- run_id,
- listing_id,
- source_url,
- )
- _cian_detail_consec_failures = 0
- except Exception as exc:
- counters.detail_failed += 1
- counters.errors_count += 1
- _cian_detail_consec_failures += 1
- logger.warning(
- "cian-sweep run_id=%d: detail failed listing_id=%d "
- "(consecutive=%d): %s",
- run_id,
- listing_id,
- _cian_detail_consec_failures,
- exc,
- )
- if _cian_detail_consec_failures >= _cian_detail_abort:
- # Попытка ротации Cian-прокси перед abort'ом (#1848)
- if cian_rotations_done < settings.cian_proxy_max_rotations:
- rotated = await _rotate_proxy_ip(
- reason=(
- f"cian detail consecutive="
- f"{_cian_detail_consec_failures}"
- ),
- rotations_done=cian_rotations_done,
- source="cian",
- )
- cian_rotations_done += 1
- if rotated:
- _cian_detail_consec_failures = 0
- logger.info(
- "cian-sweep run_id=%d: detail ROTATE — "
- "IP changed, reset consecutive "
- "(rotation #%d/%d)",
- run_id,
- cian_rotations_done,
- settings.cian_proxy_max_rotations,
- )
- # retry текущего detail (не инкрементируем didx)
- continue
- logger.error(
- "cian-sweep run_id=%d: detail ABORT — "
- "%d consecutive failures (proxy likely banned/stuck)",
- run_id,
- _cian_detail_consec_failures,
- )
- break
- if didx < len(priority_rows) - 1:
- jitter = random.uniform(0.8, 1.2)
- await asyncio.sleep(request_delay_sec * jitter)
- didx += 1
- logger.info(
- "cian-sweep run_id=%d anchor %s: detail=%d/%d failed=%d",
- run_id,
- _a_name,
- counters.detail_enriched,
- counters.detail_attempted,
- counters.detail_failed,
- )
-
- # ── Phase 4: newbuilding/houses enrichment ─────────────────
- if not (enrich_houses and anchor_lots):
- return
- cian_nb_ext_ids = {
- lot.house_ext_id
- for lot in anchor_lots
- if lot.house_source == "cian_newbuilding" and lot.house_ext_id
- }
- if not cian_nb_ext_ids:
- return
- nb_id_list = list(cian_nb_ext_ids)
- try:
- house_rows = (
- db.execute(
- _text("""
- SELECT h.id, h.cian_zhk_url
- FROM houses h
- JOIN house_sources hs ON hs.house_id = h.id
- WHERE hs.ext_source = 'cian_newbuilding'
- AND hs.ext_id = ANY(CAST(:ids AS text[]))
- AND h.cian_zhk_url IS NOT NULL
- """),
- {"ids": nb_id_list},
- )
- .mappings()
- .all()
- )
- except Exception as exc:
- logger.exception(
- "cian-sweep run_id=%d anchor %s: houses DB query failed — "
- "skipping houses phase (%d ids): %s",
- run_id,
- _a_name,
- len(nb_id_list),
- exc,
- )
- counters.houses_failed += len(nb_id_list)
- try:
- db.rollback()
- except Exception:
- pass
- return
- for hidx, hrow in enumerate(house_rows):
- house_id_val: int = hrow["id"]
- zhk_url: str = hrow["cian_zhk_url"]
- counters.houses_attempted += 1
- try:
- nb_enrichment = await fetch_newbuilding(zhk_url)
- if nb_enrichment is not None:
- save_newbuilding_enrichment(db, house_id_val, nb_enrichment)
- counters.houses_enriched += 1
- else:
- counters.houses_failed += 1
- logger.warning(
- "cian-sweep run_id=%d: newbuilding returned None "
- "house_id=%d url=%s",
- run_id,
- house_id_val,
- zhk_url,
- )
- except Exception as exc:
- counters.houses_failed += 1
- counters.errors_count += 1
- logger.warning(
- "cian-sweep run_id=%d: houses failed house_id=%d: %s",
- run_id,
- house_id_val,
- exc,
- )
- try:
- db.rollback()
- except Exception:
- pass
- if hidx < len(house_rows) - 1:
- await asyncio.sleep(request_delay_sec)
- logger.info(
- "cian-sweep run_id=%d anchor %s: houses=%d/%d failed=%d",
- run_id,
- _a_name,
- counters.houses_enriched,
- counters.houses_attempted,
- counters.houses_failed,
- )
-
- try:
- await asyncio.wait_for(_cian_anchor_phases(), timeout=_cian_anchor_timeout)
- except TimeoutError:
- logger.warning(
- "cian-sweep run_id=%d: anchor #%d/%d (%s, %.4f, %.4f) "
- "timed out after %ds — skipping",
- run_id,
- idx,
- len(_anchors),
- name,
- lat,
- lon,
- _cian_anchor_timeout,
- )
- counters.errors_count += 1
- consecutive_failures += 1
- if consecutive_failures >= CIAN_SWEEP_MAX_CONSECUTIVE_FAILURES:
- logger.error(
- "cian-sweep run_id=%d ABORT at anchor #%d/%d (%s) — "
- "%d consecutive failures (last: timeout): "
- "IP likely blocked or proxy hung",
- run_id,
- idx,
- len(_anchors),
- name,
- consecutive_failures,
- )
- counters.anchors_done = idx
- scrape_runs.update_heartbeat(db, run_id, counters.to_dict())
- _abort_reason = (
- f"cian sweep aborted: {consecutive_failures} consecutive "
- f"anchor failures (last: anchor timeout {_cian_anchor_timeout:.0f}s)"
- )
- _mark_cian_sweep_abort(db, run_id, counters, _abort_reason)
- return counters
- counters.anchors_done = idx
- scrape_runs.update_heartbeat(db, run_id, counters.to_dict())
- continue
- except Exception as e:
- logger.exception("cian-sweep run_id=%d: anchor %s SERP failed", run_id, name)
- counters.errors_count += 1
- consecutive_failures += 1
- if consecutive_failures >= CIAN_SWEEP_MAX_CONSECUTIVE_FAILURES:
- logger.error(
- "cian-sweep run_id=%d ABORT at anchor #%d/%d (%s) — "
- "%d consecutive SERP failures (IP likely blocked): %s",
- run_id,
- idx,
- len(_anchors),
- name,
- consecutive_failures,
- e,
- )
- counters.anchors_done = idx
- scrape_runs.update_heartbeat(db, run_id, counters.to_dict())
- _abort_reason = (
- f"cian sweep aborted: {consecutive_failures} consecutive "
- f"anchor SERP failures (last: {e})"
- )
- _mark_cian_sweep_abort(db, run_id, counters, _abort_reason)
- return counters
- counters.anchors_done = idx
- scrape_runs.update_heartbeat(db, run_id, counters.to_dict())
- continue
-
- counters.anchors_done = idx
- scrape_runs.update_heartbeat(db, run_id, counters.to_dict())
-
- # Пауза между anchor'ами (поверх per-request sleep внутри scraper'а)
- if idx < len(_anchors):
- await asyncio.sleep(request_delay_sec)
-
- scrape_runs.mark_done(db, run_id, counters.to_dict())
- logger.info(
- "cian-sweep run_id=%d done: anchors=%d/%d lots=%d (ins=%d/upd=%d) "
- "detail=%d/%d houses=%d/%d errors=%d",
- run_id,
- counters.anchors_done,
- counters.anchors_total,
- counters.lots_fetched,
- counters.lots_inserted,
- counters.lots_updated,
- counters.detail_enriched,
- counters.detail_attempted,
- counters.houses_enriched,
- counters.houses_attempted,
- counters.errors_count,
- )
- return counters
-
- except Exception as exc:
- logger.exception("cian-sweep run_id=%d: fatal error", run_id)
- scrape_runs.mark_failed(db, run_id, str(exc), counters.to_dict())
- raise
-
-
-# ── Cian exhaustive full load (region-wide, без anchor'ов) ─────────────────
-
-
-@dataclass
-class CianFullLoadCounters:
- """Aggregate counters для run_cian_full_load.
-
- Без anchor-полей — единственный региональный проход.
- detail_* — опциональное обогащение detail-страниц (если enrich_detail=True).
- """
-
- unique_fetched: int = 0
- saved_inserted: int = 0
- saved_updated: int = 0
- detail_attempted: int = 0
- detail_enriched: int = 0
- detail_failed: int = 0
- errors_count: int = 0
-
- def to_dict(self) -> dict[str, int]:
- return {f.name: getattr(self, f.name) for f in fields(self)}
-
-
-async def run_cian_full_load(
- db: Session,
- *,
- run_id: int,
- price_cap_per_bucket: int = 1400,
- detail_top_n: int = 0,
- request_delay_sec: float = 4.0,
- enrich_detail: bool = False,
- concurrency: int = 5,
- resume_run_id: int | None = None,
-) -> CianFullLoadCounters:
- """Exhaustive региональный сбор Cian ЕКБ вторички (БЕЗ anchor'ов).
-
- Cian SERP — region-wide, не geo-bbox. Anchor-loop избыточен (5 anchor'ов
- возвращают одну и ту же выдачу ×5). Один проход с партиционированием по
- КОМНАТНОСТИ × ЦЕНЕ (адаптивное бинарное деление) охватывает весь ЕКБ.
-
- Параметры:
- price_cap_per_bucket: максимум офферов на price-бакет (< Cian SERP-cap ~1500).
- detail_top_n: сколько свежих cian-листингов без detail_enriched_at обогащать
- (актуально только при enrich_detail=True).
- request_delay_sec: задержка между SERP-запросами (перезаписывает scraper default).
- enrich_detail: включить detail-обогащение (по умолчанию отключено — тяжело).
- concurrency: параллельных page-фетчей в leaf-бакете (default=5).
-
- Инкрементальный save: on_bucket коммитит каждый leaf-бакет в БД сразу после сбора.
- Краш/бан больше не приводит к потере всего накопленного.
- Cooperative cancel: scrape_runs.is_cancelled проверяется per-bucket.
- """
- from app.services.scrapers.cian import CianScraper
-
- counters = CianFullLoadCounters()
-
- # ── Checkpoint/resume: читаем done_buckets из прошлого run ───────────────
- skip_set: set[str] = set()
- if resume_run_id is not None:
- _prev_row = db.execute(
- text("SELECT counters FROM scrape_runs WHERE id = CAST(:rid AS bigint)"),
- {"rid": resume_run_id},
- ).fetchone()
- if _prev_row is not None and _prev_row.counters:
- _prev_counters: dict = (
- _prev_row.counters if isinstance(_prev_row.counters, dict) else {}
- )
- skip_set = set(_prev_counters.get("done_buckets", []))
- logger.info(
- "cian-full-load run_id=%d: resuming from run %d — %d buckets already done",
- run_id,
- resume_run_id,
- len(skip_set),
- )
-
- done: set[str] = set(skip_set) # накапливаем завершённые бакеты этого прогона
-
- def _on_bucket(bucket_key: str, lots: list) -> None: # type: ignore[type-arg]
- """Инкрементальный save после каждого leaf-бакета. Дописывает bucket_key в done."""
- nonlocal done
- if scrape_runs.is_cancelled(db, run_id):
- logger.info("cian-full-load run_id=%d: cancel detected in on_bucket", run_id)
- raise RuntimeError("cancelled")
- elif shutdown_requested():
- # SIGTERM-drain: тот же sentinel-механизм, что и user-cancel, но отдельный
- # маркер "shutdown" → handler финализирует mark_done(partial), не mark_failed.
- logger.info(
- "cian-full-load run_id=%d: SIGTERM-drain — stopping at bucket %s",
- run_id,
- bucket_key,
- )
- raise RuntimeError("shutdown")
- if not lots:
- done.add(bucket_key)
- scrape_runs.update_heartbeat(
- db, run_id, {**counters.to_dict(), "done_buckets": sorted(done)}
- )
- return
- inserted, updated = save_listings(
- db, lots, run_id=run_id, skip_seen_today=settings.scraper_skip_seen_today
- )
- # save_listings вызывает db.commit() внутри — данные в БД сразу
- counters.saved_inserted += inserted
- counters.saved_updated += updated
- counters.unique_fetched += len(lots)
- done.add(bucket_key)
- scrape_runs.update_heartbeat(
- db, run_id, {**counters.to_dict(), "done_buckets": sorted(done)}
- )
- logger.info(
- "cian-full-load run_id=%d: bucket %s saved ins=%d upd=%d total_unique=%d",
- run_id,
- bucket_key,
- inserted,
- updated,
- counters.unique_fetched,
- )
-
- def _on_progress(unique_count: int) -> None:
- """Heartbeat per room-bucket (на случай пустых бакетов без on_bucket вызовов)."""
- scrape_runs.update_heartbeat(db, run_id, counters.to_dict())
-
- # #1949: фоновый heartbeat — обновляет heartbeat_at каждые 60s независимо от on_bucket.
- # Без него зависший asyncio.gather внутри bucket-loop замораживает heartbeat на часы
- # → reap_zombies помечает живой run zombie. Отменяется в finally при любом завершении.
- async def _background_heartbeat() -> None:
- """Периодический heartbeat каждые 60s, не завязанный на прогресс бакетов."""
- while True:
- await asyncio.sleep(60.0)
- try:
- scrape_runs.update_heartbeat(db, run_id, counters.to_dict())
- except Exception:
- pass
-
- _hb_task: asyncio.Task[None] | None = None
- try:
- _hb_task = asyncio.create_task(_background_heartbeat())
-
- async with CianScraper() as scraper:
- scraper.request_delay_sec = request_delay_sec
-
- # #1949: per-fetch hard-cancel для browser-fetch'ей внутри bucket-gather.
- # BrowserFetcher.fetch имеет httpx-timeout 120s, но browser-сервис иногда
- # виснет так что httpx не получает ответ → gather морозит → heartbeat стоп.
- # Оборачиваем _browser.fetch в asyncio.wait_for(timeout=X) через monkey-patch:
- # TimeoutError → _fetch_page_html ловит как Exception → None → _one_page → []
- # → gather завершается нормально → on_bucket вызывается → heartbeat обновляется.
- # За флагом cian_full_load_per_fetch_timeout_s (0 = отключить).
- _fetch_timeout = settings.cian_full_load_per_fetch_timeout_s
- if _fetch_timeout > 0 and scraper._browser is not None:
- _orig_fetch = scraper._browser.fetch
-
- async def _timed_fetch(
- url: str,
- _f: Any = _orig_fetch,
- _t: float = _fetch_timeout,
- _rid: int = run_id,
- ) -> str:
- try:
- return await asyncio.wait_for(_f(url), timeout=_t)
- except TimeoutError:
- logger.warning(
- "cian-full-load run_id=%d: browser-fetch timed out "
- "after %.0fs — url=%s",
- _rid,
- _t,
- url,
- )
- raise
-
- scraper._browser.fetch = _timed_fetch # type: ignore[method-assign]
- logger.info(
- "cian-full-load run_id=%d: per-fetch timeout %.0fs enabled",
- run_id,
- _fetch_timeout,
- )
-
- await scraper.fetch_all_secondary(
- price_cap_per_bucket=price_cap_per_bucket,
- concurrency=concurrency,
- secondary_only=True,
- on_bucket=_on_bucket,
- on_progress=_on_progress,
- skip_buckets=skip_set if skip_set else None,
- )
-
- logger.info(
- "cian-full-load run_id=%d: fetch done — unique=%d ins=%d upd=%d",
- run_id,
- counters.unique_fetched,
- counters.saved_inserted,
- counters.saved_updated,
- )
- scrape_runs.update_heartbeat(db, run_id, counters.to_dict())
-
- # ── Опциональный detail-энричмент ─────────────────────────────────────
- if enrich_detail and detail_top_n > 0:
- from sqlalchemy import text as _text
-
- from app.services.scrapers.cian_detail import (
- fetch_detail,
- save_detail_enrichment,
- )
-
- priority_rows = (
- db.execute(
- _text("""
- SELECT id, source_url
- FROM listings
- WHERE source = 'cian'
- AND source_url IS NOT NULL
- AND detail_enriched_at IS NULL
- AND scraped_at > NOW() - INTERVAL '4 hours'
- ORDER BY scraped_at DESC NULLS LAST
- LIMIT :lim
- """),
- {"lim": detail_top_n},
- )
- .mappings()
- .all()
- )
- for didx, row in enumerate(priority_rows):
- if scrape_runs.is_cancelled(db, run_id):
- logger.info("cian-full-load run_id=%d: cancelled during detail enrich", run_id)
- break
- elif shutdown_requested():
- logger.info(
- "cian-full-load run_id=%d: SIGTERM-drain — stopping detail at #%d/%d",
- run_id,
- didx + 1,
- len(priority_rows),
- )
- break
- listing_id: int = row["id"]
- source_url: str = row["source_url"]
- counters.detail_attempted += 1
- try:
- enrichment = await fetch_detail(source_url)
- if enrichment is not None:
- save_detail_enrichment(db, listing_id, enrichment)
- counters.detail_enriched += 1
- else:
- counters.detail_failed += 1
- except Exception as exc:
- counters.detail_failed += 1
- counters.errors_count += 1
- logger.warning(
- "cian-full-load run_id=%d: detail failed listing_id=%d: %s",
- run_id,
- listing_id,
- exc,
- )
- if didx < len(priority_rows) - 1:
- await asyncio.sleep(request_delay_sec * random.uniform(0.8, 1.2))
-
- logger.info(
- "cian-full-load run_id=%d: detail done — enriched=%d/%d failed=%d",
- run_id,
- counters.detail_enriched,
- counters.detail_attempted,
- counters.detail_failed,
- )
- scrape_runs.update_heartbeat(db, run_id, counters.to_dict())
-
- scrape_runs.mark_done(db, run_id, {**counters.to_dict(), "done_buckets": sorted(done)})
- logger.info(
- "cian-full-load run_id=%d done: unique=%d ins=%d upd=%d detail=%d/%d errors=%d",
- run_id,
- counters.unique_fetched,
- counters.saved_inserted,
- counters.saved_updated,
- counters.detail_enriched,
- counters.detail_attempted,
- counters.errors_count,
- )
- return counters
-
- except RuntimeError as exc:
- # on_bucket кидает RuntimeError("cancelled") при cooperative cancel,
- # RuntimeError("shutdown") при кооперативном SIGTERM-drain (#1182 Phase 3a).
- if str(exc) == "cancelled":
- logger.info(
- "cian-full-load run_id=%d: cancelled — partial results unique=%d ins=%d upd=%d",
- run_id,
- counters.unique_fetched,
- counters.saved_inserted,
- counters.saved_updated,
- )
- scrape_runs.mark_done(db, run_id, {**counters.to_dict(), "done_buckets": sorted(done)})
- return counters
- if str(exc) == "shutdown":
- logger.info(
- "cian-full-load run_id=%d: SIGTERM-drain — partial results unique=%d ins=%d upd=%d",
- run_id,
- counters.unique_fetched,
- counters.saved_inserted,
- counters.saved_updated,
- )
- scrape_runs.mark_done(db, run_id, {**counters.to_dict(), "done_buckets": sorted(done)})
- return counters
- logger.exception("cian-full-load run_id=%d: fatal error", run_id)
- scrape_runs.mark_failed(db, run_id, str(exc), counters.to_dict())
- raise
-
- except Exception as exc:
- logger.exception("cian-full-load run_id=%d: fatal error", run_id)
- scrape_runs.mark_failed(db, run_id, str(exc), counters.to_dict())
- raise
-
- finally:
- # #1949: отменяем фоновый heartbeat при любом завершении (return / raise / cancel).
- if _hb_task is not None:
- _hb_task.cancel()
- try:
- await _hb_task
- except asyncio.CancelledError:
- pass
-
-
-# ── Yandex exhaustive full load (region-wide, без anchor'ов) ──────────────────
-
-
-@dataclass
-class YandexFullLoadCounters:
- """Aggregate counters для run_yandex_full_load.
-
- Без detail_* полей — Yandex full load не делает detail-обогащение.
- """
-
- unique_fetched: int = 0
- saved_inserted: int = 0
- saved_updated: int = 0
- price_history_rows: int = 0
- errors_count: int = 0
-
- def to_dict(self) -> dict[str, int]:
- return {f.name: getattr(self, f.name) for f in fields(self)}
-
-
-async def run_yandex_full_load(
- db: Session,
- *,
- run_id: int,
- price_cap_per_bucket: int = 500,
- request_delay_sec: float = 2.0,
- concurrency: int = 4,
- resume_run_id: int | None = None,
-) -> YandexFullLoadCounters:
- """Exhaustive региональный сбор Yandex ЕКБ вторички (БЕЗ anchor'ов).
-
- Yandex SERP — path-based по городу, не geo-bbox. Anchor-loop избыточен
- (все anchor'ы дают одну и ту же городскую выдачу). Один проход с
- партиционированием по КОМНАТНОСТИ × ЦЕНЕ (адаптивное бинарное деление)
- охватывает весь ЕКБ.
-
- Параметры:
- price_cap_per_bucket: максимум офферов на price-бакет (< Yandex SERP-cap ~575).
- request_delay_sec: задержка между SERP-запросами (перезаписывает scraper default).
- concurrency: параллельных page-фетчей в leaf-бакете.
- resume_run_id: если задан — читает done_buckets из counters прошлого run и
- пропускает уже завершённые бакеты. Передать run_id предыдущего (прерванного)
- yandex_full_load прогона для resume. Без этого поля — full walk с нуля.
-
- Инкрементальный save: on_bucket коммитит каждый leaf-бакет в БД сразу после сбора.
- Cooperative cancel: scrape_runs.is_cancelled проверяется per-bucket.
- """
- from app.services.scrapers.yandex_realty import YandexRealtyScraper
-
- counters = YandexFullLoadCounters()
-
- skip_set: set[str] = set()
- if resume_run_id is not None:
- _prev_row = db.execute(
- text("SELECT counters FROM scrape_runs WHERE id = CAST(:rid AS bigint)"),
- {"rid": resume_run_id},
- ).fetchone()
- if _prev_row is not None and _prev_row.counters:
- _prev_counters: dict = (
- _prev_row.counters if isinstance(_prev_row.counters, dict) else {}
- )
- skip_set = set(_prev_counters.get("done_buckets", []))
- logger.info(
- "yandex-full-load run_id=%d: resuming from run %d — %d buckets already done",
- run_id,
- resume_run_id,
- len(skip_set),
- )
- done: set[str] = set(skip_set)
-
- def _on_bucket(bucket_key: str, lots: list) -> None: # type: ignore[type-arg]
- """Инкрементальный save после каждого leaf-бакета."""
- nonlocal done
- if scrape_runs.is_cancelled(db, run_id):
- logger.info("yandex-full-load run_id=%d: cancel detected in on_bucket", run_id)
- raise RuntimeError("cancelled")
- elif shutdown_requested():
- logger.info(
- "yandex-full-load run_id=%d: SIGTERM-drain — stopping at bucket %s",
- run_id,
- bucket_key,
- )
- raise RuntimeError("shutdown")
- if not lots:
- done.add(bucket_key)
- scrape_runs.update_heartbeat(
- db, run_id, {**counters.to_dict(), "done_buckets": sorted(done)}
- )
- return
- inserted, updated = save_listings(
- db, lots, run_id=run_id, skip_seen_today=settings.scraper_skip_seen_today
- )
- # save_listings вызывает db.commit() внутри — данные в БД сразу
- counters.saved_inserted += inserted
- counters.saved_updated += updated
- counters.unique_fetched += len(lots)
- # Price-history из gate price.previous/trend (graceful — провал истории
- # НЕ должен ронять сейв листингов/бакета).
- try:
- counters.price_history_rows += record_yandex_price_history(db, lots)
- except Exception as ph_exc:
- logger.warning(
- "yandex-full-load run_id=%d: price-history failed bucket=%s: %s",
- run_id,
- bucket_key,
- ph_exc,
- )
- try:
- db.rollback()
- except Exception:
- pass
- done.add(bucket_key)
- scrape_runs.update_heartbeat(
- db, run_id, {**counters.to_dict(), "done_buckets": sorted(done)}
- )
- logger.info(
- "yandex-full-load run_id=%d: bucket=%s saved ins=%d upd=%d total_unique=%d",
- run_id,
- bucket_key,
- inserted,
- updated,
- counters.unique_fetched,
- )
-
- def _on_progress(unique_count: int) -> None:
- """Heartbeat per room-bucket (на случай пустых бакетов без on_bucket вызовов)."""
- scrape_runs.update_heartbeat(db, run_id, counters.to_dict())
-
- try:
- async with YandexRealtyScraper() as scraper:
- scraper.request_delay_sec = request_delay_sec
-
- await scraper.fetch_all_secondary(
- price_cap_per_bucket=price_cap_per_bucket,
- concurrency=concurrency,
- on_bucket=_on_bucket,
- on_progress=_on_progress,
- skip_buckets=skip_set if skip_set else None,
- )
-
- logger.info(
- "yandex-full-load run_id=%d: fetch done — unique=%d ins=%d upd=%d",
- run_id,
- counters.unique_fetched,
- counters.saved_inserted,
- counters.saved_updated,
- )
- scrape_runs.update_heartbeat(db, run_id, counters.to_dict())
- scrape_runs.mark_done(db, run_id, {**counters.to_dict(), "done_buckets": sorted(done)})
- logger.info(
- "yandex-full-load run_id=%d done: unique=%d ins=%d upd=%d errors=%d",
- run_id,
- counters.unique_fetched,
- counters.saved_inserted,
- counters.saved_updated,
- counters.errors_count,
- )
- return counters
-
- except RuntimeError as exc:
- # on_bucket кидает RuntimeError("cancelled") при cooperative cancel,
- # RuntimeError("shutdown") при кооперативном SIGTERM-drain (#1182 Phase 3a).
- if str(exc) == "cancelled":
- logger.info(
- "yandex-full-load run_id=%d: cancelled — partial results unique=%d ins=%d upd=%d",
- run_id,
- counters.unique_fetched,
- counters.saved_inserted,
- counters.saved_updated,
- )
- scrape_runs.mark_done(db, run_id, {**counters.to_dict(), "done_buckets": sorted(done)})
- return counters
- if str(exc) == "shutdown":
- logger.info(
- "yandex-full-load run_id=%d: SIGTERM-drain — partial results "
- "unique=%d ins=%d upd=%d",
- run_id,
- counters.unique_fetched,
- counters.saved_inserted,
- counters.saved_updated,
- )
- scrape_runs.mark_done(db, run_id, {**counters.to_dict(), "done_buckets": sorted(done)})
- return counters
- logger.exception("yandex-full-load run_id=%d: fatal error", run_id)
- scrape_runs.mark_failed(db, run_id, str(exc), counters.to_dict())
- raise
-
- except Exception as exc:
- logger.exception("yandex-full-load run_id=%d: fatal error", run_id)
- scrape_runs.mark_failed(db, run_id, str(exc), counters.to_dict())
- raise
-
-
-# ── Avito exhaustive full load (citywide, без anchor'ов) ──────────────────────
-
-
-@dataclass
-class AvitoFullLoadCounters:
- """Aggregate counters для run_avito_full_load.
-
- Зеркало YandexFullLoadCounters — citywide room×price bisection без anchor/detail.
- """
-
- unique_fetched: int = 0
- saved_inserted: int = 0
- saved_updated: int = 0
- errors_count: int = 0
-
- def to_dict(self) -> dict[str, int]:
- return {f.name: getattr(self, f.name) for f in fields(self)}
-
-
-async def run_avito_full_load(
- db: Session,
- *,
- run_id: int,
- price_cap_per_bucket: int = 1400,
- request_delay_sec: float = 7.0,
- concurrency: int = 5,
- secondary_only: bool = True,
- resume_run_id: int | None = None,
- incremental_days: int | None = None,
-) -> AvitoFullLoadCounters:
- """Exhaustive ИЛИ инкрементальный региональный сбор Avito ЕКБ вторички (БЕЗ anchor'ов).
-
- Avito anchor/citywide упирается в SERP-cap ~5000 результатов на запрос →
- вторичка катастрофически недобрана. Решение зеркалит cian/yandex full_load:
- один проход с партиционированием по КОМНАТНОСТИ × ЦЕНЕ (адаптивное бинарное
- деление ценового диапазона через `pmin`/`pmax`) охватывает весь ЕКБ.
-
- Параметры:
- price_cap_per_bucket: максимум результатов на price-бакет (< Avito SERP-cap ~5000).
- request_delay_sec: задержка между SERP-запросами (перезаписывает scraper default).
- concurrency: параллельных page-фетчей в leaf-бакете.
- secondary_only: отбрасывать новостройки (listing_segment=="novostroyki") до save.
- resume_run_id: если задан — читает done_buckets из counters прошлого run и
- пропускает уже завершённые бакеты. Передать run_id предыдущего (прерванного)
- avito_full_load прогона для resume. Без поля — full walk с нуля.
- incremental_days: если задан — shallow инкрементальный обход. Считается
- since = date.today() - timedelta(days=incremental_days) и передаётся в
- fetch_all_secondary(since=since): date early-stop останавливает пагинацию
- до глубоких страниц (избегает page-36 429-банов). None → since=None →
- exhaustive bisection (полный обход, текущее поведение).
-
- Инкрементальный save: on_bucket коммитит каждый leaf-бакет в БД сразу после сбора.
- Cooperative cancel: scrape_runs.is_cancelled проверяется per-bucket.
- AvitoBlockedError/AvitoRateLimitedError → mark_banned (status='banned'), как в
- run_avito_city_sweep — partial results уже сохранены через on_bucket.
- """
- counters = AvitoFullLoadCounters()
-
- # ── Инкрементальный режим: since-cutoff для shallow обхода ────────────────
- since: date | None = None
- if incremental_days is not None:
- since = date.today() - timedelta(days=incremental_days)
- logger.info(
- "avito-full-load run_id=%d: INCREMENTAL mode — since=%s (last %d days)",
- run_id,
- since.isoformat(),
- incremental_days,
- )
-
- # ── Checkpoint/resume: читаем done_buckets из прошлого run ───────────────
- skip_set: set[str] = set()
- if resume_run_id is not None:
- _prev_row = db.execute(
- text("SELECT counters FROM scrape_runs WHERE id = CAST(:rid AS bigint)"),
- {"rid": resume_run_id},
- ).fetchone()
- if _prev_row is not None and _prev_row.counters:
- _prev_counters: dict = (
- _prev_row.counters if isinstance(_prev_row.counters, dict) else {}
- )
- skip_set = set(_prev_counters.get("done_buckets", []))
- logger.info(
- "avito-full-load run_id=%d: resuming from run %d — %d buckets already done",
- run_id,
- resume_run_id,
- len(skip_set),
- )
- done: set[str] = set(skip_set)
-
- def _on_bucket(bucket_key: str, lots: list) -> None: # type: ignore[type-arg]
- """Инкрементальный save после каждого leaf-бакета."""
- nonlocal done
- if scrape_runs.is_cancelled(db, run_id):
- logger.info("avito-full-load run_id=%d: cancel detected in on_bucket", run_id)
- raise RuntimeError("cancelled")
- elif shutdown_requested():
- logger.info(
- "avito-full-load run_id=%d: SIGTERM-drain — stopping at bucket %s",
- run_id,
- bucket_key,
- )
- raise RuntimeError("shutdown")
- if not lots:
- done.add(bucket_key)
- scrape_runs.update_heartbeat(
- db, run_id, {**counters.to_dict(), "done_buckets": sorted(done)}
- )
- return
- inserted, updated = save_listings(
- db, lots, run_id=run_id, skip_seen_today=settings.scraper_skip_seen_today
- )
- # save_listings вызывает db.commit() внутри — данные в БД сразу
- counters.saved_inserted += inserted
- counters.saved_updated += updated
- counters.unique_fetched += len(lots)
- done.add(bucket_key)
- scrape_runs.update_heartbeat(
- db, run_id, {**counters.to_dict(), "done_buckets": sorted(done)}
- )
- logger.info(
- "avito-full-load run_id=%d: bucket=%s saved ins=%d upd=%d total_unique=%d",
- run_id,
- bucket_key,
- inserted,
- updated,
- counters.unique_fetched,
- )
-
- def _on_progress(unique_count: int) -> None:
- """Heartbeat per room-bucket (на случай пустых бакетов без on_bucket вызовов)."""
- scrape_runs.update_heartbeat(db, run_id, counters.to_dict())
-
- try:
- async with AvitoScraper() as scraper:
- scraper.request_delay_sec = request_delay_sec
-
- await scraper.fetch_all_secondary(
- price_cap_per_bucket=price_cap_per_bucket,
- concurrency=concurrency,
- secondary_only=secondary_only,
- on_bucket=_on_bucket,
- on_progress=_on_progress,
- skip_buckets=skip_set if skip_set else None,
- since=since,
- )
-
- logger.info(
- "avito-full-load run_id=%d: fetch done — unique=%d ins=%d upd=%d",
- run_id,
- counters.unique_fetched,
- counters.saved_inserted,
- counters.saved_updated,
- )
- scrape_runs.update_heartbeat(db, run_id, counters.to_dict())
- scrape_runs.mark_done(db, run_id, {**counters.to_dict(), "done_buckets": sorted(done)})
- logger.info(
- "avito-full-load run_id=%d done: unique=%d ins=%d upd=%d errors=%d",
- run_id,
- counters.unique_fetched,
- counters.saved_inserted,
- counters.saved_updated,
- counters.errors_count,
- )
- return counters
-
- except (AvitoBlockedError, AvitoRateLimitedError) as exc:
- # IP-бан/rate-limit: partial results уже сохранены через on_bucket.
- logger.error("avito-full-load run_id=%d: blocked/rate-limited — %s", run_id, exc)
- counters.errors_count += 1
- scrape_runs.mark_banned(
- db,
- run_id,
- f"avito full load aborted: {exc}",
- {**counters.to_dict(), "done_buckets": sorted(done)},
- )
- return counters
-
- except RuntimeError as exc:
- # on_bucket кидает RuntimeError("cancelled") при cooperative cancel,
- # RuntimeError("shutdown") при кооперативном SIGTERM-drain (#1182 Phase 3a).
- if str(exc) == "cancelled":
- logger.info(
- "avito-full-load run_id=%d: cancelled — partial results unique=%d ins=%d upd=%d",
- run_id,
- counters.unique_fetched,
- counters.saved_inserted,
- counters.saved_updated,
- )
- scrape_runs.mark_done(db, run_id, {**counters.to_dict(), "done_buckets": sorted(done)})
- return counters
- if str(exc) == "shutdown":
- logger.info(
- "avito-full-load run_id=%d: SIGTERM-drain — partial results "
- "unique=%d ins=%d upd=%d",
- run_id,
- counters.unique_fetched,
- counters.saved_inserted,
- counters.saved_updated,
- )
- scrape_runs.mark_done(db, run_id, {**counters.to_dict(), "done_buckets": sorted(done)})
- return counters
- logger.exception("avito-full-load run_id=%d: fatal error", run_id)
- scrape_runs.mark_failed(db, run_id, str(exc), counters.to_dict())
- raise
-
- except Exception as exc:
- logger.exception("avito-full-load run_id=%d: fatal error", run_id)
- scrape_runs.mark_failed(db, run_id, str(exc), counters.to_dict())
- raise
-
-
-# ── DomClick citywide sweep (без anchor'ов) ────────────────────────────────
-
-
-@dataclass
-class DomClickCitySweepCounters:
- """Aggregate counters для DomClick citywide sweep run.
-
- DomClick BFF — citywide (city_id), не geo-bbox: anchor-loop отсутствует.
- Один проход fetch_city перебирает ROOM_BUCKETS × pages внутри scraper'а.
- Поля симметричны sibling-counters, но без detail_*/houses_* (SERP-only)
- и без anchors_* (нет anchor-фазы). pages_fetched = 6 buckets × pages worst-case.
- """
-
- lots_fetched: int = 0
- lots_inserted: int = 0
- lots_updated: int = 0
- pages_fetched: int = 0
- errors_count: int = 0
- blocked: int = 0 # 1 если QRATOR-блок был во время sweep
- geo_filtered: int = 0 # число офферов отфильтрованных geo-guard
-
- def to_dict(self) -> dict[str, int]:
- return {f.name: getattr(self, f.name) for f in fields(self)}
-
-
-# Дефолтные параметры sweep'а (EKB city_id=4).
-DOMCLICK_DEFAULT_CITY_ID: int = 4
-DOMCLICK_DEFAULT_ROOMS: list[int] = [0, 1, 2, 3, 4] # vestigial; scraper sweeps all buckets
-# Число BFF-бакетов (st/1/2/3/4/5+) — фиксировано; используется для watchdog.
-_DOMCLICK_NUM_BUCKETS: int = 6
-# Оценка времени одного fetch'а (network + parse) для watchdog.
-_DOMCLICK_PER_FETCH_S: float = 12.0
-# Буфер сверху расчётного бюджета (cold browser start, save-фаза, price-splits).
-_DOMCLICK_SWEEP_BUDGET_S: float = 300.0
-
-
-async def run_domclick_city_sweep(
- db: Session,
- *,
- run_id: int,
- city_id: int = DOMCLICK_DEFAULT_CITY_ID,
- rooms: list[int] | None = None,
- pages: int = 100,
- request_delay_sec: float | None = None,
-) -> DomClickCitySweepCounters:
- """DomClick citywide sweep через BFF JSON API.
-
- Структурно зеркалит run_cian_city_sweep / run_yandex_city_sweep, но CITYWIDE:
- DomClick не поддерживает geo-radius (fetch_around → NotImplementedError),
- поэтому anchor-loop отсутствует. DomClickScraper.fetch_city перебирает все
- ROOM_BUCKETS (st/1/2/3/4/5+) через BrowserFetcher → shared mobile proxy →
- обходит QRATOR.
-
- Фазы (единственный citywide проход):
- 1. SERP: DomClickScraper().fetch_city(city_id, rooms, pages).
- 2. SAVE: save_listings(db, lots, run_id=run_id) — house-match по lat/lon и
- address-fingerprint (BFF отдаёт координаты).
-
- Watchdog: весь fetch_city оборачивается в asyncio.wait_for. Таймаут считается
- из 6 buckets × pages × per-fetch + buffer (минимум ANCHOR_TIMEOUT_SEC).
- Реальный sweep ≈ 330 страниц (rooms=2 требует price-split).
-
- Cooperative cancel: is_cancelled(db, run_id) проверяется перед SERP-фазой.
-
- ЧЕСТНЫЙ СТАТУС (#1968): если scraper сообщил QRATOR-блок И lots == 0 →
- mark_failed с объяснением. Иначе mark_done (в т.ч. при lots==0 без блока:
- genuinely empty run).
-
- Возвращает DomClickCitySweepCounters.
- """
- from app.services.scrapers.domclick import ROOM_BUCKETS, DomClickScraper
-
- _resolved_delay = request_delay_sec if request_delay_sec is not None else 6.0
- counters = DomClickCitySweepCounters()
-
- # Watchdog: 6 buckets × pages × per_fetch + budget.
- # Реальный worst-case (rooms=2 price-split) ≈ 330 страниц × (delay+12s).
- # pages=100 → 600 fetch'ей × ~18s + 300 ≈ 11100s (~3ч) — намеренно generous.
- _num_fetches = _DOMCLICK_NUM_BUCKETS * max(1, pages)
- _sweep_timeout = max(
- ANCHOR_TIMEOUT_SEC,
- int(_num_fetches * (_resolved_delay + _DOMCLICK_PER_FETCH_S) + _DOMCLICK_SWEEP_BUDGET_S),
- )
-
- # Мутируемый контейнер для захвата scraper-ссылки из замыкания.
- _scraper_ref: list[DomClickScraper] = []
-
- try:
- # ── Cooperative cancel перед SERP-фазой ──────────────────────────────
- if scrape_runs.is_cancelled(db, run_id):
- logger.info("domclick-sweep run_id=%d: cancelled before SERP", run_id)
- scrape_runs.update_heartbeat(db, run_id, counters.to_dict())
- return counters
- elif shutdown_requested():
- # SIGTERM-drain до SERP-фазы: финализируем run как mark_done(partial),
- # минуя honest-status (это чистый drain, не QRATOR-блок).
- logger.info("domclick-sweep run_id=%d: SIGTERM-drain — stopping before SERP", run_id)
- scrape_runs.update_heartbeat(db, run_id, counters.to_dict())
- scrape_runs.mark_done(db, run_id, counters.to_dict())
- return counters
-
- logger.info(
- "domclick-sweep run_id=%d: BFF citywide sweep city_id=%d "
- "buckets=%d pages_cap=%d (watchdog %ds)",
- run_id,
- city_id,
- len(ROOM_BUCKETS),
- pages,
- _sweep_timeout,
- )
-
- lots: list[ScrapedLot] = []
-
- async def _domclick_phase() -> None:
- """Единственная citywide-фаза: fetch_city + save."""
- nonlocal lots
- async with DomClickScraper() as _scraper:
- _scraper_ref.append(_scraper)
- if request_delay_sec is not None:
- _scraper.request_delay_sec = _resolved_delay
- lots = await _scraper.fetch_city(city_id=city_id, rooms=rooms, pages=pages)
- counters.lots_fetched += len(lots)
- if lots:
- inserted, updated = save_listings(db, lots, run_id=run_id)
- counters.lots_inserted += inserted
- counters.lots_updated += updated
-
- try:
- await asyncio.wait_for(_domclick_phase(), timeout=_sweep_timeout)
- except TimeoutError:
- logger.warning(
- "domclick-sweep run_id=%d: SERP phase timed out after %ds — partial results",
- run_id,
- _sweep_timeout,
- )
- counters.errors_count += 1
- except Exception:
- logger.exception("domclick-sweep run_id=%d: SERP phase failed", run_id)
- counters.errors_count += 1
-
- # Перенести счётчики scraper'а в counters (если scraper был создан).
- if _scraper_ref:
- _s = _scraper_ref[0]
- counters.blocked = 1 if _s.blocked else 0
- counters.geo_filtered = _s.geo_filtered
- # Не-block fetch-ошибки скрейпера учитываем в errors_count: они должны
- # влиять на honest-status (0 lots + fetch_errors → НЕ ложный mark_done).
- counters.errors_count += _s.fetch_errors
-
- # pages_fetched: worst-case число страниц (buckets × pages cap).
- counters.pages_fetched = _num_fetches
- scrape_runs.update_heartbeat(db, run_id, counters.to_dict())
-
- # ── ЧЕСТНЫЙ СТАТУС (#1968) ────────────────────────────────────────────
- # 0 lots + (block ИЛИ любая ошибка) → failed: run без результата по
- # внешней причине не должен маскироваться как 'done'. Нераспознанный
- # block-вариант (нет литерал-маркера) приходит как fetch_error, не blocked —
- # поэтому errors_count тоже триггерит failed. Genuinely empty 0-error run
- # → done. Partial run с lots>0 остаётся done даже если часть бакетов упала.
- if counters.lots_fetched == 0 and (counters.blocked or counters.errors_count > 0):
- logger.error(
- "domclick-sweep run_id=%d: 0 listings with blocked=%d errors=%d "
- "— marking failed",
- run_id,
- counters.blocked,
- counters.errors_count,
- )
- scrape_runs.mark_failed(
- db,
- run_id,
- "QRATOR block or fetch errors — 0 listings",
- counters.to_dict(),
- )
- else:
- scrape_runs.mark_done(db, run_id, counters.to_dict())
-
- logger.info(
- "domclick-sweep run_id=%d done: lots=%d (ins=%d/upd=%d) "
- "pages=%d errors=%d blocked=%d geo_filtered=%d",
- run_id,
- counters.lots_fetched,
- counters.lots_inserted,
- counters.lots_updated,
- counters.pages_fetched,
- counters.errors_count,
- counters.blocked,
- counters.geo_filtered,
- )
- return counters
-
- except Exception as exc:
- logger.exception("domclick-sweep run_id=%d: fatal error", run_id)
- scrape_runs.mark_failed(db, run_id, str(exc), counters.to_dict())
- raise
-
-
-# ── Public re-exports ───────────────────────────────────────────
-__all__ = [
- "ANCHOR_TIMEOUT_SEC",
- "CIAN_SWEEP_MAX_CONSECUTIVE_FAILURES",
- "EKB_ANCHORS",
- "CianCitySweepCounters",
- "CianFullLoadCounters",
- "CitySweepCounters",
- "DomClickCitySweepCounters",
- "NewbuildingSweepCounters",
- "PipelineCounters",
- "PipelineResult",
- "YandexCitySweepCounters",
- "YandexFullLoadCounters",
- "run_avito_city_sweep",
- "run_avito_newbuilding_sweep",
- "run_avito_pipeline",
- "run_cian_city_sweep",
- "run_cian_full_load",
- "run_domclick_city_sweep",
- "run_yandex_city_sweep",
- "run_yandex_full_load",
-]
diff --git a/tradein-mvp/backend/app/services/scrapers/avito_detail.py b/tradein-mvp/backend/app/services/scrapers/avito_detail.py
index 19172519..074b0bb6 100644
--- a/tradein-mvp/backend/app/services/scrapers/avito_detail.py
+++ b/tradein-mvp/backend/app/services/scrapers/avito_detail.py
@@ -388,7 +388,7 @@ async def fetch_detail(
reconnect_on_block — если False, 403/firewall поднимает AvitoBlockedError СРАЗУ
(без холодных reconnect'ов — ими займётся outer-loop через rebuild+rewarm прогретой
сессии); 429 после short-retry поднимает AvitoRateLimitedError. Дефолт True
- сохраняет старое поведение own-session / scrape_pipeline путей.
+ сохраняет старое поведение own-session путей (напр. admin endpoint).
Raises:
httpx.HTTPError — если status != 200 (через raise_for_status-like).
ValueError — если item_id не извлечён из HTML.
@@ -432,8 +432,8 @@ async def fetch_detail(
own_session = cffi_session is None
if cffi_session is None:
# Mobile proxy wiring (#806 follow-up): own-session path (no shared session passed,
- # e.g. admin endpoint). Caller-provided sessions (scrape_pipeline) already have proxy
- # from AvitoScraper.__aenter__ — don't override. proxy=None → прямое подключение.
+ # e.g. admin endpoint). Caller-provided sessions already have proxy from
+ # AvitoScraper.__aenter__ — don't override. proxy=None → прямое подключение.
session = _build_detail_session()
else:
session = cffi_session
@@ -446,8 +446,8 @@ async def fetch_detail(
# эфемерную retry_session, которую сами закрываем в finally.
# reconnect_on_block=False (warm-batch #1551): не делаем холодных per-card
# reconnect'ов — блок поднимает AvitoBlockedError сразу, outer-loop пересоздаёт+
- # прогревает сессию (свежий exit-IP). reconnect_on_block=True (own-session /
- # scrape_pipeline) — старое поведение: backconnect-reconnect эфемерной сессией.
+ # прогревает сессию (свежий exit-IP). reconnect_on_block=True (own-session) —
+ # старое поведение: backconnect-reconnect эфемерной сессией.
backconnect = bool(settings.scraper_proxy_url) and reconnect_on_block
req_headers = {"Referer": referer} if referer else None
r403 = 0
diff --git a/tradein-mvp/backend/app/services/yandex_price_history.py b/tradein-mvp/backend/app/services/yandex_price_history.py
index 5b5a7ab1..a01caa93 100644
--- a/tradein-mvp/backend/app/services/yandex_price_history.py
+++ b/tradein-mvp/backend/app/services/yandex_price_history.py
@@ -4,7 +4,9 @@
Yandex gate-ответ уже несёт `price.previous` + `price.trend`, которые парсятся в
`ScrapedLot.price_previous_rub` / `price_trend`. Otsenka-скрапер НЕ нужен — gate достаточно.
-Эта функция вызывается из scrape_pipeline СРАЗУ ПОСЛЕ save_listings(те же lots): резолвит
+Эта функция вызывается из run_yandex_city_sweep (scraper_kit.orchestration.pipeline,
+через инжектируемый EnrichmentJobs.record_yandex_price_history) СРАЗУ ПОСЛЕ
+save_listings (те же lots): резолвит
listing_id по (source='yandex', source_id), читает последнюю по change_time точку
offer_price_history и идемпотентно дописывает наблюдение, если цена изменилась.
diff --git a/tradein-mvp/backend/app/tasks/avito_detail_backfill.py b/tradein-mvp/backend/app/tasks/avito_detail_backfill.py
index a15dea3f..b7b0fc48 100644
--- a/tradein-mvp/backend/app/tasks/avito_detail_backfill.py
+++ b/tradein-mvp/backend/app/tasks/avito_detail_backfill.py
@@ -8,7 +8,8 @@ City sweep ENRICH_DETAIL only enriches top-N proximate listings per anchor.
Legacy listings (older than 2h or outside radius) are never enriched.
Solution: single snapshot SELECT at start (guarantees termination), same proxy
-session path as scrape_pipeline.py step 5. Block handling mirrors step 5:
+session path as the detail-phase of `run_avito_city_sweep`
+(scraper_kit.orchestration.pipeline). Block handling mirrors that phase:
rotate IP on every block, abort after max_consecutive_blocks (mark_done not
mark_failed -- block is temporary, retry next night via NULL detail_enriched_at).
"""
@@ -176,7 +177,7 @@ async def run_avito_detail_backfill(
session = await build_warmed_session()
elif not use_curl:
# curl_cffi legacy path (scraper_fetch_mode="curl_cffi", use_curl=False):
- # строим shared сессию через auv, как делал scrape_pipeline.
+ # строим shared сессию через auv, как делает run_avito_city_sweep (kit).
own_session = True
session = AsyncSession(
impersonate="chrome120",
@@ -280,7 +281,7 @@ async def run_avito_detail_backfill(
source_url: str = row["source_url"]
counters.attempted += 1
- # Normalise URL -> path (mirrors scrape_pipeline.py line 311)
+ # Normalise URL -> path (mirrors run_avito_city_sweep detail-phase, kit)
item_url = urlparse(source_url).path if source_url.startswith("http") else source_url
# use_curl warm-batch (#1551): интервалы по размеру SERP-страницы (~50).
diff --git a/tradein-mvp/backend/tests/scrapers/test_avito_anti_bot.py b/tradein-mvp/backend/tests/scrapers/test_avito_anti_bot.py
index b2b36e31..219ffed3 100644
--- a/tradein-mvp/backend/tests/scrapers/test_avito_anti_bot.py
+++ b/tradein-mvp/backend/tests/scrapers/test_avito_anti_bot.py
@@ -1,6 +1,6 @@
"""Test anti-bot hardening: blocked exceptions + early abort."""
-from unittest.mock import AsyncMock, MagicMock, call, patch
+from unittest.mock import AsyncMock, MagicMock, patch
import pytest
@@ -170,169 +170,13 @@ async def test_fetch_around_raises_rate_limited_on_429() -> None:
await scraper.fetch_around(56.84, 60.60, 1000, pages=1)
-# ── Pipeline abort tests ────────────────────────────────────────
-
-
-@pytest.mark.asyncio
-async def test_pipeline_aborts_after_3_consecutive_blocks() -> None:
- """#1820: 3 consecutive 403 в detail-фазе прерывают enrichment, но НЕ роняют прогон.
-
- Раньше pipeline re-raise'ил AvitoBlockedError (один блок ронял весь прогон). Теперь
- блок ловится пер-item, и лишь N consecutive блоков прерывают enrichment-фазу —
- PipelineResult возвращается, detail_failed считается, лишние detail-фетчи не делаются.
- """
- from app.services.scrape_pipeline import (
- _AVITO_PIPELINE_CONSECUTIVE_BLOCK_ABORT,
- PipelineResult,
- run_avito_pipeline,
- )
-
- mock_db = MagicMock()
- mock_scraper = MagicMock()
- mock_scraper.__aenter__ = AsyncMock(return_value=mock_scraper)
- mock_scraper.__aexit__ = AsyncMock(return_value=None)
- mock_scraper.fetch_around = AsyncMock(return_value=[])
- mock_scraper._cffi = None
-
- mock_row = MagicMock()
- mock_row.__getitem__ = lambda self, key: (
- "https://www.avito.ru/ekaterinburg/kvartiry/test-1" if key == "source_url" else None
- )
- mock_mappings = MagicMock()
- mock_mappings.all.return_value = [mock_row, mock_row, mock_row, mock_row]
- mock_execute = MagicMock()
- mock_execute.mappings.return_value = mock_mappings
- mock_db.execute.return_value = mock_execute
-
- block_error = AvitoBlockedError("HTTP 403")
- detail_mock = AsyncMock(side_effect=block_error)
-
- with (
- patch("app.services.scrape_pipeline.AvitoScraper", return_value=mock_scraper),
- patch("app.services.scrape_pipeline.fetch_detail", detail_mock),
- patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
- patch(
- "curl_cffi.requests.AsyncSession",
- return_value=AsyncMock(__aenter__=AsyncMock(), __aexit__=AsyncMock()),
- ),
- ):
- # НЕ должно бросать наружу
- result = await run_avito_pipeline(
- mock_db,
- 56.8,
- 60.6,
- 1000,
- enrich_houses=False,
- enrich_detail_top_n=4,
- )
-
- assert isinstance(result, PipelineResult)
- # enrichment прерван ровно после порога consecutive блоков (4-я строка не фетчилась)
- assert detail_mock.await_count == _AVITO_PIPELINE_CONSECUTIVE_BLOCK_ABORT
- assert result.counters.detail_attempted == _AVITO_PIPELINE_CONSECUTIVE_BLOCK_ABORT
- assert result.counters.detail_failed == _AVITO_PIPELINE_CONSECUTIVE_BLOCK_ABORT
- # в errors зафиксирован abort
- assert any("AVITO_BLOCKED" in e for e in result.counters.errors)
-
-
-@pytest.mark.asyncio
-async def test_pipeline_sleep_between_detail_requests() -> None:
- """Pipeline должен вызывать asyncio.sleep между каждым detail request."""
- from app.services.scrape_pipeline import run_avito_pipeline
-
- mock_db = MagicMock()
- mock_scraper = MagicMock()
- mock_scraper.__aenter__ = AsyncMock(return_value=mock_scraper)
- mock_scraper.__aexit__ = AsyncMock(return_value=None)
- mock_scraper.fetch_around = AsyncMock(return_value=[])
- mock_scraper._cffi = None
-
- mock_row = MagicMock()
- mock_row.__getitem__ = lambda self, key: (
- "https://www.avito.ru/ekaterinburg/kvartiry/test-1" if key == "source_url" else None
- )
- mock_mappings = MagicMock()
- mock_mappings.all.return_value = [mock_row, mock_row, mock_row]
- mock_execute = MagicMock()
- mock_execute.mappings.return_value = mock_mappings
- mock_db.execute.return_value = mock_execute
-
- mock_enrichment = MagicMock()
- sleep_mock = AsyncMock()
-
- with (
- patch("app.services.scrape_pipeline.AvitoScraper", return_value=mock_scraper),
- patch(
- "app.services.scrape_pipeline.fetch_detail",
- AsyncMock(return_value=mock_enrichment),
- ),
- patch("app.services.scrape_pipeline.save_detail_enrichment", return_value=True),
- patch("app.services.scrape_pipeline.asyncio.sleep", sleep_mock),
- patch(
- "curl_cffi.requests.AsyncSession",
- return_value=AsyncMock(__aenter__=AsyncMock(), __aexit__=AsyncMock()),
- ),
- patch("app.services.scrape_pipeline.random.uniform", return_value=1.0),
- ):
- await run_avito_pipeline(
- mock_db,
- 56.8,
- 60.6,
- 1000,
- enrich_houses=False,
- enrich_detail_top_n=3,
- request_delay_sec=7.0,
- )
-
- assert sleep_mock.call_count == 2
- for c in sleep_mock.call_args_list:
- assert c == call(7.0)
-
-
-@pytest.mark.asyncio
-async def test_city_sweep_marks_banned_on_block() -> None:
- """City sweep mark'ит run как banned при AvitoBlockedError.
-
- Migration 015 задокументировал 'banned' как 'Avito вернул 403/captcha' — именно этот сценарий.
-
- Патчим AvitoScraper.fetch_around (не run_avito_pipeline — он больше не вызывается
- из run_avito_city_sweep; sweep использует внутренние _avito_anchor_phases).
- """
- from app.services.scrape_pipeline import run_avito_city_sweep
- from app.services.scrapers.avito import AvitoScraper
-
- mock_db = MagicMock()
- mock_runs = MagicMock()
- mock_runs.is_cancelled.return_value = False
- mock_runs.update_heartbeat = MagicMock()
- mock_runs.mark_banned = MagicMock()
- mock_runs.mark_done = MagicMock()
-
- block_error = AvitoBlockedError("IP blocked at page=1")
-
- with (
- patch("app.services.scrape_pipeline.scrape_runs", mock_runs),
- patch.object(AvitoScraper, "fetch_around", AsyncMock(side_effect=block_error)),
- patch(
- "curl_cffi.requests.AsyncSession",
- return_value=AsyncMock(
- __aenter__=AsyncMock(return_value=AsyncMock()),
- __aexit__=AsyncMock(return_value=None),
- ),
- ),
- ):
- result = await run_avito_city_sweep(
- mock_db,
- run_id=42,
- anchors=[(56.84, 60.60, "Центр"), (56.79, 60.53, "ЮЗ")],
- )
-
- assert mock_runs.mark_banned.called
- assert not mock_runs.mark_done.called
- assert result.anchors_done == 1
-
-
# ── #623 firewall-детект + ротация IP ────────────────────────────
+#
+# Legacy `run_avito_pipeline` / `run_avito_city_sweep` orchestration-level tests
+# (pipeline consecutive-block abort, inter-detail sleep, city-sweep mark_banned)
+# удалены вместе с `app.services.scrape_pipeline` (#2397 Part E1) — эквивалентная
+# regression-coverage теперь в `test_scraper_kit_pipeline_parity.py` (kit
+# `run_avito_city_sweep`: ban/rotation state-machine, detail-block propagation).
_CARD_HTML = (
"<html><body>"
@@ -437,20 +281,20 @@ async def test_fetch_around_blocked_when_rotation_exhausted(
# ── #623 sweep-pipeline proxy (SERP+detail через mobile proxy) ───
-def test_avito_proxies_helper(monkeypatch: pytest.MonkeyPatch) -> None:
- """_avito_proxies: None без env, dict {http,https} при заданном AVITO_PROXY_URL.
+def test_avito_proxies_helper() -> None:
+ """Kit _avito_proxies(config): None без proxy, dict {http,https} при заданном URL.
Регрессия: sweep создаёт shared-сессию, через неё идут и SERP, и detail —
без прокси detail летит с datacenter-IP → 429 → run banned (#623 followup).
"""
- from app.services import scrape_pipeline as sp
+ from types import SimpleNamespace
- monkeypatch.setattr(sp.settings, "avito_proxy_url", None)
- assert sp._avito_proxies() is None
+ from scraper_kit.orchestration.pipeline import _avito_proxies
+
+ assert _avito_proxies(SimpleNamespace(scraper_proxy_url=None)) is None
url = "http://u:p@mproxy.site:16928"
- monkeypatch.setattr(sp.settings, "avito_proxy_url", url)
- assert sp._avito_proxies() == {"http": url, "https": url}
+ assert _avito_proxies(SimpleNamespace(scraper_proxy_url=url)) == {"http": url, "https": url}
# ── curl_cffi 429 short-retry (backconnect conn-limit transient) ──────────────
diff --git a/tradein-mvp/backend/tests/test_anchor_watchdog.py b/tradein-mvp/backend/tests/test_anchor_watchdog.py
deleted file mode 100644
index 70fb01fc..00000000
--- a/tradein-mvp/backend/tests/test_anchor_watchdog.py
+++ /dev/null
@@ -1,670 +0,0 @@
-"""Unit tests for per-anchor watchdog timeout in city sweeps (#880).
-
-Verifies that ANCHOR_TIMEOUT_SEC is applied to all three sweeps (avito / cian / yandex):
- (a) A hung anchor (asyncio.sleep(large)) doesn't freeze the whole sweep.
- (b) TimeoutError branch: errors_count incremented, anchor logged as timed-out.
- (c) Sweep proceeds to the next anchor after the timeout.
- (d) mark_done is called when the sweep finishes all anchors.
-
-All tests are network-free — they monkeypatch the inner scrape call.
-"""
-
-from __future__ import annotations
-
-import asyncio
-import os
-
-os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
-
-from typing import Any
-from unittest.mock import AsyncMock, MagicMock
-
-import pytest
-
-from app.services import scrape_pipeline
-from app.services.scrapers.base import ScrapedLot
-from app.services.scrapers.cian import CianScraper
-from app.services.scrapers.yandex_realty import YandexRealtyScraper
-
-# Two anchors: first hangs (causes timeout), second succeeds.
-TWO_ANCHORS: list[tuple[float, float, str]] = [
- (56.8400, 60.6050, "Hang"),
- (56.7950, 60.5300, "OK"),
-]
-
-
-# ── Shared fake scrape_runs ─────────────────────────────────────────────────
-
-
-class _FakeRuns:
- def __init__(self) -> None:
- self.heartbeats: list[dict[str, int]] = []
- self.done: dict[str, int] | None = None
- self.failed: tuple[str, dict[str, int]] | None = None
- self.banned: tuple[str, dict[str, int]] | None = None
-
- def is_cancelled(self, _db: Any, _run_id: int) -> bool:
- return False
-
- def update_heartbeat(self, _db: Any, _run_id: int, counters: dict[str, int]) -> None:
- self.heartbeats.append(dict(counters))
-
- def mark_done(self, _db: Any, _run_id: int, counters: dict[str, int]) -> None:
- self.done = dict(counters)
-
- def mark_failed(self, _db: Any, _run_id: int, error: str, counters: dict[str, int]) -> None:
- self.failed = (error, dict(counters))
-
- def mark_banned(self, _db: Any, _run_id: int, error: str, counters: dict[str, int]) -> None:
- self.banned = (error, dict(counters))
-
-
-def _install_runs(monkeypatch: pytest.MonkeyPatch, fake: _FakeRuns) -> None:
- monkeypatch.setattr(scrape_pipeline.scrape_runs, "is_cancelled", fake.is_cancelled)
- monkeypatch.setattr(scrape_pipeline.scrape_runs, "update_heartbeat", fake.update_heartbeat)
- monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_done", fake.mark_done)
- monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_failed", fake.mark_failed)
- monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_banned", fake.mark_banned)
-
-
-def _fake_lot(source: str, offer_id: str) -> ScrapedLot:
- return ScrapedLot(
- source=source,
- source_url=f"https://example.ru/{offer_id}/",
- source_id=offer_id,
- address="Екатеринбург, улица Тестовая, 1",
- price_rub=4_000_000,
- area_m2=42.0,
- rooms=1,
- )
-
-
-# ── CianScraper lifecycle stub ──────────────────────────────────────────────
-
-
-@pytest.fixture(autouse=True)
-def _stub_cian_lifecycle(monkeypatch: pytest.MonkeyPatch) -> None:
- def _init(self: CianScraper) -> None:
- self.request_delay_sec = 0.0
- self._cffi = None
-
- async def _aenter(self: CianScraper) -> CianScraper:
- return self
-
- async def _aexit(self: CianScraper, *_args: Any) -> None:
- return None
-
- monkeypatch.setattr(CianScraper, "__init__", _init)
- monkeypatch.setattr(CianScraper, "__aenter__", _aenter)
- monkeypatch.setattr(CianScraper, "__aexit__", _aexit)
-
-
-# ── YandexRealtyScraper lifecycle stub ─────────────────────────────────────
-
-
-@pytest.fixture(autouse=True)
-def _stub_yandex_lifecycle(monkeypatch: pytest.MonkeyPatch) -> None:
- def _init(self: YandexRealtyScraper, city: str = "ekaterinburg") -> None:
- self.city = city
- self.request_delay_sec = 0.0
- self._cffi_session = None
- self._cookies = {}
-
- async def _aenter(self: YandexRealtyScraper) -> YandexRealtyScraper:
- return self
-
- async def _aexit(self: YandexRealtyScraper, *_args: Any) -> None:
- return None
-
- monkeypatch.setattr(YandexRealtyScraper, "__init__", _init)
- monkeypatch.setattr(YandexRealtyScraper, "__aenter__", _aenter)
- monkeypatch.setattr(YandexRealtyScraper, "__aexit__", _aexit)
-
-
-# ── ANCHOR_TIMEOUT_SEC is exported ─────────────────────────────────────────
-
-
-def test_anchor_timeout_sec_exported() -> None:
- from app.services.scrape_pipeline import ANCHOR_TIMEOUT_SEC
-
- assert isinstance(ANCHOR_TIMEOUT_SEC, int)
- # Reasonable sane range: 60s–600s
- assert (
- 60 <= ANCHOR_TIMEOUT_SEC <= 600
- ), f"ANCHOR_TIMEOUT_SEC={ANCHOR_TIMEOUT_SEC} out of sane range"
-
-
-# ── Avito sweep watchdog ────────────────────────────────────────────────────
-
-
-async def test_avito_sweep_anchor_timeout_continues(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- """Первый anchor виснет → таймаут; второй anchor успешен; mark_done вызывается.
-
- run_avito_city_sweep использует внутреннюю _avito_anchor_phases (не run_avito_pipeline),
- поэтому патчим AvitoScraper.fetch_around + save_listings.
- """
- from app.services.scrapers.avito import AvitoScraper
-
- # Monkeypatch asyncio.wait_for so we can inject TimeoutError on first call only,
- # without actually waiting in CI.
- original_wait_for = asyncio.wait_for
- wf_call = 0
-
- async def fake_wait_for(coro: Any, *, timeout: float) -> Any:
- nonlocal wf_call
- wf_call += 1
- if wf_call == 1:
- # Simulate the hung anchor: cancel the coro (mimic wait_for behaviour)
- # and raise TimeoutError.
- try:
- coro.close()
- except Exception:
- pass
- raise TimeoutError
- return await original_wait_for(coro, timeout=timeout)
-
- monkeypatch.setattr(scrape_pipeline.asyncio, "wait_for", fake_wait_for)
- monkeypatch.setattr(scrape_pipeline.asyncio, "sleep", AsyncMock())
-
- # Stub AvitoScraper.fetch_around — second anchor returns 3 lots.
- fetch_calls: list[str] = []
-
- async def fake_fetch_around(
- self: AvitoScraper,
- lat: float,
- lon: float,
- radius_m: int,
- pages: int = 1,
- delay_override_sec: float | None = None,
- ) -> list[ScrapedLot]:
- fetch_calls.append(f"{lat:.4f}")
- return [_fake_lot("avito", f"{lat:.4f}-{i}") for i in range(3)]
-
- monkeypatch.setattr(AvitoScraper, "fetch_around", fake_fetch_around)
- monkeypatch.setattr(scrape_pipeline, "save_listings", lambda _db, lots, **_kw: (len(lots), 0))
-
- # Stub shared AsyncSession (used by avito sweep).
- fake_session = AsyncMock()
- fake_session.__aenter__ = AsyncMock(return_value=fake_session)
- fake_session.__aexit__ = AsyncMock(return_value=None)
- import curl_cffi.requests as _cffi_mod
-
- monkeypatch.setattr(_cffi_mod, "AsyncSession", lambda **_kw: fake_session)
-
- # Stub IMV phase (not relevant to this test).
- async def fake_imv_batch(*_a: Any, **_kw: Any) -> Any:
- result = MagicMock()
- result.checked = 0
- result.saved = 0
- result.errors = 0
- return result
-
- monkeypatch.setattr(
- "app.services.house_imv_backfill.process_houses_imv_batch",
- fake_imv_batch,
- )
-
- fake = _FakeRuns()
- _install_runs(monkeypatch, fake)
-
- counters = await scrape_pipeline.run_avito_city_sweep(
- db=MagicMock(),
- run_id=1,
- anchors=TWO_ANCHORS,
- enrich_houses=False,
- detail_top_n=0,
- enrich_imv=False,
- request_delay_sec=0.0,
- )
-
- # (a) sweep completes — doesn't hang
- # (b) first anchor timed out → errors_count >= 1
- assert counters.errors_count >= 1, "timeout должен увеличить errors_count"
- # (c) second anchor reached: lots_fetched > 0
- assert counters.lots_fetched > 0, "второй anchor должен был выполниться"
- # (d) mark_done вызван (sweep завершился)
- assert fake.done is not None, "mark_done должен быть вызван"
- assert fake.failed is None, "mark_failed не должен быть вызван"
- assert counters.anchors_done == len(TWO_ANCHORS)
-
-
-# ── Cian sweep watchdog ─────────────────────────────────────────────────────
-
-
-async def test_cian_sweep_anchor_timeout_continues(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- """Первый anchor виснет → таймаут; второй anchor успешен; mark_done вызывается."""
- serp_calls: list[str] = []
- original_wait_for = asyncio.wait_for
- wf_call = 0
-
- async def fake_wait_for(coro: Any, *, timeout: float) -> Any:
- nonlocal wf_call
- wf_call += 1
- if wf_call == 1:
- try:
- coro.close()
- except Exception:
- pass
- raise TimeoutError
- return await original_wait_for(coro, timeout=timeout)
-
- monkeypatch.setattr(scrape_pipeline.asyncio, "wait_for", fake_wait_for)
- monkeypatch.setattr(scrape_pipeline.asyncio, "sleep", AsyncMock())
-
- async def fake_fetch_multi(
- self: CianScraper, lat: float, lon: float, r: int, pages: int = 3
- ) -> list[ScrapedLot]:
- serp_calls.append(f"{lat:.4f}")
- return [_fake_lot("cian", f"{lat:.4f}-1")]
-
- monkeypatch.setattr(CianScraper, "fetch_around_multi_room", fake_fetch_multi)
- monkeypatch.setattr(
- scrape_pipeline, "save_listings", lambda _db, lots, *, run_id=None: (len(lots), 0)
- )
- monkeypatch.setattr(
- "app.services.scrapers.cian_detail.fetch_detail",
- AsyncMock(return_value=None),
- )
-
- fake = _FakeRuns()
- _install_runs(monkeypatch, fake)
-
- counters = await scrape_pipeline.run_cian_city_sweep(
- db=MagicMock(),
- run_id=2,
- anchors=TWO_ANCHORS,
- detail_top_n=0,
- enrich_houses=False,
- request_delay_sec=0.0,
- )
-
- # (a) sweep completes (doesn't hang at anchor 1)
- # (b) errors_count >= 1 (timeout counted)
- assert counters.errors_count >= 1
- # (c) second anchor was reached (SERP called once — first anchor timed out before SERP)
- # wf_call==2 means the second wait_for ran → second anchor's coro executed.
- assert wf_call == 2, f"wait_for should be called once per anchor; got {wf_call}"
- # (d) mark_done called
- assert fake.done is not None
- assert fake.failed is None
- assert counters.anchors_done == len(TWO_ANCHORS)
-
-
-# ── Yandex sweep watchdog ───────────────────────────────────────────────────
-
-
-async def test_yandex_sweep_anchor_timeout_continues(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- """Первый anchor виснет → таймаут; второй anchor успешен; mark_done вызывается."""
- serp_calls: list[str] = []
- original_wait_for = asyncio.wait_for
- wf_call = 0
-
- async def fake_wait_for(coro: Any, *, timeout: float) -> Any:
- nonlocal wf_call
- wf_call += 1
- if wf_call == 1:
- try:
- coro.close()
- except Exception:
- pass
- raise TimeoutError
- return await original_wait_for(coro, timeout=timeout)
-
- monkeypatch.setattr(scrape_pipeline.asyncio, "wait_for", fake_wait_for)
- monkeypatch.setattr(scrape_pipeline.asyncio, "sleep", AsyncMock())
-
- async def fake_fetch_multi(
- self: YandexRealtyScraper,
- lat: float,
- lon: float,
- radius_m: int = 1500,
- max_pages: int = 2,
- **_: Any,
- ) -> list[ScrapedLot]:
- serp_calls.append(f"{lat:.4f}")
- return [_fake_lot("yandex", f"{lat:.4f}-1")]
-
- monkeypatch.setattr(YandexRealtyScraper, "fetch_around_multi_room", fake_fetch_multi)
- monkeypatch.setattr(
- scrape_pipeline, "save_listings", lambda _db, lots, *, run_id=None: (len(lots), 0)
- )
-
- fake = _FakeRuns()
- _install_runs(monkeypatch, fake)
-
- counters = await scrape_pipeline.run_yandex_city_sweep(
- db=MagicMock(),
- run_id=3,
- anchors=TWO_ANCHORS,
- pages_per_anchor=1,
- enrich_address=False,
- request_delay_sec=0.0,
- )
-
- # (a) sweep completes
- # (b) timeout counted
- assert counters.errors_count >= 1
- # (c) second anchor reached
- assert wf_call == 2
- # (d) mark_done called
- assert fake.done is not None
- assert fake.failed is None
- assert counters.anchors_done == len(TWO_ANCHORS)
-
-
-# ── All three: timeout increments errors_count and is logged, sweep reaches mark_done ──
-
-
-async def test_cian_timeout_increments_counter_and_marks_done(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- """Детальная проверка: timeout anchor → errors_count+1, sweep продолжается, mark_done.
-
- Использует fake_wait_for (зеркало первых трёх тестов): первый вызов → TimeoutError
- (имитирует зависший anchor), второй вызов → реальный wait_for (нормальный anchor).
- Проверяет что:
- - errors_count увеличивается
- - второй anchor выполнился (SERP вызван)
- - mark_done вызван (sweep завершился)
- - mark_banned НЕ вызван (одиночный таймаут < порога consecutive failures)
- """
- original_wait_for = asyncio.wait_for
- wf_call = 0
-
- async def fake_wait_for(coro: Any, *, timeout: float) -> Any:
- nonlocal wf_call
- wf_call += 1
- if wf_call == 1:
- try:
- coro.close()
- except Exception:
- pass
- raise TimeoutError
- return await original_wait_for(coro, timeout=timeout)
-
- monkeypatch.setattr(scrape_pipeline.asyncio, "wait_for", fake_wait_for)
- monkeypatch.setattr(scrape_pipeline.asyncio, "sleep", AsyncMock())
-
- serp_calls: list[str] = []
-
- async def fake_fetch(
- self: CianScraper, lat: float, lon: float, r: int, pages: int = 3
- ) -> list[ScrapedLot]:
- serp_calls.append(f"{lat:.4f}")
- return [_fake_lot("cian", f"{lat:.4f}-ok")]
-
- monkeypatch.setattr(CianScraper, "fetch_around_multi_room", fake_fetch)
- monkeypatch.setattr(
- scrape_pipeline,
- "save_listings",
- lambda _db, lots, *, run_id=None: (len(lots), 0),
- )
- monkeypatch.setattr(
- "app.services.scrapers.cian_detail.fetch_detail",
- AsyncMock(return_value=None),
- )
-
- fake = _FakeRuns()
- _install_runs(monkeypatch, fake)
-
- counters = await scrape_pipeline.run_cian_city_sweep(
- db=MagicMock(),
- run_id=10,
- anchors=TWO_ANCHORS,
- detail_top_n=0,
- enrich_houses=False,
- request_delay_sec=0.0,
- )
-
- # (b) timeout branch: errors_count incremented
- assert counters.errors_count >= 1, "timeout должен увеличить errors_count"
- # (c) second anchor reached (wait_for was called twice — once per anchor)
- assert wf_call == 2, f"wait_for должен быть вызван для каждого anchor; got {wf_call}"
- # (d) sweep reached mark_done
- assert fake.done is not None, "mark_done должен быть вызван"
- assert fake.banned is None, "mark_banned не должен быть вызван для одиночного таймаута"
- assert counters.anchors_done == len(TWO_ANCHORS)
-
-
-# ── (A) Timeout scaling — avito_anchor_timeout > ANCHOR_TIMEOUT_SEC ───────────
-
-
-def test_avito_anchor_timeout_scaling() -> None:
- """_avito_anchor_timeout масштабируется по detail_top_n и enrich_houses.
-
- При detail_top_n=20 и enrich_houses=True таймаут должен быть существенно выше
- ANCHOR_TIMEOUT_SEC (240), чтобы detail-фаза не guillotine'ила SERP-фазу.
- """
- from app.services.scrape_pipeline import (
- _AVITO_HOUSES_BUDGET_S,
- _AVITO_PER_DETAIL_S,
- ANCHOR_TIMEOUT_SEC,
- )
-
- detail_top_n = 20
- enrich_houses = True
- computed = (
- float(ANCHOR_TIMEOUT_SEC)
- + detail_top_n * _AVITO_PER_DETAIL_S
- + (_AVITO_HOUSES_BUDGET_S if enrich_houses else 0.0)
- )
- # Должен быть значительно больше базового (минимум вдвое при n=20+houses)
- assert (
- computed > ANCHOR_TIMEOUT_SEC * 2
- ), f"computed timeout {computed} должен быть > 2× base {ANCHOR_TIMEOUT_SEC}"
- # Без detail и без houses — ровно ANCHOR_TIMEOUT_SEC
- computed_no_detail = float(ANCHOR_TIMEOUT_SEC) + 0 * _AVITO_PER_DETAIL_S + 0.0
- assert computed_no_detail == float(ANCHOR_TIMEOUT_SEC)
-
- # Новые константы экспортированы / импортируются без ошибок
- assert _AVITO_PER_DETAIL_S > 0
- assert _AVITO_HOUSES_BUDGET_S > 0
-
-
-# ── (B) SERP counters durable across detail-phase timeout ────────────────────
-
-
-async def test_avito_serp_counters_durable_after_detail_timeout(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- """SERP+save счётчики сохраняются в sweep-level counters даже когда detail-фаза
- вызывает TimeoutError из wait_for.
-
- Симулируем: первый anchor — SERP возвращает 5 lots, save успешен, затем wait_for
- завершается TimeoutError (detail-фаза «зависла»). Второй anchor — полный успех.
-
- Ожидаем:
- - counters.lots_fetched == 5 (SERP первого anchor'а) + лоты второго
- - counters.lots_inserted >= 1 (save первого anchor'а)
- - counters.errors_count >= 1 (timeout первого anchor'а)
- - mark_done вызван (sweep завершился)
- """
- import curl_cffi.requests as _cffi_mod
-
- from app.services.scrapers.avito import AvitoScraper
-
- # Отслеживаем вызовы fetch_around
- serp_calls: list[str] = []
- save_calls: list[int] = []
-
- original_wait_for = asyncio.wait_for
- wf_call = 0
-
- async def fake_wait_for(coro: Any, *, timeout: float) -> Any:
- nonlocal wf_call
- wf_call += 1
- if wf_call == 1:
- # Запускаем корутину до конца — она дойдёт до SERP+save,
- # а затем TimeoutError наступает как если бы detail-фаза зависла.
- # Для упрощения: просто инжектируем TimeoutError (как во всех других тестах),
- # но проверяем, что counter-патч произошёл ДО wait_for через отдельный
- # механизм: monkeypatch fetch_around + save_listings записывают данные
- # в counters до timeout-исключения.
- try:
- coro.close()
- except Exception:
- pass
- raise TimeoutError
- return await original_wait_for(coro, timeout=timeout)
-
- monkeypatch.setattr(scrape_pipeline.asyncio, "wait_for", fake_wait_for)
- monkeypatch.setattr(scrape_pipeline.asyncio, "sleep", AsyncMock())
-
- # Stub shared AsyncSession
- fake_session = AsyncMock()
- fake_session.__aenter__ = AsyncMock(return_value=fake_session)
- fake_session.__aexit__ = AsyncMock(return_value=None)
- monkeypatch.setattr(_cffi_mod, "AsyncSession", lambda **_kw: fake_session)
-
- # Stub AvitoScraper.fetch_around — возвращает lots (не AvitoBlockedError)
- async def fake_fetch_around(
- self: AvitoScraper,
- lat: float,
- lon: float,
- radius_m: int,
- pages: int = 1,
- delay_override_sec: float | None = None,
- ) -> list[ScrapedLot]:
- serp_calls.append(f"{lat:.4f}")
- return [_fake_lot("avito", f"{lat:.4f}-{i}") for i in range(5)]
-
- monkeypatch.setattr(AvitoScraper, "fetch_around", fake_fetch_around)
-
- # Stub save_listings — записывает lots count
- def fake_save_listings(db: Any, lots: Any, *, run_id: int | None = None) -> tuple[int, int]:
- save_calls.append(len(lots))
- return (len(lots), 0)
-
- monkeypatch.setattr(scrape_pipeline, "save_listings", fake_save_listings)
-
- # Stub run_avito_pipeline — используем реальный run_avito_city_sweep
- # но с внутренним патчем, поэтому НЕ monkeypatch run_avito_pipeline.
- # Важно: inner _avito_anchor_phases вызывает AvitoScraper и save_listings
- # через nonlocal counters ПЕРЕД wait_for возвращает TimeoutError.
-
- async def fake_imv_batch(*_a: Any, **_kw: Any) -> Any:
- result = MagicMock()
- result.checked = 0
- result.saved = 0
- result.errors = 0
- return result
-
- monkeypatch.setattr(
- "app.services.house_imv_backfill.process_houses_imv_batch",
- fake_imv_batch,
- )
-
- fake = _FakeRuns()
- _install_runs(monkeypatch, fake)
-
- # Stub fetch_house_catalog и fetch_detail чтобы не было реальных сетевых вызовов
- # (они могут вызваться из _avito_anchor_phases если timeout не наступит раньше)
- monkeypatch.setattr(
- scrape_pipeline,
- "fetch_house_catalog",
- AsyncMock(side_effect=RuntimeError("should not be called in timeout test")),
- )
- monkeypatch.setattr(
- scrape_pipeline,
- "fetch_detail",
- AsyncMock(side_effect=RuntimeError("should not be called in timeout test")),
- )
-
- counters = await scrape_pipeline.run_avito_city_sweep(
- db=MagicMock(),
- run_id=42,
- anchors=TWO_ANCHORS,
- enrich_houses=False,
- detail_top_n=0,
- enrich_imv=False,
- request_delay_sec=0.0,
- )
-
- # SERP первого anchor'а НЕ дойдёт до счётчиков т.к. wait_for killит coro СРАЗУ
- # (fake_wait_for делает coro.close() до любого await внутри). Это ожидаемо при
- # coro.close() — реальный сценарий другой: SERP завершается, затем detail зависает.
- # Тест проверяет sweep-level инварианты:
- # - errors_count >= 1 (timeout)
- # - sweep не падает
- # - mark_done вызван
- # - второй anchor дошёл (wf_call==2)
- assert counters.errors_count >= 1, "timeout должен увеличить errors_count"
- assert fake.done is not None, "mark_done должен быть вызван"
- assert fake.failed is None, "mark_failed не должен быть вызван"
- assert wf_call == 2, f"оба anchor'а должны пройти через wait_for; got {wf_call}"
- assert counters.anchors_done == len(TWO_ANCHORS)
-
-
-async def test_avito_serp_counters_durable_real_phases(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- """Проверяем что SERP+save счётчики попадают в sweep-level counters когда
- _avito_anchor_phases выполняется нормально (без timeout): lots_fetched/inserted
- отражают реальные данные.
-
- Патчим wait_for через реальный asyncio.wait_for с очень большим timeout —
- anchor_phases выполняется полностью. SERP возвращает 7 lots, save возвращает (7, 0).
- """
- import curl_cffi.requests as _cffi_mod
-
- from app.services.scrapers.avito import AvitoScraper
-
- monkeypatch.setattr(scrape_pipeline.asyncio, "sleep", AsyncMock())
-
- fake_session = AsyncMock()
- fake_session.__aenter__ = AsyncMock(return_value=fake_session)
- fake_session.__aexit__ = AsyncMock(return_value=None)
- monkeypatch.setattr(_cffi_mod, "AsyncSession", lambda **_kw: fake_session)
-
- async def fake_fetch_around(
- self: AvitoScraper,
- lat: float,
- lon: float,
- radius_m: int,
- pages: int = 1,
- delay_override_sec: float | None = None,
- ) -> list[ScrapedLot]:
- return [_fake_lot("avito", f"{lat:.4f}-{i}") for i in range(7)]
-
- monkeypatch.setattr(AvitoScraper, "fetch_around", fake_fetch_around)
- monkeypatch.setattr(scrape_pipeline, "save_listings", lambda _db, lots, **_kw: (len(lots), 0))
- # Не вызывать house/detail enrich
- monkeypatch.setattr(scrape_pipeline, "fetch_house_catalog", AsyncMock(return_value=None))
- monkeypatch.setattr(scrape_pipeline, "fetch_detail", AsyncMock(return_value=None))
-
- async def fake_imv_batch(*_a: Any, **_kw: Any) -> Any:
- result = MagicMock()
- result.checked = 0
- result.saved = 0
- result.errors = 0
- return result
-
- monkeypatch.setattr(
- "app.services.house_imv_backfill.process_houses_imv_batch",
- fake_imv_batch,
- )
-
- fake = _FakeRuns()
- _install_runs(monkeypatch, fake)
-
- # Один anchor — проверяем счётчики после нормального прохода
- counters = await scrape_pipeline.run_avito_city_sweep(
- db=MagicMock(),
- run_id=99,
- anchors=[(56.8400, 60.6050, "Центр")],
- enrich_houses=False,
- detail_top_n=0,
- enrich_imv=False,
- request_delay_sec=0.0,
- )
-
- assert counters.lots_fetched == 7, f"lots_fetched должен быть 7; got {counters.lots_fetched}"
- assert counters.lots_inserted == 7, f"lots_inserted должен быть 7; got {counters.lots_inserted}"
- assert counters.anchors_done == 1
- assert fake.done is not None
diff --git a/tradein-mvp/backend/tests/test_avito_newbuilding_sweep.py b/tradein-mvp/backend/tests/test_avito_newbuilding_sweep.py
index f90208c9..5aa05d27 100644
--- a/tradein-mvp/backend/tests/test_avito_newbuilding_sweep.py
+++ b/tradein-mvp/backend/tests/test_avito_newbuilding_sweep.py
@@ -6,22 +6,24 @@
newbuilding_id заполнен (DOM-маркер застройщика + zhk-якорь).
3. fetch_newbuildings break-on-empty + dedup (mirror fetch_city_wide).
4. NewbuildingSweepCounters — defaults + to_dict.
-5. run_avito_newbuilding_sweep wiring — мокаем fetch_newbuildings + save_listings,
- assert save вызван и SERP-счётчики записаны (mark_done).
+
+Legacy `run_avito_newbuilding_sweep` orchestration wiring (fetch_newbuildings +
+save_listings + mark_done) удалён вместе с `app.services.scrape_pipeline` (#2397
+Part E1) — эквивалентная regression-coverage теперь в
+`test_scraper_kit_pipeline_parity2.py::test_avito_newbuilding_sweep` (kit
+`run_avito_newbuilding_sweep`).
"""
from __future__ import annotations
import os
from pathlib import Path
-from typing import Any
-from unittest.mock import AsyncMock, MagicMock, patch
+from unittest.mock import AsyncMock, patch
import pytest
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
-from app.services import scrape_pipeline
from app.services.scrapers.avito import NOVOSTROYKA_SLUG, AvitoScraper
from app.services.scrapers.base import ScrapedLot
@@ -148,7 +150,7 @@ def test_fetch_city_wide_url_unchanged() -> None:
def test_newbuilding_counters_defaults_and_to_dict() -> None:
- from app.services.scrape_pipeline import NewbuildingSweepCounters
+ from scraper_kit.orchestration.pipeline import NewbuildingSweepCounters
c = NewbuildingSweepCounters()
assert c.lots_fetched == 0
@@ -157,89 +159,3 @@ def test_newbuilding_counters_defaults_and_to_dict() -> None:
assert c.errors_count == 0
d = c.to_dict()
assert set(d.keys()) == {"lots_fetched", "lots_inserted", "lots_updated", "errors_count"}
-
-
-# ── 5. run_avito_newbuilding_sweep wiring ───────────────────────────────────
-
-
-class _FakeRuns:
- def __init__(self) -> None:
- self.heartbeats: list[dict[str, int]] = []
- self.done: dict[str, int] | None = None
- self.failed: tuple[str, dict[str, int]] | None = None
- self.banned: tuple[str, dict[str, int]] | None = None
-
- def is_cancelled(self, _db: Any, _run_id: int) -> bool:
- return False
-
- def update_heartbeat(self, _db: Any, _run_id: int, counters: dict[str, int]) -> None:
- self.heartbeats.append(dict(counters))
-
- def mark_done(self, _db: Any, _run_id: int, counters: dict[str, int]) -> None:
- self.done = dict(counters)
-
- def mark_failed(self, _db: Any, _run_id: int, error: str, counters: dict[str, int]) -> None:
- self.failed = (error, dict(counters))
-
- def mark_banned(self, _db: Any, _run_id: int, error: str, counters: dict[str, int]) -> None:
- self.banned = (error, dict(counters))
-
-
-def _install_runs(monkeypatch: pytest.MonkeyPatch, fake: _FakeRuns) -> None:
- monkeypatch.setattr(scrape_pipeline.scrape_runs, "is_cancelled", fake.is_cancelled)
- monkeypatch.setattr(scrape_pipeline.scrape_runs, "update_heartbeat", fake.update_heartbeat)
- monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_done", fake.mark_done)
- monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_failed", fake.mark_failed)
- monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_banned", fake.mark_banned)
-
-
-@pytest.mark.asyncio
-async def test_run_avito_newbuilding_sweep_saves_and_counts(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- fake_runs = _FakeRuns()
- _install_runs(monkeypatch, fake_runs)
-
- fake_lots = [_make_lot("N1"), _make_lot("N2"), _make_lot("N3")]
-
- async def fake_fetch_newbuildings(
- self: AvitoScraper,
- pages: int = 30,
- *,
- delay_override_sec: float | None = None,
- ) -> list[ScrapedLot]:
- return list(fake_lots)
-
- monkeypatch.setattr(AvitoScraper, "fetch_newbuildings", fake_fetch_newbuildings)
-
- save_calls: list[int] = []
-
- def fake_save_listings(_db: Any, lots: Any, **_kw: Any) -> tuple[int, int]:
- save_calls.append(len(lots))
- return (len(lots), 0)
-
- monkeypatch.setattr(scrape_pipeline, "save_listings", fake_save_listings)
- # Force cffi-mode (no browser) so we don't spin up BrowserFetcher.
- monkeypatch.setattr(scrape_pipeline.settings, "scraper_fetch_mode", "cffi")
-
- # Stub shared AsyncSession (used by the sweep in cffi-mode) — no real session.
- fake_session = AsyncMock()
- fake_session.__aenter__ = AsyncMock(return_value=fake_session)
- fake_session.__aexit__ = AsyncMock(return_value=None)
- monkeypatch.setattr(scrape_pipeline, "AsyncSession", lambda **_kw: fake_session)
-
- counters = await scrape_pipeline.run_avito_newbuilding_sweep(
- MagicMock(),
- run_id=42,
- pages=2,
- request_delay_sec=0.0,
- )
-
- assert save_calls == [3]
- assert counters.lots_fetched == 3
- assert counters.lots_inserted == 3
- assert counters.lots_updated == 0
- assert fake_runs.done is not None
- assert fake_runs.done["lots_fetched"] == 3
- assert fake_runs.failed is None
- assert fake_runs.banned is None
diff --git a/tradein-mvp/backend/tests/test_ban_rotation_1790.py b/tradein-mvp/backend/tests/test_ban_rotation_1790.py
deleted file mode 100644
index b4c35122..00000000
--- a/tradein-mvp/backend/tests/test_ban_rotation_1790.py
+++ /dev/null
@@ -1,220 +0,0 @@
-"""#1790 — ban-triggered IP rotation в enrichment-фазах scrape_pipeline.
-
-Тест 1: AvitoBlockedError при доступных ротациях → вызывается _rotate_proxy_ip
- (changeip) + consecutive счётчик сбрасывается → enrichment продолжается.
-
-Тест 2: Ротаций больше нет (бюджет исчерпан) → abort enrichment (bounded N ротаций).
-
-Тест 3: Graceful per-item (#1820) сохранён — один блок без порога ротации НЕ вызывает.
-
-Без сети, без БД.
-"""
-
-from __future__ import annotations
-
-import os
-from unittest.mock import AsyncMock, MagicMock, patch
-
-import pytest
-
-os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
-
-from app.services.scrape_pipeline import (
- _AVITO_PIPELINE_CONSECUTIVE_BLOCK_ABORT,
- PipelineResult,
- run_avito_pipeline,
-)
-from app.services.scrapers.avito_exceptions import AvitoBlockedError
-
-
-def _make_lot(
- item_id: str,
- house_url: str | None = None,
-) -> object:
- from app.services.scrapers.base import ScrapedLot
-
- return ScrapedLot(
- source="avito",
- source_url=f"https://www.avito.ru/ekaterinburg/kvartiry/test_{item_id}",
- source_id=item_id,
- price_rub=5_000_000,
- address="Test address",
- house_source="avito" if house_url else None,
- house_ext_id=house_url.rstrip("/").split("/")[-1] if house_url else None,
- house_url=house_url,
- listing_segment="vtorichka",
- )
-
-
-def _block_scraper(lots: list) -> MagicMock: # type: ignore[type-arg]
- scraper = MagicMock()
- scraper.__aenter__ = AsyncMock(return_value=scraper)
- scraper.__aexit__ = AsyncMock(return_value=None)
- scraper.fetch_around = AsyncMock(return_value=lots)
- scraper._cffi = None
- return scraper
-
-
-# ---------------------------------------------------------------------------
-# Тест 1: ротация срабатывает при consecutive blocks == порог
-# ---------------------------------------------------------------------------
-
-
-@pytest.mark.asyncio
-async def test_rotation_called_on_consecutive_blocks() -> None:
- """При N подряд-блоках (порог) вызывается _rotate_proxy_ip — IP меняется.
-
- После успешной ротации consecutive счётчик сбрасывается и enrichment продолжается:
- следующий (N+1)-й дом фетчится успешно.
- """
- n = _AVITO_PIPELINE_CONSECUTIVE_BLOCK_ABORT
- lots = [
- _make_lot(str(i), f"https://www.avito.ru/catalog/houses/ekb/h{i}/{100 + i}")
- for i in range(n + 2) # n блокируются, потом ротация, потом успех
- ]
- scraper = _block_scraper(lots)
-
- # Первые n обращений — блок, (n+1)-й — успех
- fetch_house = AsyncMock(side_effect=[AvitoBlockedError("banned")] * n + [MagicMock()])
- rotate_mock = AsyncMock(return_value=True) # ротация успешна
-
- with (
- patch("app.services.scrape_pipeline.AvitoScraper", return_value=scraper),
- patch("app.services.scrape_pipeline.save_listings", return_value=(len(lots), 0)),
- patch("app.services.scrape_pipeline.fetch_house_catalog", fetch_house),
- patch(
- "app.services.scrape_pipeline.save_house_catalog_enrichment",
- return_value={"house_id": 99},
- ),
- patch("app.services.scrape_pipeline._rotate_proxy_ip", rotate_mock),
- patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
- patch(
- "curl_cffi.requests.AsyncSession",
- return_value=AsyncMock(__aenter__=AsyncMock(), __aexit__=AsyncMock()),
- ),
- ):
- result = await run_avito_pipeline(
- MagicMock(), 56.8, 60.6, 1000, enrich_houses=True, enrich_detail_top_n=0
- )
-
- assert isinstance(result, PipelineResult)
- # _rotate_proxy_ip вызвана хотя бы один раз
- assert rotate_mock.await_count >= 1
- # Enrichment продолжился после ротации — (n+1)-й дом обогащён
- assert result.counters.houses_enriched >= 1
- # Listings сохранены
- assert result.counters.lots_inserted == len(lots)
-
-
-# ---------------------------------------------------------------------------
-# Тест 2: бюджет ротаций исчерпан → abort
-# ---------------------------------------------------------------------------
-
-
-@pytest.mark.asyncio
-async def test_rotation_bounded_abort_when_budget_exhausted() -> None:
- """После max_rotations попыток ротации enrichment abort'ится (bounded).
-
- Используем avito_proxy_max_rotations=1 → только одна попытка ротации.
- После неё блоки продолжаются → abort.
- """
- n = _AVITO_PIPELINE_CONSECUTIVE_BLOCK_ABORT
- # Достаточно домов чтобы вызвать и ротацию и повторные блоки
- lots = [
- _make_lot(str(i), f"https://www.avito.ru/catalog/houses/ekb/h{i}/{200 + i}")
- for i in range(n * 3)
- ]
- scraper = _block_scraper(lots)
-
- # Все дома блокируются (не будет успешного фетча)
- fetch_house = AsyncMock(side_effect=AvitoBlockedError("banned"))
- # Ротация успешна, но дом после неё тоже блокируется
- rotate_mock = AsyncMock(return_value=True)
-
- with (
- patch("app.services.scrape_pipeline.AvitoScraper", return_value=scraper),
- patch("app.services.scrape_pipeline.save_listings", return_value=(len(lots), 0)),
- patch("app.services.scrape_pipeline.fetch_house_catalog", fetch_house),
- patch("app.services.scrape_pipeline.save_house_catalog_enrichment", return_value={}),
- patch("app.services.scrape_pipeline._rotate_proxy_ip", rotate_mock),
- patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
- patch("app.services.scrape_pipeline.settings") as mock_settings,
- patch(
- "curl_cffi.requests.AsyncSession",
- return_value=AsyncMock(__aenter__=AsyncMock(), __aexit__=AsyncMock()),
- ),
- ):
- # Лимит 1 ротация: после неё бюджет исчерпан → abort на следующей серии блоков
- mock_settings.avito_proxy_max_rotations = 1
- mock_settings.avito_proxy_rotate_url = "http://rotate.test/"
- mock_settings.avito_proxy_rotate_settle_s = 0.0
- mock_settings.scraper_fetch_mode = "curl_cffi"
- mock_settings.scraper_proxy_url = None
- mock_settings.scraper_skip_seen_today = False
- # #2202: bare MagicMock settings auto-truthy on any attribute access — pin
- # explicitly off so _rotate_avito_proxy takes the changeip path this test
- # asserts on (pool path is covered separately in test_scrape_pipeline_proxy_pool.py).
- mock_settings.use_proxy_pool_curl = False
-
- result = await run_avito_pipeline(
- MagicMock(), 56.8, 60.6, 1000, enrich_houses=True, enrich_detail_top_n=0
- )
-
- assert isinstance(result, PipelineResult)
- # Ровно 1 ротация (бюджет = 1)
- assert rotate_mock.await_count == 1
- # Enrichment abort'нут (есть AVITO_BLOCKED в ошибках)
- assert any("AVITO_BLOCKED" in e for e in result.counters.errors)
- # Listings всё равно сохранены
- assert result.counters.lots_inserted == len(lots)
-
-
-# ---------------------------------------------------------------------------
-# Тест 3: один блок (ниже порога) — ротация НЕ вызывается (#1820 graceful)
-# ---------------------------------------------------------------------------
-
-
-@pytest.mark.asyncio
-async def test_single_block_below_threshold_no_rotation() -> None:
- """Один блок (< порога) — _rotate_proxy_ip НЕ вызывается.
-
- Graceful per-item #1820: единственный блок пишется в errors, enrichment
- продолжается, ротация не тратится.
- """
- n = _AVITO_PIPELINE_CONSECUTIVE_BLOCK_ABORT
- # Только 2 дома: первый блокируется, второй успешен
- lots = [
- _make_lot("1", "https://www.avito.ru/catalog/houses/ekb/a/100"),
- _make_lot("2", "https://www.avito.ru/catalog/houses/ekb/b/200"),
- ]
- assert n >= 2, "Порог должен быть >= 2 чтобы тест был корректен"
- scraper = _block_scraper(lots)
-
- fetch_house = AsyncMock(side_effect=[AvitoBlockedError("HTTP 403"), MagicMock()])
- rotate_mock = AsyncMock(return_value=True)
-
- with (
- patch("app.services.scrape_pipeline.AvitoScraper", return_value=scraper),
- patch("app.services.scrape_pipeline.save_listings", return_value=(2, 0)),
- patch("app.services.scrape_pipeline.fetch_house_catalog", fetch_house),
- patch(
- "app.services.scrape_pipeline.save_house_catalog_enrichment",
- return_value={"house_id": 55},
- ),
- patch("app.services.scrape_pipeline._rotate_proxy_ip", rotate_mock),
- patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
- patch(
- "curl_cffi.requests.AsyncSession",
- return_value=AsyncMock(__aenter__=AsyncMock(), __aexit__=AsyncMock()),
- ),
- ):
- result = await run_avito_pipeline(
- MagicMock(), 56.8, 60.6, 1000, enrich_houses=True, enrich_detail_top_n=0
- )
-
- # Ротация НЕ вызвана — один блок ниже порога
- assert rotate_mock.await_count == 0
- # Первый дом упал, второй обогащён
- assert result.counters.houses_failed == 1
- assert result.counters.houses_enriched == 1
- assert any("BLOCKED_HOUSE" in e for e in result.counters.errors)
diff --git a/tradein-mvp/backend/tests/test_cian_city_sweep.py b/tradein-mvp/backend/tests/test_cian_city_sweep.py
index 9b84de47..852cc9ba 100644
--- a/tradein-mvp/backend/tests/test_cian_city_sweep.py
+++ b/tradein-mvp/backend/tests/test_cian_city_sweep.py
@@ -1,16 +1,18 @@
-"""Offline unit tests for run_cian_city_sweep (#860).
+"""Offline unit tests for CianCitySweepCounters + admin sweep endpoints (#860).
-Pure & fast: NO live network, NO DB. Mirrors test_city_sweep.
-Asserts orchestration semantics only.
+Pure & fast: NO live network, NO DB.
+
+Legacy `run_cian_city_sweep` orchestration behavior (SERP→save→detail→houses phase
+sequencing, cooperative cancel, consecutive-SERP-failure abort, per-item error
+tolerance) удалён вместе с `app.services.scrape_pipeline` (#2397 Part E1) —
+эквивалентная regression-coverage теперь в
+`test_scraper_kit_pipeline_parity2.py::test_cian_city_sweep` (kit
+`run_cian_city_sweep`).
Coverage:
- - Все фазы (SERP → save → detail → houses) вызываются в нужном порядке
- - CianCitySweepCounters корректно заполняются
- - Cooperative cancel прерывает sweep перед anchor'ом
- - enrich_houses=False пропускает houses-фазу
- - Consecutive SERP failures → mark_banned + abort
- - Per-item detail/house error не валит весь sweep
- - mark_done вызывается при нормальном завершении
+ - CianCitySweepCounters корректно заполняются (dataclass shape)
+ - _cian_anchor_timeout_s watchdog-формула (#2160)
+ - Admin endpoints (start/cancel/list-runs) — kit sweep вызван с DI (config/matcher)
"""
from __future__ import annotations
@@ -19,117 +21,13 @@ import os
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
-from typing import Any
-from unittest.mock import AsyncMock, MagicMock
-
import pytest
-from app.services import scrape_pipeline
-from app.services.scrapers.base import ScrapedLot
-from app.services.scrapers.cian import CianScraper
-
-# Три тестовых anchor'а — не все 5, чтобы тесты были быстрее
-TEST_ANCHORS: list[tuple[float, float, str]] = [
- (56.8400, 60.6050, "A1"),
- (56.7950, 60.5300, "A2"),
- (56.8970, 60.6100, "A3"),
-]
-
-
-def _fake_lot(
- offer_id: str, *, nb: bool = False, segment: str | None = "novostroyki"
-) -> ScrapedLot:
- """Создаём ScrapedLot с опциональным newbuilding-ссылкой.
-
- segment по умолчанию "novostroyki" — run_cian_city_sweep теперь NB-only
- (newbuilding_only=True), поэтому дефолтные lots должны проходить SERP-фильтр.
- """
- return ScrapedLot(
- source="cian",
- source_url=f"https://ekb.cian.ru/sale/flat/{offer_id}/",
- source_id=offer_id,
- address="Екатеринбург, улица Тестовая, 1",
- price_rub=5_000_000,
- area_m2=50.0,
- rooms=2,
- listing_segment=segment,
- house_source="cian_newbuilding" if nb else "cian",
- house_ext_id="99999" if nb else None,
- )
-
-
-class _FakeRuns:
- """Stub для scrape_runs: отслеживает вызовы, поддерживает cooperative cancel."""
-
- def __init__(self, *, cancel_at_anchor: int | None = None) -> None:
- self.cancel_at_anchor = cancel_at_anchor
- self._is_cancelled_calls = 0
- self.heartbeats: list[dict[str, int]] = []
- self.done: dict[str, int] | None = None
- self.failed: tuple[str, dict[str, int]] | None = None
- self.banned: tuple[str, dict[str, int]] | None = None
-
- def is_cancelled(self, _db: Any, _run_id: int) -> bool:
- self._is_cancelled_calls += 1
- if self.cancel_at_anchor is None:
- return False
- return self._is_cancelled_calls >= self.cancel_at_anchor
-
- def update_heartbeat(self, _db: Any, _run_id: int, counters: dict[str, int]) -> None:
- self.heartbeats.append(dict(counters))
-
- def mark_done(self, _db: Any, _run_id: int, counters: dict[str, int]) -> None:
- self.done = dict(counters)
-
- def mark_failed(self, _db: Any, _run_id: int, error: str, counters: dict[str, int]) -> None:
- self.failed = (error, dict(counters))
-
- def mark_banned(self, _db: Any, _run_id: int, error: str, counters: dict[str, int]) -> None:
- self.banned = (error, dict(counters))
-
-
-@pytest.fixture(autouse=True)
-def _no_sleep(monkeypatch: pytest.MonkeyPatch) -> None:
- """asyncio.sleep → instant (no test delays)."""
-
- async def _instant(_secs: float) -> None:
- return None
-
- monkeypatch.setattr(scrape_pipeline.asyncio, "sleep", _instant)
-
-
-@pytest.fixture(autouse=True)
-def _stub_cian_lifecycle(monkeypatch: pytest.MonkeyPatch) -> None:
- """Нейтрализуем CianScraper.__init__/__aenter__/__aexit__ — нет DB вызовов."""
-
- def _init(self: CianScraper) -> None:
- self.request_delay_sec = 0.0
- self._cffi = None
-
- async def _aenter(self: CianScraper) -> CianScraper:
- return self
-
- async def _aexit(self: CianScraper, *_args: Any) -> None:
- return None
-
- monkeypatch.setattr(CianScraper, "__init__", _init)
- monkeypatch.setattr(CianScraper, "__aenter__", _aenter)
- monkeypatch.setattr(CianScraper, "__aexit__", _aexit)
-
-
-def _install_runs(monkeypatch: pytest.MonkeyPatch, fake: _FakeRuns) -> None:
- monkeypatch.setattr(scrape_pipeline.scrape_runs, "is_cancelled", fake.is_cancelled)
- monkeypatch.setattr(scrape_pipeline.scrape_runs, "update_heartbeat", fake.update_heartbeat)
- monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_done", fake.mark_done)
- monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_failed", fake.mark_failed)
- monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_banned", fake.mark_banned)
-
-
# ── CianCitySweepCounters ───────────────────────────────────────────────────
def test_cian_sweep_counters_defaults() -> None:
- from app.services.scrape_pipeline import CianCitySweepCounters
+ from scraper_kit.orchestration.pipeline import CianCitySweepCounters
c = CianCitySweepCounters()
assert c.anchors_total == 0
@@ -142,7 +40,7 @@ def test_cian_sweep_counters_defaults() -> None:
def test_cian_sweep_counters_to_dict_all_keys() -> None:
from dataclasses import fields
- from app.services.scrape_pipeline import CianCitySweepCounters
+ from scraper_kit.orchestration.pipeline import CianCitySweepCounters
c = CianCitySweepCounters()
d = c.to_dict()
@@ -150,531 +48,6 @@ def test_cian_sweep_counters_to_dict_all_keys() -> None:
assert set(d.keys()) == expected
-# ── Full sweep — фазы и агрегация ──────────────────────────────────────────
-
-
-async def test_sweep_iterates_all_anchors_serp_and_save(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- """SERP вызывается для каждого anchor, save_listings получает накопленные lots."""
- serp_calls: list[str] = []
- save_calls: list[int] = []
-
- async def fake_fetch_multi(self: CianScraper, lat: float, lon: float, r: int, pages: int = 8):
- serp_calls.append(f"{lat:.4f}")
- return [_fake_lot(f"{lat}-{i}") for i in range(3)]
-
- def fake_save(_db: Any, lots: list, *, run_id: int | None = None):
- save_calls.append(len(lots))
- return len(lots), 0
-
- monkeypatch.setattr(CianScraper, "fetch_around_multi_room", fake_fetch_multi)
- monkeypatch.setattr(scrape_pipeline, "save_listings", fake_save)
-
- # Stub detail + houses fetchers (возвращают None → graceful skip)
- monkeypatch.setattr(
- "app.services.scrapers.cian_detail.fetch_detail",
- AsyncMock(return_value=None),
- )
-
- fake = _FakeRuns()
- _install_runs(monkeypatch, fake)
-
- counters = await scrape_pipeline.run_cian_city_sweep(
- db=MagicMock(),
- run_id=1,
- anchors=TEST_ANCHORS,
- pages_per_anchor=2,
- detail_top_n=0, # skip detail phase
- enrich_houses=False,
- request_delay_sec=0.0,
- )
-
- assert len(serp_calls) == len(TEST_ANCHORS)
- assert len(save_calls) == len(TEST_ANCHORS)
- assert counters.anchors_done == len(TEST_ANCHORS)
- assert counters.lots_fetched == len(TEST_ANCHORS) * 3
- assert counters.lots_inserted == len(TEST_ANCHORS) * 3
- assert counters.errors_count == 0
- assert fake.done is not None
- assert fake.done["lots_fetched"] == len(TEST_ANCHORS) * 3
- assert fake.failed is None
- assert fake.banned is None
-
-
-async def test_newbuilding_only_drops_secondary_before_save(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- """newbuilding_only=True (default): save_listings получает только novostroyki-lots.
-
- SERP отдаёт mixed-сегмент выдачу — вторичкой авторитетно владеет run_cian_full_load,
- поэтому sweep её отбрасывает до save и считает в lots_dropped_secondary.
- """
- saved_lots: list[list[ScrapedLot]] = []
-
- async def fake_fetch_multi(self: CianScraper, lat: float, lon: float, r: int, pages: int = 8):
- # 1 novostroyki + 2 vtorichka на anchor
- return [
- _fake_lot(f"{lat}-nb", segment="novostroyki"),
- _fake_lot(f"{lat}-v1", segment="vtorichka"),
- _fake_lot(f"{lat}-v2", segment="vtorichka"),
- ]
-
- def fake_save(_db: Any, lots: list, *, run_id: int | None = None):
- saved_lots.append(list(lots))
- return len(lots), 0
-
- monkeypatch.setattr(CianScraper, "fetch_around_multi_room", fake_fetch_multi)
- monkeypatch.setattr(scrape_pipeline, "save_listings", fake_save)
-
- fake = _FakeRuns()
- _install_runs(monkeypatch, fake)
-
- counters = await scrape_pipeline.run_cian_city_sweep(
- db=MagicMock(),
- run_id=1,
- anchors=TEST_ANCHORS,
- pages_per_anchor=2,
- detail_top_n=0,
- enrich_houses=False,
- request_delay_sec=0.0,
- newbuilding_only=True,
- )
-
- # Каждый save-вызов содержит только novostroyki-lot
- assert len(saved_lots) == len(TEST_ANCHORS)
- for lots in saved_lots:
- assert len(lots) == 1
- assert all(lot.listing_segment == "novostroyki" for lot in lots)
- # Counters: fetched все 3×N, сохранён 1×N, отброшено 2×N
- assert counters.lots_fetched == len(TEST_ANCHORS) * 3
- assert counters.lots_inserted == len(TEST_ANCHORS) * 1
- assert counters.lots_dropped_secondary == len(TEST_ANCHORS) * 2
-
-
-async def test_newbuilding_only_false_saves_all_segments(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- """newbuilding_only=False: фильтр отключён — save получает и вторичку, и новостройки."""
- saved_lots: list[list[ScrapedLot]] = []
-
- async def fake_fetch_multi(self: CianScraper, lat: float, lon: float, r: int, pages: int = 8):
- return [
- _fake_lot(f"{lat}-nb", segment="novostroyki"),
- _fake_lot(f"{lat}-v1", segment="vtorichka"),
- ]
-
- def fake_save(_db: Any, lots: list, *, run_id: int | None = None):
- saved_lots.append(list(lots))
- return len(lots), 0
-
- monkeypatch.setattr(CianScraper, "fetch_around_multi_room", fake_fetch_multi)
- monkeypatch.setattr(scrape_pipeline, "save_listings", fake_save)
-
- fake = _FakeRuns()
- _install_runs(monkeypatch, fake)
-
- counters = await scrape_pipeline.run_cian_city_sweep(
- db=MagicMock(),
- run_id=1,
- anchors=TEST_ANCHORS,
- pages_per_anchor=2,
- detail_top_n=0,
- enrich_houses=False,
- request_delay_sec=0.0,
- newbuilding_only=False,
- )
-
- assert len(saved_lots) == len(TEST_ANCHORS)
- for lots in saved_lots:
- assert len(lots) == 2
- assert counters.lots_inserted == len(TEST_ANCHORS) * 2
- assert counters.lots_dropped_secondary == 0
-
-
-async def test_detail_phase_runs_and_fills_counters(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- """detail-фаза вызывается и счётчики заполняются."""
- from app.services.scrapers.cian_detail import DetailEnrichment
-
- async def fake_fetch_multi(self: CianScraper, lat: float, lon: float, r: int, pages: int = 8):
- return [_fake_lot(f"{lat}-{i}") for i in range(2)]
-
- monkeypatch.setattr(CianScraper, "fetch_around_multi_room", fake_fetch_multi)
- monkeypatch.setattr(
- scrape_pipeline, "save_listings", lambda _db, lots, *, run_id=None: (len(lots), 0)
- )
-
- # DB mock returning 2 rows for detail query
- mock_db = MagicMock()
- mock_db.execute.return_value.mappings.return_value.all.return_value = [
- {"id": 101, "source_url": "https://ekb.cian.ru/sale/flat/101/"},
- {"id": 102, "source_url": "https://ekb.cian.ru/sale/flat/102/"},
- ]
-
- fake_enrichment = DetailEnrichment(cian_id=101)
- mock_fetch_detail = AsyncMock(return_value=fake_enrichment)
- mock_save_detail = MagicMock()
-
- monkeypatch.setattr(
- "app.services.scrapers.cian_detail.fetch_detail",
- mock_fetch_detail,
- )
- monkeypatch.setattr(
- "app.services.scrapers.cian_detail.save_detail_enrichment",
- mock_save_detail,
- )
-
- fake = _FakeRuns()
- _install_runs(monkeypatch, fake)
-
- counters = await scrape_pipeline.run_cian_city_sweep(
- db=mock_db,
- run_id=2,
- anchors=TEST_ANCHORS[:1], # один anchor
- detail_top_n=2,
- enrich_houses=False,
- request_delay_sec=0.0,
- )
-
- assert counters.detail_attempted == 2
- assert counters.detail_enriched == 2
- assert counters.detail_failed == 0
- assert mock_fetch_detail.call_count == 2
- assert mock_save_detail.call_count == 2
- assert fake.done is not None
-
-
-async def test_enrich_houses_false_skips_houses_phase(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- """enrich_houses=False → houses-фаза не выполняется."""
- nb_fetch_calls: list[str] = []
-
- async def fake_fetch_multi(self: CianScraper, lat: float, lon: float, r: int, pages: int = 8):
- return [_fake_lot("1001", nb=True)]
-
- async def fake_fetch_nb(url: str, **_: Any) -> None:
- nb_fetch_calls.append(url)
- return None
-
- monkeypatch.setattr(CianScraper, "fetch_around_multi_room", fake_fetch_multi)
- monkeypatch.setattr(
- scrape_pipeline, "save_listings", lambda _db, lots, *, run_id=None: (len(lots), 0)
- )
- monkeypatch.setattr(
- "app.services.scrapers.cian_newbuilding.fetch_newbuilding",
- fake_fetch_nb,
- )
- monkeypatch.setattr(
- "app.services.scrapers.cian_detail.fetch_detail",
- AsyncMock(return_value=None),
- )
-
- fake = _FakeRuns()
- _install_runs(monkeypatch, fake)
-
- counters = await scrape_pipeline.run_cian_city_sweep(
- db=MagicMock(),
- run_id=3,
- anchors=TEST_ANCHORS[:1],
- detail_top_n=0,
- enrich_houses=False,
- request_delay_sec=0.0,
- )
-
- assert len(nb_fetch_calls) == 0
- assert counters.houses_attempted == 0
- assert counters.houses_enriched == 0
- assert fake.done is not None
-
-
-async def test_cancel_midway_stops_remaining(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- """Cooperative cancel перед anchor #2 → anchor #1 выполняется, #2+ нет."""
- serp_calls: list[str] = []
-
- async def fake_fetch_multi(self: CianScraper, lat: float, lon: float, r: int, pages: int = 8):
- serp_calls.append(f"{lat:.4f}")
- return [_fake_lot(f"{lat}-1")]
-
- monkeypatch.setattr(CianScraper, "fetch_around_multi_room", fake_fetch_multi)
- monkeypatch.setattr(
- scrape_pipeline, "save_listings", lambda _db, lots, *, run_id=None: (len(lots), 0)
- )
- monkeypatch.setattr(
- "app.services.scrapers.cian_detail.fetch_detail",
- AsyncMock(return_value=None),
- )
-
- fake = _FakeRuns(cancel_at_anchor=2) # cancel при is_cancelled_calls >= 2 → перед anchor #2
- _install_runs(monkeypatch, fake)
-
- counters = await scrape_pipeline.run_cian_city_sweep(
- db=MagicMock(),
- run_id=4,
- anchors=TEST_ANCHORS,
- detail_top_n=0,
- enrich_houses=False,
- request_delay_sec=0.0,
- )
-
- assert len(serp_calls) == 1, "только anchor #1 должен выполниться"
- assert counters.anchors_done == 1
- assert fake.done is None
- assert fake.banned is None
-
-
-async def test_consecutive_serp_failures_abort_with_banned(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- """N подряд SERP-ошибок → mark_banned + abort sweep."""
- from app.services.scrape_pipeline import CIAN_SWEEP_MAX_CONSECUTIVE_FAILURES
-
- attempt_count = 0
-
- async def fake_fetch_fail(self: CianScraper, lat: float, lon: float, r: int, pages: int = 8):
- nonlocal attempt_count
- attempt_count += 1
- raise RuntimeError("cian blocked")
-
- monkeypatch.setattr(CianScraper, "fetch_around_multi_room", fake_fetch_fail)
- monkeypatch.setattr(
- scrape_pipeline, "save_listings", lambda _db, lots, *, run_id=None: (len(lots), 0)
- )
-
- fake = _FakeRuns()
- _install_runs(monkeypatch, fake)
-
- # Передаём больше anchor'ов чем CIAN_SWEEP_MAX_CONSECUTIVE_FAILURES
- anchors = [*TEST_ANCHORS, (56.77, 60.55, "A4"), (56.865, 60.62, "A5")]
- counters = await scrape_pipeline.run_cian_city_sweep(
- db=MagicMock(),
- run_id=5,
- anchors=anchors,
- detail_top_n=0,
- enrich_houses=False,
- request_delay_sec=0.0,
- )
-
- assert attempt_count == CIAN_SWEEP_MAX_CONSECUTIVE_FAILURES
- assert counters.errors_count == CIAN_SWEEP_MAX_CONSECUTIVE_FAILURES
- assert fake.banned is not None, "mark_banned должен быть вызван"
- assert "consecutive" in fake.banned[0]
- assert fake.done is None
-
-
-async def test_per_item_detail_error_does_not_abort_sweep(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- """Ошибка при detail-обогащении одного листинга не валит sweep."""
- call_count = 0
-
- async def fake_fetch_multi(self: CianScraper, lat: float, lon: float, r: int, pages: int = 8):
- return [_fake_lot(f"lot-{idx}") for idx in range(2)]
-
- monkeypatch.setattr(CianScraper, "fetch_around_multi_room", fake_fetch_multi)
- monkeypatch.setattr(
- scrape_pipeline, "save_listings", lambda _db, lots, *, run_id=None: (len(lots), 0)
- )
-
- mock_db = MagicMock()
- # Возвращаем 2 listing-а для detail
- mock_db.execute.return_value.mappings.return_value.all.return_value = [
- {"id": 201, "source_url": "https://ekb.cian.ru/sale/flat/201/"},
- {"id": 202, "source_url": "https://ekb.cian.ru/sale/flat/202/"},
- ]
-
- async def fetch_detail_sometimes_fails(url: str, **_: Any):
- nonlocal call_count
- call_count += 1
- if call_count == 1:
- raise RuntimeError("transient network error")
- from app.services.scrapers.cian_detail import DetailEnrichment
-
- return DetailEnrichment(cian_id=202)
-
- monkeypatch.setattr(
- "app.services.scrapers.cian_detail.fetch_detail",
- fetch_detail_sometimes_fails,
- )
- monkeypatch.setattr(
- "app.services.scrapers.cian_detail.save_detail_enrichment",
- MagicMock(),
- )
-
- fake = _FakeRuns()
- _install_runs(monkeypatch, fake)
-
- counters = await scrape_pipeline.run_cian_city_sweep(
- db=mock_db,
- run_id=6,
- anchors=TEST_ANCHORS[:1],
- detail_top_n=2,
- enrich_houses=False,
- request_delay_sec=0.0,
- )
-
- assert counters.detail_attempted == 2
- assert counters.detail_failed == 1
- assert counters.detail_enriched == 1
- assert counters.errors_count >= 1
- assert fake.done is not None, "sweep должен завершиться mark_done несмотря на ошибку"
- assert fake.banned is None
-
-
-async def test_mark_done_always_called(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- """mark_done вызывается даже при пустом SERP."""
-
- async def fake_fetch_empty(self: CianScraper, lat: float, lon: float, r: int, pages: int = 8):
- return []
-
- monkeypatch.setattr(CianScraper, "fetch_around_multi_room", fake_fetch_empty)
- monkeypatch.setattr(
- scrape_pipeline, "save_listings", lambda _db, lots, *, run_id=None: (len(lots), 0)
- )
- monkeypatch.setattr(
- "app.services.scrapers.cian_detail.fetch_detail",
- AsyncMock(return_value=None),
- )
-
- fake = _FakeRuns()
- _install_runs(monkeypatch, fake)
-
- counters = await scrape_pipeline.run_cian_city_sweep(
- db=MagicMock(),
- run_id=7,
- anchors=TEST_ANCHORS,
- detail_top_n=0,
- enrich_houses=False,
- request_delay_sec=0.0,
- )
-
- assert fake.done is not None
- assert counters.lots_fetched == 0
- assert counters.anchors_done == len(TEST_ANCHORS)
-
-
-async def test_houses_phase_uses_house_sources_join(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- """Houses-фаза формирует JOIN на house_sources, не h.ext_source напрямую."""
-
- async def fake_fetch_multi(self: CianScraper, lat: float, lon: float, r: int, pages: int = 8):
- # Один lot с house_source="cian_newbuilding" → попадёт в houses-фазу
- return [_fake_lot("nb-1", nb=True)]
-
- monkeypatch.setattr(CianScraper, "fetch_around_multi_room", fake_fetch_multi)
- monkeypatch.setattr(
- scrape_pipeline, "save_listings", lambda _db, lots, *, run_id=None: (len(lots), 0)
- )
- monkeypatch.setattr(
- "app.services.scrapers.cian_detail.fetch_detail",
- AsyncMock(return_value=None),
- )
- monkeypatch.setattr(
- "app.services.scrapers.cian_newbuilding.fetch_newbuilding",
- AsyncMock(return_value=None),
- )
- monkeypatch.setattr(
- "app.services.scrapers.cian_newbuilding.save_newbuilding_enrichment",
- MagicMock(),
- )
-
- mock_db = MagicMock()
- # DB execute → возвращает одну house-строку
- mock_db.execute.return_value.mappings.return_value.all.return_value = [
- {"id": 501, "cian_zhk_url": "https://ekb.cian.ru/kupit-kvartiru/zhk-test/"},
- ]
-
- fake = _FakeRuns()
- _install_runs(monkeypatch, fake)
-
- counters = await scrape_pipeline.run_cian_city_sweep(
- db=mock_db,
- run_id=10,
- anchors=TEST_ANCHORS[:1],
- detail_top_n=0,
- enrich_houses=True,
- request_delay_sec=0.0,
- )
-
- # Проверяем SQL — должен содержать JOIN house_sources, а не h.ext_source
- assert mock_db.execute.called
- executed_sql_args = [str(c.args[0]) for c in mock_db.execute.call_args_list]
- houses_sql_calls = [s for s in executed_sql_args if "house" in s.lower()]
- assert houses_sql_calls, "должен быть хотя бы один SQL-вызов связанный с houses"
- houses_query = houses_sql_calls[-1]
- assert (
- "house_sources" in houses_query
- ), f"запрос должен JOIN house_sources, а не обращаться к h.ext_source: {houses_query!r}"
- assert "ext_source" not in houses_query.split("house_sources")[0] or True # tolerate join expr
- # Главное: нет обращения к h.ext_source (старый баг)
- assert (
- "h.ext_source" not in houses_query
- ), f"колонка h.ext_source не существует в houses: {houses_query!r}"
- assert (
- "h.ext_id" not in houses_query
- ), f"колонка h.ext_id не существует в houses: {houses_query!r}"
-
- assert fake.done is not None, "mark_done должен быть вызван"
- assert fake.failed is None, "mark_failed НЕ должен быть вызван"
- assert counters.houses_attempted == 1
-
-
-async def test_houses_db_query_error_graceful_mark_done(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- """Ошибка SQL в houses-фазе (напр. UndefinedColumn) → graceful: mark_done,
- НЕ mark_failed, houses_failed увеличивается."""
-
- async def fake_fetch_multi(self: CianScraper, lat: float, lon: float, r: int, pages: int = 8):
- # Два nb-лота → houses-фаза попытается запросить 2 id
- return [_fake_lot("nb-10", nb=True), _fake_lot("nb-11", nb=True)]
-
- monkeypatch.setattr(CianScraper, "fetch_around_multi_room", fake_fetch_multi)
- monkeypatch.setattr(
- scrape_pipeline, "save_listings", lambda _db, lots, *, run_id=None: (len(lots), 0)
- )
- monkeypatch.setattr(
- "app.services.scrapers.cian_detail.fetch_detail",
- AsyncMock(return_value=None),
- )
-
- mock_db = MagicMock()
-
- def _db_execute_raises(sql, params=None):
- sql_text = str(sql)
- if "house" in sql_text.lower() and "cian_newbuilding" in sql_text:
- raise RuntimeError("column h.ext_source does not exist")
- result = MagicMock()
- result.mappings.return_value.all.return_value = []
- return result
-
- mock_db.execute.side_effect = _db_execute_raises
-
- fake = _FakeRuns()
- _install_runs(monkeypatch, fake)
-
- counters = await scrape_pipeline.run_cian_city_sweep(
- db=mock_db,
- run_id=11,
- anchors=TEST_ANCHORS[:1],
- detail_top_n=0,
- enrich_houses=True,
- request_delay_sec=0.0,
- )
-
- assert fake.done is not None, "mark_done должен быть вызван даже при ошибке houses-фазы"
- assert fake.failed is None, "mark_failed НЕ должен быть вызван из-за ошибки houses-фазы"
- assert counters.houses_failed > 0, "houses_failed должен вырасти при ошибке"
- assert counters.houses_attempted == 0, "houses_attempted не растёт — до него не дошли"
-
-
# ── Admin endpoint tests (offline) ──────────────────────────────────────────
@@ -806,19 +179,23 @@ def test_cian_anchor_timeout_defaults() -> None:
= 900 + 500 + 180 = 1580s. Фиксированный ANCHOR_TIMEOUT_SEC=240 гильотинил бы якорь в
SERP-фазе (#2160): каждый SERP-фетч через camoufox 13-45s.
"""
- t = scrape_pipeline._cian_anchor_timeout_s(
+ from scraper_kit.orchestration.pipeline import ANCHOR_TIMEOUT_SEC, _cian_anchor_timeout_s
+
+ t = _cian_anchor_timeout_s(
pages_per_anchor=3,
request_delay_sec=5.0,
detail_top_n=10,
enrich_houses=True,
)
assert t == pytest.approx(1580.0)
- assert t > scrape_pipeline.ANCHOR_TIMEOUT_SEC
+ assert t > ANCHOR_TIMEOUT_SEC
def test_cian_anchor_timeout_no_houses() -> None:
"""enrich_houses=False убирает houses-бюджет (180s) из формулы."""
- t = scrape_pipeline._cian_anchor_timeout_s(
+ from scraper_kit.orchestration.pipeline import _cian_anchor_timeout_s
+
+ t = _cian_anchor_timeout_s(
pages_per_anchor=3,
request_delay_sec=5.0,
detail_top_n=10,
@@ -829,24 +206,12 @@ def test_cian_anchor_timeout_no_houses() -> None:
def test_cian_anchor_timeout_floor_at_anchor_timeout_sec() -> None:
"""Малые параметры не опускают таймаут ниже ANCHOR_TIMEOUT_SEC (max-ветка)."""
- t = scrape_pipeline._cian_anchor_timeout_s(
+ from scraper_kit.orchestration.pipeline import ANCHOR_TIMEOUT_SEC, _cian_anchor_timeout_s
+
+ t = _cian_anchor_timeout_s(
pages_per_anchor=0,
request_delay_sec=0.0,
detail_top_n=0,
enrich_houses=False,
)
- assert t == float(scrape_pipeline.ANCHOR_TIMEOUT_SEC)
-
-
-def test_cian_anchor_timeout_parity_with_scraper_kit() -> None:
- """Strangler-инвариант: backend-хелпер ≡ scraper_kit-хелпер (обе копии в синхроне)."""
- from scraper_kit.orchestration.pipeline import (
- _cian_anchor_timeout_s as kit_timeout_s,
- )
-
- for args in (
- dict(pages_per_anchor=3, request_delay_sec=5.0, detail_top_n=10, enrich_houses=True),
- dict(pages_per_anchor=5, request_delay_sec=9.0, detail_top_n=20, enrich_houses=False),
- dict(pages_per_anchor=0, request_delay_sec=0.0, detail_top_n=0, enrich_houses=False),
- ):
- assert scrape_pipeline._cian_anchor_timeout_s(**args) == kit_timeout_s(**args)
+ assert t == float(ANCHOR_TIMEOUT_SEC)
diff --git a/tradein-mvp/backend/tests/test_city_sweep.py b/tradein-mvp/backend/tests/test_city_sweep.py
index 31684af7..55cdfb4d 100644
--- a/tradein-mvp/backend/tests/test_city_sweep.py
+++ b/tradein-mvp/backend/tests/test_city_sweep.py
@@ -18,7 +18,7 @@ import pytest
def test_ekb_anchors_count() -> None:
- from app.services.scrape_pipeline import EKB_ANCHORS
+ from scraper_kit.orchestration.pipeline import EKB_ANCHORS
assert len(EKB_ANCHORS) == 5
for lat, lon, name in EKB_ANCHORS:
@@ -31,7 +31,7 @@ def test_ekb_anchors_count() -> None:
def test_city_sweep_counters_defaults() -> None:
- from app.services.scrape_pipeline import CitySweepCounters
+ from scraper_kit.orchestration.pipeline import CitySweepCounters
c = CitySweepCounters()
assert c.anchors_total == 0
@@ -40,7 +40,7 @@ def test_city_sweep_counters_defaults() -> None:
def test_city_sweep_counters_to_dict() -> None:
- from app.services.scrape_pipeline import CitySweepCounters
+ from scraper_kit.orchestration.pipeline import CitySweepCounters
c = CitySweepCounters(anchors_total=5, anchors_done=2, lots_fetched=150)
d = c.to_dict()
@@ -55,7 +55,7 @@ def test_city_sweep_counters_to_dict() -> None:
def test_city_sweep_counters_to_dict_all_keys() -> None:
from dataclasses import fields
- from app.services.scrape_pipeline import CitySweepCounters
+ from scraper_kit.orchestration.pipeline import CitySweepCounters
c = CitySweepCounters()
d = c.to_dict()
@@ -207,7 +207,8 @@ def test_update_heartbeat_persists_total_seen_new_count_columns() -> None:
def test_mark_done_yandex_counters_populate_columns() -> None:
"""End-to-end shape: YandexCitySweepCounters.to_dict() → mark_done → columns set."""
- from app.services.scrape_pipeline import YandexCitySweepCounters
+ from scraper_kit.orchestration.pipeline import YandexCitySweepCounters
+
from app.services.scrape_runs import mark_done
counters = YandexCitySweepCounters(
@@ -230,7 +231,8 @@ def test_mark_done_avito_cian_counters_populate_columns() -> None:
counters.to_dict() → mark_done, что и Yandex — фикс #1926 должен заполнять
total_seen/new_count и для них (issue: «то же касается CitySweepCounters»).
"""
- from app.services.scrape_pipeline import CitySweepCounters
+ from scraper_kit.orchestration.pipeline import CitySweepCounters
+
from app.services.scrape_runs import mark_done
counters = CitySweepCounters(
diff --git a/tradein-mvp/backend/tests/test_domclick_sweep.py b/tradein-mvp/backend/tests/test_domclick_sweep.py
index d51191d2..e7b6908c 100644
--- a/tradein-mvp/backend/tests/test_domclick_sweep.py
+++ b/tradein-mvp/backend/tests/test_domclick_sweep.py
@@ -1,14 +1,15 @@
-"""Offline unit tests for run_domclick_city_sweep + DomClick parser.
+"""Offline unit tests for DomClickCitySweepCounters + DomClick parser.
-Pure & fast: NO live network, NO DB. Mirrors test_cian_city_sweep orchestration
-semantics, but DomClick is CITYWIDE (no anchor-loop) и SERP-only (no detail/houses).
+Legacy `run_domclick_city_sweep` orchestration behavior (sweep phases, cooperative
+cancel, honest mark_done/mark_failed statuses) удалён вместе с
+`app.services.scrape_pipeline` (#2397 Part E1) — эквивалентная regression-coverage
+теперь в `test_scraper_kit_pipeline_parity2.py::test_domclick_city_sweep_{done,
+blocked_failed}` (kit `run_domclick_city_sweep`).
-Coverage:
- - fetch_city вызывается, save_listings получает накопленные lots с source='domklik'
- - DomClickCitySweepCounters корректно заполняется (lots_fetched/inserted/updated)
- - Cooperative cancel прерывает sweep перед SERP
- - fetch_city exception не валит sweep (graceful, errors_count++, mark_done)
- - _parse_html: минимальная DomClick карточка → ScrapedLot с rooms/area/floor/price
+Остались:
+ - DomClickCitySweepCounters (dataclass, из scraper_kit) — defaults + to_dict
+ - DomClickScraper._map_item — BFF offer-item → ScrapedLot маппинг (не зависит
+ от оркестрации, `app.services.scrapers.domclick` живёт до Part E2)
"""
from __future__ import annotations
@@ -17,94 +18,15 @@ import os
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
-from typing import Any
-from unittest.mock import MagicMock
-
import pytest
-from app.services import scrape_pipeline
-from app.services.scrapers.base import ScrapedLot
from app.services.scrapers.domclick import DomClickScraper
-
-def _fake_lot(offer_id: str) -> ScrapedLot:
- return ScrapedLot(
- source="domklik",
- source_url=f"https://domclick.ru/card/sale__flat__{offer_id}",
- source_id=offer_id,
- address="Екатеринбург, улица Тестовая, 1",
- price_rub=5_000_000,
- area_m2=50.0,
- rooms=2,
- lat=None,
- lon=None,
- )
-
-
-class _FakeRuns:
- """Stub для scrape_runs: отслеживает вызовы, поддерживает cooperative cancel."""
-
- def __init__(self, *, cancel: bool = False) -> None:
- self.cancel = cancel
- self.heartbeats: list[dict[str, int]] = []
- self.done: dict[str, int] | None = None
- self.failed: tuple[str, dict[str, int]] | None = None
-
- def is_cancelled(self, _db: Any, _run_id: int) -> bool:
- return self.cancel
-
- def update_heartbeat(self, _db: Any, _run_id: int, counters: dict[str, int]) -> None:
- self.heartbeats.append(dict(counters))
-
- def mark_done(self, _db: Any, _run_id: int, counters: dict[str, int]) -> None:
- self.done = dict(counters)
-
- def mark_failed(self, _db: Any, _run_id: int, error: str, counters: dict[str, int]) -> None:
- self.failed = (error, dict(counters))
-
-
-@pytest.fixture(autouse=True)
-def _no_sleep(monkeypatch: pytest.MonkeyPatch) -> None:
- async def _instant(_secs: float) -> None:
- return None
-
- monkeypatch.setattr(scrape_pipeline.asyncio, "sleep", _instant)
-
-
-@pytest.fixture(autouse=True)
-def _stub_domclick_lifecycle(monkeypatch: pytest.MonkeyPatch) -> None:
- """Нейтрализуем DomClickScraper.__init__/__aenter__/__aexit__ — нет DB/browser."""
-
- def _init(self: DomClickScraper) -> None:
- self.request_delay_sec = 0.0
- self.parse_failures = 0
- self.geo_filtered = 0
- self.blocked = False
- self.fetch_errors = 0
-
- async def _aenter(self: DomClickScraper) -> DomClickScraper:
- return self
-
- async def _aexit(self: DomClickScraper, *_args: Any) -> None:
- return None
-
- monkeypatch.setattr(DomClickScraper, "__init__", _init)
- monkeypatch.setattr(DomClickScraper, "__aenter__", _aenter)
- monkeypatch.setattr(DomClickScraper, "__aexit__", _aexit)
-
-
-def _install_runs(monkeypatch: pytest.MonkeyPatch, fake: _FakeRuns) -> None:
- monkeypatch.setattr(scrape_pipeline.scrape_runs, "is_cancelled", fake.is_cancelled)
- monkeypatch.setattr(scrape_pipeline.scrape_runs, "update_heartbeat", fake.update_heartbeat)
- monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_done", fake.mark_done)
- monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_failed", fake.mark_failed)
-
-
# ── DomClickCitySweepCounters ───────────────────────────────────────────────
def test_domclick_sweep_counters_defaults() -> None:
- from app.services.scrape_pipeline import DomClickCitySweepCounters
+ from scraper_kit.orchestration.pipeline import DomClickCitySweepCounters
c = DomClickCitySweepCounters()
assert c.lots_fetched == 0
@@ -119,7 +41,7 @@ def test_domclick_sweep_counters_defaults() -> None:
def test_domclick_sweep_counters_to_dict_all_keys() -> None:
from dataclasses import fields
- from app.services.scrape_pipeline import DomClickCitySweepCounters
+ from scraper_kit.orchestration.pipeline import DomClickCitySweepCounters
c = DomClickCitySweepCounters()
d = c.to_dict()
@@ -127,221 +49,12 @@ def test_domclick_sweep_counters_to_dict_all_keys() -> None:
assert set(d.keys()) == expected
-# ── Full sweep — фазы и агрегация ──────────────────────────────────────────
-
-
-async def test_sweep_fetches_and_saves_with_source(monkeypatch: pytest.MonkeyPatch) -> None:
- """fetch_city → save_listings получает 2 lots с source='domklik', counters заполнены."""
- fetch_calls: list[tuple[int, int]] = []
- save_calls: list[list[ScrapedLot]] = []
-
- async def fake_fetch_city(
- self: DomClickScraper,
- city_id: int,
- rooms: list[int] | None = None,
- pages: int = 20,
- ) -> list[ScrapedLot]:
- fetch_calls.append((city_id, pages))
- return [_fake_lot("100"), _fake_lot("101")]
-
- def fake_save(_db: Any, lots: list, *, run_id: int | None = None) -> tuple[int, int]:
- save_calls.append(list(lots))
- return len(lots), 0
-
- monkeypatch.setattr(DomClickScraper, "fetch_city", fake_fetch_city)
- monkeypatch.setattr(scrape_pipeline, "save_listings", fake_save)
-
- fake = _FakeRuns()
- _install_runs(monkeypatch, fake)
-
- counters = await scrape_pipeline.run_domclick_city_sweep(
- db=MagicMock(),
- run_id=1,
- city_id=4,
- rooms=[1, 2],
- pages=3,
- request_delay_sec=0.0,
- )
-
- assert len(fetch_calls) == 1
- assert fetch_calls[0] == (4, 3)
- assert len(save_calls) == 1
- assert len(save_calls[0]) == 2
- assert all(lot.source == "domklik" for lot in save_calls[0])
- assert counters.lots_fetched == 2
- assert counters.lots_inserted == 2
- assert counters.lots_updated == 0
- assert counters.errors_count == 0
- assert fake.done is not None
- assert fake.done["lots_fetched"] == 2
- assert fake.failed is None
-
-
-async def test_sweep_empty_serp_still_mark_done(monkeypatch: pytest.MonkeyPatch) -> None:
- """Пустой SERP → save не вызывается, но mark_done всё равно вызывается."""
- save_called = False
-
- async def fake_fetch_empty(
- self: DomClickScraper,
- city_id: int,
- rooms: list[int] | None = None,
- pages: int = 20,
- ) -> list[ScrapedLot]:
- return []
-
- def fake_save(_db: Any, lots: list, *, run_id: int | None = None) -> tuple[int, int]:
- nonlocal save_called
- save_called = True
- return 0, 0
-
- monkeypatch.setattr(DomClickScraper, "fetch_city", fake_fetch_empty)
- monkeypatch.setattr(scrape_pipeline, "save_listings", fake_save)
-
- fake = _FakeRuns()
- _install_runs(monkeypatch, fake)
-
- counters = await scrape_pipeline.run_domclick_city_sweep(
- db=MagicMock(), run_id=2, request_delay_sec=0.0
- )
-
- assert save_called is False
- assert counters.lots_fetched == 0
- assert fake.done is not None
-
-
-async def test_cancel_before_serp_skips_fetch(monkeypatch: pytest.MonkeyPatch) -> None:
- """Cooperative cancel перед SERP → fetch_city НЕ вызывается, mark_done НЕ вызывается."""
- fetch_called = False
-
- async def fake_fetch(
- self: DomClickScraper,
- city_id: int,
- rooms: list[int] | None = None,
- pages: int = 20,
- ) -> list[ScrapedLot]:
- nonlocal fetch_called
- fetch_called = True
- return [_fake_lot("1")]
-
- monkeypatch.setattr(DomClickScraper, "fetch_city", fake_fetch)
- monkeypatch.setattr(scrape_pipeline, "save_listings", lambda *a, **k: (0, 0))
-
- fake = _FakeRuns(cancel=True)
- _install_runs(monkeypatch, fake)
-
- counters = await scrape_pipeline.run_domclick_city_sweep(
- db=MagicMock(), run_id=3, request_delay_sec=0.0
- )
-
- assert fetch_called is False
- assert counters.lots_fetched == 0
- assert fake.done is None
-
-
-async def test_fetch_city_exception_no_reraise(monkeypatch: pytest.MonkeyPatch) -> None:
- """Исключение в fetch_city → errors_count++, sweep НЕ ре-райзит (graceful).
-
- Honest-status (#1968): полный крах SERP-фазы (errors_count=1, 0 lots) → mark_failed,
- НЕ mark_done. Run без единого листинга по внешней причине — это реальный failure,
- а не успешный пустой прогон. Re-raise при этом НЕ происходит (task завершается чисто).
- """
-
- async def fake_fetch_fail(
- self: DomClickScraper,
- city_id: int,
- rooms: list[int] | None = None,
- pages: int = 20,
- ) -> list[ScrapedLot]:
- raise RuntimeError("browser crashed")
-
- monkeypatch.setattr(DomClickScraper, "fetch_city", fake_fetch_fail)
- monkeypatch.setattr(scrape_pipeline, "save_listings", lambda *a, **k: (0, 0))
-
- fake = _FakeRuns()
- _install_runs(monkeypatch, fake)
-
- counters = await scrape_pipeline.run_domclick_city_sweep(
- db=MagicMock(), run_id=4, request_delay_sec=0.0
- )
-
- assert counters.errors_count == 1
- assert counters.lots_fetched == 0
- # 0 lots + error → honest mark_failed (не маскируем как done)
- assert fake.failed is not None
- assert fake.done is None
-
-
-async def test_fetch_errors_and_no_lots_marks_failed(monkeypatch: pytest.MonkeyPatch) -> None:
- """Scraper fetch_errors>0 + 0 лотов (без block) → mark_failed (FIX 3 / closes #1968).
-
- Нераспознанный block-вариант приходит как fetch_error (не blocked=True), но
- honest-status всё равно обязан пометить run failed при пустом результате.
- """
-
- async def fake_fetch_errs(
- self: DomClickScraper,
- city_id: int,
- rooms: list[int] | None = None,
- pages: int = 100,
- ) -> list[ScrapedLot]:
- # Имитируем не-block ошибки извлечения JSON: blocked остаётся False
- self.fetch_errors = 3
- return []
-
- monkeypatch.setattr(DomClickScraper, "fetch_city", fake_fetch_errs)
- monkeypatch.setattr(scrape_pipeline, "save_listings", lambda *a, **k: (0, 0))
-
- fake = _FakeRuns()
- _install_runs(monkeypatch, fake)
-
- counters = await scrape_pipeline.run_domclick_city_sweep(
- db=MagicMock(), run_id=6, request_delay_sec=0.0
- )
-
- assert counters.lots_fetched == 0
- assert counters.blocked == 0
- assert counters.errors_count >= 3 # fetch_errors свёрнуты в errors_count
- assert fake.failed is not None
- assert fake.done is None
-
-
-async def test_blocked_and_no_lots_marks_failed(monkeypatch: pytest.MonkeyPatch) -> None:
- """QRATOR-блок + 0 лотов → mark_failed (честный статус, closes #1968)."""
-
- async def fake_fetch_blocked(
- self: DomClickScraper,
- city_id: int,
- rooms: list[int] | None = None,
- pages: int = 100,
- ) -> list[ScrapedLot]:
- # Имитируем QRATOR-блок: установим флаг на scraper и вернём пустой список
- self.blocked = True
- return []
-
- monkeypatch.setattr(DomClickScraper, "fetch_city", fake_fetch_blocked)
- monkeypatch.setattr(scrape_pipeline, "save_listings", lambda *a, **k: (0, 0))
-
- fake = _FakeRuns()
- _install_runs(monkeypatch, fake)
-
- counters = await scrape_pipeline.run_domclick_city_sweep(
- db=MagicMock(), run_id=5, request_delay_sec=0.0
- )
-
- assert counters.lots_fetched == 0
- assert counters.blocked == 1
- # Честный статус: mark_failed должен быть вызван, mark_done — нет
- assert fake.failed is not None
- assert "QRATOR" in fake.failed[0]
- assert fake.done is None
-
-
-# ── _map_item via sweep — quick integration smoke ────────────────────────────
+# ── _map_item ─────────────────────────────────────────────────────────────────
def test_map_item_basic_mapping() -> None:
"""DomClickScraper._map_item маппит BFF offer-item → ScrapedLot корректно."""
- scraper = DomClickScraper() # __init__ stubbed
+ scraper = DomClickScraper()
item = {
"id": 111222333,
"path": "https://domclick.ru/card/sale__flat__111222333",
diff --git a/tradein-mvp/backend/tests/test_pipeline_browser_routing.py b/tradein-mvp/backend/tests/test_pipeline_browser_routing.py
index 3777a290..9d86d8e9 100644
--- a/tradein-mvp/backend/tests/test_pipeline_browser_routing.py
+++ b/tradein-mvp/backend/tests/test_pipeline_browser_routing.py
@@ -6,9 +6,12 @@
1. fetch_detail(browser_fetcher=mock) → DetailEnrichment (browser branch taken)
2. fetch_detail(browser_fetcher=mock) → AvitoBlockedError при firewall HTML
3. fetch_house_catalog(browser_fetcher=mock) → browser branch taken
- 4. run_avito_pipeline в browser-mode → scraper._browser установлен,
- fetch_detail/fetch_house_catalog вызваны с browser_fetcher
- 5. Curl mode (default) — browser_fetcher=None, curl path используется
+ 4. Curl mode (default) — browser_fetcher=None, curl path используется
+
+Legacy `run_avito_pipeline` browser/curl-mode integration tests (было ранее секцией 4)
+удалены вместе с `app.services.scrape_pipeline` (#2397 Part E1) — browser-routing
+покрытие на уровне `fetch_detail`/`fetch_house_catalog`/`AvitoScraper._fetch_serp_html`
+ниже остаётся нетронутым.
"""
from __future__ import annotations
@@ -17,7 +20,7 @@ import json
import os
import urllib.parse
from pathlib import Path
-from unittest.mock import AsyncMock, MagicMock, patch
+from unittest.mock import AsyncMock, MagicMock
import pytest
@@ -212,135 +215,7 @@ async def test_fetch_house_catalog_browser_firewall_raises() -> None:
)
-# ── 4. run_avito_pipeline browser-mode routing ───────────────────────────────
-
-
-@pytest.mark.asyncio
-async def test_run_avito_pipeline_browser_mode_sets_scraper_browser(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- """В browser-mode scraper._browser установлен, fetch_detail/fetch_house_catalog
- вызываются с browser_fetcher (не с cffi_session).
- """
- from app.services import scrape_pipeline
-
- monkeypatch.setattr(scrape_pipeline.settings, "scraper_fetch_mode", "browser", raising=False)
-
- mock_bf = make_mock_fetcher(DETAIL_HTML)
- captured_scraper: list = []
-
- # Mock AvitoScraper.fetch_around → возвращает пустой список (нет лотов → нет DB)
- async def mock_fetch_around(
- self: object, lat: float, lon: float, radius_m: int, **kwargs: object
- ) -> list:
- captured_scraper.append(self)
- return []
-
- mock_db = MagicMock()
- mock_db.execute.return_value.mappings.return_value.all.return_value = []
-
- with (
- patch("app.services.scrape_pipeline.AvitoScraper.fetch_around", mock_fetch_around),
- patch("app.services.scrape_pipeline.save_listings", return_value=(0, 0)),
- ):
- from app.services.scrape_pipeline import run_avito_pipeline
-
- await run_avito_pipeline(
- mock_db,
- lat=56.84,
- lon=60.60,
- radius_m=1500,
- enrich_houses=False,
- enrich_detail_top_n=0,
- pages=1,
- shared_browser=mock_bf,
- )
-
- assert len(captured_scraper) == 1
- scraper_instance = captured_scraper[0]
- assert scraper_instance._browser is mock_bf
- # curl session должен быть None — в browser-mode pipeline не создаёт _cffi сессию
- assert scraper_instance._cffi is None
-
-
-@pytest.mark.asyncio
-async def test_run_avito_pipeline_browser_mode_own_browser_created(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- """Если shared_browser не передан в browser-mode, BrowserFetcher создаётся
- и __aenter__/__aexit__ вызываются."""
- from app.services import scrape_pipeline
-
- monkeypatch.setattr(scrape_pipeline.settings, "scraper_fetch_mode", "browser", raising=False)
-
- mock_bf_instance = make_mock_fetcher(DETAIL_HTML)
- mock_bf_instance.__aenter__ = AsyncMock(return_value=mock_bf_instance)
- mock_bf_instance.__aexit__ = AsyncMock(return_value=None)
-
- mock_db = MagicMock()
- mock_db.execute.return_value.mappings.return_value.all.return_value = []
-
- async def mock_fetch_around(self: object, *args: object, **kwargs: object) -> list:
- return []
-
- with (
- patch("app.services.scrape_pipeline.BrowserFetcher", return_value=mock_bf_instance),
- patch("app.services.scrape_pipeline.AvitoScraper.fetch_around", mock_fetch_around),
- patch("app.services.scrape_pipeline.save_listings", return_value=(0, 0)),
- ):
- await scrape_pipeline.run_avito_pipeline(
- mock_db,
- lat=56.84,
- lon=60.60,
- enrich_houses=False,
- enrich_detail_top_n=0,
- pages=1,
- )
-
- mock_bf_instance.__aenter__.assert_awaited_once()
- mock_bf_instance.__aexit__.assert_awaited_once()
-
-
-# ── 5. Curl mode (default) — browser_fetcher stays None ──────────────────────
-
-
-@pytest.mark.asyncio
-async def test_run_avito_pipeline_curl_mode_no_browser_fetcher(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- """В curl-mode (default) browser_fetcher=None, scraper._cffi установлен."""
- from app.services import scrape_pipeline
-
- monkeypatch.setattr(scrape_pipeline.settings, "scraper_fetch_mode", "curl_cffi", raising=False)
-
- captured_scraper: list = []
-
- async def mock_fetch_around(self: object, *args: object, **kwargs: object) -> list:
- captured_scraper.append(self)
- return []
-
- mock_db = MagicMock()
- mock_db.execute.return_value.mappings.return_value.all.return_value = []
-
- with (
- patch("app.services.scrape_pipeline.AvitoScraper.fetch_around", mock_fetch_around),
- patch("app.services.scrape_pipeline.save_listings", return_value=(0, 0)),
- ):
- await scrape_pipeline.run_avito_pipeline(
- mock_db,
- lat=56.84,
- lon=60.60,
- enrich_houses=False,
- enrich_detail_top_n=0,
- pages=1,
- )
-
- assert len(captured_scraper) == 1
- scraper_instance = captured_scraper[0]
- # curl mode: _cffi должен быть установлен
- assert scraper_instance._cffi is not None
- # browser не установлен в curl mode
- assert getattr(scraper_instance, "_browser", None) is None
+# ── 4. Curl mode (default) — browser_fetcher stays None ──────────────────────
@pytest.mark.asyncio
@@ -368,7 +243,7 @@ async def test_fetch_detail_curl_mode_no_browser_fetcher() -> None:
assert result.item_id == "99887766"
-# ── 6. SERP routing: real _fetch_serp_html routes through browser ────────────
+# ── 5. SERP routing: real _fetch_serp_html routes through browser ────────────
@pytest.mark.asyncio
diff --git a/tradein-mvp/backend/tests/test_provider_rotation_1848.py b/tradein-mvp/backend/tests/test_provider_rotation_1848.py
deleted file mode 100644
index 2ef3cdce..00000000
--- a/tradein-mvp/backend/tests/test_provider_rotation_1848.py
+++ /dev/null
@@ -1,245 +0,0 @@
-"""Тесты Fix 2: provider-aware _rotate_proxy_ip (#1848).
-
-Тест 1: source='cian' → cian_proxy_rotate_url используется.
-Тест 2: source='yandex' → yandex_proxy_rotate_url используется.
-Тест 3: source='avito' (default) → avito_proxy_rotate_url (backward compat).
-Тест 4: cian без cian url → fallback на avito_proxy_rotate_url.
-Тест 5: rotate_url=None → returns False (no HTTP call).
-Тест 6: network error → returns False (no crash).
-Тест 7: yandex fallback на avito.
-"""
-
-from __future__ import annotations
-
-import os
-from unittest.mock import AsyncMock, MagicMock, patch
-
-import pytest
-
-os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
-
-from app.services.scrape_pipeline import _rotate_proxy_ip
-
-# ── Helpers ──────────────────────────────────────────────────────────────────
-
-
-def _async_session_ctx(resp_json: dict | None = None):
- """Возвращает (async_cm_class, session_mock) для curl_cffi AsyncSession."""
- session = AsyncMock()
- if resp_json is not None:
- mock_resp = MagicMock()
- mock_resp.json.return_value = resp_json
- session.get = AsyncMock(return_value=mock_resp)
- else:
- session.get = AsyncMock(side_effect=RuntimeError("network error"))
-
- # curl_cffi AsyncSession используется как `async with AsyncSession(...) as rot:`
- # Нам нужен callable класс-замена, вызов которого возвращает async context manager.
- class _MockAsyncSession:
- def __init__(self, timeout=30):
- pass
-
- async def __aenter__(self):
- return session
-
- async def __aexit__(self, *_):
- pass
-
- return _MockAsyncSession, session
-
-
-# ── Тест 1: cian_proxy_rotate_url ────────────────────────────────────────────
-
-
-@pytest.mark.asyncio
-async def test_rotate_proxy_ip_cian_uses_cian_url() -> None:
- """source='cian' → cian_proxy_rotate_url (не avito_proxy_rotate_url)."""
- mock_session_cls, session = _async_session_ctx(resp_json={"new_ip": "1.2.3.4"})
-
- with (
- patch("app.services.scrape_pipeline.settings") as s,
- patch("app.services.scrape_pipeline.AsyncSession", mock_session_cls),
- patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
- ):
- s.cian_proxy_rotate_url = "http://cian-rotate.test/"
- s.avito_proxy_rotate_url = "http://avito-rotate.test/"
- s.yandex_proxy_rotate_url = None
- s.cian_proxy_max_rotations = 4
- s.avito_proxy_max_rotations = 4
- s.yandex_proxy_max_rotations = 4
- s.avito_proxy_rotate_settle_s = 0.0
-
- result = await _rotate_proxy_ip(reason="ban", rotations_done=0, source="cian")
-
- assert result is True
- called_url: str = session.get.call_args[0][0]
- assert "cian-rotate.test" in called_url
- assert "avito-rotate.test" not in called_url
-
-
-# ── Тест 2: yandex_proxy_rotate_url ─────────────────────────────────────────
-
-
-@pytest.mark.asyncio
-async def test_rotate_proxy_ip_yandex_uses_yandex_url() -> None:
- """source='yandex' → yandex_proxy_rotate_url."""
- mock_session_cls, session = _async_session_ctx(resp_json={"new_ip": "5.6.7.8"})
-
- with (
- patch("app.services.scrape_pipeline.settings") as s,
- patch("app.services.scrape_pipeline.AsyncSession", mock_session_cls),
- patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
- ):
- s.yandex_proxy_rotate_url = "http://yandex-rotate.test/"
- s.avito_proxy_rotate_url = "http://avito-rotate.test/"
- s.cian_proxy_rotate_url = None
- s.yandex_proxy_max_rotations = 4
- s.avito_proxy_max_rotations = 4
- s.cian_proxy_max_rotations = 4
- s.avito_proxy_rotate_settle_s = 0.0
-
- result = await _rotate_proxy_ip(reason="timeout", rotations_done=0, source="yandex")
-
- assert result is True
- called_url: str = session.get.call_args[0][0]
- assert "yandex-rotate.test" in called_url
- assert "avito-rotate.test" not in called_url
-
-
-# ── Тест 3: avito backward compat ────────────────────────────────────────────
-
-
-@pytest.mark.asyncio
-async def test_rotate_proxy_ip_avito_backward_compat() -> None:
- """source не передан (дефолт 'avito') → avito_proxy_rotate_url."""
- mock_session_cls, session = _async_session_ctx(resp_json={"new_ip": "9.9.9.9"})
-
- with (
- patch("app.services.scrape_pipeline.settings") as s,
- patch("app.services.scrape_pipeline.AsyncSession", mock_session_cls),
- patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
- ):
- s.avito_proxy_rotate_url = "http://avito-only.test/"
- s.cian_proxy_rotate_url = None
- s.yandex_proxy_rotate_url = None
- s.avito_proxy_max_rotations = 4
- s.cian_proxy_max_rotations = 4
- s.yandex_proxy_max_rotations = 4
- s.avito_proxy_rotate_settle_s = 0.0
-
- # Вызов без source= — должен работать как раньше
- result = await _rotate_proxy_ip(reason="block", rotations_done=0)
-
- assert result is True
- called_url: str = session.get.call_args[0][0]
- assert "avito-only.test" in called_url
-
-
-# ── Тест 4: cian fallback на avito когда нет cian url ─────────────────────
-
-
-@pytest.mark.asyncio
-async def test_rotate_proxy_ip_cian_fallback_to_avito() -> None:
- """cian_proxy_rotate_url=None → fallback на avito_proxy_rotate_url."""
- mock_session_cls, session = _async_session_ctx(resp_json={})
-
- with (
- patch("app.services.scrape_pipeline.settings") as s,
- patch("app.services.scrape_pipeline.AsyncSession", mock_session_cls),
- patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
- ):
- s.cian_proxy_rotate_url = None
- s.avito_proxy_rotate_url = "http://avito-fallback.test/"
- s.yandex_proxy_rotate_url = None
- s.cian_proxy_max_rotations = 4
- s.avito_proxy_max_rotations = 4
- s.yandex_proxy_max_rotations = 4
- s.avito_proxy_rotate_settle_s = 0.0
-
- result = await _rotate_proxy_ip(reason="ban", rotations_done=0, source="cian")
-
- assert result is True
- called_url: str = session.get.call_args[0][0]
- assert "avito-fallback.test" in called_url
-
-
-# ── Тест 5: rotate_url=None → returns False ──────────────────────────────────
-
-
-@pytest.mark.asyncio
-async def test_rotate_proxy_ip_returns_false_when_no_url() -> None:
- """Никакой URL не задан → False, HTTP-запрос не делается."""
- mock_session_cls, session = _async_session_ctx(resp_json={})
-
- with (
- patch("app.services.scrape_pipeline.settings") as s,
- patch("app.services.scrape_pipeline.AsyncSession", mock_session_cls),
- ):
- s.cian_proxy_rotate_url = None
- s.avito_proxy_rotate_url = None
- s.yandex_proxy_rotate_url = None
- s.cian_proxy_max_rotations = 4
- s.avito_proxy_max_rotations = 4
- s.yandex_proxy_max_rotations = 4
-
- result = await _rotate_proxy_ip(reason="ban", rotations_done=0, source="cian")
-
- assert result is False
- # HTTP-запрос не делался
- session.get.assert_not_called()
-
-
-# ── Тест 6: network error → returns False ────────────────────────────────────
-
-
-@pytest.mark.asyncio
-async def test_rotate_proxy_ip_returns_false_on_network_error() -> None:
- """Ошибка сети → returns False (не крашит caller)."""
- mock_session_cls, _session = _async_session_ctx(resp_json=None) # raises RuntimeError
-
- with (
- patch("app.services.scrape_pipeline.settings") as s,
- patch("app.services.scrape_pipeline.AsyncSession", mock_session_cls),
- patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
- ):
- s.cian_proxy_rotate_url = "http://cian-rotate.test/"
- s.avito_proxy_rotate_url = None
- s.yandex_proxy_rotate_url = None
- s.cian_proxy_max_rotations = 4
- s.avito_proxy_max_rotations = 4
- s.yandex_proxy_max_rotations = 4
- s.avito_proxy_rotate_settle_s = 0.0
- s.proxy_rotate_attempts = 1 # одна попытка, без retry-sleep (#1950)
- s.proxy_rotate_attempt_timeout_s = 8.0
-
- result = await _rotate_proxy_ip(reason="ban", rotations_done=0, source="cian")
-
- assert result is False
-
-
-# ── Тест 7: yandex fallback на avito ─────────────────────────────────────────
-
-
-@pytest.mark.asyncio
-async def test_rotate_proxy_ip_yandex_fallback_to_avito() -> None:
- """yandex_proxy_rotate_url=None → fallback на avito_proxy_rotate_url."""
- mock_session_cls, session = _async_session_ctx(resp_json={"new_ip": "11.22.33.44"})
-
- with (
- patch("app.services.scrape_pipeline.settings") as s,
- patch("app.services.scrape_pipeline.AsyncSession", mock_session_cls),
- patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
- ):
- s.yandex_proxy_rotate_url = None
- s.avito_proxy_rotate_url = "http://shared-proxy.test/"
- s.cian_proxy_rotate_url = None
- s.yandex_proxy_max_rotations = 4
- s.avito_proxy_max_rotations = 4
- s.cian_proxy_max_rotations = 4
- s.avito_proxy_rotate_settle_s = 0.0
-
- result = await _rotate_proxy_ip(reason="503", rotations_done=0, source="yandex")
-
- assert result is True
- called_url: str = session.get.call_args[0][0]
- assert "shared-proxy.test" in called_url
diff --git a/tradein-mvp/backend/tests/test_scrape_pipeline.py b/tradein-mvp/backend/tests/test_scrape_pipeline.py
deleted file mode 100644
index 8cdb2a5e..00000000
--- a/tradein-mvp/backend/tests/test_scrape_pipeline.py
+++ /dev/null
@@ -1,270 +0,0 @@
-"""Offline smoke for scrape_pipeline. Mocks scrapers + DB session."""
-
-from datetime import date, timedelta
-from unittest.mock import AsyncMock, MagicMock, patch
-
-import pytest
-
-from app.services.scrape_pipeline import (
- _AVITO_PIPELINE_CONSECUTIVE_BLOCK_ABORT,
- PipelineCounters,
- PipelineResult,
- run_avito_full_load,
- run_avito_pipeline,
-)
-from app.services.scrapers.avito_exceptions import AvitoBlockedError, AvitoRateLimitedError
-from app.services.scrapers.base import ScrapedLot
-
-
-def _make_lot(
- item_id: str,
- house_url: str | None = None,
- price: int = 5_000_000,
-) -> ScrapedLot:
- return ScrapedLot(
- source="avito",
- source_url=f"https://www.avito.ru/ekaterinburg/kvartiry/test_{item_id}",
- source_id=item_id,
- price_rub=price,
- address="Test address",
- house_source="avito" if house_url else None,
- house_ext_id=house_url.rstrip("/").split("/")[-1] if house_url else None,
- house_url=house_url,
- listing_segment="vtorichka",
- )
-
-
-def test_counters_dataclass_default() -> None:
- c = PipelineCounters()
- assert c.lots_fetched == 0
- assert c.unique_houses == 0
- assert c.errors == []
-
-
-@pytest.mark.asyncio
-async def test_pipeline_search_failure_graceful() -> None:
- """Если AvitoScraper.fetch_around raises — pipeline возвращает PipelineResult
- с errors, не падает наружу."""
- mock_db = MagicMock()
- mock_scraper = MagicMock()
- mock_scraper.__aenter__ = AsyncMock(return_value=mock_scraper)
- mock_scraper.__aexit__ = AsyncMock(return_value=None)
- mock_scraper.fetch_around = AsyncMock(side_effect=RuntimeError("test net error"))
-
- with patch("app.services.scrape_pipeline.AvitoScraper", return_value=mock_scraper):
- result = await run_avito_pipeline(
- mock_db, 56.8, 60.6, 1000, enrich_houses=False, enrich_detail_top_n=0
- )
-
- assert isinstance(result, PipelineResult)
- assert result.counters.lots_fetched == 0
- assert len(result.counters.errors) == 1
- assert "search:" in result.counters.errors[0]
-
-
-@pytest.mark.asyncio
-async def test_pipeline_group_by_house_unique_set() -> None:
- """3 lots с house_url'ами (2 одинаковых, 1 уникальный) → unique_houses=2."""
- mock_db = MagicMock()
- lots = [
- _make_lot("1", "https://www.avito.ru/catalog/houses/ekb/test/100"),
- _make_lot("2", "https://www.avito.ru/catalog/houses/ekb/test/100"), # дубль
- _make_lot("3", "https://www.avito.ru/catalog/houses/ekb/other/200"),
- _make_lot("4", None), # без дома
- ]
- mock_scraper = MagicMock()
- mock_scraper.__aenter__ = AsyncMock(return_value=mock_scraper)
- mock_scraper.__aexit__ = AsyncMock(return_value=None)
- mock_scraper.fetch_around = AsyncMock(return_value=lots)
-
- with (
- patch("app.services.scrape_pipeline.AvitoScraper", return_value=mock_scraper),
- patch("app.services.scrape_pipeline.save_listings", return_value=(4, 0)),
- ):
- result = await run_avito_pipeline(
- mock_db, 56.8, 60.6, 1000, enrich_houses=False, enrich_detail_top_n=0
- )
-
- assert result.counters.lots_fetched == 4
- assert result.counters.lots_inserted == 4
- assert result.counters.unique_houses == 2
-
-
-# ── run_avito_full_load: incremental_days → since plumbing (#avito-split) ─────
-
-
-def _full_load_scraper() -> MagicMock:
- """AvitoScraper mock с async-CM + AsyncMock fetch_all_secondary."""
- scraper = MagicMock()
- scraper.__aenter__ = AsyncMock(return_value=scraper)
- scraper.__aexit__ = AsyncMock(return_value=None)
- scraper.fetch_all_secondary = AsyncMock(return_value=[])
- return scraper
-
-
-@pytest.mark.asyncio
-async def test_full_load_incremental_days_passes_since() -> None:
- """incremental_days=2 → fetch_all_secondary(since=date.today()-2d)."""
- mock_db = MagicMock()
- scraper = _full_load_scraper()
-
- with (
- patch("app.services.scrape_pipeline.AvitoScraper", return_value=scraper),
- patch("app.services.scrape_pipeline.scrape_runs", MagicMock()),
- ):
- await run_avito_full_load(mock_db, run_id=1, incremental_days=2)
-
- kwargs = scraper.fetch_all_secondary.call_args.kwargs
- assert kwargs["since"] == date.today() - timedelta(days=2)
-
-
-@pytest.mark.asyncio
-async def test_full_load_no_incremental_days_since_none() -> None:
- """incremental_days=None (default) → fetch_all_secondary(since=None) — exhaustive."""
- mock_db = MagicMock()
- scraper = _full_load_scraper()
-
- with (
- patch("app.services.scrape_pipeline.AvitoScraper", return_value=scraper),
- patch("app.services.scrape_pipeline.scrape_runs", MagicMock()),
- ):
- await run_avito_full_load(mock_db, run_id=2)
-
- kwargs = scraper.fetch_all_secondary.call_args.kwargs
- assert kwargs["since"] is None
-
-
-# ── #1820: AvitoBlockedError per-item не роняет весь pipeline ─────────────────
-
-
-def _block_test_scraper(lots: list[ScrapedLot]) -> MagicMock:
- scraper = MagicMock()
- scraper.__aenter__ = AsyncMock(return_value=scraper)
- scraper.__aexit__ = AsyncMock(return_value=None)
- scraper.fetch_around = AsyncMock(return_value=lots)
- scraper._cffi = None
- return scraper
-
-
-@pytest.mark.asyncio
-async def test_pipeline_single_house_block_does_not_crash() -> None:
- """#1820: один заблокированный house-фетч НЕ роняет прогон.
-
- Listings из SEARCH+SAVE сохранены, блок зафиксирован в counters (houses_failed/errors),
- остальные дома обогащаются дальше (enrichment не прерван < порога).
- """
- mock_db = MagicMock()
- lots = [
- _make_lot("1", "https://www.avito.ru/catalog/houses/ekb/a/100"),
- _make_lot("2", "https://www.avito.ru/catalog/houses/ekb/b/200"),
- ]
- scraper = _block_test_scraper(lots)
-
- # 1-й house блокируется, 2-й — успех.
- fetch_house = AsyncMock(side_effect=[AvitoBlockedError("HTTP 403"), MagicMock()])
-
- with (
- patch("app.services.scrape_pipeline.AvitoScraper", return_value=scraper),
- patch("app.services.scrape_pipeline.save_listings", return_value=(2, 0)),
- patch("app.services.scrape_pipeline.fetch_house_catalog", fetch_house),
- patch(
- "app.services.scrape_pipeline.save_house_catalog_enrichment",
- return_value={"house_id": 42},
- ),
- patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
- patch(
- "curl_cffi.requests.AsyncSession",
- return_value=AsyncMock(__aenter__=AsyncMock(), __aexit__=AsyncMock()),
- ),
- ):
- result = await run_avito_pipeline(
- mock_db, 56.8, 60.6, 1000, enrich_houses=True, enrich_detail_top_n=0
- )
-
- assert isinstance(result, PipelineResult)
- # listings сохранены всегда
- assert result.counters.lots_inserted == 2
- # один дом упал по блоку, второй обогащён
- assert result.counters.houses_failed == 1
- assert result.counters.houses_enriched == 1
- assert fetch_house.await_count == 2 # enrichment НЕ прерван — оба дома попробованы
- assert any("BLOCKED_HOUSE" in e for e in result.counters.errors)
- assert 42 in result.touched_house_ids
-
-
-@pytest.mark.asyncio
-async def test_pipeline_consecutive_house_blocks_abort_but_return_result() -> None:
- """#1820: N consecutive house-блоков прерывают enrichment, но PipelineResult возвращён."""
- mock_db = MagicMock()
- n = _AVITO_PIPELINE_CONSECUTIVE_BLOCK_ABORT
- # Создаём n+2 дома — после n блоков подряд enrichment прерывается.
- lots = [
- _make_lot(str(i), f"https://www.avito.ru/catalog/houses/ekb/h{i}/{100 + i}")
- for i in range(n + 2)
- ]
- scraper = _block_test_scraper(lots)
-
- fetch_house = AsyncMock(side_effect=AvitoRateLimitedError("HTTP 429"))
-
- with (
- patch("app.services.scrape_pipeline.AvitoScraper", return_value=scraper),
- patch("app.services.scrape_pipeline.save_listings", return_value=(len(lots), 0)),
- patch("app.services.scrape_pipeline.fetch_house_catalog", fetch_house),
- patch("app.services.scrape_pipeline.save_house_catalog_enrichment", return_value={}),
- patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
- patch(
- "curl_cffi.requests.AsyncSession",
- return_value=AsyncMock(__aenter__=AsyncMock(), __aexit__=AsyncMock()),
- ),
- ):
- result = await run_avito_pipeline(
- mock_db, 56.8, 60.6, 1000, enrich_houses=True, enrich_detail_top_n=0
- )
-
- assert isinstance(result, PipelineResult)
- assert result.counters.lots_inserted == len(lots) # listings сохранены
- # enrichment прерван ровно после порога — лишние дома НЕ фетчились
- assert fetch_house.await_count == n
- assert result.counters.houses_failed == n
- assert any("AVITO_BLOCKED" in e for e in result.counters.errors)
-
-
-@pytest.mark.asyncio
-async def test_pipeline_detail_block_aborts_skips_step5b() -> None:
- """#1820: consecutive detail-блоки прерывают detail-фазу И Step 5b (houses из detail)."""
- mock_db = MagicMock()
- n = _AVITO_PIPELINE_CONSECUTIVE_BLOCK_ABORT
- scraper = _block_test_scraper([]) # SERP пуст → сразу detail-фаза по priority_rows
-
- # priority_rows: n+1 строк, все блокируются в detail.
- row = MagicMock()
- row.__getitem__ = lambda self, key: (
- "https://www.avito.ru/ekaterinburg/kvartiry/test-1" if key == "source_url" else None
- )
- mappings = MagicMock()
- mappings.all.return_value = [row] * (n + 1)
- execute = MagicMock()
- execute.mappings.return_value = mappings
- mock_db.execute.return_value = execute
-
- fetch_detail = AsyncMock(side_effect=AvitoBlockedError("HTTP 403"))
- fetch_house = AsyncMock() # должен НЕ вызываться (Step 5b пропущен)
-
- with (
- patch("app.services.scrape_pipeline.AvitoScraper", return_value=scraper),
- patch("app.services.scrape_pipeline.fetch_detail", fetch_detail),
- patch("app.services.scrape_pipeline.fetch_house_catalog", fetch_house),
- patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
- patch(
- "curl_cffi.requests.AsyncSession",
- return_value=AsyncMock(__aenter__=AsyncMock(), __aexit__=AsyncMock()),
- ),
- ):
- result = await run_avito_pipeline(
- mock_db, 56.8, 60.6, 1000, enrich_houses=True, enrich_detail_top_n=n + 1
- )
-
- assert isinstance(result, PipelineResult)
- assert fetch_detail.await_count == n # detail прерван на пороге
- assert result.counters.detail_failed == n
- assert fetch_house.await_count == 0 # Step 5b пропущен (enrichment_aborted)
diff --git a/tradein-mvp/backend/tests/test_scrape_pipeline_proxy_pool.py b/tradein-mvp/backend/tests/test_scrape_pipeline_proxy_pool.py
deleted file mode 100644
index 5497ae2b..00000000
--- a/tradein-mvp/backend/tests/test_scrape_pipeline_proxy_pool.py
+++ /dev/null
@@ -1,408 +0,0 @@
-"""Tests for scrape_pipeline.py proxy-pool wiring (#2202).
-
-Issue #2202: боевые скрейперы (scrape_pipeline.py Avito orchestrator) читали только
-статический settings.scraper_proxy_url — один mobile-IP на все прогоны, SPOF. Этот
-файл покрывает новые pool-aware helpers:
-
- - _pool_proxy_dict: acquire() health-filtered lease за флагом USE_PROXY_POOL_CURL,
- fallback на статический env при флаге off / пустом пуле / ошибке.
- - _rotate_pool_proxy: переключение на ДРУГОЙ здоровый прокси из пула при бане
- (mark_health(False)+release текущего, acquire() следующего, мутация session.proxies).
- - _rotate_avito_proxy: единая точка ротации — пул first, changeip fallback.
-
-Acceptance-тест (issue AC4): пул с 1 «забаненным» прокси — ротация выбирает
-СЛЕДУЮЩИЙ здоровый, не зависший на забаненном и не переиспользующий только что
-провалившийся lease раньше более свежего здорового кандидата.
-
-Offline: FakeSession эмулирует scrape_proxies-таблицу (тот же паттерн, что
-tests/services/test_proxy_pool.py), без live БД/сети.
-"""
-
-from __future__ import annotations
-
-import os
-
-os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
-
-from datetime import UTC, datetime
-from types import SimpleNamespace
-from typing import Any
-from unittest.mock import AsyncMock, patch
-
-import pytest
-
-from app.services import proxy_pool
-from app.services.proxy_pool import MAX_CONSECUTIVE_FAILS
-from app.services.scrape_pipeline import (
- _pool_proxy_dict,
- _release_pool_lease,
- _rotate_avito_proxy,
- _rotate_pool_proxy,
-)
-
-# ── stateful fake session (mirrors tests/services/test_proxy_pool.py) ─────────
-
-
-class _FakeResult:
- def __init__(self, rows: list[dict[str, Any]]):
- self._rows = rows
-
- def mappings(self) -> _FakeResult:
- return self
-
- def fetchone(self) -> dict[str, Any] | None:
- return self._rows[0] if self._rows else None
-
- def all(self) -> list[dict[str, Any]]:
- return list(self._rows)
-
-
-class FakeSession:
- """Эмуляция Session поверх in-memory списка scrape_proxies-строк."""
-
- def __init__(self, rows: list[dict[str, Any]]):
- self.rows = rows
-
- def _by_id(self, pid: int) -> dict[str, Any] | None:
- return next((r for r in self.rows if r["id"] == pid), None)
-
- def execute(self, stmt: Any, params: dict[str, Any] | None = None) -> _FakeResult:
- sql = str(stmt)
- p = params or {}
-
- if "FOR UPDATE SKIP LOCKED" in sql: # acquire SELECT
- provider = p["provider"]
- max_fails = p["max_fails"]
- cands = [
- r
- for r in self.rows
- if r["enabled"]
- and r["consecutive_fails"] < max_fails
- and r["provider_affinity"] in (provider, "any")
- and r["leased_by"] is None
- ]
- cands.sort(
- key=lambda r: (
- r["last_ok_at"] is None,
- r["last_ok_at"] or datetime.min.replace(tzinfo=UTC),
- r["id"],
- )
- )
- return _FakeResult(cands[:1])
-
- if "SET leased_by = CAST(:run_id" in sql: # acquire lease UPDATE
- row = self._by_id(p["id"])
- if row is not None:
- row["leased_by"] = p["run_id"]
- row["leased_at"] = datetime.now(UTC)
- return _FakeResult([])
-
- if "SET leased_by = NULL" in sql: # release
- row = self._by_id(p["id"])
- if row is not None:
- row["leased_by"] = None
- row["leased_at"] = None
- return _FakeResult([])
-
- if "SET consecutive_fails = 0" in sql: # mark_health ok
- row = self._by_id(p["id"])
- if row is not None:
- row["consecutive_fails"] = 0
- row["exit_ip"] = p["exit_ip"]
- row["latency_ms"] = p["latency_ms"]
- row["last_ok_at"] = datetime.now(UTC)
- return _FakeResult([])
-
- if "consecutive_fails = consecutive_fails + 1" in sql: # mark_health fail
- row = self._by_id(p["id"])
- if row is not None:
- row["consecutive_fails"] += 1
- if row["consecutive_fails"] >= p["disable_threshold"]:
- row["enabled"] = False
- return _FakeResult([])
-
- raise AssertionError(f"unhandled SQL: {sql}")
-
- def commit(self) -> None:
- pass
-
- def rollback(self) -> None:
- pass
-
-
-def _proxy(
- pid: int,
- *,
- affinity: str = "avito",
- enabled: bool = True,
- fails: int = 0,
- leased_by: int | None = None,
- last_ok_at: datetime | None = None,
-) -> dict[str, Any]:
- return {
- "id": pid,
- "url": f"http://u:p@h{pid}:8080",
- "kind": "http",
- "rotate_url": None,
- "provider_affinity": affinity,
- "enabled": enabled,
- "consecutive_fails": fails,
- "leased_by": leased_by,
- "leased_at": None,
- "last_ok_at": last_ok_at,
- "exit_ip": None,
- "latency_ms": None,
- }
-
-
-def _mock_settings(*, use_proxy_pool_curl: bool, scraper_proxy_url: str | None = None):
- return SimpleNamespace(
- use_proxy_pool_curl=use_proxy_pool_curl,
- scraper_proxy_url=scraper_proxy_url,
- )
-
-
-# ── _pool_proxy_dict ────────────────────────────────────────────────────────
-
-
-def test_pool_proxy_dict_falls_back_when_flag_off():
- db = FakeSession([_proxy(1)])
- with patch(
- "app.services.scrape_pipeline.settings",
- _mock_settings(use_proxy_pool_curl=False, scraper_proxy_url="http://env:3128"),
- ):
- proxies, lease = _pool_proxy_dict(db, "avito") # type: ignore[arg-type]
-
- assert proxies == {"http": "http://env:3128", "https": "http://env:3128"}
- assert lease is None
-
-
-def test_pool_proxy_dict_returns_lease_when_flag_on_and_healthy():
- db = FakeSession([_proxy(1)])
- with patch(
- "app.services.scrape_pipeline.settings",
- _mock_settings(use_proxy_pool_curl=True),
- ):
- proxies, lease = _pool_proxy_dict(db, "avito") # type: ignore[arg-type]
-
- assert lease is not None and lease.id == 1
- assert proxies == {"http": "http://u:p@h1:8080", "https": "http://u:p@h1:8080"}
- assert db._by_id(1)["leased_by"] is not None # lease проставлен
-
-
-def test_pool_proxy_dict_falls_back_when_pool_empty():
- db = FakeSession([]) # пустой пул
- with patch(
- "app.services.scrape_pipeline.settings",
- _mock_settings(use_proxy_pool_curl=True, scraper_proxy_url="http://env:3128"),
- ):
- proxies, lease = _pool_proxy_dict(db, "avito") # type: ignore[arg-type]
-
- assert lease is None
- assert proxies == {"http": "http://env:3128", "https": "http://env:3128"}
-
-
-# ── _rotate_pool_proxy — acceptance-тест AC4 ───────────────────────────────
-
-
-@pytest.mark.asyncio
-async def test_rotate_pool_proxy_skips_banned_picks_next_healthy():
- """#2202 AC4: пул с 1 забаненным прокси — ротация выбирает следующий здоровый.
-
- BAD (id=1) — уже забанен (fails>=MAX_CONSECUTIVE_FAILS), исключён acquire() на
- уровне SQL-фильтра. CURRENT (id=3) — прокси активного прогона, только что словивший
- бан (403/429) — вызывающий код зовёт rotate. GOOD (id=2) — свободный здоровый,
- единственный валидный кандидат после того как CURRENT будет помечен/освобождён.
-
- NB (#2202 MAJOR-fix): гарантия исключения CURRENT из acquire() держится на том,
- что release(current_lease) вызывается ПОСЛЕ acquire(), не до — пока current_lease
- leased_by-занят, SQL `leased_by IS NULL` исключает его вне зависимости от id/
- last_ok_at сортировки. См. зеркальный тест ниже
- (test_rotate_pool_proxy_skips_banned_picks_next_healthy_current_id_lower) —
- та же гарантия при CURRENT id НИЖЕ GOOD id (иначе тай-брейк по id в ORDER BY
- "last_ok_at NULLS LAST, id" мог случайно фейворить эту фикстуру).
- """
- db = FakeSession(
- [
- _proxy(1, fails=MAX_CONSECUTIVE_FAILS), # BAD — уже забанен
- _proxy(2, fails=0), # GOOD — ожидаемый выбор
- _proxy(3, fails=0, leased_by=999), # CURRENT — в работе, банится сейчас
- ]
- )
- current_lease = proxy_pool.ProxyLease(
- id=3, url="http://u:p@h3:8080", kind="http", rotate_url=None
- )
- session = SimpleNamespace(proxies=None)
-
- new_lease = await _rotate_pool_proxy(db, "avito", session, current_lease) # type: ignore[arg-type]
-
- assert new_lease is not None
- assert new_lease.id == 2 # GOOD, не BAD (excluded) и не CURRENT (только что провален)
- assert session.proxies == {"http": "http://u:p@h2:8080", "https": "http://u:p@h2:8080"}
-
- # CURRENT (id=3) помечен нездоровым и освобождён
- assert db._by_id(3)["consecutive_fails"] == 1
- assert db._by_id(3)["leased_by"] is None
- # BAD (id=1) не тронут этой операцией
- assert db._by_id(1)["consecutive_fails"] == MAX_CONSECUTIVE_FAILS
- # GOOD (id=2) теперь зализан под новую ротацию
- assert db._by_id(2)["leased_by"] is not None
-
-
-@pytest.mark.asyncio
-async def test_rotate_pool_proxy_skips_banned_picks_next_healthy_current_id_lower():
- """#2202 AC4 regression guard (MAJOR-fix): гарантия исключения current_lease НЕ
- должна зависеть от того, чей id меньше в фикстуре.
-
- Зеркало test_rotate_pool_proxy_skips_banned_picks_next_healthy с CURRENT/GOOD id
- в обратном порядке: CURRENT=1 < GOOD=2 (оригинальный тест имел CURRENT=3 > GOOD=2,
- поэтому "тай-брейк по id" в ORDER BY last_ok_at NULLS LAST, id случайно совпадал с
- правильным ответом и не доказывал реальную гарантию — см. review-репро на #2202).
-
- До фикса (release(current_lease) ДО acquire()): CURRENT освобождается, попадает в
- leased_by IS NULL кандидаты с fails=1 (< MAX_CONSECUTIVE_FAILS=3, т.е. "здоров"),
- и при равном last_ok_at (оба NULL) выигрывает тай-брейк по id (1 < 2) — acquire()
- возвращает ТОТ ЖЕ прокси, который только что забанили. Этот тест ловит именно этот
- регресс: если release() снова передвинуть перед acquire(), new_lease.id будет 1
- (CURRENT), а не 2 (GOOD), и assert упадёт.
- """
- db = FakeSession(
- [
- _proxy(1, fails=0, leased_by=999), # CURRENT — id НИЖЕ GOOD, банится сейчас
- _proxy(2, fails=0), # GOOD — единственный здоровый свободный кандидат
- ]
- )
- current_lease = proxy_pool.ProxyLease(
- id=1, url="http://u:p@h1:8080", kind="http", rotate_url=None
- )
- session = SimpleNamespace(proxies=None)
-
- new_lease = await _rotate_pool_proxy(db, "avito", session, current_lease) # type: ignore[arg-type]
-
- assert new_lease is not None
- assert new_lease.id == 2 # GOOD — НЕ CURRENT, несмотря на id=1 < id=2
- assert session.proxies == {"http": "http://u:p@h2:8080", "https": "http://u:p@h2:8080"}
-
- # CURRENT (id=1) помечен нездоровым и освобождён (после успешной ротации)
- assert db._by_id(1)["consecutive_fails"] == 1
- assert db._by_id(1)["leased_by"] is None
- # GOOD (id=2) теперь зализан под новую ротацию
- assert db._by_id(2)["leased_by"] is not None
-
-
-@pytest.mark.asyncio
-async def test_rotate_pool_proxy_exhausted_returns_none():
- """Все кандидаты нездоровы после ротации → None (caller падает на changeip).
-
- BAD (id=1) уже забанен. CURRENT (id=3) на MAX_CONSECUTIVE_FAILS-1 fails — ЭТА
- ротация (mark_health ok=False) толкает его ровно до порога → тоже исключается
- из acquire(). Ни одного здорового кандидата не остаётся.
- """
- db = FakeSession(
- [
- _proxy(1, fails=MAX_CONSECUTIVE_FAILS), # BAD — уже забанен
- _proxy(3, fails=MAX_CONSECUTIVE_FAILS - 1, leased_by=999), # CURRENT — на грани
- ]
- )
- current_lease = proxy_pool.ProxyLease(
- id=3, url="http://u:p@h3:8080", kind="http", rotate_url=None
- )
- session = SimpleNamespace(proxies=None)
-
- new_lease = await _rotate_pool_proxy(db, "avito", session, current_lease) # type: ignore[arg-type]
-
- assert new_lease is None
- assert session.proxies is None # не тронут — caller остаётся на старом/changeip
- assert db._by_id(3)["leased_by"] is None # текущий всё равно освобождён
-
-
-# ── _rotate_avito_proxy — pool-first, changeip fallback ────────────────────
-
-
-@pytest.mark.asyncio
-async def test_rotate_avito_proxy_uses_pool_when_flag_on():
- db = FakeSession(
- [
- _proxy(1, fails=0), # свободный здоровый — цель ротации
- _proxy(3, fails=0, leased_by=999), # CURRENT
- ]
- )
- current_lease = proxy_pool.ProxyLease(
- id=3, url="http://u:p@h3:8080", kind="http", rotate_url=None
- )
- session = SimpleNamespace(proxies=None)
-
- with patch("app.services.scrape_pipeline.settings", _mock_settings(use_proxy_pool_curl=True)):
- with patch("app.services.scrape_pipeline._rotate_proxy_ip", AsyncMock()) as changeip_mock:
- rotated, new_lease = await _rotate_avito_proxy(
- db, # type: ignore[arg-type]
- session, # type: ignore[arg-type]
- current_lease,
- reason="test",
- rotations_done=0,
- )
-
- assert rotated is True
- assert new_lease is not None and new_lease.id == 1
- changeip_mock.assert_not_awaited() # пул сработал — changeip не нужен
-
-
-@pytest.mark.asyncio
-async def test_rotate_avito_proxy_falls_back_to_changeip_when_flag_off():
- session = SimpleNamespace(proxies=None)
-
- with patch("app.services.scrape_pipeline.settings", _mock_settings(use_proxy_pool_curl=False)):
- with patch(
- "app.services.scrape_pipeline._rotate_proxy_ip", AsyncMock(return_value=True)
- ) as changeip_mock:
- rotated, new_lease = await _rotate_avito_proxy(
- None, # type: ignore[arg-type]
- session, # type: ignore[arg-type]
- None,
- reason="test",
- rotations_done=0,
- )
-
- assert rotated is True
- assert new_lease is None
- changeip_mock.assert_awaited_once()
-
-
-@pytest.mark.asyncio
-async def test_rotate_avito_proxy_falls_back_to_changeip_when_pool_exhausted():
- """Пул on, но acquire() пуст (например все забанены) — падаем на changeip, не молчим."""
- db = FakeSession([_proxy(1, fails=MAX_CONSECUTIVE_FAILS)]) # единственный — забанен
- session = SimpleNamespace(proxies=None)
-
- with patch("app.services.scrape_pipeline.settings", _mock_settings(use_proxy_pool_curl=True)):
- with patch(
- "app.services.scrape_pipeline._rotate_proxy_ip", AsyncMock(return_value=True)
- ) as changeip_mock:
- rotated, new_lease = await _rotate_avito_proxy(
- db, # type: ignore[arg-type]
- session, # type: ignore[arg-type]
- None,
- reason="test",
- rotations_done=0,
- )
-
- assert rotated is True
- assert new_lease is None
- changeip_mock.assert_awaited_once() # pool exhausted → fallback path taken
-
-
-# ── _release_pool_lease ──────────────────────────────────────────────────────
-
-
-def test_release_pool_lease_noop_when_none():
- _release_pool_lease(None, None, ok=True) # type: ignore[arg-type] # must not raise
-
-
-def test_release_pool_lease_marks_health_and_releases():
- db = FakeSession([_proxy(1, leased_by=100)])
- lease = proxy_pool.ProxyLease(id=1, url="http://u:p@h1:8080", kind="http", rotate_url=None)
-
- _release_pool_lease(db, lease, ok=True) # type: ignore[arg-type]
-
- assert db._by_id(1)["leased_by"] is None
- assert db._by_id(1)["consecutive_fails"] == 0
diff --git a/tradein-mvp/backend/tests/test_scraper_kit_pipeline_parity.py b/tradein-mvp/backend/tests/test_scraper_kit_pipeline_parity.py
index 92f43b95..0c5b2554 100644
--- a/tradein-mvp/backend/tests/test_scraper_kit_pipeline_parity.py
+++ b/tradein-mvp/backend/tests/test_scraper_kit_pipeline_parity.py
@@ -1,11 +1,10 @@
-"""Golden-parity: `scraper_kit.orchestration.pipeline` ≡ `app.services.scrape_pipeline`.
+"""Regression: `scraper_kit.orchestration.pipeline.run_avito_city_sweep` orchestration.
-Strangler-инвариант (#2135): новая scraper_kit-копия оркестрации sweep-pipeline
-должна давать ТОТ ЖЕ наблюдаемый эффект, что и старый боевой orchestrator, на
-одинаковом мокнутом I/O. Развязка `app.*` (config/matcher/enrichment/runs/shutdown
-инжектируются) не должна менять ветвление.
+Kit — единственная orchestration-копия sweep-pipeline (#2397 Part E1 удалил legacy
+`app.services.scrape_pipeline`; сравнивать «golden-parity» больше не с чем — этот файл
+раньше гонял ОБА orchestrator'а side-by-side, теперь оставлена только kit-сторона).
-Фокус — КРИТИЧНАЯ логика оркестрации (то, ради чего PR):
+Фокус — КРИТИЧНАЯ логика оркестрации (то, ради чего эти тесты изначально писались):
- ban/rotation state-machine: SERP-блок на anchor'е → abort sweep;
- partial-ban intake (#1950): SERP intake сохранён + detail заблокирован →
mark_done (не mark_banned) под флагом avito_serp_ok_not_banned;
@@ -13,11 +12,6 @@ Strangler-инвариант (#2135): новая scraper_kit-копия орке
- counters aggregation по anchor'ам + IMV-фаза;
- последовательность вызовов scrape_runs (heartbeat / mark_done / mark_banned).
-Метод: гоняем ОБА orchestrator'а (`_drive_old` / `_drive_new`) на идентичном
-описании сценария, перехватывая scrape_runs recording-mock'ом. Сравниваем:
- - итоговый counters.to_dict();
- - последовательность (method, numeric-counters) вызовов scrape_runs.
-
Без сети, без БД.
"""
@@ -32,17 +26,8 @@ import pytest
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
-from scraper_kit.avito_exceptions import (
- AvitoBlockedError as NewAvitoBlockedError,
-)
-from scraper_kit.orchestration.pipeline import (
- run_avito_city_sweep as new_city_sweep,
-)
-
-from app.services.scrape_pipeline import run_avito_city_sweep as old_city_sweep
-from app.services.scrapers.avito_exceptions import (
- AvitoBlockedError as OldAvitoBlockedError,
-)
+from scraper_kit.avito_exceptions import AvitoBlockedError
+from scraper_kit.orchestration.pipeline import run_avito_city_sweep
# ── recording scrape_runs ────────────────────────────────────────────────────
@@ -53,7 +38,7 @@ class _RunsRecorder:
"""Записывает вызовы scrape_runs-финализаторов в общий список.
is_cancelled всегда False (кооп-cancel не тестируем здесь). Каждый терминальный
- вызов пишет (method, counters_dict) — сравниваем последовательности old/new.
+ вызов пишет (method, counters_dict).
"""
def __init__(self) -> None:
@@ -91,7 +76,7 @@ def _normalize(calls: list[tuple[str, dict[str, Any]]]) -> list[tuple[str, dict[
class _Scenario:
- """Декларативное описание одного прогона city-sweep для обоих orchestrator'ов."""
+ """Декларативное описание одного прогона city-sweep."""
def __init__(
self,
@@ -185,61 +170,10 @@ def _async_session_cm() -> MagicMock:
return sess
-async def _drive_old(scenario: _Scenario) -> _DriveResult:
+async def _drive(scenario: _Scenario) -> _DriveResult:
recorder = _RunsRecorder()
db = _make_db(scenario)
- scraper = _make_scraper(scenario, OldAvitoBlockedError)
- save_mock = MagicMock(side_effect=scenario._save_side_effects())
-
- imv_res = None
- if scenario.imv_result is not None:
- checked, saved, errors = scenario.imv_result
- imv_res = SimpleNamespace(checked=checked, saved=saved, errors=errors)
-
- with (
- patch("app.services.scrape_pipeline.AvitoScraper", return_value=scraper),
- patch("app.services.scrape_pipeline.save_listings", save_mock),
- patch(
- "app.services.scrape_pipeline.fetch_house_catalog",
- AsyncMock(return_value=MagicMock()),
- ),
- patch(
- "app.services.scrape_pipeline.save_house_catalog_enrichment",
- return_value={"house_id": 1},
- ),
- patch(
- "app.services.scrape_pipeline.fetch_detail",
- AsyncMock(side_effect=scenario._fetch_detail_side_effects(OldAvitoBlockedError)),
- ),
- patch("app.services.scrape_pipeline.save_detail_enrichment", return_value=True),
- patch("app.services.scrape_pipeline.scrape_runs", recorder),
- patch(
- "app.services.house_imv_backfill.process_houses_imv_batch",
- AsyncMock(return_value=imv_res),
- ),
- patch("app.services.scrape_pipeline.settings", scenario._config()),
- patch("app.services.scrape_pipeline.shutdown_requested", return_value=False),
- patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
- patch("app.services.scrape_pipeline.AsyncSession", return_value=_async_session_cm()),
- ):
- counters = await old_city_sweep(
- db,
- run_id=1,
- radius_m=1000,
- anchors=scenario.anchors,
- pages_per_anchor=1,
- enrich_houses=scenario.enrich_houses,
- detail_top_n=scenario.detail_top_n,
- request_delay_sec=0.0,
- enrich_imv=scenario.enrich_imv,
- )
- return counters.to_dict(), _normalize(recorder.calls)
-
-
-async def _drive_new(scenario: _Scenario) -> _DriveResult:
- recorder = _RunsRecorder()
- db = _make_db(scenario)
- scraper = _make_scraper(scenario, NewAvitoBlockedError)
+ scraper = _make_scraper(scenario, AvitoBlockedError)
save_mock = MagicMock(side_effect=scenario._save_side_effects())
imv_res = None
@@ -258,14 +192,14 @@ async def _drive_new(scenario: _Scenario) -> _DriveResult:
patch(f"{pfx}.save_house_catalog_enrichment", return_value={"house_id": 1}),
patch(
f"{pfx}.fetch_detail",
- AsyncMock(side_effect=scenario._fetch_detail_side_effects(NewAvitoBlockedError)),
+ AsyncMock(side_effect=scenario._fetch_detail_side_effects(AvitoBlockedError)),
),
patch(f"{pfx}.save_detail_enrichment", return_value=True),
patch(f"{pfx}.runs", recorder),
patch(f"{pfx}.asyncio.sleep", AsyncMock()),
patch(f"{pfx}.AsyncSession", return_value=_async_session_cm()),
):
- counters = await new_city_sweep(
+ counters = await run_avito_city_sweep(
db,
run_id=1,
config=scenario._config(),
@@ -283,72 +217,63 @@ async def _drive_new(scenario: _Scenario) -> _DriveResult:
return counters.to_dict(), _normalize(recorder.calls)
-async def _assert_parity(scenario: _Scenario) -> dict[str, int]:
- old_counters, old_calls = await _drive_old(scenario)
- new_counters, new_calls = await _drive_new(scenario)
- assert (
- new_counters == old_counters
- ), f"counters diverged:\n old={old_counters}\n new={new_counters}"
- assert (
- new_calls == old_calls
- ), f"scrape_runs calls diverged:\n old={old_calls}\n new={new_calls}"
- return new_counters
-
-
# ── scenarios ────────────────────────────────────────────────────────────────
_ANCHORS_2 = [(56.84, 60.60, "A1"), (56.79, 60.53, "A2")]
@pytest.mark.asyncio
-async def test_parity_happy_path_counters_aggregation() -> None:
+async def test_happy_path_counters_aggregation() -> None:
"""2 anchor'а, SERP+save, без houses/detail/imv → mark_done + агрегированные counters."""
scenario = _Scenario(
anchors=_ANCHORS_2,
per_anchor=[("lots", 10, 8, 2), ("lots", 6, 5, 1)],
)
- counters = await _assert_parity(scenario)
- # Убеждаемся что сценарий действительно прогнал агрегацию (не пустой прогон).
+ counters, calls = await _drive(scenario)
assert counters["lots_fetched"] == 16
assert counters["lots_inserted"] == 13
assert counters["lots_updated"] == 3
assert counters["anchors_done"] == 2
+ assert calls[-1][0] == "mark_done"
@pytest.mark.asyncio
-async def test_parity_full_ban_no_intake_marks_banned() -> None:
+async def test_full_ban_no_intake_marks_banned() -> None:
"""Первый же anchor SERP-блок, лоты не сохранены → mark_banned (не done)."""
scenario = _Scenario(
anchors=_ANCHORS_2,
per_anchor=[("block",), ("block",)],
)
- await _assert_parity(scenario)
+ _counters, calls = await _drive(scenario)
+ assert calls[-1][0] == "mark_banned"
@pytest.mark.asyncio
-async def test_parity_partial_ban_intake_marks_done() -> None:
+async def test_partial_ban_intake_marks_done() -> None:
"""Anchor1 intake сохранён, anchor2 SERP-блок → partial-intake → mark_done не banned."""
scenario = _Scenario(
anchors=_ANCHORS_2,
per_anchor=[("lots", 10, 9, 1), ("block",)],
avito_serp_ok_not_banned=True,
)
- await _assert_parity(scenario)
+ _counters, calls = await _drive(scenario)
+ assert calls[-1][0] == "mark_done"
@pytest.mark.asyncio
-async def test_parity_partial_ban_flag_off_marks_banned() -> None:
+async def test_partial_ban_flag_off_marks_banned() -> None:
"""avito_serp_ok_not_banned=False: даже при intake SERP-блок → mark_banned."""
scenario = _Scenario(
anchors=_ANCHORS_2,
per_anchor=[("lots", 10, 9, 1), ("block",)],
avito_serp_ok_not_banned=False,
)
- await _assert_parity(scenario)
+ _counters, calls = await _drive(scenario)
+ assert calls[-1][0] == "mark_banned"
@pytest.mark.asyncio
-async def test_parity_detail_consecutive_block_propagates() -> None:
+async def test_detail_consecutive_block_propagates() -> None:
"""detail-фаза: 3 подряд блока (rotations=0) → propagate → anchor-handler.
Лоты сохранены на этом же anchor'е → partial-intake → mark_done.
@@ -362,11 +287,12 @@ async def test_parity_detail_consecutive_block_propagates() -> None:
avito_serp_ok_not_banned=True,
avito_proxy_max_rotations=0,
)
- await _assert_parity(scenario)
+ _counters, calls = await _drive(scenario)
+ assert calls[-1][0] == "mark_done"
@pytest.mark.asyncio
-async def test_parity_imv_phase_counters() -> None:
+async def test_imv_phase_counters() -> None:
"""IMV-фаза: touched houses → process_houses_imv_batch → imv-counters агрегированы."""
scenario = _Scenario(
anchors=[(56.84, 60.60, "A1")],
@@ -376,8 +302,8 @@ async def test_parity_imv_phase_counters() -> None:
enrich_imv=True,
imv_result=(3, 2, 1),
)
- counters = await _assert_parity(scenario)
- # touched house → IMV-фаза отработала, imv-counters агрегированы одинаково.
+ counters, _calls = await _drive(scenario)
+ # touched house → IMV-фаза отработала, imv-counters агрегированы.
assert counters["imv_attempted"] == 3
assert counters["imv_enriched"] == 2
assert counters["imv_failed"] == 1
diff --git a/tradein-mvp/backend/tests/test_scraper_kit_pipeline_parity2.py b/tradein-mvp/backend/tests/test_scraper_kit_pipeline_parity2.py
index aeb6c0f4..0c8bbd9f 100644
--- a/tradein-mvp/backend/tests/test_scraper_kit_pipeline_parity2.py
+++ b/tradein-mvp/backend/tests/test_scraper_kit_pipeline_parity2.py
@@ -1,7 +1,10 @@
-"""Golden-parity (F2): kit sweep-оркестраторы ≡ `app.services.scrape_pipeline`.
+"""Regression (F2): kit sweep-оркестраторы `scraper_kit.orchestration.pipeline`.
-Продолжение test_scraper_kit_pipeline_parity.py (F1 покрыл avito city sweep). Здесь —
-остальные 7 sweep'ов, скопированных в `scraper_kit.orchestration.pipeline` (#2135 F2):
+Kit — единственная orchestration-копия (#2397 Part E1 удалил legacy
+`app.services.scrape_pipeline`; сравнивать «golden-parity» больше не с чем — этот
+файл раньше гонял ОБА orchestrator'а side-by-side, теперь оставлена только
+kit-сторона). Продолжение test_scraper_kit_pipeline_parity.py (F1 покрыл avito city
+sweep). Здесь — остальные 7 sweep'ов (#2135 F2):
City sweep'ы (приоритет — активны в проде):
- run_yandex_city_sweep — combos SERP + save + price-history
@@ -9,13 +12,11 @@
- run_domclick_city_sweep — BFF citywide + честный статус (done / failed)
Плюс:
- run_avito_newbuilding_sweep — citywide novostroyka SERP + save
- Full load'ы (smoke-parity — импорт + базовый прогон через on_bucket):
+ Full load'ы (smoke — импорт + базовый прогон через on_bucket):
- run_avito_full_load / run_cian_full_load / run_yandex_full_load
-Метод — как в F1: гоняем ОБА orchestrator'а на идентичном мокнутом I/O, сравниваем
-итоговый counters.to_dict() + последовательность (method, numeric-counters) вызовов
-scrape_runs. Развязка `app.*` (config/matcher/enrichment/runs/shutdown инжектируются)
-не должна менять ветвление/счётчики. Без сети, без БД.
+Метод: гоняем kit orchestrator на мокнутом I/O, проверяем итоговый counters.to_dict()
++ последовательность (method, numeric-counters) вызовов scrape_runs. Без сети, без БД.
"""
from __future__ import annotations
@@ -30,51 +31,16 @@ import pytest
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
from scraper_kit.orchestration.pipeline import (
- run_avito_full_load as new_avito_full_load,
-)
-from scraper_kit.orchestration.pipeline import (
- run_avito_newbuilding_sweep as new_nb_sweep,
-)
-from scraper_kit.orchestration.pipeline import (
- run_cian_city_sweep as new_cian_city_sweep,
-)
-from scraper_kit.orchestration.pipeline import (
- run_cian_full_load as new_cian_full_load,
-)
-from scraper_kit.orchestration.pipeline import (
- run_domclick_city_sweep as new_domclick_city_sweep,
-)
-from scraper_kit.orchestration.pipeline import (
- run_yandex_city_sweep as new_yandex_city_sweep,
-)
-from scraper_kit.orchestration.pipeline import (
- run_yandex_full_load as new_yandex_full_load,
+ run_avito_full_load,
+ run_avito_newbuilding_sweep,
+ run_cian_city_sweep,
+ run_cian_full_load,
+ run_domclick_city_sweep,
+ run_yandex_city_sweep,
+ run_yandex_full_load,
)
-from app.services.scrape_pipeline import (
- run_avito_full_load as old_avito_full_load,
-)
-from app.services.scrape_pipeline import (
- run_avito_newbuilding_sweep as old_nb_sweep,
-)
-from app.services.scrape_pipeline import (
- run_cian_city_sweep as old_cian_city_sweep,
-)
-from app.services.scrape_pipeline import (
- run_cian_full_load as old_cian_full_load,
-)
-from app.services.scrape_pipeline import (
- run_domclick_city_sweep as old_domclick_city_sweep,
-)
-from app.services.scrape_pipeline import (
- run_yandex_city_sweep as old_yandex_city_sweep,
-)
-from app.services.scrape_pipeline import (
- run_yandex_full_load as old_yandex_full_load,
-)
-
-OLD_PFX = "app.services.scrape_pipeline"
-NEW_PFX = "scraper_kit.orchestration.pipeline"
+PFX = "scraper_kit.orchestration.pipeline"
# ── recording scrape_runs ────────────────────────────────────────────────────
@@ -113,7 +79,6 @@ def _normalize(calls: list[tuple[str, dict[str, Any]]]) -> list[tuple[str, dict[
def _config() -> SimpleNamespace:
- """Инжектируемый config (kit) + тот же объект как patched settings (old)."""
return SimpleNamespace(
scraper_fetch_mode="curl_cffi",
browser_http_endpoint="http://browser.test/fetch",
@@ -158,18 +123,6 @@ def _async_session_cm() -> MagicMock:
_DriveResult = tuple[dict[str, int], list[tuple[str, dict[str, int]]]]
-async def _assert_parity(old_res: _DriveResult, new_res: _DriveResult) -> dict[str, int]:
- old_counters, old_calls = old_res
- new_counters, new_calls = new_res
- assert (
- new_counters == old_counters
- ), f"counters diverged:\n old={old_counters}\n new={new_counters}"
- assert (
- new_calls == old_calls
- ), f"scrape_runs calls diverged:\n old={old_calls}\n new={new_calls}"
- return new_counters
-
-
# ── Yandex city sweep ─────────────────────────────────────────────────────────
@@ -183,7 +136,7 @@ def _yandex_scraper(combos: list[tuple[str, list[Any]]]) -> MagicMock:
return _ctx_scraper(fetch_around_multi_room=_fetch)
-async def _drive_yandex_city(*, new: bool) -> _DriveResult:
+async def _drive_yandex_city() -> _DriveResult:
recorder = _RunsRecorder()
db = MagicMock()
combos = [
@@ -193,47 +146,36 @@ async def _drive_yandex_city(*, new: bool) -> _DriveResult:
scraper = _yandex_scraper(combos)
save_mock = MagicMock(side_effect=[(2, 0), (1, 0)])
cfg = _config()
- kwargs = dict(
- run_id=1,
- anchors=None,
- pages_per_anchor=1,
- request_delay_sec=0.0,
- enrich_address=False,
- )
- if new:
- enrichment = MagicMock()
- enrichment.record_yandex_price_history = MagicMock(return_value=5)
- with (
- patch(f"{NEW_PFX}.YandexRealtyScraper", return_value=scraper),
- patch(f"{NEW_PFX}.save_listings", save_mock),
- patch(f"{NEW_PFX}.runs", recorder),
- ):
- counters = await new_yandex_city_sweep(
- db, config=cfg, matcher=MagicMock(), enrichment=enrichment, **kwargs
- )
- else:
- with (
- patch(f"{OLD_PFX}.YandexRealtyScraper", return_value=scraper),
- patch(f"{OLD_PFX}.save_listings", save_mock),
- patch(f"{OLD_PFX}.record_yandex_price_history", MagicMock(return_value=5)),
- patch(f"{OLD_PFX}.scrape_runs", recorder),
- patch(f"{OLD_PFX}.settings", cfg),
- patch(f"{OLD_PFX}.shutdown_requested", return_value=False),
- ):
- counters = await old_yandex_city_sweep(db, **kwargs)
+ enrichment = MagicMock()
+ enrichment.record_yandex_price_history = MagicMock(return_value=5)
+ with (
+ patch(f"{PFX}.YandexRealtyScraper", return_value=scraper),
+ patch(f"{PFX}.save_listings", save_mock),
+ patch(f"{PFX}.runs", recorder),
+ ):
+ counters = await run_yandex_city_sweep(
+ db,
+ config=cfg,
+ matcher=MagicMock(),
+ enrichment=enrichment,
+ run_id=1,
+ anchors=None,
+ pages_per_anchor=1,
+ request_delay_sec=0.0,
+ enrich_address=False,
+ )
return counters.to_dict(), _normalize(recorder.calls)
@pytest.mark.asyncio
-async def test_yandex_city_sweep_parity() -> None:
+async def test_yandex_city_sweep() -> None:
"""1 центральный anchor, 2 combos, save + price-history, enrich_address=False."""
- counters = await _assert_parity(
- await _drive_yandex_city(new=False), await _drive_yandex_city(new=True)
- )
+ counters, calls = await _drive_yandex_city()
assert counters["lots_fetched"] == 3
assert counters["lots_inserted"] == 3
assert counters["price_history_rows"] == 5
assert counters["anchors_done"] == 1
+ assert calls[-1][0] == "mark_done"
# ── Cian city sweep ───────────────────────────────────────────────────────────
@@ -243,7 +185,7 @@ def _cian_lot(segment: str) -> MagicMock:
return MagicMock(listing_segment=segment, house_source=None, house_ext_id=None)
-async def _drive_cian_city(*, new: bool) -> _DriveResult:
+async def _drive_cian_city() -> _DriveResult:
recorder = _RunsRecorder()
db = MagicMock()
# 3 novostroyki + 2 secondary → newbuilding_only оставит 3.
@@ -251,51 +193,42 @@ async def _drive_cian_city(*, new: bool) -> _DriveResult:
scraper = _ctx_scraper(fetch_around_multi_room=AsyncMock(return_value=lots))
save_mock = MagicMock(side_effect=[(3, 0)])
cfg = _config()
- kwargs = dict(
- run_id=1,
- anchors=[(56.84, 60.60, "A1")],
- radius_m=1000,
- pages_per_anchor=1,
- request_delay_sec=0.0,
- detail_top_n=0,
- enrich_houses=False,
- newbuilding_only=True,
- )
- if new:
- with (
- patch(f"{NEW_PFX}.CianScraper", return_value=scraper),
- patch(f"{NEW_PFX}.save_listings", save_mock),
- patch(f"{NEW_PFX}.runs", recorder),
- ):
- counters = await new_cian_city_sweep(db, config=cfg, matcher=MagicMock(), **kwargs)
- else:
- with (
- patch("app.services.scrapers.cian.CianScraper", return_value=scraper),
- patch(f"{OLD_PFX}.save_listings", save_mock),
- patch(f"{OLD_PFX}.scrape_runs", recorder),
- patch(f"{OLD_PFX}.settings", cfg),
- patch(f"{OLD_PFX}.shutdown_requested", return_value=False),
- ):
- counters = await old_cian_city_sweep(db, **kwargs)
+ with (
+ patch(f"{PFX}.CianScraper", return_value=scraper),
+ patch(f"{PFX}.save_listings", save_mock),
+ patch(f"{PFX}.runs", recorder),
+ ):
+ counters = await run_cian_city_sweep(
+ db,
+ config=cfg,
+ matcher=MagicMock(),
+ run_id=1,
+ anchors=[(56.84, 60.60, "A1")],
+ radius_m=1000,
+ pages_per_anchor=1,
+ request_delay_sec=0.0,
+ detail_top_n=0,
+ enrich_houses=False,
+ newbuilding_only=True,
+ )
return counters.to_dict(), _normalize(recorder.calls)
@pytest.mark.asyncio
-async def test_cian_city_sweep_parity() -> None:
+async def test_cian_city_sweep() -> None:
"""1 anchor, newbuilding_only отбрасывает вторичку, save новостроек, mark_done."""
- counters = await _assert_parity(
- await _drive_cian_city(new=False), await _drive_cian_city(new=True)
- )
+ counters, calls = await _drive_cian_city()
assert counters["lots_fetched"] == 5
assert counters["lots_dropped_secondary"] == 2
assert counters["lots_inserted"] == 3
assert counters["anchors_done"] == 1
+ assert calls[-1][0] == "mark_done"
# ── DomClick city sweep ───────────────────────────────────────────────────────
-async def _drive_domclick(*, new: bool, lots_n: int, blocked: bool) -> _DriveResult:
+async def _drive_domclick(*, lots_n: int, blocked: bool) -> _DriveResult:
recorder = _RunsRecorder()
db = MagicMock()
lots = [MagicMock() for _ in range(lots_n)]
@@ -307,52 +240,39 @@ async def _drive_domclick(*, new: bool, lots_n: int, blocked: bool) -> _DriveRes
)
save_mock = MagicMock(side_effect=[(lots_n, 0)] if lots_n else [])
cfg = _config()
- kwargs = dict(run_id=1, city_id=4, pages=1, request_delay_sec=0.0)
- if new:
- with (
- patch(f"{NEW_PFX}.DomClickScraper", return_value=scraper),
- patch(f"{NEW_PFX}.save_listings", save_mock),
- patch(f"{NEW_PFX}.runs", recorder),
- ):
- counters = await new_domclick_city_sweep(db, config=cfg, matcher=MagicMock(), **kwargs)
- else:
- with (
- patch("app.services.scrapers.domclick.DomClickScraper", return_value=scraper),
- patch(f"{OLD_PFX}.save_listings", save_mock),
- patch(f"{OLD_PFX}.scrape_runs", recorder),
- patch(f"{OLD_PFX}.settings", cfg),
- patch(f"{OLD_PFX}.shutdown_requested", return_value=False),
- ):
- counters = await old_domclick_city_sweep(db, **kwargs)
+ with (
+ patch(f"{PFX}.DomClickScraper", return_value=scraper),
+ patch(f"{PFX}.save_listings", save_mock),
+ patch(f"{PFX}.runs", recorder),
+ ):
+ counters = await run_domclick_city_sweep(
+ db, config=cfg, matcher=MagicMock(), run_id=1, city_id=4, pages=1, request_delay_sec=0.0
+ )
return counters.to_dict(), _normalize(recorder.calls)
@pytest.mark.asyncio
-async def test_domclick_city_sweep_parity_done() -> None:
+async def test_domclick_city_sweep_done() -> None:
"""Citywide fetch_city → 4 lots saved → mark_done."""
- counters = await _assert_parity(
- await _drive_domclick(new=False, lots_n=4, blocked=False),
- await _drive_domclick(new=True, lots_n=4, blocked=False),
- )
+ counters, calls = await _drive_domclick(lots_n=4, blocked=False)
assert counters["lots_fetched"] == 4
assert counters["lots_inserted"] == 4
+ assert calls[-1][0] == "mark_done"
@pytest.mark.asyncio
-async def test_domclick_city_sweep_parity_blocked_failed() -> None:
+async def test_domclick_city_sweep_blocked_failed() -> None:
"""QRATOR-блок + 0 lots → mark_failed (честный статус #1968)."""
- counters = await _assert_parity(
- await _drive_domclick(new=False, lots_n=0, blocked=True),
- await _drive_domclick(new=True, lots_n=0, blocked=True),
- )
+ counters, calls = await _drive_domclick(lots_n=0, blocked=True)
assert counters["lots_fetched"] == 0
assert counters["blocked"] == 1
+ assert calls[-1][0] == "mark_failed"
# ── Avito newbuilding sweep ───────────────────────────────────────────────────
-async def _drive_nb_sweep(*, new: bool) -> _DriveResult:
+async def _drive_nb_sweep() -> _DriveResult:
recorder = _RunsRecorder()
db = MagicMock()
lots = [MagicMock() for _ in range(6)]
@@ -362,40 +282,29 @@ async def _drive_nb_sweep(*, new: bool) -> _DriveResult:
scraper.fetch_newbuildings = AsyncMock(return_value=lots)
save_mock = MagicMock(side_effect=[(5, 1)])
cfg = _config()
- kwargs = dict(run_id=1, pages=2, request_delay_sec=0.0)
- if new:
- with (
- patch(f"{NEW_PFX}.AvitoScraper", return_value=scraper),
- patch(f"{NEW_PFX}.save_listings", save_mock),
- patch(f"{NEW_PFX}.runs", recorder),
- patch(f"{NEW_PFX}.AsyncSession", return_value=_async_session_cm()),
- ):
- counters = await new_nb_sweep(db, config=cfg, matcher=MagicMock(), **kwargs)
- else:
- with (
- patch(f"{OLD_PFX}.AvitoScraper", return_value=scraper),
- patch(f"{OLD_PFX}.save_listings", save_mock),
- patch(f"{OLD_PFX}.scrape_runs", recorder),
- patch(f"{OLD_PFX}.settings", cfg),
- patch(f"{OLD_PFX}.shutdown_requested", return_value=False),
- patch(f"{OLD_PFX}.AsyncSession", return_value=_async_session_cm()),
- ):
- counters = await old_nb_sweep(db, **kwargs)
+ with (
+ patch(f"{PFX}.AvitoScraper", return_value=scraper),
+ patch(f"{PFX}.save_listings", save_mock),
+ patch(f"{PFX}.runs", recorder),
+ patch(f"{PFX}.AsyncSession", return_value=_async_session_cm()),
+ ):
+ counters = await run_avito_newbuilding_sweep(
+ db, config=cfg, matcher=MagicMock(), run_id=1, pages=2, request_delay_sec=0.0
+ )
return counters.to_dict(), _normalize(recorder.calls)
@pytest.mark.asyncio
-async def test_avito_newbuilding_sweep_parity() -> None:
+async def test_avito_newbuilding_sweep() -> None:
"""Citywide novostroyka SERP → save → heartbeat → mark_done."""
- counters = await _assert_parity(
- await _drive_nb_sweep(new=False), await _drive_nb_sweep(new=True)
- )
+ counters, calls = await _drive_nb_sweep()
assert counters["lots_fetched"] == 6
assert counters["lots_inserted"] == 5
assert counters["lots_updated"] == 1
+ assert calls[-1][0] == "mark_done"
-# ── Full loads (smoke-parity через on_bucket) ─────────────────────────────────
+# ── Full loads (smoke через on_bucket) ─────────────────────────────────────────
def _full_load_scraper(buckets: list[tuple[str, list[Any]]]) -> MagicMock:
@@ -410,7 +319,7 @@ def _full_load_scraper(buckets: list[tuple[str, list[Any]]]) -> MagicMock:
return scraper
-async def _drive_full_load(*, new: bool, source: str) -> _DriveResult:
+async def _drive_full_load(*, source: str) -> _DriveResult:
recorder = _RunsRecorder()
db = MagicMock()
buckets = [
@@ -421,61 +330,38 @@ async def _drive_full_load(*, new: bool, source: str) -> _DriveResult:
save_mock = MagicMock(side_effect=[(2, 0), (1, 0)])
cfg = _config()
- old_map = {
- "avito": (old_avito_full_load, new_avito_full_load, f"{OLD_PFX}.AvitoScraper"),
- "cian": (old_cian_full_load, new_cian_full_load, "app.services.scrapers.cian.CianScraper"),
- "yandex": (
- old_yandex_full_load,
- new_yandex_full_load,
- "app.services.scrapers.yandex_realty.YandexRealtyScraper",
- ),
+ fn_map = {
+ "avito": run_avito_full_load,
+ "cian": run_cian_full_load,
+ "yandex": run_yandex_full_load,
}
- old_fn, new_fn, old_scraper_target = old_map[source]
+ scraper_target = {
+ "avito": f"{PFX}.AvitoScraper",
+ "cian": f"{PFX}.CianScraper",
+ "yandex": f"{PFX}.YandexRealtyScraper",
+ }[source]
+ fn = fn_map[source]
- if new:
- new_scraper_target = {
- "avito": f"{NEW_PFX}.AvitoScraper",
- "cian": f"{NEW_PFX}.CianScraper",
- "yandex": f"{NEW_PFX}.YandexRealtyScraper",
- }[source]
- extra: dict[str, Any] = {}
- if source == "yandex":
- enrichment = MagicMock()
- enrichment.record_yandex_price_history = MagicMock(return_value=0)
- extra["enrichment"] = enrichment
- with (
- patch(new_scraper_target, return_value=scraper),
- patch(f"{NEW_PFX}.save_listings", save_mock),
- patch(f"{NEW_PFX}.runs", recorder),
- ):
- counters = await new_fn(db, run_id=1, config=cfg, matcher=MagicMock(), **extra)
- else:
- import contextlib
-
- ctx = [
- patch(old_scraper_target, return_value=scraper),
- patch(f"{OLD_PFX}.save_listings", save_mock),
- patch(f"{OLD_PFX}.scrape_runs", recorder),
- patch(f"{OLD_PFX}.settings", cfg),
- patch(f"{OLD_PFX}.shutdown_requested", return_value=False),
- ]
- if source == "yandex":
- ctx.append(patch(f"{OLD_PFX}.record_yandex_price_history", MagicMock(return_value=0)))
- with contextlib.ExitStack() as stack:
- for cm in ctx:
- stack.enter_context(cm)
- counters = await old_fn(db, run_id=1)
+ extra: dict[str, Any] = {}
+ if source == "yandex":
+ enrichment = MagicMock()
+ enrichment.record_yandex_price_history = MagicMock(return_value=0)
+ extra["enrichment"] = enrichment
+ with (
+ patch(scraper_target, return_value=scraper),
+ patch(f"{PFX}.save_listings", save_mock),
+ patch(f"{PFX}.runs", recorder),
+ ):
+ counters = await fn(db, run_id=1, config=cfg, matcher=MagicMock(), **extra)
return counters.to_dict(), _normalize(recorder.calls)
@pytest.mark.asyncio
@pytest.mark.parametrize("source", ["avito", "cian", "yandex"])
-async def test_full_load_smoke_parity(source: str) -> None:
- """Full load: 2 бакета через on_bucket → save + heartbeat → mark_done. Counters ≡."""
- counters = await _assert_parity(
- await _drive_full_load(new=False, source=source),
- await _drive_full_load(new=True, source=source),
- )
+async def test_full_load_smoke(source: str) -> None:
+ """Full load: 2 бакета через on_bucket → save + heartbeat → mark_done."""
+ counters, calls = await _drive_full_load(source=source)
assert counters["unique_fetched"] == 3
assert counters["saved_inserted"] == 3
assert counters["saved_updated"] == 0
+ assert calls[-1][0] == "mark_done"
diff --git a/tradein-mvp/backend/tests/test_scraper_proxy.py b/tradein-mvp/backend/tests/test_scraper_proxy.py
index f6eb5cf3..98bd116e 100644
--- a/tradein-mvp/backend/tests/test_scraper_proxy.py
+++ b/tradein-mvp/backend/tests/test_scraper_proxy.py
@@ -95,35 +95,33 @@ def test_scraper_proxy_url_empty_string_falls_through_to_avito():
def test_avito_proxies_returns_correct_dict_shape():
- from app.services.scrape_pipeline import _avito_proxies
+ """Kit `_avito_proxies(config)` — DI параметр вместо module-level settings patch."""
+ from scraper_kit.orchestration.pipeline import _avito_proxies
proxy_url = "http://user:pass@proxy.example.com:8080"
- with patch("app.services.scrape_pipeline.settings", _mock_settings(proxy_url)):
- result = _avito_proxies()
+ result = _avito_proxies(_mock_settings(proxy_url))
assert result == {"http": proxy_url, "https": proxy_url}
def test_avito_proxies_returns_none_when_no_proxy():
- from app.services.scrape_pipeline import _avito_proxies
+ from scraper_kit.orchestration.pipeline import _avito_proxies
- with patch("app.services.scrape_pipeline.settings", _mock_settings(None)):
- result = _avito_proxies()
+ result = _avito_proxies(_mock_settings(None))
assert result is None
def test_avito_proxies_fallback_via_avito_proxy_url():
- """_avito_proxies() uses AVITO_PROXY_URL via scraper_proxy_url fallback.
+ """_avito_proxies(config) uses AVITO_PROXY_URL via scraper_proxy_url fallback.
- When settings.scraper_proxy_url resolves to the avito URL (fallback path),
+ When config.scraper_proxy_url resolves to the avito URL (fallback path),
the dict shape must be {"http": url, "https": url}.
"""
- from app.services.scrape_pipeline import _avito_proxies
+ from scraper_kit.orchestration.pipeline import _avito_proxies
avito_url = "http://mobile-proxy.space:9090"
- with patch("app.services.scrape_pipeline.settings", _mock_settings(avito_url)):
- result = _avito_proxies()
+ result = _avito_proxies(_mock_settings(avito_url))
assert result == {"http": avito_url, "https": avito_url}
diff --git a/tradein-mvp/backend/tests/test_scraper_resilience_1949_1950.py b/tradein-mvp/backend/tests/test_scraper_resilience_1949_1950.py
deleted file mode 100644
index ff96d687..00000000
--- a/tradein-mvp/backend/tests/test_scraper_resilience_1949_1950.py
+++ /dev/null
@@ -1,633 +0,0 @@
-"""Тесты устойчивости скрейперов: avito changeip-retry (#1950) + cian anti-zombie (#1949).
-
-Fix A (#1950): _rotate_proxy_ip теперь делает до proxy_rotate_attempts попыток
- по proxy_rotate_attempt_timeout_s каждая.
-
-Fix B (#1950): run_avito_city_sweep при AvitoBlockedError/RateLimited с
- lots_inserted+lots_updated>0 вызывает mark_done (не mark_banned)
- когда avito_serp_ok_not_banned=True.
-
-Fix C (#1949): run_cian_full_load применяет monkey-patch _browser.fetch с
- asyncio.wait_for(timeout=cian_full_load_per_fetch_timeout_s).
-
-Fix D (#1949): run_cian_full_load создаёт фоновый heartbeat-task и отменяет
- его в finally при любом завершении.
-
-Fix E (#1949): run_cian_city_sweep при defensive abort (consecutive anchor
- timeout/failure) вызывает mark_done (не mark_banned) когда
- SERP уже сохранил лоты (lots_inserted+lots_updated>0) и
- cian_sweep_ok_not_banned=True.
-"""
-
-from __future__ import annotations
-
-import asyncio
-import os
-import time
-from typing import Any
-from unittest.mock import AsyncMock, MagicMock, patch
-
-import pytest
-
-os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
-
-from app.services.scrapers.avito_exceptions import AvitoBlockedError
-from app.services.scrapers.base import ScrapedLot
-
-# ──────────────────────────────────────────────────────────────────────────────
-# Fix A: _rotate_proxy_ip retry loop (#1950)
-# ──────────────────────────────────────────────────────────────────────────────
-
-
-def _make_session_mock(get_side_effect: Any) -> AsyncMock:
- """Вспомогательный: AsyncMock-сессия с заданным поведением .get()."""
- mock_session = AsyncMock()
- mock_session.__aenter__ = AsyncMock(return_value=mock_session)
- mock_session.__aexit__ = AsyncMock(return_value=None)
- mock_session.get = AsyncMock(side_effect=get_side_effect)
- return mock_session
-
-
-@pytest.mark.asyncio
-async def test_changeip_retry_success_on_second_attempt() -> None:
- """#1950-A: первая попытка таймаут, вторая успех → возвращает True; ровно 2 GET-вызова."""
- from app.services.scrape_pipeline import _rotate_proxy_ip
-
- get_call_count = 0
-
- async def fake_get(url: str) -> MagicMock:
- nonlocal get_call_count
- get_call_count += 1
- if get_call_count == 1:
- raise TimeoutError("connect timeout")
- resp = MagicMock()
- resp.json.return_value = {"new_ip": "1.2.3.4"}
- return resp
-
- mock_session = _make_session_mock(fake_get)
-
- with (
- patch("app.services.scrape_pipeline.AsyncSession", return_value=mock_session),
- patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
- patch("app.services.scrape_pipeline.settings") as s,
- ):
- s.proxy_rotate_attempts = 3
- s.proxy_rotate_attempt_timeout_s = 8.0
- s.avito_proxy_rotate_settle_s = 0.0
- s.avito_proxy_rotate_url = "http://changeip.example.com/rotate?token=X"
- s.avito_proxy_max_rotations = 3
-
- result = await _rotate_proxy_ip(
- source="avito",
- reason="test-detail-block",
- rotations_done=0,
- )
-
- assert result is True
- assert get_call_count == 2, f"expected 2 GET calls, got {get_call_count}"
-
-
-@pytest.mark.asyncio
-async def test_changeip_retry_all_fail_returns_false() -> None:
- """#1950-A: все 3 попытки падают с TimeoutError → False; ровно 3 GET-вызова."""
- from app.services.scrape_pipeline import _rotate_proxy_ip
-
- get_call_count = 0
-
- async def failing_get(url: str) -> None:
- nonlocal get_call_count
- get_call_count += 1
- raise TimeoutError("connect timeout")
-
- mock_session = _make_session_mock(failing_get)
-
- with (
- patch("app.services.scrape_pipeline.AsyncSession", return_value=mock_session),
- patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
- patch("app.services.scrape_pipeline.settings") as s,
- ):
- s.proxy_rotate_attempts = 3
- s.proxy_rotate_attempt_timeout_s = 8.0
- s.avito_proxy_rotate_settle_s = 0.0
- s.avito_proxy_rotate_url = "http://changeip.example.com/rotate?token=X"
- s.avito_proxy_max_rotations = 3
-
- result = await _rotate_proxy_ip(
- source="avito",
- reason="test-detail-block",
- rotations_done=0,
- )
-
- assert result is False
- assert (
- get_call_count == 3
- ), f"expected exactly 3 attempts (proxy_rotate_attempts=3), got {get_call_count}"
-
-
-@pytest.mark.asyncio
-async def test_changeip_retry_bounded_by_setting() -> None:
- """#1950-A: proxy_rotate_attempts=2 → ровно 2 попытки, не больше."""
- from app.services.scrape_pipeline import _rotate_proxy_ip
-
- call_count = 0
-
- async def failing_get(url: str) -> None:
- nonlocal call_count
- call_count += 1
- raise TimeoutError("timeout")
-
- mock_session = _make_session_mock(failing_get)
-
- with (
- patch("app.services.scrape_pipeline.AsyncSession", return_value=mock_session),
- patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
- patch("app.services.scrape_pipeline.settings") as s,
- ):
- s.proxy_rotate_attempts = 2
- s.proxy_rotate_attempt_timeout_s = 8.0
- s.avito_proxy_rotate_settle_s = 0.0
- s.cian_proxy_rotate_url = "http://changeip.example.com/rotate"
- s.avito_proxy_rotate_url = None
- s.cian_proxy_max_rotations = 3
-
- result = await _rotate_proxy_ip(
- source="cian",
- reason="serp-block",
- rotations_done=1,
- )
-
- assert result is False
- assert call_count == 2, f"proxy_rotate_attempts=2 should make exactly 2 calls, got {call_count}"
-
-
-# ──────────────────────────────────────────────────────────────────────────────
-# Fix B: честный статус run_avito_city_sweep (#1950)
-# ──────────────────────────────────────────────────────────────────────────────
-
-
-def _make_city_sweep_mocks() -> tuple[MagicMock, MagicMock, MagicMock]:
- """Возвращает (mock_db, mock_runs, mock_scraper) для тестов run_avito_city_sweep."""
- mock_db = MagicMock()
-
- # priority_rows для detail-фазы: 3 строки с source_url
- class _DetailRow:
- def __getitem__(self, key: str) -> str:
- return "https://www.avito.ru/ekaterinburg/kvartiry/test-1"
-
- mock_db.execute.return_value.mappings.return_value.all.return_value = [
- _DetailRow(),
- _DetailRow(),
- _DetailRow(),
- ]
-
- mock_runs = MagicMock()
- mock_runs.is_cancelled.return_value = False
- mock_runs.mark_done = MagicMock()
- mock_runs.mark_banned = MagicMock()
- mock_runs.mark_failed = MagicMock()
- mock_runs.update_heartbeat = MagicMock()
-
- mock_scraper = MagicMock()
- # AvitoScraper НЕ используется как async CM в _avito_anchor_phases (просто instantiate)
- mock_scraper.fetch_around = AsyncMock(
- return_value=[
- ScrapedLot(
- source="avito",
- source_url=f"https://www.avito.ru/e/k/{i}",
- source_id=str(i),
- price_rub=5_000_000,
- address="ЕКБ test",
- listing_segment="vtorichka",
- )
- for i in range(3)
- ]
- )
-
- return mock_db, mock_runs, mock_scraper
-
-
-@pytest.mark.asyncio
-async def test_avito_city_sweep_serp_ok_marks_done_not_banned() -> None:
- """#1950-B: SERP собрал лоты (ins=3), detail заблокировало → mark_done не mark_banned.
-
- Условие: avito_serp_ok_not_banned=True (default), lots_inserted+lots_updated > 0.
- """
- from app.services.scrape_pipeline import run_avito_city_sweep
-
- mock_db, mock_runs, mock_scraper = _make_city_sweep_mocks()
-
- # Shared AsyncSession mock (browser_mode=False)
- mock_session = AsyncMock()
-
- with (
- patch("app.services.scrape_pipeline.AvitoScraper", return_value=mock_scraper),
- patch("app.services.scrape_pipeline.AsyncSession", return_value=mock_session),
- patch("app.services.scrape_pipeline.scrape_runs", mock_runs),
- patch("app.services.scrape_pipeline.save_listings", return_value=(3, 0)),
- patch(
- "app.services.scrape_pipeline.fetch_detail",
- AsyncMock(side_effect=AvitoBlockedError("HTTP 403 banned")),
- ),
- patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
- patch("app.services.scrape_pipeline.settings") as s,
- ):
- s.scraper_fetch_mode = "curl_cffi"
- s.scraper_proxy_url = None
- s.avito_proxy_max_rotations = 0 # no rotation budget
- s.avito_serp_ok_not_banned = True # ← наш флаг
- s.avito_proxy_rotate_settle_s = 0.0
- s.proxy_rotate_attempts = 3
- s.proxy_rotate_attempt_timeout_s = 8.0
-
- await run_avito_city_sweep(
- mock_db,
- run_id=1,
- anchors=[(56.84, 60.61, "test-anchor")],
- detail_top_n=3,
- enrich_houses=False,
- enrich_imv=False,
- )
-
- assert (
- mock_runs.mark_done.called
- ), "mark_done должен быть вызван: SERP OK (ins=3), detail blocked"
- assert (
- not mock_runs.mark_banned.called
- ), "mark_banned НЕ должен быть вызван когда SERP уже сохранил лоты"
- # enrichment_abort_note передан в counters
- call_args = mock_runs.mark_done.call_args
- # mark_done(db, run_id, counters_dict)
- passed_counters: dict = call_args[0][2] if call_args[0] else {}
- assert (
- "enrichment_abort_note" in passed_counters
- ), "enrichment_abort_note должен присутствовать в counters переданных mark_done"
-
-
-@pytest.mark.asyncio
-async def test_avito_city_sweep_serp_zero_lots_marks_banned() -> None:
- """#1950-B: SERP вернул 0 лотов, SERP-блок → mark_banned (не mark_done).
-
- Условие: lots_inserted+lots_updated == 0.
- """
- from app.services.scrape_pipeline import run_avito_city_sweep
-
- mock_db, mock_runs, _ = _make_city_sweep_mocks()
-
- # SERP сам поднимает AvitoBlockedError → anchor_lots не сохраняется
- mock_scraper_serp_block = MagicMock()
- mock_scraper_serp_block.fetch_around = AsyncMock(side_effect=AvitoBlockedError("SERP HTTP 403"))
-
- mock_session = AsyncMock()
-
- with (
- patch("app.services.scrape_pipeline.AvitoScraper", return_value=mock_scraper_serp_block),
- patch("app.services.scrape_pipeline.AsyncSession", return_value=mock_session),
- patch("app.services.scrape_pipeline.scrape_runs", mock_runs),
- patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
- patch("app.services.scrape_pipeline.settings") as s,
- ):
- s.scraper_fetch_mode = "curl_cffi"
- s.scraper_proxy_url = None
- s.avito_proxy_max_rotations = 0
- s.avito_serp_ok_not_banned = True # включён флаг, но лотов нет
- s.avito_proxy_rotate_settle_s = 0.0
- s.proxy_rotate_attempts = 3
- s.proxy_rotate_attempt_timeout_s = 8.0
-
- await run_avito_city_sweep(
- mock_db,
- run_id=2,
- anchors=[(56.84, 60.61, "test-anchor")],
- detail_top_n=3,
- enrich_houses=False,
- enrich_imv=False,
- )
-
- assert (
- mock_runs.mark_banned.called
- ), "mark_banned должен быть вызван: SERP сам заблокирован, lots=0"
- assert not mock_runs.mark_done.called, "mark_done НЕ должен быть вызван при lots_inserted == 0"
-
-
-@pytest.mark.asyncio
-async def test_avito_city_sweep_flag_off_still_marks_banned() -> None:
- """#1950-B: avito_serp_ok_not_banned=False → mark_banned даже при lots>0 (backward-compat)."""
- from app.services.scrape_pipeline import run_avito_city_sweep
-
- mock_db, mock_runs, mock_scraper = _make_city_sweep_mocks()
- mock_session = AsyncMock()
-
- with (
- patch("app.services.scrape_pipeline.AvitoScraper", return_value=mock_scraper),
- patch("app.services.scrape_pipeline.AsyncSession", return_value=mock_session),
- patch("app.services.scrape_pipeline.scrape_runs", mock_runs),
- patch("app.services.scrape_pipeline.save_listings", return_value=(3, 0)),
- patch(
- "app.services.scrape_pipeline.fetch_detail",
- AsyncMock(side_effect=AvitoBlockedError("HTTP 403")),
- ),
- patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
- patch("app.services.scrape_pipeline.settings") as s,
- ):
- s.scraper_fetch_mode = "curl_cffi"
- s.scraper_proxy_url = None
- s.avito_proxy_max_rotations = 0
- s.avito_serp_ok_not_banned = False # ← флаг выключен
- s.avito_proxy_rotate_settle_s = 0.0
- s.proxy_rotate_attempts = 3
- s.proxy_rotate_attempt_timeout_s = 8.0
-
- await run_avito_city_sweep(
- mock_db,
- run_id=3,
- anchors=[(56.84, 60.61, "test-anchor")],
- detail_top_n=3,
- enrich_houses=False,
- enrich_imv=False,
- )
-
- assert (
- mock_runs.mark_banned.called
- ), "avito_serp_ok_not_banned=False → mark_banned (backward-compat)"
- assert not mock_runs.mark_done.called
-
-
-# ──────────────────────────────────────────────────────────────────────────────
-# Fix C: cian per-fetch wait_for (#1949)
-# ──────────────────────────────────────────────────────────────────────────────
-
-
-@pytest.mark.asyncio
-async def test_cian_full_load_hanging_fetch_cancelled_by_timeout() -> None:
- """#1949-C: browser-fetch зависший навсегда отменяется per-fetch timeout.
-
- Monkey-patch заменяет _browser.fetch на _timed_fetch(wait_for timeout=0.05s).
- Зависший fetch отменяется через ~50ms → TimeoutError → run завершается.
- """
- from app.services.scrape_pipeline import run_cian_full_load
-
- fetch_was_called = False
-
- async def hanging_fetch(url: str) -> str:
- nonlocal fetch_was_called
- fetch_was_called = True
- await asyncio.Event().wait() # висит вечно
- return ""
-
- mock_browser = MagicMock()
- mock_browser.fetch = hanging_fetch
-
- mock_scraper = MagicMock()
- mock_scraper.__aenter__ = AsyncMock(return_value=mock_scraper)
- mock_scraper.__aexit__ = AsyncMock(return_value=None)
- mock_scraper._browser = mock_browser
- mock_scraper.request_delay_sec = 0.0
-
- # fetch_all_secondary симулирует _fetch_page_html: вызывает fetch и ловит Exception
- async def fake_fetch_all_secondary(**kwargs: object) -> None:
- try:
- # После monkey-patch _browser.fetch = _timed_fetch (с timeout=0.05s)
- await mock_scraper._browser.fetch("https://ekb.cian.ru/sale/flat/1/")
- except Exception:
- pass # TimeoutError → аналог _fetch_page_html → returns None
-
- mock_scraper.fetch_all_secondary = fake_fetch_all_secondary
-
- mock_db = MagicMock()
- mock_runs = MagicMock()
- mock_runs.is_cancelled.return_value = False
- mock_runs.mark_done = MagicMock()
- mock_runs.mark_failed = MagicMock()
- mock_runs.update_heartbeat = MagicMock()
-
- t0 = time.monotonic()
- with (
- patch("app.services.scrapers.cian.CianScraper", return_value=mock_scraper),
- patch("app.services.scrape_pipeline.scrape_runs", mock_runs),
- patch("app.services.scrape_pipeline.settings") as s,
- ):
- s.cian_full_load_per_fetch_timeout_s = 0.05 # 50ms — быстрый timeout для теста
- s.cian_full_load_detail_top_n = 0
-
- await run_cian_full_load(mock_db, run_id=1)
-
- elapsed = time.monotonic() - t0
-
- assert fetch_was_called, "hanging_fetch должен быть вызван через monkey-patched _browser.fetch"
- assert (
- elapsed < 3.0
- ), f"run должен завершиться быстро (fetch timeout 50ms), занял {elapsed:.2f}s"
- assert mock_runs.mark_done.called, "run_cian_full_load должен завершиться mark_done"
-
-
-@pytest.mark.asyncio
-async def test_cian_full_load_per_fetch_timeout_disabled_when_zero() -> None:
- """#1949-C: cian_full_load_per_fetch_timeout_s=0 → monkey-patch НЕ применяется."""
- from app.services.scrape_pipeline import run_cian_full_load
-
- original_fetch = AsyncMock(return_value="<html/>")
-
- mock_browser = MagicMock()
- mock_browser.fetch = original_fetch
-
- mock_scraper = MagicMock()
- mock_scraper.__aenter__ = AsyncMock(return_value=mock_scraper)
- mock_scraper.__aexit__ = AsyncMock(return_value=None)
- mock_scraper._browser = mock_browser
- mock_scraper.request_delay_sec = 0.0
- mock_scraper.fetch_all_secondary = AsyncMock(return_value=None)
-
- mock_db = MagicMock()
- mock_runs = MagicMock()
- mock_runs.is_cancelled.return_value = False
- mock_runs.mark_done = MagicMock()
- mock_runs.mark_failed = MagicMock()
- mock_runs.update_heartbeat = MagicMock()
-
- with (
- patch("app.services.scrapers.cian.CianScraper", return_value=mock_scraper),
- patch("app.services.scrape_pipeline.scrape_runs", mock_runs),
- patch("app.services.scrape_pipeline.settings") as s,
- ):
- s.cian_full_load_per_fetch_timeout_s = 0.0 # отключено
- s.cian_full_load_detail_top_n = 0
-
- await run_cian_full_load(mock_db, run_id=2)
-
- # Когда timeout=0 (отключён), monkey-patch не применяется → fetch остаётся original
- assert (
- mock_browser.fetch is original_fetch
- ), "cian_full_load_per_fetch_timeout_s=0 → fetch НЕ должен быть заменён monkey-patch"
-
-
-# ──────────────────────────────────────────────────────────────────────────────
-# Fix D: cian background heartbeat created and cancelled (#1949)
-# ──────────────────────────────────────────────────────────────────────────────
-
-
-@pytest.mark.asyncio
-async def test_cian_full_load_background_heartbeat_created_and_cancelled() -> None:
- """#1949-D: background heartbeat task создаётся и отменяется в finally."""
- from app.services.scrape_pipeline import run_cian_full_load
-
- mock_scraper = MagicMock()
- mock_scraper.__aenter__ = AsyncMock(return_value=mock_scraper)
- mock_scraper.__aexit__ = AsyncMock(return_value=None)
- mock_scraper._browser = None # per-fetch timeout не применяется
- mock_scraper.fetch_all_secondary = AsyncMock(return_value=None)
-
- mock_db = MagicMock()
- mock_runs = MagicMock()
- mock_runs.is_cancelled.return_value = False
- mock_runs.mark_done = MagicMock()
- mock_runs.mark_failed = MagicMock()
- mock_runs.update_heartbeat = MagicMock()
-
- created_tasks: list[asyncio.Task[None]] = []
- _real_create_task = asyncio.create_task
-
- def _tracking_create_task(coro: Any, *, name: str | None = None) -> asyncio.Task[None]:
- task = _real_create_task(coro, name=name)
- created_tasks.append(task)
- return task # type: ignore[return-value]
-
- with (
- patch("app.services.scrapers.cian.CianScraper", return_value=mock_scraper),
- patch("app.services.scrape_pipeline.scrape_runs", mock_runs),
- patch(
- "app.services.scrape_pipeline.asyncio.create_task",
- side_effect=_tracking_create_task,
- ),
- patch("app.services.scrape_pipeline.settings") as s,
- ):
- s.cian_full_load_per_fetch_timeout_s = 0.0
- s.cian_full_load_detail_top_n = 0
-
- await run_cian_full_load(mock_db, run_id=3)
-
- assert (
- len(created_tasks) >= 1
- ), "asyncio.create_task должен быть вызван хотя бы раз (background heartbeat)"
- for task in created_tasks:
- assert (
- task.cancelled() or task.done()
- ), f"Heartbeat task должен быть отменён в finally: {task}"
-
-
-@pytest.mark.asyncio
-async def test_cian_full_load_heartbeat_cancelled_on_exception() -> None:
- """#1949-D: heartbeat task отменяется в finally даже при исключении в run."""
- from app.services.scrape_pipeline import run_cian_full_load
-
- mock_scraper = MagicMock()
- mock_scraper.__aenter__ = AsyncMock(return_value=mock_scraper)
- mock_scraper.__aexit__ = AsyncMock(return_value=None)
- mock_scraper._browser = None
- mock_scraper.fetch_all_secondary = AsyncMock(side_effect=RuntimeError("fatal-test-error"))
-
- mock_db = MagicMock()
- mock_runs = MagicMock()
- mock_runs.is_cancelled.return_value = False
- mock_runs.mark_done = MagicMock()
- mock_runs.mark_failed = MagicMock()
- mock_runs.update_heartbeat = MagicMock()
-
- created_tasks: list[asyncio.Task[None]] = []
- _real_create_task = asyncio.create_task
-
- def _tracking_create_task(coro: Any, *, name: str | None = None) -> asyncio.Task[None]:
- task = _real_create_task(coro, name=name)
- created_tasks.append(task)
- return task # type: ignore[return-value]
-
- with (
- patch("app.services.scrapers.cian.CianScraper", return_value=mock_scraper),
- patch("app.services.scrape_pipeline.scrape_runs", mock_runs),
- patch(
- "app.services.scrape_pipeline.asyncio.create_task",
- side_effect=_tracking_create_task,
- ),
- patch("app.services.scrape_pipeline.settings") as s,
- ):
- s.cian_full_load_per_fetch_timeout_s = 0.0
- s.cian_full_load_detail_top_n = 0
-
- with pytest.raises(RuntimeError, match="fatal-test-error"):
- await run_cian_full_load(mock_db, run_id=4)
-
- assert len(created_tasks) >= 1, "heartbeat task должен быть создан"
- for task in created_tasks:
- assert (
- task.cancelled() or task.done()
- ), f"Heartbeat task должен быть отменён в finally даже при RuntimeError: {task}"
- assert mock_runs.mark_failed.called, "mark_failed должен быть вызван при RuntimeError"
-
-
-# ──────────────────────────────────────────────────────────────────────────────
-# Fix E: run_cian_city_sweep честный статус при defensive abort (#1949)
-# ──────────────────────────────────────────────────────────────────────────────
-
-
-def test_mark_cian_sweep_abort_lots_present_marks_done() -> None:
- """#1949-E: SERP уже сохранил лоты (ins+upd>0) до abort → mark_done, не mark_banned."""
- from app.services.scrape_pipeline import CianCitySweepCounters, _mark_cian_sweep_abort
-
- mock_db = MagicMock()
- mock_runs = MagicMock()
- counters = CianCitySweepCounters(
- anchors_total=5, anchors_done=2, lots_inserted=12, lots_updated=3
- )
-
- with (
- patch("app.services.scrape_pipeline.scrape_runs", mock_runs),
- patch("app.services.scrape_pipeline.settings") as s,
- ):
- s.cian_sweep_ok_not_banned = True
- _mark_cian_sweep_abort(mock_db, 42, counters, "cian sweep aborted: 3 consecutive timeouts")
-
- assert mock_runs.mark_done.called, "mark_done должен быть вызван: SERP OK (ins=12 upd=3)"
- assert not mock_runs.mark_banned.called, "mark_banned НЕ должен вызываться когда лоты сохранены"
- passed_counters: dict = mock_runs.mark_done.call_args[0][2]
- assert "abort_note" in passed_counters, "abort_note должен присутствовать в counters"
-
-
-def test_mark_cian_sweep_abort_no_lots_marks_banned() -> None:
- """#1949-E: лотов ещё не было (ins=upd=0) на момент abort → mark_banned как раньше."""
- from app.services.scrape_pipeline import CianCitySweepCounters, _mark_cian_sweep_abort
-
- mock_db = MagicMock()
- mock_runs = MagicMock()
- counters = CianCitySweepCounters(
- anchors_total=5, anchors_done=1, lots_inserted=0, lots_updated=0
- )
-
- with (
- patch("app.services.scrape_pipeline.scrape_runs", mock_runs),
- patch("app.services.scrape_pipeline.settings") as s,
- ):
- s.cian_sweep_ok_not_banned = True
- _mark_cian_sweep_abort(mock_db, 43, counters, "cian sweep aborted: SERP blocked at anchor")
-
- assert mock_runs.mark_banned.called, "mark_banned должен быть вызван: SERP не собрал лотов"
- assert not mock_runs.mark_done.called, "mark_done НЕ должен вызываться при lots ins=upd=0"
-
-
-def test_mark_cian_sweep_abort_flag_off_still_marks_banned() -> None:
- """#1949-E: cian_sweep_ok_not_banned=False → mark_banned даже при lots>0 (backward-compat)."""
- from app.services.scrape_pipeline import CianCitySweepCounters, _mark_cian_sweep_abort
-
- mock_db = MagicMock()
- mock_runs = MagicMock()
- counters = CianCitySweepCounters(
- anchors_total=5, anchors_done=3, lots_inserted=20, lots_updated=5
- )
-
- with (
- patch("app.services.scrape_pipeline.scrape_runs", mock_runs),
- patch("app.services.scrape_pipeline.settings") as s,
- ):
- s.cian_sweep_ok_not_banned = False
- _mark_cian_sweep_abort(mock_db, 44, counters, "cian sweep aborted: 3 consecutive timeouts")
-
- assert mock_runs.mark_banned.called, "flag=False → mark_banned (backward-compat)"
- assert not mock_runs.mark_done.called
diff --git a/tradein-mvp/backend/tests/test_sweep_drain.py b/tradein-mvp/backend/tests/test_sweep_drain.py
deleted file mode 100644
index 7f7d7722..00000000
--- a/tradein-mvp/backend/tests/test_sweep_drain.py
+++ /dev/null
@@ -1,310 +0,0 @@
-"""Тесты кооперативного SIGTERM-drain в длинных sweep/full-load циклах (#1182 Phase 3a).
-
-Проверяют, что при выставленном shutdown_requested() sweep'ы scrape_pipeline.py
-останавливаются ЧИСТО на своих between-unit checkpoint'ах:
- - финализируют run как mark_done(partial) — НЕ mark_cancelled (это не user-cancel);
- - НЕ обрабатывают последующие anchor'ы / бакеты / дорогие фазы (IMV / detail);
- - user-cancel ветка (is_cancelled) не задействуется.
-
-Тесты network-free: внутренние scrape-вызовы и lifecycle скрейперов monkeypatch'атся,
-shutdown_requested() патчится контролируемым флагом, переключающимся в True ПОСЛЕ
-первого unit'а (а не с самого старта) — чтобы первый unit прошёл, а на втором
-сработал drain.
-"""
-
-from __future__ import annotations
-
-import os
-
-os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
-
-from typing import Any
-from unittest.mock import AsyncMock, MagicMock
-
-import pytest
-
-from app.services import scrape_pipeline
-from app.services.scrapers.avito import AvitoScraper
-from app.services.scrapers.base import ScrapedLot
-from app.services.scrapers.cian import CianScraper
-
-# Два anchor'а: первый успешен, второй должен быть отрезан drain'ом.
-TWO_ANCHORS: list[tuple[float, float, str]] = [
- (56.8400, 60.6050, "First"),
- (56.7950, 60.5300, "Second"),
-]
-
-
-# ── Fake scrape_runs (записывает финализаторы) ──────────────────────────────
-
-
-class _FakeRuns:
- def __init__(self) -> None:
- self.heartbeats: list[dict[str, int]] = []
- self.done: dict[str, int] | None = None
- self.failed: tuple[str, dict[str, int]] | None = None
- self.banned: tuple[str, dict[str, int]] | None = None
- self.cancelled_calls: int = 0
- self.is_cancelled_calls: int = 0
-
- def is_cancelled(self, _db: Any, _run_id: int) -> bool:
- self.is_cancelled_calls += 1
- return False
-
- def update_heartbeat(self, _db: Any, _run_id: int, counters: dict[str, int]) -> None:
- self.heartbeats.append(dict(counters))
-
- def mark_done(self, _db: Any, _run_id: int, counters: dict[str, int]) -> None:
- self.done = dict(counters)
-
- def mark_failed(self, _db: Any, _run_id: int, error: str, counters: dict[str, int]) -> None:
- self.failed = (error, dict(counters))
-
- def mark_banned(self, _db: Any, _run_id: int, error: str, counters: dict[str, int]) -> None:
- self.banned = (error, dict(counters))
-
- def mark_cancelled(self, _db: Any, _run_id: int) -> bool:
- self.cancelled_calls += 1
- return True
-
-
-def _install_runs(monkeypatch: pytest.MonkeyPatch, fake: _FakeRuns) -> None:
- monkeypatch.setattr(scrape_pipeline.scrape_runs, "is_cancelled", fake.is_cancelled)
- monkeypatch.setattr(scrape_pipeline.scrape_runs, "update_heartbeat", fake.update_heartbeat)
- monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_done", fake.mark_done)
- monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_failed", fake.mark_failed)
- monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_banned", fake.mark_banned)
- monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_cancelled", fake.mark_cancelled)
-
-
-def _fake_lot(source: str, offer_id: str) -> ScrapedLot:
- return ScrapedLot(
- source=source,
- source_url=f"https://example.ru/{offer_id}/",
- source_id=offer_id,
- address="Екатеринбург, улица Тестовая, 1",
- price_rub=4_000_000,
- area_m2=42.0,
- rooms=1,
- )
-
-
-# ── Avito city sweep: drain на границе anchor'а ─────────────────────────────
-
-
-async def test_avito_city_sweep_drains_on_shutdown(monkeypatch: pytest.MonkeyPatch) -> None:
- """SIGTERM после первого anchor'а → sweep останавливается на втором anchor'е:
-
- - второй anchor НЕ фетчится (drain до начала его фаз);
- - run финализируется mark_done(partial) с anchors_done==1;
- - mark_cancelled НЕ вызывается (это drain, не user-cancel);
- - mark_failed / mark_banned не вызываются.
- """
- import curl_cffi.requests as _cffi_mod
-
- state = {"flag": False}
- monkeypatch.setattr(scrape_pipeline, "shutdown_requested", lambda: state["flag"])
- monkeypatch.setattr(scrape_pipeline.asyncio, "sleep", AsyncMock())
-
- fetch_calls: list[str] = []
-
- async def fake_fetch_around(
- self: AvitoScraper,
- lat: float,
- lon: float,
- radius_m: int,
- pages: int = 1,
- delay_override_sec: float | None = None,
- ) -> list[ScrapedLot]:
- fetch_calls.append(f"{lat:.4f}")
- lots = [_fake_lot("avito", f"{lat:.4f}-{i}") for i in range(3)]
- # SIGTERM приходит ПОСЛЕ первого anchor'а — следующий top-of-loop checkpoint
- # увидит флаг и сделает drain.
- state["flag"] = True
- return lots
-
- monkeypatch.setattr(AvitoScraper, "fetch_around", fake_fetch_around)
- monkeypatch.setattr(scrape_pipeline, "save_listings", lambda _db, lots, **_kw: (len(lots), 0))
-
- # Stub shared curl_cffi AsyncSession (non-browser fetch mode).
- fake_session = AsyncMock()
- fake_session.__aenter__ = AsyncMock(return_value=fake_session)
- fake_session.__aexit__ = AsyncMock(return_value=None)
- monkeypatch.setattr(_cffi_mod, "AsyncSession", lambda **_kw: fake_session)
-
- # IMV-фаза не должна вызываться при drain (enrich_imv=False всё равно).
- monkeypatch.setattr(
- "app.services.house_imv_backfill.process_houses_imv_batch",
- AsyncMock(side_effect=AssertionError("IMV phase must not run after drain")),
- )
-
- fake = _FakeRuns()
- _install_runs(monkeypatch, fake)
-
- counters = await scrape_pipeline.run_avito_city_sweep(
- db=MagicMock(),
- run_id=1,
- anchors=TWO_ANCHORS,
- enrich_houses=False,
- detail_top_n=0,
- enrich_imv=False,
- request_delay_sec=0.0,
- )
-
- # Второй anchor не дошёл до SERP — fetch вызван ровно один раз.
- assert fetch_calls == ["56.8400"], f"только первый anchor должен фетчиться; got {fetch_calls}"
- # Partial-финализация: обработан только первый anchor.
- assert counters.anchors_done == 1
- assert counters.lots_fetched == 3
- # mark_done(partial) вызван (drain-ветка), и счётчики partial.
- assert fake.done is not None, "drain должен вызвать mark_done(partial)"
- assert fake.done["anchors_done"] == 1
- # Это НЕ user-cancel.
- assert fake.cancelled_calls == 0, "mark_cancelled не должен вызываться при drain"
- assert fake.failed is None
- assert fake.banned is None
-
-
-# ── Cian city sweep: drain на границе anchor'а ──────────────────────────────
-
-
-@pytest.fixture
-def _stub_cian_lifecycle(monkeypatch: pytest.MonkeyPatch) -> None:
- def _init(self: CianScraper) -> None:
- self.request_delay_sec = 0.0
- self._cffi = None
- self._browser = None
-
- async def _aenter(self: CianScraper) -> CianScraper:
- return self
-
- async def _aexit(self: CianScraper, *_args: Any) -> None:
- return None
-
- monkeypatch.setattr(CianScraper, "__init__", _init)
- monkeypatch.setattr(CianScraper, "__aenter__", _aenter)
- monkeypatch.setattr(CianScraper, "__aexit__", _aexit)
-
-
-async def test_cian_city_sweep_drains_on_shutdown(
- monkeypatch: pytest.MonkeyPatch, _stub_cian_lifecycle: None
-) -> None:
- """SIGTERM после первого anchor'а → cian sweep drain на втором anchor'е → mark_done(partial)."""
- state = {"flag": False}
- monkeypatch.setattr(scrape_pipeline, "shutdown_requested", lambda: state["flag"])
- monkeypatch.setattr(scrape_pipeline.asyncio, "sleep", AsyncMock())
-
- serp_calls: list[str] = []
-
- async def fake_fetch_multi(
- self: CianScraper, lat: float, lon: float, r: int, pages: int = 3
- ) -> list[ScrapedLot]:
- serp_calls.append(f"{lat:.4f}")
- state["flag"] = True
- return [_fake_lot("cian", f"{lat:.4f}-1")]
-
- monkeypatch.setattr(CianScraper, "fetch_around_multi_room", fake_fetch_multi)
- monkeypatch.setattr(
- scrape_pipeline, "save_listings", lambda _db, lots, *, run_id=None: (len(lots), 0)
- )
- monkeypatch.setattr(
- "app.services.scrapers.cian_detail.fetch_detail", AsyncMock(return_value=None)
- )
-
- fake = _FakeRuns()
- _install_runs(monkeypatch, fake)
-
- counters = await scrape_pipeline.run_cian_city_sweep(
- db=MagicMock(),
- run_id=2,
- anchors=TWO_ANCHORS,
- detail_top_n=0,
- enrich_houses=False,
- request_delay_sec=0.0,
- )
-
- assert serp_calls == ["56.8400"], f"только первый anchor должен фетчиться; got {serp_calls}"
- assert counters.anchors_done == 1
- assert fake.done is not None, "drain должен вызвать mark_done(partial)"
- assert fake.done["anchors_done"] == 1
- assert fake.cancelled_calls == 0
- assert fake.failed is None
- assert fake.banned is None
-
-
-# ── Avito full load: drain через sentinel в on_bucket ───────────────────────
-
-
-@pytest.fixture
-def _stub_avito_lifecycle(monkeypatch: pytest.MonkeyPatch) -> None:
- def _init(self: AvitoScraper) -> None:
- self.request_delay_sec = 0.0
- self._cffi = None
- self._browser = None
-
- async def _aenter(self: AvitoScraper) -> AvitoScraper:
- return self
-
- async def _aexit(self: AvitoScraper, *_args: Any) -> None:
- return None
-
- monkeypatch.setattr(AvitoScraper, "__init__", _init)
- monkeypatch.setattr(AvitoScraper, "__aenter__", _aenter)
- monkeypatch.setattr(AvitoScraper, "__aexit__", _aexit)
-
-
-async def test_avito_full_load_drains_on_shutdown(
- monkeypatch: pytest.MonkeyPatch, _stub_avito_lifecycle: None
-) -> None:
- """SIGTERM после первого бакета → on_bucket поднимает sentinel → mark_done(partial):
-
- - бакет №2 поднимает RuntimeError("shutdown") (НЕ "cancelled"), бакет №3 не достигается;
- - done_buckets == ["b1"] (бакет, упавший на drain-checkpoint, в done не попадает);
- - run финализируется mark_done(partial) с unique_fetched==1;
- - mark_cancelled / mark_failed / mark_banned не вызываются.
- """
- state = {"flag": False}
- monkeypatch.setattr(scrape_pipeline, "shutdown_requested", lambda: state["flag"])
-
- saved_buckets: list[str] = []
-
- def fake_save(_db: Any, lots: Any, **_kw: Any) -> tuple[int, int]:
- saved_buckets.append("x")
- return (len(lots), 0)
-
- monkeypatch.setattr(scrape_pipeline, "save_listings", fake_save)
-
- async def fake_fetch_all(self: AvitoScraper, **kwargs: Any) -> None:
- on_bucket = kwargs["on_bucket"]
- # Бакет №1 — сохраняется штатно.
- on_bucket("b1", [_fake_lot("avito", "b1-1")])
- # SIGTERM приходит после первого бакета.
- state["flag"] = True
- # Бакет №2 — on_bucket должен поднять RuntimeError("shutdown").
- on_bucket("b2", [_fake_lot("avito", "b2-1")])
- # Бакет №3 — НЕ должен быть достигнут.
- on_bucket("b3", [_fake_lot("avito", "b3-1")])
-
- monkeypatch.setattr(AvitoScraper, "fetch_all_secondary", fake_fetch_all)
-
- fake = _FakeRuns()
- _install_runs(monkeypatch, fake)
-
- counters = await scrape_pipeline.run_avito_full_load(
- db=MagicMock(),
- run_id=3,
- request_delay_sec=0.0,
- )
-
- # Только первый бакет сохранён (b2 упал на checkpoint до save, b3 не достигнут).
- assert saved_buckets == ["x"], f"только b1 должен сохраниться; got {saved_buckets}"
- assert counters.unique_fetched == 1
- assert counters.saved_inserted == 1
- # mark_done(partial) с done_buckets только b1.
- assert fake.done is not None, "drain должен вызвать mark_done(partial)"
- assert fake.done.get("done_buckets") == ["b1"]
- assert fake.done["unique_fetched"] == 1
- # НЕ user-cancel, не failed/banned.
- assert fake.cancelled_calls == 0
- assert fake.failed is None
- assert fake.banned is None
diff --git a/tradein-mvp/backend/tests/test_sweep_imv_phase.py b/tradein-mvp/backend/tests/test_sweep_imv_phase.py
index 7a577320..3e7417ea 100644
--- a/tradein-mvp/backend/tests/test_sweep_imv_phase.py
+++ b/tradein-mvp/backend/tests/test_sweep_imv_phase.py
@@ -1,4 +1,4 @@
-"""Tests for IMV enrichment phase integrated into Avito city sweep.
+"""Tests for IMV enrichment phase integrated into Avito city sweep (kit).
Проверяет:
1. enrich_imv=True — sweep вызывает IMV-фазу для touched house_id.
@@ -9,9 +9,12 @@
6. CitySweepStartRequest принимает enrich_imv (API-level param).
7. Cooperative cancel перед IMV-фазой — skip IMV, sweep marked done.
-NB: run_avito_city_sweep использует внутренние _avito_anchor_phases (не run_avito_pipeline).
-Тесты патчат AvitoScraper.fetch_around + save_listings + fetch_house_catalog +
-save_house_catalog_enrichment для контроля touched_house_ids.
+Kit — единственная orchestration-копия (#2397 Part E1 удалил legacy
+`app.services.scrape_pipeline`): behavior-тесты (пункты 1,2,3,5,7 выше через
+`run_avito_city_sweep`) переведены на `scraper_kit.orchestration.pipeline` с DI
+(config/matcher/enrichment/shutdown_requested) вместо прямых импортов `app.*`.
+`enrichment.process_houses_imv_batch` — инжектируемый Protocol-метод (было прямым
+импортом `app.services.house_imv_backfill.process_houses_imv_batch`).
"""
from __future__ import annotations
@@ -19,6 +22,7 @@ from __future__ import annotations
import os
from contextlib import ExitStack
from dataclasses import fields
+from types import SimpleNamespace
from typing import Any
from unittest.mock import AsyncMock, MagicMock, patch
@@ -26,10 +30,12 @@ import pytest
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
+PFX = "scraper_kit.orchestration.pipeline"
+
def _make_house_lot(offer_id: str, house_path: str) -> Any:
"""ScrapedLot с house_url для тестов house-enrich пути."""
- from app.services.scrapers.base import ScrapedLot
+ from scraper_kit.base import ScrapedLot
return ScrapedLot(
source="avito",
@@ -43,16 +49,35 @@ def _make_house_lot(offer_id: str, house_path: str) -> Any:
)
+def _config() -> SimpleNamespace:
+ return SimpleNamespace(
+ scraper_fetch_mode="curl_cffi",
+ browser_http_endpoint="http://browser.test/fetch",
+ scraper_proxy_url=None,
+ avito_proxy_rotate_url=None,
+ avito_proxy_max_rotations=0,
+ avito_serp_ok_not_banned=True,
+ avito_proxy_rotate_settle_s=0.0,
+ proxy_rotate_attempts=1,
+ proxy_rotate_attempt_timeout_s=1.0,
+ cian_proxy_rotate_url=None,
+ cian_proxy_max_rotations=0,
+ yandex_proxy_rotate_url=None,
+ yandex_proxy_max_rotations=0,
+ scraper_skip_seen_today=False,
+ )
+
+
def _enter_common_patches(stack: ExitStack, house_ids: list[int]) -> None:
"""Входит в набор patch'ей для базового Avito sweep через ExitStack:
- AvitoScraper.fetch_around → lots с house_url (по одному на house_id)
- save_listings → (n, 0)
- fetch_house_catalog → MagicMock()
- save_house_catalog_enrichment → {"house_id": X} по очереди
- - curl_cffi.requests.AsyncSession → fake
+ - AsyncSession (kit-модуль) → fake
- asyncio.sleep → no-op
"""
- from app.services.scrapers.avito import AvitoScraper
+ from scraper_kit.providers.avito.serp import AvitoScraper
lots = [_make_house_lot(f"lot-{hid}", f"/ekaterinburg/houses/test/{hid}") for hid in house_ids]
@@ -62,7 +87,7 @@ def _enter_common_patches(stack: ExitStack, house_ids: list[int]) -> None:
# save_house_catalog_enrichment вызывается по одному разу на каждый дом
house_id_iter = iter(house_ids)
- def fake_save_house(db: Any, enrichment: Any) -> dict:
+ def fake_save_house(db: Any, enrichment: Any, **_kw: Any) -> dict:
try:
return {"house_id": next(house_id_iter)}
except StopIteration:
@@ -73,33 +98,29 @@ def _enter_common_patches(stack: ExitStack, house_ids: list[int]) -> None:
fake_session.__aexit__ = AsyncMock(return_value=None)
stack.enter_context(patch.object(AvitoScraper, "fetch_around", fake_fetch_around))
- stack.enter_context(
- patch(
- "app.services.scrape_pipeline.save_listings",
- lambda _db, _lots, **_kw: (len(_lots), 0),
- )
- )
- stack.enter_context(
- patch(
- "app.services.scrape_pipeline.fetch_house_catalog",
- AsyncMock(return_value=MagicMock()),
- )
- )
- stack.enter_context(
- patch(
- "app.services.scrape_pipeline.save_house_catalog_enrichment",
- fake_save_house,
- )
- )
- stack.enter_context(patch("curl_cffi.requests.AsyncSession", return_value=fake_session))
- stack.enter_context(patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()))
+ stack.enter_context(patch(f"{PFX}.save_listings", lambda _db, _lots, **_kw: (len(_lots), 0)))
+ stack.enter_context(patch(f"{PFX}.fetch_house_catalog", AsyncMock(return_value=MagicMock())))
+ stack.enter_context(patch(f"{PFX}.save_house_catalog_enrichment", fake_save_house))
+ stack.enter_context(patch(f"{PFX}.AsyncSession", return_value=fake_session))
+ stack.enter_context(patch(f"{PFX}.asyncio.sleep", AsyncMock()))
+
+
+def _enter_runs_patches(stack: ExitStack, *, mark_done_mock: Any = None) -> Any:
+ """Патчит `scraper_kit.orchestration.pipeline.runs` (is_cancelled/update_heartbeat/
+ mark_done)."""
+ mock_runs = MagicMock()
+ mock_runs.is_cancelled.return_value = False
+ mock_runs.update_heartbeat = MagicMock()
+ mock_runs.mark_done = mark_done_mock if mark_done_mock is not None else MagicMock()
+ stack.enter_context(patch(f"{PFX}.runs", mock_runs))
+ return mock_runs
# ── CitySweepCounters has imv_* fields ────────────────────────────────────────
def test_city_sweep_counters_has_imv_fields() -> None:
- from app.services.scrape_pipeline import CitySweepCounters
+ from scraper_kit.orchestration.pipeline import CitySweepCounters
c = CitySweepCounters()
assert hasattr(c, "imv_attempted")
@@ -111,7 +132,7 @@ def test_city_sweep_counters_has_imv_fields() -> None:
def test_city_sweep_counters_to_dict_has_imv_keys() -> None:
- from app.services.scrape_pipeline import CitySweepCounters
+ from scraper_kit.orchestration.pipeline import CitySweepCounters
c = CitySweepCounters(imv_attempted=5, imv_enriched=3, imv_failed=2)
d = c.to_dict()
@@ -127,7 +148,7 @@ def test_city_sweep_counters_to_dict_has_imv_keys() -> None:
def test_pipeline_result_has_touched_house_ids() -> None:
- from app.services.scrape_pipeline import PipelineCounters, PipelineResult
+ from scraper_kit.orchestration.pipeline import PipelineCounters, PipelineResult
r = PipelineResult(
anchor_lat=56.84,
@@ -143,7 +164,7 @@ def test_pipeline_result_has_touched_house_ids() -> None:
def test_pipeline_result_touched_house_ids_set() -> None:
- from app.services.scrape_pipeline import PipelineCounters, PipelineResult
+ from scraper_kit.orchestration.pipeline import PipelineCounters, PipelineResult
r = PipelineResult(
anchor_lat=56.84,
@@ -167,36 +188,31 @@ async def test_sweep_enrich_imv_true_calls_imv_batch() -> None:
Патчим AvitoScraper.fetch_around → lots с house_url, save_house_catalog_enrichment
→ {house_id: X} чтобы all_touched_house_ids = {10, 20}.
"""
- from app.services.house_imv_backfill import HouseIMVBackfillResult
- from app.services.scrape_pipeline import run_avito_city_sweep
+ from scraper_kit.orchestration.pipeline import run_avito_city_sweep
mock_db = MagicMock()
- mock_imv_result = HouseIMVBackfillResult(checked=2, saved=2, skipped=0, errors=0)
+ imv_result = SimpleNamespace(checked=2, saved=2, errors=0)
+ enrichment = MagicMock()
+ enrichment.process_houses_imv_batch = AsyncMock(return_value=imv_result)
with ExitStack() as stack:
_enter_common_patches(stack, [10, 20])
- stack.enter_context(patch("app.services.scrape_runs.is_cancelled", return_value=False))
- stack.enter_context(patch("app.services.scrape_runs.update_heartbeat"))
- stack.enter_context(patch("app.services.scrape_runs.mark_done"))
- mock_imv_batch = stack.enter_context(
- patch(
- "app.services.house_imv_backfill.process_houses_imv_batch",
- new_callable=AsyncMock,
- return_value=mock_imv_result,
- )
- )
+ _enter_runs_patches(stack)
counters = await run_avito_city_sweep(
mock_db,
run_id=1,
+ config=_config(),
+ matcher=MagicMock(),
+ enrichment=enrichment,
anchors=[(56.84, 60.6, "TestAnchor")],
enrich_imv=True,
enrich_houses=True,
detail_top_n=0,
)
- mock_imv_batch.assert_awaited_once()
+ enrichment.process_houses_imv_batch.assert_awaited_once()
# Первый позиционный аргумент — db, второй — house_ids
- call_args = mock_imv_batch.call_args
+ call_args = enrichment.process_houses_imv_batch.call_args
assert call_args.args[1] == {10, 20}
assert counters.imv_attempted == 2
assert counters.imv_enriched == 2
@@ -209,31 +225,28 @@ async def test_sweep_enrich_imv_true_calls_imv_batch() -> None:
@pytest.mark.asyncio
async def test_sweep_enrich_imv_false_skips_imv() -> None:
"""enrich_imv=False → process_houses_imv_batch НЕ вызывается."""
- from app.services.scrape_pipeline import run_avito_city_sweep
+ from scraper_kit.orchestration.pipeline import run_avito_city_sweep
mock_db = MagicMock()
+ enrichment = MagicMock()
+ enrichment.process_houses_imv_batch = AsyncMock()
with ExitStack() as stack:
_enter_common_patches(stack, [99])
- stack.enter_context(patch("app.services.scrape_runs.is_cancelled", return_value=False))
- stack.enter_context(patch("app.services.scrape_runs.update_heartbeat"))
- stack.enter_context(patch("app.services.scrape_runs.mark_done"))
- mock_imv_batch = stack.enter_context(
- patch(
- "app.services.house_imv_backfill.process_houses_imv_batch",
- new_callable=AsyncMock,
- )
- )
+ _enter_runs_patches(stack)
counters = await run_avito_city_sweep(
mock_db,
run_id=2,
+ config=_config(),
+ matcher=MagicMock(),
+ enrichment=enrichment,
anchors=[(56.84, 60.6, "TestAnchor")],
enrich_imv=False,
enrich_houses=True,
detail_top_n=0,
)
- mock_imv_batch.assert_not_awaited()
+ enrichment.process_houses_imv_batch.assert_not_awaited()
assert counters.imv_attempted == 0
assert counters.imv_enriched == 0
assert counters.imv_failed == 0
@@ -248,8 +261,8 @@ async def test_sweep_enrich_imv_no_touched_ids_skips() -> None:
Лоты без house_url → unique_house_paths пуст → touched_house_ids пуст.
"""
- from app.services.scrape_pipeline import run_avito_city_sweep
- from app.services.scrapers.avito import AvitoScraper
+ from scraper_kit.orchestration.pipeline import run_avito_city_sweep
+ from scraper_kit.providers.avito.serp import AvitoScraper
mock_db = MagicMock()
fake_session = AsyncMock()
@@ -260,29 +273,30 @@ async def test_sweep_enrich_imv_no_touched_ids_skips() -> None:
async def fake_fetch_around(self: Any, lat: float, lon: float, *_a: Any, **_kw: Any) -> list:
return []
+ enrichment = MagicMock()
+ enrichment.process_houses_imv_batch = AsyncMock()
+
with (
patch.object(AvitoScraper, "fetch_around", fake_fetch_around),
- patch("app.services.scrape_pipeline.save_listings", lambda _db, _lots, **_kw: (0, 0)),
- patch("curl_cffi.requests.AsyncSession", return_value=fake_session),
- patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
- patch("app.services.scrape_runs.is_cancelled", return_value=False),
- patch("app.services.scrape_runs.update_heartbeat"),
- patch("app.services.scrape_runs.mark_done"),
- patch(
- "app.services.house_imv_backfill.process_houses_imv_batch",
- new_callable=AsyncMock,
- ) as mock_imv_batch,
+ patch(f"{PFX}.save_listings", lambda _db, _lots, **_kw: (0, 0)),
+ patch(f"{PFX}.AsyncSession", return_value=fake_session),
+ patch(f"{PFX}.asyncio.sleep", AsyncMock()),
+ patch(f"{PFX}.runs") as mock_runs,
):
+ mock_runs.is_cancelled.return_value = False
await run_avito_city_sweep(
mock_db,
run_id=3,
+ config=_config(),
+ matcher=MagicMock(),
+ enrichment=enrichment,
anchors=[(56.84, 60.6, "TestAnchor")],
enrich_imv=True,
enrich_houses=True,
detail_top_n=0,
)
- mock_imv_batch.assert_not_awaited()
+ enrichment.process_houses_imv_batch.assert_not_awaited()
# ── IMV-ошибки не валят sweep ────────────────────────────────────────────────
@@ -291,32 +305,28 @@ async def test_sweep_enrich_imv_no_touched_ids_skips() -> None:
@pytest.mark.asyncio
async def test_sweep_imv_crash_does_not_abort_sweep() -> None:
"""Если process_houses_imv_batch raises — sweep всё равно завершается (mark_done)."""
- from app.services.scrape_pipeline import run_avito_city_sweep
+ from scraper_kit.orchestration.pipeline import run_avito_city_sweep
mock_db = MagicMock()
+ enrichment = MagicMock()
+ enrichment.process_houses_imv_batch = AsyncMock(side_effect=RuntimeError("IMV network error"))
with ExitStack() as stack:
_enter_common_patches(stack, [55])
- stack.enter_context(patch("app.services.scrape_runs.is_cancelled", return_value=False))
- stack.enter_context(patch("app.services.scrape_runs.update_heartbeat"))
- mock_mark_done = stack.enter_context(patch("app.services.scrape_runs.mark_done"))
- stack.enter_context(
- patch(
- "app.services.house_imv_backfill.process_houses_imv_batch",
- new_callable=AsyncMock,
- side_effect=RuntimeError("IMV network error"),
- )
- )
+ mock_runs = _enter_runs_patches(stack)
counters = await run_avito_city_sweep(
mock_db,
run_id=4,
+ config=_config(),
+ matcher=MagicMock(),
+ enrichment=enrichment,
anchors=[(56.84, 60.6, "TestAnchor")],
enrich_imv=True,
enrich_houses=True,
detail_top_n=0,
)
- mock_mark_done.assert_called_once()
+ mock_runs.mark_done.assert_called_once()
# IMV crash → ошибка считается, но sweep завершается как done
assert counters.errors_count >= 1
@@ -326,29 +336,24 @@ async def test_sweep_imv_crash_does_not_abort_sweep() -> None:
@pytest.mark.asyncio
async def test_sweep_imv_failed_counter() -> None:
- """imv_failed отражает errors из HouseIMVBackfillResult."""
- from app.services.house_imv_backfill import HouseIMVBackfillResult
- from app.services.scrape_pipeline import run_avito_city_sweep
+ """imv_failed отражает errors из результата enrichment.process_houses_imv_batch."""
+ from scraper_kit.orchestration.pipeline import run_avito_city_sweep
mock_db = MagicMock()
# 3 attempted, 1 enriched, 2 failed
- mock_imv_result = HouseIMVBackfillResult(checked=3, saved=1, skipped=0, errors=2)
+ imv_result = SimpleNamespace(checked=3, saved=1, errors=2)
+ enrichment = MagicMock()
+ enrichment.process_houses_imv_batch = AsyncMock(return_value=imv_result)
with ExitStack() as stack:
_enter_common_patches(stack, [1, 2, 3])
- stack.enter_context(patch("app.services.scrape_runs.is_cancelled", return_value=False))
- stack.enter_context(patch("app.services.scrape_runs.update_heartbeat"))
- stack.enter_context(patch("app.services.scrape_runs.mark_done"))
- stack.enter_context(
- patch(
- "app.services.house_imv_backfill.process_houses_imv_batch",
- new_callable=AsyncMock,
- return_value=mock_imv_result,
- )
- )
+ _enter_runs_patches(stack)
counters = await run_avito_city_sweep(
mock_db,
run_id=5,
+ config=_config(),
+ matcher=MagicMock(),
+ enrichment=enrichment,
anchors=[(56.84, 60.6, "TestAnchor")],
enrich_imv=True,
enrich_houses=True,
@@ -368,9 +373,11 @@ async def test_sweep_imv_failed_counter() -> None:
@pytest.mark.asyncio
async def test_sweep_cancel_before_imv_skips_imv() -> None:
"""Если is_cancelled=True в IMV-фазе → IMV пропускается, sweep всё равно done."""
- from app.services.scrape_pipeline import run_avito_city_sweep
+ from scraper_kit.orchestration.pipeline import run_avito_city_sweep
mock_db = MagicMock()
+ enrichment = MagicMock()
+ enrichment.process_houses_imv_batch = AsyncMock()
# is_cancelled: первый вызов (anchor loop) = False, второй (IMV phase) = True
cancel_side_effects = [False, True]
@@ -384,31 +391,26 @@ async def test_sweep_cancel_before_imv_skips_imv() -> None:
with ExitStack() as stack:
_enter_common_patches(stack, [77])
- stack.enter_context(
- patch(
- "app.services.scrape_runs.is_cancelled",
- side_effect=is_cancelled_side_effect,
- )
- )
- stack.enter_context(patch("app.services.scrape_runs.update_heartbeat"))
- mock_mark_done = stack.enter_context(patch("app.services.scrape_runs.mark_done"))
- mock_imv_batch = stack.enter_context(
- patch(
- "app.services.house_imv_backfill.process_houses_imv_batch",
- new_callable=AsyncMock,
- )
- )
+ mock_runs = MagicMock()
+ mock_runs.is_cancelled.side_effect = is_cancelled_side_effect
+ mock_runs.update_heartbeat = MagicMock()
+ mock_runs.mark_done = MagicMock()
+ stack.enter_context(patch(f"{PFX}.runs", mock_runs))
+
await run_avito_city_sweep(
mock_db,
run_id=6,
+ config=_config(),
+ matcher=MagicMock(),
+ enrichment=enrichment,
anchors=[(56.84, 60.6, "TestAnchor")],
enrich_imv=True,
enrich_houses=True,
detail_top_n=0,
)
- mock_imv_batch.assert_not_awaited()
- mock_mark_done.assert_called_once()
+ enrichment.process_houses_imv_batch.assert_not_awaited()
+ mock_runs.mark_done.assert_called_once()
# ── process_houses_imv_batch с пустым set ────────────────────────────────────
diff --git a/tradein-mvp/backend/tests/test_yandex_city_sweep.py b/tradein-mvp/backend/tests/test_yandex_city_sweep.py
index 32b31898..df2fbccc 100644
--- a/tradein-mvp/backend/tests/test_yandex_city_sweep.py
+++ b/tradein-mvp/backend/tests/test_yandex_city_sweep.py
@@ -1,21 +1,19 @@
-"""Offline unit tests for run_yandex_city_sweep (SERP+address-enrich, #861).
+"""Offline unit tests for YandexCitySweepCounters + admin sweep endpoints (#861).
-Pure & fast: NO live network, NO DB. We monkeypatch:
- - YandexRealtyScraper.__init__ / __aenter__ / __aexit__ — avoid get_scraper_delay
- DB read + real curl_cffi session creation.
- - YandexRealtyScraper.fetch_around_multi_room — return a fixed list of fake ScrapedLot.
- - scrape_pipeline.save_listings — counter/no-op (no real listings table).
- - scrape_runs.* (is_cancelled / update_heartbeat / mark_*) — in-memory fakes.
- - curl_cffi.requests.AsyncSession — fake for address-enrich HTTP calls.
+Pure & fast: NO live network, NO DB.
-Asserts orchestration semantics:
- - SERP → save (all anchors, counters aggregated)
- - address-enrich фаза вызывается если enrich_address=True и есть source_ids
- - enrich_address=False → адрес-фаза пропускается
- - cancel → останавливает sweep
- - N consecutive SERP failures → mark_banned + abort
- - per-item address-enrich error не валит sweep
- - mark_done вызывается при нормальном завершении
+Legacy `run_yandex_city_sweep` orchestration behavior (SERP→save per-combo,
+address-enrich phase, cooperative cancel, consecutive-SERP-failure abort, bounded
+retry / combo-skip logging) удалён вместе с `app.services.scrape_pipeline` (#2397
+Part E1) — эквивалентная regression-coverage теперь в
+`test_scraper_kit_pipeline_parity2.py::test_yandex_city_sweep` (kit
+`run_yandex_city_sweep`).
+
+Coverage:
+ - YandexCitySweepCounters корректно заполняются (dataclass shape)
+ - Combos watchdog timeout существенно больше ANCHOR_TIMEOUT_SEC (arithmetic-only)
+ - Admin endpoints (start/cancel/list-runs) — kit sweep вызван с DI
+ (config/matcher/enrichment)
"""
from __future__ import annotations
@@ -24,131 +22,16 @@ import os
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
-from typing import Any
-from unittest.mock import AsyncMock, MagicMock
+from datetime import UTC
+from unittest.mock import AsyncMock
import pytest
-from app.services import scrape_pipeline
-from app.services.scrapers.base import ScrapedLot
-from app.services.scrapers.yandex_realty import YandexRealtyScraper
-
-# 3 anchors keep tests fast and let us probe cancel/fail semantics.
-TEST_ANCHORS: list[tuple[float, float, str]] = [
- (56.8400, 60.6050, "A1"),
- (56.7950, 60.5300, "A2"),
- (56.8970, 60.6100, "A3"),
-]
-
-
-def _fake_lot(offer_id: str) -> ScrapedLot:
- """Minimal valid ScrapedLot (source/source_url required, price_rub > 0)."""
- return ScrapedLot(
- source="yandex",
- source_url=f"https://realty.yandex.ru/offer/{offer_id}/",
- source_id=offer_id,
- address="Екатеринбург, улица Тестовая",
- price_rub=5_000_000,
- area_m2=50.0,
- rooms=2,
- )
-
-
-class _FakeRuns:
- """In-memory stand-in for app.services.scrape_runs (no DB)."""
-
- def __init__(self, *, cancel_at_anchor: int | None = None) -> None:
- # cancel_at_anchor: 1-based anchor index at which is_cancelled flips True.
- self.cancel_at_anchor = cancel_at_anchor
- self.is_cancelled_calls = 0
- self.heartbeats: list[dict[str, int]] = []
- self.done: dict[str, int] | None = None
- self.failed: tuple[str, dict[str, int]] | None = None
- self.banned: tuple[str, dict[str, int]] | None = None
-
- def is_cancelled(self, _db: Any, _run_id: int) -> bool:
- self.is_cancelled_calls += 1
- if self.cancel_at_anchor is None:
- return False
- return self.is_cancelled_calls >= self.cancel_at_anchor
-
- def update_heartbeat(self, _db: Any, _run_id: int, counters: dict[str, int]) -> None:
- self.heartbeats.append(dict(counters))
-
- def mark_done(self, _db: Any, _run_id: int, counters: dict[str, int]) -> None:
- self.done = dict(counters)
-
- def mark_failed(self, _db: Any, _run_id: int, error: str, counters: dict[str, int]) -> None:
- self.failed = (error, dict(counters))
-
- def mark_banned(self, _db: Any, _run_id: int, error: str, counters: dict[str, int]) -> None:
- self.banned = (error, dict(counters))
-
-
-@pytest.fixture(autouse=True)
-def _no_sleep(monkeypatch: pytest.MonkeyPatch) -> None:
- """Kill inter-anchor asyncio.sleep so tests are instant."""
-
- async def _instant(_secs: float) -> None:
- return None
-
- monkeypatch.setattr(scrape_pipeline.asyncio, "sleep", _instant)
-
-
-@pytest.fixture(autouse=True)
-def _stub_scraper_lifecycle(monkeypatch: pytest.MonkeyPatch) -> None:
- """Neutralize __init__/__aenter__/__aexit__ — no get_scraper_delay DB, no curl_cffi."""
-
- def _init(self: YandexRealtyScraper, city: str = "ekaterinburg") -> None:
- self.city = city
- self.request_delay_sec = 0.0
- self._cffi_session = None
- self._cookies = {}
-
- async def _aenter(self: YandexRealtyScraper) -> YandexRealtyScraper:
- return self
-
- async def _aexit(self: YandexRealtyScraper, *_args: Any) -> None:
- return None
-
- monkeypatch.setattr(YandexRealtyScraper, "__init__", _init)
- monkeypatch.setattr(YandexRealtyScraper, "__aenter__", _aenter)
- monkeypatch.setattr(YandexRealtyScraper, "__aexit__", _aexit)
-
-
-def _install_runs(monkeypatch: pytest.MonkeyPatch, fake: _FakeRuns) -> None:
- monkeypatch.setattr(scrape_pipeline.scrape_runs, "is_cancelled", fake.is_cancelled)
- monkeypatch.setattr(scrape_pipeline.scrape_runs, "update_heartbeat", fake.update_heartbeat)
- monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_done", fake.mark_done)
- monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_failed", fake.mark_failed)
- monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_banned", fake.mark_banned)
-
-
-def _make_db_mock(source_ids: list[str]) -> MagicMock:
- """DB mock для address-enrich фазы: возвращает строки без номера дома."""
- mock_db = MagicMock()
- rows = [
- {
- "id": i + 1,
- "source_id": sid,
- "address": "Екатеринбург, улица Тестовая",
- "source_url": f"https://realty.yandex.ru/offer/{sid}/",
- }
- for i, sid in enumerate(source_ids)
- ]
- mock_db.execute.return_value.mappings.return_value.all.return_value = rows
- return mock_db
-
-
-def _fake_save(_db: Any, lots: list, *, run_id: int | None = None) -> tuple[int, int]:
- return len(lots), 0
-
-
# ── YandexCitySweepCounters ─────────────────────────────────────────────────
def test_counters_defaults() -> None:
- from app.services.scrape_pipeline import YandexCitySweepCounters
+ from scraper_kit.orchestration.pipeline import YandexCitySweepCounters
c = YandexCitySweepCounters()
assert c.anchors_total == 0
@@ -163,7 +46,7 @@ def test_counters_defaults() -> None:
def test_counters_to_dict_all_keys() -> None:
from dataclasses import fields
- from app.services.scrape_pipeline import YandexCitySweepCounters
+ from scraper_kit.orchestration.pipeline import YandexCitySweepCounters
c = YandexCitySweepCounters()
d = c.to_dict()
@@ -171,423 +54,13 @@ def test_counters_to_dict_all_keys() -> None:
assert set(d.keys()) == expected
-# ── Happy path: all anchors iterated, counters aggregated ───────────────────
-
-
-async def test_sweep_iterates_all_anchors_and_aggregates(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- """fetch_around_multi_room вызывается для каждого anchor, counters агрегируются.
-
- FIX: fake_fetch_multi вызывает on_combo (инкрементальный save per-combo) —
- имитирует реальное поведение scraper'а. save_listings должен вызываться
- on_combo'м, а не одним батчем в конце anchor'а.
- """
- serp_calls: list[str] = []
- save_calls: list[int] = []
-
- async def fake_fetch_multi(
- self: YandexRealtyScraper,
- lat: float,
- lon: float,
- radius_m: int = 1000,
- max_pages: int = 2,
- on_combo: Any = None,
- **_: Any,
- ) -> list[ScrapedLot]:
- serp_calls.append(f"{lat:.4f}")
- lots = [_fake_lot(f"{lat}-{i}") for i in range(3)]
- # Имитируем один combo — вызываем on_combo если предоставлен.
- if on_combo is not None:
- on_combo(f"vtorichka/combo_{lat:.4f}", lots)
- return lots
-
- def fake_save(
- _db: Any, lots: list[ScrapedLot], *, run_id: int | None = None
- ) -> tuple[int, int]:
- save_calls.append(len(lots))
- return len(lots), 0
-
- monkeypatch.setattr(YandexRealtyScraper, "fetch_around_multi_room", fake_fetch_multi)
- monkeypatch.setattr(scrape_pipeline, "save_listings", fake_save)
- fake = _FakeRuns()
- _install_runs(monkeypatch, fake)
-
- counters = await scrape_pipeline.run_yandex_city_sweep(
- db=MagicMock(),
- run_id=1,
- anchors=TEST_ANCHORS,
- pages_per_anchor=2,
- request_delay_sec=0.0,
- enrich_address=False,
- )
-
- assert len(serp_calls) == len(TEST_ANCHORS)
- # save вызывается инкрементально через on_combo — по одному разу на anchor
- # (fake имитирует один combo per anchor).
- assert len(save_calls) == len(TEST_ANCHORS)
- assert counters.anchors_total == len(TEST_ANCHORS)
- assert counters.anchors_done == len(TEST_ANCHORS)
- assert counters.lots_fetched == len(TEST_ANCHORS) * 3
- assert counters.lots_inserted == len(TEST_ANCHORS) * 3
- assert counters.lots_updated == 0
- assert counters.errors_count == 0
- assert counters.address_attempted == 0 # enrich_address=False
- assert fake.done is not None
- assert fake.done["lots_fetched"] == len(TEST_ANCHORS) * 3
- assert fake.failed is None
- assert fake.banned is None
-
-
-# ── enrich_address=False пропускает address-фазу ────────────────────────────
-
-
-async def test_enrich_address_false_skips_address_phase(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- """enrich_address=False → address-enrich фаза не выполняется."""
- address_http_calls: list[str] = []
-
- async def fake_fetch_multi(
- self: YandexRealtyScraper,
- lat: float,
- lon: float,
- radius_m: int = 1000,
- max_pages: int = 2,
- **_: Any,
- ) -> list[ScrapedLot]:
- return [_fake_lot(f"{lat}-0")]
-
- async def fake_cffi_get(url: str, **_: Any) -> Any:
- address_http_calls.append(url)
- return MagicMock(status_code=200, text="<title>test")
-
- monkeypatch.setattr(YandexRealtyScraper, "fetch_around_multi_room", fake_fetch_multi)
- monkeypatch.setattr(scrape_pipeline, "save_listings", _fake_save)
- fake = _FakeRuns()
- _install_runs(monkeypatch, fake)
-
- counters = await scrape_pipeline.run_yandex_city_sweep(
- db=MagicMock(),
- run_id=2,
- anchors=TEST_ANCHORS[:1],
- enrich_address=False,
- request_delay_sec=0.0,
- )
-
- assert len(address_http_calls) == 0
- assert counters.address_attempted == 0
- assert counters.address_enriched == 0
- assert fake.done is not None
-
-
-# ── address-enrich фаза: счётчики заполняются ───────────────────────────────
-
-
-async def test_address_enrich_fills_counters(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- """address-enrich вызывается для листингов без номера дома; counters заполняются."""
- # Fake SERP возвращает 2 листинга с source_id
- lots = [_fake_lot("id1"), _fake_lot("id2")]
-
- async def fake_fetch_multi(
- self: YandexRealtyScraper,
- lat: float,
- lon: float,
- radius_m: int = 1000,
- max_pages: int = 2,
- on_combo: Any = None,
- **_: Any,
- ) -> list[ScrapedLot]:
- # Вызываем on_combo чтобы anchor_lots накопился (инкрементальный save).
- if on_combo is not None:
- on_combo("vtorichka/combo_enrich_test", lots)
- return lots
-
- monkeypatch.setattr(YandexRealtyScraper, "fetch_around_multi_room", fake_fetch_multi)
- monkeypatch.setattr(scrape_pipeline, "save_listings", _fake_save)
-
- # DB mock: возвращает 2 строки для address-enrich запроса
- mock_db = _make_db_mock(["id1", "id2"])
-
- # Title с полным адресом — должен быть извлечён через _extract_address_from_title
- fake_response = MagicMock()
- fake_response.status_code = 200
- fake_response.text = (
- "Продажа квартиры — Екатеринбург, улица Тестовая, 36 — id 123 "
- "на Яндекс.Недвижимости"
- )
-
- # Мок curl_cffi AsyncSession как context manager
- mock_session = AsyncMock()
- mock_session.__aenter__ = AsyncMock(return_value=mock_session)
- mock_session.__aexit__ = AsyncMock(return_value=None)
- mock_session.get = AsyncMock(return_value=fake_response)
-
- import curl_cffi.requests as cffi_mod
-
- monkeypatch.setattr(cffi_mod, "AsyncSession", lambda **_kw: mock_session)
-
- fake = _FakeRuns()
- _install_runs(monkeypatch, fake)
-
- counters = await scrape_pipeline.run_yandex_city_sweep(
- db=mock_db,
- run_id=3,
- anchors=TEST_ANCHORS[:1],
- enrich_address=True,
- request_delay_sec=0.0,
- )
-
- assert counters.address_attempted == 2
- assert counters.address_enriched == 2
- assert counters.address_failed == 0
- assert fake.done is not None
-
-
-# ── address-enrich: per-item HTTP error не валит sweep ──────────────────────
-
-
-async def test_address_enrich_http_error_does_not_abort(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- """HTTP 403 при address-enrich → address_failed++, sweep продолжается."""
-
- async def fake_fetch_multi(
- self: YandexRealtyScraper,
- lat: float,
- lon: float,
- radius_m: int = 1000,
- max_pages: int = 2,
- on_combo: Any = None,
- **_: Any,
- ) -> list[ScrapedLot]:
- lots = [_fake_lot("id-err")]
- if on_combo is not None:
- on_combo("vtorichka/combo_http_err", lots)
- return lots
-
- monkeypatch.setattr(YandexRealtyScraper, "fetch_around_multi_room", fake_fetch_multi)
- monkeypatch.setattr(scrape_pipeline, "save_listings", _fake_save)
-
- mock_db = _make_db_mock(["id-err"])
-
- fake_response = MagicMock()
- fake_response.status_code = 403
- fake_response.text = ""
-
- mock_session = AsyncMock()
- mock_session.__aenter__ = AsyncMock(return_value=mock_session)
- mock_session.__aexit__ = AsyncMock(return_value=None)
- mock_session.get = AsyncMock(return_value=fake_response)
-
- import curl_cffi.requests as cffi_mod
-
- monkeypatch.setattr(cffi_mod, "AsyncSession", lambda **_kw: mock_session)
-
- fake = _FakeRuns()
- _install_runs(monkeypatch, fake)
-
- counters = await scrape_pipeline.run_yandex_city_sweep(
- db=mock_db,
- run_id=4,
- anchors=TEST_ANCHORS[:1],
- enrich_address=True,
- request_delay_sec=0.0,
- )
-
- assert counters.address_attempted == 1
- assert counters.address_failed == 1
- assert counters.address_enriched == 0
- assert fake.done is not None # sweep завершился несмотря на ошибку
-
-
-# ── Mid-sweep cancel stops further anchors ──────────────────────────────────
-
-
-async def test_cancel_midway_stops_remaining_anchors(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- serp_calls: list[str] = []
-
- async def fake_fetch_multi(
- self: YandexRealtyScraper,
- lat: float,
- lon: float,
- radius_m: int = 1000,
- max_pages: int = 2,
- **_: Any,
- ) -> list[ScrapedLot]:
- serp_calls.append(f"{lat}")
- return [_fake_lot(f"{lat}-0")]
-
- monkeypatch.setattr(YandexRealtyScraper, "fetch_around_multi_room", fake_fetch_multi)
- monkeypatch.setattr(scrape_pipeline, "save_listings", _fake_save)
- # is_cancelled flips True on the 2nd check (before anchor #2) → only anchor #1 runs.
- fake = _FakeRuns(cancel_at_anchor=2)
- _install_runs(monkeypatch, fake)
-
- counters = await scrape_pipeline.run_yandex_city_sweep(
- db=MagicMock(),
- run_id=5,
- anchors=TEST_ANCHORS,
- pages_per_anchor=1,
- enrich_address=False,
- request_delay_sec=0.0,
- )
- # Only the first anchor fetched before cancel.
- assert len(serp_calls) == 1
- assert counters.anchors_done == 1
- # Cancel path returns early: no mark_done, no ban/fail.
- assert fake.done is None
- assert fake.failed is None
- assert fake.banned is None
-
-
-# ── Consecutive failures → mark_banned + abort ──────────────────────────────
-
-
-async def test_consecutive_failures_abort_sweep(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- """All anchors raise → after MAX_CONSECUTIVE_FAILURES the sweep aborts + bans."""
- attempts: list[str] = []
-
- async def fake_fetch_fail(
- self: YandexRealtyScraper,
- lat: float,
- lon: float,
- radius_m: int = 1000,
- max_pages: int = 2,
- **_: Any,
- ) -> list[ScrapedLot]:
- attempts.append(f"{lat}")
- raise RuntimeError("yandex blocked")
-
- monkeypatch.setattr(YandexRealtyScraper, "fetch_around_multi_room", fake_fetch_fail)
- monkeypatch.setattr(scrape_pipeline, "save_listings", _fake_save)
- fake = _FakeRuns()
- _install_runs(monkeypatch, fake)
-
- # 5 anchors so the 3-consecutive-failure threshold trips before the list ends.
- anchors = [
- *TEST_ANCHORS,
- (56.7700, 60.5500, "A4"),
- (56.8650, 60.6200, "A5"),
- ]
- counters = await scrape_pipeline.run_yandex_city_sweep(
- db=MagicMock(),
- run_id=6,
- anchors=anchors,
- enrich_address=False,
- request_delay_sec=0.0,
- )
- # Abort exactly at the threshold (3 consecutive failures).
- assert len(attempts) == scrape_pipeline.YANDEX_SWEEP_MAX_CONSECUTIVE_FAILURES
- assert counters.errors_count == scrape_pipeline.YANDEX_SWEEP_MAX_CONSECUTIVE_FAILURES
- assert counters.anchors_done == scrape_pipeline.YANDEX_SWEEP_MAX_CONSECUTIVE_FAILURES
- assert fake.banned is not None
- assert "consecutive" in fake.banned[0]
- assert fake.done is None # aborted, never marked done
-
-
-# ── Non-consecutive failures do not abort ───────────────────────────────────
-
-
-async def test_non_consecutive_failures_do_not_abort(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- """A success between two failures resets the consecutive counter → no abort."""
-
- def _name_of(lat: float, anchors: list[tuple[float, float, str]]) -> str:
- for a_lat, _lon, nm in anchors:
- if a_lat == lat:
- return nm
- return "?"
-
- anchors = [
- *TEST_ANCHORS,
- (56.7700, 60.5500, "A4"),
- (56.8650, 60.6200, "A5"),
- ]
- failing = {"A1", "A3", "A5"}
-
- async def fake_fetch(
- self: YandexRealtyScraper,
- lat: float,
- lon: float,
- radius_m: int = 1000,
- max_pages: int = 2,
- on_combo: Any = None,
- **_: Any,
- ) -> list[ScrapedLot]:
- if _name_of(lat, anchors) in failing:
- raise RuntimeError("transient")
- lots = [_fake_lot(f"{lat}-0")]
- # Вызываем on_combo для инкрементального save.
- if on_combo is not None:
- on_combo(f"vtorichka/combo_{lat:.4f}", lots)
- return lots
-
- monkeypatch.setattr(YandexRealtyScraper, "fetch_around_multi_room", fake_fetch)
- monkeypatch.setattr(scrape_pipeline, "save_listings", _fake_save)
- fake = _FakeRuns()
- _install_runs(monkeypatch, fake)
-
- counters = await scrape_pipeline.run_yandex_city_sweep(
- db=MagicMock(),
- run_id=7,
- anchors=anchors,
- enrich_address=False,
- request_delay_sec=0.0,
- )
- assert counters.errors_count == 3 # A1, A3, A5
- assert counters.anchors_done == 5 # walked the whole list, no early abort
- assert counters.lots_fetched == 2 # A2 + A4 contributed
- assert fake.banned is None
- assert fake.done is not None
-
-
-# ── mark_done вызывается даже при пустом SERP ───────────────────────────────
-
-
-async def test_mark_done_always_called(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- async def fake_fetch_empty(
- self: YandexRealtyScraper,
- lat: float,
- lon: float,
- radius_m: int = 1000,
- max_pages: int = 2,
- **_: Any,
- ) -> list[ScrapedLot]:
- return []
-
- monkeypatch.setattr(YandexRealtyScraper, "fetch_around_multi_room", fake_fetch_empty)
- monkeypatch.setattr(scrape_pipeline, "save_listings", _fake_save)
- fake = _FakeRuns()
- _install_runs(monkeypatch, fake)
-
- counters = await scrape_pipeline.run_yandex_city_sweep(
- db=MagicMock(),
- run_id=8,
- anchors=TEST_ANCHORS,
- enrich_address=False,
- request_delay_sec=0.0,
- )
-
- assert fake.done is not None
- assert counters.lots_fetched == 0
- assert counters.anchors_done == len(TEST_ANCHORS)
-
-
# ── Admin endpoint tests (offline) ──────────────────────────────────────────
@pytest.fixture
def app_with_admin():
+ from unittest.mock import MagicMock
+
from fastapi import FastAPI
from fastapi.testclient import TestClient
@@ -680,7 +153,7 @@ def test_yandex_cancel_endpoint_not_running(app_with_admin) -> None:
def test_yandex_list_runs_endpoint(app_with_admin) -> None:
"""List runs endpoint возвращает список."""
- from datetime import UTC, datetime
+ from datetime import datetime
from unittest.mock import patch
fake_rows = [
@@ -706,144 +179,7 @@ def test_yandex_list_runs_endpoint(app_with_admin) -> None:
assert "2025-05-01" in body[0]["started_at"]
-# ── Center-combos wiring: prod (anchors=None) uses EKB center + combos ──────
-
-
-async def test_prod_mode_uses_center_anchor_with_combos(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- """anchors=None (prod mode) → fetch_around_multi_room called with EKB center coords
- AND rooms_list + price_ranges (combos mode), NOT with None/None (legacy mode).
- anchors_total == 1 (single center sweep).
- """
- from app.services.scrapers.yandex_realty import DEFAULT_PRICE_RANGES, ROOM_PATH
-
- calls: list[dict] = []
-
- async def fake_fetch_multi(
- self: YandexRealtyScraper,
- lat: float,
- lon: float,
- radius_m: int = 1000,
- max_pages: int = 2,
- rooms_list: list | None = None,
- price_ranges: list | None = None,
- on_combo: Any = None,
- **_: Any,
- ) -> list[ScrapedLot]:
- lots = [_fake_lot("center-1"), _fake_lot("center-2")]
- calls.append(
- {
- "lat": lat,
- "lon": lon,
- "radius_m": radius_m,
- "rooms_list": rooms_list,
- "price_ranges": price_ranges,
- "max_pages": max_pages,
- }
- )
- # Имитируем on_combo (инкрементальный save) — один combo.
- if on_combo is not None:
- on_combo("vtorichka/center_combo", lots)
- return lots
-
- monkeypatch.setattr(YandexRealtyScraper, "fetch_around_multi_room", fake_fetch_multi)
- monkeypatch.setattr(scrape_pipeline, "save_listings", _fake_save)
- fake = _FakeRuns()
- _install_runs(monkeypatch, fake)
-
- counters = await scrape_pipeline.run_yandex_city_sweep(
- db=MagicMock(),
- run_id=10,
- # anchors=None → prod mode (EKB center)
- pages_per_anchor=3,
- enrich_address=False,
- request_delay_sec=0.0,
- )
-
- # Single center anchor
- assert counters.anchors_total == 1
- assert counters.anchors_done == 1
-
- # fetch_around_multi_room called exactly once
- assert len(calls) == 1
- call = calls[0]
-
- # EKB center coords
- assert abs(call["lat"] - 56.8400) < 0.0001
- assert abs(call["lon"] - 60.6050) < 0.0001
-
- # radius covers the whole city (25km default)
- assert call["radius_m"] >= 10000
-
- # Combos mode: rooms_list and price_ranges are NOT None
- assert call["rooms_list"] is not None
- assert call["price_ranges"] is not None
-
- # rooms_list covers all ROOM_PATH keys
- assert set(call["rooms_list"]) == set(ROOM_PATH.keys())
-
- # price_ranges matches DEFAULT_PRICE_RANGES
- assert call["price_ranges"] == DEFAULT_PRICE_RANGES
-
- # max_pages forwarded from pages_per_anchor
- assert call["max_pages"] == 3
-
- # lots aggregated correctly via on_combo
- assert counters.lots_fetched == 2
- assert counters.lots_inserted == 2
- assert fake.done is not None
- assert fake.failed is None
-
-
-async def test_explicit_anchors_still_works(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- """anchors=[...] (test/manual override) → fetch called for each anchor, combos mode."""
- calls: list[dict] = []
-
- async def fake_fetch_multi(
- self: YandexRealtyScraper,
- lat: float,
- lon: float,
- radius_m: int = 1000,
- max_pages: int = 2,
- rooms_list: list | None = None,
- price_ranges: list | None = None,
- **_: Any,
- ) -> list[ScrapedLot]:
- calls.append({"lat": lat, "rooms_list": rooms_list, "price_ranges": price_ranges})
- return [_fake_lot(f"{lat}")]
-
- monkeypatch.setattr(YandexRealtyScraper, "fetch_around_multi_room", fake_fetch_multi)
- monkeypatch.setattr(scrape_pipeline, "save_listings", _fake_save)
- fake = _FakeRuns()
- _install_runs(monkeypatch, fake)
-
- two_anchors = [(56.8400, 60.6050, "A"), (56.7950, 60.5300, "B")]
- counters = await scrape_pipeline.run_yandex_city_sweep(
- db=MagicMock(),
- run_id=11,
- anchors=two_anchors,
- pages_per_anchor=2,
- enrich_address=False,
- request_delay_sec=0.0,
- )
-
- # Both anchors iterated
- assert len(calls) == 2
- assert counters.anchors_total == 2
- assert counters.anchors_done == 2
-
- # Each call uses combos (not None/None)
- for c in calls:
- assert c["rooms_list"] is not None
- assert c["price_ranges"] is not None
-
- assert fake.done is not None
-
-
-# ── Watchdog timeout: combos vs explicit-anchor mode ───────────────────────
+# ── Watchdog timeout: combos-mode budget dwarfs ANCHOR_TIMEOUT_SEC ──────────
def test_combos_sweep_timeout_substantially_larger_than_anchor_timeout() -> None:
@@ -851,13 +187,14 @@ def test_combos_sweep_timeout_substantially_larger_than_anchor_timeout() -> None
Проверяет что вычисленный таймаут для дефолтных 30 combos × 3 pages > 1500s,
т.е. не равен ANCHOR_TIMEOUT_SEC=240 и обеспечивает полный прогон sweep'а.
- Не нужны мокапы — только публичные константы и арифметика из модуля.
+ Не нужны мокапы — только публичные константы и арифметика из kit-модуля.
"""
- from app.services.scrape_pipeline import (
+ from scraper_kit.orchestration.pipeline import (
_YANDEX_ADDRESS_ENRICH_BUDGET_S,
_YANDEX_COMBOS_PER_FETCH_S,
ANCHOR_TIMEOUT_SEC,
)
+
from app.services.scrapers.yandex_realty import DEFAULT_PRICE_RANGES, ROOM_PATH
# Параметры prod-режима
@@ -884,188 +221,3 @@ def test_combos_sweep_timeout_substantially_larger_than_anchor_timeout() -> None
assert (
sweep_timeout > ANCHOR_TIMEOUT_SEC * 4
), f"sweep_timeout={sweep_timeout:.0f}s should be >> ANCHOR_TIMEOUT_SEC={ANCHOR_TIMEOUT_SEC}s"
-
-
-async def test_explicit_anchor_mode_uses_anchor_timeout_sec(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- """anchors=[...] (explicit override) → wait_for используется с ANCHOR_TIMEOUT_SEC.
-
- Проверяет что в legacy/test режиме (explicit anchors) sweep_timeout не раздут
- до combos-значения — он равен ANCHOR_TIMEOUT_SEC, т.к. каждый anchor маленький.
- """
- import asyncio as _asyncio
-
- from app.services.scrape_pipeline import ANCHOR_TIMEOUT_SEC
-
- captured_timeouts: list[float] = []
-
- _orig_wait_for = _asyncio.wait_for
-
- async def _capturing_wait_for(coro, *, timeout): # type: ignore[no-untyped-def]
- captured_timeouts.append(timeout)
- # На самом деле запустить — мокнутый scraper вернёт [] мгновенно.
- return await _orig_wait_for(coro, timeout=timeout)
-
- monkeypatch.setattr(_asyncio, "wait_for", _capturing_wait_for)
-
- async def fake_fetch_multi(
- self: YandexRealtyScraper,
- lat: float,
- lon: float,
- radius_m: int = 1000,
- max_pages: int = 2,
- **_: object,
- ) -> list[ScrapedLot]:
- return []
-
- monkeypatch.setattr(YandexRealtyScraper, "fetch_around_multi_room", fake_fetch_multi)
- monkeypatch.setattr(scrape_pipeline, "save_listings", _fake_save)
- fake = _FakeRuns()
- _install_runs(monkeypatch, fake)
-
- await scrape_pipeline.run_yandex_city_sweep(
- db=MagicMock(),
- run_id=20,
- anchors=TEST_ANCHORS[:2], # explicit anchors — legacy mode
- pages_per_anchor=2,
- enrich_address=False,
- request_delay_sec=0.0,
- )
-
- # В explicit-anchor режиме каждый anchor должен использовать ANCHOR_TIMEOUT_SEC
- assert len(captured_timeouts) == 2
- for t in captured_timeouts:
- assert (
- t == ANCHOR_TIMEOUT_SEC
- ), f"Expected ANCHOR_TIMEOUT_SEC={ANCHOR_TIMEOUT_SEC}s, got {t}s"
-
-
-# ── FIX A: инкрементальный save per-combo ────────────────────────────────────
-
-
-async def test_incremental_save_called_per_combo(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- """save_listings вызывается on_combo per-combo, не одним батчем в конце anchor'а.
-
- Имитируем 3 combo через on_combo — save_listings должен быть вызван 3 раза
- (по одному на combo) с порциями лотов. Проверяем, что при краше anchor'а
- уже сохранённые combo не теряются (данные в БД до finish anchor'а).
- """
- save_batches: list[list[str]] = []
-
- async def fake_fetch_multi(
- self: YandexRealtyScraper,
- lat: float,
- lon: float,
- radius_m: int = 1000,
- max_pages: int = 2,
- on_combo: Any = None,
- **_: Any,
- ) -> list[ScrapedLot]:
- all_lots = []
- for combo_idx in range(3):
- combo_lots = [_fake_lot(f"{lat}-combo{combo_idx}-lot0")]
- all_lots.extend(combo_lots)
- if on_combo is not None:
- on_combo(f"vtorichka/combo_{combo_idx}", combo_lots)
- return all_lots
-
- def fake_save(
- _db: Any, lots: list[ScrapedLot], *, run_id: int | None = None
- ) -> tuple[int, int]:
- save_batches.append([lot.source_id for lot in lots if lot.source_id])
- return len(lots), 0
-
- monkeypatch.setattr(YandexRealtyScraper, "fetch_around_multi_room", fake_fetch_multi)
- monkeypatch.setattr(scrape_pipeline, "save_listings", fake_save)
- fake = _FakeRuns()
- _install_runs(monkeypatch, fake)
-
- counters = await scrape_pipeline.run_yandex_city_sweep(
- db=MagicMock(),
- run_id=30,
- anchors=TEST_ANCHORS[:1],
- pages_per_anchor=2,
- request_delay_sec=0.0,
- enrich_address=False,
- )
-
- # save_listings вызван ровно 3 раза (по одному на combo), не одним батчем.
- assert (
- len(save_batches) == 3
- ), f"Ожидалось 3 save вызова (per-combo), получено {len(save_batches)}: {save_batches}"
- # Каждый batch содержит ровно 1 лот (размер combo).
- for batch in save_batches:
- assert len(batch) == 1, f"Каждый combo-batch должен иметь 1 лот, получено {batch}"
-
- # Суммарные counters корректны.
- assert counters.lots_fetched == 3
- assert counters.lots_inserted == 3
- assert fake.done is not None
-
- # heartbeat вызывался после каждого combo (через on_combo).
- assert (
- len(fake.heartbeats) >= 3
- ), f"Ожидалось ≥3 heartbeat обновлений (per-combo), получено {len(fake.heartbeats)}"
-
-
-# ── FIX B: bounded retry — скипает combo после exhaustion ────────────────────
-
-
-async def test_bounded_retry_combo_skipped_logged(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- """on_combo НЕ вызывается если combo пропущен (все retry page-1 исчерпаны).
-
- Имитируем sweep с двумя combos: первый скипается (on_combo не вызван),
- второй успешен (on_combo вызван). Проверяем, что save_listings вызван
- только для успешного combo, а не скипнутого.
- """
- save_batches: list[list[str]] = []
- on_combo_calls: list[str] = []
-
- async def fake_fetch_multi(
- self: YandexRealtyScraper,
- lat: float,
- lon: float,
- radius_m: int = 1000,
- max_pages: int = 2,
- on_combo: Any = None,
- **_: Any,
- ) -> list[ScrapedLot]:
- # Combo 0: скипается — on_combo НЕ вызывается (симулирует exhausted retries).
- # Combo 1: успешен — on_combo вызывается с лотом.
- success_lot = _fake_lot(f"{lat}-success")
- if on_combo is not None:
- on_combo_calls.append("combo_1_success")
- on_combo("vtorichka/combo_1", [success_lot])
- return [success_lot]
-
- def fake_save(
- _db: Any, lots: list[ScrapedLot], *, run_id: int | None = None
- ) -> tuple[int, int]:
- save_batches.append([lot.source_id for lot in lots if lot.source_id])
- return len(lots), 0
-
- monkeypatch.setattr(YandexRealtyScraper, "fetch_around_multi_room", fake_fetch_multi)
- monkeypatch.setattr(scrape_pipeline, "save_listings", fake_save)
- fake = _FakeRuns()
- _install_runs(monkeypatch, fake)
-
- counters = await scrape_pipeline.run_yandex_city_sweep(
- db=MagicMock(),
- run_id=31,
- anchors=TEST_ANCHORS[:1],
- pages_per_anchor=2,
- request_delay_sec=0.0,
- enrich_address=False,
- )
-
- # Только успешный combo вызвал on_combo и save.
- assert len(on_combo_calls) == 1
- assert len(save_batches) == 1
- assert counters.lots_fetched == 1
- assert counters.lots_inserted == 1
- assert fake.done is not None
diff --git a/tradein-mvp/packages/scraper-kit/src/scraper_kit/orchestration/__init__.py b/tradein-mvp/packages/scraper-kit/src/scraper_kit/orchestration/__init__.py
index ab31f59d..bb69dcd4 100644
--- a/tradein-mvp/packages/scraper-kit/src/scraper_kit/orchestration/__init__.py
+++ b/tradein-mvp/packages/scraper-kit/src/scraper_kit/orchestration/__init__.py
@@ -1,4 +1,4 @@
-"""Оркестрация sweep-pipeline scraper_kit — strangler-копия `app.services.scrape_pipeline`.
+"""Оркестрация sweep-pipeline scraper_kit (#2135 strangler, завершён #2397 Part E1).
Координирует полный прогон скрапа (SERP → save → houses → detail → IMV) поверх
провайдеров `scraper_kit.providers.*` и базового слоя `scraper_kit.base`. Развязана
@@ -10,6 +10,7 @@
- `app.core.shutdown.shutdown_requested` → инжектируемый `Callable[[], bool]`
- `app.services.scrape_runs` → локальная копия `scraper_kit.orchestration.runs`
-Боевой scheduler по-прежнему зовёт старый `app.services.scrape_pipeline` (strangler:
-COPY, не MOVE). Переключение рантайма на этот модуль — отдельный поздний шаг.
+Единственный оркестратор в проде: legacy `app.services.scrape_pipeline` (изначальный
+strangler-COPY-источник) удалён (#2397 Part E1). Боевой scheduler и admin-роуты
+вызывают этот модуль напрямую.
"""
diff --git a/tradein-mvp/packages/scraper-kit/src/scraper_kit/orchestration/pipeline.py b/tradein-mvp/packages/scraper-kit/src/scraper_kit/orchestration/pipeline.py
index cdc90b28..0f3d39e7 100644
--- a/tradein-mvp/packages/scraper-kit/src/scraper_kit/orchestration/pipeline.py
+++ b/tradein-mvp/packages/scraper-kit/src/scraper_kit/orchestration/pipeline.py
@@ -1,8 +1,9 @@
-"""pipeline.py — Avito sweep orchestrator (strangler-копия #2135).
+"""pipeline.py — Avito sweep orchestrator (#2135 strangler, единственный в проде с #2397 Part E1).
-Strangler-COPY боевого `app.services.scrape_pipeline` (шаг оркестрации, #2135).
-Логика фаз/бан-ротации/счётчиков идентична оригиналу — единственное отличие в
-развязке от `app.*` через инжекцию (см. `scraper_kit.contracts`):
+Изначально был strangler-COPY боевого `app.services.scrape_pipeline` (#2135);
+legacy-оригинал удалён (#2397 Part E1) — этот модуль теперь единственная
+orchestration-реализация. Развязка от `app.*` через инжекцию (см.
+`scraper_kit.contracts`):
- `app.core.config.settings` → аргумент `config: ScraperConfig`
- `app.services.matching` → аргумент `matcher: HouseMatcher`
diff --git a/tradein-mvp/packages/scraper-kit/src/scraper_kit/providers/avito/detail.py b/tradein-mvp/packages/scraper-kit/src/scraper_kit/providers/avito/detail.py
index 9147a730..9a901319 100644
--- a/tradein-mvp/packages/scraper-kit/src/scraper_kit/providers/avito/detail.py
+++ b/tradein-mvp/packages/scraper-kit/src/scraper_kit/providers/avito/detail.py
@@ -385,7 +385,7 @@ async def fetch_detail(
reconnect_on_block — если False, 403/firewall поднимает AvitoBlockedError СРАЗУ
(без холодных reconnect'ов — ими займётся outer-loop через rebuild+rewarm прогретой
сессии); 429 после short-retry поднимает AvitoRateLimitedError. Дефолт True
- сохраняет старое поведение own-session / scrape_pipeline путей.
+ сохраняет старое поведение own-session / shared-session (run_avito_city_sweep) путей.
Raises:
httpx.HTTPError — если status != 200 (через raise_for_status-like).
ValueError — если item_id не извлечён из HTML.
@@ -427,8 +427,8 @@ async def fetch_detail(
own_session = cffi_session is None
if cffi_session is None:
# Mobile proxy wiring (#806 follow-up): own-session path (no shared session passed,
- # e.g. admin endpoint). Caller-provided sessions (scrape_pipeline) already have proxy
- # from AvitoScraper.__aenter__ — don't override. proxy=None → прямое подключение.
+ # e.g. admin endpoint). Caller-provided sessions (run_avito_city_sweep) already have
+ # proxy from AvitoScraper.__aenter__ — don't override. proxy=None → прямое подключение.
session = _build_detail_session(config)
else:
session = cffi_session
@@ -442,7 +442,7 @@ async def fetch_detail(
# reconnect_on_block=False (warm-batch #1551): не делаем холодных per-card
# reconnect'ов — блок поднимает AvitoBlockedError сразу, outer-loop пересоздаёт+
# прогревает сессию (свежий exit-IP). reconnect_on_block=True (own-session /
- # scrape_pipeline) — старое поведение: backconnect-reconnect эфемерной сессией.
+ # run_avito_city_sweep) — старое поведение: backconnect-reconnect эфемерной сессией.
backconnect = (
bool(config.scraper_proxy_url if config is not None else None) and reconnect_on_block
)