diff --git a/tradein-mvp/backend/app/api/v1/admin.py b/tradein-mvp/backend/app/api/v1/admin.py index 595b68b0..3a3f4f88 100644 --- a/tradein-mvp/backend/app/api/v1/admin.py +++ b/tradein-mvp/backend/app/api/v1/admin.py @@ -27,7 +27,6 @@ from app.services.scrape_pipeline import ( run_avito_city_sweep, run_cian_city_sweep, run_cian_full_load, - run_n1_city_sweep, run_yandex_city_sweep, run_yandex_full_load, ) @@ -44,7 +43,6 @@ from app.services.scrapers.avito_imv import ( from app.services.scrapers.base import save_listings from app.services.scrapers.browser_fetcher import BrowserFetcher from app.services.scrapers.cian import CianScraper -from app.services.scrapers.n1 import N1Scraper from app.services.scrapers.yandex_detail import YandexDetailScraper from app.services.scrapers.yandex_newbuilding import YandexNewbuildingScraper from app.services.scrapers.yandex_realty import YandexRealtyScraper @@ -67,14 +65,14 @@ def _assert_allowed_url(url: str) -> None: raise HTTPException(status_code=400, detail="host not allowed") -_ALL_SOURCES = ["avito", "cian", "yandex", "n1"] +_ALL_SOURCES = ["avito", "cian", "yandex"] class ScrapeRequest(BaseModel): lat: float lon: float radius_m: int = Field(default=1000, ge=100, le=20000) - sources: list[Literal["avito", "cian", "yandex", "n1"]] = Field( + sources: list[Literal["avito", "cian", "yandex"]] = Field( default_factory=lambda: list(_ALL_SOURCES) ) # Если True — скрейп Yandex по 5 сегментам комнат, не одним общим запросом. @@ -83,8 +81,6 @@ class ScrapeRequest(BaseModel): deep_yandex: bool = False # Если True — скрейп Cian по 4 сегментам комнат отдельно (~4× лотов). multi_room_cian: bool = False - # Если True — скрейп N1 по 4 сегментам комнат отдельно (~4× лотов). - multi_room_n1: bool = False class ScrapeResult(BaseModel): @@ -120,7 +116,6 @@ async def scrape_around( "avito": AvitoScraper, "cian": CianScraper, "yandex": YandexRealtyScraper, - "n1": N1Scraper, }.get(source) if scraper_cls is None: continue @@ -141,10 +136,6 @@ async def scrape_around( lots = await scraper.fetch_around_multi_room( payload.lat, payload.lon, payload.radius_m ) - elif source == "n1" and payload.multi_room_n1: - lots = await scraper.fetch_around_multi_room( - payload.lat, payload.lon, payload.radius_m - ) else: lots = await scraper.fetch_around(payload.lat, payload.lon, payload.radius_m) inserted, updated = save_listings(db, lots) @@ -792,72 +783,6 @@ def cancel_avito_city_sweep( return {"ok": True, "run_id": run_id, "cancelled": cancelled} -# ── N1.ru city sweep (#575) — on-demand trigger + runs list ────────────────── - - -@router.post("/scrape/n1", response_model=CitySweepStartResponse) -async def start_n1_city_sweep( - payload: CitySweepStartRequest, - background_tasks: BackgroundTasks, - db: Annotated[Session, Depends(get_db)], -) -> CitySweepStartResponse: - """Запустить N1.ru ЕКБ city sweep в background (#575). Returns run_id. - - Зеркало avito-city-sweep, но N1 — простой SERP (search → save), без - houses/detail-enrich: поля enrich_houses/detail_top_n из payload игнорируются. - Помимо этого эндпоинта N1 крутится по расписанию (scrape_schedules - source='n1_city_sweep', seed 084). - """ - run_id = runs_mod.create_run(db, source="n1_city_sweep", params=payload.model_dump()) - - async def _sweep_task() -> None: - sweep_db = SessionLocal() - try: - await run_n1_city_sweep( - sweep_db, - run_id=run_id, - radius_m=payload.radius_m, - pages_per_anchor=payload.pages_per_anchor, - request_delay_sec=payload.request_delay_sec, - ) - except Exception: - logger.exception("n1-sweep background task run_id=%d crashed", run_id) - finally: - sweep_db.close() - - background_tasks.add_task(_sweep_task) - logger.info("n1-sweep queued run_id=%d params=%s", run_id, payload.model_dump()) - return CitySweepStartResponse( - run_id=run_id, - status="running", - pages_per_anchor=payload.pages_per_anchor, - detail_top_n=0, # N1 sweep не имеет detail-шага - ) - - -@router.get("/scrape/n1/runs", response_model=list[ScrapeRunRow]) -def list_n1_city_sweep_runs( - db: Annotated[Session, Depends(get_db)], - limit: int = 10, -) -> list[ScrapeRunRow]: - """Список последних N N1-sweep runs (для UI polling). Default limit=10.""" - rows = runs_mod.list_recent(db, source="n1_city_sweep", limit=limit) - return [ - ScrapeRunRow( - run_id=r["run_id"], - source=r["source"], - status=r["status"], - params=r.get("params"), - counters=r.get("counters"), - error=r.get("error"), - started_at=r["started_at"].isoformat() if r.get("started_at") else None, - finished_at=r["finished_at"].isoformat() if r.get("finished_at") else None, - heartbeat_at=r["heartbeat_at"].isoformat() if r.get("heartbeat_at") else None, - ) - for r in rows - ] - - # ── Cian city sweep (#860) — on-demand trigger + runs list ─────────────────── diff --git a/tradein-mvp/backend/app/services/scheduler.py b/tradein-mvp/backend/app/services/scheduler.py index 767f9361..2017fff4 100644 --- a/tradein-mvp/backend/app/services/scheduler.py +++ b/tradein-mvp/backend/app/services/scheduler.py @@ -10,7 +10,6 @@ Sources: - avito_city_sweep → run_avito_city_sweep (scrape_pipeline.py) - yandex_city_sweep → run_yandex_city_sweep (scrape_pipeline.py, #561; shipped DORMANT) - - n1_city_sweep → run_n1_city_sweep (scrape_pipeline.py, #575) - cian_history_backfill → run_cian_backfill (this module, #560) - rosreestr_dkp_import → import_rosreestr_dkp (this module, #563) - listing_source_snapshot → snapshot_listing_sources (tasks/listing_source_snapshot.py, #570; @@ -81,7 +80,6 @@ from app.services import scrape_runs as runs_mod from app.services.scrape_pipeline import ( run_avito_city_sweep, run_cian_city_sweep, - run_n1_city_sweep, run_yandex_city_sweep, ) @@ -333,39 +331,6 @@ async def trigger_yandex_city_sweep_run(db: Session, schedule_row: dict[str, Any return run_id -async def trigger_n1_city_sweep_run(db: Session, schedule_row: dict[str, Any]) -> int | None: - """Создать новый scrape_runs + launch run_n1_city_sweep в asyncio.create_task (#575). - - Зеркало trigger_yandex_city_sweep_run — N1 sweep простой (search → save, без - houses/detail). Returns run_id (или None если skip — есть running run). - """ - run_id = _claim_run(db, schedule_row) - if run_id is None: - return None - - params = schedule_row.get("default_params") or {} - - async def _run() -> None: - run_db = SessionLocal() - try: - await run_n1_city_sweep( - run_db, - run_id=run_id, - pages_per_anchor=int(params.get("pages_per_anchor", 2)), - request_delay_sec=float(params.get("request_delay_sec", 7.0)), - radius_m=int(params.get("radius_m", 1500)), - ) - except Exception: - logger.exception("scheduler: run_n1_city_sweep crashed run_id=%d", run_id) - finally: - run_db.close() - - task = asyncio.create_task(_run()) - task.add_done_callback(lambda t: t.exception() if not t.cancelled() else None) - logger.info("scheduler: triggered n1_city_sweep run_id=%d", run_id) - return run_id - - async def trigger_cian_backfill_run(db: Session, schedule_row: dict[str, Any]) -> int | None: """Создать scrape_runs + launch run_cian_backfill в asyncio.create_task. @@ -1289,8 +1254,6 @@ async def scheduler_loop() -> None: await trigger_avito_city_sweep_run(db, sch) elif source == "yandex_city_sweep": await trigger_yandex_city_sweep_run(db, sch) - elif source == "n1_city_sweep": - await trigger_n1_city_sweep_run(db, sch) elif source == "cian_history_backfill": await trigger_cian_backfill_run(db, sch) elif source == "cian_city_sweep": diff --git a/tradein-mvp/backend/app/services/scrape_pipeline.py b/tradein-mvp/backend/app/services/scrape_pipeline.py index f9c373f9..9af1ec8b 100644 --- a/tradein-mvp/backend/app/services/scrape_pipeline.py +++ b/tradein-mvp/backend/app/services/scrape_pipeline.py @@ -46,7 +46,6 @@ from app.services.scrapers.avito_exceptions import AvitoBlockedError, AvitoRateL from app.services.scrapers.avito_houses import fetch_house_catalog, save_house_catalog_enrichment from app.services.scrapers.base import ScrapedLot, save_listings from app.services.scrapers.browser_fetcher import BrowserFetcher -from app.services.scrapers.n1 import N1Scraper from app.services.scrapers.yandex_realty import YandexRealtyScraper logger = logging.getLogger(__name__) @@ -1037,155 +1036,6 @@ async def run_yandex_city_sweep( raise -# ── N1.ru city sweep (#575) ───────────────────────────────────────── - - -@dataclass -class N1CitySweepCounters: - """Aggregate counters для N1.ru city sweep run. - - Как Yandex: N1 SERP — простой search → save, без house-catalog / detail-enrich - шага, поэтому houses/detail-полей нет. - """ - - anchors_total: int = 0 - anchors_done: int = 0 - pages_fetched: int = 0 - lots_fetched: int = 0 - lots_inserted: int = 0 - lots_updated: int = 0 - errors_count: int = 0 - - def to_dict(self) -> dict[str, int]: - return {f.name: getattr(self, f.name) for f in fields(self)} - - -# Defensive abort: N1Scraper.fetch_around глотает ошибки и возвращает [] (нет -# block/rate exception-классов). Прерываем sweep после N подряд anchor'ов, -# упавших с исключением — сигнал системного блока IP. Пустой результат (0 lots) -# — НЕ ошибка, счётчик не растёт (зеркало yandex). -N1_SWEEP_MAX_CONSECUTIVE_FAILURES = 3 - - -async def run_n1_city_sweep( - db: Session, - *, - run_id: int, - anchors: list[tuple[float, float, str]] | None = None, - radius_m: int = 1500, - pages_per_anchor: int = 2, - request_delay_sec: float | None = None, -) -> N1CitySweepCounters: - """N1.ru city sweep: iterate anchors × pages → save (#575). - - Зеркало run_yandex_city_sweep — N1 это региональный SERP-портал ЕКБ без - house-catalog / detail-enrich шага: - per anchor: N1Scraper().fetch_around(lat, lon, radius_m, page=p) - → save_listings(db, lots) → accumulate counters. - - Anti-bot / cancel семантика (зеркало yandex/avito sweep): - - Cooperative cancel: is_cancelled(db, run_id) перед каждым anchor. - - N1Scraper не определяет block/rate exception-классов (fetch_around ловит - ошибки внутри → []). Per-anchor ловим broad Exception, логируем, continue; - защитно прерываем (early abort + mark_banned) при N подряд исключениях. - - request_delay_sec применяется как доп. asyncio.sleep МЕЖДУ anchor'ами - (N1Scraper сам спит self.request_delay_sec между страницами). - """ - _anchors = anchors if anchors is not None else EKB_ANCHORS - counters = N1CitySweepCounters(anchors_total=len(_anchors)) - inter_anchor_delay = request_delay_sec if request_delay_sec is not None else 7.0 - consecutive_failures = 0 - - try: - for idx, (lat, lon, name) in enumerate(_anchors, start=1): - if scrape_runs.is_cancelled(db, run_id): - logger.info( - "n1-sweep run_id=%d: cancelled at anchor #%d/%d (%s)", - run_id, - idx, - len(_anchors), - name, - ) - scrape_runs.update_heartbeat(db, run_id, counters.to_dict()) - return counters - - logger.info( - "n1-sweep run_id=%d anchor #%d/%d (%s, %.4f, %.4f)", - run_id, - idx, - len(_anchors), - name, - lat, - lon, - ) - try: - anchor_lots: list[ScrapedLot] = [] - async with N1Scraper() as scraper: - for page in range(1, pages_per_anchor + 1): - page_lots = await scraper.fetch_around(lat, lon, radius_m, page=page) - counters.pages_fetched += 1 - if not page_lots: - break - anchor_lots.extend(page_lots) - - counters.lots_fetched += len(anchor_lots) - if anchor_lots: - inserted, updated = save_listings(db, anchor_lots, run_id=run_id) - counters.lots_inserted += inserted - counters.lots_updated += updated - consecutive_failures = 0 - except Exception as e: - logger.exception("n1-sweep run_id=%d: anchor %s failed", run_id, name) - counters.errors_count += 1 - consecutive_failures += 1 - if consecutive_failures >= N1_SWEEP_MAX_CONSECUTIVE_FAILURES: - logger.error( - "n1-sweep run_id=%d ABORT at anchor #%d/%d (%s) — " - "%d consecutive failures (IP likely blocked): %s", - run_id, - idx, - len(_anchors), - name, - consecutive_failures, - e, - ) - counters.anchors_done = idx - scrape_runs.update_heartbeat(db, run_id, counters.to_dict()) - scrape_runs.mark_banned( - db, - run_id, - f"n1 sweep aborted: {consecutive_failures} consecutive " - f"anchor failures (last: {e})", - counters.to_dict(), - ) - return counters - - counters.anchors_done = idx - scrape_runs.update_heartbeat(db, run_id, counters.to_dict()) - - if idx < len(_anchors): - await asyncio.sleep(inter_anchor_delay) - - scrape_runs.mark_done(db, run_id, counters.to_dict()) - logger.info( - "n1-sweep run_id=%d done: anchors=%d/%d pages=%d lots=%d (ins=%d/upd=%d) errors=%d", - run_id, - counters.anchors_done, - counters.anchors_total, - counters.pages_fetched, - counters.lots_fetched, - counters.lots_inserted, - counters.lots_updated, - counters.errors_count, - ) - return counters - - except Exception as exc: - logger.exception("n1-sweep run_id=%d: fatal error", run_id) - scrape_runs.mark_failed(db, run_id, str(exc), counters.to_dict()) - raise - - # ── Cian city sweep (#860) ────────────────────────────────────────────────── diff --git a/tradein-mvp/backend/app/services/scraper_settings.py b/tradein-mvp/backend/app/services/scraper_settings.py index 42baf593..cdea04dc 100644 --- a/tradein-mvp/backend/app/services/scraper_settings.py +++ b/tradein-mvp/backend/app/services/scraper_settings.py @@ -32,7 +32,6 @@ _DEFAULT_DELAY_BY_SOURCE: dict[str, float] = { "avito": 7.0, "cian": 5.0, "domklik": 8.0, - "n1": 5.0, "yandex": 5.0, "yandex_detail": 5.0, "yandex_newbuilding": 5.0, diff --git a/tradein-mvp/backend/app/services/scrapers/n1.py b/tradein-mvp/backend/app/services/scrapers/n1.py deleted file mode 100644 index be2dfe51..00000000 --- a/tradein-mvp/backend/app/services/scrapers/n1.py +++ /dev/null @@ -1,271 +0,0 @@ -"""N1.ru scraper — вторичка ЕКБ (ekaterinburg.n1.ru). - -N1.ru — региональный портал недвижимости (хорошо представлен на Урале). -SSR HTML с DOM-карточками `data-test="offers-list-item"`. - -Использует curl_cffi (impersonate=chrome120) — как Cian/Avito. - -Карточка: - - a[href^="/view/NNN/"] — ссылка на объявление + offer_id - текст ссылки = "1/11 2-к, Репина, 75/2 стр." (этаж/этажность + комнаты + адрес) - Этаж/этажность — leading-токен «N/M» перед сегментом комнат. - - [data-test="offers-list-item-param-total-area"] — "79 м2" - - [data-test="offers-list-item-param-price"] — "15 700 000" - - img.src — фото - -Координаты карточки не отдаёт → оставляем lat/lon = None, -geocode-missing проставит реальные по адресу. -""" - -from __future__ import annotations - -import logging -import re -from typing import Any - -from curl_cffi.requests import AsyncSession -from selectolax.parser import HTMLParser - -from app.services.scraper_settings import get_scraper_delay -from app.services.scrapers.base import BaseScraper, ScrapedLot - -logger = logging.getLogger(__name__) - - -class N1Scraper(BaseScraper): - """N1.ru vtorichka parser. Источник = 'n1'.""" - - name = "n1" - base_url = "https://ekaterinburg.n1.ru" - # Класс-дефолт; реальное значение загружается из scraper_settings при создании экземпляра. - request_delay_sec = 5.0 - - def __init__(self) -> None: - super().__init__() - self.request_delay_sec = get_scraper_delay(self.name) - self._cffi: AsyncSession | None = None - - async def __aenter__(self) -> N1Scraper: - await super().__aenter__() - self._cffi = AsyncSession( - impersonate="chrome120", - timeout=25, - headers={ - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", - "Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8", - "Sec-Fetch-Dest": "document", - "Sec-Fetch-Mode": "navigate", - "Sec-Fetch-Site": "none", - "Upgrade-Insecure-Requests": "1", - }, - ) - return self - - async def __aexit__(self, *args: Any) -> None: - if self._cffi is not None: - await self._cffi.close() - await super().__aexit__(*args) - - async def fetch_around( - self, - lat: float, - lon: float, - radius_m: int = 1000, - rooms: int | None = None, - page: int = 1, - ) -> list[ScrapedLot]: - self._anchor_lat = lat - self._anchor_lon = lon - self._anchor_radius_m = radius_m - - url = self._build_url(rooms, page) - try: - assert self._cffi is not None - response = await self._cffi.get(url) - except Exception: - logger.exception("n1 fetch failed for %s", url) - return [] - if response.status_code != 200: - logger.warning("n1 returned %d for %s", response.status_code, url) - return [] - - lots = self._parse_html(response.text) - logger.info( - "n1: %d lots around (%.4f, %.4f) rooms=%s page=%d", - len(lots), - lat, - lon, - rooms, - page, - ) - await self.sleep_between_requests() - return lots - - async def fetch_around_multi_room( - self, - lat: float, - lon: float, - radius_m: int = 1000, - pages: int = 8, - ) -> list[ScrapedLot]: - """Скрейп N1 по сегментам комнат 1/2/3/4 × N страниц — расширяет выборку.""" - seen: dict[str, ScrapedLot] = {} - for rooms in (1, 2, 3, 4): - for page in range(1, pages + 1): - try: - lots = await self.fetch_around(lat, lon, radius_m, rooms=rooms, page=page) - except Exception: - logger.exception("n1 multi-room fetch failed rooms=%s page=%d", rooms, page) - continue - if not lots: - break # пустая страница → дальше нет смысла - for lot in lots: - key = lot.source_id or lot.source_url - if key and key not in seen: - seen[key] = lot - logger.info("n1 multi-room: %d unique lots around (%.4f, %.4f)", len(seen), lat, lon) - return list(seen.values()) - - def _build_url(self, rooms: int | None = None, page: int = 1) -> str: - """URL каталога вторички ЕКБ. - - N1 фильтр комнат: /kupit/kvartiry/vtorichka/1-komnatnye/ итд. - Пагинация: ?page=N. - """ - rooms_segment = { - 1: "1-komnatnye/", - 2: "2-komnatnye/", - 3: "3-komnatnye/", - 4: "4-komnatnye/", - }.get(rooms or 0, "") - url = f"{self.base_url}/kupit/kvartiry/vtorichka/{rooms_segment}" - if page > 1: - url += f"?page={page}" - return url - - def _parse_html(self, html: str) -> list[ScrapedLot]: - tree = HTMLParser(html) - cards = tree.css('[data-test="offers-list-item"]') - return [lot for lot in (self._card_to_lot(c) for c in cards) if lot is not None] - - def _card_to_lot(self, card: Any) -> ScrapedLot | None: - try: - # Ссылка на объявление /view/NNN/ - link = card.css_first('a[href*="/view/"]') - if link is None: - return None - href = link.attributes.get("href", "") - m_id = re.search(r"/view/(\d+)/", href) - offer_id = m_id.group(1) if m_id else None - url = href if href.startswith("http") else f"{self.base_url}{href}" - - # Заголовок ссылки: "1/11 2-к, Репина, 75/2 стр." (этаж/этажность + комнаты + адрес) - title = link.text(strip=True) or "" - floor, total_floors = _extract_floor_from_title(title) - rooms = _extract_rooms(title) - # Адрес = title без ведущего «N/M» и без префикса комнат - address = re.sub( - r"^\s*(\d+-к|студи[яи])\s*,?\s*", "", title, flags=re.IGNORECASE - ).strip() - if not address: - address = "Екатеринбург (N1)" - - # Площадь - area_el = card.css_first('[data-test="offers-list-item-param-total-area"]') - area = _extract_area(area_el.text(strip=True) if area_el else "") - - # Цена - price_el = card.css_first('[data-test="offers-list-item-param-price"]') - price = _extract_price(price_el.text(strip=True) if price_el else "") - if not price: - return None - - # Координаты НЕ ставим — N1 card не отдаёт. Оставляем None, - # geocode-missing проставит реальные по address (N1 отдаёт всё ЕКБ, - # jitter вокруг якоря был бы фейком). - lat = lon = None - - # Фото - photos: list[str] = [] - for img in card.css("img")[:5]: - src = img.attributes.get("src", "") - if src and src.startswith("http") and src not in photos: - photos.append(src) - - return ScrapedLot( - source="n1", - source_url=url, - source_id=offer_id, - address=address, - lat=lat, - lon=lon, - rooms=rooms, - area_m2=area, - floor=floor, - total_floors=total_floors, - price_rub=price, - photo_urls=photos, - raw_payload={"title": title}, - ) - except Exception: - logger.exception("n1 card parse failed") - return None - - -# ── Helpers ───────────────────────────────────────────────────────────────── - -_RE_ROOMS = re.compile(r"(\d+)\s*-?\s*к", re.IGNORECASE) -_RE_STUDIO = re.compile(r"студи", re.IGNORECASE) -_RE_AREA = re.compile(r"([\d]+(?:[.,]\d+)?)") -_RE_DIGITS = re.compile(r"\d") - -# Ведущий токен «N/M» в тексте карточки N1: «1/11 2-к, Репина, 75/2 стр.» -# Якорь ^ гарантирует: только если N/M стоит В НАЧАЛЕ строки (до сегмента комнат). -# Дробь в адресе вида «75/2» в середине строки НЕ совпадёт. -_RE_FLOOR_LEADING = re.compile(r"^\s*(\d{1,3})\s*/\s*(\d{1,3})\b") - - -def _extract_rooms(text: str) -> int | None: - if _RE_STUDIO.search(text): - return 0 - m = _RE_ROOMS.search(text) - return int(m.group(1)) if m else None - - -def _extract_area(text: str) -> float | None: - m = _RE_AREA.search(text) - if m: - try: - return float(m.group(1).replace(",", ".")) - except ValueError: - return None - return None - - -def _extract_price(text: str) -> int | None: - digits = "".join(_RE_DIGITS.findall(text)) - return int(digits) if digits else None - - -def _extract_floor_from_title(text: str) -> tuple[int | None, int | None]: - """Извлекает этаж и этажность из ведущего токена «N/M» в тексте карточки N1. - - N1 SERP отдаёт текст вида «1/11 2-к, Репина, 75/2 стр.» — этаж/этажность - идут первыми, до сегмента комнат. Регулярка привязана к началу строки (^), - поэтому дробь в номере дома («75/2») в середине строки не захватывается. - - Sanity-проверки: - - floor >= 1 - - total_floors >= floor - - total_floors <= 100 - - Возвращает (None, None) при отсутствии совпадения или нарушении диапазона. - """ - m = _RE_FLOOR_LEADING.match(text) - if m is None: - return None, None - floor = int(m.group(1)) - total_floors = int(m.group(2)) - if floor < 1 or total_floors < floor or total_floors > 100: - return None, None - return floor, total_floors diff --git a/tradein-mvp/backend/data/sql/114_disable_n1_sweep.sql b/tradein-mvp/backend/data/sql/114_disable_n1_sweep.sql new file mode 100644 index 00000000..29c520c6 --- /dev/null +++ b/tradein-mvp/backend/data/sql/114_disable_n1_sweep.sql @@ -0,0 +1,18 @@ +-- Миграция 114: деактивировать N1.ru city sweep. +-- +-- N1.ru — региональный портал ЕКБ — переехал в Cian (подтверждено владельцем 2026-06). +-- Активный скрап прекращается. Строку НЕ удаляем — нужна для audit-истории run'ов. +-- Исторические listings source='n1' (389 записей в БД) сохраняются без изменений. + +BEGIN; + +UPDATE scrape_schedules +SET enabled = false +WHERE source = 'n1_city_sweep' + AND enabled = true; -- idempotent: повторный запуск не трогает уже-disabled + +COMMENT ON TABLE scrape_schedules IS + 'Периодические задачи: avito/yandex/cian city sweep, rosreestr import, ' + 'listing_source_snapshot. N1 city sweep деактивирован (2026-06, переехал в cian).'; + +COMMIT; diff --git a/tradein-mvp/backend/tests/test_n1_city_sweep.py b/tradein-mvp/backend/tests/test_n1_city_sweep.py deleted file mode 100644 index e24b4f4d..00000000 --- a/tradein-mvp/backend/tests/test_n1_city_sweep.py +++ /dev/null @@ -1,228 +0,0 @@ -"""Offline unit tests for run_n1_city_sweep (#575). - -Pure & fast: NO live network, NO DB. Mirrors test_yandex_city_sweep — monkeypatch -N1Scraper lifecycle + fetch_around, scrape_pipeline.save_listings, scrape_runs.*. -Asserts orchestration semantics only (live N1 behavior verified by post-deploy QA). - -NB: run_n1_city_sweep пагинирует page=1..pages_per_anchor (1-based), в отличие от -yandex (0-based) — тесты учитывают это. -""" - -import os - -os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test") - -from typing import Any - -import pytest - -from app.services import scrape_pipeline -from app.services.scrapers.base import ScrapedLot -from app.services.scrapers.n1 import N1Scraper - -TEST_ANCHORS: list[tuple[float, float, str]] = [ - (56.8400, 60.6050, "A1"), - (56.7950, 60.5300, "A2"), - (56.8970, 60.6100, "A3"), -] - - -def _fake_lot(offer_id: str) -> ScrapedLot: - return ScrapedLot( - source="n1", - source_url=f"https://ekb.n1.ru/view/{offer_id}/", - source_id=offer_id, - address="Екатеринбург, улица Тестовая, 1", - price_rub=5_000_000, - area_m2=50.0, - rooms=2, - ) - - -class _FakeRuns: - def __init__(self, *, cancel_at_anchor: int | None = None) -> None: - self.cancel_at_anchor = cancel_at_anchor - self.is_cancelled_calls = 0 - self.heartbeats: list[dict[str, int]] = [] - self.done: dict[str, int] | None = None - self.failed: tuple[str, dict[str, int]] | None = None - self.banned: tuple[str, dict[str, int]] | None = None - - def is_cancelled(self, _db: Any, _run_id: int) -> bool: - self.is_cancelled_calls += 1 - if self.cancel_at_anchor is None: - return False - return self.is_cancelled_calls >= self.cancel_at_anchor - - def update_heartbeat(self, _db: Any, _run_id: int, counters: dict[str, int]) -> None: - self.heartbeats.append(dict(counters)) - - def mark_done(self, _db: Any, _run_id: int, counters: dict[str, int]) -> None: - self.done = dict(counters) - - def mark_failed(self, _db: Any, _run_id: int, error: str, counters: dict[str, int]) -> None: - self.failed = (error, dict(counters)) - - def mark_banned(self, _db: Any, _run_id: int, error: str, counters: dict[str, int]) -> None: - self.banned = (error, dict(counters)) - - -@pytest.fixture(autouse=True) -def _no_sleep(monkeypatch: pytest.MonkeyPatch) -> None: - async def _instant(_secs: float) -> None: - return None - - monkeypatch.setattr(scrape_pipeline.asyncio, "sleep", _instant) - - -@pytest.fixture(autouse=True) -def _stub_scraper_lifecycle(monkeypatch: pytest.MonkeyPatch) -> None: - """Neutralize __init__/__aenter__/__aexit__ — no get_scraper_delay DB, no curl_cffi.""" - - def _init(self: N1Scraper) -> None: - self.request_delay_sec = 0.0 - self._cffi = None - - async def _aenter(self: N1Scraper) -> N1Scraper: - return self - - async def _aexit(self: N1Scraper, *_args: Any) -> None: - return None - - monkeypatch.setattr(N1Scraper, "__init__", _init) - monkeypatch.setattr(N1Scraper, "__aenter__", _aenter) - monkeypatch.setattr(N1Scraper, "__aexit__", _aexit) - - -def _install_runs(monkeypatch: pytest.MonkeyPatch, fake: _FakeRuns) -> None: - monkeypatch.setattr(scrape_pipeline.scrape_runs, "is_cancelled", fake.is_cancelled) - monkeypatch.setattr(scrape_pipeline.scrape_runs, "update_heartbeat", fake.update_heartbeat) - monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_done", fake.mark_done) - monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_failed", fake.mark_failed) - monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_banned", fake.mark_banned) - - -async def test_sweep_iterates_all_anchors_and_aggregates(monkeypatch: pytest.MonkeyPatch) -> None: - fetch_calls: list[tuple[float, float, int]] = [] - save_calls: list[int] = [] - - async def fake_fetch(self, lat, lon, radius_m=1000, rooms=None, page=1): - fetch_calls.append((lat, lon, page)) - return [_fake_lot(f"{lat}-{page}-{i}") for i in range(2)] - - def fake_save(_db, lots, *, run_id=None): - save_calls.append(len(lots)) - return len(lots), 0 - - monkeypatch.setattr(N1Scraper, "fetch_around", fake_fetch) - monkeypatch.setattr(scrape_pipeline, "save_listings", fake_save) - fake = _FakeRuns() - _install_runs(monkeypatch, fake) - - counters = await scrape_pipeline.run_n1_city_sweep( - db=object(), run_id=1, anchors=TEST_ANCHORS, pages_per_anchor=2, request_delay_sec=0.0 - ) - - assert len(fetch_calls) == len(TEST_ANCHORS) * 2 - assert [p for _, _, p in fetch_calls[:2]] == [1, 2] # 1-based pagination - assert len(save_calls) == len(TEST_ANCHORS) - assert counters.anchors_done == len(TEST_ANCHORS) - assert counters.pages_fetched == len(TEST_ANCHORS) * 2 - assert counters.lots_fetched == 12 - assert counters.lots_inserted == 12 - assert counters.errors_count == 0 - assert fake.done is not None and fake.done["lots_fetched"] == 12 - assert fake.failed is None and fake.banned is None - - -async def test_save_listings_receives_run_id(monkeypatch: pytest.MonkeyPatch) -> None: - seen: list[int | None] = [] - - async def fake_fetch(self, lat, lon, radius_m=1000, rooms=None, page=1): - return [_fake_lot(f"{lat}-{page}")] - - def fake_save(_db, lots, *, run_id=None): - seen.append(run_id) - return len(lots), 0 - - monkeypatch.setattr(N1Scraper, "fetch_around", fake_fetch) - monkeypatch.setattr(scrape_pipeline, "save_listings", fake_save) - _install_runs(monkeypatch, _FakeRuns()) - - await scrape_pipeline.run_n1_city_sweep( - db=object(), run_id=777, anchors=TEST_ANCHORS, pages_per_anchor=1 - ) - assert seen == [777, 777, 777] - - -async def test_empty_page_stops_pagination(monkeypatch: pytest.MonkeyPatch) -> None: - pages_seen: list[int] = [] - - async def fake_fetch(self, lat, lon, radius_m=1000, rooms=None, page=1): - pages_seen.append(page) - if page == 1: - return [_fake_lot(f"{lat}-1")] - return [] # page 2 empty → stop - - monkeypatch.setattr(N1Scraper, "fetch_around", fake_fetch) - monkeypatch.setattr( - scrape_pipeline, "save_listings", lambda _db, lots, *, run_id=None: (len(lots), 0) - ) - fake = _FakeRuns() - _install_runs(monkeypatch, fake) - - counters = await scrape_pipeline.run_n1_city_sweep( - db=object(), run_id=2, anchors=TEST_ANCHORS, pages_per_anchor=5 - ) - assert pages_seen == [1, 2, 1, 2, 1, 2] - assert counters.pages_fetched == 6 - assert counters.lots_fetched == 3 - assert counters.anchors_done == 3 - - -async def test_cancel_midway_stops_remaining(monkeypatch: pytest.MonkeyPatch) -> None: - fetch_calls: list[str] = [] - - async def fake_fetch(self, lat, lon, radius_m=1000, rooms=None, page=1): - fetch_calls.append(f"{lat}") - return [_fake_lot(f"{lat}-{page}")] - - monkeypatch.setattr(N1Scraper, "fetch_around", fake_fetch) - monkeypatch.setattr( - scrape_pipeline, "save_listings", lambda _db, lots, *, run_id=None: (len(lots), 0) - ) - fake = _FakeRuns(cancel_at_anchor=2) - _install_runs(monkeypatch, fake) - - counters = await scrape_pipeline.run_n1_city_sweep( - db=object(), run_id=3, anchors=TEST_ANCHORS, pages_per_anchor=1 - ) - assert len(fetch_calls) == 1 - assert counters.anchors_done == 1 - assert fake.done is None and fake.banned is None - - -async def test_consecutive_failures_abort_sweep(monkeypatch: pytest.MonkeyPatch) -> None: - attempts: list[str] = [] - - async def fake_fetch(self, lat, lon, radius_m=1000, rooms=None, page=1): - attempts.append(f"{lat}") - raise RuntimeError("n1 blocked") - - monkeypatch.setattr(N1Scraper, "fetch_around", fake_fetch) - monkeypatch.setattr( - scrape_pipeline, "save_listings", lambda _db, lots, *, run_id=None: (len(lots), 0) - ) - fake = _FakeRuns() - _install_runs(monkeypatch, fake) - - anchors = [*TEST_ANCHORS, (56.77, 60.55, "A4"), (56.865, 60.62, "A5")] - counters = await scrape_pipeline.run_n1_city_sweep( - db=object(), run_id=5, anchors=anchors, pages_per_anchor=1 - ) - n = scrape_pipeline.N1_SWEEP_MAX_CONSECUTIVE_FAILURES - assert len(attempts) == n - assert counters.errors_count == n - assert counters.anchors_done == n - assert fake.banned is not None and "consecutive" in fake.banned[0] - assert fake.done is None diff --git a/tradein-mvp/backend/tests/test_n1_floor_parse.py b/tradein-mvp/backend/tests/test_n1_floor_parse.py deleted file mode 100644 index df89d9f4..00000000 --- a/tradein-mvp/backend/tests/test_n1_floor_parse.py +++ /dev/null @@ -1,169 +0,0 @@ -"""Unit tests for N1 floor/total_floors extraction (#795). - -Tests _extract_floor_from_title() directly (pure function, no network/DB). -Also covers _card_to_lot integration via a minimal fake card HTML node. -""" - -from __future__ import annotations - -import pytest - -from app.services.scrapers.n1 import _extract_floor_from_title - -# ── _extract_floor_from_title: direct unit tests ──────────────────────────── - - -@pytest.mark.parametrize( - "text, expected_floor, expected_total", - [ - # Typical N1 card: floor 1 of 11 - ("1/11 2-к, Репина, 75/2 стр.", 1, 11), - # Floor in middle of building - ("5/9 3-к, Ленина, 10 стр.", 5, 9), - # Top floor - ("17/17 1-к, Малышева, 12", 17, 17), - # Leading whitespace tolerated - (" 3/16 студия, Вайнера, 5", 3, 16), - # Three-digit total floors (edge of range) - ("2/100 2-к, Тверитина, 8", 2, 100), - ], -) -def test_extract_floor_happy_path(text: str, expected_floor: int, expected_total: int) -> None: - floor, total = _extract_floor_from_title(text) - assert floor == expected_floor - assert total == expected_total - - -@pytest.mark.parametrize( - "text", - [ - # No leading N/M at all — classic old format without floor token - ("2-к, Репина, 75/2 стр."), - # House number fraction in middle — must NOT be parsed as floor - ("Репина, 75/2 стр."), - # N/M not at string start (preceded by other text) - ("квартира 3/9, Ленина, 1"), - # floor=0 is invalid - ("0/9 1-к, Тестовая, 1"), - # total_floors < floor — sanity fail - ("9/5 2-к, Тестовая, 1"), - # total_floors > 100 — sanity fail - ("2/101 1-к, Тестовая, 1"), - # Empty string - (""), - # Only digits, no slash - ("5 2-к, Тестовая, 1"), - ], -) -def test_extract_floor_returns_none(text: str) -> None: - floor, total = _extract_floor_from_title(text) - assert floor is None - assert total is None - - -def test_house_number_fraction_not_misparsed() -> None: - """«75/2 стр.» appearing after rooms segment must not match floor regex.""" - # The fraction 75/2 comes AFTER the rooms prefix, not at string start. - floor, total = _extract_floor_from_title("3-к, Репина, 75/2 стр.") - assert floor is None - assert total is None - - -def test_non_leading_fraction_after_comma() -> None: - """Fraction embedded mid-string via comma-separated address token.""" - floor, total = _extract_floor_from_title("ул. 8 Марта, 51/3, 2-к") - assert floor is None - assert total is None - - -# ── Integration: _card_to_lot via fake selectolax node ────────────────────── - - -class _FakeAttr(dict): - """Minimal attributes dict for a fake HTML node.""" - - -class _FakeNode: - """Minimal selectolax node stub for _card_to_lot tests.""" - - def __init__( - self, - *, - href: str = "/view/12345/", - title_text: str = "1/11 2-к, Репина, 75/2 стр.", - area_text: str = "52 м2", - price_text: str = "8 500 000", - img_src: str = "https://img.n1.ru/photo.jpg", - ) -> None: - self._href = href - self._title_text = title_text - self._area_text = area_text - self._price_text = price_text - self._img_src = img_src - - def css_first(self, selector: str) -> _FakeNode | None: - if 'a[href*="/view/"]' in selector: - return self - if "total-area" in selector: - return _FakeTextNode(self._area_text) - if "param-price" in selector: - return _FakeTextNode(self._price_text) - return None - - def css(self, selector: str) -> list[_FakeNode]: - if selector == "img": - return [_FakeImgNode(self._img_src)] - return [] - - @property - def attributes(self) -> dict: - return {"href": self._href} - - def text(self, strip: bool = False) -> str: - t = self._title_text - return t.strip() if strip else t - - -class _FakeTextNode: - def __init__(self, text: str) -> None: - self._text = text - - def text(self, strip: bool = False) -> str: - return self._text.strip() if strip else self._text - - -class _FakeImgNode: - def __init__(self, src: str) -> None: - self.attributes = {"src": src} - - -def _call_card_to_lot(card: _FakeNode): # type: ignore[return] - from app.services.scrapers.n1 import N1Scraper - - scraper = object.__new__(N1Scraper) - scraper.base_url = "https://ekaterinburg.n1.ru" # type: ignore[attr-defined] - return scraper._card_to_lot(card) # type: ignore[attr-defined] - - -def test_card_to_lot_parses_floor() -> None: - card = _FakeNode(title_text="1/11 2-к, Репина, 75/2 стр.") - lot = _call_card_to_lot(card) - assert lot is not None - assert lot.floor == 1 - assert lot.total_floors == 11 - - -def test_card_to_lot_floor_none_when_no_token() -> None: - card = _FakeNode(title_text="2-к, Репина, 75/2 стр.") - lot = _call_card_to_lot(card) - assert lot is not None - assert lot.floor is None - assert lot.total_floors is None - - -def test_card_to_lot_listing_date_is_none() -> None: - """N1 SERP has no date — listing_date must remain None.""" - card = _FakeNode(title_text="3/9 1-к, Малышева, 12") - lot = _call_card_to_lot(card) - assert lot is not None - assert lot.listing_date is None diff --git a/tradein-mvp/backend/tests/test_scraper_settings.py b/tradein-mvp/backend/tests/test_scraper_settings.py index a465382c..2b83f9c1 100644 --- a/tradein-mvp/backend/tests/test_scraper_settings.py +++ b/tradein-mvp/backend/tests/test_scraper_settings.py @@ -149,7 +149,7 @@ def test_invalidate_cache_specific_source(): mock_open, _ = _mock_session_uniform(6.0) with patch(_PATCH_TARGET, mock_open): ss.get_scraper_delay("yandex") # +2 (yandex + global) - ss.invalidate_cache("yandex") # clears yandex key + ss.invalidate_cache("yandex") # clears yandex key ss.get_scraper_delay("yandex") # +1 (yandex stale; global still cached) assert mock_open.call_count == 3 @@ -159,7 +159,7 @@ def test_invalidate_cache_all(): mock_open, _ = _mock_session_uniform(6.0) with patch(_PATCH_TARGET, mock_open): ss.get_scraper_delay("yandex") # +2 (yandex + global) - ss.invalidate_cache() # clears everything + ss.invalidate_cache() # clears everything ss.get_scraper_delay("yandex") # +2 (both stale) assert mock_open.call_count == 4 @@ -169,8 +169,8 @@ def test_invalidate_alias_invalidates_umbrella(): mock_open, _ = _mock_session_uniform(6.0) with patch(_PATCH_TARGET, mock_open): ss.get_scraper_delay("yandex_detail") # +2 (yandex + global) - ss.invalidate_cache("yandex_detail") # clears yandex (via alias) - ss.get_scraper_delay("yandex_detail") # +1 (yandex stale; global fresh) + ss.invalidate_cache("yandex_detail") # clears yandex (via alias) + ss.get_scraper_delay("yandex_detail") # +1 (yandex stale; global fresh) assert mock_open.call_count == 3 @@ -214,11 +214,11 @@ def test_missing_per_source_row_uses_class_default(monkeypatch): """Missing source row → _DEFAULT_DELAY_BY_SOURCE. global=0 → no floor.""" invalidate_cache() - mock_db = _make_mock_session({"n1": None, "global": 0.0}) + mock_db = _make_mock_session({"domklik": None, "global": 0.0}) monkeypatch.setattr(ss_mod, "_open_session", lambda: mock_db) - delay = get_scraper_delay("n1") - assert delay == ss_mod._DEFAULT_DELAY_BY_SOURCE["n1"] + delay = get_scraper_delay("domklik") + assert delay == ss_mod._DEFAULT_DELAY_BY_SOURCE["domklik"] def test_global_missing_row_treated_as_zero(monkeypatch):