diff --git a/tradein-mvp/backend/app/api/v1/admin.py b/tradein-mvp/backend/app/api/v1/admin.py index 89a881a5..b111832e 100644 --- a/tradein-mvp/backend/app/api/v1/admin.py +++ b/tradein-mvp/backend/app/api/v1/admin.py @@ -19,6 +19,7 @@ from app.services.geocoder import geocode from app.services.scrapers.avito import AvitoScraper from app.services.scrapers.base import save_listings from app.services.scrapers.cian import CianScraper +from app.services.scrapers.n1 import N1Scraper from app.services.scrapers.yandex_realty import YandexRealtyScraper logger = logging.getLogger(__name__) @@ -30,10 +31,11 @@ class ScrapeRequest(BaseModel): lat: float lon: float radius_m: int = Field(default=1000, ge=100, le=20000) - sources: list[Literal["avito", "cian", "yandex"]] = Field(default_factory=lambda: ["avito", "cian", "yandex"]) + sources: list[Literal["avito", "cian", "yandex", "n1"]] = Field(default_factory=lambda: ["avito", "cian", "yandex", "n1"]) multi_room_yandex: bool = False # Если True — скрейп Yandex по 5 сегментам комнат, не одним общим запросом. deep_yandex: bool = False # Если True — Yandex полный обход 5 rooms × 3 sorts × 2 pages = 30 запросов / ~150s. multi_room_cian: bool = False # Если True — скрейп Cian по 4 сегментам комнат отдельно (~4× лотов). + multi_room_n1: bool = False # Если True — скрейп N1 по 4 сегментам комнат отдельно (~4× лотов). class ScrapeResult(BaseModel): @@ -69,6 +71,7 @@ async def scrape_around( "avito": AvitoScraper, "cian": CianScraper, "yandex": YandexRealtyScraper, + "n1": N1Scraper, }.get(source) if scraper_cls is None: continue @@ -87,6 +90,10 @@ async def scrape_around( lots = await scraper.fetch_around_multi_room( payload.lat, payload.lon, payload.radius_m ) + elif source == "n1" and payload.multi_room_n1: + lots = await scraper.fetch_around_multi_room( + payload.lat, payload.lon, payload.radius_m + ) else: lots = await scraper.fetch_around( payload.lat, payload.lon, payload.radius_m diff --git a/tradein-mvp/backend/app/services/scrapers/n1.py b/tradein-mvp/backend/app/services/scrapers/n1.py new file mode 100644 index 00000000..09891ba9 --- /dev/null +++ b/tradein-mvp/backend/app/services/scrapers/n1.py @@ -0,0 +1,218 @@ +"""N1.ru scraper — вторичка ЕКБ (ekaterinburg.n1.ru). + +N1.ru — региональный портал недвижимости (хорошо представлен на Урале). +SSR HTML с DOM-карточками `data-test="offers-list-item"`. + +Использует curl_cffi (impersonate=chrome120) — как Cian/Avito. + +Карточка: + - a[href^="/view/NNN/"] — ссылка на объявление + offer_id + текст ссылки = "3-к, Репина, 75/2 стр." (комнаты + адрес) + - [data-test="offers-list-item-param-total-area"] — "79 м2" + - [data-test="offers-list-item-param-price"] — "15 700 000" + - img.src — фото + +Координаты карточки не отдаёт → anchor+jitter (как Avito). +""" + +from __future__ import annotations + +import logging +import random +import re +from typing import Any + +from curl_cffi.requests import AsyncSession +from selectolax.parser import HTMLParser + +from app.services.scrapers.base import BaseScraper, ScrapedLot + +logger = logging.getLogger(__name__) + + +class N1Scraper(BaseScraper): + """N1.ru vtorichka parser. Источник = 'n1'.""" + + name = "n1" + base_url = "https://ekaterinburg.n1.ru" + request_delay_sec = 5.0 + + def __init__(self) -> None: + super().__init__() + self._cffi: AsyncSession | None = None + + async def __aenter__(self) -> "N1Scraper": + await super().__aenter__() + self._cffi = AsyncSession( + impersonate="chrome120", + timeout=25, + headers={ + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8", + "Sec-Fetch-Dest": "document", + "Sec-Fetch-Mode": "navigate", + "Sec-Fetch-Site": "none", + "Upgrade-Insecure-Requests": "1", + }, + ) + return self + + async def __aexit__(self, *args: Any) -> None: + if self._cffi is not None: + await self._cffi.close() + await super().__aexit__(*args) + + async def fetch_around( + self, lat: float, lon: float, radius_m: int = 1000, rooms: int | None = None, + ) -> list[ScrapedLot]: + self._anchor_lat = lat + self._anchor_lon = lon + self._anchor_radius_m = radius_m + + url = self._build_url(rooms) + try: + assert self._cffi is not None + response = await self._cffi.get(url) + except Exception: # noqa: BLE001 + logger.exception("n1 fetch failed for %s", url) + return [] + if response.status_code != 200: + logger.warning("n1 returned %d for %s", response.status_code, url) + return [] + + lots = self._parse_html(response.text) + logger.info( + "n1: %d lots around (%.4f, %.4f) rooms=%s", len(lots), lat, lon, rooms + ) + await self.sleep_between_requests() + return lots + + async def fetch_around_multi_room( + self, lat: float, lon: float, radius_m: int = 1000 + ) -> list[ScrapedLot]: + """Скрейп N1 по сегментам комнат 1/2/3/4 — расширяет выборку.""" + seen: dict[str, ScrapedLot] = {} + for rooms in (1, 2, 3, 4): + try: + lots = await self.fetch_around(lat, lon, radius_m, rooms=rooms) + except Exception: + logger.exception("n1 multi-room fetch failed for rooms=%s", rooms) + continue + for lot in lots: + key = lot.source_id or lot.source_url + if key and key not in seen: + seen[key] = lot + logger.info( + "n1 multi-room: %d unique lots around (%.4f, %.4f)", len(seen), lat, lon + ) + return list(seen.values()) + + def _build_url(self, rooms: int | None = None) -> str: + """URL каталога вторички ЕКБ. + + N1 фильтр комнат: /kupit/kvartiry/vtorichka/1-komnatnye/ итд. + """ + rooms_segment = { + 1: "1-komnatnye/", + 2: "2-komnatnye/", + 3: "3-komnatnye/", + 4: "4-komnatnye/", + }.get(rooms or 0, "") + return f"{self.base_url}/kupit/kvartiry/vtorichka/{rooms_segment}" + + def _parse_html(self, html: str) -> list[ScrapedLot]: + tree = HTMLParser(html) + cards = tree.css('[data-test="offers-list-item"]') + return [ + lot for lot in (self._card_to_lot(c) for c in cards) if lot is not None + ] + + def _card_to_lot(self, card: Any) -> ScrapedLot | None: + try: + # Ссылка на объявление /view/NNN/ + link = card.css_first('a[href*="/view/"]') + if link is None: + return None + href = link.attributes.get("href", "") + m_id = re.search(r"/view/(\d+)/", href) + offer_id = m_id.group(1) if m_id else None + url = href if href.startswith("http") else f"{self.base_url}{href}" + + # Заголовок ссылки: "3-к, Репина, 75/2 стр." + title = link.text(strip=True) or "" + rooms = _extract_rooms(title) + # Адрес = title без префикса комнат + address = re.sub(r"^\s*(\d+-к|студи[яи])\s*,?\s*", "", title, flags=re.IGNORECASE).strip() + if not address: + address = "Екатеринбург (N1)" + + # Площадь + area_el = card.css_first('[data-test="offers-list-item-param-total-area"]') + area = _extract_area(area_el.text(strip=True) if area_el else "") + + # Цена + price_el = card.css_first('[data-test="offers-list-item-param-price"]') + price = _extract_price(price_el.text(strip=True) if price_el else "") + if not price: + return None + + # Координаты: anchor + jitter (N1 card не отдаёт coords) + lat = lon = None + if hasattr(self, "_anchor_lat"): + jitter_deg = (self._anchor_radius_m / 1000) / 111 + lat = self._anchor_lat + (random.random() - 0.5) * jitter_deg + lon = self._anchor_lon + (random.random() - 0.5) * jitter_deg / 2 + + # Фото + photos: list[str] = [] + for img in card.css("img")[:5]: + src = img.attributes.get("src", "") + if src and src.startswith("http") and src not in photos: + photos.append(src) + + return ScrapedLot( + source="n1", + source_url=url, + source_id=offer_id, + address=address, + lat=lat, + lon=lon, + rooms=rooms, + area_m2=area, + price_rub=price, + photo_urls=photos, + raw_payload={"title": title}, + ) + except Exception: # noqa: BLE001 + logger.exception("n1 card parse failed") + return None + + +# ── Helpers ───────────────────────────────────────────────────────────────── + +_RE_ROOMS = re.compile(r"(\d+)\s*-?\s*к", re.IGNORECASE) +_RE_STUDIO = re.compile(r"студи", re.IGNORECASE) +_RE_AREA = re.compile(r"([\d]+(?:[.,]\d+)?)") +_RE_DIGITS = re.compile(r"\d") + + +def _extract_rooms(text: str) -> int | None: + if _RE_STUDIO.search(text): + return 0 + m = _RE_ROOMS.search(text) + return int(m.group(1)) if m else None + + +def _extract_area(text: str) -> float | None: + m = _RE_AREA.search(text) + if m: + try: + return float(m.group(1).replace(",", ".")) + except ValueError: + return None + return None + + +def _extract_price(text: str) -> int | None: + digits = "".join(_RE_DIGITS.findall(text)) + return int(digits) if digits else None diff --git a/tradein-mvp/deploy/cron-scrape.sh b/tradein-mvp/deploy/cron-scrape.sh old mode 100644 new mode 100755 index e2c43e3e..7ebaa949 --- a/tradein-mvp/deploy/cron-scrape.sh +++ b/tradein-mvp/deploy/cron-scrape.sh @@ -1,20 +1,20 @@ #!/usr/bin/env bash -# Периодический скрейп Yandex+Avito по 5 якорям ЕКБ. -# Запускается из cron. Логи в /var/log/tradein-scrape.log +# Периодический скрейп всех площадок по 5 якорям ЕКБ. +# Запускается из cron на ПРОД-хосте (46.173.16.127). +# Запрос к backend идёт через `docker exec tradein-backend curl localhost:8000` +# — не зависит от Caddy и не выставляет admin/scrape наружу. # -# Usage: ./cron-scrape.sh [yandex|avito|all] [deep|fast] +# Usage: ./cron-scrape.sh [all|avito|cian|yandex|n1] [deep|fast] # -# Примеры в crontab: -# # Yandex deep раз в 3 часа -# 0 */3 * * * /opt/tradein-mvp/deploy/cron-scrape.sh yandex deep -# # Avito быстро каждый час -# 30 * * * * /opt/tradein-mvp/deploy/cron-scrape.sh avito fast +# Crontab пример: +# 0 */3 * * * /opt/gendesign/tradein-mvp/deploy/cron-scrape.sh all deep +# 30 * * * * /opt/gendesign/tradein-mvp/deploy/cron-scrape.sh all fast set -euo pipefail SOURCE="${1:-all}" MODE="${2:-fast}" -API="${API:-http://127.0.0.1:8091}" +CONTAINER="${TRADEIN_BACKEND_CONTAINER:-tradein-backend}" LOG="${LOG:-/var/log/tradein-scrape.log}" log() { @@ -22,8 +22,6 @@ log() { } # 5 якорей по районам ЕКБ. lat lon name -# Для Yandex DATE_DESC возвращает одинаковую выдачу регардлесс — но Cian/Avito -# реально фильтруют по координатам, так что multi-anchor нужен для них. ANCHORS=( "56.8400 60.6050 Центр" "56.7950 60.5300 ЮЗ" @@ -39,40 +37,29 @@ build_payload() { yandex) sources_json='["yandex"]' ;; avito) sources_json='["avito"]' ;; cian) sources_json='["cian"]' ;; - all) sources_json='["avito","cian","yandex"]' ;; - *) sources_json='["avito","cian","yandex"]' ;; + n1) sources_json='["n1"]' ;; + all|*) sources_json='["avito","cian","yandex","n1"]' ;; esac if [[ "$mode" == "deep" ]]; then - echo "{\"lat\":$lat,\"lon\":$lon,\"radius_m\":1500,\"sources\":$sources_json,\"multi_room_yandex\":true,\"deep_yandex\":true,\"multi_room_cian\":true}" + echo "{\"lat\":$lat,\"lon\":$lon,\"radius_m\":1500,\"sources\":$sources_json,\"multi_room_yandex\":true,\"deep_yandex\":true,\"multi_room_cian\":true,\"multi_room_n1\":true}" else - echo "{\"lat\":$lat,\"lon\":$lon,\"radius_m\":1500,\"sources\":$sources_json,\"multi_room_yandex\":true,\"multi_room_cian\":true}" + echo "{\"lat\":$lat,\"lon\":$lon,\"radius_m\":1500,\"sources\":$sources_json,\"multi_room_yandex\":true,\"multi_room_cian\":true,\"multi_room_n1\":true}" fi } -log "=== cron-scrape start: source=$SOURCE mode=$MODE ===" +log "=== cron-scrape start: source=$SOURCE mode=$MODE container=$CONTAINER ===" -# Circuit breaker: если Avito IP-rate-limited (3 нуля подряд) — пропускаем оставшиеся. -zero_fetch_streak=0 for anchor in "${ANCHORS[@]}"; do - if [[ "$zero_fetch_streak" -ge 3 ]]; then - log "⚠ circuit-breaker tripped (3× zero-fetch in a row) — skipping remaining anchors" - break - fi read -r lat lon name <<< "$anchor" payload=$(build_payload "$lat" "$lon" "$SOURCE" "$MODE") - log "→ $name ($lat, $lon) payload=$payload" - resp=$(curl -s -X POST "$API/api/v1/admin/scrape" \ + log "→ $name ($lat, $lon)" + # curl выполняется ВНУТРИ backend-контейнера → localhost:8000, без публичного доступа. + resp=$(docker exec "$CONTAINER" curl -s -X POST \ + http://localhost:8000/api/v1/admin/scrape \ -H "Content-Type: application/json" \ -d "$payload" \ - -m 300 || echo '{"error":"curl_timeout"}') + -m 600 2>/dev/null || echo '{"error":"exec_or_timeout"}') log "← $name $resp" - # Парсим total_fetched — если 0, инкремент streak; иначе reset - fetched=$(echo "$resp" | grep -oE '"total_fetched":[0-9]+' | grep -oE '[0-9]+' || echo 0) - if [[ "${fetched:-0}" == "0" ]]; then - zero_fetch_streak=$((zero_fetch_streak + 1)) - else - zero_fetch_streak=0 - fi sleep 5 done