tradein-mvp/backend/app/api/v1/admin.py

@@ -19,6 +19,7 @@ from app.services.geocoder import geocode
 from app.services.scrapers.avito import AvitoScraper
 from app.services.scrapers.base import save_listings
 from app.services.scrapers.cian import CianScraper
+from app.services.scrapers.n1 import N1Scraper
 from app.services.scrapers.yandex_realty import YandexRealtyScraper
 
 logger = logging.getLogger(__name__)
@@ -30,10 +31,11 @@ class ScrapeRequest(BaseModel):
     lat: float
     lon: float
     radius_m: int = Field(default=1000, ge=100, le=20000)
-    sources: list[Literal["avito", "cian", "yandex"]] = Field(default_factory=lambda: ["avito", "cian", "yandex"])
+    sources: list[Literal["avito", "cian", "yandex", "n1"]] = Field(default_factory=lambda: ["avito", "cian", "yandex", "n1"])
     multi_room_yandex: bool = False  # Если True — скрейп Yandex по 5 сегментам комнат, не одним общим запросом.
     deep_yandex: bool = False  # Если True — Yandex полный обход 5 rooms × 3 sorts × 2 pages = 30 запросов / ~150s.
     multi_room_cian: bool = False  # Если True — скрейп Cian по 4 сегментам комнат отдельно (~4× лотов).
+    multi_room_n1: bool = False  # Если True — скрейп N1 по 4 сегментам комнат отдельно (~4× лотов).
 
 
 class ScrapeResult(BaseModel):
@@ -69,6 +71,7 @@ async def scrape_around(
             "avito": AvitoScraper,
             "cian": CianScraper,
             "yandex": YandexRealtyScraper,
+            "n1": N1Scraper,
         }.get(source)
         if scraper_cls is None:
             continue
@@ -87,6 +90,10 @@ async def scrape_around(
                 lots = await scraper.fetch_around_multi_room(
                     payload.lat, payload.lon, payload.radius_m
                 )
+            elif source == "n1" and payload.multi_room_n1:
+                lots = await scraper.fetch_around_multi_room(
+                    payload.lat, payload.lon, payload.radius_m
+                )
             else:
                 lots = await scraper.fetch_around(
                     payload.lat, payload.lon, payload.radius_m

tradein-mvp/backend/app/services/scrapers/n1.py

@@ -0,0 +1,218 @@
+"""N1.ru scraper — вторичка ЕКБ (ekaterinburg.n1.ru).
+
+N1.ru — региональный портал недвижимости (хорошо представлен на Урале).
+SSR HTML с DOM-карточками `data-test="offers-list-item"`.
+
+Использует curl_cffi (impersonate=chrome120) — как Cian/Avito.
+
+Карточка:
+  - a[href^="/view/NNN/"]  — ссылка на объявление + offer_id
+    текст ссылки = "3-к, Репина, 75/2 стр." (комнаты + адрес)
+  - [data-test="offers-list-item-param-total-area"]  — "79 м2"
+  - [data-test="offers-list-item-param-price"]       — "15 700 000"
+  - img.src — фото
+
+Координаты карточки не отдаёт → anchor+jitter (как Avito).
+"""
+
+from __future__ import annotations
+
+import logging
+import random
+import re
+from typing import Any
+
+from curl_cffi.requests import AsyncSession
+from selectolax.parser import HTMLParser
+
+from app.services.scrapers.base import BaseScraper, ScrapedLot
+
+logger = logging.getLogger(__name__)
+
+
+class N1Scraper(BaseScraper):
+    """N1.ru vtorichka parser. Источник = 'n1'."""
+
+    name = "n1"
+    base_url = "https://ekaterinburg.n1.ru"
+    request_delay_sec = 5.0
+
+    def __init__(self) -> None:
+        super().__init__()
+        self._cffi: AsyncSession | None = None
+
+    async def __aenter__(self) -> "N1Scraper":
+        await super().__aenter__()
+        self._cffi = AsyncSession(
+            impersonate="chrome120",
+            timeout=25,
+            headers={
+                "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+                "Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8",
+                "Sec-Fetch-Dest": "document",
+                "Sec-Fetch-Mode": "navigate",
+                "Sec-Fetch-Site": "none",
+                "Upgrade-Insecure-Requests": "1",
+            },
+        )
+        return self
+
+    async def __aexit__(self, *args: Any) -> None:
+        if self._cffi is not None:
+            await self._cffi.close()
+        await super().__aexit__(*args)
+
+    async def fetch_around(
+        self, lat: float, lon: float, radius_m: int = 1000, rooms: int | None = None,
+    ) -> list[ScrapedLot]:
+        self._anchor_lat = lat
+        self._anchor_lon = lon
+        self._anchor_radius_m = radius_m
+
+        url = self._build_url(rooms)
+        try:
+            assert self._cffi is not None
+            response = await self._cffi.get(url)
+        except Exception:  # noqa: BLE001
+            logger.exception("n1 fetch failed for %s", url)
+            return []
+        if response.status_code != 200:
+            logger.warning("n1 returned %d for %s", response.status_code, url)
+            return []
+
+        lots = self._parse_html(response.text)
+        logger.info(
+            "n1: %d lots around (%.4f, %.4f) rooms=%s", len(lots), lat, lon, rooms
+        )
+        await self.sleep_between_requests()
+        return lots
+
+    async def fetch_around_multi_room(
+        self, lat: float, lon: float, radius_m: int = 1000
+    ) -> list[ScrapedLot]:
+        """Скрейп N1 по сегментам комнат 1/2/3/4 — расширяет выборку."""
+        seen: dict[str, ScrapedLot] = {}
+        for rooms in (1, 2, 3, 4):
+            try:
+                lots = await self.fetch_around(lat, lon, radius_m, rooms=rooms)
+            except Exception:
+                logger.exception("n1 multi-room fetch failed for rooms=%s", rooms)
+                continue
+            for lot in lots:
+                key = lot.source_id or lot.source_url
+                if key and key not in seen:
+                    seen[key] = lot
+        logger.info(
+            "n1 multi-room: %d unique lots around (%.4f, %.4f)", len(seen), lat, lon
+        )
+        return list(seen.values())
+
+    def _build_url(self, rooms: int | None = None) -> str:
+        """URL каталога вторички ЕКБ.
+
+        N1 фильтр комнат: /kupit/kvartiry/vtorichka/1-komnatnye/ итд.
+        """
+        rooms_segment = {
+            1: "1-komnatnye/",
+            2: "2-komnatnye/",
+            3: "3-komnatnye/",
+            4: "4-komnatnye/",
+        }.get(rooms or 0, "")
+        return f"{self.base_url}/kupit/kvartiry/vtorichka/{rooms_segment}"
+
+    def _parse_html(self, html: str) -> list[ScrapedLot]:
+        tree = HTMLParser(html)
+        cards = tree.css('[data-test="offers-list-item"]')
+        return [
+            lot for lot in (self._card_to_lot(c) for c in cards) if lot is not None
+        ]
+
+    def _card_to_lot(self, card: Any) -> ScrapedLot | None:
+        try:
+            # Ссылка на объявление /view/NNN/
+            link = card.css_first('a[href*="/view/"]')
+            if link is None:
+                return None
+            href = link.attributes.get("href", "")
+            m_id = re.search(r"/view/(\d+)/", href)
+            offer_id = m_id.group(1) if m_id else None
+            url = href if href.startswith("http") else f"{self.base_url}{href}"
+
+            # Заголовок ссылки: "3-к, Репина, 75/2 стр."
+            title = link.text(strip=True) or ""
+            rooms = _extract_rooms(title)
+            # Адрес = title без префикса комнат
+            address = re.sub(r"^\s*(\d+-к|студи[яи])\s*,?\s*", "", title, flags=re.IGNORECASE).strip()
+            if not address:
+                address = "Екатеринбург (N1)"
+
+            # Площадь
+            area_el = card.css_first('[data-test="offers-list-item-param-total-area"]')
+            area = _extract_area(area_el.text(strip=True) if area_el else "")
+
+            # Цена
+            price_el = card.css_first('[data-test="offers-list-item-param-price"]')
+            price = _extract_price(price_el.text(strip=True) if price_el else "")
+            if not price:
+                return None
+
+            # Координаты: anchor + jitter (N1 card не отдаёт coords)
+            lat = lon = None
+            if hasattr(self, "_anchor_lat"):
+                jitter_deg = (self._anchor_radius_m / 1000) / 111
+                lat = self._anchor_lat + (random.random() - 0.5) * jitter_deg
+                lon = self._anchor_lon + (random.random() - 0.5) * jitter_deg / 2
+
+            # Фото
+            photos: list[str] = []
+            for img in card.css("img")[:5]:
+                src = img.attributes.get("src", "")
+                if src and src.startswith("http") and src not in photos:
+                    photos.append(src)
+
+            return ScrapedLot(
+                source="n1",
+                source_url=url,
+                source_id=offer_id,
+                address=address,
+                lat=lat,
+                lon=lon,
+                rooms=rooms,
+                area_m2=area,
+                price_rub=price,
+                photo_urls=photos,
+                raw_payload={"title": title},
+            )
+        except Exception:  # noqa: BLE001
+            logger.exception("n1 card parse failed")
+            return None
+
+
+# ── Helpers ─────────────────────────────────────────────────────────────────
+
+_RE_ROOMS = re.compile(r"(\d+)\s*-?\s*к", re.IGNORECASE)
+_RE_STUDIO = re.compile(r"студи", re.IGNORECASE)
+_RE_AREA = re.compile(r"([\d]+(?:[.,]\d+)?)")
+_RE_DIGITS = re.compile(r"\d")
+
+
+def _extract_rooms(text: str) -> int | None:
+    if _RE_STUDIO.search(text):
+        return 0
+    m = _RE_ROOMS.search(text)
+    return int(m.group(1)) if m else None
+
+
+def _extract_area(text: str) -> float | None:
+    m = _RE_AREA.search(text)
+    if m:
+        try:
+            return float(m.group(1).replace(",", "."))
+        except ValueError:
+            return None
+    return None
+
+
+def _extract_price(text: str) -> int | None:
+    digits = "".join(_RE_DIGITS.findall(text))
+    return int(digits) if digits else None

tradein-mvp/deploy/cron-scrape.sh

@@ -1,20 +1,20 @@
 #!/usr/bin/env bash
-# Периодический скрейп Yandex+Avito по 5 якорям ЕКБ.
+# Периодический скрейп всех площадок по 5 якорям ЕКБ.
-# Запускается из cron. Логи в /var/log/tradein-scrape.log
+# Запускается из cron на ПРОД-хосте (46.173.16.127).
+# Запрос к backend идёт через `docker exec tradein-backend curl localhost:8000`
+# — не зависит от Caddy и не выставляет admin/scrape наружу.
 #
-# Usage: ./cron-scrape.sh [yandex|avito|all] [deep|fast]
+# Usage: ./cron-scrape.sh [all|avito|cian|yandex|n1] [deep|fast]
 #
-# Примеры в crontab:
+# Crontab пример:
-#   # Yandex deep раз в 3 часа
+#   0  */3 * * * /opt/gendesign/tradein-mvp/deploy/cron-scrape.sh all deep
-#   0 */3 * * * /opt/tradein-mvp/deploy/cron-scrape.sh yandex deep
+#   30 *   * * * /opt/gendesign/tradein-mvp/deploy/cron-scrape.sh all fast
-#   # Avito быстро каждый час
-#   30 * * * * /opt/tradein-mvp/deploy/cron-scrape.sh avito fast
 
 set -euo pipefail
 
 SOURCE="${1:-all}"
 MODE="${2:-fast}"
-API="${API:-http://127.0.0.1:8091}"
+CONTAINER="${TRADEIN_BACKEND_CONTAINER:-tradein-backend}"
 LOG="${LOG:-/var/log/tradein-scrape.log}"
 
 log() {
@@ -22,8 +22,6 @@ log() {
 }
 
 # 5 якорей по районам ЕКБ. lat lon name
-# Для Yandex DATE_DESC возвращает одинаковую выдачу регардлесс — но Cian/Avito
-# реально фильтруют по координатам, так что multi-anchor нужен для них.
 ANCHORS=(
   "56.8400 60.6050 Центр"
   "56.7950 60.5300 ЮЗ"
@@ -39,40 +37,29 @@ build_payload() {
     yandex) sources_json='["yandex"]' ;;
     avito)  sources_json='["avito"]' ;;
     cian)   sources_json='["cian"]' ;;
-    all)    sources_json='["avito","cian","yandex"]' ;;
+    n1)     sources_json='["n1"]' ;;
-    *)      sources_json='["avito","cian","yandex"]' ;;
+    all|*)  sources_json='["avito","cian","yandex","n1"]' ;;
   esac
   if [[ "$mode" == "deep" ]]; then
-    echo "{\"lat\":$lat,\"lon\":$lon,\"radius_m\":1500,\"sources\":$sources_json,\"multi_room_yandex\":true,\"deep_yandex\":true,\"multi_room_cian\":true}"
+    echo "{\"lat\":$lat,\"lon\":$lon,\"radius_m\":1500,\"sources\":$sources_json,\"multi_room_yandex\":true,\"deep_yandex\":true,\"multi_room_cian\":true,\"multi_room_n1\":true}"
   else
-    echo "{\"lat\":$lat,\"lon\":$lon,\"radius_m\":1500,\"sources\":$sources_json,\"multi_room_yandex\":true,\"multi_room_cian\":true}"
+    echo "{\"lat\":$lat,\"lon\":$lon,\"radius_m\":1500,\"sources\":$sources_json,\"multi_room_yandex\":true,\"multi_room_cian\":true,\"multi_room_n1\":true}"
   fi
 }
 
-log "=== cron-scrape start: source=$SOURCE mode=$MODE ==="
+log "=== cron-scrape start: source=$SOURCE mode=$MODE container=$CONTAINER ==="
 
-# Circuit breaker: если Avito IP-rate-limited (3 нуля подряд) — пропускаем оставшиеся.
-zero_fetch_streak=0
 for anchor in "${ANCHORS[@]}"; do
-  if [[ "$zero_fetch_streak" -ge 3 ]]; then
-    log "⚠ circuit-breaker tripped (3× zero-fetch in a row) — skipping remaining anchors"
-    break
-  fi
   read -r lat lon name <<< "$anchor"
   payload=$(build_payload "$lat" "$lon" "$SOURCE" "$MODE")
-  log "→ $name ($lat, $lon) payload=$payload"
+  log "→ $name ($lat, $lon)"
-  resp=$(curl -s -X POST "$API/api/v1/admin/scrape" \
+  # curl выполняется ВНУТРИ backend-контейнера → localhost:8000, без публичного доступа.
+  resp=$(docker exec "$CONTAINER" curl -s -X POST \
+    http://localhost:8000/api/v1/admin/scrape \
     -H "Content-Type: application/json" \
     -d "$payload" \
-    -m 300 || echo '{"error":"curl_timeout"}')
+    -m 600 2>/dev/null || echo '{"error":"exec_or_timeout"}')
   log "← $name $resp"
-  # Парсим total_fetched — если 0, инкремент streak; иначе reset
-  fetched=$(echo "$resp" | grep -oE '"total_fetched":[0-9]+' | grep -oE '[0-9]+' || echo 0)
-  if [[ "${fetched:-0}" == "0" ]]; then
-    zero_fetch_streak=$((zero_fetch_streak + 1))
-  else
-    zero_fetch_streak=0
-  fi
   sleep 5
 done