feat(tradein): N1.ru scraper + cron для 4 источников #377
3 changed files with 245 additions and 33 deletions
|
|
@ -19,6 +19,7 @@ from app.services.geocoder import geocode
|
|||
from app.services.scrapers.avito import AvitoScraper
|
||||
from app.services.scrapers.base import save_listings
|
||||
from app.services.scrapers.cian import CianScraper
|
||||
from app.services.scrapers.n1 import N1Scraper
|
||||
from app.services.scrapers.yandex_realty import YandexRealtyScraper
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
@ -30,10 +31,11 @@ class ScrapeRequest(BaseModel):
|
|||
lat: float
|
||||
lon: float
|
||||
radius_m: int = Field(default=1000, ge=100, le=20000)
|
||||
sources: list[Literal["avito", "cian", "yandex"]] = Field(default_factory=lambda: ["avito", "cian", "yandex"])
|
||||
sources: list[Literal["avito", "cian", "yandex", "n1"]] = Field(default_factory=lambda: ["avito", "cian", "yandex", "n1"])
|
||||
multi_room_yandex: bool = False # Если True — скрейп Yandex по 5 сегментам комнат, не одним общим запросом.
|
||||
deep_yandex: bool = False # Если True — Yandex полный обход 5 rooms × 3 sorts × 2 pages = 30 запросов / ~150s.
|
||||
multi_room_cian: bool = False # Если True — скрейп Cian по 4 сегментам комнат отдельно (~4× лотов).
|
||||
multi_room_n1: bool = False # Если True — скрейп N1 по 4 сегментам комнат отдельно (~4× лотов).
|
||||
|
||||
|
||||
class ScrapeResult(BaseModel):
|
||||
|
|
@ -69,6 +71,7 @@ async def scrape_around(
|
|||
"avito": AvitoScraper,
|
||||
"cian": CianScraper,
|
||||
"yandex": YandexRealtyScraper,
|
||||
"n1": N1Scraper,
|
||||
}.get(source)
|
||||
if scraper_cls is None:
|
||||
continue
|
||||
|
|
@ -87,6 +90,10 @@ async def scrape_around(
|
|||
lots = await scraper.fetch_around_multi_room(
|
||||
payload.lat, payload.lon, payload.radius_m
|
||||
)
|
||||
elif source == "n1" and payload.multi_room_n1:
|
||||
lots = await scraper.fetch_around_multi_room(
|
||||
payload.lat, payload.lon, payload.radius_m
|
||||
)
|
||||
else:
|
||||
lots = await scraper.fetch_around(
|
||||
payload.lat, payload.lon, payload.radius_m
|
||||
|
|
|
|||
218
tradein-mvp/backend/app/services/scrapers/n1.py
Normal file
218
tradein-mvp/backend/app/services/scrapers/n1.py
Normal file
|
|
@ -0,0 +1,218 @@
|
|||
"""N1.ru scraper — вторичка ЕКБ (ekaterinburg.n1.ru).
|
||||
|
||||
N1.ru — региональный портал недвижимости (хорошо представлен на Урале).
|
||||
SSR HTML с DOM-карточками `data-test="offers-list-item"`.
|
||||
|
||||
Использует curl_cffi (impersonate=chrome120) — как Cian/Avito.
|
||||
|
||||
Карточка:
|
||||
- a[href^="/view/NNN/"] — ссылка на объявление + offer_id
|
||||
текст ссылки = "3-к, Репина, 75/2 стр." (комнаты + адрес)
|
||||
- [data-test="offers-list-item-param-total-area"] — "79 м2"
|
||||
- [data-test="offers-list-item-param-price"] — "15 700 000"
|
||||
- img.src — фото
|
||||
|
||||
Координаты карточки не отдаёт → anchor+jitter (как Avito).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import random
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from curl_cffi.requests import AsyncSession
|
||||
from selectolax.parser import HTMLParser
|
||||
|
||||
from app.services.scrapers.base import BaseScraper, ScrapedLot
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class N1Scraper(BaseScraper):
|
||||
"""N1.ru vtorichka parser. Источник = 'n1'."""
|
||||
|
||||
name = "n1"
|
||||
base_url = "https://ekaterinburg.n1.ru"
|
||||
request_delay_sec = 5.0
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self._cffi: AsyncSession | None = None
|
||||
|
||||
async def __aenter__(self) -> "N1Scraper":
|
||||
await super().__aenter__()
|
||||
self._cffi = AsyncSession(
|
||||
impersonate="chrome120",
|
||||
timeout=25,
|
||||
headers={
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
"Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8",
|
||||
"Sec-Fetch-Dest": "document",
|
||||
"Sec-Fetch-Mode": "navigate",
|
||||
"Sec-Fetch-Site": "none",
|
||||
"Upgrade-Insecure-Requests": "1",
|
||||
},
|
||||
)
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *args: Any) -> None:
|
||||
if self._cffi is not None:
|
||||
await self._cffi.close()
|
||||
await super().__aexit__(*args)
|
||||
|
||||
async def fetch_around(
|
||||
self, lat: float, lon: float, radius_m: int = 1000, rooms: int | None = None,
|
||||
) -> list[ScrapedLot]:
|
||||
self._anchor_lat = lat
|
||||
self._anchor_lon = lon
|
||||
self._anchor_radius_m = radius_m
|
||||
|
||||
url = self._build_url(rooms)
|
||||
try:
|
||||
assert self._cffi is not None
|
||||
response = await self._cffi.get(url)
|
||||
except Exception: # noqa: BLE001
|
||||
logger.exception("n1 fetch failed for %s", url)
|
||||
return []
|
||||
if response.status_code != 200:
|
||||
logger.warning("n1 returned %d for %s", response.status_code, url)
|
||||
return []
|
||||
|
||||
lots = self._parse_html(response.text)
|
||||
logger.info(
|
||||
"n1: %d lots around (%.4f, %.4f) rooms=%s", len(lots), lat, lon, rooms
|
||||
)
|
||||
await self.sleep_between_requests()
|
||||
return lots
|
||||
|
||||
async def fetch_around_multi_room(
|
||||
self, lat: float, lon: float, radius_m: int = 1000
|
||||
) -> list[ScrapedLot]:
|
||||
"""Скрейп N1 по сегментам комнат 1/2/3/4 — расширяет выборку."""
|
||||
seen: dict[str, ScrapedLot] = {}
|
||||
for rooms in (1, 2, 3, 4):
|
||||
try:
|
||||
lots = await self.fetch_around(lat, lon, radius_m, rooms=rooms)
|
||||
except Exception:
|
||||
logger.exception("n1 multi-room fetch failed for rooms=%s", rooms)
|
||||
continue
|
||||
for lot in lots:
|
||||
key = lot.source_id or lot.source_url
|
||||
if key and key not in seen:
|
||||
seen[key] = lot
|
||||
logger.info(
|
||||
"n1 multi-room: %d unique lots around (%.4f, %.4f)", len(seen), lat, lon
|
||||
)
|
||||
return list(seen.values())
|
||||
|
||||
def _build_url(self, rooms: int | None = None) -> str:
|
||||
"""URL каталога вторички ЕКБ.
|
||||
|
||||
N1 фильтр комнат: /kupit/kvartiry/vtorichka/1-komnatnye/ итд.
|
||||
"""
|
||||
rooms_segment = {
|
||||
1: "1-komnatnye/",
|
||||
2: "2-komnatnye/",
|
||||
3: "3-komnatnye/",
|
||||
4: "4-komnatnye/",
|
||||
}.get(rooms or 0, "")
|
||||
return f"{self.base_url}/kupit/kvartiry/vtorichka/{rooms_segment}"
|
||||
|
||||
def _parse_html(self, html: str) -> list[ScrapedLot]:
|
||||
tree = HTMLParser(html)
|
||||
cards = tree.css('[data-test="offers-list-item"]')
|
||||
return [
|
||||
lot for lot in (self._card_to_lot(c) for c in cards) if lot is not None
|
||||
]
|
||||
|
||||
def _card_to_lot(self, card: Any) -> ScrapedLot | None:
|
||||
try:
|
||||
# Ссылка на объявление /view/NNN/
|
||||
link = card.css_first('a[href*="/view/"]')
|
||||
if link is None:
|
||||
return None
|
||||
href = link.attributes.get("href", "")
|
||||
m_id = re.search(r"/view/(\d+)/", href)
|
||||
offer_id = m_id.group(1) if m_id else None
|
||||
url = href if href.startswith("http") else f"{self.base_url}{href}"
|
||||
|
||||
# Заголовок ссылки: "3-к, Репина, 75/2 стр."
|
||||
title = link.text(strip=True) or ""
|
||||
rooms = _extract_rooms(title)
|
||||
# Адрес = title без префикса комнат
|
||||
address = re.sub(r"^\s*(\d+-к|студи[яи])\s*,?\s*", "", title, flags=re.IGNORECASE).strip()
|
||||
if not address:
|
||||
address = "Екатеринбург (N1)"
|
||||
|
||||
# Площадь
|
||||
area_el = card.css_first('[data-test="offers-list-item-param-total-area"]')
|
||||
area = _extract_area(area_el.text(strip=True) if area_el else "")
|
||||
|
||||
# Цена
|
||||
price_el = card.css_first('[data-test="offers-list-item-param-price"]')
|
||||
price = _extract_price(price_el.text(strip=True) if price_el else "")
|
||||
if not price:
|
||||
return None
|
||||
|
||||
# Координаты: anchor + jitter (N1 card не отдаёт coords)
|
||||
lat = lon = None
|
||||
if hasattr(self, "_anchor_lat"):
|
||||
jitter_deg = (self._anchor_radius_m / 1000) / 111
|
||||
lat = self._anchor_lat + (random.random() - 0.5) * jitter_deg
|
||||
lon = self._anchor_lon + (random.random() - 0.5) * jitter_deg / 2
|
||||
|
||||
# Фото
|
||||
photos: list[str] = []
|
||||
for img in card.css("img")[:5]:
|
||||
src = img.attributes.get("src", "")
|
||||
if src and src.startswith("http") and src not in photos:
|
||||
photos.append(src)
|
||||
|
||||
return ScrapedLot(
|
||||
source="n1",
|
||||
source_url=url,
|
||||
source_id=offer_id,
|
||||
address=address,
|
||||
lat=lat,
|
||||
lon=lon,
|
||||
rooms=rooms,
|
||||
area_m2=area,
|
||||
price_rub=price,
|
||||
photo_urls=photos,
|
||||
raw_payload={"title": title},
|
||||
)
|
||||
except Exception: # noqa: BLE001
|
||||
logger.exception("n1 card parse failed")
|
||||
return None
|
||||
|
||||
|
||||
# ── Helpers ─────────────────────────────────────────────────────────────────
|
||||
|
||||
_RE_ROOMS = re.compile(r"(\d+)\s*-?\s*к", re.IGNORECASE)
|
||||
_RE_STUDIO = re.compile(r"студи", re.IGNORECASE)
|
||||
_RE_AREA = re.compile(r"([\d]+(?:[.,]\d+)?)")
|
||||
_RE_DIGITS = re.compile(r"\d")
|
||||
|
||||
|
||||
def _extract_rooms(text: str) -> int | None:
|
||||
if _RE_STUDIO.search(text):
|
||||
return 0
|
||||
m = _RE_ROOMS.search(text)
|
||||
return int(m.group(1)) if m else None
|
||||
|
||||
|
||||
def _extract_area(text: str) -> float | None:
|
||||
m = _RE_AREA.search(text)
|
||||
if m:
|
||||
try:
|
||||
return float(m.group(1).replace(",", "."))
|
||||
except ValueError:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def _extract_price(text: str) -> int | None:
|
||||
digits = "".join(_RE_DIGITS.findall(text))
|
||||
return int(digits) if digits else None
|
||||
51
tradein-mvp/deploy/cron-scrape.sh
Normal file → Executable file
51
tradein-mvp/deploy/cron-scrape.sh
Normal file → Executable file
|
|
@ -1,20 +1,20 @@
|
|||
#!/usr/bin/env bash
|
||||
# Периодический скрейп Yandex+Avito по 5 якорям ЕКБ.
|
||||
# Запускается из cron. Логи в /var/log/tradein-scrape.log
|
||||
# Периодический скрейп всех площадок по 5 якорям ЕКБ.
|
||||
# Запускается из cron на ПРОД-хосте (46.173.16.127).
|
||||
# Запрос к backend идёт через `docker exec tradein-backend curl localhost:8000`
|
||||
# — не зависит от Caddy и не выставляет admin/scrape наружу.
|
||||
#
|
||||
# Usage: ./cron-scrape.sh [yandex|avito|all] [deep|fast]
|
||||
# Usage: ./cron-scrape.sh [all|avito|cian|yandex|n1] [deep|fast]
|
||||
#
|
||||
# Примеры в crontab:
|
||||
# # Yandex deep раз в 3 часа
|
||||
# 0 */3 * * * /opt/tradein-mvp/deploy/cron-scrape.sh yandex deep
|
||||
# # Avito быстро каждый час
|
||||
# 30 * * * * /opt/tradein-mvp/deploy/cron-scrape.sh avito fast
|
||||
# Crontab пример:
|
||||
# 0 */3 * * * /opt/gendesign/tradein-mvp/deploy/cron-scrape.sh all deep
|
||||
# 30 * * * * /opt/gendesign/tradein-mvp/deploy/cron-scrape.sh all fast
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SOURCE="${1:-all}"
|
||||
MODE="${2:-fast}"
|
||||
API="${API:-http://127.0.0.1:8091}"
|
||||
CONTAINER="${TRADEIN_BACKEND_CONTAINER:-tradein-backend}"
|
||||
LOG="${LOG:-/var/log/tradein-scrape.log}"
|
||||
|
||||
log() {
|
||||
|
|
@ -22,8 +22,6 @@ log() {
|
|||
}
|
||||
|
||||
# 5 якорей по районам ЕКБ. lat lon name
|
||||
# Для Yandex DATE_DESC возвращает одинаковую выдачу регардлесс — но Cian/Avito
|
||||
# реально фильтруют по координатам, так что multi-anchor нужен для них.
|
||||
ANCHORS=(
|
||||
"56.8400 60.6050 Центр"
|
||||
"56.7950 60.5300 ЮЗ"
|
||||
|
|
@ -39,40 +37,29 @@ build_payload() {
|
|||
yandex) sources_json='["yandex"]' ;;
|
||||
avito) sources_json='["avito"]' ;;
|
||||
cian) sources_json='["cian"]' ;;
|
||||
all) sources_json='["avito","cian","yandex"]' ;;
|
||||
*) sources_json='["avito","cian","yandex"]' ;;
|
||||
n1) sources_json='["n1"]' ;;
|
||||
all|*) sources_json='["avito","cian","yandex","n1"]' ;;
|
||||
esac
|
||||
if [[ "$mode" == "deep" ]]; then
|
||||
echo "{\"lat\":$lat,\"lon\":$lon,\"radius_m\":1500,\"sources\":$sources_json,\"multi_room_yandex\":true,\"deep_yandex\":true,\"multi_room_cian\":true}"
|
||||
echo "{\"lat\":$lat,\"lon\":$lon,\"radius_m\":1500,\"sources\":$sources_json,\"multi_room_yandex\":true,\"deep_yandex\":true,\"multi_room_cian\":true,\"multi_room_n1\":true}"
|
||||
else
|
||||
echo "{\"lat\":$lat,\"lon\":$lon,\"radius_m\":1500,\"sources\":$sources_json,\"multi_room_yandex\":true,\"multi_room_cian\":true}"
|
||||
echo "{\"lat\":$lat,\"lon\":$lon,\"radius_m\":1500,\"sources\":$sources_json,\"multi_room_yandex\":true,\"multi_room_cian\":true,\"multi_room_n1\":true}"
|
||||
fi
|
||||
}
|
||||
|
||||
log "=== cron-scrape start: source=$SOURCE mode=$MODE ==="
|
||||
log "=== cron-scrape start: source=$SOURCE mode=$MODE container=$CONTAINER ==="
|
||||
|
||||
# Circuit breaker: если Avito IP-rate-limited (3 нуля подряд) — пропускаем оставшиеся.
|
||||
zero_fetch_streak=0
|
||||
for anchor in "${ANCHORS[@]}"; do
|
||||
if [[ "$zero_fetch_streak" -ge 3 ]]; then
|
||||
log "⚠ circuit-breaker tripped (3× zero-fetch in a row) — skipping remaining anchors"
|
||||
break
|
||||
fi
|
||||
read -r lat lon name <<< "$anchor"
|
||||
payload=$(build_payload "$lat" "$lon" "$SOURCE" "$MODE")
|
||||
log "→ $name ($lat, $lon) payload=$payload"
|
||||
resp=$(curl -s -X POST "$API/api/v1/admin/scrape" \
|
||||
log "→ $name ($lat, $lon)"
|
||||
# curl выполняется ВНУТРИ backend-контейнера → localhost:8000, без публичного доступа.
|
||||
resp=$(docker exec "$CONTAINER" curl -s -X POST \
|
||||
http://localhost:8000/api/v1/admin/scrape \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "$payload" \
|
||||
-m 300 || echo '{"error":"curl_timeout"}')
|
||||
-m 600 2>/dev/null || echo '{"error":"exec_or_timeout"}')
|
||||
log "← $name $resp"
|
||||
# Парсим total_fetched — если 0, инкремент streak; иначе reset
|
||||
fetched=$(echo "$resp" | grep -oE '"total_fetched":[0-9]+' | grep -oE '[0-9]+' || echo 0)
|
||||
if [[ "${fetched:-0}" == "0" ]]; then
|
||||
zero_fetch_streak=$((zero_fetch_streak + 1))
|
||||
else
|
||||
zero_fetch_streak=0
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue