42a0c31f27
Добавлен 4-й источник N1.ru (ekaterinburg.n1.ru) — региональный портал, хорошо представлен на Урале. DOM-парсинг карточек data-test=offers-list-item через curl_cffi. Multi-room сегментация 1/2/3/4к. cron-scrape.sh переписан под прод: запрос к backend через docker exec (без публичного admin/scrape), все 4 источника (avito/cian/yandex/n1).
218 lines
7.9 KiB
Python
218 lines
7.9 KiB
Python
"""N1.ru scraper — вторичка ЕКБ (ekaterinburg.n1.ru).
|
||
|
||
N1.ru — региональный портал недвижимости (хорошо представлен на Урале).
|
||
SSR HTML с DOM-карточками `data-test="offers-list-item"`.
|
||
|
||
Использует curl_cffi (impersonate=chrome120) — как Cian/Avito.
|
||
|
||
Карточка:
|
||
- a[href^="/view/NNN/"] — ссылка на объявление + offer_id
|
||
текст ссылки = "3-к, Репина, 75/2 стр." (комнаты + адрес)
|
||
- [data-test="offers-list-item-param-total-area"] — "79 м2"
|
||
- [data-test="offers-list-item-param-price"] — "15 700 000"
|
||
- img.src — фото
|
||
|
||
Координаты карточки не отдаёт → anchor+jitter (как Avito).
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import logging
|
||
import random
|
||
import re
|
||
from typing import Any
|
||
|
||
from curl_cffi.requests import AsyncSession
|
||
from selectolax.parser import HTMLParser
|
||
|
||
from app.services.scrapers.base import BaseScraper, ScrapedLot
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
|
||
class N1Scraper(BaseScraper):
|
||
"""N1.ru vtorichka parser. Источник = 'n1'."""
|
||
|
||
name = "n1"
|
||
base_url = "https://ekaterinburg.n1.ru"
|
||
request_delay_sec = 5.0
|
||
|
||
def __init__(self) -> None:
|
||
super().__init__()
|
||
self._cffi: AsyncSession | None = None
|
||
|
||
async def __aenter__(self) -> "N1Scraper":
|
||
await super().__aenter__()
|
||
self._cffi = AsyncSession(
|
||
impersonate="chrome120",
|
||
timeout=25,
|
||
headers={
|
||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||
"Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8",
|
||
"Sec-Fetch-Dest": "document",
|
||
"Sec-Fetch-Mode": "navigate",
|
||
"Sec-Fetch-Site": "none",
|
||
"Upgrade-Insecure-Requests": "1",
|
||
},
|
||
)
|
||
return self
|
||
|
||
async def __aexit__(self, *args: Any) -> None:
|
||
if self._cffi is not None:
|
||
await self._cffi.close()
|
||
await super().__aexit__(*args)
|
||
|
||
async def fetch_around(
|
||
self, lat: float, lon: float, radius_m: int = 1000, rooms: int | None = None,
|
||
) -> list[ScrapedLot]:
|
||
self._anchor_lat = lat
|
||
self._anchor_lon = lon
|
||
self._anchor_radius_m = radius_m
|
||
|
||
url = self._build_url(rooms)
|
||
try:
|
||
assert self._cffi is not None
|
||
response = await self._cffi.get(url)
|
||
except Exception: # noqa: BLE001
|
||
logger.exception("n1 fetch failed for %s", url)
|
||
return []
|
||
if response.status_code != 200:
|
||
logger.warning("n1 returned %d for %s", response.status_code, url)
|
||
return []
|
||
|
||
lots = self._parse_html(response.text)
|
||
logger.info(
|
||
"n1: %d lots around (%.4f, %.4f) rooms=%s", len(lots), lat, lon, rooms
|
||
)
|
||
await self.sleep_between_requests()
|
||
return lots
|
||
|
||
async def fetch_around_multi_room(
|
||
self, lat: float, lon: float, radius_m: int = 1000
|
||
) -> list[ScrapedLot]:
|
||
"""Скрейп N1 по сегментам комнат 1/2/3/4 — расширяет выборку."""
|
||
seen: dict[str, ScrapedLot] = {}
|
||
for rooms in (1, 2, 3, 4):
|
||
try:
|
||
lots = await self.fetch_around(lat, lon, radius_m, rooms=rooms)
|
||
except Exception:
|
||
logger.exception("n1 multi-room fetch failed for rooms=%s", rooms)
|
||
continue
|
||
for lot in lots:
|
||
key = lot.source_id or lot.source_url
|
||
if key and key not in seen:
|
||
seen[key] = lot
|
||
logger.info(
|
||
"n1 multi-room: %d unique lots around (%.4f, %.4f)", len(seen), lat, lon
|
||
)
|
||
return list(seen.values())
|
||
|
||
def _build_url(self, rooms: int | None = None) -> str:
|
||
"""URL каталога вторички ЕКБ.
|
||
|
||
N1 фильтр комнат: /kupit/kvartiry/vtorichka/1-komnatnye/ итд.
|
||
"""
|
||
rooms_segment = {
|
||
1: "1-komnatnye/",
|
||
2: "2-komnatnye/",
|
||
3: "3-komnatnye/",
|
||
4: "4-komnatnye/",
|
||
}.get(rooms or 0, "")
|
||
return f"{self.base_url}/kupit/kvartiry/vtorichka/{rooms_segment}"
|
||
|
||
def _parse_html(self, html: str) -> list[ScrapedLot]:
|
||
tree = HTMLParser(html)
|
||
cards = tree.css('[data-test="offers-list-item"]')
|
||
return [
|
||
lot for lot in (self._card_to_lot(c) for c in cards) if lot is not None
|
||
]
|
||
|
||
def _card_to_lot(self, card: Any) -> ScrapedLot | None:
|
||
try:
|
||
# Ссылка на объявление /view/NNN/
|
||
link = card.css_first('a[href*="/view/"]')
|
||
if link is None:
|
||
return None
|
||
href = link.attributes.get("href", "")
|
||
m_id = re.search(r"/view/(\d+)/", href)
|
||
offer_id = m_id.group(1) if m_id else None
|
||
url = href if href.startswith("http") else f"{self.base_url}{href}"
|
||
|
||
# Заголовок ссылки: "3-к, Репина, 75/2 стр."
|
||
title = link.text(strip=True) or ""
|
||
rooms = _extract_rooms(title)
|
||
# Адрес = title без префикса комнат
|
||
address = re.sub(r"^\s*(\d+-к|студи[яи])\s*,?\s*", "", title, flags=re.IGNORECASE).strip()
|
||
if not address:
|
||
address = "Екатеринбург (N1)"
|
||
|
||
# Площадь
|
||
area_el = card.css_first('[data-test="offers-list-item-param-total-area"]')
|
||
area = _extract_area(area_el.text(strip=True) if area_el else "")
|
||
|
||
# Цена
|
||
price_el = card.css_first('[data-test="offers-list-item-param-price"]')
|
||
price = _extract_price(price_el.text(strip=True) if price_el else "")
|
||
if not price:
|
||
return None
|
||
|
||
# Координаты: anchor + jitter (N1 card не отдаёт coords)
|
||
lat = lon = None
|
||
if hasattr(self, "_anchor_lat"):
|
||
jitter_deg = (self._anchor_radius_m / 1000) / 111
|
||
lat = self._anchor_lat + (random.random() - 0.5) * jitter_deg
|
||
lon = self._anchor_lon + (random.random() - 0.5) * jitter_deg / 2
|
||
|
||
# Фото
|
||
photos: list[str] = []
|
||
for img in card.css("img")[:5]:
|
||
src = img.attributes.get("src", "")
|
||
if src and src.startswith("http") and src not in photos:
|
||
photos.append(src)
|
||
|
||
return ScrapedLot(
|
||
source="n1",
|
||
source_url=url,
|
||
source_id=offer_id,
|
||
address=address,
|
||
lat=lat,
|
||
lon=lon,
|
||
rooms=rooms,
|
||
area_m2=area,
|
||
price_rub=price,
|
||
photo_urls=photos,
|
||
raw_payload={"title": title},
|
||
)
|
||
except Exception: # noqa: BLE001
|
||
logger.exception("n1 card parse failed")
|
||
return None
|
||
|
||
|
||
# ── Helpers ─────────────────────────────────────────────────────────────────
|
||
|
||
_RE_ROOMS = re.compile(r"(\d+)\s*-?\s*к", re.IGNORECASE)
|
||
_RE_STUDIO = re.compile(r"студи", re.IGNORECASE)
|
||
_RE_AREA = re.compile(r"([\d]+(?:[.,]\d+)?)")
|
||
_RE_DIGITS = re.compile(r"\d")
|
||
|
||
|
||
def _extract_rooms(text: str) -> int | None:
|
||
if _RE_STUDIO.search(text):
|
||
return 0
|
||
m = _RE_ROOMS.search(text)
|
||
return int(m.group(1)) if m else None
|
||
|
||
|
||
def _extract_area(text: str) -> float | None:
|
||
m = _RE_AREA.search(text)
|
||
if m:
|
||
try:
|
||
return float(m.group(1).replace(",", "."))
|
||
except ValueError:
|
||
return None
|
||
return None
|
||
|
||
|
||
def _extract_price(text: str) -> int | None:
|
||
digits = "".join(_RE_DIGITS.findall(text))
|
||
return int(digits) if digits else None
|