02267d0306
- FastAPI backend: PostGIS estimator + 3 scrapers (Avito/Cian/Yandex)
- Next.js 15 frontend: tradein.html mockup design, basePath=/trade-in
- WeasyPrint PDF (Брусника-style 4-page report)
- Address autocomplete с typo-tolerance + 6 EKB presets
- Изолированный docker stack gendesign-tradein (отдельная postgres БД)
- Caddy inline routes: gendsgn.ru/trade-in/* и /trade-in/api/v1/*
- Forgejo Actions: .forgejo/workflows/deploy-tradein.yml (shell-based GHCR login)
- Триггер только по paths: tradein-mvp/** (не пересекается с deploy.yml)
- Образы: ghcr.io/lekss361/gendesign-tradein-{backend,frontend}:latest
Первый запуск на сервере (вручную, один раз):
- создать /opt/gendesign/tradein-mvp/.env.runtime (postgres pwd, contact email)
- docker network create gendesign_shared (если нет)
- docker compose -p gendesign-tradein up -d
- docker compose -p gendesign exec caddy caddy reload
409 lines
17 KiB
Python
409 lines
17 KiB
Python
"""Avito.ru scraper — парсинг вторички вокруг точки.
|
||
|
||
Стратегия: HTML scrape карточек объявлений + извлечение JSON `__initialState__`
|
||
который Avito вставляет в `<script>` тег.
|
||
|
||
URL pattern (EKB):
|
||
https://www.avito.ru/ekaterinburg/kvartiry/prodam-ASgBAgICAUSSA8YQ
|
||
?geoCoords=56.838,60.605
|
||
&radius=1 # в км
|
||
&s=104 # sort by date
|
||
&p=1 # страница
|
||
|
||
ВАЖНО: Avito банит httpx (403/429) по TLS fingerprint от server IP.
|
||
Используем curl_cffi с impersonate='chrome120' — настоящий Chrome TLS ClientHello.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import logging
|
||
import random
|
||
import re
|
||
from datetime import date, datetime
|
||
from typing import Any
|
||
from urllib.parse import urlencode, urljoin
|
||
|
||
from curl_cffi.requests import AsyncSession
|
||
from selectolax.parser import HTMLParser
|
||
|
||
from app.services.scrapers.base import BaseScraper, ScrapedLot
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
|
||
class AvitoScraper(BaseScraper):
|
||
"""Avito vtorichka parser. Источник = 'avito'.
|
||
|
||
Использует curl_cffi с impersonate=chrome120 для обхода TLS fingerprint бана.
|
||
"""
|
||
|
||
name = "avito"
|
||
base_url = "https://www.avito.ru"
|
||
# Avito жёстко мониторит — спим долго между запросами
|
||
request_delay_sec = 7.0
|
||
|
||
def __init__(self) -> None:
|
||
super().__init__()
|
||
self._cffi: AsyncSession | None = None
|
||
|
||
async def __aenter__(self) -> "AvitoScraper":
|
||
await super().__aenter__()
|
||
self._cffi = AsyncSession(
|
||
impersonate="chrome120",
|
||
timeout=25,
|
||
headers={
|
||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||
"Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8",
|
||
"Cache-Control": "max-age=0",
|
||
"Sec-Fetch-Dest": "document",
|
||
"Sec-Fetch-Mode": "navigate",
|
||
"Sec-Fetch-Site": "none",
|
||
"Sec-Fetch-User": "?1",
|
||
"Upgrade-Insecure-Requests": "1",
|
||
},
|
||
)
|
||
return self
|
||
|
||
async def __aexit__(self, *args: Any) -> None:
|
||
if self._cffi is not None:
|
||
await self._cffi.close()
|
||
await super().__aexit__(*args)
|
||
|
||
# ── Public ──────────────────────────────────────────────────────────────
|
||
async def fetch_around(
|
||
self, lat: float, lon: float, radius_m: int = 1000
|
||
) -> list[ScrapedLot]:
|
||
"""Найти объявления Авито вокруг (lat, lon) в radius_m метрах.
|
||
|
||
Avito работает в км — конвертируем. Минимум 1 км.
|
||
Возвращаем макс 50 объявлений (страница 1).
|
||
"""
|
||
radius_km = max(1, round(radius_m / 1000))
|
||
# Сохраняем центр для fallback заполнения coords (Avito HTML state не даёт coords)
|
||
self._anchor_lat = lat
|
||
self._anchor_lon = lon
|
||
self._anchor_radius_m = radius_m
|
||
|
||
# Strategy A: попробовать через mobile API (проще, меньше защиты)
|
||
try:
|
||
lots = await self._fetch_via_mobile_api(lat, lon, radius_km)
|
||
if lots:
|
||
logger.info("avito mobile_api: %d lots around (%.4f, %.4f)", len(lots), lat, lon)
|
||
return lots
|
||
except Exception as e: # noqa: BLE001
|
||
logger.warning("avito mobile_api failed: %s — falling back to HTML scrape", e)
|
||
|
||
# Strategy B: HTML scrape (web) через curl_cffi
|
||
url = self._build_web_url(lat, lon, radius_km, page=1)
|
||
try:
|
||
assert self._cffi is not None
|
||
response = await self._cffi.get(url)
|
||
if response.status_code != 200:
|
||
logger.warning(
|
||
"avito HTML returned %d for %s", response.status_code, url
|
||
)
|
||
return []
|
||
except Exception: # noqa: BLE001
|
||
logger.exception("avito HTML fetch failed for %s", url)
|
||
return []
|
||
|
||
lots = self._parse_html(response.text, source_url_base=url)
|
||
logger.info("avito html_scrape: %d lots around (%.4f, %.4f)", len(lots), lat, lon)
|
||
await self.sleep_between_requests()
|
||
return lots
|
||
|
||
# ── Strategy A: mobile API ──────────────────────────────────────────────
|
||
async def _fetch_via_mobile_api(
|
||
self, lat: float, lon: float, radius_km: int
|
||
) -> list[ScrapedLot]:
|
||
"""Avito mobile API endpoint: https://m.avito.ru/api/15/items
|
||
|
||
Возвращает JSON с полным structured data — проще, чем HTML scrape.
|
||
Может быть закрыт за auth — обрабатываем в except выше.
|
||
"""
|
||
params = {
|
||
"categoryId": 24, # квартиры
|
||
"locationId": 653240, # ЕКБ (Avito internal)
|
||
"params[201]": 1, # тип: продам (вторичка)
|
||
"geoCoords": f"{lat},{lon}",
|
||
"radius": radius_km,
|
||
"page": 1,
|
||
"limit": 50,
|
||
"sort": "date",
|
||
}
|
||
url = f"https://m.avito.ru/api/15/items?{urlencode(params)}"
|
||
assert self._cffi is not None
|
||
response = await self._cffi.get(url, headers={"Accept": "application/json"})
|
||
if response.status_code != 200:
|
||
return []
|
||
|
||
data = response.json()
|
||
items = data.get("result", {}).get("items", []) if isinstance(data, dict) else []
|
||
return [self._mobile_item_to_lot(item) for item in items if isinstance(item, dict)]
|
||
|
||
def _mobile_item_to_lot(self, item: dict[str, Any]) -> ScrapedLot | None:
|
||
"""Маппинг Avito mobile API item → ScrapedLot.
|
||
|
||
Avito mobile API meanwhile отдаёт title с метаданными квартиры, но
|
||
address/coords могут быть в разных полях — пробуем все.
|
||
"""
|
||
try:
|
||
title = item.get("title", "")
|
||
# price может быть int или dict {value: N}
|
||
price_field = item.get("price")
|
||
if isinstance(price_field, dict):
|
||
price = price_field.get("value")
|
||
else:
|
||
price = price_field
|
||
if not price:
|
||
return None
|
||
|
||
rooms = _extract_rooms_from_title(title)
|
||
area = _extract_area_from_title(title)
|
||
floor, total = _extract_floor_from_title(title)
|
||
|
||
# Address — пробуем все возможные пути
|
||
address = (
|
||
item.get("address")
|
||
or item.get("location", {}).get("name") if isinstance(item.get("location"), dict) else ""
|
||
)
|
||
if not address and isinstance(item.get("geo"), dict):
|
||
address = item["geo"].get("formatted_address") or item["geo"].get("address", "")
|
||
if not address:
|
||
# Avito mobile API часто отдаёт address в "location.references" или "geo.geoReferences"
|
||
geo = item.get("geo") or {}
|
||
if isinstance(geo.get("geoReferences"), list) and geo["geoReferences"]:
|
||
address = geo["geoReferences"][0].get("content", "")
|
||
|
||
# Coords — тоже пробуем все варианты
|
||
coords = item.get("coords") or item.get("geo", {}).get("coords") or {}
|
||
lat = coords.get("lat") if isinstance(coords, dict) else None
|
||
lon = coords.get("lng") or coords.get("lon") if isinstance(coords, dict) else None
|
||
|
||
# Photos
|
||
photos = []
|
||
for img in (item.get("images") or [])[:5]:
|
||
if isinstance(img, dict):
|
||
photos.append(img.get("1280x960") or img.get("636x476") or img.get("864x648", ""))
|
||
elif isinstance(img, str):
|
||
photos.append(img)
|
||
|
||
return ScrapedLot(
|
||
source="avito",
|
||
source_url=urljoin("https://www.avito.ru", item.get("uri", "")),
|
||
source_id=str(item.get("id")),
|
||
address=address or None,
|
||
lat=lat,
|
||
lon=lon,
|
||
rooms=rooms,
|
||
area_m2=area,
|
||
floor=floor,
|
||
total_floors=total,
|
||
price_rub=int(price),
|
||
photo_urls=[p for p in photos if p],
|
||
raw_payload=item, # сохраняем ВСЁ для retrofit и дебага
|
||
)
|
||
except Exception: # noqa: BLE001
|
||
logger.exception("avito mobile item parse failed: %s", item.get("id"))
|
||
return None
|
||
|
||
# ── Strategy B: HTML scrape ─────────────────────────────────────────────
|
||
def _build_web_url(self, lat: float, lon: float, radius_km: int, page: int = 1) -> str:
|
||
params = {
|
||
"geoCoords": f"{lat},{lon}",
|
||
"radius": radius_km,
|
||
"s": 104, # sort by date
|
||
"p": page,
|
||
}
|
||
return (
|
||
f"{self.base_url}/ekaterinburg/kvartiry/prodam-ASgBAgICAUSSA8YQ?{urlencode(params)}"
|
||
)
|
||
|
||
def _parse_html(self, html: str, source_url_base: str) -> list[ScrapedLot]:
|
||
"""Парсим карточки объявлений из HTML.
|
||
|
||
Avito embed-ит JSON в `<script>` теге с initial state — пробуем найти.
|
||
Если структура поменялась — fallback на DOM scrape карточек.
|
||
"""
|
||
# Попробуем сначала __initialState__ JSON
|
||
state = _extract_avito_state(html)
|
||
if state:
|
||
items = _items_from_state(state)
|
||
if items:
|
||
return [
|
||
lot
|
||
for lot in (self._state_item_to_lot(it) for it in items)
|
||
if lot is not None
|
||
]
|
||
|
||
# Fallback: DOM scrape
|
||
tree = HTMLParser(html)
|
||
cards = tree.css('[data-marker="item"]')
|
||
lots: list[ScrapedLot] = []
|
||
for card in cards:
|
||
lot = self._dom_card_to_lot(card, source_url_base)
|
||
if lot is not None:
|
||
lots.append(lot)
|
||
return lots
|
||
|
||
def _state_item_to_lot(self, item: dict[str, Any]) -> ScrapedLot | None:
|
||
"""Парсинг из __initialState__ JSON. Avito отдаёт минимум полей в нём
|
||
(часто только title) — fallback на anchor координаты с jitter."""
|
||
try:
|
||
price = item.get("priceDetailed", {}).get("value") or item.get("price")
|
||
if not price:
|
||
return None
|
||
title = item.get("title", "")
|
||
rooms = _extract_rooms_from_title(title)
|
||
area = _extract_area_from_title(title)
|
||
floor, total = _extract_floor_from_title(title)
|
||
coords = item.get("coords") or {}
|
||
location = item.get("location", {}) if isinstance(item.get("location"), dict) else {}
|
||
|
||
lat = coords.get("lat") if isinstance(coords, dict) else None
|
||
lon = coords.get("lng") if isinstance(coords, dict) else None
|
||
|
||
# Fallback: если Avito state не даёт coords — используем anchor с jitter.
|
||
# Это приближение, но позволяет PostGIS ST_DWithin работать в правильном радиусе.
|
||
if (lat is None or lon is None) and hasattr(self, "_anchor_lat"):
|
||
# Jitter в радиусе scrape — chunk радиуса в координатах (1° lat ≈ 111км)
|
||
jitter_deg = (self._anchor_radius_m / 1000) / 111 # км → градусы
|
||
lat = self._anchor_lat + (random.random() - 0.5) * jitter_deg
|
||
lon = self._anchor_lon + (random.random() - 0.5) * jitter_deg / 2 # коррекция по широте
|
||
|
||
return ScrapedLot(
|
||
source="avito",
|
||
source_url=urljoin("https://www.avito.ru", item.get("urlPath", "")),
|
||
source_id=str(item.get("id")),
|
||
address=location.get("name") or item.get("address") or "Екатеринбург (Avito)",
|
||
lat=lat,
|
||
lon=lon,
|
||
rooms=rooms,
|
||
area_m2=area,
|
||
floor=floor,
|
||
total_floors=total,
|
||
price_rub=int(price),
|
||
photo_urls=[
|
||
img.get("1280x960") or img.get("636x476", "")
|
||
for img in item.get("images", [])[:5]
|
||
if isinstance(img, dict)
|
||
],
|
||
raw_payload=item,
|
||
)
|
||
except Exception: # noqa: BLE001
|
||
return None
|
||
|
||
def _dom_card_to_lot(self, card: Any, source_url_base: str) -> ScrapedLot | None:
|
||
"""Парсинг через DOM (fallback). Avito state часто пустой — items падают сюда."""
|
||
try:
|
||
link_el = card.css_first('a[data-marker="item-title"]')
|
||
if link_el is None:
|
||
return None
|
||
href = link_el.attributes.get("href", "")
|
||
url = urljoin("https://www.avito.ru", href)
|
||
title = link_el.text(strip=True)
|
||
|
||
price_el = card.css_first('meta[itemprop="price"]')
|
||
price = int(price_el.attributes.get("content", 0)) if price_el is not None else 0
|
||
if price <= 0:
|
||
return None
|
||
|
||
rooms = _extract_rooms_from_title(title)
|
||
area = _extract_area_from_title(title)
|
||
floor, total = _extract_floor_from_title(title)
|
||
|
||
addr_el = card.css_first('[class*="geo-address"]')
|
||
address = addr_el.text(strip=True) if addr_el is not None else None
|
||
|
||
# Anchor jitter — Avito DOM не отдаёт точные coords, используем центр scrape ±jitter
|
||
lat = lon = None
|
||
if hasattr(self, "_anchor_lat"):
|
||
jitter_deg = (self._anchor_radius_m / 1000) / 111
|
||
lat = self._anchor_lat + (random.random() - 0.5) * jitter_deg
|
||
lon = self._anchor_lon + (random.random() - 0.5) * jitter_deg / 2
|
||
|
||
return ScrapedLot(
|
||
source="avito",
|
||
source_url=url,
|
||
source_id=card.attributes.get("data-item-id"),
|
||
address=address or "Екатеринбург (Avito)",
|
||
lat=lat,
|
||
lon=lon,
|
||
rooms=rooms,
|
||
area_m2=area,
|
||
floor=floor,
|
||
total_floors=total,
|
||
price_rub=price,
|
||
raw_payload={"title": title},
|
||
)
|
||
except Exception: # noqa: BLE001
|
||
return None
|
||
|
||
|
||
# ── Helpers: title parser ───────────────────────────────────────────────────
|
||
_RE_ROOMS = re.compile(r"(\d)-к\.?\s*(квартира|кв\.)", re.IGNORECASE)
|
||
_RE_STUDIO = re.compile(r"студи[яиюей]", re.IGNORECASE)
|
||
_RE_AREA = re.compile(r"(\d+[.,]?\d*)\s*м[²2]", re.IGNORECASE)
|
||
_RE_FLOOR = re.compile(r"(\d+)\s*/\s*(\d+)\s*эт\.?", re.IGNORECASE)
|
||
|
||
|
||
def _extract_rooms_from_title(title: str) -> int | None:
|
||
if _RE_STUDIO.search(title):
|
||
return 0
|
||
m = _RE_ROOMS.search(title)
|
||
if m:
|
||
return int(m.group(1))
|
||
return None
|
||
|
||
|
||
def _extract_area_from_title(title: str) -> float | None:
|
||
m = _RE_AREA.search(title)
|
||
if m:
|
||
return float(m.group(1).replace(",", "."))
|
||
return None
|
||
|
||
|
||
def _extract_floor_from_title(title: str) -> tuple[int | None, int | None]:
|
||
m = _RE_FLOOR.search(title)
|
||
if m:
|
||
return int(m.group(1)), int(m.group(2))
|
||
return None, None
|
||
|
||
|
||
# ── Avito __initialState__ extraction ──────────────────────────────────────
|
||
_RE_AVITO_STATE = re.compile(
|
||
r"window\.__initialState__\s*=\s*\"([^\"]+)\"", re.DOTALL
|
||
)
|
||
|
||
|
||
def _extract_avito_state(html: str) -> dict[str, Any] | None:
|
||
"""Avito embed-ит initial state как escaped JSON в большом <script>."""
|
||
m = _RE_AVITO_STATE.search(html)
|
||
if not m:
|
||
return None
|
||
try:
|
||
raw = m.group(1).encode("utf-8").decode("unicode_escape")
|
||
return json.loads(raw)
|
||
except (UnicodeDecodeError, json.JSONDecodeError):
|
||
return None
|
||
|
||
|
||
def _items_from_state(state: dict[str, Any]) -> list[dict[str, Any]]:
|
||
"""Вытаскиваем items из state — структура у Avito гуляет, ищем рекурсивно."""
|
||
# типичный путь: state.items.catalog или state.search.results
|
||
for path in [
|
||
("items", "catalog"),
|
||
("search", "results"),
|
||
("catalog", "items"),
|
||
("singleCard", "items"),
|
||
]:
|
||
cursor: Any = state
|
||
for key in path:
|
||
if not isinstance(cursor, dict):
|
||
cursor = None
|
||
break
|
||
cursor = cursor.get(key)
|
||
if isinstance(cursor, list) and cursor:
|
||
return cursor
|
||
return []
|