334 lines
14 KiB
Python
334 lines
14 KiB
Python
"""Базовый framework для парсеров объявлений (Avito / Cian / DomKlik / ...).
|
||
|
||
Общие задачи:
|
||
- httpx.AsyncClient с realistic browser headers + UA rotation
|
||
- tenacity retry с exp backoff
|
||
- Sleep между запросами (anti-ban)
|
||
- ScrapedLot — единая Pydantic схема всех источников
|
||
- Запись в `listings` Postgres с дедупом по dedup_hash
|
||
|
||
Каждый конкретный парсер (avito.py, cian.py, ...) наследуется от BaseScraper
|
||
и реализует `_fetch_page()` и `_parse_listings()`.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import hashlib
|
||
import logging
|
||
import random
|
||
from abc import ABC, abstractmethod
|
||
from datetime import date
|
||
from typing import Any
|
||
|
||
import httpx
|
||
from pydantic import BaseModel, Field
|
||
from sqlalchemy import text
|
||
from sqlalchemy.orm import Session
|
||
from tenacity import retry, stop_after_attempt, wait_exponential
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
|
||
# ── Realistic browser User-Agents (Apr 2026 versions) ──────────────────────
|
||
_USER_AGENTS: list[str] = [
|
||
# Chrome on macOS
|
||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36",
|
||
# Chrome on Windows
|
||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36",
|
||
# Firefox on macOS
|
||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 14.5; rv:127.0) Gecko/20100101 Firefox/127.0",
|
||
# Safari on macOS
|
||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 14_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Safari/605.1.15",
|
||
# Edge on Windows
|
||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36 Edg/132.0.0.0",
|
||
]
|
||
|
||
|
||
def random_user_agent() -> str:
|
||
return random.choice(_USER_AGENTS)
|
||
|
||
|
||
# ── Унифицированная схема результата ────────────────────────────────────────
|
||
class ScrapedLot(BaseModel):
|
||
"""Результат парсинга одного объявления.
|
||
|
||
Все источники приводятся к этому виду перед записью в Postgres.
|
||
Поля optional если источник их не отдаёт.
|
||
"""
|
||
|
||
source: str # 'avito' / 'cian' / 'domklik' / ...
|
||
source_url: str # ссылка на оригинал — кликается из UI
|
||
source_id: str | None = None # внутренний id источника (для update)
|
||
|
||
# Локация — должна быть хотя бы address ИЛИ (lat, lon)
|
||
address: str | None = None
|
||
lat: float | None = None
|
||
lon: float | None = None
|
||
|
||
# Параметры квартиры
|
||
rooms: int | None = None # 0 = студия
|
||
area_m2: float | None = None
|
||
floor: int | None = None
|
||
total_floors: int | None = None
|
||
year_built: int | None = None
|
||
house_type: str | None = None
|
||
repair_state: str | None = None
|
||
has_balcony: bool | None = None
|
||
kadastr_num: str | None = None
|
||
|
||
# Avito house linking (Stage 2a — search → houses)
|
||
house_source: str | None = None # 'avito' / 'cian'
|
||
house_ext_id: str | None = None # Avito's '3171365'
|
||
house_url: str | None = None # full URL of /catalog/houses/...
|
||
listing_segment: str | None = None # 'vtorichka' / 'novostroyki'
|
||
|
||
# Цена (обязательно)
|
||
price_rub: int = Field(gt=0)
|
||
price_per_m2: int | None = None
|
||
|
||
# Метаданные
|
||
listing_date: date | None = None
|
||
days_on_market: int | None = None
|
||
photo_urls: list[str] = Field(default_factory=list)
|
||
raw_payload: dict[str, Any] | None = None
|
||
|
||
# Cian-specific extensions (Stage 2 of CianScraper v1)
|
||
living_area_m2: float | None = None
|
||
bedrooms_count: int | None = None
|
||
balconies_count: int | None = None
|
||
loggias_count: int | None = None
|
||
description_minhash: str | None = None
|
||
cadastral_number: str | None = None
|
||
building_cadastral_number: str | None = None
|
||
phones: list[dict] = Field(default_factory=list)
|
||
is_homeowner: bool | None = None
|
||
is_pro_seller: bool | None = None
|
||
bargain_allowed: bool | None = None
|
||
sale_type: str | None = None
|
||
metro_stations: list[dict] = Field(default_factory=list)
|
||
|
||
def compute_dedup_hash(self) -> str:
|
||
"""SHA256(source + (source_id или source_url) + price_rub) — стабильный uniqueness key.
|
||
|
||
Cian URL содержит random `?context=...` токены — поэтому ключ берём
|
||
source_id (offer_id 330200428), а не source_url. Если source_id нет —
|
||
откатываемся на source_url (Yandex, Avito с offerId).
|
||
"""
|
||
h = hashlib.sha256()
|
||
h.update(self.source.encode())
|
||
h.update(b"|")
|
||
key = self.source_id if self.source_id else self.source_url
|
||
h.update((key or "").encode())
|
||
h.update(b"|")
|
||
h.update(str(self.price_rub).encode())
|
||
return h.hexdigest()
|
||
|
||
def compute_price_per_m2(self) -> int | None:
|
||
"""price_per_m2 = price_rub / area_m2 если area задана."""
|
||
if self.area_m2 and self.area_m2 > 0:
|
||
return int(self.price_rub / self.area_m2)
|
||
return None
|
||
|
||
|
||
# ── BaseScraper ─────────────────────────────────────────────────────────────
|
||
class BaseScraper(ABC):
|
||
"""Базовый класс — наследовать для каждого источника.
|
||
|
||
Subclass должен реализовать:
|
||
- `name` (class attr) — 'avito' / 'cian' / ...
|
||
- `_fetch_page(client, ...)` — async GET к источнику
|
||
- `_parse_listings(html_or_json, ...)` — html → list[ScrapedLot]
|
||
"""
|
||
|
||
name: str = "base"
|
||
base_url: str = ""
|
||
request_delay_sec: float = 5.0 # sleep между запросами (anti-ban)
|
||
|
||
def __init__(self) -> None:
|
||
self._client: httpx.AsyncClient | None = None
|
||
|
||
async def __aenter__(self) -> BaseScraper:
|
||
self._client = httpx.AsyncClient(
|
||
timeout=20.0,
|
||
follow_redirects=True,
|
||
headers={
|
||
"User-Agent": random_user_agent(),
|
||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||
"Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8",
|
||
"Accept-Encoding": "gzip, deflate, br",
|
||
"Connection": "keep-alive",
|
||
"Upgrade-Insecure-Requests": "1",
|
||
},
|
||
)
|
||
return self
|
||
|
||
async def __aexit__(self, *args: Any) -> None:
|
||
if self._client is not None:
|
||
await self._client.aclose()
|
||
|
||
@retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=2, min=2, max=30))
|
||
async def _http_get(self, url: str, **kwargs: Any) -> httpx.Response:
|
||
"""GET с retry. Subclass-ы используют это вместо raw httpx."""
|
||
assert self._client is not None, "Use scraper as async context manager"
|
||
response = await self._client.get(url, **kwargs)
|
||
# 403/429 — ретрай поможет (рандомный UA в новом контексте), 5xx тоже
|
||
if response.status_code in {403, 429, 502, 503, 504}:
|
||
response.raise_for_status()
|
||
return response
|
||
|
||
async def sleep_between_requests(self) -> None:
|
||
"""Случайный jitter в районе self.request_delay_sec — чтобы парсер
|
||
не выглядел как detstvo-bot."""
|
||
jitter = random.uniform(0.7, 1.5)
|
||
await asyncio.sleep(self.request_delay_sec * jitter)
|
||
|
||
@abstractmethod
|
||
async def fetch_around(
|
||
self, lat: float, lon: float, radius_m: int = 1000
|
||
) -> list[ScrapedLot]:
|
||
"""Главный метод — собрать объявления вокруг (lat, lon) в радиусе radius_m.
|
||
|
||
Returns:
|
||
Список ScrapedLot. Может быть пустым.
|
||
"""
|
||
...
|
||
|
||
|
||
# ── Запись пачки результатов в Postgres ─────────────────────────────────────
|
||
def save_listings(db: Session, lots: list[ScrapedLot]) -> tuple[int, int]:
|
||
"""Пишем list[ScrapedLot] в `listings` с upsert по dedup_hash.
|
||
|
||
Returns:
|
||
(inserted, updated) — counters для логов.
|
||
"""
|
||
if not lots:
|
||
return 0, 0
|
||
|
||
inserted = 0
|
||
updated = 0
|
||
|
||
for lot in lots:
|
||
ppm2 = lot.price_per_m2 or lot.compute_price_per_m2()
|
||
dedup = lot.compute_dedup_hash()
|
||
|
||
result = db.execute(
|
||
text(
|
||
"""
|
||
INSERT INTO listings (
|
||
source, source_url, source_id, dedup_hash,
|
||
address, lat, lon, region_code,
|
||
rooms, area_m2, floor, total_floors, year_built,
|
||
house_type, repair_state, has_balcony, kadastr_num,
|
||
house_source, house_ext_id, house_url, listing_segment,
|
||
price_rub, price_per_m2,
|
||
listing_date, days_on_market, photo_urls, raw_payload,
|
||
living_area_m2, bedrooms_count, balconies_count, loggias_count,
|
||
description_minhash, cadastral_number, building_cadastral_number,
|
||
phones, is_homeowner, is_pro_seller,
|
||
bargain_allowed, sale_type, metro_stations,
|
||
scraped_at, last_seen_at
|
||
) VALUES (
|
||
:source, :source_url, :source_id, :dedup,
|
||
:address, :lat, :lon, 66,
|
||
:rooms, :area_m2, :floor, :total_floors, :year_built,
|
||
:house_type, :repair_state, :has_balcony, :kadastr,
|
||
:house_source, :house_ext_id, :house_url, :listing_segment,
|
||
:price_rub, :ppm2,
|
||
:listing_date, :days_on_market,
|
||
CAST(:photos AS jsonb),
|
||
CAST(:raw AS jsonb),
|
||
:living_area_m2, :bedrooms_count, :balconies_count, :loggias_count,
|
||
:description_minhash, :cadastral_number, :building_cadastral_number,
|
||
CAST(:phones AS jsonb), :is_homeowner, :is_pro_seller,
|
||
:bargain_allowed, :sale_type, CAST(:metro_stations AS jsonb),
|
||
NOW(), NOW()
|
||
)
|
||
ON CONFLICT (dedup_hash) DO UPDATE
|
||
SET last_seen_at = NOW(),
|
||
is_active = true,
|
||
-- если цена изменилась — обновляем
|
||
price_rub = EXCLUDED.price_rub,
|
||
price_per_m2 = EXCLUDED.price_per_m2,
|
||
-- Cian-specific: обновляем при каждом re-scrape
|
||
living_area_m2 = EXCLUDED.living_area_m2,
|
||
bedrooms_count = EXCLUDED.bedrooms_count,
|
||
balconies_count = EXCLUDED.balconies_count,
|
||
loggias_count = EXCLUDED.loggias_count,
|
||
description_minhash = EXCLUDED.description_minhash,
|
||
cadastral_number = EXCLUDED.cadastral_number,
|
||
building_cadastral_number = EXCLUDED.building_cadastral_number,
|
||
phones = EXCLUDED.phones,
|
||
is_homeowner = EXCLUDED.is_homeowner,
|
||
is_pro_seller = EXCLUDED.is_pro_seller,
|
||
bargain_allowed = EXCLUDED.bargain_allowed,
|
||
sale_type = EXCLUDED.sale_type,
|
||
metro_stations = EXCLUDED.metro_stations
|
||
RETURNING (xmax = 0) AS inserted
|
||
"""
|
||
),
|
||
{
|
||
"source": lot.source,
|
||
"source_url": lot.source_url,
|
||
"source_id": lot.source_id,
|
||
"dedup": dedup,
|
||
"address": lot.address,
|
||
"lat": lot.lat,
|
||
"lon": lot.lon,
|
||
"rooms": lot.rooms,
|
||
"area_m2": lot.area_m2,
|
||
"floor": lot.floor,
|
||
"total_floors": lot.total_floors,
|
||
"year_built": lot.year_built,
|
||
"house_type": lot.house_type,
|
||
"repair_state": lot.repair_state,
|
||
"has_balcony": lot.has_balcony,
|
||
"kadastr": lot.kadastr_num,
|
||
"house_source": lot.house_source,
|
||
"house_ext_id": lot.house_ext_id,
|
||
"house_url": lot.house_url,
|
||
"listing_segment": lot.listing_segment,
|
||
"price_rub": lot.price_rub,
|
||
"ppm2": ppm2,
|
||
"listing_date": lot.listing_date,
|
||
"days_on_market": lot.days_on_market,
|
||
"photos": _to_json(lot.photo_urls),
|
||
"raw": _to_json(lot.raw_payload) if lot.raw_payload else None,
|
||
# Cian-specific columns — None for non-Cian scrapers (→ SQL NULL)
|
||
"living_area_m2": lot.living_area_m2,
|
||
"bedrooms_count": lot.bedrooms_count,
|
||
"balconies_count": lot.balconies_count,
|
||
"loggias_count": lot.loggias_count,
|
||
"description_minhash": lot.description_minhash,
|
||
"cadastral_number": lot.cadastral_number,
|
||
"building_cadastral_number": lot.building_cadastral_number,
|
||
"phones": _to_json(lot.phones) if lot.phones else None,
|
||
"is_homeowner": lot.is_homeowner,
|
||
"is_pro_seller": lot.is_pro_seller,
|
||
"bargain_allowed": lot.bargain_allowed,
|
||
"sale_type": lot.sale_type,
|
||
"metro_stations": _to_json(lot.metro_stations) if lot.metro_stations else None,
|
||
},
|
||
).fetchone()
|
||
|
||
if result is not None and result.inserted:
|
||
inserted += 1
|
||
else:
|
||
updated += 1
|
||
|
||
db.commit()
|
||
logger.info(
|
||
"save_listings: source=%s inserted=%d updated=%d (total %d)",
|
||
lots[0].source if lots else "?",
|
||
inserted,
|
||
updated,
|
||
len(lots),
|
||
)
|
||
return inserted, updated
|
||
|
||
|
||
def _to_json(value: Any) -> str:
|
||
"""JSON serialization helper — для jsonb колонок."""
|
||
import json
|
||
|
||
return json.dumps(value, ensure_ascii=False, default=str)
|