feat(tradein): avito_detail.py — detail page parser (30+ fields incl. domoteka)
Stage 2b of AvitoScraper_v2. - fetch_detail(item_url) — async HTTP via curl_cffi.AsyncSession (chrome120) - parse_detail_html(html, source_url) — pure selectolax parser (fixture-testable) - DetailEnrichment dataclass — 30+ fields: * Identity (item_id, title, price, publish_date, views) * Apartment params (rooms/area/floor/balcony_loggia/room_layout/bathroom/windows/repair/sale_type/mortgage) * Location (lat/lon/avito_location_id/metro_stations[]/address_full) * House params (house_type/total_floors_house/lifts/concierge/closed_yard/house_catalog_url) * Description * Domoteka 5 fields (owners_count + owners_at_least + last_owner_change_date + encumbrances_clean + registry_match) * Gallery (photo_urls[]) - save_detail_enrichment(db, e) — UPDATE listings SET 25+ cols WHERE source_id, sets detail_enriched_at=NOW() - pytest test_avito_detail_parse — 10 fixture tests, all passing
This commit is contained in:
lekss361
parent
47f4cc7d58
commit
7a7a72754e
3 changed files with 912 additions and 0 deletions
737
tradein-mvp/backend/app/services/scrapers/avito_detail.py
Normal file
737
tradein-mvp/backend/app/services/scrapers/avito_detail.py
Normal file
|
|
@ -0,0 +1,737 @@
|
|||
"""Avito detail page parser — Stage 2b.
|
||||
|
||||
Извлекает 30+ полей из страницы объявления:
|
||||
- Identity (item_id, title, price, publish_date, views)
|
||||
- Apartment params (rooms, area, floor, balcony_loggia, room_layout,
|
||||
bathroom_type, windows_view, repair_state, sale_type, mortgage_available)
|
||||
- Location (lat, lon, avito_location_id, metro_stations[], address_full)
|
||||
- House params (house_type, total_floors_house, lifts, concierge, closed_yard,
|
||||
house_catalog_url) — собираются но НЕ сохраняются в БД (Stage 2c)
|
||||
- Description
|
||||
- Domoteka 5 полей (owners_count, owners_at_least, last_owner_change_date,
|
||||
encumbrances_clean, registry_match)
|
||||
- Gallery (photo_urls[])
|
||||
|
||||
Точка входа для batch-обогащения:
|
||||
enrichment = await fetch_detail(item_url)
|
||||
saved = save_detail_enrichment(db, enrichment)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import date
|
||||
from typing import Any
|
||||
from urllib.parse import urljoin
|
||||
|
||||
from curl_cffi.requests import AsyncSession
|
||||
from selectolax.parser import HTMLParser, Node
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
AVITO_BASE = "https://www.avito.ru"
|
||||
|
||||
# ── Русские месяцы ─────────────────────────────────────────────────────────
|
||||
RUS_MONTHS: dict[str, int] = {
|
||||
"января": 1,
|
||||
"февраля": 2,
|
||||
"марта": 3,
|
||||
"апреля": 4,
|
||||
"мая": 5,
|
||||
"июня": 6,
|
||||
"июля": 7,
|
||||
"августа": 8,
|
||||
"сентября": 9,
|
||||
"октября": 10,
|
||||
"ноября": 11,
|
||||
"декабря": 12,
|
||||
}
|
||||
|
||||
# ── Regex ────────────────────────────────────────────────────────────────────
|
||||
_ITEM_ID_RE = re.compile(r"№\s*(\d+)")
|
||||
_VIEWS_TOTAL_RE = re.compile(r"(\d+)\s+просмотр")
|
||||
_VIEWS_TODAY_RE = re.compile(r"\+\s*(\d+)\s+сегодня")
|
||||
_PUBLISH_DATE_RE = re.compile(
|
||||
r"(\d{1,2})\s+(января|февраля|марта|апреля|мая|июня|"
|
||||
r"июля|августа|сентября|октября|ноября|декабря)\s+в",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_FLOAT_RE = re.compile(r"[\d.,]+")
|
||||
_INT_RE = re.compile(r"\d+")
|
||||
_FLOOR_SPLIT_RE = re.compile(r"(\d+)\s+из\s+(\d+)")
|
||||
|
||||
OWNERS_RE = re.compile(r"^(\d+)\s+собственник(?:\s+или\s+больше)?", re.IGNORECASE)
|
||||
OWNER_CHANGE_RE = re.compile(
|
||||
r"смена собственника\s+(\d+)\s+(\w+)\s+(\d{4})", re.IGNORECASE
|
||||
)
|
||||
ENCUMBRANCES_RE = re.compile(r"Не\s+найдены\s+ограничения", re.IGNORECASE)
|
||||
REGISTRY_MATCH_RE = re.compile(r"Совпадают\s+площадь", re.IGNORECASE)
|
||||
|
||||
METRO_RE = re.compile(
|
||||
r"([А-ЯЁ][а-яё]+(?:ская|инская|итская|овская|енская))\s*"
|
||||
r"(?:(\d+)[–\-](\d+)|от\s+(\d+))\s*мин",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
# ── Маппинг RUS полей params-блока ───────────────────────────────────────────
|
||||
# (rus_label) -> (field_name, conversion_type)
|
||||
RUS_FIELD_MAP: dict[str, tuple[str, str | None]] = {
|
||||
"Количество комнат": ("rooms", "int"),
|
||||
"Общая площадь": ("area_m2", "float_m2"),
|
||||
"Площадь кухни": ("kitchen_area_m2", "float_m2"),
|
||||
"Этаж": ("floor_split", None),
|
||||
"Балкон или лоджия": ("balcony_loggia", "balcony"),
|
||||
"Тип комнат": ("room_layout", "layout"),
|
||||
"Санузел": ("bathroom_type", "bathroom"),
|
||||
"Окна": ("windows_view", "windows"),
|
||||
"Ремонт": ("repair_state", "repair"),
|
||||
"Способ продажи": ("sale_type", "sale_type"),
|
||||
"Условия продажи": ("sale_conditions_raw", None),
|
||||
"Тип дома": ("house_type", "house_type"),
|
||||
"Этажей в доме": ("total_floors_house", "int"),
|
||||
"Пассажирский лифт": ("passenger_elevators", "int"),
|
||||
"Грузовой лифт": ("cargo_elevators", "int"),
|
||||
"В доме": ("in_house_features_raw", None),
|
||||
"Двор": ("yard_features_raw", None),
|
||||
}
|
||||
|
||||
# ── Enum-маппинги ─────────────────────────────────────────────────────────────
|
||||
_BALCONY_MAP: dict[str, str] = {
|
||||
"балкон": "balcony",
|
||||
"лоджия": "loggia",
|
||||
"балкон, лоджия": "both",
|
||||
"лоджия, балкон": "both",
|
||||
"нет": "none",
|
||||
}
|
||||
|
||||
_LAYOUT_MAP: dict[str, str] = {
|
||||
"изолированные": "isolated",
|
||||
"смежные": "adjacent",
|
||||
"смешанные": "mixed",
|
||||
}
|
||||
|
||||
_BATHROOM_MAP: dict[str, str] = {
|
||||
"совмещенный": "combined",
|
||||
"раздельный": "separate",
|
||||
"совмещенный, раздельный": "both",
|
||||
"раздельный, совмещенный": "both",
|
||||
}
|
||||
|
||||
_WINDOWS_MAP: dict[str, str] = {
|
||||
"во двор": "yard",
|
||||
"на улицу": "street",
|
||||
"во двор и на улицу": "both",
|
||||
"на улицу и во двор": "both",
|
||||
"в парк": "park",
|
||||
}
|
||||
|
||||
_REPAIR_MAP: dict[str, str] = {
|
||||
"косметический": "cosmetic",
|
||||
"евро": "euro",
|
||||
"дизайнерский": "designer",
|
||||
"требуется": "required",
|
||||
"без ремонта": "required",
|
||||
}
|
||||
|
||||
_SALE_TYPE_MAP: dict[str, str] = {
|
||||
"свободная": "free",
|
||||
"альтернатива": "alternative",
|
||||
"аукцион": "auction",
|
||||
}
|
||||
|
||||
_HOUSE_TYPE_MAP: dict[str, str] = {
|
||||
"монолитный": "monolith",
|
||||
"панельный": "panel",
|
||||
"кирпичный": "brick",
|
||||
"монолитно-кирпичный": "monolith_brick",
|
||||
"блочный": "block",
|
||||
"деревянный": "wood",
|
||||
}
|
||||
|
||||
|
||||
# ── DetailEnrichment dataclass ────────────────────────────────────────────────
|
||||
@dataclass
|
||||
class DetailEnrichment:
|
||||
"""Результат парсинга detail page Avito.
|
||||
|
||||
Содержит 30+ полей из страницы объявления.
|
||||
save_detail_enrichment() пишет подмножество в listings (house-level поля пропускаются
|
||||
— они canonical из Houses Catalog Stage 2c).
|
||||
"""
|
||||
|
||||
# Identity
|
||||
item_id: str
|
||||
source_url: str
|
||||
title: str | None = None
|
||||
price_rub: int | None = None
|
||||
publish_date: date | None = None
|
||||
views_total: int | None = None
|
||||
views_today: int | None = None
|
||||
|
||||
# Apartment params
|
||||
rooms: int | None = None
|
||||
area_m2: float | None = None
|
||||
kitchen_area_m2: float | None = None
|
||||
floor: int | None = None
|
||||
total_floors: int | None = None
|
||||
balcony_loggia: str | None = None
|
||||
room_layout: str | None = None
|
||||
bathroom_type: str | None = None
|
||||
windows_view: str | None = None
|
||||
repair_state: str | None = None
|
||||
sale_type: str | None = None
|
||||
mortgage_available: bool | None = None
|
||||
|
||||
# House params (собираем, но НЕ сохраняем в listings — Stage 2c)
|
||||
house_type: str | None = None
|
||||
total_floors_house: int | None = None
|
||||
passenger_elevators: int | None = None
|
||||
cargo_elevators: int | None = None
|
||||
has_concierge: bool | None = None
|
||||
closed_yard: bool | None = None
|
||||
house_catalog_url: str | None = None
|
||||
|
||||
# Location
|
||||
address_full: str | None = None
|
||||
lat: float | None = None
|
||||
lon: float | None = None
|
||||
avito_location_id: int | None = None
|
||||
metro_stations: list[dict[str, Any]] = field(default_factory=list)
|
||||
|
||||
# Description
|
||||
description: str | None = None
|
||||
|
||||
# Domoteka (5 полей)
|
||||
owners_count: int | None = None
|
||||
owners_at_least: bool | None = None
|
||||
last_owner_change_date: date | None = None
|
||||
encumbrances_clean: bool | None = None
|
||||
registry_match: bool | None = None
|
||||
|
||||
# Gallery
|
||||
photo_urls: list[str] = field(default_factory=list)
|
||||
|
||||
# Raw для retrofit / debug
|
||||
raw_html_meta: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
# ── fetch_detail ──────────────────────────────────────────────────────────────
|
||||
async def fetch_detail(
|
||||
item_url: str,
|
||||
*,
|
||||
cffi_session: AsyncSession | None = None,
|
||||
) -> DetailEnrichment:
|
||||
"""GET <avito>/{item_url} → parse HTML via selectolax → DetailEnrichment.
|
||||
|
||||
Если cffi_session не передана — создаёт новую (impersonate='chrome120').
|
||||
Raises:
|
||||
httpx.HTTPError — если status != 200 (через raise_for_status-like).
|
||||
ValueError — если item_id не извлечён из HTML.
|
||||
"""
|
||||
own_session = cffi_session is None
|
||||
session = cffi_session or AsyncSession(
|
||||
impersonate="chrome120",
|
||||
timeout=25,
|
||||
headers={
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
"Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8",
|
||||
"Cache-Control": "max-age=0",
|
||||
"Sec-Fetch-Dest": "document",
|
||||
"Sec-Fetch-Mode": "navigate",
|
||||
"Sec-Fetch-Site": "none",
|
||||
"Sec-Fetch-User": "?1",
|
||||
"Upgrade-Insecure-Requests": "1",
|
||||
},
|
||||
)
|
||||
|
||||
full_url = item_url if item_url.startswith("http") else urljoin(AVITO_BASE, item_url)
|
||||
try:
|
||||
response = await session.get(full_url)
|
||||
if response.status_code != 200:
|
||||
raise ValueError(
|
||||
f"avito detail HTTP {response.status_code} for {full_url}"
|
||||
)
|
||||
enrichment = parse_detail_html(response.text, full_url)
|
||||
return enrichment
|
||||
finally:
|
||||
if own_session:
|
||||
await session.close()
|
||||
|
||||
|
||||
# ── parse_detail_html ─────────────────────────────────────────────────────────
|
||||
def parse_detail_html(html: str, source_url: str) -> DetailEnrichment:
|
||||
"""Pure parser — отдельно для тестов на fixtures без сети.
|
||||
|
||||
Raises:
|
||||
ValueError — если item_id не извлечён.
|
||||
"""
|
||||
tree = HTMLParser(html)
|
||||
|
||||
# ── Identity ────────────────────────────────────────────────────────────
|
||||
item_id = _extract_item_id(tree)
|
||||
if not item_id:
|
||||
raise ValueError(f"Cannot extract item_id from {source_url}")
|
||||
|
||||
title = _text(tree, "[data-marker='item-view/title-info']")
|
||||
price_rub = _extract_price(tree)
|
||||
publish_date, views_total, views_today = _extract_meta(tree)
|
||||
|
||||
# ── Location ────────────────────────────────────────────────────────────
|
||||
map_el = tree.css_first("[data-marker='item-map-wrapper']")
|
||||
lat: float | None = None
|
||||
lon: float | None = None
|
||||
avito_location_id: int | None = None
|
||||
if map_el is not None:
|
||||
lat = _try_float(map_el.attributes.get("data-map-lat"))
|
||||
lon = _try_float(map_el.attributes.get("data-map-lon"))
|
||||
avito_location_id = _try_int(map_el.attributes.get("data-location-id"))
|
||||
|
||||
address_full = _text(tree, "[itemprop='address']")
|
||||
|
||||
# ── Description ─────────────────────────────────────────────────────────
|
||||
description = _text(tree, "[data-marker='item-view/item-description']")
|
||||
|
||||
# ── Metro from description ───────────────────────────────────────────────
|
||||
metro_stations: list[dict[str, Any]] = []
|
||||
if description:
|
||||
metro_stations = _extract_metro(description)
|
||||
|
||||
# ── House catalog link ───────────────────────────────────────────────────
|
||||
house_catalog_url: str | None = None
|
||||
house_link = tree.css_first("[data-marker='nd-jk-details-button']")
|
||||
if house_link is not None:
|
||||
href = house_link.attributes.get("href", "")
|
||||
if href:
|
||||
house_catalog_url = urljoin(AVITO_BASE, href)
|
||||
|
||||
# ── Photo gallery ────────────────────────────────────────────────────────
|
||||
photo_urls: list[str] = []
|
||||
for img in tree.css("[data-marker='image-preview/item'] img"):
|
||||
src = img.attributes.get("src") or ""
|
||||
if src and src not in photo_urls:
|
||||
photo_urls.append(src)
|
||||
|
||||
# ── Params blocks ────────────────────────────────────────────────────────
|
||||
params_block = tree.css_first("[data-marker='item-view/item-params']")
|
||||
apt_params: dict[str, str] = {}
|
||||
house_params: dict[str, str] = {}
|
||||
if params_block is not None:
|
||||
ul_els = params_block.css("ul")
|
||||
if len(ul_els) >= 1:
|
||||
apt_params = _parse_params_ul(ul_els[0])
|
||||
if len(ul_els) >= 2:
|
||||
house_params = _parse_params_ul(ul_els[1])
|
||||
|
||||
# Merge house_params into apt_params — некоторые страницы совмещают всё в одном ul
|
||||
all_params = {**apt_params, **house_params}
|
||||
|
||||
# ── Parse apartment fields ───────────────────────────────────────────────
|
||||
rooms: int | None = None
|
||||
area_m2: float | None = None
|
||||
kitchen_area_m2: float | None = None
|
||||
floor: int | None = None
|
||||
total_floors: int | None = None
|
||||
balcony_loggia: str | None = None
|
||||
room_layout: str | None = None
|
||||
bathroom_type: str | None = None
|
||||
windows_view: str | None = None
|
||||
repair_state: str | None = None
|
||||
sale_type: str | None = None
|
||||
mortgage_available: bool | None = None
|
||||
house_type: str | None = None
|
||||
total_floors_house: int | None = None
|
||||
passenger_elevators: int | None = None
|
||||
cargo_elevators: int | None = None
|
||||
has_concierge: bool | None = None
|
||||
closed_yard: bool | None = None
|
||||
|
||||
for rus_label, val in all_params.items():
|
||||
mapping = RUS_FIELD_MAP.get(rus_label)
|
||||
if mapping is None:
|
||||
continue
|
||||
field_name, _conv = mapping
|
||||
|
||||
if field_name == "rooms":
|
||||
rooms = _try_int(val)
|
||||
elif field_name == "area_m2":
|
||||
area_m2 = _parse_float_m2(val)
|
||||
elif field_name == "kitchen_area_m2":
|
||||
kitchen_area_m2 = _parse_float_m2(val)
|
||||
elif field_name == "floor_split":
|
||||
floor, total_floors = _parse_floor_split(val)
|
||||
elif field_name == "balcony_loggia":
|
||||
balcony_loggia = _map_lower(val, _BALCONY_MAP)
|
||||
elif field_name == "room_layout":
|
||||
room_layout = _map_lower(val, _LAYOUT_MAP)
|
||||
elif field_name == "bathroom_type":
|
||||
bathroom_type = _map_lower(val, _BATHROOM_MAP)
|
||||
elif field_name == "windows_view":
|
||||
windows_view = _map_lower(val, _WINDOWS_MAP)
|
||||
elif field_name == "repair_state":
|
||||
repair_state = _map_lower(val, _REPAIR_MAP)
|
||||
elif field_name == "sale_type":
|
||||
sale_type = _map_lower(val, _SALE_TYPE_MAP)
|
||||
elif field_name == "sale_conditions_raw":
|
||||
if "возможна ипотека" in val.lower():
|
||||
mortgage_available = True
|
||||
elif field_name == "house_type":
|
||||
house_type = _map_lower(val, _HOUSE_TYPE_MAP)
|
||||
elif field_name == "total_floors_house":
|
||||
total_floors_house = _try_int(val)
|
||||
elif field_name == "passenger_elevators":
|
||||
passenger_elevators = _try_int(val)
|
||||
elif field_name == "cargo_elevators":
|
||||
cargo_elevators = _try_int(val)
|
||||
elif field_name == "in_house_features_raw":
|
||||
if "консьерж" in val.lower():
|
||||
has_concierge = True
|
||||
elif field_name == "yard_features_raw":
|
||||
if "закрытая территория" in val.lower():
|
||||
closed_yard = True
|
||||
|
||||
# ── Domoteka ─────────────────────────────────────────────────────────────
|
||||
owners_count: int | None = None
|
||||
owners_at_least: bool | None = None
|
||||
last_owner_change_date: date | None = None
|
||||
encumbrances_clean: bool | None = None
|
||||
registry_match: bool | None = None
|
||||
|
||||
domoteka_block = tree.css_first("[data-marker='domoteka-entry-block']")
|
||||
if domoteka_block is not None:
|
||||
badges = domoteka_block.css("p[data-marker='TeaserData.item']")
|
||||
for badge in badges:
|
||||
badge_text = badge.text(strip=True)
|
||||
if not badge_text:
|
||||
continue
|
||||
|
||||
m_owners = OWNERS_RE.match(badge_text)
|
||||
if m_owners:
|
||||
owners_count = int(m_owners.group(1))
|
||||
owners_at_least = "или больше" in badge_text.lower()
|
||||
continue
|
||||
|
||||
m_change = OWNER_CHANGE_RE.search(badge_text)
|
||||
if m_change:
|
||||
try:
|
||||
day = int(m_change.group(1))
|
||||
month_word = m_change.group(2).lower()
|
||||
year = int(m_change.group(3))
|
||||
month = RUS_MONTHS.get(month_word)
|
||||
if month:
|
||||
last_owner_change_date = date(year, month, day)
|
||||
except (ValueError, KeyError):
|
||||
logger.warning("Cannot parse owner change date: %s", badge_text)
|
||||
continue
|
||||
|
||||
if ENCUMBRANCES_RE.search(badge_text):
|
||||
encumbrances_clean = True
|
||||
continue
|
||||
|
||||
if REGISTRY_MATCH_RE.search(badge_text):
|
||||
registry_match = True
|
||||
continue
|
||||
|
||||
return DetailEnrichment(
|
||||
item_id=item_id,
|
||||
source_url=source_url,
|
||||
title=title,
|
||||
price_rub=price_rub,
|
||||
publish_date=publish_date,
|
||||
views_total=views_total,
|
||||
views_today=views_today,
|
||||
rooms=rooms,
|
||||
area_m2=area_m2,
|
||||
kitchen_area_m2=kitchen_area_m2,
|
||||
floor=floor,
|
||||
total_floors=total_floors,
|
||||
balcony_loggia=balcony_loggia,
|
||||
room_layout=room_layout,
|
||||
bathroom_type=bathroom_type,
|
||||
windows_view=windows_view,
|
||||
repair_state=repair_state,
|
||||
sale_type=sale_type,
|
||||
mortgage_available=mortgage_available,
|
||||
house_type=house_type,
|
||||
total_floors_house=total_floors_house,
|
||||
passenger_elevators=passenger_elevators,
|
||||
cargo_elevators=cargo_elevators,
|
||||
has_concierge=has_concierge,
|
||||
closed_yard=closed_yard,
|
||||
house_catalog_url=house_catalog_url,
|
||||
address_full=address_full,
|
||||
lat=lat,
|
||||
lon=lon,
|
||||
avito_location_id=avito_location_id,
|
||||
metro_stations=metro_stations,
|
||||
description=description,
|
||||
owners_count=owners_count,
|
||||
owners_at_least=owners_at_least,
|
||||
last_owner_change_date=last_owner_change_date,
|
||||
encumbrances_clean=encumbrances_clean,
|
||||
registry_match=registry_match,
|
||||
photo_urls=photo_urls,
|
||||
)
|
||||
|
||||
|
||||
# ── save_detail_enrichment ────────────────────────────────────────────────────
|
||||
def save_detail_enrichment(db: Session, e: DetailEnrichment) -> bool:
|
||||
"""UPDATE listings SET <25+ cols> WHERE source='avito' AND source_id=:item_id.
|
||||
|
||||
Returns True если строка обновлена, False если listing не найден.
|
||||
Устанавливает detail_enriched_at=NOW().
|
||||
House-level поля (passenger_elevators, has_concierge, closed_yard,
|
||||
total_floors_house, house_type) игнорируются — canonical из Houses Catalog (Stage 2c).
|
||||
"""
|
||||
result = db.execute(
|
||||
text("""
|
||||
UPDATE listings SET
|
||||
kitchen_area_m2 = COALESCE(:kitchen_area_m2, kitchen_area_m2),
|
||||
balcony_loggia = COALESCE(:balcony_loggia, balcony_loggia),
|
||||
room_layout = COALESCE(:room_layout, room_layout),
|
||||
bathroom_type = COALESCE(:bathroom_type, bathroom_type),
|
||||
windows_view = COALESCE(:windows_view, windows_view),
|
||||
sale_type = COALESCE(:sale_type, sale_type),
|
||||
mortgage_available = COALESCE(:mortgage_available, mortgage_available),
|
||||
views_total = COALESCE(:views_total, views_total),
|
||||
views_today = COALESCE(:views_today, views_today),
|
||||
publish_date = COALESCE(:publish_date, publish_date),
|
||||
description = COALESCE(:description, description),
|
||||
owners_count = COALESCE(:owners_count, owners_count),
|
||||
owners_at_least = COALESCE(:owners_at_least, owners_at_least),
|
||||
last_owner_change_date = COALESCE(:last_owner_change_date, last_owner_change_date),
|
||||
encumbrances_clean = COALESCE(:encumbrances_clean, encumbrances_clean),
|
||||
registry_match = COALESCE(:registry_match, registry_match),
|
||||
metro_stations = COALESCE(CAST(:metro_stations AS jsonb), metro_stations),
|
||||
avito_location_id = COALESCE(:avito_location_id, avito_location_id),
|
||||
lat = COALESCE(:lat, lat),
|
||||
lon = COALESCE(:lon, lon),
|
||||
address = COALESCE(:address, address),
|
||||
rooms = COALESCE(:rooms, rooms),
|
||||
area_m2 = COALESCE(:area_m2, area_m2),
|
||||
floor = COALESCE(:floor, floor),
|
||||
total_floors = COALESCE(:total_floors, total_floors),
|
||||
repair_state = COALESCE(:repair_state, repair_state),
|
||||
price_rub = COALESCE(:price_rub, price_rub),
|
||||
detail_enriched_at = NOW()
|
||||
WHERE source = 'avito' AND source_id = :item_id
|
||||
"""),
|
||||
{
|
||||
"item_id": e.item_id,
|
||||
"kitchen_area_m2": e.kitchen_area_m2,
|
||||
"balcony_loggia": e.balcony_loggia,
|
||||
"room_layout": e.room_layout,
|
||||
"bathroom_type": e.bathroom_type,
|
||||
"windows_view": e.windows_view,
|
||||
"sale_type": e.sale_type,
|
||||
"mortgage_available": e.mortgage_available,
|
||||
"views_total": e.views_total,
|
||||
"views_today": e.views_today,
|
||||
"publish_date": e.publish_date,
|
||||
"description": e.description,
|
||||
"owners_count": e.owners_count,
|
||||
"owners_at_least": e.owners_at_least,
|
||||
"last_owner_change_date": e.last_owner_change_date,
|
||||
"encumbrances_clean": e.encumbrances_clean,
|
||||
"registry_match": e.registry_match,
|
||||
"metro_stations": (
|
||||
json.dumps(e.metro_stations, ensure_ascii=False) if e.metro_stations else None
|
||||
),
|
||||
"avito_location_id": e.avito_location_id,
|
||||
"lat": e.lat,
|
||||
"lon": e.lon,
|
||||
"address": e.address_full,
|
||||
"rooms": e.rooms,
|
||||
"area_m2": e.area_m2,
|
||||
"floor": e.floor,
|
||||
"total_floors": e.total_floors,
|
||||
"repair_state": e.repair_state,
|
||||
"price_rub": e.price_rub,
|
||||
},
|
||||
)
|
||||
db.commit()
|
||||
found = result.rowcount > 0
|
||||
if not found:
|
||||
logger.warning(
|
||||
"save_detail_enrichment: listing not found in DB for item_id=%s", e.item_id
|
||||
)
|
||||
else:
|
||||
logger.info("save_detail_enrichment: updated listing item_id=%s", e.item_id)
|
||||
return found
|
||||
|
||||
|
||||
# ── Internal helpers ─────────────────────────────────────────────────────────
|
||||
def _text(tree: HTMLParser, selector: str) -> str | None:
|
||||
"""Вернуть stripped text первого совпавшего элемента или None."""
|
||||
el = tree.css_first(selector)
|
||||
if el is None:
|
||||
return None
|
||||
t = el.text(strip=True)
|
||||
return t or None
|
||||
|
||||
|
||||
def _try_float(val: Any) -> float | None:
|
||||
if val is None:
|
||||
return None
|
||||
try:
|
||||
return float(str(val).replace(",", "."))
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def _try_int(val: Any) -> int | None:
|
||||
if val is None:
|
||||
return None
|
||||
s = str(val).strip()
|
||||
m = _INT_RE.search(s)
|
||||
if m:
|
||||
try:
|
||||
return int(m.group())
|
||||
except ValueError:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def _parse_float_m2(val: str) -> float | None:
|
||||
"""'90.1 м²' → 90.1. '75 м²' → 75.0."""
|
||||
m = _FLOAT_RE.search(val)
|
||||
if m:
|
||||
try:
|
||||
return float(m.group().replace(",", "."))
|
||||
except ValueError:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def _parse_floor_split(val: str) -> tuple[int | None, int | None]:
|
||||
"""'20 из 26' → (20, 26)."""
|
||||
m = _FLOOR_SPLIT_RE.search(val)
|
||||
if m:
|
||||
return _try_int(m.group(1)), _try_int(m.group(2))
|
||||
return None, None
|
||||
|
||||
|
||||
def _map_lower(val: str, mapping: dict[str, str]) -> str | None:
|
||||
"""Нижний регистр + strip → lookup в mapping. None если не найдено."""
|
||||
key = val.strip().lower()
|
||||
return mapping.get(key)
|
||||
|
||||
|
||||
def _extract_item_id(tree: HTMLParser) -> str | None:
|
||||
"""Извлечь item_id из [data-marker='item-view/item-id'] text."""
|
||||
el = tree.css_first("[data-marker='item-view/item-id']")
|
||||
if el is None:
|
||||
return None
|
||||
text_val = el.text(strip=True)
|
||||
m = _ITEM_ID_RE.search(text_val)
|
||||
return m.group(1) if m else None
|
||||
|
||||
|
||||
def _extract_price(tree: HTMLParser) -> int | None:
|
||||
"""Извлечь price из [itemprop='price'] content attribute."""
|
||||
el = tree.css_first("[itemprop='price']")
|
||||
if el is None:
|
||||
return None
|
||||
content = el.attributes.get("content")
|
||||
return _try_int(content)
|
||||
|
||||
|
||||
def _extract_meta(tree: HTMLParser) -> tuple[date | None, int | None, int | None]:
|
||||
"""Извлечь publish_date, views_total, views_today из item-id блока."""
|
||||
el = tree.css_first("[data-marker='item-view/item-id']")
|
||||
if el is None:
|
||||
return None, None, None
|
||||
|
||||
# Поиск по всему тексту включая дочерние элементы
|
||||
full_text = el.text(strip=False)
|
||||
|
||||
publish_date: date | None = None
|
||||
m_date = _PUBLISH_DATE_RE.search(full_text)
|
||||
if m_date:
|
||||
try:
|
||||
day = int(m_date.group(1))
|
||||
month_word = m_date.group(2).lower()
|
||||
month = RUS_MONTHS.get(month_word)
|
||||
# Год — текущий (Avito не показывает год для свежих объявлений)
|
||||
import datetime
|
||||
|
||||
current_year = datetime.date.today().year
|
||||
if month:
|
||||
publish_date = date(current_year, month, day)
|
||||
except (ValueError, KeyError):
|
||||
pass
|
||||
|
||||
views_total: int | None = None
|
||||
m_total = _VIEWS_TOTAL_RE.search(full_text)
|
||||
if m_total:
|
||||
views_total = int(m_total.group(1))
|
||||
|
||||
views_today: int | None = None
|
||||
m_today = _VIEWS_TODAY_RE.search(full_text)
|
||||
if m_today:
|
||||
views_today = int(m_today.group(1))
|
||||
|
||||
# Попробуем найти views в отдельных data-marker элементах (новый Avito layout)
|
||||
if views_total is None:
|
||||
v_el = tree.css_first("[data-marker='item-view/total-views']")
|
||||
if v_el is not None:
|
||||
m = _VIEWS_TOTAL_RE.search(v_el.text(strip=True))
|
||||
if m:
|
||||
views_total = int(m.group(1))
|
||||
|
||||
if views_today is None:
|
||||
t_el = tree.css_first("[data-marker='item-view/today-views']")
|
||||
if t_el is not None:
|
||||
m = _VIEWS_TODAY_RE.search(t_el.text(strip=True))
|
||||
if m:
|
||||
views_today = int(m.group(1))
|
||||
|
||||
return publish_date, views_total, views_today
|
||||
|
||||
|
||||
def _parse_params_ul(ul: Node) -> dict[str, str]:
|
||||
"""Парсим <ul> с <li>Label: Value</li> → {label: value} dict."""
|
||||
result: dict[str, str] = {}
|
||||
for li in ul.css("li"):
|
||||
text_val = li.text(strip=True)
|
||||
if ":" in text_val:
|
||||
label, _, value = text_val.partition(":")
|
||||
label = label.strip()
|
||||
value = value.strip()
|
||||
if label and value:
|
||||
result[label] = value
|
||||
return result
|
||||
|
||||
|
||||
def _extract_metro(text_val: str) -> list[dict[str, Any]]:
|
||||
"""Извлечь станции метро из текста описания.
|
||||
|
||||
Примеры: 'Чкаловская 11-15 мин пешком', 'от 5 мин пешком'.
|
||||
"""
|
||||
stations: list[dict[str, Any]] = []
|
||||
for m in METRO_RE.finditer(text_val):
|
||||
name = m.group(1)
|
||||
if m.group(2) and m.group(3):
|
||||
# диапазон: 11-15
|
||||
min_from = int(m.group(2))
|
||||
min_to = int(m.group(3))
|
||||
elif m.group(4):
|
||||
# "от N мин"
|
||||
min_from = None
|
||||
min_to = int(m.group(4))
|
||||
else:
|
||||
continue
|
||||
|
||||
stations.append(
|
||||
{
|
||||
"name": name,
|
||||
"min_to": min_to,
|
||||
"min_from": min_from,
|
||||
"mode": "walk",
|
||||
}
|
||||
)
|
||||
return stations
|
||||
0
tradein-mvp/backend/tests/__init__.py
Normal file
0
tradein-mvp/backend/tests/__init__.py
Normal file
175
tradein-mvp/backend/tests/test_avito_detail_parse.py
Normal file
175
tradein-mvp/backend/tests/test_avito_detail_parse.py
Normal file
|
|
@ -0,0 +1,175 @@
|
|||
"""Smoke тесты для avito_detail parser.
|
||||
|
||||
Загружает minimal HTML fixture и проверяет selector + conversion logic
|
||||
без сетевых запросов.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from app.services.scrapers.avito_detail import DetailEnrichment, parse_detail_html
|
||||
|
||||
MINIMAL_HTML = """
|
||||
<html><body>
|
||||
<div data-marker="item-view/item-id">№ 7986882804 · 18 мая в 13:40 · 1899 просмотров (+ 12 сегодня)</div>
|
||||
<span itemprop="price" content="11990000">11 990 000 ₽</span>
|
||||
<div data-marker="item-map-wrapper" data-map-lat="56.797713" data-map-lon="60.609135" data-location-id="654070"></div>
|
||||
<div data-marker="item-view/item-params">
|
||||
<ul>
|
||||
<li>Количество комнат: 3</li>
|
||||
<li>Общая площадь: 75 м²</li>
|
||||
<li>Площадь кухни: 15 м²</li>
|
||||
<li>Этаж: 20 из 26</li>
|
||||
<li>Балкон или лоджия: лоджия</li>
|
||||
<li>Ремонт: евро</li>
|
||||
<li>Способ продажи: свободная</li>
|
||||
<li>Условия продажи: возможна ипотека</li>
|
||||
</ul>
|
||||
<ul>
|
||||
<li>Тип дома: монолитный</li>
|
||||
<li>Этажей в доме: 26</li>
|
||||
</ul>
|
||||
</div>
|
||||
<a data-marker="nd-jk-details-button" href="/catalog/houses/ekaterinburg/ul_postovskogo_17a/3171365">Дом</a>
|
||||
<div data-marker="domoteka-entry-block">
|
||||
<p data-marker="TeaserData.item">1 собственник</p>
|
||||
<p data-marker="TeaserData.item">Последняя смена собственника 5 мая 2020</p>
|
||||
<p data-marker="TeaserData.item">Не найдены ограничения и обременения</p>
|
||||
<p data-marker="TeaserData.item">Совпадают площадь, адрес и этаж</p>
|
||||
</div>
|
||||
<div data-marker="item-view/item-description">Отличная квартира в новом ЖК. Чкаловская 11-15 мин пешком.</div>
|
||||
</body></html>
|
||||
"""
|
||||
|
||||
SOURCE_URL = "https://www.avito.ru/ekaterinburg/kvartiry/test_7986882804"
|
||||
|
||||
|
||||
def test_minimal_parse() -> None:
|
||||
result = parse_detail_html(MINIMAL_HTML, SOURCE_URL)
|
||||
assert isinstance(result, DetailEnrichment)
|
||||
assert result.item_id == "7986882804"
|
||||
assert result.price_rub == 11990000
|
||||
assert result.rooms == 3
|
||||
assert result.area_m2 == 75.0
|
||||
assert result.kitchen_area_m2 == 15.0
|
||||
assert result.floor == 20
|
||||
assert result.total_floors == 26
|
||||
assert result.balcony_loggia == "loggia"
|
||||
assert result.repair_state == "euro"
|
||||
assert result.sale_type == "free"
|
||||
assert result.mortgage_available is True
|
||||
assert result.house_type == "monolith"
|
||||
assert result.total_floors_house == 26
|
||||
assert result.lat == 56.797713
|
||||
assert result.lon == 60.609135
|
||||
assert result.avito_location_id == 654070
|
||||
assert result.house_catalog_url is not None and "3171365" in result.house_catalog_url
|
||||
assert result.owners_count == 1
|
||||
assert result.owners_at_least is False
|
||||
assert result.last_owner_change_date is not None and result.last_owner_change_date.year == 2020
|
||||
assert result.encumbrances_clean is True
|
||||
assert result.registry_match is True
|
||||
assert result.views_total == 1899
|
||||
assert result.views_today == 12
|
||||
|
||||
|
||||
def test_metro_extraction() -> None:
|
||||
result = parse_detail_html(MINIMAL_HTML, SOURCE_URL)
|
||||
# "Чкаловская 11-15 мин пешком" → metro_stations
|
||||
assert len(result.metro_stations) >= 1
|
||||
station = result.metro_stations[0]
|
||||
assert station["name"] == "Чкаловская"
|
||||
assert station["min_from"] == 11
|
||||
assert station["min_to"] == 15
|
||||
assert station["mode"] == "walk"
|
||||
|
||||
|
||||
def test_owners_at_least_false() -> None:
|
||||
"""Один собственник (без 'или больше') → owners_at_least=False."""
|
||||
result = parse_detail_html(MINIMAL_HTML, SOURCE_URL)
|
||||
assert result.owners_count == 1
|
||||
assert result.owners_at_least is False
|
||||
|
||||
|
||||
def test_owners_at_least_true() -> None:
|
||||
"""'3 собственника или больше' → owners_at_least=True."""
|
||||
html = """
|
||||
<html><body>
|
||||
<div data-marker="item-view/item-id">№ 1234567890</div>
|
||||
<span itemprop="price" content="5000000">5 000 000 ₽</span>
|
||||
<div data-marker="domoteka-entry-block">
|
||||
<p data-marker="TeaserData.item">3 собственника или больше</p>
|
||||
</div>
|
||||
</body></html>
|
||||
"""
|
||||
result = parse_detail_html(html, "https://www.avito.ru/test_1234567890")
|
||||
assert result.owners_count == 3
|
||||
assert result.owners_at_least is True
|
||||
|
||||
|
||||
def test_missing_domoteka_all_none() -> None:
|
||||
"""Если нет domoteka-блока — все 5 полей None."""
|
||||
html = """
|
||||
<html><body>
|
||||
<div data-marker="item-view/item-id">№ 9999999999</div>
|
||||
<span itemprop="price" content="3000000">3 000 000 ₽</span>
|
||||
</body></html>
|
||||
"""
|
||||
result = parse_detail_html(html, "https://www.avito.ru/test_9999999999")
|
||||
assert result.owners_count is None
|
||||
assert result.owners_at_least is None
|
||||
assert result.last_owner_change_date is None
|
||||
assert result.encumbrances_clean is None
|
||||
assert result.registry_match is None
|
||||
|
||||
|
||||
def test_invalid_html_raises_value_error() -> None:
|
||||
"""HTML без item_id → ValueError."""
|
||||
html = "<html><body><p>Нет айди</p></body></html>"
|
||||
with pytest.raises(ValueError, match="Cannot extract item_id"):
|
||||
parse_detail_html(html, "https://www.avito.ru/test")
|
||||
|
||||
|
||||
def test_house_catalog_url_full() -> None:
|
||||
"""house_catalog_url должен быть absolute URL."""
|
||||
result = parse_detail_html(MINIMAL_HTML, SOURCE_URL)
|
||||
assert result.house_catalog_url is not None
|
||||
assert result.house_catalog_url.startswith("https://www.avito.ru")
|
||||
|
||||
|
||||
def test_bathroom_type_combined() -> None:
|
||||
html = """
|
||||
<html><body>
|
||||
<div data-marker="item-view/item-id">№ 1111111111</div>
|
||||
<span itemprop="price" content="4000000">4 000 000 ₽</span>
|
||||
<div data-marker="item-view/item-params">
|
||||
<ul>
|
||||
<li>Санузел: совмещенный</li>
|
||||
<li>Окна: во двор и на улицу</li>
|
||||
<li>Тип комнат: изолированные</li>
|
||||
</ul>
|
||||
</div>
|
||||
</body></html>
|
||||
"""
|
||||
result = parse_detail_html(html, "https://www.avito.ru/test_1111111111")
|
||||
assert result.bathroom_type == "combined"
|
||||
assert result.windows_view == "both"
|
||||
assert result.room_layout == "isolated"
|
||||
|
||||
|
||||
def test_photo_urls_extracted() -> None:
|
||||
html = """
|
||||
<html><body>
|
||||
<div data-marker="item-view/item-id">№ 2222222222</div>
|
||||
<span itemprop="price" content="7000000">7 000 000 ₽</span>
|
||||
<div data-marker="image-preview/item"><img src="https://img.avito.ru/img1.jpg"/></div>
|
||||
<div data-marker="image-preview/item"><img src="https://img.avito.ru/img2.jpg"/></div>
|
||||
</body></html>
|
||||
"""
|
||||
result = parse_detail_html(html, "https://www.avito.ru/test_2222222222")
|
||||
assert len(result.photo_urls) == 2
|
||||
assert result.photo_urls[0] == "https://img.avito.ru/img1.jpg"
|
||||
|
||||
|
||||
def test_source_url_preserved() -> None:
|
||||
result = parse_detail_html(MINIMAL_HTML, SOURCE_URL)
|
||||
assert result.source_url == SOURCE_URL
|
||||
Loading…
Add table
Reference in a new issue