feat(scraper-kit): copy yandex providers with protocol injection, strangler (#2133 yandex) #2158
7 changed files with 3237 additions and 0 deletions
|
|
@ -0,0 +1,364 @@
|
|||
"""Golden-parity: `scraper_kit.providers.yandex.*` парсинг ≡ `app.services.scrapers.yandex_*`.
|
||||
|
||||
Strangler-инвариант (#2133): новая scraper_kit-копия yandex-провайдера должна давать
|
||||
БАЙТ-ИДЕНТИЧНЫЙ результат парсинга старому боевому коду на одинаковом входе.
|
||||
Развязка (settings→ScraperConfig, get_scraper_delay→delay_provider,
|
||||
house_type_normalizer→providers.yandex.shared, base/browser_fetcher/helpers→scraper_kit.*)
|
||||
НЕ меняет распарсенные данные.
|
||||
|
||||
Гоняем ОБА модуля на одних фикстурах и сравниваем результат:
|
||||
- SERP `_entity_to_lot` (gate-API entity → ScrapedLot) на репрезентативных entity;
|
||||
- SERP `_parse_gate_json` (payload → list[ScrapedLot]) с новостройкой/вторичкой;
|
||||
- `normalize_house_type` (SCREAMING/camelCase → канон);
|
||||
- detail `parse` (INITIAL_STATE.offerCard.card + JSON-LD → DetailEnrichment) на
|
||||
реальных offer-HTML фикстурах;
|
||||
- valuation `parse` (house-meta + history → YandexValuationResult) на синтетике;
|
||||
- newbuilding `parse` (ЖК-лендинг → YandexNewbuildingInfo) на синтетике;
|
||||
- `_build_url` (gate-API URL builder) — детерминированность.
|
||||
|
||||
Идентичность результата = kit-парсер верно скопирован (развязка не задела логику).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import inspect
|
||||
import os
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
# Старый app.services.scrapers.yandex_* импортирует app.core.config.settings=Settings(),
|
||||
# которому нужен DATABASE_URL. Офлайн-парсинг БД не трогает — фиктивный DSN достаточен.
|
||||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
|
||||
|
||||
# НОВЫЙ (scraper_kit) и СТАРЫЙ (app.services.scrapers) yandex-парсеры — оба в одном
|
||||
# тесте; ruff сортирует scraper_kit (first-party) выше app-импортов.
|
||||
from scraper_kit.providers.yandex.detail import YandexDetailScraper as NewDetailScraper
|
||||
from scraper_kit.providers.yandex.newbuilding import (
|
||||
YandexNewbuildingScraper as NewNewbuildingScraper,
|
||||
)
|
||||
from scraper_kit.providers.yandex.serp import YandexRealtyScraper as NewRealtyScraper
|
||||
from scraper_kit.providers.yandex.serp import _entity_to_lot as new_entity_to_lot
|
||||
from scraper_kit.providers.yandex.serp import _parse_gate_json as new_parse_gate_json
|
||||
from scraper_kit.providers.yandex.shared import normalize_house_type as new_normalize_house_type
|
||||
from scraper_kit.providers.yandex.valuation import (
|
||||
YandexValuationScraper as NewValuationScraper,
|
||||
)
|
||||
|
||||
from app.services.scrapers.house_type_normalizer import (
|
||||
normalize_house_type as old_normalize_house_type,
|
||||
)
|
||||
from app.services.scrapers.yandex_detail import YandexDetailScraper as OldDetailScraper
|
||||
from app.services.scrapers.yandex_newbuilding import (
|
||||
YandexNewbuildingScraper as OldNewbuildingScraper,
|
||||
)
|
||||
from app.services.scrapers.yandex_realty import YandexRealtyScraper as OldRealtyScraper
|
||||
from app.services.scrapers.yandex_realty import _entity_to_lot as old_entity_to_lot
|
||||
from app.services.scrapers.yandex_realty import _parse_gate_json as old_parse_gate_json
|
||||
from app.services.scrapers.yandex_valuation import (
|
||||
YandexValuationScraper as OldValuationScraper,
|
||||
)
|
||||
|
||||
_FIXTURES = Path(__file__).parent / "fixtures"
|
||||
|
||||
# Инжектируемый конфиг для kit-скрапперов (duck-typed namespace, structurally
|
||||
# удовлетворяет scraper_kit.contracts.ScraperConfig в объёме читаемых полей).
|
||||
_KIT_CONFIG = SimpleNamespace(
|
||||
scraper_proxy_url=None,
|
||||
yandex_proxy_rotate_url=None,
|
||||
avito_proxy_rotate_url=None,
|
||||
)
|
||||
|
||||
|
||||
# ── Fixtures: gate-API entities (mirror test_yandex_realty_serp) ─────────────
|
||||
|
||||
_ENTITY_FULL: dict = {
|
||||
"offerId": 4740475460451078271,
|
||||
"url": "//realty.yandex.ru/offer/4740475460451078271",
|
||||
"price": {"currency": "RUR", "value": 4500000, "valuePerPart": 119048},
|
||||
"area": {"value": 37.8, "unit": "SQUARE_METER"},
|
||||
"livingSpace": {"value": 20, "unit": "SQUARE_METER"},
|
||||
"kitchenSpace": {"value": 12.7, "unit": "SQUARE_METER"},
|
||||
"roomsTotal": 1,
|
||||
"floorsOffered": [8],
|
||||
"floorsTotal": 25,
|
||||
"ceilingHeight": 2.7,
|
||||
"building": {
|
||||
"builtYear": 2016,
|
||||
"buildingType": "MONOLIT",
|
||||
"siteId": 280938,
|
||||
"siteName": "Peremen",
|
||||
},
|
||||
"location": {
|
||||
"geocoderAddress": "Ekaterinburg, ulitsa Evgenia Savkova, 8",
|
||||
"point": {"latitude": 56.79512, "longitude": 60.49186},
|
||||
},
|
||||
"mainImages": [
|
||||
"//avatars.mds.yandex.net/get-realty-offers/11904018/abc/main",
|
||||
"//avatars.mds.yandex.net/get-realty-offers/14717586/def/main",
|
||||
],
|
||||
}
|
||||
|
||||
_ENTITY_STUDIO: dict = {
|
||||
"offerId": 9999999,
|
||||
"url": "//realty.yandex.ru/offer/9999999",
|
||||
"price": {"value": 3200000, "valuePerPart": 114285},
|
||||
"area": {"value": 28.0},
|
||||
"livingSpace": None,
|
||||
"kitchenSpace": None,
|
||||
"roomsTotal": None, # studio -> rooms=0
|
||||
"floorsOffered": [2],
|
||||
"floorsTotal": 9,
|
||||
"ceilingHeight": None,
|
||||
"building": {"builtYear": 2010, "buildingType": "PANEL", "siteId": None, "siteName": None},
|
||||
"location": {
|
||||
"geocoderAddress": "Ekaterinburg, ulitsa Lenina, 5",
|
||||
"point": {"latitude": 56.838, "longitude": 60.597},
|
||||
},
|
||||
"mainImages": [],
|
||||
}
|
||||
|
||||
_ENTITY_RICH_OWNER: dict = {
|
||||
"offerId": 5500000000000000001,
|
||||
"url": "//realty.yandex.ru/offer/5500000000000000001",
|
||||
"creationDate": "2026-05-24T15:10:27Z",
|
||||
"description": "Продаётся светлая квартира с видом на парк.",
|
||||
"author": {"category": "OWNER", "agentName": "Иван", "organization": None},
|
||||
"price": {"value": 7200000, "valuePerPart": 150000, "trend": "DECREASED", "previous": 7500000},
|
||||
"predictions": {"predictedPrice": {"value": "7554000", "min": "7100000", "max": "8000000"}},
|
||||
"area": {"value": 48.0},
|
||||
"roomsTotal": 2,
|
||||
"floorsOffered": [5],
|
||||
"floorsTotal": 12,
|
||||
"building": {"builtYear": 2018, "buildingType": "MONOLIT"},
|
||||
"location": {
|
||||
"geocoderAddress": "Ekaterinburg, ulitsa Mira, 10",
|
||||
"point": {"latitude": 56.84, "longitude": 60.61},
|
||||
"metroList": [
|
||||
{"name": "Ploshchad 1905 goda", "timeToMetro": 7},
|
||||
{"name": "Geologicheskaya", "timeToMetro": 12},
|
||||
],
|
||||
},
|
||||
"mainImages": [],
|
||||
}
|
||||
|
||||
_ENTITY_RICH_AGENCY: dict = {
|
||||
"offerId": 5500000000000000002,
|
||||
"url": "//realty.yandex.ru/offer/5500000000000000002",
|
||||
"creationDate": "2026-06-01T09:00:00Z",
|
||||
"description": "Агентская продажа.",
|
||||
"author": {"category": "AGENCY", "agentName": "Петров", "organization": "Агентство «Диал»"},
|
||||
"price": {"value": 9900000, "trend": "UNCHANGED"},
|
||||
"area": {"value": 60.0},
|
||||
"roomsTotal": 3,
|
||||
"floorsOffered": [3],
|
||||
"floorsTotal": 9,
|
||||
"building": {"builtYear": 2012, "buildingType": "BRICK"},
|
||||
"location": {
|
||||
"geocoderAddress": "Ekaterinburg, ulitsa Lenina, 1",
|
||||
"point": {"latitude": 56.83, "longitude": 60.6},
|
||||
"metro": {"name": "Dinamo", "timeToMetro": 4},
|
||||
},
|
||||
"mainImages": [],
|
||||
}
|
||||
|
||||
_ALL_ENTITIES = [_ENTITY_FULL, _ENTITY_STUDIO, _ENTITY_RICH_OWNER, _ENTITY_RICH_AGENCY]
|
||||
|
||||
|
||||
def _make_gate_payload(entities: list, pager: dict | None = None) -> dict:
|
||||
if pager is None:
|
||||
pager = {"page": 0, "pageSize": 20, "totalItems": len(entities), "totalPages": 1}
|
||||
return {"response": {"search": {"offers": {"entities": entities, "pager": pager}}}}
|
||||
|
||||
|
||||
def _dump_lots(lots: list[Any]) -> list[dict[str, Any]]:
|
||||
return [lot.model_dump() for lot in lots]
|
||||
|
||||
|
||||
# ── SERP _entity_to_lot parity ───────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_entity_to_lot_parity_all_entities() -> None:
|
||||
"""gate-API entity → ScrapedLot идентичен на всех репрезентативных entity."""
|
||||
for entity in _ALL_ENTITIES:
|
||||
old_lot = old_entity_to_lot(entity)
|
||||
new_lot = new_entity_to_lot(entity)
|
||||
assert (old_lot is None) == (new_lot is None)
|
||||
assert old_lot is not None and new_lot is not None
|
||||
assert old_lot.model_dump() == new_lot.model_dump()
|
||||
|
||||
|
||||
def test_entity_to_lot_parity_new_flat_segments() -> None:
|
||||
"""new_flat=YES/NO → listing_segment идентичен (новостройка/вторичка)."""
|
||||
for new_flat in ("NO", "YES"):
|
||||
old_lot = old_entity_to_lot(_ENTITY_FULL, new_flat=new_flat)
|
||||
new_lot = new_entity_to_lot(_ENTITY_FULL, new_flat=new_flat)
|
||||
assert old_lot is not None and new_lot is not None
|
||||
assert old_lot.model_dump() == new_lot.model_dump()
|
||||
|
||||
|
||||
def test_entity_to_lot_parity_ceiling_out_of_range() -> None:
|
||||
"""ceilingHeight мусор (18 / 1.6) → одинаково дропается в None (#2007)."""
|
||||
for bad_ceiling in (18, 1.6):
|
||||
entity = {**_ENTITY_FULL, "ceilingHeight": bad_ceiling}
|
||||
old_lot = old_entity_to_lot(entity)
|
||||
new_lot = new_entity_to_lot(entity)
|
||||
assert old_lot is not None and new_lot is not None
|
||||
assert old_lot.model_dump() == new_lot.model_dump()
|
||||
|
||||
|
||||
# ── SERP _parse_gate_json parity ─────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_parse_gate_json_parity() -> None:
|
||||
"""payload → list[ScrapedLot] идентичен (вторичка и новостройка)."""
|
||||
payload = _make_gate_payload(_ALL_ENTITIES)
|
||||
for new_flat in ("NO", "YES"):
|
||||
old_lots = old_parse_gate_json(payload, page_param=2, new_flat=new_flat)
|
||||
new_lots = new_parse_gate_json(payload, page_param=2, new_flat=new_flat)
|
||||
assert len(old_lots) == len(new_lots) > 0
|
||||
assert _dump_lots(old_lots) == _dump_lots(new_lots)
|
||||
|
||||
|
||||
# ── normalize_house_type parity ──────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_normalize_house_type_parity() -> None:
|
||||
"""SCREAMING / camelCase / канон / мусор → идентичный результат."""
|
||||
raws = [
|
||||
None,
|
||||
"",
|
||||
"MONOLIT",
|
||||
"BRICK",
|
||||
"PANEL",
|
||||
"MONOLIT_BRICK",
|
||||
"monolithBrick",
|
||||
"gasSilicateBlock",
|
||||
"monolith_brick",
|
||||
"monolith",
|
||||
"stalin",
|
||||
"OTHER",
|
||||
" MONOLIT ",
|
||||
]
|
||||
for raw in raws:
|
||||
assert old_normalize_house_type(raw) == new_normalize_house_type(raw)
|
||||
|
||||
|
||||
# ── detail parse parity (реальные offer-HTML фикстуры) ───────────────────────
|
||||
|
||||
_SECONDARY_ID = "3402418396407468801"
|
||||
_STUDIO_ID = "7811691822126445498"
|
||||
|
||||
|
||||
def _load_offer(offer_id: str) -> str:
|
||||
return (_FIXTURES / f"yandex_offer_{offer_id}.html").read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def _offer_url(offer_id: str) -> str:
|
||||
return f"https://realty.yandex.ru/offer/{offer_id}/"
|
||||
|
||||
|
||||
def test_detail_parse_parity() -> None:
|
||||
"""INITIAL_STATE.offerCard.card + JSON-LD → DetailEnrichment идентичен."""
|
||||
old_scraper = OldDetailScraper()
|
||||
new_scraper = NewDetailScraper()
|
||||
for offer_id in (_SECONDARY_ID, _STUDIO_ID):
|
||||
html = _load_offer(offer_id)
|
||||
url = _offer_url(offer_id)
|
||||
old_res = old_scraper.parse(html, offer_url=url)
|
||||
new_res = new_scraper.parse(html, offer_url=url)
|
||||
assert old_res is not None and new_res is not None
|
||||
assert old_res.model_dump() == new_res.model_dump()
|
||||
|
||||
|
||||
# ── valuation parse parity (синтетика) ───────────────────────────────────────
|
||||
|
||||
_VALUATION_HTML = """
|
||||
<html><body>
|
||||
Дом 2016 года 25 этажей 2,7 м потолки Лифт 42 объекта Панорама
|
||||
48,5 м², 1-комнатная, 5 этаж 5,1 млн ₽ 105 155 ₽ за м² 23.10.2023
|
||||
В экспозиции 945 дней Снято 24.05.2024
|
||||
32 м², студия, 2 этаж 3,2 млн ₽ 100 000 ₽ за м² 01.02.2024 В продаже
|
||||
</body></html>
|
||||
"""
|
||||
|
||||
|
||||
def test_valuation_parse_parity() -> None:
|
||||
"""house-meta + history (chunked-text fallback) → идентичный результат."""
|
||||
old_scraper = OldValuationScraper()
|
||||
new_scraper = NewValuationScraper(config=_KIT_CONFIG)
|
||||
kwargs = dict(
|
||||
address="Екатеринбург, ул. Ленина, 5",
|
||||
offer_category="APARTMENT",
|
||||
offer_type="SELL",
|
||||
page=1,
|
||||
source_url="https://realty.yandex.ru/otsenka-kvartiry-po-adresu-onlayn/?address=test",
|
||||
)
|
||||
old_res = old_scraper.parse(_VALUATION_HTML, **kwargs)
|
||||
new_res = new_scraper.parse(_VALUATION_HTML, **kwargs)
|
||||
assert old_res.model_dump() == new_res.model_dump()
|
||||
|
||||
|
||||
# ── newbuilding parse parity (синтетика) ─────────────────────────────────────
|
||||
|
||||
_NEWBUILDING_HTML = """
|
||||
<html><body>
|
||||
<h1>ЖК «Татлин»</h1>
|
||||
Екатеринбург, ул. Черепанова 4.3 из 5 1505 оценок Смотреть все 353 отзыва
|
||||
56.855312 60.576668
|
||||
<h2>О комплексе</h2>
|
||||
<div>Дом комфорт-класса, монолитный. Введён в эксплуатацию в июнь 2023.
|
||||
35-этажные башни на участке 1,5 га. Три корпуса.</div>
|
||||
</body></html>
|
||||
"""
|
||||
|
||||
|
||||
def test_newbuilding_parse_parity() -> None:
|
||||
"""ЖК-лендинг → YandexNewbuildingInfo идентичен."""
|
||||
old_scraper = OldNewbuildingScraper()
|
||||
new_scraper = NewNewbuildingScraper()
|
||||
old_res = old_scraper.parse(
|
||||
_NEWBUILDING_HTML,
|
||||
jk_slug="tatlin",
|
||||
jk_id="1592987",
|
||||
source_url="https://realty.yandex.ru/ekaterinburg/kupit/novostrojka/tatlin-1592987/",
|
||||
)
|
||||
new_res = new_scraper.parse(
|
||||
_NEWBUILDING_HTML,
|
||||
jk_slug="tatlin",
|
||||
jk_id="1592987",
|
||||
source_url="https://realty.yandex.ru/ekaterinburg/kupit/novostrojka/tatlin-1592987/",
|
||||
)
|
||||
assert old_res.model_dump() == new_res.model_dump()
|
||||
|
||||
|
||||
# ── _build_url parity ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_build_url_parity() -> None:
|
||||
"""gate-API URL builder идентичен на репрезентативных комбо."""
|
||||
old_scraper = OldRealtyScraper()
|
||||
new_scraper = NewRealtyScraper(config=_KIT_CONFIG)
|
||||
combos = [
|
||||
dict(page=1),
|
||||
dict(page=3, rooms="2", price_min=5_000_000, price_max=7_000_000),
|
||||
dict(page=1, rooms="studio", new_flat="YES"),
|
||||
dict(page=2, rooms="4+", price_min=25_000_000),
|
||||
]
|
||||
for combo in combos:
|
||||
assert old_scraper._build_url(**combo) == new_scraper._build_url(**combo)
|
||||
|
||||
|
||||
# ── strangler-guard: kit-провайдер не импортирует app.* ──────────────────────
|
||||
|
||||
|
||||
def test_kit_yandex_has_no_app_imports() -> None:
|
||||
"""Ни один модуль scraper_kit.providers.yandex.* не импортирует app.* (развязка)."""
|
||||
from scraper_kit.providers.yandex import detail, newbuilding, serp, shared, valuation
|
||||
|
||||
for mod in (serp, detail, valuation, newbuilding, shared):
|
||||
source = inspect.getsource(mod)
|
||||
for line in source.splitlines():
|
||||
stripped = line.strip()
|
||||
assert not stripped.startswith("from app"), f"{mod.__name__}: {line!r}"
|
||||
assert not stripped.startswith("import app"), f"{mod.__name__}: {line!r}"
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
"""Yandex-провайдер scraper_kit — strangler-копия `app.services.scrapers.yandex_*`.
|
||||
|
||||
Модули:
|
||||
- serp — SERP-sweep ядро (YandexRealtyScraper, gate-API JSON → ScrapedLot,
|
||||
price-range walk, combos)
|
||||
- detail — detail-страница объявления (INITIAL_STATE.offerCard.card парс →
|
||||
DetailEnrichment)
|
||||
- valuation — анонимная оценка/история дома (/otsenka-...)
|
||||
- newbuilding — ЖК-лендинг (YandexNewbuildingScraper, slug-resolve через SERP)
|
||||
- shared — локальная копия normalize_house_type (канон house_type)
|
||||
|
||||
Развязка от `app.*`:
|
||||
- `app.core.config.settings` → инжектируемый `ScraperConfig` (scraper_kit.contracts)
|
||||
- `app.services.scraper_settings.get_scraper_delay` → инжектируемый `delay_provider`
|
||||
- `app.services.scrapers.base` → `scraper_kit.base`
|
||||
- `app.services.scrapers.browser_fetcher` → `scraper_kit.browser_fetcher`
|
||||
- `app.services.scrapers.price_brackets` → `scraper_kit.price_brackets`
|
||||
- `app.services.scrapers.repair_state_normalizer` → `scraper_kit.repair_state_normalizer`
|
||||
- `app.services.scrapers.yandex_helpers` → `scraper_kit.yandex_helpers`
|
||||
- `app.services.scrapers.house_type_normalizer` → локальный `scraper_kit.providers.yandex.shared`
|
||||
"""
|
||||
|
|
@ -0,0 +1,671 @@
|
|||
"""Yandex Realty detail page scraper.
|
||||
|
||||
Fetches /offer/<id>/ and extracts Product JSON-LD + DOM sections into
|
||||
a DetailEnrichment Pydantic model. Used by enrichment pipeline
|
||||
(Wave 5+ matching / Wave 6 estimator).
|
||||
|
||||
Strangler-копия `app.services.scrapers.yandex_detail` (#2133). Развязка от `app.*`:
|
||||
- `app.services.scraper_settings.get_scraper_delay` → инжектируемый `delay_provider`
|
||||
- `app.services.scrapers.base` → `scraper_kit.base`
|
||||
- `app.services.scrapers.repair_state_normalizer` → `scraper_kit.repair_state_normalizer`
|
||||
- `app.services.scrapers.yandex_helpers` → `scraper_kit.yandex_helpers`
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
from datetime import date
|
||||
from typing import Any
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
from selectolax.parser import HTMLParser, Node
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from scraper_kit.base import BaseScraper
|
||||
from scraper_kit.repair_state_normalizer import infer_repair_state_from_text
|
||||
from scraper_kit.yandex_helpers import (
|
||||
RE_AGENCY_FOUNDED,
|
||||
RE_AGENCY_OBJECTS,
|
||||
RE_METRO_WALK,
|
||||
RE_VIEWS,
|
||||
RE_YEAR,
|
||||
find_ld_by_type,
|
||||
parse_house_type,
|
||||
parse_ru_date,
|
||||
parse_rub,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ── Pydantic models ───────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class MetroStation(BaseModel):
|
||||
name: str
|
||||
walk_min: int | None = None
|
||||
|
||||
|
||||
class DetailEnrichment(BaseModel):
|
||||
"""Enrichment payload from a Yandex detail page."""
|
||||
|
||||
offer_id: str
|
||||
source_url: str
|
||||
|
||||
# Pricing — Product JSON-LD `offers.price` is the exact int
|
||||
price_rub: int | None = None
|
||||
price_per_m2: int | None = None
|
||||
|
||||
# Title + basic params
|
||||
title: str | None = None
|
||||
rooms: int | None = None
|
||||
area_m2: float | None = None
|
||||
living_area_m2: float | None = None
|
||||
kitchen_area_m2: float | None = None
|
||||
ceiling_height: float | None = None # meters, e.g. 2.55
|
||||
floor: int | None = None
|
||||
total_floors: int | None = None
|
||||
|
||||
# Address (full)
|
||||
address: str | None = None
|
||||
|
||||
# Description (full text)
|
||||
description: str | None = None
|
||||
|
||||
# Repair state — enum, inferred from description text (#622).
|
||||
# Yandex не отдаёт структурного поля ремонта, поэтому только инференс.
|
||||
repair_state: str | None = None
|
||||
|
||||
# Stats
|
||||
views_total: int | None = None
|
||||
publish_date: date | None = None
|
||||
publish_date_relative: str | None = None
|
||||
|
||||
# Agency block (OfferCardAuthorInfo)
|
||||
agency_name: str | None = None
|
||||
agency_founded_year: int | None = None
|
||||
agency_objects_count: int | None = None
|
||||
seller_name: str | None = None # last text line before "Агентство «...»"
|
||||
|
||||
# Metro stations from "Расположение" section
|
||||
metro_stations: list[MetroStation] = Field(default_factory=list)
|
||||
|
||||
# Photos — 8 sizes from Product.image[]
|
||||
photo_urls: list[str] = Field(default_factory=list)
|
||||
|
||||
# Newbuilding linkage
|
||||
newbuilding_url: str | None = None
|
||||
newbuilding_id: str | None = None
|
||||
|
||||
# NLP from description (best-effort)
|
||||
house_type_nlp: str | None = None
|
||||
year_built_hint: int | None = None
|
||||
metro_walk_min: int | None = None
|
||||
|
||||
# Raw payload (trimmed)
|
||||
raw_payload: dict[str, Any] | None = None
|
||||
|
||||
|
||||
# ── Scraper ───────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class YandexDetailScraper(BaseScraper):
|
||||
"""Detail page scraper for realty.yandex.ru."""
|
||||
|
||||
name = "yandex_detail"
|
||||
base_url = "https://realty.yandex.ru"
|
||||
request_delay_sec = 5.0 # class default; instance value injected via delay_provider
|
||||
|
||||
def __init__(self, *, delay_provider: Callable[[str], float] | None = None) -> None:
|
||||
super().__init__()
|
||||
# Strangler-инжекция (#2133): провайдер задержки приходит снаружи вместо
|
||||
# прямого импорта app.services.scraper_settings.get_scraper_delay.
|
||||
if delay_provider is not None:
|
||||
self.request_delay_sec = delay_provider(self.name)
|
||||
|
||||
# BaseScraper requires fetch_around — detail isn't geo-based, raise NotImplementedError
|
||||
async def fetch_around(self, lat: float, lon: float, radius_m: int = 1000) -> list: # type: ignore[override]
|
||||
raise NotImplementedError(
|
||||
"YandexDetailScraper is offer-id-based; use fetch_detail(offer_url) instead."
|
||||
)
|
||||
|
||||
async def fetch_detail(self, offer_url: str) -> DetailEnrichment | None:
|
||||
try:
|
||||
response = await self._http_get(offer_url)
|
||||
except Exception:
|
||||
logger.exception("yandex detail fetch failed: %s", offer_url)
|
||||
return None
|
||||
if response.status_code != 200:
|
||||
logger.warning("yandex detail returned %d for %s", response.status_code, offer_url)
|
||||
return None
|
||||
result = self.parse(response.text, offer_url=offer_url)
|
||||
await self.sleep_between_requests()
|
||||
return result
|
||||
|
||||
def parse(self, html: str, offer_url: str) -> DetailEnrichment | None:
|
||||
offer_id_match = re.search(r"/offer/(\d+)/?", offer_url)
|
||||
if not offer_id_match:
|
||||
logger.warning("offer_url has no /offer/<id>/: %s", offer_url)
|
||||
return None
|
||||
offer_id = offer_id_match.group(1)
|
||||
|
||||
tree = HTMLParser(html)
|
||||
|
||||
# --- Product JSON-LD (authoritative price) ---
|
||||
product = find_ld_by_type(html, "Product") or {}
|
||||
offers_ld = product.get("offers") or {}
|
||||
if isinstance(offers_ld, list) and offers_ld:
|
||||
offers_ld = offers_ld[0]
|
||||
price_ld = offers_ld.get("price") if isinstance(offers_ld, dict) else None
|
||||
try:
|
||||
price_rub = int(price_ld) if price_ld else None
|
||||
except (TypeError, ValueError):
|
||||
price_rub = None
|
||||
|
||||
# Photos from JSON-LD image[] (typically 8 size variants)
|
||||
images = product.get("image") or []
|
||||
if isinstance(images, str):
|
||||
images = [images]
|
||||
photo_urls = [u for u in images if isinstance(u, str)]
|
||||
|
||||
# --- Title + summary ---
|
||||
title_node = tree.css_first("h1")
|
||||
title = title_node.text(strip=True) if title_node else None
|
||||
|
||||
rooms, area_m2, floor, total_floors = _parse_title(title or "")
|
||||
|
||||
# --- Structural offer card (window.INITIAL_STATE → offerCard.card) ---
|
||||
# Authoritative source for area/floor/ceiling/kitchen — the h1 title
|
||||
# misses non-standard layouts (студия / свободная планировка) and never
|
||||
# carries ceiling/kitchen/living. Title stays as fallback below.
|
||||
living_area_m2: float | None = None
|
||||
kitchen_area_m2: float | None = None
|
||||
ceiling_height: float | None = None
|
||||
card = _extract_offer_card(html, offer_id)
|
||||
if card is not None:
|
||||
(
|
||||
c_rooms,
|
||||
c_area,
|
||||
c_living,
|
||||
c_kitchen,
|
||||
c_ceiling,
|
||||
c_floor,
|
||||
c_total_floors,
|
||||
) = _parse_card_fields(card)
|
||||
# Structural source wins; title only fills the gaps it left.
|
||||
rooms = c_rooms if c_rooms is not None else rooms
|
||||
area_m2 = c_area if c_area is not None else area_m2
|
||||
living_area_m2 = c_living
|
||||
kitchen_area_m2 = c_kitchen
|
||||
ceiling_height = c_ceiling
|
||||
floor = c_floor if c_floor is not None else floor
|
||||
total_floors = c_total_floors if c_total_floors is not None else total_floors
|
||||
|
||||
# --- OfferCardSummary text block ---
|
||||
summary_node = tree.css_first('[data-test="OfferCardSummary"]')
|
||||
summary_text = summary_node.text(strip=True) if summary_node else ""
|
||||
|
||||
# Views + relative publish date from summary text
|
||||
views_match = RE_VIEWS.search(summary_text)
|
||||
views_total = int(views_match.group(1)) if views_match else None
|
||||
publish_date = parse_ru_date(summary_text)
|
||||
publish_date_relative = _extract_relative_date(summary_text)
|
||||
|
||||
# price_per_m2 — from summary text if absent in LD
|
||||
price_per_m2: int | None = None
|
||||
ppm2_match = re.search(r"(\d[\d\s]+)\s*₽\s*за\s*м²", summary_text)
|
||||
if ppm2_match:
|
||||
price_per_m2 = parse_rub(ppm2_match.group(1))
|
||||
|
||||
# --- OfferCardAuthorInfo (agency block) ---
|
||||
author_node = tree.css_first('[data-test="OfferCardAuthorInfo"]')
|
||||
agency_name: str | None = None
|
||||
agency_founded_year: int | None = None
|
||||
agency_objects_count: int | None = None
|
||||
seller_name: str | None = None
|
||||
if author_node is not None:
|
||||
author_text = author_node.text(strip=True)
|
||||
agency_h2 = author_node.css_first("h2")
|
||||
agency_name = agency_h2.text(strip=True) if agency_h2 else None
|
||||
founded_m = RE_AGENCY_FOUNDED.search(author_text)
|
||||
if founded_m:
|
||||
agency_founded_year = int(founded_m.group(1))
|
||||
objects_m = RE_AGENCY_OBJECTS.search(author_text)
|
||||
if objects_m:
|
||||
agency_objects_count = int(objects_m.group(1))
|
||||
# seller_name — last text line before "Агентство"
|
||||
seller_name = _extract_seller_name(summary_text, agency_name)
|
||||
|
||||
# --- Description section (after H2 "Описание") ---
|
||||
description = _find_section_text(tree, "Описание")
|
||||
|
||||
# --- Repair state: инференс из описания (#622), Yandex без структурного поля ---
|
||||
repair_state = infer_repair_state_from_text(description or summary_text)
|
||||
|
||||
# --- Address ---
|
||||
address = _extract_address(summary_text)
|
||||
|
||||
# --- Metro stations from "Расположение" section ---
|
||||
location_text = _find_section_text(tree, "Расположение") or ""
|
||||
metro_stations = _parse_metro_stations(location_text)
|
||||
|
||||
# --- Newbuilding link ---
|
||||
nb_url: str | None = None
|
||||
nb_id: str | None = None
|
||||
nb_link = tree.css_first('a[href*="/kupit/novostrojka/"]')
|
||||
if nb_link is not None:
|
||||
nb_href = nb_link.attributes.get("href", "")
|
||||
nb_match = re.search(r"/novostrojka/[\w-]+?-(\d+)/?", nb_href)
|
||||
if nb_match:
|
||||
nb_id = nb_match.group(1)
|
||||
nb_url = (
|
||||
nb_href if nb_href.startswith("http") else f"https://realty.yandex.ru{nb_href}"
|
||||
)
|
||||
|
||||
# --- NLP best-effort from description ---
|
||||
nlp_text = description or summary_text
|
||||
house_type_nlp = parse_house_type(nlp_text)
|
||||
year_hint_m = RE_YEAR.search(nlp_text or "")
|
||||
year_built_hint = int(year_hint_m.group(1)) if year_hint_m else None
|
||||
walk_m = RE_METRO_WALK.search(nlp_text or "")
|
||||
metro_walk_min = int(walk_m.group(1)) if walk_m else None
|
||||
|
||||
return DetailEnrichment(
|
||||
offer_id=offer_id,
|
||||
source_url=offer_url,
|
||||
price_rub=price_rub,
|
||||
price_per_m2=price_per_m2,
|
||||
title=title,
|
||||
rooms=rooms,
|
||||
area_m2=area_m2,
|
||||
living_area_m2=living_area_m2,
|
||||
kitchen_area_m2=kitchen_area_m2,
|
||||
ceiling_height=ceiling_height,
|
||||
floor=floor,
|
||||
total_floors=total_floors,
|
||||
address=address,
|
||||
description=description,
|
||||
repair_state=repair_state,
|
||||
views_total=views_total,
|
||||
publish_date=publish_date,
|
||||
publish_date_relative=publish_date_relative,
|
||||
agency_name=agency_name,
|
||||
agency_founded_year=agency_founded_year,
|
||||
agency_objects_count=agency_objects_count,
|
||||
seller_name=seller_name,
|
||||
metro_stations=metro_stations,
|
||||
photo_urls=photo_urls,
|
||||
newbuilding_url=nb_url,
|
||||
newbuilding_id=nb_id,
|
||||
house_type_nlp=house_type_nlp,
|
||||
year_built_hint=year_built_hint,
|
||||
metro_walk_min=metro_walk_min,
|
||||
raw_payload={
|
||||
"summary_text": summary_text[:1000],
|
||||
"description_len": len(description) if description else 0,
|
||||
"photo_count": len(photo_urls),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
# ── Helpers ───────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _parse_title(title: str) -> tuple[int | None, float | None, int | None, int | None]:
|
||||
"""Extract (rooms, area_m2, floor, total_floors) from h1 text."""
|
||||
rooms: int | None = None
|
||||
area_m2: float | None = None
|
||||
floor: int | None = None
|
||||
total_floors: int | None = None
|
||||
|
||||
area_m = re.search(r"(\d+[.,]?\d*)\s*м²", title)
|
||||
if area_m:
|
||||
area_m2 = float(area_m.group(1).replace(",", "."))
|
||||
|
||||
if re.search(r"студи[яюй]", title, re.IGNORECASE):
|
||||
rooms = 0
|
||||
else:
|
||||
rooms_m = re.search(r"(\d+)\s*-?\s*комнатн", title, re.IGNORECASE)
|
||||
if rooms_m:
|
||||
try:
|
||||
rooms = int(rooms_m.group(1))
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
floor_m = re.search(r"(\d+)\s+этаж\s+из\s+(\d+)", title, re.IGNORECASE)
|
||||
if floor_m:
|
||||
floor = int(floor_m.group(1))
|
||||
total_floors = int(floor_m.group(2))
|
||||
|
||||
return rooms, area_m2, floor, total_floors
|
||||
|
||||
|
||||
def _extract_js_object(html: str, marker: str) -> str | None:
|
||||
"""Return the JSON object literal assigned to `marker` (e.g. window.INITIAL_STATE).
|
||||
|
||||
Brace-matches from the first ``{`` after ``marker = ...`` to its balanced
|
||||
close, respecting string literals/escapes. Returns the raw JSON text or None.
|
||||
"""
|
||||
i = html.find(marker)
|
||||
if i < 0:
|
||||
return None
|
||||
eq = html.find("=", i)
|
||||
if eq < 0:
|
||||
return None
|
||||
start = html.find("{", eq)
|
||||
if start < 0:
|
||||
return None
|
||||
bal = 0
|
||||
in_str = False
|
||||
esc = False
|
||||
k = start
|
||||
n = len(html)
|
||||
while k < n:
|
||||
c = html[k]
|
||||
if in_str:
|
||||
if esc:
|
||||
esc = False
|
||||
elif c == "\\":
|
||||
esc = True
|
||||
elif c == '"':
|
||||
in_str = False
|
||||
else:
|
||||
if c == '"':
|
||||
in_str = True
|
||||
elif c == "{":
|
||||
bal += 1
|
||||
elif c == "}":
|
||||
bal -= 1
|
||||
if bal == 0:
|
||||
return html[start : k + 1]
|
||||
k += 1
|
||||
return None
|
||||
|
||||
|
||||
def _find_card_by_offer_id(obj: Any, offer_id: str) -> dict[str, Any] | None:
|
||||
"""Recursively locate the offer dict whose offerId matches and carries `area`."""
|
||||
if isinstance(obj, dict):
|
||||
if obj.get("offerId") == offer_id and "area" in obj:
|
||||
return obj
|
||||
for v in obj.values():
|
||||
found = _find_card_by_offer_id(v, offer_id)
|
||||
if found is not None:
|
||||
return found
|
||||
elif isinstance(obj, list):
|
||||
for v in obj:
|
||||
found = _find_card_by_offer_id(v, offer_id)
|
||||
if found is not None:
|
||||
return found
|
||||
return None
|
||||
|
||||
|
||||
def _extract_offer_card(html: str, offer_id: str) -> dict[str, Any] | None:
|
||||
"""Extract the offer card object from window.INITIAL_STATE.
|
||||
|
||||
Yandex embeds full structured offer data in ``window.INITIAL_STATE`` under
|
||||
``offerCard.card``. This is the authoritative source for area / floor /
|
||||
ceiling / kitchen — unlike the h1 title which misses non-standard layouts
|
||||
and never carries ceiling/kitchen. Returns the card dict or None.
|
||||
"""
|
||||
blob = _extract_js_object(html, "window.INITIAL_STATE")
|
||||
if not blob:
|
||||
return None
|
||||
try:
|
||||
state = json.loads(blob)
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
logger.warning("yandex detail: INITIAL_STATE failed to parse for offer %s", offer_id)
|
||||
return None
|
||||
# Fast path: canonical location.
|
||||
offer_card = state.get("offerCard") if isinstance(state, dict) else None
|
||||
if isinstance(offer_card, dict):
|
||||
card = offer_card.get("card")
|
||||
if isinstance(card, dict) and card.get("offerId") == offer_id:
|
||||
return card
|
||||
# Fallback: deep search (page structure may differ).
|
||||
return _find_card_by_offer_id(state, offer_id)
|
||||
|
||||
|
||||
def _space_value(node: Any) -> float | None:
|
||||
"""Yandex area fields are ``{"value": N, "unit": "SQUARE_METER"}`` → float."""
|
||||
if isinstance(node, dict):
|
||||
val = node.get("value")
|
||||
if isinstance(val, int | float):
|
||||
return float(val)
|
||||
elif isinstance(node, int | float):
|
||||
return float(node)
|
||||
return None
|
||||
|
||||
|
||||
def _parse_card_fields(
|
||||
card: dict[str, Any],
|
||||
) -> tuple[
|
||||
int | None,
|
||||
float | None,
|
||||
float | None,
|
||||
float | None,
|
||||
float | None,
|
||||
int | None,
|
||||
int | None,
|
||||
]:
|
||||
"""Extract (rooms, area, living, kitchen, ceiling, floor, total_floors) from card.
|
||||
|
||||
rooms: ``roomsTotal``, or 0 when ``house.studio`` is truthy (студии не
|
||||
несут roomsTotal). floor: first element of ``floorsOffered``.
|
||||
"""
|
||||
area = _space_value(card.get("area"))
|
||||
living = _space_value(card.get("livingSpace"))
|
||||
kitchen = _space_value(card.get("kitchenSpace"))
|
||||
|
||||
ceiling_raw = card.get("ceilingHeight")
|
||||
ceiling: float | None = None
|
||||
if isinstance(ceiling_raw, int | float):
|
||||
ceiling = float(ceiling_raw)
|
||||
|
||||
rooms_raw = card.get("roomsTotal")
|
||||
rooms: int | None = None
|
||||
if isinstance(rooms_raw, int):
|
||||
rooms = rooms_raw
|
||||
house = card.get("house")
|
||||
if isinstance(house, dict) and house.get("studio"):
|
||||
rooms = 0
|
||||
|
||||
total_floors_raw = card.get("floorsTotal")
|
||||
total_floors = total_floors_raw if isinstance(total_floors_raw, int) else None
|
||||
|
||||
floor: int | None = None
|
||||
floors_offered = card.get("floorsOffered")
|
||||
if isinstance(floors_offered, list) and floors_offered:
|
||||
first = floors_offered[0]
|
||||
if isinstance(first, int):
|
||||
floor = first
|
||||
|
||||
return rooms, area, living, kitchen, ceiling, floor, total_floors
|
||||
|
||||
|
||||
def _find_section_text(tree: HTMLParser, heading: str) -> str | None:
|
||||
"""Find the text content of a <section>/<div> whose preceding h2/h3 matches heading.
|
||||
|
||||
Yandex page structure varies; this scans h2/h3 nodes, then returns the
|
||||
concatenated text of subsequent sibling blocks until the next heading.
|
||||
"""
|
||||
for h in tree.css("h2, h3"):
|
||||
if heading.lower() in (h.text(strip=True) or "").lower():
|
||||
# collect subsequent siblings until the next h2/h3
|
||||
parts: list[str] = []
|
||||
node: Node | None = h.next
|
||||
while node is not None:
|
||||
tag = (node.tag or "").lower()
|
||||
if tag in {"h2", "h3"}:
|
||||
break
|
||||
txt = node.text(strip=True) if hasattr(node, "text") else ""
|
||||
if txt:
|
||||
parts.append(txt)
|
||||
node = node.next
|
||||
return " ".join(parts).strip() or None
|
||||
return None
|
||||
|
||||
|
||||
def _extract_address(summary_text: str) -> str | None:
|
||||
"""Best-effort address extraction from summary block."""
|
||||
# Pattern: "Россия, Свердловская область, Екатеринбург, улица Х, д. N"
|
||||
m = re.search(r"(Россия[^•]+?)(?:•|\d+\s+просмотр|$)", summary_text)
|
||||
if m:
|
||||
addr = m.group(1).strip().rstrip(",").strip()
|
||||
return addr if len(addr) > 10 else None
|
||||
return None
|
||||
|
||||
|
||||
def _parse_metro_stations(location_text: str) -> list[MetroStation]:
|
||||
"""Parse 'Уральская 11 мин. Динамо 16 мин.' → list of MetroStation."""
|
||||
stations: list[MetroStation] = []
|
||||
# name (1+ Cyrillic words) + space + N + space + мин(.|у|ут)
|
||||
for m in re.finditer(r"([А-ЯЁ][А-Яа-яё\s-]+?)\s+(\d+)\s*мин", location_text):
|
||||
name = m.group(1).strip()
|
||||
if 2 <= len(name) <= 40:
|
||||
stations.append(MetroStation(name=name, walk_min=int(m.group(2))))
|
||||
if len(stations) >= 5:
|
||||
break
|
||||
return stations
|
||||
|
||||
|
||||
def _extract_relative_date(summary_text: str) -> str | None:
|
||||
"""Capture phrases like '6 часов назад' / 'вчера' / '3 дня назад'."""
|
||||
m = re.search(
|
||||
r"(\d+\s+(?:минут|час|часов|часа|день|дня|дней|недел[ьюи])\s+назад"
|
||||
r"|вчера|сегодня|позавчера)",
|
||||
summary_text,
|
||||
re.IGNORECASE,
|
||||
)
|
||||
return m.group(1).strip() if m else None
|
||||
|
||||
|
||||
def _extract_seller_name(summary_text: str, agency_name: str | None) -> str | None:
|
||||
"""Heuristic: line right before 'Агентство ...' in summary text."""
|
||||
if not agency_name or agency_name not in summary_text:
|
||||
return None
|
||||
head = summary_text.split(agency_name, 1)[0]
|
||||
# last short token sequence (likely "Имя Фамилия")
|
||||
m = re.findall(r"([А-ЯЁ][а-яё]+(?:\s+[А-ЯЁ][а-яё]+){1,2})", head)
|
||||
return m[-1] if m else None
|
||||
|
||||
|
||||
# ── Save helper ───────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def save_detail_enrichment(db: Session, listing_id: int, e: DetailEnrichment) -> bool:
|
||||
"""Persist Yandex DetailEnrichment to listings row.
|
||||
|
||||
UPDATE listings SET <col> = COALESCE(:val, <col>), ..., detail_enriched_at = NOW()
|
||||
WHERE id = listing_id.
|
||||
|
||||
COALESCE semantics: keeps existing non-NULL value if new value is NULL (never
|
||||
overwrites a populated column with NULL). area_m2 from detail is more accurate
|
||||
than SERP, but COALESCE preserves SERP value if detail returns NULL — acceptable.
|
||||
|
||||
Returns True if the UPDATE touched at least one row (listing_id found in DB).
|
||||
"""
|
||||
metro_json: str | None = None
|
||||
if e.metro_stations:
|
||||
metro_json = json.dumps(
|
||||
[s.model_dump(exclude_none=True) for s in e.metro_stations],
|
||||
ensure_ascii=False,
|
||||
)
|
||||
|
||||
photo_json: str | None = None
|
||||
if e.photo_urls:
|
||||
photo_json = json.dumps(e.photo_urls, ensure_ascii=False)
|
||||
|
||||
result = db.execute(
|
||||
text("""
|
||||
UPDATE listings SET
|
||||
rooms = COALESCE(CAST(:rooms AS int), rooms),
|
||||
area_m2 = COALESCE(CAST(:area_m2 AS numeric), area_m2),
|
||||
living_area_m2 = COALESCE(
|
||||
CAST(:living_area_m2 AS numeric),
|
||||
living_area_m2
|
||||
),
|
||||
kitchen_area_m2 = COALESCE(
|
||||
CAST(:kitchen_area_m2 AS numeric),
|
||||
kitchen_area_m2
|
||||
),
|
||||
ceiling_height = COALESCE(
|
||||
CAST(:ceiling_height AS numeric),
|
||||
ceiling_height
|
||||
),
|
||||
floor = COALESCE(CAST(:floor AS int), floor),
|
||||
total_floors = COALESCE(CAST(:total_floors AS int), total_floors),
|
||||
address = COALESCE(CAST(:address AS text), address),
|
||||
description = COALESCE(CAST(:description AS text), description),
|
||||
repair_state = COALESCE(CAST(:repair_state AS text),repair_state),
|
||||
publish_date = COALESCE(CAST(:publish_date AS date),publish_date),
|
||||
views_total_yandex = COALESCE(CAST(:views_total AS int), views_total_yandex),
|
||||
publish_date_relative= COALESCE(
|
||||
CAST(:pub_date_rel AS text),
|
||||
publish_date_relative
|
||||
),
|
||||
agency_name = COALESCE(CAST(:agency_name AS text), agency_name),
|
||||
agency_founded_year = COALESCE(
|
||||
CAST(:agency_founded_year AS int),
|
||||
agency_founded_year
|
||||
),
|
||||
agency_objects_count = COALESCE(
|
||||
CAST(:agency_objects_count AS int),
|
||||
agency_objects_count
|
||||
),
|
||||
metro_stations = COALESCE(
|
||||
CAST(:metro_stations AS jsonb),
|
||||
metro_stations
|
||||
),
|
||||
photo_urls = COALESCE(
|
||||
CAST(:photo_urls AS jsonb),
|
||||
photo_urls
|
||||
),
|
||||
newbuilding_url = COALESCE(
|
||||
CAST(:newbuilding_url AS text),
|
||||
newbuilding_url
|
||||
),
|
||||
newbuilding_id = COALESCE(
|
||||
CAST(:newbuilding_id AS text),
|
||||
newbuilding_id
|
||||
),
|
||||
detail_enriched_at = NOW()
|
||||
WHERE id = CAST(:listing_id AS bigint)
|
||||
"""),
|
||||
{
|
||||
"listing_id": listing_id,
|
||||
"rooms": e.rooms,
|
||||
"area_m2": e.area_m2,
|
||||
"living_area_m2": e.living_area_m2,
|
||||
"kitchen_area_m2": e.kitchen_area_m2,
|
||||
"ceiling_height": e.ceiling_height,
|
||||
"floor": e.floor,
|
||||
"total_floors": e.total_floors,
|
||||
"address": e.address,
|
||||
"description": e.description,
|
||||
"repair_state": e.repair_state,
|
||||
"publish_date": e.publish_date,
|
||||
"views_total": e.views_total,
|
||||
"pub_date_rel": e.publish_date_relative,
|
||||
"agency_name": e.agency_name,
|
||||
"agency_founded_year": e.agency_founded_year,
|
||||
"agency_objects_count": e.agency_objects_count,
|
||||
"metro_stations": metro_json,
|
||||
"photo_urls": photo_json,
|
||||
"newbuilding_url": e.newbuilding_url,
|
||||
"newbuilding_id": e.newbuilding_id,
|
||||
},
|
||||
)
|
||||
db.commit()
|
||||
saved = (result.rowcount or 0) > 0
|
||||
logger.info(
|
||||
"yandex detail enrichment saved listing_id=%s (metro=%d photos=%d saved=%s)",
|
||||
listing_id,
|
||||
len(e.metro_stations),
|
||||
len(e.photo_urls),
|
||||
saved,
|
||||
)
|
||||
return saved
|
||||
|
|
@ -0,0 +1,438 @@
|
|||
"""Yandex Realty ЖК landing page parser.
|
||||
|
||||
URL pattern: /{city}/kupit/novostrojka/<slug>-<id>/
|
||||
Reference target: ЖК Татлин (id=1592987, slug=tatlin) — comfort+, June 2023,
|
||||
PRINZIP, rating 4.3, 1505 ratings, 353 text reviews, coords (56.855312, 60.576668).
|
||||
|
||||
Fetch strategy (#974):
|
||||
- Все network-запросы (ЖК-лендинг + SERP slug-resolve) идут через BrowserFetcher
|
||||
(tradein-browser camoufox), а НЕ через httpx/_http_get.
|
||||
- Yandex Realty — JS-heavy / anti-bot; curl_cffi / httpx не получают данные.
|
||||
- BrowserFetcher.fetch(url) → str (полный HTML); вызывающий код парсит через parse().
|
||||
|
||||
Strangler-копия `app.services.scrapers.yandex_newbuilding` (#2133). Развязка от `app.*`:
|
||||
- `app.services.scraper_settings.get_scraper_delay` → инжектируемый `delay_provider`
|
||||
- `app.services.scrapers.base` → `scraper_kit.base`
|
||||
- `app.services.scrapers.browser_fetcher` → `scraper_kit.browser_fetcher`
|
||||
- `app.services.scrapers.yandex_helpers` → `scraper_kit.yandex_helpers`
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
from selectolax.parser import HTMLParser, Node
|
||||
|
||||
from scraper_kit.base import BaseScraper
|
||||
from scraper_kit.browser_fetcher import BrowserFetcher
|
||||
from scraper_kit.yandex_helpers import (
|
||||
RE_JK_ID,
|
||||
parse_house_class,
|
||||
parse_house_type,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ── Models ───────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class JKMetroStation(BaseModel):
|
||||
name: str
|
||||
walk_min: int | None = None
|
||||
|
||||
|
||||
class YandexNewbuildingInfo(BaseModel):
|
||||
"""ЖК landing scrape payload."""
|
||||
|
||||
ext_id: str # '1592987'
|
||||
ext_slug: str # 'tatlin'
|
||||
source_url: str
|
||||
|
||||
# Identification
|
||||
name: str | None = None # 'ЖК «Татлин»'
|
||||
address: str | None = None # 'Екатеринбург, ул. Черепанова / ул. Готвальда'
|
||||
|
||||
# Coords (inline in HTML — Yandex unique vs Avito/Cian)
|
||||
lat: float | None = None
|
||||
lon: float | None = None
|
||||
|
||||
# Classification + commission
|
||||
house_class: str | None = None # 'comfort_plus' etc.
|
||||
commission_year: int | None = None
|
||||
commission_month: str | None = None # 'июнь' (RU)
|
||||
|
||||
# Footprint
|
||||
total_floors: int | None = None # 35
|
||||
corpus_count: int | None = None # 3
|
||||
total_area_ha: float | None = None # 1.5
|
||||
|
||||
# Developer
|
||||
developer_name: str | None = None # 'PRINZIP'
|
||||
developer_url: str | None = None
|
||||
developer_other_jk: list[str] = Field(default_factory=list)
|
||||
|
||||
# Reviews
|
||||
rating: float | None = None # 4.3
|
||||
ratings_count: int | None = None # 1505 — оценок
|
||||
text_reviews_count: int | None = None # 353 — текстовых отзывов
|
||||
|
||||
# NLP / description
|
||||
description: str | None = None
|
||||
|
||||
# Metro
|
||||
metro_stations: list[JKMetroStation] = Field(default_factory=list)
|
||||
|
||||
# House type (best-effort NLP from description)
|
||||
house_type: str | None = None
|
||||
|
||||
raw_payload: dict[str, Any] | None = None
|
||||
|
||||
|
||||
# ── Regex constants ───────────────────────────────────────────────────────────
|
||||
|
||||
RE_FLOORS_TOWERS = re.compile(r"(\d+)-этажн\w+\s+башн", re.IGNORECASE)
|
||||
RE_AREA_HA = re.compile(r"участке\s+([\d,]+)\s*га", re.IGNORECASE)
|
||||
RE_COMMISSION = re.compile(
|
||||
r"(?:введ[её]н|сдан)\s+в\s+эксплуатацию\s+в\s+(\w+)\s+(\d{4})", re.IGNORECASE
|
||||
)
|
||||
# Yandex drift (#974 follow-up): рейтинг рендерится без пробела перед «из»
|
||||
# ("4.3из 5" / "4,5из 5", nbsp внутри) → \s* (а не \s+) перед «из», иначе rating=NULL
|
||||
# при заполненных ratings_count/text_reviews (verified live 2026-06-15, ЖК Татлин/Рио).
|
||||
RE_RATING = re.compile(r"(\d[.,]\d)\s*из\s+5")
|
||||
RE_RATINGS_COUNT = re.compile(r"(\d+)\s+оценок", re.IGNORECASE)
|
||||
RE_TEXT_REVIEWS = re.compile(r"Смотреть\s+все\s+(\d+)\s+отзыв", re.IGNORECASE)
|
||||
RE_CORPUS_COUNT_WORD = re.compile(
|
||||
r"(одн[ау]|две|три|четыре|пять|шесть|семь|восемь|\d+)\s+(?:[\d\s-]*)\s*-?\s*этажн",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
RE_METRO_INLINE = re.compile(
|
||||
r"([А-ЯЁ][А-Яа-яё\s-]{2,30}?)\s+(\d+)\s*мин",
|
||||
)
|
||||
|
||||
# Slug из href: /{city}/kupit/novostrojka/<slug>-<id>/
|
||||
# Используется в resolve_yandex_jk_slug для извлечения slug из SERP href.
|
||||
_JK_SLUG_RE = re.compile(
|
||||
r"/(?:ekaterinburg|moskva|spb|[a-z-]+)/kupit/novostrojka/([a-z0-9-]+)-(\d+)/"
|
||||
)
|
||||
|
||||
# Ekb-specific coord ranges (extend per-city later)
|
||||
LAT_RANGE = (55.5, 57.5)
|
||||
LON_RANGE = (59.5, 61.5)
|
||||
|
||||
# Word → number map
|
||||
_WORD_NUM: dict[str, int] = {
|
||||
"одна": 1,
|
||||
"одну": 1,
|
||||
"одной": 1,
|
||||
"две": 2,
|
||||
"три": 3,
|
||||
"четыре": 4,
|
||||
"пять": 5,
|
||||
"шесть": 6,
|
||||
"семь": 7,
|
||||
"восемь": 8,
|
||||
}
|
||||
|
||||
|
||||
# ── Scraper class ─────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class YandexNewbuildingScraper(BaseScraper):
|
||||
name = "yandex_realty_nb"
|
||||
base_url = "https://realty.yandex.ru"
|
||||
request_delay_sec = 5.0 # class default; instance value injected via delay_provider
|
||||
|
||||
def __init__(self, *, delay_provider: Callable[[str], float] | None = None) -> None:
|
||||
super().__init__()
|
||||
# Strangler-инжекция (#2133): провайдер задержки приходит снаружи вместо
|
||||
# прямого импорта app.services.scraper_settings.get_scraper_delay.
|
||||
if delay_provider is not None:
|
||||
self.request_delay_sec = delay_provider(self.name)
|
||||
|
||||
async def fetch_around(self, lat: float, lon: float, radius_m: int = 1000) -> list: # type: ignore[override]
|
||||
raise NotImplementedError(
|
||||
"YandexNewbuildingScraper is JK-slug-based; use fetch_jk(slug, id) instead."
|
||||
)
|
||||
|
||||
async def fetch_jk(
|
||||
self, jk_slug: str, jk_id: str, city: str = "ekaterinburg"
|
||||
) -> YandexNewbuildingInfo | None:
|
||||
"""Загрузить ЖК-лендинг через BrowserFetcher и распарсить.
|
||||
|
||||
Yandex Realty — JS/anti-bot: curl/httpx не получают данные. Запрос идёт
|
||||
через tradein-browser (camoufox) контейнер — единственный рабочий путь (#974).
|
||||
"""
|
||||
url = f"{self.base_url}/{city}/kupit/novostrojka/{jk_slug}-{jk_id}/"
|
||||
try:
|
||||
async with BrowserFetcher(source="yandex") as fetcher:
|
||||
html = await fetcher.fetch(url)
|
||||
except Exception:
|
||||
logger.exception("yandex nb browser fetch failed: %s", url)
|
||||
return None
|
||||
if not html or len(html) < 500:
|
||||
logger.warning(
|
||||
"yandex nb browser returned empty/tiny HTML (%d bytes): %s",
|
||||
len(html) if html else 0,
|
||||
url,
|
||||
)
|
||||
return None
|
||||
result = self.parse(html, jk_slug=jk_slug, jk_id=jk_id, source_url=url)
|
||||
return result
|
||||
|
||||
def parse(self, html: str, jk_slug: str, jk_id: str, source_url: str) -> YandexNewbuildingInfo:
|
||||
tree = HTMLParser(html)
|
||||
body = tree.body
|
||||
body_text = body.text(strip=True) if body else ""
|
||||
|
||||
# Header
|
||||
h1 = tree.css_first("h1")
|
||||
name = h1.text(strip=True) if h1 else None
|
||||
|
||||
# Sections (heading-based extraction)
|
||||
location_text = _find_section_text(tree, "Расположение") or ""
|
||||
description_section = (
|
||||
_find_section_text(tree, "О комплексе")
|
||||
or _find_section_text(tree, "Расположение, транспортная доступность")
|
||||
or location_text
|
||||
)
|
||||
|
||||
# Coords — inline scan within plausible range
|
||||
lat, lon = _extract_coords(html)
|
||||
|
||||
# Class
|
||||
house_class = parse_house_class(description_section) or parse_house_class(body_text)
|
||||
|
||||
# Commission
|
||||
commission_year: int | None = None
|
||||
commission_month: str | None = None
|
||||
c_m = RE_COMMISSION.search(description_section) or RE_COMMISSION.search(body_text)
|
||||
if c_m:
|
||||
commission_month = c_m.group(1).lower()
|
||||
try:
|
||||
commission_year = int(c_m.group(2))
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
# Floors + corpus + area
|
||||
floors_m = RE_FLOORS_TOWERS.search(description_section) or RE_FLOORS_TOWERS.search(
|
||||
body_text
|
||||
)
|
||||
total_floors = int(floors_m.group(1)) if floors_m else None
|
||||
|
||||
corpus_count = _parse_corpus_count(description_section or body_text)
|
||||
|
||||
area_m = RE_AREA_HA.search(description_section) or RE_AREA_HA.search(body_text)
|
||||
total_area_ha = float(area_m.group(1).replace(",", ".")) if area_m else None
|
||||
|
||||
# Developer
|
||||
dev_link = tree.css_first('[data-test="CARD_DEV_BADGE_DEVELOPER_LINK"]')
|
||||
developer_name: str | None = None
|
||||
developer_url: str | None = None
|
||||
if dev_link is not None:
|
||||
developer_name = dev_link.text(strip=True) or None
|
||||
href = dev_link.attributes.get("href", "")
|
||||
if href:
|
||||
developer_url = (
|
||||
href if href.startswith("http") else f"https://realty.yandex.ru{href}"
|
||||
)
|
||||
|
||||
# Developer's other JKs
|
||||
other_jk_block = tree.css_first('[data-test="CardDevSites"]')
|
||||
developer_other_jk: list[str] = []
|
||||
if other_jk_block is not None:
|
||||
for jk_a in other_jk_block.css("a"):
|
||||
t = jk_a.text(strip=True)
|
||||
if t and t not in developer_other_jk:
|
||||
developer_other_jk.append(t)
|
||||
|
||||
# Reviews
|
||||
rating_m = RE_RATING.search(body_text)
|
||||
rating = float(rating_m.group(1).replace(",", ".")) if rating_m else None
|
||||
rcount_m = RE_RATINGS_COUNT.search(body_text)
|
||||
ratings_count = int(rcount_m.group(1)) if rcount_m else None
|
||||
treviews_m = RE_TEXT_REVIEWS.search(body_text)
|
||||
text_reviews_count = int(treviews_m.group(1)) if treviews_m else None
|
||||
|
||||
# Address (header area before lat/lon block)
|
||||
address = _extract_jk_address(body_text, name)
|
||||
|
||||
# Metro stations
|
||||
metro_stations = _parse_metro(body_text)
|
||||
|
||||
# NLP house_type fallback
|
||||
house_type = parse_house_type(description_section) or parse_house_type(body_text)
|
||||
|
||||
return YandexNewbuildingInfo(
|
||||
ext_id=jk_id,
|
||||
ext_slug=jk_slug,
|
||||
source_url=source_url,
|
||||
name=name,
|
||||
address=address,
|
||||
lat=lat,
|
||||
lon=lon,
|
||||
house_class=house_class,
|
||||
commission_year=commission_year,
|
||||
commission_month=commission_month,
|
||||
total_floors=total_floors,
|
||||
corpus_count=corpus_count,
|
||||
total_area_ha=total_area_ha,
|
||||
developer_name=developer_name,
|
||||
developer_url=developer_url,
|
||||
developer_other_jk=developer_other_jk[:10],
|
||||
rating=rating,
|
||||
ratings_count=ratings_count,
|
||||
text_reviews_count=text_reviews_count,
|
||||
description=description_section or None,
|
||||
metro_stations=metro_stations,
|
||||
house_type=house_type,
|
||||
raw_payload={
|
||||
"description_len": len(description_section or ""),
|
||||
"body_len": len(body_text),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
# ── Slug resolution via SERP ──────────────────────────────────────────────────
|
||||
|
||||
|
||||
async def resolve_yandex_jk_slug(
|
||||
jk_id: str,
|
||||
city: str = "ekaterinburg",
|
||||
) -> str | None:
|
||||
"""Найти Yandex Realty slug для ЖК по его ext_id (jk_id) через SERP.
|
||||
|
||||
Стратегия (#974 — зеркало resolve_cian_zhk_url_via_search):
|
||||
1. Запросить поисковую страницу Yandex Realty через BrowserFetcher.
|
||||
URL: /ekaterinburg/kupit/novostrojka/?siteId=<jk_id>
|
||||
2. В HTML найти первую ссылку вида /<city>/kupit/novostrojka/<slug>-<jk_id>/
|
||||
через regex _JK_SLUG_RE.
|
||||
3. Вернуть slug или None при любой ошибке.
|
||||
|
||||
Caller несёт ответственность за anti-bot sleep (зеркало cian_newbuilding.py).
|
||||
BrowserFetcher обязателен — Yandex Realty JS/anti-bot, httpx/curl не работают.
|
||||
|
||||
Returns:
|
||||
slug (str без id-суффикса), или None при ошибке / не найден.
|
||||
"""
|
||||
# Yandex Realty SERP: фильтр по siteId → первый результат = нужный ЖК.
|
||||
# Альтернативный путь через Яндекс Поиск (web SERP) менее надёжен из-за
|
||||
# вариативности разметки. Прямой realty.yandex.ru SERP — стабильнее.
|
||||
serp_url = f"https://realty.yandex.ru/{city}/kupit/novostrojka/?siteId={jk_id}"
|
||||
try:
|
||||
async with BrowserFetcher(source="yandex") as fetcher:
|
||||
html = await fetcher.fetch(serp_url)
|
||||
except Exception as exc:
|
||||
logger.warning("resolve_yandex_jk_slug jk_id=%s browser fetch failed: %s", jk_id, exc)
|
||||
return None
|
||||
|
||||
if not html:
|
||||
logger.warning("resolve_yandex_jk_slug jk_id=%s: empty HTML from browser", jk_id)
|
||||
return None
|
||||
|
||||
# Ищем ссылку вида /{city}/kupit/novostrojka/<slug>-<id>/
|
||||
# Ограничиваем: id в ссылке должен совпадать с искомым jk_id.
|
||||
for m in _JK_SLUG_RE.finditer(html):
|
||||
if m.group(2) == str(jk_id):
|
||||
slug = m.group(1)
|
||||
logger.info("resolve_yandex_jk_slug jk_id=%s → slug=%s", jk_id, slug)
|
||||
return slug
|
||||
|
||||
logger.warning(
|
||||
"resolve_yandex_jk_slug jk_id=%s: no matching slug in SERP HTML (markup drift?)",
|
||||
jk_id,
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
# ── helpers ───────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _extract_coords(html: str) -> tuple[float | None, float | None]:
|
||||
"""Scan HTML for any `<lat>.\\d+` and `<lon>.\\d+` matching plausible city range."""
|
||||
lats = [
|
||||
float(m)
|
||||
for m in re.findall(r"\b(\d{2}\.\d{4,8})\b", html)
|
||||
if LAT_RANGE[0] <= float(m) <= LAT_RANGE[1]
|
||||
]
|
||||
lons = [
|
||||
float(m)
|
||||
for m in re.findall(r"\b(\d{2}\.\d{4,8})\b", html)
|
||||
if LON_RANGE[0] <= float(m) <= LON_RANGE[1]
|
||||
]
|
||||
lat = lats[0] if lats else None
|
||||
lon = lons[0] if lons else None
|
||||
return lat, lon
|
||||
|
||||
|
||||
def _find_section_text(tree: HTMLParser, heading: str) -> str | None:
|
||||
for h in tree.css("h2, h3"):
|
||||
if heading.lower() in (h.text(strip=True) or "").lower():
|
||||
parts: list[str] = []
|
||||
node: Node | None = h.next
|
||||
while node is not None:
|
||||
tag = (node.tag or "").lower()
|
||||
if tag in {"h2", "h3"}:
|
||||
break
|
||||
txt = node.text(strip=True) if hasattr(node, "text") else ""
|
||||
if txt:
|
||||
parts.append(txt)
|
||||
node = node.next
|
||||
return " ".join(parts).strip() or None
|
||||
return None
|
||||
|
||||
|
||||
def _parse_corpus_count(text: str) -> int | None:
|
||||
m = RE_CORPUS_COUNT_WORD.search(text)
|
||||
if not m:
|
||||
return None
|
||||
token = m.group(1).lower()
|
||||
if token.isdigit():
|
||||
return int(token)
|
||||
return _WORD_NUM.get(token)
|
||||
|
||||
|
||||
def _parse_metro(text: str) -> list[JKMetroStation]:
|
||||
stations: list[JKMetroStation] = []
|
||||
for m in RE_METRO_INLINE.finditer(text):
|
||||
name = m.group(1).strip()
|
||||
# filter common noise
|
||||
if not 2 <= len(name) <= 40 or "ходьб" in name.lower() or "минут" in name.lower():
|
||||
continue
|
||||
try:
|
||||
stations.append(JKMetroStation(name=name, walk_min=int(m.group(2))))
|
||||
except ValueError:
|
||||
continue
|
||||
if len(stations) >= 5:
|
||||
break
|
||||
return stations
|
||||
|
||||
|
||||
def _extract_jk_address(body_text: str, name: str | None) -> str | None:
|
||||
"""Find address line — typically right after JK name, before metro entries."""
|
||||
if name and name in body_text:
|
||||
after_name = body_text.split(name, 1)[1]
|
||||
# Match 'Город, ул. X / ул. Y' until first metro mention
|
||||
m = re.match(
|
||||
r"\s*([А-ЯЁ][^•]{5,150}?)(?:\s+[А-ЯЁ][а-яё]+\s+\d+\s*мин|\d+\s+оценок|$)",
|
||||
after_name,
|
||||
)
|
||||
if m:
|
||||
addr = m.group(1).strip().rstrip(",").strip()
|
||||
if 10 < len(addr) < 200:
|
||||
return addr
|
||||
return None
|
||||
|
||||
|
||||
# Expose RE_JK_ID at module level for external use (e.g. SERP link → newbuilding detail)
|
||||
__all__ = [
|
||||
"RE_JK_ID",
|
||||
"JKMetroStation",
|
||||
"YandexNewbuildingInfo",
|
||||
"YandexNewbuildingScraper",
|
||||
"resolve_yandex_jk_slug",
|
||||
]
|
||||
File diff suppressed because it is too large
Load diff
|
|
@ -0,0 +1,95 @@
|
|||
"""Нормализация house_type → каноничный enum для listings.house_type.
|
||||
|
||||
Strangler-копия `app.services.scrapers.house_type_normalizer` (#2133). Локальная в
|
||||
yandex-провайдере, потому что house_type_normalizer ещё не перенесён в core-слой
|
||||
scraper_kit (это отдельный шаг консолидации E). Копия — pure-функция без `app.*`
|
||||
зависимостей; идентична источнику.
|
||||
|
||||
Целевой канон (сверено с estimator._IMV_HOUSE_TYPE_MAP, estimator.py:146):
|
||||
panel / brick / monolith / monolith_brick / block / wood.
|
||||
|
||||
Зачем (#2007): estimator применяет soft-penalty по house_type — аналог с
|
||||
house_type != target штрафуется. Yandex SERP отдаёт SCREAMING-значения
|
||||
(MONOLIT / BRICK / PANEL / ...), которые НИКОГДА не равны каноничным
|
||||
(monolith / brick / panel / ...) → ~70% yandex-аналогов получали ложный штраф
|
||||
и фактически выпадали из скоринга. Нормализация на ингесте чинит это.
|
||||
|
||||
Важно про None: неизвестное / 'other' / '' → None, НЕ 'other'. В estimator
|
||||
`house_type IS NULL` нейтрально (без штрафа), а любое не-канон значение всегда
|
||||
!= target → ложный штраф. Поэтому unknown лучше схлопнуть в NULL.
|
||||
|
||||
Источники raw-значений:
|
||||
- yandex SERP: building.buildingType — SCREAMING_SNAKE (MONOLIT, MONOLIT_BRICK, ...)
|
||||
- cian SERP: building.materialType — camelCase (monolith, monolithBrick,
|
||||
gasSilicateBlock, ...) — переиспользуется в #2008 (cian.py:815).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Канон listings.house_type — выровнен по estimator._IMV_HOUSE_TYPE_MAP.
|
||||
_CANON: frozenset[str] = frozenset(
|
||||
{"panel", "brick", "monolith", "monolith_brick", "block", "wood"}
|
||||
)
|
||||
|
||||
# raw-токен → канон. Ключи — точные значения вокабуляров источников; сравнение
|
||||
# регистронезависимое (см. _LOOKUP ниже), но строго по полному токену енума,
|
||||
# без fuzzy-матчинга. Неизвестные токены сюда НЕ попадают → normalize вернёт None.
|
||||
_RAW_TO_CANON: dict[str, str] = {
|
||||
# ── yandex SERP (SCREAMING_SNAKE buildingType) ───────────────────────────
|
||||
"MONOLIT": "monolith",
|
||||
"BRICK": "brick",
|
||||
"PANEL": "panel",
|
||||
"MONOLIT_BRICK": "monolith_brick",
|
||||
"BLOCK": "block",
|
||||
"WOOD": "wood",
|
||||
# ── cian SERP (camelCase materialType) — reuse в #2008 ───────────────────
|
||||
"monolith": "monolith",
|
||||
"brick": "brick",
|
||||
"panel": "panel",
|
||||
"block": "block",
|
||||
"wood": "wood",
|
||||
"monolithBrick": "monolith_brick",
|
||||
"gasSilicateBlock": "block",
|
||||
"aerocreteBlock": "block",
|
||||
"foamConcreteBlock": "block",
|
||||
"stalin": "brick", # «сталинка» — кирпич
|
||||
}
|
||||
|
||||
# Регистронезависимый lookup. Лоуэркейс-ключи не коллизят между вокабулярами:
|
||||
# 'MONOLIT'→'monolit' и 'monolith'→'monolith' — разные ключи.
|
||||
_LOOKUP: dict[str, str] = {k.lower(): v for k, v in _RAW_TO_CANON.items()}
|
||||
|
||||
|
||||
def normalize_house_type(raw: str | None) -> str | None:
|
||||
"""Преобразовать raw house_type в каноничный enum.
|
||||
|
||||
Уже-каноничное значение возвращается as-is (идемпотентность — нужна при
|
||||
ре-обработке и для backfill-миграции). Неизвестное / 'other' / пустое /
|
||||
None → None (НЕ 'other': см. docstring модуля про estimator soft-penalty).
|
||||
|
||||
Args:
|
||||
raw: сырое значение из парсера (e.g. «MONOLIT», «monolithBrick») или None.
|
||||
|
||||
Returns:
|
||||
Одно из panel / brick / monolith / monolith_brick / block / wood, либо None.
|
||||
"""
|
||||
if raw is None:
|
||||
return None
|
||||
stripped = raw.strip()
|
||||
if not stripped:
|
||||
return None
|
||||
# Pass-through: уже каноничное значение (idempotency — в т.ч. 'monolith_brick',
|
||||
# которого нет среди raw-ключей карты).
|
||||
if stripped in _CANON:
|
||||
return stripped
|
||||
# Регистронезависимый exact-token lookup по обоим вокабулярам.
|
||||
canon = _LOOKUP.get(stripped) or _LOOKUP.get(stripped.lower())
|
||||
if canon is None:
|
||||
# Неизвестное / 'other' — нормально (NULL нейтрально для estimator).
|
||||
# debug, не warning: 'other' встречается массово, warning засорил бы лог.
|
||||
logger.debug("house_type_normalizer: unmapped raw value %r — stored as NULL", raw)
|
||||
return canon
|
||||
|
|
@ -0,0 +1,543 @@
|
|||
"""Yandex Realty anonymous valuation/history scraper.
|
||||
|
||||
URL pattern:
|
||||
GET /otsenka-kvartiry-po-adresu-onlayn/?address=...&offerCategory=...&offerType=...&page=N
|
||||
|
||||
No cookies, no auth required (unlike Cian Calculator or Avito IMV).
|
||||
|
||||
Extracts:
|
||||
- House metadata: year_built, total_floors, house_type, ceiling_height, has_lift, total_objects
|
||||
- Historical offers: area, rooms, floor, start/last price + per-m2, publish_date DMY, exposure
|
||||
|
||||
Two-strategy history extraction:
|
||||
1. data-test container selectors (CSS, if Yandex adds them)
|
||||
2. Fallback: text chunks around DD.MM.YYYY date matches with dedup by (date, area, floor)
|
||||
|
||||
Strangler-копия `app.services.scrapers.yandex_valuation` (#2133). Развязка от `app.*`:
|
||||
- `app.core.config.settings` → инжектируемый `ScraperConfig` (scraper_proxy_url)
|
||||
- `app.services.scraper_settings.get_scraper_delay` → инжектируемый `delay_provider`
|
||||
- `app.services.scrapers.base` → `scraper_kit.base`
|
||||
- `app.services.scrapers.yandex_helpers` → `scraper_kit.yandex_helpers`
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
from datetime import date
|
||||
from typing import TYPE_CHECKING, Any
|
||||
from urllib.parse import urlencode
|
||||
|
||||
from curl_cffi.requests import AsyncSession as _CurlCffiSession
|
||||
from pydantic import BaseModel, Field
|
||||
from selectolax.parser import HTMLParser
|
||||
|
||||
from scraper_kit.base import BaseScraper
|
||||
from scraper_kit.yandex_helpers import (
|
||||
parse_dmy,
|
||||
parse_house_type,
|
||||
parse_rub,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from scraper_kit.contracts import ScraperConfig
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Models
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class ValuationHouseMeta(BaseModel):
|
||||
"""House metadata extracted from Yandex valuation page."""
|
||||
|
||||
year_built: int | None = None
|
||||
total_floors: int | None = None
|
||||
house_type: str | None = None # panel/brick/monolith/...
|
||||
ceiling_height: float | None = None # in meters, e.g. 2.5
|
||||
has_lift: bool | None = None
|
||||
total_objects: int | None = None # 'N объектов' (full archive count)
|
||||
has_panorama: bool = False # 'Панорама' label present
|
||||
|
||||
def validate_match(
|
||||
self,
|
||||
expected_year_built: int | None = None,
|
||||
expected_total_floors: int | None = None,
|
||||
) -> float:
|
||||
"""Return confidence 0..1 that this house meta matches expected values.
|
||||
|
||||
Used after fetching valuation by address to detect when Yandex returned a
|
||||
different house (ambiguous address geocoding). Tolerance ±1 year, ±1 floor.
|
||||
|
||||
Both expected=None → 1.0 (no check). Mismatch on any dimension → 0.0 for it.
|
||||
"""
|
||||
score = 0.0
|
||||
checks = 0
|
||||
if expected_year_built is not None:
|
||||
checks += 1
|
||||
if self.year_built is not None and abs(self.year_built - expected_year_built) <= 1:
|
||||
score += 1.0
|
||||
if expected_total_floors is not None:
|
||||
checks += 1
|
||||
if (
|
||||
self.total_floors is not None
|
||||
and abs(self.total_floors - expected_total_floors) <= 1
|
||||
):
|
||||
score += 1.0
|
||||
if checks == 0:
|
||||
return 1.0
|
||||
return score / checks
|
||||
|
||||
|
||||
class ValuationHistoryItem(BaseModel):
|
||||
"""One historical offer entry from the valuation page."""
|
||||
|
||||
area_m2: float | None = None
|
||||
rooms: int | None = None
|
||||
floor: int | None = None
|
||||
start_price: int | None = None
|
||||
start_price_per_m2: int | None = None
|
||||
last_price: int | None = None
|
||||
last_price_per_m2: int | None = None
|
||||
publish_date: date | None = None
|
||||
removed_date: date | None = None # ← NEW
|
||||
exposure_days: int | None = None
|
||||
status: str | None = None # 'В продаже' / 'Снято'
|
||||
|
||||
|
||||
class YandexValuationResult(BaseModel):
|
||||
"""Full result of one /otsenka-... GET."""
|
||||
|
||||
address: str
|
||||
offer_category: str
|
||||
offer_type: str
|
||||
page: int
|
||||
source_url: str
|
||||
house: ValuationHouseMeta
|
||||
history_items: list[ValuationHistoryItem] = Field(default_factory=list)
|
||||
raw_payload: dict[str, Any] | None = None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Regex constants
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
RE_YEAR_BUILT = re.compile(r"Дом\s+(\d{4})\s+года", re.IGNORECASE)
|
||||
RE_FLOORS = re.compile(r"(\d+)\s+этажей", re.IGNORECASE)
|
||||
RE_CEILING = re.compile(r"([\d,]+)\s*м\s+потолки", re.IGNORECASE)
|
||||
RE_TOTAL_OBJECTS = re.compile(r"(\d+)\s+объект", re.IGNORECASE)
|
||||
|
||||
# Lookbehind blocks digit-only adjacency (rejects year-concat like '20244,6')
|
||||
# while still allowing tokens preceded by punctuation/letter. Min 2 digits
|
||||
# rejects sub-fragments like '2,2' that come from inside '52,2'. Max 4 digits
|
||||
# whole part catches obvious junk (penthouse extremes are <500 m² in practice;
|
||||
# sanity cap from PR #538 backs this up).
|
||||
RE_ITEM_AREA = re.compile(r"(?<!\d)(\d{2,4}(?:[.,]\d{1,2})?)\s*м²")
|
||||
RE_ITEM_ROOMS = re.compile(r"(\d+)\s*-\s*комнатн", re.IGNORECASE)
|
||||
RE_ITEM_STUDIO = re.compile(r"студи[яюй]", re.IGNORECASE)
|
||||
RE_ITEM_FLOOR = re.compile(r"(\d+)\s*этаж", re.IGNORECASE)
|
||||
RE_ITEM_EXPOSURE = re.compile(r"экспозиции\s+(\d+)\s+дн", re.IGNORECASE)
|
||||
RE_ITEM_STATUS = re.compile(r"(В\s+продаже|Снят[оа])", re.IGNORECASE)
|
||||
|
||||
# Matches total-price tokens (rubles) — excludes per-m2 tokens by negative lookahead
|
||||
_RE_PRICE_TOKEN = re.compile(r"(?:\d[\d\s]*\d|\d)(?:[.,]\d+)?\s*(?:млн)?\s*₽(?!\s*за\s*м²)")
|
||||
_RE_PPM2_TOKEN = re.compile(r"\d[\d\s]*\s*₽\s*за\s*м²")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Scraper
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class YandexValuationScraper(BaseScraper):
|
||||
"""Anonymous Yandex valuation/history scraper.
|
||||
|
||||
Fetches https://realty.yandex.ru/otsenka-kvartiry-po-adresu-onlayn/ by address.
|
||||
Pagination via ?page=N. Use fetch_house_history() directly;
|
||||
fetch_around() is not applicable to this tool.
|
||||
"""
|
||||
|
||||
name = "yandex_valuation"
|
||||
base_url = "https://realty.yandex.ru"
|
||||
valuation_path = "/otsenka-kvartiry-po-adresu-onlayn/"
|
||||
request_delay_sec = 5.0 # class default; instance value injected via delay_provider
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: ScraperConfig,
|
||||
*,
|
||||
delay_provider: Callable[[str], float] | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
# Strangler-инжекция (#2133): конфиг (прокси-egress) и провайдер задержки
|
||||
# приходят снаружи вместо app.core.config.settings /
|
||||
# app.services.scraper_settings.get_scraper_delay.
|
||||
self._config = config
|
||||
if delay_provider is not None:
|
||||
self.request_delay_sec = delay_provider(self.name)
|
||||
self._cffi_session: _CurlCffiSession | None = None
|
||||
|
||||
async def __aenter__(self) -> YandexValuationScraper: # type: ignore[override]
|
||||
# Override: Yandex valuation endpoint gates SSR data on Chrome TLS
|
||||
# fingerprint. Plain httpx returns shell HTML (CSR-only).
|
||||
# Sibling: yandex_realty.py / scripts/local-sweep-ekb-yandex.py
|
||||
# Mobile proxy wiring (#806 follow-up): route via mobile proxy to avoid
|
||||
# datacenter-IP blocks. proxy=None → прямое подключение (dev).
|
||||
_proxy_url = self._config.scraper_proxy_url
|
||||
_proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None
|
||||
self._cffi_session = _CurlCffiSession(impersonate="chrome120", proxies=_proxies)
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *args: object) -> None: # type: ignore[override]
|
||||
if self._cffi_session is not None:
|
||||
await self._cffi_session.close()
|
||||
self._cffi_session = None
|
||||
|
||||
async def _http_get(self, url: str, **kwargs: object) -> object: # type: ignore[override]
|
||||
"""curl_cffi-based GET with Chrome120 impersonation.
|
||||
|
||||
Returns curl_cffi Response (compatible API: .status_code, .text).
|
||||
Caller must check status_code; no automatic retry (BaseScraper.retry
|
||||
decorator is per-method and not inherited cleanly when overridden).
|
||||
"""
|
||||
if self._cffi_session is None:
|
||||
raise RuntimeError("YandexValuationScraper must be used as async context manager")
|
||||
kwargs.setdefault("timeout", 30)
|
||||
return await self._cffi_session.get(url, **kwargs)
|
||||
|
||||
async def fetch_around(self, lat: float, lon: float, radius_m: int = 1000) -> list: # type: ignore[override]
|
||||
raise NotImplementedError(
|
||||
"YandexValuationScraper is address-based; use fetch_house_history() instead."
|
||||
)
|
||||
|
||||
async def fetch_house_history(
|
||||
self,
|
||||
address: str,
|
||||
offer_category: str = "APARTMENT",
|
||||
offer_type: str = "SELL",
|
||||
page: int = 1,
|
||||
) -> YandexValuationResult | None:
|
||||
"""Fetch and parse one page of house history for the given address.
|
||||
|
||||
Args:
|
||||
address: Postal address string (will be URL-encoded).
|
||||
offer_category: 'APARTMENT' / 'ROOMS' / etc.
|
||||
offer_type: 'SELL' / 'RENT'.
|
||||
page: 1-based pagination index.
|
||||
|
||||
Returns:
|
||||
YandexValuationResult or None on HTTP error / network failure.
|
||||
"""
|
||||
params = {
|
||||
"address": address,
|
||||
"offerCategory": offer_category,
|
||||
"offerType": offer_type,
|
||||
"page": page,
|
||||
}
|
||||
url = f"{self.base_url}{self.valuation_path}?{urlencode(params)}"
|
||||
try:
|
||||
response = await self._http_get(url)
|
||||
except Exception:
|
||||
logger.exception("yandex valuation fetch failed: %s", url)
|
||||
return None
|
||||
if response.status_code != 200:
|
||||
logger.warning("yandex valuation returned %d for %s", response.status_code, url)
|
||||
return None
|
||||
result = self.parse(
|
||||
response.text,
|
||||
address=address,
|
||||
offer_category=offer_category,
|
||||
offer_type=offer_type,
|
||||
page=page,
|
||||
source_url=url,
|
||||
)
|
||||
await self.sleep_between_requests()
|
||||
return result
|
||||
|
||||
def parse(
|
||||
self,
|
||||
html: str,
|
||||
address: str,
|
||||
offer_category: str,
|
||||
offer_type: str,
|
||||
page: int,
|
||||
source_url: str,
|
||||
) -> YandexValuationResult:
|
||||
"""Parse raw HTML into YandexValuationResult. Pure function — usable in unit tests."""
|
||||
html_normalized = html.replace("\xa0", " ")
|
||||
tree = HTMLParser(html_normalized)
|
||||
body = tree.body
|
||||
body_text = (body.text(strip=True) if body else "").replace("\xa0", " ")
|
||||
|
||||
house = self._parse_house_meta(body_text)
|
||||
history_items = self._parse_history_items(tree, body_text)
|
||||
|
||||
return YandexValuationResult(
|
||||
address=address,
|
||||
offer_category=offer_category,
|
||||
offer_type=offer_type,
|
||||
page=page,
|
||||
source_url=source_url,
|
||||
house=house,
|
||||
history_items=history_items,
|
||||
raw_payload={
|
||||
"body_len": len(body_text),
|
||||
"items_count": len(history_items),
|
||||
},
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _parse_house_meta(body_text: str) -> ValuationHouseMeta:
|
||||
"""Extract house-level metadata from page body text."""
|
||||
year_m = RE_YEAR_BUILT.search(body_text)
|
||||
floors_m = RE_FLOORS.search(body_text)
|
||||
ceiling_m = RE_CEILING.search(body_text)
|
||||
objects_m = RE_TOTAL_OBJECTS.search(body_text)
|
||||
return ValuationHouseMeta(
|
||||
year_built=int(year_m.group(1)) if year_m else None,
|
||||
total_floors=int(floors_m.group(1)) if floors_m else None,
|
||||
house_type=parse_house_type(body_text),
|
||||
ceiling_height=(float(ceiling_m.group(1).replace(",", ".")) if ceiling_m else None),
|
||||
has_lift="Лифт" in body_text,
|
||||
total_objects=int(objects_m.group(1)) if objects_m else None,
|
||||
has_panorama="Панорама" in body_text,
|
||||
)
|
||||
|
||||
def _parse_history_items(self, tree: HTMLParser, body_text: str) -> list[ValuationHistoryItem]:
|
||||
"""Extract historical offer items.
|
||||
|
||||
Strategy 1 (preferred, structured): each row at .OffersArchiveSearchOffers__row,
|
||||
with 6 cells (.OffersArchiveSearchOffers__cell) in fixed order:
|
||||
[0] area+rooms, [1] floor, [2] start_price+ppm2, [3] last_price+ppm2,
|
||||
[4] publish_date+exposure, [5] status/removed_date.
|
||||
Yields per-cell parsing — no chunked-text ambiguity.
|
||||
|
||||
Strategy 2 (last-resort fallback): legacy chunked-text scan around
|
||||
DD.MM.YYYY anchors. Kept for the rare case Yandex changes class names;
|
||||
triggers a logger.warning so we notice.
|
||||
"""
|
||||
items: list[ValuationHistoryItem] = []
|
||||
for row in tree.css(".OffersArchiveSearchOffers__row"):
|
||||
item = self._parse_row_cells(row)
|
||||
if item:
|
||||
items.append(item)
|
||||
if items:
|
||||
return items
|
||||
# Fallback (only if structured selector failed)
|
||||
logger.warning(
|
||||
"yandex_valuation: .OffersArchiveSearchOffers__row matched 0 items — "
|
||||
"falling back to chunked-text scan"
|
||||
)
|
||||
return self._parse_items_from_chunked_text(body_text)
|
||||
|
||||
@classmethod
|
||||
def _parse_row_cells(cls, row) -> ValuationHistoryItem | None:
|
||||
"""Parse one .OffersArchiveSearchOffers__row into a ValuationHistoryItem.
|
||||
|
||||
Cell layout is positional (no labels). Returns None if the row doesn't
|
||||
match the expected 6-cell structure (defensive — Yandex may A/B-test).
|
||||
"""
|
||||
cells = row.css(".OffersArchiveSearchOffers__cell")
|
||||
if len(cells) < 5:
|
||||
return None
|
||||
cell_texts = [(c.text(strip=True) or "").replace("\xa0", " ") for c in cells]
|
||||
|
||||
# [0] photo (skipped — empty cell)
|
||||
# [1] title — single cell contains area + rooms + floor jammed together,
|
||||
# e.g. "48,5 м², 1-комнатная19 этаж" (real Yandex layout, 2026-05-24)
|
||||
title_text = cell_texts[1] if len(cell_texts) > 1 else ""
|
||||
|
||||
area_m = RE_ITEM_AREA.search(title_text)
|
||||
area_m2: float | None = float(area_m.group(1).replace(",", ".")) if area_m else None
|
||||
# Sanity drop (belt-and-suspenders from PR #538)
|
||||
if area_m2 is not None and (area_m2 > 10_000 or area_m2 <= 0):
|
||||
area_m2 = None
|
||||
|
||||
rooms: int | None = None
|
||||
if RE_ITEM_STUDIO.search(title_text):
|
||||
rooms = 0
|
||||
else:
|
||||
rooms_m = RE_ITEM_ROOMS.search(title_text)
|
||||
if rooms_m:
|
||||
rooms = int(rooms_m.group(1))
|
||||
|
||||
floor_m = RE_ITEM_FLOOR.search(title_text)
|
||||
floor: int | None = int(floor_m.group(1)) if floor_m else None
|
||||
|
||||
# [2] start price + ppm2 — "5,1 млн ₽105 155 ₽ за м²" (no separator)
|
||||
start_price: int | None = None
|
||||
start_ppm2: int | None = None
|
||||
if len(cell_texts) > 2:
|
||||
tokens = _RE_PRICE_TOKEN.findall(cell_texts[2])
|
||||
if tokens:
|
||||
start_price = parse_rub(tokens[0])
|
||||
ppm2_tokens = _RE_PPM2_TOKEN.findall(cell_texts[2])
|
||||
if ppm2_tokens:
|
||||
start_ppm2 = parse_rub(ppm2_tokens[0])
|
||||
|
||||
# [3] last price + ppm2
|
||||
last_price: int | None = None
|
||||
last_ppm2: int | None = None
|
||||
if len(cell_texts) > 3:
|
||||
tokens = _RE_PRICE_TOKEN.findall(cell_texts[3])
|
||||
if tokens:
|
||||
last_price = parse_rub(tokens[0])
|
||||
ppm2_tokens = _RE_PPM2_TOKEN.findall(cell_texts[3])
|
||||
if ppm2_tokens:
|
||||
last_ppm2 = parse_rub(ppm2_tokens[0])
|
||||
|
||||
# [4] publish_date + exposure ("23.10.2023В экспозиции 945 дней")
|
||||
publish_date = None
|
||||
exposure_days: int | None = None
|
||||
if len(cell_texts) > 4:
|
||||
publish_date = parse_dmy(cell_texts[4])
|
||||
expo_m = RE_ITEM_EXPOSURE.search(cell_texts[4])
|
||||
if expo_m:
|
||||
exposure_days = int(expo_m.group(1))
|
||||
|
||||
# [5] status or removed_date
|
||||
removed_date = None
|
||||
status: str | None = None
|
||||
if len(cell_texts) > 5:
|
||||
last_cell = cell_texts[5]
|
||||
status_m = RE_ITEM_STATUS.search(last_cell)
|
||||
if status_m:
|
||||
status = status_m.group(1)
|
||||
removed_date = parse_dmy(last_cell)
|
||||
|
||||
# Sort dates chronologically (in case Yandex flips them — same fix as PR #541)
|
||||
if publish_date and removed_date and removed_date < publish_date:
|
||||
publish_date, removed_date = removed_date, publish_date
|
||||
|
||||
# Row is valid only if we got area OR start_price
|
||||
if area_m2 is None and start_price is None:
|
||||
return None
|
||||
|
||||
return ValuationHistoryItem(
|
||||
area_m2=area_m2,
|
||||
rooms=rooms,
|
||||
floor=floor,
|
||||
start_price=start_price,
|
||||
start_price_per_m2=start_ppm2,
|
||||
last_price=last_price,
|
||||
last_price_per_m2=last_ppm2,
|
||||
publish_date=publish_date,
|
||||
removed_date=removed_date,
|
||||
exposure_days=exposure_days,
|
||||
status=status,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _parse_item_text(text: str) -> ValuationHistoryItem | None:
|
||||
"""Parse a single text block into a ValuationHistoryItem.
|
||||
|
||||
Returns None if neither area nor start_price can be extracted
|
||||
(item is considered invalid/noise).
|
||||
"""
|
||||
if not text or len(text) < 20:
|
||||
return None
|
||||
|
||||
# area + rooms
|
||||
area_m = RE_ITEM_AREA.search(text)
|
||||
area_m2: float | None = float(area_m.group(1).replace(",", ".")) if area_m else None
|
||||
# Sanity cap — Yandex page text sometimes concatenates digits across DOM
|
||||
# boundaries (e.g. "2025\xa0106,7 м²" → 2_025_106.7). Flats over 10_000 m²
|
||||
# are impossible; DB column is NUMERIC(8,2) which overflows at 10^6.
|
||||
# Drop the value rather than block the whole save batch.
|
||||
if area_m2 is not None and (area_m2 > 10_000 or area_m2 <= 0):
|
||||
logger.warning(
|
||||
"yandex_valuation: dropping nonsensical area_m2=%s from item chunk",
|
||||
area_m2,
|
||||
)
|
||||
area_m2 = None
|
||||
|
||||
rooms: int | None = None
|
||||
if RE_ITEM_STUDIO.search(text):
|
||||
rooms = 0
|
||||
else:
|
||||
rooms_m = RE_ITEM_ROOMS.search(text)
|
||||
if rooms_m:
|
||||
rooms = int(rooms_m.group(1))
|
||||
|
||||
floor_m = RE_ITEM_FLOOR.search(text)
|
||||
floor = int(floor_m.group(1)) if floor_m else None
|
||||
|
||||
# Prices — find first 2 price tokens (start, last)
|
||||
price_tokens = _RE_PRICE_TOKEN.findall(text)
|
||||
start_price = parse_rub(price_tokens[0]) if len(price_tokens) >= 1 else None
|
||||
last_price = parse_rub(price_tokens[1]) if len(price_tokens) >= 2 else None
|
||||
|
||||
ppm2_tokens = _RE_PPM2_TOKEN.findall(text)
|
||||
start_ppm2 = parse_rub(ppm2_tokens[0]) if len(ppm2_tokens) >= 1 else None
|
||||
last_ppm2 = parse_rub(ppm2_tokens[1]) if len(ppm2_tokens) >= 2 else None
|
||||
|
||||
# Extract ALL DD.MM.YYYY dates: first → publish_date, second → removed_date
|
||||
date_matches = list(re.finditer(r"\d{2}\.\d{2}\.\d{4}", text))
|
||||
dates_parsed: list[date] = []
|
||||
for m in date_matches:
|
||||
d = parse_dmy(m.group(0))
|
||||
if d is not None:
|
||||
dates_parsed.append(d)
|
||||
# Sort dates chronologically — chunked-text scan returns them in page-text
|
||||
# order, but semantically publish_date is earliest and removed_date is
|
||||
# latest. Without this, ~1% of rows on prod show removed < publish.
|
||||
if dates_parsed:
|
||||
dates_sorted = sorted(d for d in dates_parsed if d is not None)
|
||||
publish_date = dates_sorted[0] if dates_sorted else None
|
||||
removed_date = dates_sorted[1] if len(dates_sorted) >= 2 else None
|
||||
else:
|
||||
publish_date = None
|
||||
removed_date = None
|
||||
expo_m = RE_ITEM_EXPOSURE.search(text)
|
||||
exposure_days = int(expo_m.group(1)) if expo_m else None
|
||||
|
||||
status_m = RE_ITEM_STATUS.search(text)
|
||||
status = status_m.group(1) if status_m else None
|
||||
|
||||
# Item is valid only if we got at least area or price
|
||||
if area_m2 is None and start_price is None:
|
||||
return None
|
||||
|
||||
return ValuationHistoryItem(
|
||||
area_m2=area_m2,
|
||||
rooms=rooms,
|
||||
floor=floor,
|
||||
start_price=start_price,
|
||||
start_price_per_m2=start_ppm2,
|
||||
last_price=last_price,
|
||||
last_price_per_m2=last_ppm2,
|
||||
publish_date=publish_date,
|
||||
removed_date=removed_date,
|
||||
exposure_days=exposure_days,
|
||||
status=status,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _parse_items_from_chunked_text(cls, body_text: str) -> list[ValuationHistoryItem]:
|
||||
"""Fallback: split body text into chunks around DD.MM.YYYY dates and parse each chunk.
|
||||
|
||||
Window: 200 chars before + 100 chars after each date match.
|
||||
Deduplicates by (publish_date, area_m2, floor).
|
||||
Caps output at 30 items per page to avoid runaway extraction.
|
||||
"""
|
||||
items: list[ValuationHistoryItem] = []
|
||||
for m in re.finditer(r"\d{2}\.\d{2}\.\d{4}", body_text):
|
||||
start = max(0, m.start() - 200)
|
||||
end = min(len(body_text), m.end() + 100)
|
||||
chunk = body_text[start:end]
|
||||
item = cls._parse_item_text(chunk)
|
||||
if item:
|
||||
items.append(item)
|
||||
|
||||
# Dedup by (publish_date, area_m2, floor)
|
||||
seen: set[tuple[Any, Any, Any]] = set()
|
||||
deduped: list[ValuationHistoryItem] = []
|
||||
for item in items:
|
||||
key = (item.publish_date, item.area_m2, item.floor)
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
deduped.append(item)
|
||||
return deduped[:30]
|
||||
Loading…
Add table
Reference in a new issue