662 lines
26 KiB
Python
662 lines
26 KiB
Python
"""Yandex Realty detail page scraper.
|
||
|
||
Fetches /offer/<id>/ and extracts Product JSON-LD + DOM sections into
|
||
a DetailEnrichment Pydantic model. Used by enrichment pipeline
|
||
(Wave 5+ matching / Wave 6 estimator).
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import logging
|
||
import re
|
||
from datetime import date
|
||
from typing import Any
|
||
|
||
from pydantic import BaseModel, Field
|
||
from selectolax.parser import HTMLParser, Node
|
||
from sqlalchemy import text
|
||
from sqlalchemy.orm import Session
|
||
|
||
from app.services.scraper_settings import get_scraper_delay
|
||
from app.services.scrapers.base import BaseScraper
|
||
from app.services.scrapers.repair_state_normalizer import infer_repair_state_from_text
|
||
from app.services.scrapers.yandex_helpers import (
|
||
RE_AGENCY_FOUNDED,
|
||
RE_AGENCY_OBJECTS,
|
||
RE_METRO_WALK,
|
||
RE_VIEWS,
|
||
RE_YEAR,
|
||
find_ld_by_type,
|
||
parse_house_type,
|
||
parse_ru_date,
|
||
parse_rub,
|
||
)
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
|
||
# ── Pydantic models ───────────────────────────────────────────────────────────
|
||
|
||
|
||
class MetroStation(BaseModel):
|
||
name: str
|
||
walk_min: int | None = None
|
||
|
||
|
||
class DetailEnrichment(BaseModel):
|
||
"""Enrichment payload from a Yandex detail page."""
|
||
|
||
offer_id: str
|
||
source_url: str
|
||
|
||
# Pricing — Product JSON-LD `offers.price` is the exact int
|
||
price_rub: int | None = None
|
||
price_per_m2: int | None = None
|
||
|
||
# Title + basic params
|
||
title: str | None = None
|
||
rooms: int | None = None
|
||
area_m2: float | None = None
|
||
living_area_m2: float | None = None
|
||
kitchen_area_m2: float | None = None
|
||
ceiling_height: float | None = None # meters, e.g. 2.55
|
||
floor: int | None = None
|
||
total_floors: int | None = None
|
||
|
||
# Address (full)
|
||
address: str | None = None
|
||
|
||
# Description (full text)
|
||
description: str | None = None
|
||
|
||
# Repair state — enum, inferred from description text (#622).
|
||
# Yandex не отдаёт структурного поля ремонта, поэтому только инференс.
|
||
repair_state: str | None = None
|
||
|
||
# Stats
|
||
views_total: int | None = None
|
||
publish_date: date | None = None
|
||
publish_date_relative: str | None = None
|
||
|
||
# Agency block (OfferCardAuthorInfo)
|
||
agency_name: str | None = None
|
||
agency_founded_year: int | None = None
|
||
agency_objects_count: int | None = None
|
||
seller_name: str | None = None # last text line before "Агентство «...»"
|
||
|
||
# Metro stations from "Расположение" section
|
||
metro_stations: list[MetroStation] = Field(default_factory=list)
|
||
|
||
# Photos — 8 sizes from Product.image[]
|
||
photo_urls: list[str] = Field(default_factory=list)
|
||
|
||
# Newbuilding linkage
|
||
newbuilding_url: str | None = None
|
||
newbuilding_id: str | None = None
|
||
|
||
# NLP from description (best-effort)
|
||
house_type_nlp: str | None = None
|
||
year_built_hint: int | None = None
|
||
metro_walk_min: int | None = None
|
||
|
||
# Raw payload (trimmed)
|
||
raw_payload: dict[str, Any] | None = None
|
||
|
||
|
||
# ── Scraper ───────────────────────────────────────────────────────────────────
|
||
|
||
|
||
class YandexDetailScraper(BaseScraper):
|
||
"""Detail page scraper for realty.yandex.ru."""
|
||
|
||
name = "yandex_detail"
|
||
base_url = "https://realty.yandex.ru"
|
||
request_delay_sec = 5.0 # class default; instance value loaded from scraper_settings
|
||
|
||
def __init__(self) -> None:
|
||
super().__init__()
|
||
self.request_delay_sec = get_scraper_delay(self.name)
|
||
|
||
# BaseScraper requires fetch_around — detail isn't geo-based, raise NotImplementedError
|
||
async def fetch_around(self, lat: float, lon: float, radius_m: int = 1000) -> list: # type: ignore[override]
|
||
raise NotImplementedError(
|
||
"YandexDetailScraper is offer-id-based; use fetch_detail(offer_url) instead."
|
||
)
|
||
|
||
async def fetch_detail(self, offer_url: str) -> DetailEnrichment | None:
|
||
try:
|
||
response = await self._http_get(offer_url)
|
||
except Exception:
|
||
logger.exception("yandex detail fetch failed: %s", offer_url)
|
||
return None
|
||
if response.status_code != 200:
|
||
logger.warning("yandex detail returned %d for %s", response.status_code, offer_url)
|
||
return None
|
||
result = self.parse(response.text, offer_url=offer_url)
|
||
await self.sleep_between_requests()
|
||
return result
|
||
|
||
def parse(self, html: str, offer_url: str) -> DetailEnrichment | None:
|
||
offer_id_match = re.search(r"/offer/(\d+)/?", offer_url)
|
||
if not offer_id_match:
|
||
logger.warning("offer_url has no /offer/<id>/: %s", offer_url)
|
||
return None
|
||
offer_id = offer_id_match.group(1)
|
||
|
||
tree = HTMLParser(html)
|
||
|
||
# --- Product JSON-LD (authoritative price) ---
|
||
product = find_ld_by_type(html, "Product") or {}
|
||
offers_ld = product.get("offers") or {}
|
||
if isinstance(offers_ld, list) and offers_ld:
|
||
offers_ld = offers_ld[0]
|
||
price_ld = offers_ld.get("price") if isinstance(offers_ld, dict) else None
|
||
try:
|
||
price_rub = int(price_ld) if price_ld else None
|
||
except (TypeError, ValueError):
|
||
price_rub = None
|
||
|
||
# Photos from JSON-LD image[] (typically 8 size variants)
|
||
images = product.get("image") or []
|
||
if isinstance(images, str):
|
||
images = [images]
|
||
photo_urls = [u for u in images if isinstance(u, str)]
|
||
|
||
# --- Title + summary ---
|
||
title_node = tree.css_first("h1")
|
||
title = title_node.text(strip=True) if title_node else None
|
||
|
||
rooms, area_m2, floor, total_floors = _parse_title(title or "")
|
||
|
||
# --- Structural offer card (window.INITIAL_STATE → offerCard.card) ---
|
||
# Authoritative source for area/floor/ceiling/kitchen — the h1 title
|
||
# misses non-standard layouts (студия / свободная планировка) and never
|
||
# carries ceiling/kitchen/living. Title stays as fallback below.
|
||
living_area_m2: float | None = None
|
||
kitchen_area_m2: float | None = None
|
||
ceiling_height: float | None = None
|
||
card = _extract_offer_card(html, offer_id)
|
||
if card is not None:
|
||
(
|
||
c_rooms,
|
||
c_area,
|
||
c_living,
|
||
c_kitchen,
|
||
c_ceiling,
|
||
c_floor,
|
||
c_total_floors,
|
||
) = _parse_card_fields(card)
|
||
# Structural source wins; title only fills the gaps it left.
|
||
rooms = c_rooms if c_rooms is not None else rooms
|
||
area_m2 = c_area if c_area is not None else area_m2
|
||
living_area_m2 = c_living
|
||
kitchen_area_m2 = c_kitchen
|
||
ceiling_height = c_ceiling
|
||
floor = c_floor if c_floor is not None else floor
|
||
total_floors = c_total_floors if c_total_floors is not None else total_floors
|
||
|
||
# --- OfferCardSummary text block ---
|
||
summary_node = tree.css_first('[data-test="OfferCardSummary"]')
|
||
summary_text = summary_node.text(strip=True) if summary_node else ""
|
||
|
||
# Views + relative publish date from summary text
|
||
views_match = RE_VIEWS.search(summary_text)
|
||
views_total = int(views_match.group(1)) if views_match else None
|
||
publish_date = parse_ru_date(summary_text)
|
||
publish_date_relative = _extract_relative_date(summary_text)
|
||
|
||
# price_per_m2 — from summary text if absent in LD
|
||
price_per_m2: int | None = None
|
||
ppm2_match = re.search(r"(\d[\d\s]+)\s*₽\s*за\s*м²", summary_text)
|
||
if ppm2_match:
|
||
price_per_m2 = parse_rub(ppm2_match.group(1))
|
||
|
||
# --- OfferCardAuthorInfo (agency block) ---
|
||
author_node = tree.css_first('[data-test="OfferCardAuthorInfo"]')
|
||
agency_name: str | None = None
|
||
agency_founded_year: int | None = None
|
||
agency_objects_count: int | None = None
|
||
seller_name: str | None = None
|
||
if author_node is not None:
|
||
author_text = author_node.text(strip=True)
|
||
agency_h2 = author_node.css_first("h2")
|
||
agency_name = agency_h2.text(strip=True) if agency_h2 else None
|
||
founded_m = RE_AGENCY_FOUNDED.search(author_text)
|
||
if founded_m:
|
||
agency_founded_year = int(founded_m.group(1))
|
||
objects_m = RE_AGENCY_OBJECTS.search(author_text)
|
||
if objects_m:
|
||
agency_objects_count = int(objects_m.group(1))
|
||
# seller_name — last text line before "Агентство"
|
||
seller_name = _extract_seller_name(summary_text, agency_name)
|
||
|
||
# --- Description section (after H2 "Описание") ---
|
||
description = _find_section_text(tree, "Описание")
|
||
|
||
# --- Repair state: инференс из описания (#622), Yandex без структурного поля ---
|
||
repair_state = infer_repair_state_from_text(description or summary_text)
|
||
|
||
# --- Address ---
|
||
address = _extract_address(summary_text)
|
||
|
||
# --- Metro stations from "Расположение" section ---
|
||
location_text = _find_section_text(tree, "Расположение") or ""
|
||
metro_stations = _parse_metro_stations(location_text)
|
||
|
||
# --- Newbuilding link ---
|
||
nb_url: str | None = None
|
||
nb_id: str | None = None
|
||
nb_link = tree.css_first('a[href*="/kupit/novostrojka/"]')
|
||
if nb_link is not None:
|
||
nb_href = nb_link.attributes.get("href", "")
|
||
nb_match = re.search(r"/novostrojka/[\w-]+?-(\d+)/?", nb_href)
|
||
if nb_match:
|
||
nb_id = nb_match.group(1)
|
||
nb_url = (
|
||
nb_href if nb_href.startswith("http") else f"https://realty.yandex.ru{nb_href}"
|
||
)
|
||
|
||
# --- NLP best-effort from description ---
|
||
nlp_text = description or summary_text
|
||
house_type_nlp = parse_house_type(nlp_text)
|
||
year_hint_m = RE_YEAR.search(nlp_text or "")
|
||
year_built_hint = int(year_hint_m.group(1)) if year_hint_m else None
|
||
walk_m = RE_METRO_WALK.search(nlp_text or "")
|
||
metro_walk_min = int(walk_m.group(1)) if walk_m else None
|
||
|
||
return DetailEnrichment(
|
||
offer_id=offer_id,
|
||
source_url=offer_url,
|
||
price_rub=price_rub,
|
||
price_per_m2=price_per_m2,
|
||
title=title,
|
||
rooms=rooms,
|
||
area_m2=area_m2,
|
||
living_area_m2=living_area_m2,
|
||
kitchen_area_m2=kitchen_area_m2,
|
||
ceiling_height=ceiling_height,
|
||
floor=floor,
|
||
total_floors=total_floors,
|
||
address=address,
|
||
description=description,
|
||
repair_state=repair_state,
|
||
views_total=views_total,
|
||
publish_date=publish_date,
|
||
publish_date_relative=publish_date_relative,
|
||
agency_name=agency_name,
|
||
agency_founded_year=agency_founded_year,
|
||
agency_objects_count=agency_objects_count,
|
||
seller_name=seller_name,
|
||
metro_stations=metro_stations,
|
||
photo_urls=photo_urls,
|
||
newbuilding_url=nb_url,
|
||
newbuilding_id=nb_id,
|
||
house_type_nlp=house_type_nlp,
|
||
year_built_hint=year_built_hint,
|
||
metro_walk_min=metro_walk_min,
|
||
raw_payload={
|
||
"summary_text": summary_text[:1000],
|
||
"description_len": len(description) if description else 0,
|
||
"photo_count": len(photo_urls),
|
||
},
|
||
)
|
||
|
||
|
||
# ── Helpers ───────────────────────────────────────────────────────────────────
|
||
|
||
|
||
def _parse_title(title: str) -> tuple[int | None, float | None, int | None, int | None]:
|
||
"""Extract (rooms, area_m2, floor, total_floors) from h1 text."""
|
||
rooms: int | None = None
|
||
area_m2: float | None = None
|
||
floor: int | None = None
|
||
total_floors: int | None = None
|
||
|
||
area_m = re.search(r"(\d+[.,]?\d*)\s*м²", title)
|
||
if area_m:
|
||
area_m2 = float(area_m.group(1).replace(",", "."))
|
||
|
||
if re.search(r"студи[яюй]", title, re.IGNORECASE):
|
||
rooms = 0
|
||
else:
|
||
rooms_m = re.search(r"(\d+)\s*-?\s*комнатн", title, re.IGNORECASE)
|
||
if rooms_m:
|
||
try:
|
||
rooms = int(rooms_m.group(1))
|
||
except ValueError:
|
||
pass
|
||
|
||
floor_m = re.search(r"(\d+)\s+этаж\s+из\s+(\d+)", title, re.IGNORECASE)
|
||
if floor_m:
|
||
floor = int(floor_m.group(1))
|
||
total_floors = int(floor_m.group(2))
|
||
|
||
return rooms, area_m2, floor, total_floors
|
||
|
||
|
||
def _extract_js_object(html: str, marker: str) -> str | None:
|
||
"""Return the JSON object literal assigned to `marker` (e.g. window.INITIAL_STATE).
|
||
|
||
Brace-matches from the first ``{`` after ``marker = ...`` to its balanced
|
||
close, respecting string literals/escapes. Returns the raw JSON text or None.
|
||
"""
|
||
i = html.find(marker)
|
||
if i < 0:
|
||
return None
|
||
eq = html.find("=", i)
|
||
if eq < 0:
|
||
return None
|
||
start = html.find("{", eq)
|
||
if start < 0:
|
||
return None
|
||
bal = 0
|
||
in_str = False
|
||
esc = False
|
||
k = start
|
||
n = len(html)
|
||
while k < n:
|
||
c = html[k]
|
||
if in_str:
|
||
if esc:
|
||
esc = False
|
||
elif c == "\\":
|
||
esc = True
|
||
elif c == '"':
|
||
in_str = False
|
||
else:
|
||
if c == '"':
|
||
in_str = True
|
||
elif c == "{":
|
||
bal += 1
|
||
elif c == "}":
|
||
bal -= 1
|
||
if bal == 0:
|
||
return html[start : k + 1]
|
||
k += 1
|
||
return None
|
||
|
||
|
||
def _find_card_by_offer_id(obj: Any, offer_id: str) -> dict[str, Any] | None:
|
||
"""Recursively locate the offer dict whose offerId matches and carries `area`."""
|
||
if isinstance(obj, dict):
|
||
if obj.get("offerId") == offer_id and "area" in obj:
|
||
return obj
|
||
for v in obj.values():
|
||
found = _find_card_by_offer_id(v, offer_id)
|
||
if found is not None:
|
||
return found
|
||
elif isinstance(obj, list):
|
||
for v in obj:
|
||
found = _find_card_by_offer_id(v, offer_id)
|
||
if found is not None:
|
||
return found
|
||
return None
|
||
|
||
|
||
def _extract_offer_card(html: str, offer_id: str) -> dict[str, Any] | None:
|
||
"""Extract the offer card object from window.INITIAL_STATE.
|
||
|
||
Yandex embeds full structured offer data in ``window.INITIAL_STATE`` under
|
||
``offerCard.card``. This is the authoritative source for area / floor /
|
||
ceiling / kitchen — unlike the h1 title which misses non-standard layouts
|
||
and never carries ceiling/kitchen. Returns the card dict or None.
|
||
"""
|
||
blob = _extract_js_object(html, "window.INITIAL_STATE")
|
||
if not blob:
|
||
return None
|
||
try:
|
||
state = json.loads(blob)
|
||
except (json.JSONDecodeError, ValueError):
|
||
logger.warning("yandex detail: INITIAL_STATE failed to parse for offer %s", offer_id)
|
||
return None
|
||
# Fast path: canonical location.
|
||
offer_card = state.get("offerCard") if isinstance(state, dict) else None
|
||
if isinstance(offer_card, dict):
|
||
card = offer_card.get("card")
|
||
if isinstance(card, dict) and card.get("offerId") == offer_id:
|
||
return card
|
||
# Fallback: deep search (page structure may differ).
|
||
return _find_card_by_offer_id(state, offer_id)
|
||
|
||
|
||
def _space_value(node: Any) -> float | None:
|
||
"""Yandex area fields are ``{"value": N, "unit": "SQUARE_METER"}`` → float."""
|
||
if isinstance(node, dict):
|
||
val = node.get("value")
|
||
if isinstance(val, int | float):
|
||
return float(val)
|
||
elif isinstance(node, int | float):
|
||
return float(node)
|
||
return None
|
||
|
||
|
||
def _parse_card_fields(
|
||
card: dict[str, Any],
|
||
) -> tuple[
|
||
int | None,
|
||
float | None,
|
||
float | None,
|
||
float | None,
|
||
float | None,
|
||
int | None,
|
||
int | None,
|
||
]:
|
||
"""Extract (rooms, area, living, kitchen, ceiling, floor, total_floors) from card.
|
||
|
||
rooms: ``roomsTotal``, or 0 when ``house.studio`` is truthy (студии не
|
||
несут roomsTotal). floor: first element of ``floorsOffered``.
|
||
"""
|
||
area = _space_value(card.get("area"))
|
||
living = _space_value(card.get("livingSpace"))
|
||
kitchen = _space_value(card.get("kitchenSpace"))
|
||
|
||
ceiling_raw = card.get("ceilingHeight")
|
||
ceiling: float | None = None
|
||
if isinstance(ceiling_raw, int | float):
|
||
ceiling = float(ceiling_raw)
|
||
|
||
rooms_raw = card.get("roomsTotal")
|
||
rooms: int | None = None
|
||
if isinstance(rooms_raw, int):
|
||
rooms = rooms_raw
|
||
house = card.get("house")
|
||
if isinstance(house, dict) and house.get("studio"):
|
||
rooms = 0
|
||
|
||
total_floors_raw = card.get("floorsTotal")
|
||
total_floors = total_floors_raw if isinstance(total_floors_raw, int) else None
|
||
|
||
floor: int | None = None
|
||
floors_offered = card.get("floorsOffered")
|
||
if isinstance(floors_offered, list) and floors_offered:
|
||
first = floors_offered[0]
|
||
if isinstance(first, int):
|
||
floor = first
|
||
|
||
return rooms, area, living, kitchen, ceiling, floor, total_floors
|
||
|
||
|
||
def _find_section_text(tree: HTMLParser, heading: str) -> str | None:
|
||
"""Find the text content of a <section>/<div> whose preceding h2/h3 matches heading.
|
||
|
||
Yandex page structure varies; this scans h2/h3 nodes, then returns the
|
||
concatenated text of subsequent sibling blocks until the next heading.
|
||
"""
|
||
for h in tree.css("h2, h3"):
|
||
if heading.lower() in (h.text(strip=True) or "").lower():
|
||
# collect subsequent siblings until the next h2/h3
|
||
parts: list[str] = []
|
||
node: Node | None = h.next
|
||
while node is not None:
|
||
tag = (node.tag or "").lower()
|
||
if tag in {"h2", "h3"}:
|
||
break
|
||
txt = node.text(strip=True) if hasattr(node, "text") else ""
|
||
if txt:
|
||
parts.append(txt)
|
||
node = node.next
|
||
return " ".join(parts).strip() or None
|
||
return None
|
||
|
||
|
||
def _extract_address(summary_text: str) -> str | None:
|
||
"""Best-effort address extraction from summary block."""
|
||
# Pattern: "Россия, Свердловская область, Екатеринбург, улица Х, д. N"
|
||
m = re.search(r"(Россия[^•]+?)(?:•|\d+\s+просмотр|$)", summary_text)
|
||
if m:
|
||
addr = m.group(1).strip().rstrip(",").strip()
|
||
return addr if len(addr) > 10 else None
|
||
return None
|
||
|
||
|
||
def _parse_metro_stations(location_text: str) -> list[MetroStation]:
|
||
"""Parse 'Уральская 11 мин. Динамо 16 мин.' → list of MetroStation."""
|
||
stations: list[MetroStation] = []
|
||
# name (1+ Cyrillic words) + space + N + space + мин(.|у|ут)
|
||
for m in re.finditer(r"([А-ЯЁ][А-Яа-яё\s-]+?)\s+(\d+)\s*мин", location_text):
|
||
name = m.group(1).strip()
|
||
if 2 <= len(name) <= 40:
|
||
stations.append(MetroStation(name=name, walk_min=int(m.group(2))))
|
||
if len(stations) >= 5:
|
||
break
|
||
return stations
|
||
|
||
|
||
def _extract_relative_date(summary_text: str) -> str | None:
|
||
"""Capture phrases like '6 часов назад' / 'вчера' / '3 дня назад'."""
|
||
m = re.search(
|
||
r"(\d+\s+(?:минут|час|часов|часа|день|дня|дней|недел[ьюи])\s+назад"
|
||
r"|вчера|сегодня|позавчера)",
|
||
summary_text,
|
||
re.IGNORECASE,
|
||
)
|
||
return m.group(1).strip() if m else None
|
||
|
||
|
||
def _extract_seller_name(summary_text: str, agency_name: str | None) -> str | None:
|
||
"""Heuristic: line right before 'Агентство ...' in summary text."""
|
||
if not agency_name or agency_name not in summary_text:
|
||
return None
|
||
head = summary_text.split(agency_name, 1)[0]
|
||
# last short token sequence (likely "Имя Фамилия")
|
||
m = re.findall(r"([А-ЯЁ][а-яё]+(?:\s+[А-ЯЁ][а-яё]+){1,2})", head)
|
||
return m[-1] if m else None
|
||
|
||
|
||
# ── Save helper ───────────────────────────────────────────────────────────────
|
||
|
||
|
||
def save_detail_enrichment(db: Session, listing_id: int, e: DetailEnrichment) -> bool:
|
||
"""Persist Yandex DetailEnrichment to listings row.
|
||
|
||
UPDATE listings SET <col> = COALESCE(:val, <col>), ..., detail_enriched_at = NOW()
|
||
WHERE id = listing_id.
|
||
|
||
COALESCE semantics: keeps existing non-NULL value if new value is NULL (never
|
||
overwrites a populated column with NULL). area_m2 from detail is more accurate
|
||
than SERP, but COALESCE preserves SERP value if detail returns NULL — acceptable.
|
||
|
||
Returns True if the UPDATE touched at least one row (listing_id found in DB).
|
||
"""
|
||
metro_json: str | None = None
|
||
if e.metro_stations:
|
||
metro_json = json.dumps(
|
||
[s.model_dump(exclude_none=True) for s in e.metro_stations],
|
||
ensure_ascii=False,
|
||
)
|
||
|
||
photo_json: str | None = None
|
||
if e.photo_urls:
|
||
photo_json = json.dumps(e.photo_urls, ensure_ascii=False)
|
||
|
||
result = db.execute(
|
||
text("""
|
||
UPDATE listings SET
|
||
rooms = COALESCE(CAST(:rooms AS int), rooms),
|
||
area_m2 = COALESCE(CAST(:area_m2 AS numeric), area_m2),
|
||
living_area_m2 = COALESCE(
|
||
CAST(:living_area_m2 AS numeric),
|
||
living_area_m2
|
||
),
|
||
kitchen_area_m2 = COALESCE(
|
||
CAST(:kitchen_area_m2 AS numeric),
|
||
kitchen_area_m2
|
||
),
|
||
ceiling_height = COALESCE(
|
||
CAST(:ceiling_height AS numeric),
|
||
ceiling_height
|
||
),
|
||
floor = COALESCE(CAST(:floor AS int), floor),
|
||
total_floors = COALESCE(CAST(:total_floors AS int), total_floors),
|
||
address = COALESCE(CAST(:address AS text), address),
|
||
description = COALESCE(CAST(:description AS text), description),
|
||
repair_state = COALESCE(CAST(:repair_state AS text),repair_state),
|
||
publish_date = COALESCE(CAST(:publish_date AS date),publish_date),
|
||
views_total_yandex = COALESCE(CAST(:views_total AS int), views_total_yandex),
|
||
publish_date_relative= COALESCE(
|
||
CAST(:pub_date_rel AS text),
|
||
publish_date_relative
|
||
),
|
||
agency_name = COALESCE(CAST(:agency_name AS text), agency_name),
|
||
agency_founded_year = COALESCE(
|
||
CAST(:agency_founded_year AS int),
|
||
agency_founded_year
|
||
),
|
||
agency_objects_count = COALESCE(
|
||
CAST(:agency_objects_count AS int),
|
||
agency_objects_count
|
||
),
|
||
metro_stations = COALESCE(
|
||
CAST(:metro_stations AS jsonb),
|
||
metro_stations
|
||
),
|
||
photo_urls = COALESCE(
|
||
CAST(:photo_urls AS jsonb),
|
||
photo_urls
|
||
),
|
||
newbuilding_url = COALESCE(
|
||
CAST(:newbuilding_url AS text),
|
||
newbuilding_url
|
||
),
|
||
newbuilding_id = COALESCE(
|
||
CAST(:newbuilding_id AS text),
|
||
newbuilding_id
|
||
),
|
||
detail_enriched_at = NOW()
|
||
WHERE id = CAST(:listing_id AS bigint)
|
||
"""),
|
||
{
|
||
"listing_id": listing_id,
|
||
"rooms": e.rooms,
|
||
"area_m2": e.area_m2,
|
||
"living_area_m2": e.living_area_m2,
|
||
"kitchen_area_m2": e.kitchen_area_m2,
|
||
"ceiling_height": e.ceiling_height,
|
||
"floor": e.floor,
|
||
"total_floors": e.total_floors,
|
||
"address": e.address,
|
||
"description": e.description,
|
||
"repair_state": e.repair_state,
|
||
"publish_date": e.publish_date,
|
||
"views_total": e.views_total,
|
||
"pub_date_rel": e.publish_date_relative,
|
||
"agency_name": e.agency_name,
|
||
"agency_founded_year": e.agency_founded_year,
|
||
"agency_objects_count": e.agency_objects_count,
|
||
"metro_stations": metro_json,
|
||
"photo_urls": photo_json,
|
||
"newbuilding_url": e.newbuilding_url,
|
||
"newbuilding_id": e.newbuilding_id,
|
||
},
|
||
)
|
||
db.commit()
|
||
saved = (result.rowcount or 0) > 0
|
||
logger.info(
|
||
"yandex detail enrichment saved listing_id=%s (metro=%d photos=%d saved=%s)",
|
||
listing_id,
|
||
len(e.metro_stations),
|
||
len(e.photo_urls),
|
||
saved,
|
||
)
|
||
return saved
|