fix(scrapers): cian detail — извлечение kitchen_area_m2 (part of #1553) #1555
2 changed files with 119 additions and 2 deletions
|
|
@ -44,6 +44,7 @@ class DetailEnrichment:
|
|||
ceiling_height: float | None = None # meters
|
||||
repair_type: str | None = None # 'cosmetic' / 'design' / 'no' — raw Cian value
|
||||
repair_state: str | None = None # enum: needs_repair/standard/good/excellent
|
||||
kitchen_area_m2: float | None = None # offer.kitchenArea (м²)
|
||||
views_total: int | None = None # Cian stats.totalViewsFormattedString → int
|
||||
views_today: int | None = None
|
||||
|
||||
|
|
@ -119,6 +120,7 @@ async def fetch_detail(
|
|||
ceiling_height=_parse_float(offer.get("ceilingHeight")),
|
||||
repair_type=_extract_repair_type(offer),
|
||||
repair_state=_extract_repair_state(offer),
|
||||
kitchen_area_m2=_parse_float(offer.get("kitchenArea")),
|
||||
raw_offer=offer,
|
||||
)
|
||||
|
||||
|
|
@ -259,6 +261,7 @@ def save_detail_enrichment(db: Session, listing_id: int, enrichment: DetailEnric
|
|||
ceiling_height = COALESCE(:ch, ceiling_height),
|
||||
repair_type = COALESCE(:rt, repair_type),
|
||||
repair_state = COALESCE(:rs, repair_state),
|
||||
kitchen_area_m2 = COALESCE(CAST(:ka AS double precision), kitchen_area_m2),
|
||||
views_total = COALESCE(:vt, views_total),
|
||||
views_today = COALESCE(:vd, views_today)
|
||||
WHERE id = CAST(:lid AS bigint)
|
||||
|
|
@ -271,6 +274,7 @@ def save_detail_enrichment(db: Session, listing_id: int, enrichment: DetailEnric
|
|||
"ch": enrichment.ceiling_height,
|
||||
"rt": enrichment.repair_type,
|
||||
"rs": enrichment.repair_state,
|
||||
"ka": enrichment.kitchen_area_m2,
|
||||
"vt": enrichment.views_total,
|
||||
"vd": enrichment.views_today,
|
||||
},
|
||||
|
|
@ -376,8 +380,7 @@ def save_detail_enrichment(db: Session, listing_id: int, enrichment: DetailEnric
|
|||
|
||||
db.commit()
|
||||
logger.info(
|
||||
"Cian detail enrichment saved for listing_id=%s "
|
||||
"(price_changes=%d, agent=%s, snapshot=%s)",
|
||||
"Cian detail enrichment saved for listing_id=%s (price_changes=%d, agent=%s, snapshot=%s)",
|
||||
listing_id,
|
||||
len(enrichment.price_changes),
|
||||
bool(enrichment.agent_profile),
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
"""Tests for cian_detail.py — Stage 5 of CianScraper v1."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
|
@ -12,6 +13,7 @@ from app.services.scrapers.cian_detail import (
|
|||
_extract_repair_state,
|
||||
_parse_views,
|
||||
fetch_detail,
|
||||
save_detail_enrichment,
|
||||
)
|
||||
|
||||
# ── _parse_views ──────────────────────────────────────────────────────────────
|
||||
|
|
@ -349,3 +351,115 @@ async def test_fetch_detail_extracts_newbuilding_link(monkeypatch):
|
|||
assert result is not None
|
||||
assert result.newbuilding_data is not None
|
||||
assert result.newbuilding_data["id"] == 54016
|
||||
|
||||
|
||||
# ── kitchen_area_m2 extraction ────────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_detail_extracts_kitchen_area(monkeypatch):
|
||||
"""kitchenArea из offer → kitchen_area_m2 в DetailEnrichment."""
|
||||
fake_state = {
|
||||
"offerData": {
|
||||
"offer": {
|
||||
"cianId": 333333,
|
||||
"kitchenArea": 9.5,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
monkeypatch.setattr(
|
||||
"app.services.scrapers.cian_detail.extract_state",
|
||||
lambda html, mfe, key: fake_state if key == "defaultState" else None,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"app.services.scrapers.cian_detail.extract_all_states",
|
||||
lambda html: {},
|
||||
)
|
||||
|
||||
response = MagicMock()
|
||||
response.status_code = 200
|
||||
response.text = "<html></html>"
|
||||
|
||||
session = MagicMock()
|
||||
session.get = AsyncMock(return_value=response)
|
||||
session.close = AsyncMock()
|
||||
|
||||
result = await fetch_detail("https://ekb.cian.ru/sale/flat/333333/", session=session)
|
||||
assert result is not None
|
||||
assert result.kitchen_area_m2 == 9.5
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_detail_kitchen_area_none_when_missing(monkeypatch):
|
||||
"""Если kitchenArea отсутствует в offer — kitchen_area_m2 остаётся None."""
|
||||
fake_state = {
|
||||
"offerData": {
|
||||
"offer": {
|
||||
"cianId": 444444,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
monkeypatch.setattr(
|
||||
"app.services.scrapers.cian_detail.extract_state",
|
||||
lambda html, mfe, key: fake_state if key == "defaultState" else None,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"app.services.scrapers.cian_detail.extract_all_states",
|
||||
lambda html: {},
|
||||
)
|
||||
|
||||
response = MagicMock()
|
||||
response.status_code = 200
|
||||
response.text = "<html></html>"
|
||||
|
||||
session = MagicMock()
|
||||
session.get = AsyncMock(return_value=response)
|
||||
session.close = AsyncMock()
|
||||
|
||||
result = await fetch_detail("https://ekb.cian.ru/sale/flat/444444/", session=session)
|
||||
assert result is not None
|
||||
assert result.kitchen_area_m2 is None
|
||||
|
||||
|
||||
# ── save_detail_enrichment — kitchen_area_m2 persisted ───────────────────────
|
||||
|
||||
|
||||
def test_save_detail_enrichment_writes_kitchen_area():
|
||||
"""save_detail_enrichment передаёт kitchen_area_m2 в UPDATE listings."""
|
||||
db = MagicMock()
|
||||
|
||||
# Мокаем fetchone для snapshot-запроса — price_rub=None → snapshot не пишется
|
||||
db.execute.return_value.fetchone.return_value = None
|
||||
|
||||
enrichment = DetailEnrichment(
|
||||
windows_view_type=None,
|
||||
separate_wcs_count=None,
|
||||
combined_wcs_count=None,
|
||||
ceiling_height=None,
|
||||
repair_type=None,
|
||||
repair_state=None,
|
||||
kitchen_area_m2=8.3,
|
||||
views_total=None,
|
||||
views_today=None,
|
||||
)
|
||||
|
||||
save_detail_enrichment(db, listing_id=999, enrichment=enrichment)
|
||||
|
||||
# Проверяем что UPDATE listings вызван с параметром ka=8.3
|
||||
update_call_params = db.execute.call_args_list[0][0][1]
|
||||
assert update_call_params["ka"] == 8.3
|
||||
|
||||
|
||||
def test_save_detail_enrichment_kitchen_area_none_passthrough():
|
||||
"""Если kitchen_area_m2 = None — параметр ka=None, COALESCE сохраняет существующее."""
|
||||
db = MagicMock()
|
||||
db.execute.return_value.fetchone.return_value = None
|
||||
|
||||
enrichment = DetailEnrichment(kitchen_area_m2=None)
|
||||
|
||||
save_detail_enrichment(db, listing_id=1001, enrichment=enrichment)
|
||||
|
||||
update_call_params = db.execute.call_args_list[0][0][1]
|
||||
assert update_call_params["ka"] is None
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue