fix(tradein): cian detail-парсер не извлекал kitchen_area_m2 из offer.kitchenArea
All checks were successful
CI / changes (push) Successful in 8s
CI / openapi-codegen-check (push) Has been skipped
CI / backend-tests (pull_request) Has been skipped
CI / backend-tests (push) Has been skipped
CI / frontend-tests (push) Has been skipped
CI / changes (pull_request) Successful in 7s
CI / frontend-tests (pull_request) Has been skipped
CI / openapi-codegen-check (pull_request) Has been skipped
All checks were successful
CI / changes (push) Successful in 8s
CI / openapi-codegen-check (push) Has been skipped
CI / backend-tests (pull_request) Has been skipped
CI / backend-tests (push) Has been skipped
CI / frontend-tests (push) Has been skipped
CI / changes (pull_request) Successful in 7s
CI / frontend-tests (pull_request) Has been skipped
CI / openapi-codegen-check (pull_request) Has been skipped
Добавлено поле kitchen_area_m2 в DetailEnrichment dataclass,
извлечение через _parse_float(offer.get("kitchenArea")) в fetch_detail,
и COALESCE(CAST(:ka AS double precision), kitchen_area_m2) в UPDATE listings
внутри save_detail_enrichment.
Имя поля kitchenArea — стандартное cian offer API (live-verify не делался,
прокси недоступен в dev-окружении).
Closes #1553
This commit is contained in:
parent
d76df54ece
commit
b390c8334c
2 changed files with 119 additions and 2 deletions
|
|
@ -44,6 +44,7 @@ class DetailEnrichment:
|
||||||
ceiling_height: float | None = None # meters
|
ceiling_height: float | None = None # meters
|
||||||
repair_type: str | None = None # 'cosmetic' / 'design' / 'no' — raw Cian value
|
repair_type: str | None = None # 'cosmetic' / 'design' / 'no' — raw Cian value
|
||||||
repair_state: str | None = None # enum: needs_repair/standard/good/excellent
|
repair_state: str | None = None # enum: needs_repair/standard/good/excellent
|
||||||
|
kitchen_area_m2: float | None = None # offer.kitchenArea (м²)
|
||||||
views_total: int | None = None # Cian stats.totalViewsFormattedString → int
|
views_total: int | None = None # Cian stats.totalViewsFormattedString → int
|
||||||
views_today: int | None = None
|
views_today: int | None = None
|
||||||
|
|
||||||
|
|
@ -119,6 +120,7 @@ async def fetch_detail(
|
||||||
ceiling_height=_parse_float(offer.get("ceilingHeight")),
|
ceiling_height=_parse_float(offer.get("ceilingHeight")),
|
||||||
repair_type=_extract_repair_type(offer),
|
repair_type=_extract_repair_type(offer),
|
||||||
repair_state=_extract_repair_state(offer),
|
repair_state=_extract_repair_state(offer),
|
||||||
|
kitchen_area_m2=_parse_float(offer.get("kitchenArea")),
|
||||||
raw_offer=offer,
|
raw_offer=offer,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -259,6 +261,7 @@ def save_detail_enrichment(db: Session, listing_id: int, enrichment: DetailEnric
|
||||||
ceiling_height = COALESCE(:ch, ceiling_height),
|
ceiling_height = COALESCE(:ch, ceiling_height),
|
||||||
repair_type = COALESCE(:rt, repair_type),
|
repair_type = COALESCE(:rt, repair_type),
|
||||||
repair_state = COALESCE(:rs, repair_state),
|
repair_state = COALESCE(:rs, repair_state),
|
||||||
|
kitchen_area_m2 = COALESCE(CAST(:ka AS double precision), kitchen_area_m2),
|
||||||
views_total = COALESCE(:vt, views_total),
|
views_total = COALESCE(:vt, views_total),
|
||||||
views_today = COALESCE(:vd, views_today)
|
views_today = COALESCE(:vd, views_today)
|
||||||
WHERE id = CAST(:lid AS bigint)
|
WHERE id = CAST(:lid AS bigint)
|
||||||
|
|
@ -271,6 +274,7 @@ def save_detail_enrichment(db: Session, listing_id: int, enrichment: DetailEnric
|
||||||
"ch": enrichment.ceiling_height,
|
"ch": enrichment.ceiling_height,
|
||||||
"rt": enrichment.repair_type,
|
"rt": enrichment.repair_type,
|
||||||
"rs": enrichment.repair_state,
|
"rs": enrichment.repair_state,
|
||||||
|
"ka": enrichment.kitchen_area_m2,
|
||||||
"vt": enrichment.views_total,
|
"vt": enrichment.views_total,
|
||||||
"vd": enrichment.views_today,
|
"vd": enrichment.views_today,
|
||||||
},
|
},
|
||||||
|
|
@ -376,8 +380,7 @@ def save_detail_enrichment(db: Session, listing_id: int, enrichment: DetailEnric
|
||||||
|
|
||||||
db.commit()
|
db.commit()
|
||||||
logger.info(
|
logger.info(
|
||||||
"Cian detail enrichment saved for listing_id=%s "
|
"Cian detail enrichment saved for listing_id=%s (price_changes=%d, agent=%s, snapshot=%s)",
|
||||||
"(price_changes=%d, agent=%s, snapshot=%s)",
|
|
||||||
listing_id,
|
listing_id,
|
||||||
len(enrichment.price_changes),
|
len(enrichment.price_changes),
|
||||||
bool(enrichment.agent_profile),
|
bool(enrichment.agent_profile),
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,5 @@
|
||||||
"""Tests for cian_detail.py — Stage 5 of CianScraper v1."""
|
"""Tests for cian_detail.py — Stage 5 of CianScraper v1."""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from unittest.mock import AsyncMock, MagicMock
|
from unittest.mock import AsyncMock, MagicMock
|
||||||
|
|
@ -12,6 +13,7 @@ from app.services.scrapers.cian_detail import (
|
||||||
_extract_repair_state,
|
_extract_repair_state,
|
||||||
_parse_views,
|
_parse_views,
|
||||||
fetch_detail,
|
fetch_detail,
|
||||||
|
save_detail_enrichment,
|
||||||
)
|
)
|
||||||
|
|
||||||
# ── _parse_views ──────────────────────────────────────────────────────────────
|
# ── _parse_views ──────────────────────────────────────────────────────────────
|
||||||
|
|
@ -349,3 +351,115 @@ async def test_fetch_detail_extracts_newbuilding_link(monkeypatch):
|
||||||
assert result is not None
|
assert result is not None
|
||||||
assert result.newbuilding_data is not None
|
assert result.newbuilding_data is not None
|
||||||
assert result.newbuilding_data["id"] == 54016
|
assert result.newbuilding_data["id"] == 54016
|
||||||
|
|
||||||
|
|
||||||
|
# ── kitchen_area_m2 extraction ────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_fetch_detail_extracts_kitchen_area(monkeypatch):
|
||||||
|
"""kitchenArea из offer → kitchen_area_m2 в DetailEnrichment."""
|
||||||
|
fake_state = {
|
||||||
|
"offerData": {
|
||||||
|
"offer": {
|
||||||
|
"cianId": 333333,
|
||||||
|
"kitchenArea": 9.5,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"app.services.scrapers.cian_detail.extract_state",
|
||||||
|
lambda html, mfe, key: fake_state if key == "defaultState" else None,
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"app.services.scrapers.cian_detail.extract_all_states",
|
||||||
|
lambda html: {},
|
||||||
|
)
|
||||||
|
|
||||||
|
response = MagicMock()
|
||||||
|
response.status_code = 200
|
||||||
|
response.text = "<html></html>"
|
||||||
|
|
||||||
|
session = MagicMock()
|
||||||
|
session.get = AsyncMock(return_value=response)
|
||||||
|
session.close = AsyncMock()
|
||||||
|
|
||||||
|
result = await fetch_detail("https://ekb.cian.ru/sale/flat/333333/", session=session)
|
||||||
|
assert result is not None
|
||||||
|
assert result.kitchen_area_m2 == 9.5
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_fetch_detail_kitchen_area_none_when_missing(monkeypatch):
|
||||||
|
"""Если kitchenArea отсутствует в offer — kitchen_area_m2 остаётся None."""
|
||||||
|
fake_state = {
|
||||||
|
"offerData": {
|
||||||
|
"offer": {
|
||||||
|
"cianId": 444444,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"app.services.scrapers.cian_detail.extract_state",
|
||||||
|
lambda html, mfe, key: fake_state if key == "defaultState" else None,
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"app.services.scrapers.cian_detail.extract_all_states",
|
||||||
|
lambda html: {},
|
||||||
|
)
|
||||||
|
|
||||||
|
response = MagicMock()
|
||||||
|
response.status_code = 200
|
||||||
|
response.text = "<html></html>"
|
||||||
|
|
||||||
|
session = MagicMock()
|
||||||
|
session.get = AsyncMock(return_value=response)
|
||||||
|
session.close = AsyncMock()
|
||||||
|
|
||||||
|
result = await fetch_detail("https://ekb.cian.ru/sale/flat/444444/", session=session)
|
||||||
|
assert result is not None
|
||||||
|
assert result.kitchen_area_m2 is None
|
||||||
|
|
||||||
|
|
||||||
|
# ── save_detail_enrichment — kitchen_area_m2 persisted ───────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def test_save_detail_enrichment_writes_kitchen_area():
|
||||||
|
"""save_detail_enrichment передаёт kitchen_area_m2 в UPDATE listings."""
|
||||||
|
db = MagicMock()
|
||||||
|
|
||||||
|
# Мокаем fetchone для snapshot-запроса — price_rub=None → snapshot не пишется
|
||||||
|
db.execute.return_value.fetchone.return_value = None
|
||||||
|
|
||||||
|
enrichment = DetailEnrichment(
|
||||||
|
windows_view_type=None,
|
||||||
|
separate_wcs_count=None,
|
||||||
|
combined_wcs_count=None,
|
||||||
|
ceiling_height=None,
|
||||||
|
repair_type=None,
|
||||||
|
repair_state=None,
|
||||||
|
kitchen_area_m2=8.3,
|
||||||
|
views_total=None,
|
||||||
|
views_today=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
save_detail_enrichment(db, listing_id=999, enrichment=enrichment)
|
||||||
|
|
||||||
|
# Проверяем что UPDATE listings вызван с параметром ka=8.3
|
||||||
|
update_call_params = db.execute.call_args_list[0][0][1]
|
||||||
|
assert update_call_params["ka"] == 8.3
|
||||||
|
|
||||||
|
|
||||||
|
def test_save_detail_enrichment_kitchen_area_none_passthrough():
|
||||||
|
"""Если kitchen_area_m2 = None — параметр ka=None, COALESCE сохраняет существующее."""
|
||||||
|
db = MagicMock()
|
||||||
|
db.execute.return_value.fetchone.return_value = None
|
||||||
|
|
||||||
|
enrichment = DetailEnrichment(kitchen_area_m2=None)
|
||||||
|
|
||||||
|
save_detail_enrichment(db, listing_id=1001, enrichment=enrichment)
|
||||||
|
|
||||||
|
update_call_params = db.execute.call_args_list[0][0][1]
|
||||||
|
assert update_call_params["ka"] is None
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue