bot-backend
b390c8334c
All checks were successful
CI / changes (push) Successful in 8s
CI / openapi-codegen-check (push) Has been skipped
CI / backend-tests (pull_request) Has been skipped
CI / backend-tests (push) Has been skipped
CI / frontend-tests (push) Has been skipped
CI / changes (pull_request) Successful in 7s
CI / frontend-tests (pull_request) Has been skipped
CI / openapi-codegen-check (pull_request) Has been skipped
Добавлено поле kitchen_area_m2 в DetailEnrichment dataclass,
извлечение через _parse_float(offer.get("kitchenArea")) в fetch_detail,
и COALESCE(CAST(:ka AS double precision), kitchen_area_m2) в UPDATE listings
внутри save_detail_enrichment.
Имя поля kitchenArea — стандартное cian offer API (live-verify не делался,
прокси недоступен в dev-окружении).
Closes #1553
465 lines
15 KiB
Python
465 lines
15 KiB
Python
"""Tests for cian_detail.py — Stage 5 of CianScraper v1."""
|
||
|
||
from __future__ import annotations
|
||
|
||
from unittest.mock import AsyncMock, MagicMock
|
||
|
||
import pytest
|
||
|
||
from app.services.scrapers.cian_detail import (
|
||
DetailEnrichment,
|
||
_extract_agent_profile,
|
||
_extract_price_changes,
|
||
_extract_repair_state,
|
||
_parse_views,
|
||
fetch_detail,
|
||
save_detail_enrichment,
|
||
)
|
||
|
||
# ── _parse_views ──────────────────────────────────────────────────────────────
|
||
|
||
|
||
def test_parse_views_space_formatted():
|
||
assert _parse_views("1 234") == 1234
|
||
|
||
|
||
def test_parse_views_plain():
|
||
assert _parse_views("567") == 567
|
||
|
||
|
||
def test_parse_views_none():
|
||
assert _parse_views(None) is None
|
||
|
||
|
||
def test_parse_views_invalid():
|
||
assert _parse_views("invalid") is None
|
||
|
||
|
||
def test_parse_views_comma_formatted():
|
||
assert _parse_views("1,234") == 1234
|
||
|
||
|
||
# ── _extract_repair_state — структурное поле в приоритете, fallback на текст (#622) ──
|
||
|
||
|
||
def test_repair_state_structured_wins_over_description():
|
||
"""repairType присутствует → используется он, описание игнорируется."""
|
||
offer = {
|
||
"repairType": "cosmetic", # → standard
|
||
"description": "Шикарный дизайнерский ремонт", # инференс дал бы excellent
|
||
}
|
||
assert _extract_repair_state(offer) == "standard"
|
||
|
||
|
||
def test_repair_state_falls_back_to_description_when_structured_missing():
|
||
"""repairType отсутствует → инференс из описания."""
|
||
offer = {
|
||
"repairType": None,
|
||
"description": "Сделан свежий евроремонт, заезжай и живи",
|
||
}
|
||
assert _extract_repair_state(offer) == "good"
|
||
|
||
|
||
def test_repair_state_none_when_both_missing():
|
||
"""Нет structured и нет сигнала в тексте → None (не фабрикуем)."""
|
||
offer = {"description": "Светлая квартира рядом с парком"}
|
||
assert _extract_repair_state(offer) is None
|
||
|
||
|
||
# ── _extract_price_changes ────────────────────────────────────────────────────
|
||
|
||
|
||
def test_extract_price_changes_normalizes():
|
||
offer = {
|
||
"priceChanges": [
|
||
{"changeTime": "2026-04-01T00:00:00Z", "price": 5000000, "diffPercent": -2.5},
|
||
{"changeTime": "2026-04-10T00:00:00Z", "price": 4900000, "diffPercent": -2.0},
|
||
]
|
||
}
|
||
result = _extract_price_changes(offer, {})
|
||
assert len(result) == 2
|
||
assert result[0]["change_time"] == "2026-04-01T00:00:00Z"
|
||
assert result[0]["price_rub"] == 5000000
|
||
assert result[0]["diff_percent"] == -2.5
|
||
|
||
|
||
def test_extract_price_changes_handles_empty_offer():
|
||
assert _extract_price_changes({}, {}) == []
|
||
|
||
|
||
def test_extract_price_changes_handles_none_value():
|
||
assert _extract_price_changes({"priceChanges": None}, {}) == []
|
||
|
||
|
||
def test_extract_price_changes_falls_back_to_state():
|
||
state = {
|
||
"priceChanges": [
|
||
{"changeTime": "2026-03-01T00:00:00Z", "price": 6000000, "diffPercent": 0},
|
||
]
|
||
}
|
||
result = _extract_price_changes({}, state)
|
||
assert len(result) == 1
|
||
assert result[0]["price_rub"] == 6000000
|
||
|
||
|
||
def test_extract_price_changes_skips_non_dict_entries():
|
||
offer = {"priceChanges": [{"changeTime": "2026-04-01T00:00:00Z", "price": 5000000}, "bad"]}
|
||
result = _extract_price_changes(offer, {})
|
||
assert len(result) == 1
|
||
|
||
|
||
# ── _extract_agent_profile ────────────────────────────────────────────────────
|
||
|
||
|
||
def test_extract_agent_profile_from_agent_key():
|
||
offer = {
|
||
"agent": {
|
||
"id": 12345,
|
||
"companyName": "Новосёл",
|
||
"isPro": True,
|
||
"skills": [{"id": 1, "name": "Продажа"}],
|
||
"accountType": "specialist",
|
||
}
|
||
}
|
||
profile = _extract_agent_profile(offer)
|
||
assert profile is not None
|
||
assert profile["ext_agent_id"] == 12345
|
||
assert profile["company_name"] == "Новосёл"
|
||
assert profile["is_pro"] is True
|
||
assert profile["account_type"] == "specialist"
|
||
|
||
|
||
def test_extract_agent_profile_from_user_key():
|
||
offer = {
|
||
"user": {
|
||
"cianUserId": 99999,
|
||
"companyName": "Агентство Икс",
|
||
"isPro": False,
|
||
"accountType": "agency",
|
||
}
|
||
}
|
||
profile = _extract_agent_profile(offer)
|
||
assert profile is not None
|
||
assert profile["ext_agent_id"] == 99999
|
||
assert profile["company_name"] == "Агентство Икс"
|
||
|
||
|
||
def test_extract_agent_profile_returns_none_when_no_agent():
|
||
assert _extract_agent_profile({}) is None
|
||
assert _extract_agent_profile({"agent": {}, "user": {}}) is None
|
||
|
||
|
||
# ── DetailEnrichment defaults ─────────────────────────────────────────────────
|
||
|
||
|
||
def test_dataclass_defaults():
|
||
enrich = DetailEnrichment()
|
||
assert enrich.cian_id is None
|
||
assert enrich.price_changes == []
|
||
assert enrich.raw_sister_states == {}
|
||
assert enrich.bti_data is None
|
||
assert enrich.agent_profile is None
|
||
assert enrich.newbuilding_data is None
|
||
|
||
|
||
# ── fetch_detail integration (mocked) ────────────────────────────────────────
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_detail_returns_none_on_no_state(monkeypatch):
|
||
"""If state extraction returns None → fetch_detail returns None gracefully."""
|
||
monkeypatch.setattr(
|
||
"app.services.scrapers.cian_detail.extract_state",
|
||
lambda html, mfe, key: None,
|
||
)
|
||
|
||
response = MagicMock()
|
||
response.status_code = 200
|
||
response.text = "<html></html>"
|
||
|
||
session = MagicMock()
|
||
session.get = AsyncMock(return_value=response)
|
||
session.close = AsyncMock()
|
||
|
||
result = await fetch_detail("https://ekb.cian.ru/sale/flat/123/", session=session)
|
||
assert result is None
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_detail_returns_none_on_non_200(monkeypatch):
|
||
"""Non-200 HTTP response → returns None, no exception raised."""
|
||
response = MagicMock()
|
||
response.status_code = 403
|
||
response.text = ""
|
||
|
||
session = MagicMock()
|
||
session.get = AsyncMock(return_value=response)
|
||
session.close = AsyncMock()
|
||
|
||
result = await fetch_detail("https://ekb.cian.ru/sale/flat/456/", session=session)
|
||
assert result is None
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_detail_extracts_core_fields(monkeypatch):
|
||
"""Verify core fields are extracted correctly from mocked state."""
|
||
fake_state = {
|
||
"offerData": {
|
||
"offer": {
|
||
"cianId": 327830237,
|
||
"windowsViewType": "yardAndStreet",
|
||
"separateWcsCount": 1,
|
||
"combinedWcsCount": 0,
|
||
"ceilingHeight": 2.65,
|
||
"repairType": "cosmetic",
|
||
"priceChanges": [
|
||
{
|
||
"changeTime": "2026-04-01T00:00:00Z",
|
||
"price": 5500000,
|
||
"diffPercent": -1.8,
|
||
}
|
||
],
|
||
"agent": {
|
||
"id": 7654321,
|
||
"companyName": "Первичка Плюс",
|
||
"isPro": True,
|
||
"skills": [],
|
||
"accountType": "specialist",
|
||
},
|
||
},
|
||
"stats": {
|
||
"totalViewsFormattedString": "1 042",
|
||
"todayViewsFormattedString": "7",
|
||
},
|
||
}
|
||
}
|
||
|
||
monkeypatch.setattr(
|
||
"app.services.scrapers.cian_detail.extract_state",
|
||
lambda html, mfe, key: fake_state if key == "defaultState" else None,
|
||
)
|
||
monkeypatch.setattr(
|
||
"app.services.scrapers.cian_detail.extract_all_states",
|
||
lambda html: {},
|
||
)
|
||
|
||
response = MagicMock()
|
||
response.status_code = 200
|
||
response.text = "<html></html>"
|
||
|
||
session = MagicMock()
|
||
session.get = AsyncMock(return_value=response)
|
||
session.close = AsyncMock()
|
||
|
||
result = await fetch_detail("https://ekb.cian.ru/sale/flat/327830237/", session=session)
|
||
assert result is not None
|
||
assert result.cian_id == 327830237
|
||
assert result.windows_view_type == "yardAndStreet"
|
||
assert result.separate_wcs_count == 1
|
||
assert result.combined_wcs_count == 0
|
||
assert result.ceiling_height == 2.65
|
||
assert result.repair_type == "cosmetic"
|
||
assert result.views_total == 1042
|
||
assert result.views_today == 7
|
||
assert len(result.price_changes) == 1
|
||
assert result.price_changes[0]["price_rub"] == 5500000
|
||
assert result.agent_profile is not None
|
||
assert result.agent_profile["ext_agent_id"] == 7654321
|
||
assert result.agent_profile["company_name"] == "Первичка Плюс"
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_detail_extracts_bti_from_sister_states(monkeypatch):
|
||
"""BTI sister state is captured into bti_data."""
|
||
fake_state = {
|
||
"offerData": {
|
||
"offer": {
|
||
"cianId": 111111,
|
||
}
|
||
}
|
||
}
|
||
fake_all_states = {
|
||
"frontend-offer-card": {
|
||
"bti": {
|
||
"houseData": {
|
||
"seriesName": "П-44Т",
|
||
"floorsCount": 17,
|
||
"flatCount": 120,
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
monkeypatch.setattr(
|
||
"app.services.scrapers.cian_detail.extract_state",
|
||
lambda html, mfe, key: fake_state if key == "defaultState" else None,
|
||
)
|
||
monkeypatch.setattr(
|
||
"app.services.scrapers.cian_detail.extract_all_states",
|
||
lambda html: fake_all_states,
|
||
)
|
||
|
||
response = MagicMock()
|
||
response.status_code = 200
|
||
response.text = "<html></html>"
|
||
|
||
session = MagicMock()
|
||
session.get = AsyncMock(return_value=response)
|
||
session.close = AsyncMock()
|
||
|
||
result = await fetch_detail("https://ekb.cian.ru/sale/flat/111111/", session=session)
|
||
assert result is not None
|
||
assert result.bti_data is not None
|
||
assert result.bti_data["seriesName"] == "П-44Т"
|
||
assert "bti" in result.raw_sister_states
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_detail_extracts_newbuilding_link(monkeypatch):
|
||
"""Newbuilding reference is captured from offer.newbuilding."""
|
||
fake_state = {
|
||
"offerData": {
|
||
"offer": {
|
||
"cianId": 222222,
|
||
"newbuilding": {
|
||
"id": 54016,
|
||
"name": "Екатерининский парк",
|
||
"url": "https://zhk-ekaterininskiy-park-ekb-i.cian.ru/",
|
||
},
|
||
}
|
||
}
|
||
}
|
||
|
||
monkeypatch.setattr(
|
||
"app.services.scrapers.cian_detail.extract_state",
|
||
lambda html, mfe, key: fake_state if key == "defaultState" else None,
|
||
)
|
||
monkeypatch.setattr(
|
||
"app.services.scrapers.cian_detail.extract_all_states",
|
||
lambda html: {},
|
||
)
|
||
|
||
response = MagicMock()
|
||
response.status_code = 200
|
||
response.text = "<html></html>"
|
||
|
||
session = MagicMock()
|
||
session.get = AsyncMock(return_value=response)
|
||
session.close = AsyncMock()
|
||
|
||
result = await fetch_detail("https://ekb.cian.ru/sale/flat/222222/", session=session)
|
||
assert result is not None
|
||
assert result.newbuilding_data is not None
|
||
assert result.newbuilding_data["id"] == 54016
|
||
|
||
|
||
# ── kitchen_area_m2 extraction ────────────────────────────────────────────────
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_detail_extracts_kitchen_area(monkeypatch):
|
||
"""kitchenArea из offer → kitchen_area_m2 в DetailEnrichment."""
|
||
fake_state = {
|
||
"offerData": {
|
||
"offer": {
|
||
"cianId": 333333,
|
||
"kitchenArea": 9.5,
|
||
}
|
||
}
|
||
}
|
||
|
||
monkeypatch.setattr(
|
||
"app.services.scrapers.cian_detail.extract_state",
|
||
lambda html, mfe, key: fake_state if key == "defaultState" else None,
|
||
)
|
||
monkeypatch.setattr(
|
||
"app.services.scrapers.cian_detail.extract_all_states",
|
||
lambda html: {},
|
||
)
|
||
|
||
response = MagicMock()
|
||
response.status_code = 200
|
||
response.text = "<html></html>"
|
||
|
||
session = MagicMock()
|
||
session.get = AsyncMock(return_value=response)
|
||
session.close = AsyncMock()
|
||
|
||
result = await fetch_detail("https://ekb.cian.ru/sale/flat/333333/", session=session)
|
||
assert result is not None
|
||
assert result.kitchen_area_m2 == 9.5
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_detail_kitchen_area_none_when_missing(monkeypatch):
|
||
"""Если kitchenArea отсутствует в offer — kitchen_area_m2 остаётся None."""
|
||
fake_state = {
|
||
"offerData": {
|
||
"offer": {
|
||
"cianId": 444444,
|
||
}
|
||
}
|
||
}
|
||
|
||
monkeypatch.setattr(
|
||
"app.services.scrapers.cian_detail.extract_state",
|
||
lambda html, mfe, key: fake_state if key == "defaultState" else None,
|
||
)
|
||
monkeypatch.setattr(
|
||
"app.services.scrapers.cian_detail.extract_all_states",
|
||
lambda html: {},
|
||
)
|
||
|
||
response = MagicMock()
|
||
response.status_code = 200
|
||
response.text = "<html></html>"
|
||
|
||
session = MagicMock()
|
||
session.get = AsyncMock(return_value=response)
|
||
session.close = AsyncMock()
|
||
|
||
result = await fetch_detail("https://ekb.cian.ru/sale/flat/444444/", session=session)
|
||
assert result is not None
|
||
assert result.kitchen_area_m2 is None
|
||
|
||
|
||
# ── save_detail_enrichment — kitchen_area_m2 persisted ───────────────────────
|
||
|
||
|
||
def test_save_detail_enrichment_writes_kitchen_area():
|
||
"""save_detail_enrichment передаёт kitchen_area_m2 в UPDATE listings."""
|
||
db = MagicMock()
|
||
|
||
# Мокаем fetchone для snapshot-запроса — price_rub=None → snapshot не пишется
|
||
db.execute.return_value.fetchone.return_value = None
|
||
|
||
enrichment = DetailEnrichment(
|
||
windows_view_type=None,
|
||
separate_wcs_count=None,
|
||
combined_wcs_count=None,
|
||
ceiling_height=None,
|
||
repair_type=None,
|
||
repair_state=None,
|
||
kitchen_area_m2=8.3,
|
||
views_total=None,
|
||
views_today=None,
|
||
)
|
||
|
||
save_detail_enrichment(db, listing_id=999, enrichment=enrichment)
|
||
|
||
# Проверяем что UPDATE listings вызван с параметром ka=8.3
|
||
update_call_params = db.execute.call_args_list[0][0][1]
|
||
assert update_call_params["ka"] == 8.3
|
||
|
||
|
||
def test_save_detail_enrichment_kitchen_area_none_passthrough():
|
||
"""Если kitchen_area_m2 = None — параметр ka=None, COALESCE сохраняет существующее."""
|
||
db = MagicMock()
|
||
db.execute.return_value.fetchone.return_value = None
|
||
|
||
enrichment = DetailEnrichment(kitchen_area_m2=None)
|
||
|
||
save_detail_enrichment(db, listing_id=1001, enrichment=enrichment)
|
||
|
||
update_call_params = db.execute.call_args_list[0][0][1]
|
||
assert update_call_params["ka"] is None
|