6d99e6a41a
Stage 7 (Wave 5). Scraper fetches https://www.cian.ru/kalkulator-nedvizhimosti/ with curl_cffi (impersonate=chrome120) + Cian session cookies from DB (Stage 4). Parses estimation.{sale,rent}.data, estimationChart 7-point time-series, houseInfo 15 items, managementCompany, filters.houseId. Writes to external_valuations (026+029 schema) with 24h TTL + ON CONFLICT upsert. 28 unit tests covering cache key, state parsing, floor/repair enums, cache hit/miss.
401 lines
13 KiB
Python
401 lines
13 KiB
Python
"""Tests for cian_valuation.py — Stage 7 Cian Valuation Calculator scraper."""
|
||
import os
|
||
|
||
# Settings требует DATABASE_URL при инициализации (fail-fast).
|
||
# Для offline unit-тестов задаём dummy DSN до любого импорта из app.
|
||
os.environ.setdefault("DATABASE_URL", "postgresql://test:test@localhost/test_db")
|
||
os.environ.setdefault("COOKIE_ENCRYPTION_KEY", "test-key-for-tests")
|
||
|
||
from unittest.mock import MagicMock
|
||
|
||
import pytest
|
||
|
||
from app.services.scrapers.cian_valuation import (
|
||
CianValuationResult,
|
||
_parse_change_pct,
|
||
_parse_num,
|
||
_parse_valuation_state,
|
||
compute_cache_key,
|
||
estimate_via_cian_valuation,
|
||
)
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# compute_cache_key
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_cache_key_stable():
|
||
"""Нормализация address: lower + strip дают одинаковый ключ."""
|
||
k1 = compute_cache_key("ЕКБ ул. Ленина 1", 50.0, 2, 5, "cosmetic", "sale")
|
||
k2 = compute_cache_key("екб ул. ленина 1 ", 50.0, 2, 5, "cosmetic", "sale")
|
||
assert k1 == k2
|
||
|
||
|
||
def test_cache_key_distinct_on_deal_type():
|
||
k1 = compute_cache_key("addr", 50.0, 2, 5, "cosmetic", "sale")
|
||
k2 = compute_cache_key("addr", 50.0, 2, 5, "cosmetic", "rent")
|
||
assert k1 != k2
|
||
|
||
|
||
def test_cache_key_distinct_on_area():
|
||
k1 = compute_cache_key("addr", 50.0, 2, 5, "cosmetic", "sale")
|
||
k2 = compute_cache_key("addr", 51.0, 2, 5, "cosmetic", "sale")
|
||
assert k1 != k2
|
||
|
||
|
||
def test_cache_key_is_hex_string():
|
||
k = compute_cache_key("addr", 50.0, 2, 5, "cosmetic", "sale")
|
||
assert len(k) == 64
|
||
assert all(c in "0123456789abcdef" for c in k)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# _parse_num
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_parse_num_float():
|
||
assert _parse_num(5.5) == 5.5
|
||
|
||
|
||
def test_parse_num_str():
|
||
assert _parse_num("100") == 100.0
|
||
|
||
|
||
def test_parse_num_none():
|
||
assert _parse_num(None) is None
|
||
|
||
|
||
def test_parse_num_bad_str():
|
||
assert _parse_num("bad") is None
|
||
|
||
|
||
def test_parse_num_int():
|
||
assert _parse_num(0) == 0.0
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# _parse_change_pct
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_parse_change_pct_positive():
|
||
assert _parse_change_pct("8,3%") == pytest.approx(8.3)
|
||
|
||
|
||
def test_parse_change_pct_negative():
|
||
assert _parse_change_pct("-2,5%") == pytest.approx(-2.5)
|
||
|
||
|
||
def test_parse_change_pct_empty():
|
||
assert _parse_change_pct("") is None
|
||
|
||
|
||
def test_parse_change_pct_dot_format():
|
||
assert _parse_change_pct("1.5%") == pytest.approx(1.5)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# _parse_valuation_state — основные сценарии
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def _make_state(
|
||
is_authenticated: bool = True,
|
||
user_id: int = 12345,
|
||
sale_price: int = 7500000,
|
||
sale_accuracy: int = 94,
|
||
rent_price: int = 35000,
|
||
rent_accuracy: int = 93,
|
||
chart_change: str = "increase",
|
||
chart_change_value: str = "8,3%",
|
||
house_id: int = 54016,
|
||
) -> dict:
|
||
return {
|
||
"user": {"isAuthenticated": is_authenticated, "userId": user_id},
|
||
"estimation": {
|
||
"sale": {
|
||
"data": {
|
||
"price": sale_price,
|
||
"accuracy": sale_accuracy,
|
||
"priceFrom": 6900000,
|
||
"priceTo": 8100000,
|
||
"priceSqm": 150000,
|
||
"dealType": "sale",
|
||
"filtersHash": "abc123",
|
||
},
|
||
"isError": False,
|
||
"isFetching": False,
|
||
},
|
||
"rent": {
|
||
"data": {
|
||
"price": rent_price,
|
||
"accuracy": rent_accuracy,
|
||
"priceFrom": 32000,
|
||
"priceTo": 38000,
|
||
"taxPrice": 1400,
|
||
"dealType": "rent",
|
||
},
|
||
},
|
||
},
|
||
"estimationChart": {
|
||
"data": {
|
||
"title": {
|
||
"title": "Стоимость этой квартиры увеличилась на {{change}} за полгода",
|
||
"changeValue": chart_change_value,
|
||
"change": chart_change,
|
||
},
|
||
"chartData": {
|
||
"maxXValue": 1777593600000,
|
||
"minXValue": 1761955200000,
|
||
"data": [
|
||
{"date": 1761955200000, "price": 6033438, "priceFormatted": "6,0 млн ₽"},
|
||
{"date": 1764547200000, "price": 6013573, "priceFormatted": "6,0 млн ₽"},
|
||
{"date": 1767225600000, "price": 6116587, "priceFormatted": "6,1 млн ₽"},
|
||
{"date": 1769904000000, "price": 6236983, "priceFormatted": "6,2 млн ₽"},
|
||
{"date": 1772323200000, "price": 6232172, "priceFormatted": "6,2 млн ₽"},
|
||
{"date": 1775001600000, "price": 6331112, "priceFormatted": "6,3 млн ₽"},
|
||
{"date": 1777593600000, "price": 6532872, "priceFormatted": "6,5 млн ₽"},
|
||
],
|
||
},
|
||
},
|
||
},
|
||
"houseInfo": {
|
||
"data": {
|
||
"items": [
|
||
{"title": "Год постройки", "value": 2005},
|
||
{"title": "Тип дома", "value": "Кирпичный"},
|
||
],
|
||
"isRenovation": False,
|
||
"isEmergency": False,
|
||
},
|
||
},
|
||
"filters": {"houseId": house_id, "roomsCount": 2},
|
||
"managementCompany": {
|
||
"data": {
|
||
"name": "ТСЖ «Учителей, 18»",
|
||
"address": "ул. Учителей, 18",
|
||
"phones": ["+7 (343) 000-00-00"],
|
||
"email": "tsj@example.ru",
|
||
"chiefName": "Иванов И.И.",
|
||
"openingHours": [{"Пн–Пт": ["9:00", "18:00"]}],
|
||
},
|
||
},
|
||
"page": {"pageType": "valuationPage"},
|
||
}
|
||
|
||
|
||
def test_parse_valuation_state_extracts_sale():
|
||
state = _make_state()
|
||
result = _parse_valuation_state(state)
|
||
|
||
assert result.is_authenticated is True
|
||
assert result.user_id == 12345
|
||
assert result.sale_price_rub == 7500000.0
|
||
assert result.sale_accuracy == 94.0
|
||
assert result.sale_price_from == 6900000.0
|
||
assert result.sale_price_to == 8100000.0
|
||
assert result.sale_price_sqm == 150000.0
|
||
assert result.filters_hash == "abc123"
|
||
|
||
|
||
def test_parse_valuation_state_extracts_rent():
|
||
state = _make_state()
|
||
result = _parse_valuation_state(state)
|
||
|
||
assert result.rent_price_rub == 35000.0
|
||
assert result.rent_accuracy == 93.0
|
||
assert result.rent_price_from == 32000.0
|
||
assert result.rent_price_to == 38000.0
|
||
assert result.rent_tax_price == 1400.0
|
||
|
||
|
||
def test_parse_valuation_state_chart_7_points():
|
||
state = _make_state()
|
||
result = _parse_valuation_state(state)
|
||
|
||
assert len(result.chart) == 7
|
||
# First point: unix ms 1761955200000 → 2025-11-01 (UTC)
|
||
assert result.chart[0]["month_date"] == "2025-11-01"
|
||
assert result.chart[0]["price"] == 6033438.0
|
||
|
||
|
||
def test_parse_valuation_state_chart_change_increase():
|
||
state = _make_state(chart_change="increase", chart_change_value="8,3%")
|
||
result = _parse_valuation_state(state)
|
||
|
||
assert result.chart_change_pct == pytest.approx(8.3)
|
||
assert result.chart_change_direction == "increase"
|
||
|
||
|
||
def test_parse_valuation_state_chart_change_decrease():
|
||
state = _make_state(chart_change="decrease", chart_change_value="-2,5%")
|
||
result = _parse_valuation_state(state)
|
||
|
||
assert result.chart_change_pct == pytest.approx(-2.5)
|
||
assert result.chart_change_direction == "decrease"
|
||
|
||
|
||
def test_parse_valuation_state_house_info():
|
||
state = _make_state()
|
||
result = _parse_valuation_state(state)
|
||
|
||
assert len(result.house_info) == 2
|
||
assert result.house_info[0]["title"] == "Год постройки"
|
||
assert result.house_info[0]["value"] == 2005
|
||
|
||
|
||
def test_parse_valuation_state_external_house_id():
|
||
state = _make_state(house_id=54016)
|
||
result = _parse_valuation_state(state)
|
||
|
||
assert result.external_house_id == 54016
|
||
|
||
|
||
def test_parse_valuation_state_management_company():
|
||
state = _make_state()
|
||
result = _parse_valuation_state(state)
|
||
|
||
assert result.management_company is not None
|
||
assert result.management_company["name"] == "ТСЖ «Учителей, 18»"
|
||
assert result.management_company["chiefName"] == "Иванов И.И."
|
||
|
||
|
||
def test_parse_valuation_state_no_auth():
|
||
state = _make_state(is_authenticated=False)
|
||
result = _parse_valuation_state(state)
|
||
|
||
assert result.is_authenticated is False
|
||
# Values still parsed — auth check is done by caller, not parser
|
||
assert result.sale_price_rub == 7500000.0
|
||
|
||
|
||
def test_parse_valuation_state_empty_estimation():
|
||
state = {"user": {"isAuthenticated": True, "userId": 1}, "estimation": {}}
|
||
result = _parse_valuation_state(state)
|
||
|
||
assert result.sale_price_rub is None
|
||
assert result.rent_price_rub is None
|
||
assert result.chart == []
|
||
|
||
|
||
def test_parse_valuation_state_missing_management_company():
|
||
state = _make_state()
|
||
del state["managementCompany"]
|
||
result = _parse_valuation_state(state)
|
||
|
||
assert result.management_company is None
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# CianValuationResult dataclass defaults
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_dataclass_defaults():
|
||
r = CianValuationResult()
|
||
|
||
assert r.chart == []
|
||
assert r.house_info == []
|
||
assert r.is_authenticated is False
|
||
assert r.sale_price_rub is None
|
||
assert r.management_company is None
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# estimate_via_cian_valuation — integration (mocked)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_estimate_returns_none_when_no_cookies(monkeypatch):
|
||
"""Если load_session returns None → graceful return None без crash."""
|
||
monkeypatch.setattr(
|
||
"app.services.scrapers.cian_valuation.load_session",
|
||
lambda db: None,
|
||
)
|
||
db = MagicMock()
|
||
# cache miss
|
||
db.execute.return_value.mappings.return_value.first.return_value = None
|
||
|
||
result = await estimate_via_cian_valuation(
|
||
db,
|
||
address="ЕКБ",
|
||
total_area=50.0,
|
||
rooms_count=2,
|
||
floor=5,
|
||
use_cache=True,
|
||
)
|
||
assert result is None
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_cache_hit_returns_cached(monkeypatch):
|
||
"""Если cache hit → return без fetching (load_session не вызывается)."""
|
||
db = MagicMock()
|
||
cached_row = {
|
||
"raw_payload": {"cached": True},
|
||
"sale_price_rub": 7000000,
|
||
"sale_accuracy": 80,
|
||
"rent_price_rub": 30000,
|
||
"rent_accuracy": 65,
|
||
"chart": [{"month_date": "2026-05-01", "price": 145000}],
|
||
"chart_change_pct": 0.5,
|
||
"chart_change_direction": "increase",
|
||
"external_house_id": 999,
|
||
"filters_hash": "xyz789",
|
||
}
|
||
db.execute.return_value.mappings.return_value.first.return_value = cached_row
|
||
|
||
# load_session должен НЕ вызываться при cache hit
|
||
monkeypatch.setattr(
|
||
"app.services.scrapers.cian_valuation.load_session",
|
||
lambda db: pytest.fail("load_session should not be called on cache hit"),
|
||
)
|
||
|
||
result = await estimate_via_cian_valuation(
|
||
db,
|
||
address="ЕКБ",
|
||
total_area=50.0,
|
||
rooms_count=2,
|
||
floor=5,
|
||
use_cache=True,
|
||
)
|
||
|
||
assert result is not None
|
||
assert result.sale_price_rub == 7000000.0
|
||
assert result.rent_price_rub == 30000.0
|
||
assert result.external_house_id == 999
|
||
assert result.filters_hash == "xyz789"
|
||
assert result.is_authenticated is True # cache implies authenticated
|
||
assert len(result.chart) == 1
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_estimate_use_cache_false_skips_cache(monkeypatch):
|
||
"""use_cache=False → cache lookup не вызывается, идём сразу в load_session."""
|
||
load_session_called = []
|
||
|
||
def fake_load_session(db: object) -> None:
|
||
load_session_called.append(True)
|
||
return None # simulate no cookies → None result
|
||
|
||
monkeypatch.setattr(
|
||
"app.services.scrapers.cian_valuation.load_session",
|
||
fake_load_session,
|
||
)
|
||
|
||
db = MagicMock()
|
||
result = await estimate_via_cian_valuation(
|
||
db,
|
||
address="ЕКБ",
|
||
total_area=50.0,
|
||
rooms_count=2,
|
||
floor=5,
|
||
use_cache=False,
|
||
)
|
||
|
||
assert result is None
|
||
assert load_session_called # load_session был вызван (cache bypass worked)
|
||
# db.execute НЕ должен быть вызван для cache lookup
|
||
db.execute.assert_not_called()
|