lekss361
b26e0ff7c3
Stage 8 of YandexRealtyScraper v1. Adds Yandex Valuation (anonymous, no cookies) as 6th on-demand source in estimate_quality. Mirrors Avito IMV pattern: 24h cache via external_valuations(source='yandex_valuation', cache_key) [UNIQUE 026:34], idempotent history persistence via house_placement_history(source, ext_item_id) [UNIQUE 017:50] with synthesized SHA256(address|date|area|floor|prices)[:32] ext_item_id. Graceful degradation on geocode/fetch/cache/save errors. house_id=NULL — matching deferred to Stage 8.x. 11 new tests + 4 IMV regression = 15 green.
197 lines
7 KiB
Python
197 lines
7 KiB
Python
"""Tests for Yandex Valuation integration in estimator.py."""
|
||
import os
|
||
|
||
# Settings requires DATABASE_URL at init time. Set dummy DSN before any app import.
|
||
os.environ.setdefault("DATABASE_URL", "postgresql://test:test@localhost/test_db")
|
||
|
||
from datetime import date
|
||
from unittest.mock import AsyncMock, MagicMock, patch
|
||
|
||
import pytest
|
||
|
||
from app.services.estimator import (
|
||
YANDEX_VALUATION_CACHE_TTL_HOURS,
|
||
_get_or_fetch_yandex_valuation_cached,
|
||
_save_yandex_history_items,
|
||
_yandex_valuation_cache_key,
|
||
)
|
||
from app.services.scrapers.yandex_valuation import (
|
||
ValuationHistoryItem,
|
||
ValuationHouseMeta,
|
||
YandexValuationResult,
|
||
)
|
||
|
||
|
||
def _sample_result(address: str = "Екатеринбург, ул. Учителей, 18") -> YandexValuationResult:
|
||
return YandexValuationResult(
|
||
address=address,
|
||
offer_category="APARTMENT",
|
||
offer_type="SELL",
|
||
page=1,
|
||
source_url=(
|
||
f"https://realty.yandex.ru/otsenka-kvartiry-po-adresu-onlayn/?address={address}"
|
||
),
|
||
house=ValuationHouseMeta(
|
||
year_built=1981,
|
||
total_floors=9,
|
||
house_type="panel",
|
||
ceiling_height=2.5,
|
||
has_lift=True,
|
||
total_objects=12,
|
||
),
|
||
history_items=[
|
||
ValuationHistoryItem(
|
||
area_m2=36.8,
|
||
rooms=1,
|
||
floor=8,
|
||
start_price=4_500_000,
|
||
start_price_per_m2=122_283,
|
||
last_price=4_400_000,
|
||
last_price_per_m2=119_566,
|
||
publish_date=date(2026, 2, 17),
|
||
exposure_days=96,
|
||
status="В продаже",
|
||
),
|
||
ValuationHistoryItem(
|
||
area_m2=64.4,
|
||
rooms=2,
|
||
floor=1,
|
||
start_price=6_580_000,
|
||
start_price_per_m2=102_174,
|
||
last_price=6_100_000,
|
||
last_price_per_m2=94_721,
|
||
publish_date=date(2026, 2, 2),
|
||
exposure_days=111,
|
||
status="В продаже",
|
||
),
|
||
],
|
||
)
|
||
|
||
|
||
def test_cache_key_deterministic():
|
||
k1 = _yandex_valuation_cache_key("addr X", "APARTMENT", "SELL")
|
||
k2 = _yandex_valuation_cache_key("addr X", "APARTMENT", "SELL")
|
||
assert k1 == k2
|
||
assert len(k1) == 64
|
||
|
||
|
||
def test_cache_key_different_for_different_inputs():
|
||
a = _yandex_valuation_cache_key("addr X", "APARTMENT", "SELL")
|
||
b = _yandex_valuation_cache_key("addr Y", "APARTMENT", "SELL")
|
||
c = _yandex_valuation_cache_key("addr X", "HOUSE", "SELL")
|
||
d = _yandex_valuation_cache_key("addr X", "APARTMENT", "RENT")
|
||
assert len({a, b, c, d}) == 4
|
||
|
||
|
||
def test_ttl_default_24h():
|
||
assert YANDEX_VALUATION_CACHE_TTL_HOURS == 24
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_cache_hit_returns_deserialized_result():
|
||
"""When cache row exists and hasn't expired, return parsed YandexValuationResult."""
|
||
db = MagicMock()
|
||
result = _sample_result()
|
||
db.execute.return_value.mappings.return_value.first.return_value = {
|
||
"raw_payload": result.model_dump(mode="json"),
|
||
"fetched_at": None,
|
||
}
|
||
with patch("app.services.estimator.YandexValuationScraper") as mock_scraper_cls:
|
||
out = await _get_or_fetch_yandex_valuation_cached(db, address=result.address)
|
||
mock_scraper_cls.assert_not_called() # no fetch on hit
|
||
assert out is not None
|
||
assert out.address == result.address
|
||
assert len(out.history_items) == 2
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_cache_miss_triggers_fetch_and_persist():
|
||
"""On cache miss, fetch via scraper + INSERT into external_valuations."""
|
||
db = MagicMock()
|
||
db.execute.return_value.mappings.return_value.first.return_value = None # miss
|
||
fresh = _sample_result()
|
||
fake_scraper = MagicMock()
|
||
fake_scraper.__aenter__ = AsyncMock(return_value=fake_scraper)
|
||
fake_scraper.__aexit__ = AsyncMock(return_value=False)
|
||
fake_scraper.fetch_house_history = AsyncMock(return_value=fresh)
|
||
with patch("app.services.estimator.YandexValuationScraper", return_value=fake_scraper):
|
||
out = await _get_or_fetch_yandex_valuation_cached(db, address=fresh.address)
|
||
assert out is fresh
|
||
fake_scraper.fetch_house_history.assert_awaited_once()
|
||
# 2 DB calls: 1 SELECT (miss) + 1 INSERT/UPSERT
|
||
assert db.execute.call_count >= 2
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_error_returns_none_gracefully():
|
||
db = MagicMock()
|
||
db.execute.return_value.mappings.return_value.first.return_value = None
|
||
fake_scraper = MagicMock()
|
||
fake_scraper.__aenter__ = AsyncMock(return_value=fake_scraper)
|
||
fake_scraper.__aexit__ = AsyncMock(return_value=False)
|
||
fake_scraper.fetch_house_history = AsyncMock(side_effect=RuntimeError("network"))
|
||
with patch("app.services.estimator.YandexValuationScraper", return_value=fake_scraper):
|
||
out = await _get_or_fetch_yandex_valuation_cached(db, address="addr")
|
||
assert out is None
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_returns_none_propagates_none():
|
||
db = MagicMock()
|
||
db.execute.return_value.mappings.return_value.first.return_value = None
|
||
fake_scraper = MagicMock()
|
||
fake_scraper.__aenter__ = AsyncMock(return_value=fake_scraper)
|
||
fake_scraper.__aexit__ = AsyncMock(return_value=False)
|
||
fake_scraper.fetch_house_history = AsyncMock(return_value=None)
|
||
with patch("app.services.estimator.YandexValuationScraper", return_value=fake_scraper):
|
||
out = await _get_or_fetch_yandex_valuation_cached(db, address="addr")
|
||
assert out is None
|
||
|
||
|
||
def test_save_history_items_inserts_each():
|
||
db = MagicMock()
|
||
result = _sample_result()
|
||
saved = _save_yandex_history_items(db, result)
|
||
assert saved == 2
|
||
# 2 INSERTs + 1 commit
|
||
assert db.execute.call_count == 2
|
||
db.commit.assert_called_once()
|
||
|
||
|
||
def test_save_history_items_empty_no_commit():
|
||
db = MagicMock()
|
||
result = YandexValuationResult(
|
||
address="x",
|
||
offer_category="APARTMENT",
|
||
offer_type="SELL",
|
||
page=1,
|
||
source_url="https://x",
|
||
house=ValuationHouseMeta(),
|
||
history_items=[],
|
||
)
|
||
saved = _save_yandex_history_items(db, result)
|
||
assert saved == 0
|
||
db.execute.assert_not_called()
|
||
db.commit.assert_not_called()
|
||
|
||
|
||
def test_save_history_items_ext_id_stable_across_calls():
|
||
"""Same result → same ext_item_id sequence → idempotent UPSERT."""
|
||
db1 = MagicMock()
|
||
db2 = MagicMock()
|
||
result = _sample_result()
|
||
_save_yandex_history_items(db1, result)
|
||
_save_yandex_history_items(db2, result)
|
||
# Both got same call params (ext_id derived from same seed)
|
||
ext_ids_1 = [c.args[1]["ext_id"] for c in db1.execute.call_args_list]
|
||
ext_ids_2 = [c.args[1]["ext_id"] for c in db2.execute.call_args_list]
|
||
assert ext_ids_1 == ext_ids_2
|
||
|
||
|
||
def test_save_history_items_db_error_continues():
|
||
"""One item failing doesn't abort the others."""
|
||
db = MagicMock()
|
||
db.execute.side_effect = [RuntimeError("first row fails"), None]
|
||
result = _sample_result()
|
||
saved = _save_yandex_history_items(db, result)
|
||
assert saved == 1 # second item still saved
|