bot-backend
17d558b18c
All checks were successful
Deploy Trade-In / changes (push) Successful in 12s
Deploy Trade-In / build-frontend (push) Has been skipped
Deploy Trade-In / build-browser (push) Has been skipped
Deploy Trade-In / test (push) Successful in 3m27s
Deploy Trade-In / build-backend (push) Successful in 1m33s
Deploy Trade-In / deploy (push) Successful in 2m12s
270 lines
10 KiB
Python
270 lines
10 KiB
Python
"""Tests for Yandex Valuation integration in estimator.py."""
|
||
|
||
import os
|
||
|
||
# Settings requires DATABASE_URL at init time. Set dummy DSN before any app import.
|
||
os.environ.setdefault("DATABASE_URL", "postgresql://test:test@localhost/test_db")
|
||
|
||
from datetime import date
|
||
from unittest.mock import AsyncMock, MagicMock, patch
|
||
|
||
import pytest
|
||
from scraper_kit.providers.yandex.valuation import (
|
||
ValuationHistoryItem,
|
||
ValuationHouseMeta,
|
||
YandexValuationResult,
|
||
)
|
||
|
||
from app.services.estimator import (
|
||
YANDEX_VALUATION_CACHE_TTL_HOURS,
|
||
_get_or_fetch_yandex_valuation_cached,
|
||
_save_yandex_history_items,
|
||
_yandex_valuation_cache_key,
|
||
)
|
||
from app.services.scraper_settings import get_scraper_delay
|
||
|
||
|
||
def _history_rows(db) -> list[dict]:
|
||
"""Строки батча house_placement_history из мока сессии (фильтр по SQL, не по позиции)."""
|
||
for call in db.execute.call_args_list:
|
||
if "INSERT INTO house_placement_history" in str(call.args[0]):
|
||
return call.args[1]
|
||
return []
|
||
|
||
|
||
def _sample_result(address: str = "Екатеринбург, ул. Учителей, 18") -> YandexValuationResult:
|
||
return YandexValuationResult(
|
||
address=address,
|
||
offer_category="APARTMENT",
|
||
offer_type="SELL",
|
||
page=1,
|
||
source_url=(
|
||
f"https://realty.yandex.ru/otsenka-kvartiry-po-adresu-onlayn/?address={address}"
|
||
),
|
||
house=ValuationHouseMeta(
|
||
year_built=1981,
|
||
total_floors=9,
|
||
house_type="panel",
|
||
ceiling_height=2.5,
|
||
has_lift=True,
|
||
total_objects=12,
|
||
),
|
||
history_items=[
|
||
ValuationHistoryItem(
|
||
area_m2=36.8,
|
||
rooms=1,
|
||
floor=8,
|
||
start_price=4_500_000,
|
||
start_price_per_m2=122_283,
|
||
last_price=4_400_000,
|
||
last_price_per_m2=119_566,
|
||
publish_date=date(2026, 2, 17),
|
||
exposure_days=96,
|
||
status="В продаже",
|
||
),
|
||
ValuationHistoryItem(
|
||
area_m2=64.4,
|
||
rooms=2,
|
||
floor=1,
|
||
start_price=6_580_000,
|
||
start_price_per_m2=102_174,
|
||
last_price=6_100_000,
|
||
last_price_per_m2=94_721,
|
||
publish_date=date(2026, 2, 2),
|
||
exposure_days=111,
|
||
status="В продаже",
|
||
),
|
||
],
|
||
)
|
||
|
||
|
||
def test_cache_key_deterministic():
|
||
k1 = _yandex_valuation_cache_key("addr X", "APARTMENT", "SELL")
|
||
k2 = _yandex_valuation_cache_key("addr X", "APARTMENT", "SELL")
|
||
assert k1 == k2
|
||
assert len(k1) == 64
|
||
|
||
|
||
def test_cache_key_different_for_different_inputs():
|
||
a = _yandex_valuation_cache_key("addr X", "APARTMENT", "SELL")
|
||
b = _yandex_valuation_cache_key("addr Y", "APARTMENT", "SELL")
|
||
c = _yandex_valuation_cache_key("addr X", "HOUSE", "SELL")
|
||
d = _yandex_valuation_cache_key("addr X", "APARTMENT", "RENT")
|
||
assert len({a, b, c, d}) == 4
|
||
|
||
|
||
def test_ttl_default_24h():
|
||
assert YANDEX_VALUATION_CACHE_TTL_HOURS == 24
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_cache_hit_returns_deserialized_result():
|
||
"""When cache row exists and hasn't expired, return parsed YandexValuationResult."""
|
||
db = MagicMock()
|
||
result = _sample_result()
|
||
db.execute.return_value.mappings.return_value.first.return_value = {
|
||
"raw_payload": result.model_dump(mode="json"),
|
||
"fetched_at": None,
|
||
}
|
||
with patch("app.services.estimator.YandexValuationScraper") as mock_scraper_cls:
|
||
out = await _get_or_fetch_yandex_valuation_cached(db, address=result.address)
|
||
mock_scraper_cls.assert_not_called() # no fetch on hit
|
||
assert out is not None
|
||
assert out.address == result.address
|
||
assert len(out.history_items) == 2
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_cache_miss_triggers_fetch_and_persist():
|
||
"""On cache miss, fetch via scraper + INSERT into external_valuations."""
|
||
from app.services.scraper_adapters import RealScraperConfig
|
||
|
||
db = MagicMock()
|
||
db.execute.return_value.mappings.return_value.first.return_value = None # miss
|
||
fresh = _sample_result()
|
||
fake_scraper = MagicMock()
|
||
fake_scraper.__aenter__ = AsyncMock(return_value=fake_scraper)
|
||
fake_scraper.__aexit__ = AsyncMock(return_value=False)
|
||
fake_scraper.fetch_house_history = AsyncMock(return_value=fresh)
|
||
with patch(
|
||
"app.services.estimator.YandexValuationScraper", return_value=fake_scraper
|
||
) as mock_scraper_cls:
|
||
out = await _get_or_fetch_yandex_valuation_cached(db, address=fresh.address)
|
||
assert out is fresh
|
||
fake_scraper.fetch_house_history.assert_awaited_once()
|
||
# #2337 regression guard (Group E4): kit YandexValuationScraper requires config=
|
||
# (TypeError if omitted) AND silently falls back to a hardcoded 5.0s throttle
|
||
# instead of the DB-configured anti-ban delay unless delay_provider= is passed.
|
||
# assert_called alone wouldn't catch either kwarg being dropped later.
|
||
call_args, call_kwargs = mock_scraper_cls.call_args
|
||
assert isinstance(call_args[0], RealScraperConfig)
|
||
assert call_kwargs.get("delay_provider") is get_scraper_delay
|
||
# 2 DB calls: 1 SELECT (miss) + 1 INSERT/UPSERT
|
||
assert db.execute.call_count >= 2
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_error_returns_none_gracefully():
|
||
db = MagicMock()
|
||
db.execute.return_value.mappings.return_value.first.return_value = None
|
||
fake_scraper = MagicMock()
|
||
fake_scraper.__aenter__ = AsyncMock(return_value=fake_scraper)
|
||
fake_scraper.__aexit__ = AsyncMock(return_value=False)
|
||
fake_scraper.fetch_house_history = AsyncMock(side_effect=RuntimeError("network"))
|
||
with patch("app.services.estimator.YandexValuationScraper", return_value=fake_scraper):
|
||
out = await _get_or_fetch_yandex_valuation_cached(db, address="addr")
|
||
assert out is None
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_returns_none_propagates_none():
|
||
db = MagicMock()
|
||
db.execute.return_value.mappings.return_value.first.return_value = None
|
||
fake_scraper = MagicMock()
|
||
fake_scraper.__aenter__ = AsyncMock(return_value=fake_scraper)
|
||
fake_scraper.__aexit__ = AsyncMock(return_value=False)
|
||
fake_scraper.fetch_house_history = AsyncMock(return_value=None)
|
||
with patch("app.services.estimator.YandexValuationScraper", return_value=fake_scraper):
|
||
out = await _get_or_fetch_yandex_valuation_cached(db, address="addr")
|
||
assert out is None
|
||
|
||
|
||
def test_save_history_items_inserts_each():
|
||
db = MagicMock()
|
||
result = _sample_result()
|
||
with patch(
|
||
"app.services.estimator.match_or_create_house",
|
||
return_value=(1, 0.9, "fingerprint"),
|
||
):
|
||
saved = _save_yandex_history_items(db, result)
|
||
assert saved == 2
|
||
# 1 batch INSERT (executemany). Фильтруем по SQL, а не по позиции вызова:
|
||
# #2674 однажды уже сдвинул позицию, добавив второй execute перед вставкой.
|
||
rows = _history_rows(db)
|
||
assert isinstance(rows, list) and len(rows) == 2
|
||
# Один коммит — батч истории. Второй (UPDATE houses.has_panorama) ушёл вместе
|
||
# с колонкой, хвост #2674, мигр. 259.
|
||
assert db.commit.call_count == 1
|
||
|
||
|
||
def test_save_history_items_empty_no_commit():
|
||
"""Пустая история + НЕподтверждённая страница → дом резолвится, но не пишется ничего.
|
||
|
||
#2674 (ревью): ранний возврат по пустой истории раньше стоял ПЕРВЫМ и заодно
|
||
отрезал резолв дома для отрисованных страниц без объявлений (~10%). Теперь
|
||
match_or_create_house вызывается до возврата — а записей по-прежнему ноль:
|
||
истории нет, вставлять нечего.
|
||
"""
|
||
db = MagicMock()
|
||
result = YandexValuationResult(
|
||
address="x",
|
||
offer_category="APARTMENT",
|
||
offer_type="SELL",
|
||
page=1,
|
||
source_url="https://x",
|
||
house=ValuationHouseMeta(),
|
||
history_items=[],
|
||
)
|
||
with patch(
|
||
"app.services.estimator.match_or_create_house",
|
||
return_value=(1, 0.9, "fingerprint"),
|
||
) as m:
|
||
saved = _save_yandex_history_items(db, result)
|
||
assert saved == 0
|
||
m.assert_called_once()
|
||
db.execute.assert_not_called()
|
||
db.commit.assert_not_called()
|
||
|
||
|
||
def test_save_history_items_ext_id_stable_across_calls():
|
||
"""Same result → same ext_item_id sequence → idempotent UPSERT."""
|
||
db1 = MagicMock()
|
||
db2 = MagicMock()
|
||
result = _sample_result()
|
||
with patch(
|
||
"app.services.estimator.match_or_create_house",
|
||
return_value=(1, 0.9, "fingerprint"),
|
||
):
|
||
_save_yandex_history_items(db1, result)
|
||
_save_yandex_history_items(db2, result)
|
||
# Both calls pass a list-of-dicts (batch executemany); ext_id extracted from each row
|
||
ext_ids_1 = [
|
||
r["ext_id"]
|
||
for c in db1.execute.call_args_list
|
||
if isinstance(c.args[1], list)
|
||
for r in c.args[1]
|
||
]
|
||
ext_ids_2 = [
|
||
r["ext_id"]
|
||
for c in db2.execute.call_args_list
|
||
if isinstance(c.args[1], list)
|
||
for r in c.args[1]
|
||
]
|
||
assert ext_ids_1 == ext_ids_2
|
||
assert len(ext_ids_1) == 2
|
||
|
||
|
||
def test_save_history_items_db_error_rolls_back_batch():
|
||
"""Any item failing rolls back the whole batch — batch semantics (finding #5).
|
||
|
||
#2674: side_effect адресуем по SQL, а не по позиции вызова. Урок остаётся в силе
|
||
и после сноса has_panorama (мигр. 259): позиционный side_effect молча проверял бы
|
||
не тот путь, стоит появиться любому новому execute перед вставкой истории.
|
||
"""
|
||
db = MagicMock()
|
||
|
||
def _fail_history(sql, *args, **kwargs):
|
||
if "INSERT INTO house_placement_history" in str(sql):
|
||
raise RuntimeError("first row fails")
|
||
return MagicMock()
|
||
|
||
db.execute.side_effect = _fail_history
|
||
result = _sample_result()
|
||
with patch(
|
||
"app.services.estimator.match_or_create_house",
|
||
return_value=(1, 0.9, "fingerprint"),
|
||
):
|
||
saved = _save_yandex_history_items(db, result)
|
||
assert saved == 0 # whole batch rolled back
|
||
db.rollback.assert_called_once()
|
||
assert db.commit.call_count == 0
|