All checks were successful
CI / changes (pull_request) Successful in 9s
CI Trade-In / changes (pull_request) Successful in 9s
CI / backend-tests (pull_request) Has been skipped
CI / frontend-tests (pull_request) Has been skipped
CI / openapi-codegen-check (pull_request) Has been skipped
CI Trade-In / frontend-checks (pull_request) Has been skipped
CI Trade-In / backend-tests (pull_request) Successful in 1m48s
Group E4 (final, highest-risk step of scraper_kit migration epic #2277): estimator.py and house_imv_backfill.py's avito_imv/cian_valuation/ yandex_valuation call sites now import from scraper_kit.providers.* instead of app.services.scrapers.*, following the exact wiring proven safe by E1/E2/E3 (#2334/#2335/#2336): - estimator.py's avito IMV (_get_or_fetch_imv_cached, both call sites) and house_imv_backfill.py's _process_one_house: add config=RealScraperConfig() — kit's evaluate_via_imv silently drops the configured proxy without it. - estimator.py's cian valuation (Stage 9): add config=RealScraperConfig() — mandatory kwarg on the kit function (TypeError if omitted). - estimator.py's yandex valuation: add config=RealScraperConfig() (mandatory) and delay_provider=get_scraper_delay — without it kit silently falls back to a hardcoded 5.0s throttle instead of the DB-configured anti-ban delay. - All exception classes imported consistently from the same kit module as evaluate_via_imv (not just the function) — mixing legacy/kit exception classes would break `except IMVAddressNotFoundError` etc. via identity mismatch (caught by an existing test that assumed the legacy class, fixed alongside). Observability: both cian_valuation and yandex_valuation graceful-degradation except-blocks upgraded from logger.warning to logger.exception. GlitchTip's LoggingIntegration listens at event_level=ERROR (main.py/scheduler_main.py) — a WARNING never reaches GlitchTip as an event regardless of exc_info, so a future config-wiring mistake at these call sites needs ERROR level to be visible in monitoring. house_imv_backfill.py: RealScraperConfig is imported lazily inside _process_one_house (not at module level) to avoid a circular import — app.services.scraper_adapters imports backfill_house_imv/ process_houses_imv_batch from this module at module level. Verified via direct import in both orders plus a full `app.main` import. Also fixes a stale docstring claiming process_houses_imv_batch is "not wired into scheduler" — it is, via scrape_pipeline.py's run_avito_city_sweep. Test updates: 2 pre-existing tests (test_backfill_wave2.py) mocked the legacy IMVAddressNotFoundError/IMVEvaluation/IMVGeo classes, now updated to import from scraper_kit to match the production exception identity. Added config=/delay_provider= regression-guard asserts to the relevant estimator and backfill tests, mirroring the existing #2306 cian_price_history pattern. Legacy app/services/scrapers/{avito_imv,cian_valuation,yandex_valuation}.py are untouched and still imported by cian_history_backfill.py's valuation block (separately scoped, not touched here) — revert is a clean single-commit revert, no schema/data migration involved.
114 lines
3.8 KiB
Python
114 lines
3.8 KiB
Python
"""Offline smoke for estimator IMV integration. Mocks IMV fetch + DB."""
|
|
|
|
import os
|
|
|
|
# Settings требует DATABASE_URL при инициализации (fail-fast, C-3).
|
|
# Для offline unit-тестов задаём dummy DSN до любого импорта из app.
|
|
os.environ.setdefault("DATABASE_URL", "postgresql://test:test@localhost/test_db")
|
|
|
|
from unittest.mock import AsyncMock, MagicMock, patch
|
|
|
|
import anyio
|
|
|
|
from app.services.estimator import (
|
|
_IMV_HOUSE_TYPE_MAP,
|
|
_IMV_REPAIR_MAP,
|
|
_get_or_fetch_imv_cached,
|
|
)
|
|
|
|
|
|
def test_imv_house_type_map() -> None:
|
|
assert _IMV_HOUSE_TYPE_MAP["panel"] == "panel"
|
|
assert _IMV_HOUSE_TYPE_MAP["monolithic"] == "monolith"
|
|
assert _IMV_HOUSE_TYPE_MAP[None] is None
|
|
assert _IMV_HOUSE_TYPE_MAP.get("unknown") is None
|
|
|
|
|
|
def test_imv_repair_map() -> None:
|
|
assert _IMV_REPAIR_MAP["needs_repair"] == "required"
|
|
assert _IMV_REPAIR_MAP["excellent"] == "designer"
|
|
assert _IMV_REPAIR_MAP[None] is None
|
|
|
|
|
|
def test_imv_cache_miss_calls_evaluate() -> None:
|
|
"""Cache miss → calls evaluate_via_imv + save_imv_evaluation."""
|
|
from scraper_kit.providers.avito.imv import IMVEvaluation, IMVGeo
|
|
|
|
from app.services.scraper_adapters import RealScraperConfig
|
|
|
|
mock_db = MagicMock()
|
|
mock_db.execute.return_value.mappings.return_value.first.return_value = None # no cache hit
|
|
|
|
fake_result = IMVEvaluation(
|
|
cache_key="x" * 64,
|
|
address="ЕКБ test",
|
|
rooms=2,
|
|
area_m2=42.0,
|
|
floor=4,
|
|
floor_at_home=5,
|
|
house_type="panel",
|
|
renovation_type="cosmetic",
|
|
has_balcony=True,
|
|
has_loggia=False,
|
|
geo=IMVGeo(geo_hash="JWT"),
|
|
recommended_price=6_290_000,
|
|
lower_price=6_100_000,
|
|
higher_price=6_600_000,
|
|
market_count=800,
|
|
)
|
|
|
|
async def _run() -> None:
|
|
mock_evaluate = AsyncMock(return_value=fake_result)
|
|
with (
|
|
patch("app.services.estimator.evaluate_via_imv", new=mock_evaluate),
|
|
patch("app.services.estimator.save_imv_evaluation", return_value=1),
|
|
):
|
|
result = await _get_or_fetch_imv_cached(
|
|
mock_db,
|
|
address="ЕКБ test",
|
|
rooms=2,
|
|
area_m2=42.0,
|
|
floor=4,
|
|
floor_at_home=5,
|
|
house_type="panel",
|
|
renovation_type="cosmetic",
|
|
has_balcony=True,
|
|
has_loggia=False,
|
|
)
|
|
assert result is not None
|
|
assert result.recommended_price == 6_290_000
|
|
# #2337 regression guard (Group E4): kit evaluate_via_imv silently drops the
|
|
# Avito proxy (direct connection) unless config=RealScraperConfig() is passed
|
|
# at the call site — assert_called alone wouldn't catch someone dropping
|
|
# that kwarg later (same pattern as #2306 cian_price_history guards).
|
|
_, call_kwargs = mock_evaluate.call_args
|
|
assert isinstance(call_kwargs.get("config"), RealScraperConfig)
|
|
|
|
anyio.run(_run)
|
|
|
|
|
|
def test_imv_fetch_failure_returns_none_gracefully() -> None:
|
|
"""Network failure → returns None, не raises."""
|
|
mock_db = MagicMock()
|
|
mock_db.execute.return_value.mappings.return_value.first.return_value = None
|
|
|
|
async def _run() -> None:
|
|
with patch(
|
|
"app.services.estimator.evaluate_via_imv",
|
|
new=AsyncMock(side_effect=RuntimeError("test net error")),
|
|
):
|
|
result = await _get_or_fetch_imv_cached(
|
|
mock_db,
|
|
address="X",
|
|
rooms=2,
|
|
area_m2=42.0,
|
|
floor=4,
|
|
floor_at_home=5,
|
|
house_type="panel",
|
|
renovation_type="cosmetic",
|
|
has_balcony=True,
|
|
has_loggia=False,
|
|
)
|
|
assert result is None
|
|
|
|
anyio.run(_run)
|