"""Tests for app.tasks.yandex_detail_backfill (the KIT task — already imports `scraper_kit.providers.yandex.detail.{YandexDetailScraper,save_detail_enrichment}`). Fetch mechanism changed: curl_cffi AsyncSession(chrome120+proxy) + YandexDetailScraper.parse (instead of old fetch_detail path). Mocks target session.get and scraper.parse. Легаси `app.services.scrapers.yandex_detail` удалён (#2277 финальный шаг scraper_kit-миграции) — `save_detail_enrichment` coverage-тесты внизу файла переведены на kit-эквивалент (тот же, что реально вызывает эта задача). """ from __future__ import annotations import json import os import sys from unittest.mock import AsyncMock, MagicMock, patch os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test") _wp_mock = MagicMock() sys.modules.setdefault("weasyprint", _wp_mock) import pytest # noqa: E402 from app.tasks.yandex_detail_backfill import ( # noqa: E402 YandexDetailBackfillResult, run_yandex_detail_backfill, ) # --------------------------------------------------------------------------- # Path constants for patching # --------------------------------------------------------------------------- _ASYNC_SESSION = "app.tasks.yandex_detail_backfill.AsyncSession" _PARSE = "app.tasks.yandex_detail_backfill.YandexDetailScraper.parse" _SAVE = "app.tasks.yandex_detail_backfill.save_detail_enrichment" _RUNS = "app.tasks.yandex_detail_backfill.runs_mod" _SLEEP = "app.tasks.yandex_detail_backfill.asyncio.sleep" _SETTINGS = "app.tasks.yandex_detail_backfill.settings" # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- def _make_snapshot(n: int) -> list[dict]: return [ {"id": i + 1, "source_url": f"https://realty.yandex.ru/offer/{i + 1}/"} for i in range(n) ] def _mock_db(snapshot: list[dict]) -> MagicMock: """Fake Session: first execute() returns snapshot via .mappings().all().""" db = MagicMock() sel = MagicMock() sel.mappings.return_value.all.return_value = snapshot db.execute.return_value = sel return db def _make_resp(status: int = 200, text: str = "ok") -> MagicMock: """Fake curl_cffi response.""" resp = MagicMock() resp.status_code = status resp.text = text return resp def _make_session_ctx(get_side_effect) -> MagicMock: """Build AsyncSession context-manager mock with session.get side_effect.""" session = AsyncMock() session.get = AsyncMock(side_effect=get_side_effect) ctx = MagicMock() ctx.__aenter__ = AsyncMock(return_value=session) ctx.__aexit__ = AsyncMock(return_value=None) session_cls = MagicMock(return_value=ctx) return session_cls, session def _mock_settings(proxy: str | None = "http://proxy:3128") -> MagicMock: s = MagicMock() s.scraper_proxy_url = proxy return s # --------------------------------------------------------------------------- # Tests: run_yandex_detail_backfill # --------------------------------------------------------------------------- @pytest.mark.asyncio async def test_backfill_empty_snapshot_marks_done() -> None: """Empty snapshot -> mark_done immediately, no session.get calls.""" db = _mock_db([]) runs = MagicMock() session_cls, session = _make_session_ctx([]) with ( patch(_ASYNC_SESSION, session_cls), patch(_RUNS, runs), patch(_SETTINGS, _mock_settings()), ): result = await run_yandex_detail_backfill( db, run_id=1, params={"batch_size": 10, "budget_sec": 60} ) assert isinstance(result, YandexDetailBackfillResult) assert result.attempted == 0 assert result.enriched == 0 session.get.assert_not_called() runs.mark_done.assert_called_once() runs.mark_failed.assert_not_called() @pytest.mark.asyncio async def test_backfill_processes_snapshot_to_completion() -> None: """3 listings -> all fetched (200) + parse success -> enriched=3, mark_done.""" snapshot = _make_snapshot(3) db = _mock_db(snapshot) runs = MagicMock() mock_enrichment = MagicMock() responses = [_make_resp(200), _make_resp(200), _make_resp(200)] session_cls, session = _make_session_ctx(responses) with ( patch(_ASYNC_SESSION, session_cls), patch(_PARSE, return_value=mock_enrichment), patch(_RUNS, runs), patch(_SAVE, return_value=True), patch(_SLEEP, new_callable=AsyncMock), patch(_SETTINGS, _mock_settings()), ): result = await run_yandex_detail_backfill( db, run_id=2, params={"batch_size": 10, "budget_sec": 3600} ) assert result.attempted == 3 assert result.enriched == 3 assert result.failed == 0 assert session.get.call_count == 3 runs.mark_done.assert_called_once() runs.mark_failed.assert_not_called() @pytest.mark.asyncio async def test_backfill_parse_none_abort_after_max_consecutive() -> None: """5 consecutive parse→None results -> abort, mark_done (NOT mark_failed).""" snapshot = _make_snapshot(10) db = _mock_db(snapshot) runs = MagicMock() responses = [_make_resp(200)] * 10 session_cls, _session = _make_session_ctx(responses) with ( patch(_ASYNC_SESSION, session_cls), patch(_PARSE, return_value=None), patch(_RUNS, runs), patch(_SLEEP, new_callable=AsyncMock), patch(_SETTINGS, _mock_settings()), ): result = await run_yandex_detail_backfill( db, run_id=3, params={"batch_size": 10, "budget_sec": 3600, "max_consecutive_blocks": 5}, ) assert result.attempted == 5 assert result.failed == 5 assert result.enriched == 0 runs.mark_done.assert_called_once() runs.mark_failed.assert_not_called() @pytest.mark.asyncio async def test_backfill_parse_none_resets_on_success() -> None: """1 parse→None + 1 success + 1 parse→None -> consecutive resets, no abort.""" snapshot = _make_snapshot(3) db = _mock_db(snapshot) runs = MagicMock() mock_enrichment = MagicMock() responses = [_make_resp(200)] * 3 session_cls, _session = _make_session_ctx(responses) parse_results = [None, mock_enrichment, None] with ( patch(_ASYNC_SESSION, session_cls), patch(_PARSE, side_effect=parse_results), patch(_RUNS, runs), patch(_SAVE, return_value=True), patch(_SLEEP, new_callable=AsyncMock), patch(_SETTINGS, _mock_settings()), ): result = await run_yandex_detail_backfill( db, run_id=4, params={"batch_size": 10, "budget_sec": 3600, "max_consecutive_blocks": 5}, ) assert result.attempted == 3 assert result.enriched == 1 assert result.failed == 2 runs.mark_done.assert_called_once() @pytest.mark.asyncio async def test_backfill_non200_counts_as_fail_and_aborts() -> None: """5 consecutive HTTP 403 responses -> abort after max_consecutive_blocks.""" snapshot = _make_snapshot(10) db = _mock_db(snapshot) runs = MagicMock() responses = [_make_resp(403)] * 10 session_cls, _session = _make_session_ctx(responses) with ( patch(_ASYNC_SESSION, session_cls), patch(_PARSE, return_value=MagicMock()), patch(_RUNS, runs), patch(_SLEEP, new_callable=AsyncMock), patch(_SETTINGS, _mock_settings()), ): result = await run_yandex_detail_backfill( db, run_id=5, params={"batch_size": 10, "budget_sec": 3600, "max_consecutive_blocks": 5}, ) assert result.failed == 5 assert result.enriched == 0 runs.mark_done.assert_called_once() runs.mark_failed.assert_not_called() @pytest.mark.asyncio async def test_backfill_budget_guard_stops_loop() -> None: """Budget expired before first listing -> session.get not called.""" snapshot = _make_snapshot(5) db = _mock_db(snapshot) runs = MagicMock() responses = [_make_resp(200)] * 5 session_cls, session = _make_session_ctx(responses) mono_values = iter([0.0, 999.0, 999.0]) with ( patch(_ASYNC_SESSION, session_cls), patch(_RUNS, runs), patch("app.tasks.yandex_detail_backfill.time.monotonic", side_effect=mono_values), patch(_SETTINGS, _mock_settings()), ): await run_yandex_detail_backfill(db, run_id=6, params={"batch_size": 5, "budget_sec": 1}) session.get.assert_not_called() runs.mark_done.assert_called_once() @pytest.mark.asyncio async def test_backfill_top_level_exception_marks_failed() -> None: """db.execute raises -> mark_failed called, exception re-raised.""" db = MagicMock() db.execute.side_effect = RuntimeError("DB connection lost") runs = MagicMock() with ( patch(_RUNS, runs), patch(_SETTINGS, _mock_settings()), ): with pytest.raises(RuntimeError, match="DB connection lost"): await run_yandex_detail_backfill( db, run_id=7, params={"batch_size": 5, "budget_sec": 60} ) runs.mark_failed.assert_called_once() runs.mark_done.assert_not_called() @pytest.mark.asyncio async def test_backfill_fetch_exception_continues() -> None: """session.get raises on first listing -> failed++, loop continues for second.""" snapshot = _make_snapshot(2) db = _mock_db(snapshot) runs = MagicMock() mock_enrichment = MagicMock() # First call raises, second returns 200 OK responses = [RuntimeError("connection reset"), _make_resp(200)] session_cls, _session = _make_session_ctx(responses) with ( patch(_ASYNC_SESSION, session_cls), patch(_PARSE, return_value=mock_enrichment), patch(_RUNS, runs), patch(_SAVE, return_value=True), patch(_SLEEP, new_callable=AsyncMock), patch(_SETTINGS, _mock_settings()), ): result = await run_yandex_detail_backfill( db, run_id=8, params={"batch_size": 10, "budget_sec": 3600} ) assert result.failed == 1 assert result.enriched == 1 assert result.attempted == 2 # fetch exception is caught by inner try/except (not DB) — no rollback needed db.rollback.assert_not_called() runs.mark_done.assert_called_once() @pytest.mark.asyncio async def test_backfill_no_proxy_when_settings_none() -> None: """scraper_proxy_url=None -> AsyncSession called with proxies=None.""" snapshot = _make_snapshot(1) db = _mock_db(snapshot) runs = MagicMock() mock_enrichment = MagicMock() responses = [_make_resp(200)] session_cls, _session = _make_session_ctx(responses) with ( patch(_ASYNC_SESSION, session_cls), patch(_PARSE, return_value=mock_enrichment), patch(_RUNS, runs), patch(_SAVE, return_value=True), patch(_SLEEP, new_callable=AsyncMock), patch(_SETTINGS, _mock_settings(proxy=None)), ): result = await run_yandex_detail_backfill( db, run_id=9, params={"batch_size": 10, "budget_sec": 3600} ) assert result.enriched == 1 # Verify AsyncSession was constructed with proxies=None call_kwargs = session_cls.call_args[1] assert call_kwargs.get("proxies") is None # --------------------------------------------------------------------------- # Tests: save_detail_enrichment (kit — same function the task above actually calls; # легаси `app.services.scrapers.yandex_detail` удалён #2277 финальный шаг миграции) # --------------------------------------------------------------------------- def test_save_detail_enrichment_maps_fields_to_update() -> None: """save_detail_enrichment calls db.execute with UPDATE and commits.""" from scraper_kit.providers.yandex.detail import ( DetailEnrichment, MetroStation, save_detail_enrichment, ) enrichment = DetailEnrichment( offer_id="12345", source_url="https://realty.yandex.ru/offer/12345/", rooms=2, area_m2=54.5, floor=3, total_floors=9, address="Екатеринбург, ул. Ленина, 10", description="Хорошая квартира", repair_state="standard", publish_date=None, views_total=42, publish_date_relative="вчера", agency_name="Агентство «Тест»", agency_founded_year=2005, agency_objects_count=150, metro_stations=[MetroStation(name="Чкаловская", walk_min=11)], photo_urls=["https://example.com/1.jpg", "https://example.com/2.jpg"], newbuilding_url="https://realty.yandex.ru/kupit/novostrojka/test-12345/", newbuilding_id="12345", ) db = MagicMock() result_mock = MagicMock() result_mock.rowcount = 1 db.execute.return_value = result_mock saved = save_detail_enrichment(db, listing_id=99, e=enrichment) assert saved is True db.execute.assert_called_once() db.commit.assert_called_once() call_args = db.execute.call_args params = call_args[0][1] assert params["rooms"] == 2 assert params["area_m2"] == 54.5 assert params["agency_name"] == "Агентство «Тест»" metro_val = params["metro_stations"] assert metro_val is not None parsed_metro = json.loads(metro_val) assert parsed_metro[0]["name"] == "Чкаловская" assert parsed_metro[0]["walk_min"] == 11 photo_val = params["photo_urls"] assert photo_val is not None parsed_photos = json.loads(photo_val) assert len(parsed_photos) == 2 assert parsed_photos[0] == "https://example.com/1.jpg" def test_save_detail_enrichment_empty_metro_and_photos() -> None: """Empty metro_stations and photo_urls -> NULL passed for both jsonb columns.""" from scraper_kit.providers.yandex.detail import DetailEnrichment, save_detail_enrichment enrichment = DetailEnrichment( offer_id="99", source_url="https://realty.yandex.ru/offer/99/", ) db = MagicMock() result_mock = MagicMock() result_mock.rowcount = 1 db.execute.return_value = result_mock save_detail_enrichment(db, listing_id=1, e=enrichment) params = db.execute.call_args[0][1] assert params["metro_stations"] is None assert params["photo_urls"] is None def test_save_detail_enrichment_rowcount_zero_returns_false() -> None: """rowcount=0 (listing_id not found) -> returns False.""" from scraper_kit.providers.yandex.detail import DetailEnrichment, save_detail_enrichment enrichment = DetailEnrichment( offer_id="404", source_url="https://realty.yandex.ru/offer/404/", ) db = MagicMock() result_mock = MagicMock() result_mock.rowcount = 0 db.execute.return_value = result_mock saved = save_detail_enrichment(db, listing_id=404, e=enrichment) assert saved is False