"""Tests for app.tasks.yandex_detail_backfill and save_detail_enrichment.""" from __future__ import annotations import json import os import sys from unittest.mock import AsyncMock, MagicMock, patch os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test") _wp_mock = MagicMock() sys.modules.setdefault("weasyprint", _wp_mock) import pytest # noqa: E402 from app.tasks.yandex_detail_backfill import ( # noqa: E402 YandexDetailBackfillResult, run_yandex_detail_backfill, ) # --------------------------------------------------------------------------- # Path constants for patching # --------------------------------------------------------------------------- _FETCH = "app.tasks.yandex_detail_backfill.YandexDetailScraper" _SAVE = "app.tasks.yandex_detail_backfill.save_detail_enrichment" _RUNS = "app.tasks.yandex_detail_backfill.runs_mod" _SLEEP = "app.tasks.yandex_detail_backfill.asyncio.sleep" # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- def _make_snapshot(n: int) -> list[dict]: return [ {"id": i + 1, "source_url": f"https://realty.yandex.ru/offer/{i + 1}/"} for i in range(n) ] def _mock_db(snapshot: list[dict]) -> MagicMock: """Fake Session: first execute() returns snapshot via .mappings().all().""" db = MagicMock() sel = MagicMock() sel.mappings.return_value.all.return_value = snapshot db.execute.return_value = sel return db # --------------------------------------------------------------------------- # Tests: run_yandex_detail_backfill # --------------------------------------------------------------------------- @pytest.mark.asyncio async def test_backfill_empty_snapshot_marks_done() -> None: """Empty snapshot -> mark_done immediately, no fetch calls.""" db = _mock_db([]) runs = MagicMock() mock_scraper_cls = MagicMock() mock_scraper_inst = AsyncMock() mock_scraper_cls.return_value.__aenter__ = AsyncMock(return_value=mock_scraper_inst) mock_scraper_cls.return_value.__aexit__ = AsyncMock(return_value=None) with ( patch(_FETCH, mock_scraper_cls), patch(_RUNS, runs), ): result = await run_yandex_detail_backfill( db, run_id=1, params={"batch_size": 10, "budget_sec": 60} ) assert isinstance(result, YandexDetailBackfillResult) assert result.attempted == 0 assert result.enriched == 0 mock_scraper_inst.fetch_detail.assert_not_called() runs.mark_done.assert_called_once() runs.mark_failed.assert_not_called() @pytest.mark.asyncio async def test_backfill_processes_snapshot_to_completion() -> None: """3 listings -> all fetched and enriched, mark_done called.""" snapshot = _make_snapshot(3) db = _mock_db(snapshot) runs = MagicMock() mock_enrichment = MagicMock() mock_scraper_cls = MagicMock() mock_scraper_inst = AsyncMock() mock_scraper_inst.fetch_detail = AsyncMock(return_value=mock_enrichment) mock_scraper_cls.return_value.__aenter__ = AsyncMock(return_value=mock_scraper_inst) mock_scraper_cls.return_value.__aexit__ = AsyncMock(return_value=None) with ( patch(_FETCH, mock_scraper_cls), patch(_RUNS, runs), patch(_SAVE, return_value=True), patch(_SLEEP, new_callable=AsyncMock), ): result = await run_yandex_detail_backfill( db, run_id=2, params={"batch_size": 10, "budget_sec": 3600} ) assert result.attempted == 3 assert result.enriched == 3 assert result.failed == 0 assert mock_scraper_inst.fetch_detail.call_count == 3 runs.mark_done.assert_called_once() runs.mark_failed.assert_not_called() @pytest.mark.asyncio async def test_backfill_none_abort_after_max_consecutive() -> None: """5 consecutive None results -> abort, mark_done (NOT mark_failed).""" snapshot = _make_snapshot(10) db = _mock_db(snapshot) runs = MagicMock() mock_scraper_cls = MagicMock() mock_scraper_inst = AsyncMock() mock_scraper_inst.fetch_detail = AsyncMock(return_value=None) mock_scraper_cls.return_value.__aenter__ = AsyncMock(return_value=mock_scraper_inst) mock_scraper_cls.return_value.__aexit__ = AsyncMock(return_value=None) with ( patch(_FETCH, mock_scraper_cls), patch(_RUNS, runs), patch(_SLEEP, new_callable=AsyncMock), ): result = await run_yandex_detail_backfill( db, run_id=3, params={"batch_size": 10, "budget_sec": 3600, "max_consecutive_blocks": 5}, ) assert result.attempted == 5 assert result.failed == 5 assert result.enriched == 0 runs.mark_done.assert_called_once() runs.mark_failed.assert_not_called() @pytest.mark.asyncio async def test_backfill_none_resets_on_success() -> None: """1 None + 1 success + 1 None -> consecutive resets, no abort after 1 success.""" snapshot = _make_snapshot(3) db = _mock_db(snapshot) runs = MagicMock() mock_enrichment = MagicMock() mock_scraper_cls = MagicMock() mock_scraper_inst = AsyncMock() mock_scraper_inst.fetch_detail = AsyncMock(side_effect=[None, mock_enrichment, None]) mock_scraper_cls.return_value.__aenter__ = AsyncMock(return_value=mock_scraper_inst) mock_scraper_cls.return_value.__aexit__ = AsyncMock(return_value=None) with ( patch(_FETCH, mock_scraper_cls), patch(_RUNS, runs), patch(_SAVE, return_value=True), patch(_SLEEP, new_callable=AsyncMock), ): result = await run_yandex_detail_backfill( db, run_id=4, params={"batch_size": 10, "budget_sec": 3600, "max_consecutive_blocks": 5}, ) assert result.attempted == 3 assert result.enriched == 1 assert result.failed == 2 runs.mark_done.assert_called_once() @pytest.mark.asyncio async def test_backfill_budget_guard_stops_loop() -> None: """Budget expired before first listing -> fetch_detail not called.""" snapshot = _make_snapshot(5) db = _mock_db(snapshot) runs = MagicMock() mock_scraper_cls = MagicMock() mock_scraper_inst = AsyncMock() mock_scraper_cls.return_value.__aenter__ = AsyncMock(return_value=mock_scraper_inst) mock_scraper_cls.return_value.__aexit__ = AsyncMock(return_value=None) mono_values = iter([0.0, 999.0, 999.0]) with ( patch(_FETCH, mock_scraper_cls), patch(_RUNS, runs), patch("app.tasks.yandex_detail_backfill.time.monotonic", side_effect=mono_values), ): await run_yandex_detail_backfill(db, run_id=5, params={"batch_size": 5, "budget_sec": 1}) mock_scraper_inst.fetch_detail.assert_not_called() runs.mark_done.assert_called_once() @pytest.mark.asyncio async def test_backfill_top_level_exception_marks_failed() -> None: """db.execute raises -> mark_failed called, exception re-raised.""" db = MagicMock() db.execute.side_effect = RuntimeError("DB connection lost") runs = MagicMock() with ( patch(_RUNS, runs), ): with pytest.raises(RuntimeError, match="DB connection lost"): await run_yandex_detail_backfill( db, run_id=6, params={"batch_size": 5, "budget_sec": 60} ) runs.mark_failed.assert_called_once() runs.mark_done.assert_not_called() @pytest.mark.asyncio async def test_backfill_fetch_exception_continues() -> None: """RuntimeError on one listing -> failed++, db.rollback(), loop continues for next.""" snapshot = _make_snapshot(2) db = _mock_db(snapshot) runs = MagicMock() mock_enrichment = MagicMock() mock_scraper_cls = MagicMock() mock_scraper_inst = AsyncMock() mock_scraper_inst.fetch_detail = AsyncMock( side_effect=[RuntimeError("parse error"), mock_enrichment] ) mock_scraper_cls.return_value.__aenter__ = AsyncMock(return_value=mock_scraper_inst) mock_scraper_cls.return_value.__aexit__ = AsyncMock(return_value=None) with ( patch(_FETCH, mock_scraper_cls), patch(_RUNS, runs), patch(_SAVE, return_value=True), patch(_SLEEP, new_callable=AsyncMock), ): result = await run_yandex_detail_backfill( db, run_id=7, params={"batch_size": 10, "budget_sec": 3600} ) assert result.failed == 1 assert result.enriched == 1 assert result.attempted == 2 db.rollback.assert_called() runs.mark_done.assert_called_once() # --------------------------------------------------------------------------- # Tests: save_detail_enrichment # --------------------------------------------------------------------------- def test_save_detail_enrichment_maps_fields_to_update() -> None: """save_detail_enrichment calls db.execute with UPDATE and commits.""" from app.services.scrapers.yandex_detail import ( DetailEnrichment, MetroStation, save_detail_enrichment, ) enrichment = DetailEnrichment( offer_id="12345", source_url="https://realty.yandex.ru/offer/12345/", rooms=2, area_m2=54.5, floor=3, total_floors=9, address="Екатеринбург, ул. Ленина, 10", description="Хорошая квартира", repair_state="standard", publish_date=None, views_total=42, publish_date_relative="вчера", agency_name="Агентство «Тест»", agency_founded_year=2005, agency_objects_count=150, metro_stations=[MetroStation(name="Чкаловская", walk_min=11)], photo_urls=["https://example.com/1.jpg", "https://example.com/2.jpg"], newbuilding_url="https://realty.yandex.ru/kupit/novostrojka/test-12345/", newbuilding_id="12345", ) db = MagicMock() result_mock = MagicMock() result_mock.rowcount = 1 db.execute.return_value = result_mock saved = save_detail_enrichment(db, listing_id=99, e=enrichment) assert saved is True db.execute.assert_called_once() db.commit.assert_called_once() # Verify the SQL call contains metro_stations and photo_urls as JSON call_args = db.execute.call_args params = call_args[0][1] # positional second arg to execute() assert params["rooms"] == 2 assert params["area_m2"] == 54.5 assert params["agency_name"] == "Агентство «Тест»" # metro_stations must be JSON-serialised metro_val = params["metro_stations"] assert metro_val is not None parsed_metro = json.loads(metro_val) assert parsed_metro[0]["name"] == "Чкаловская" assert parsed_metro[0]["walk_min"] == 11 # photo_urls must be JSON-serialised photo_val = params["photo_urls"] assert photo_val is not None parsed_photos = json.loads(photo_val) assert len(parsed_photos) == 2 assert parsed_photos[0] == "https://example.com/1.jpg" def test_save_detail_enrichment_empty_metro_and_photos() -> None: """Empty metro_stations and photo_urls -> NULL passed for both jsonb columns.""" from app.services.scrapers.yandex_detail import DetailEnrichment, save_detail_enrichment enrichment = DetailEnrichment( offer_id="99", source_url="https://realty.yandex.ru/offer/99/", ) db = MagicMock() result_mock = MagicMock() result_mock.rowcount = 1 db.execute.return_value = result_mock save_detail_enrichment(db, listing_id=1, e=enrichment) params = db.execute.call_args[0][1] assert params["metro_stations"] is None assert params["photo_urls"] is None def test_save_detail_enrichment_rowcount_zero_returns_false() -> None: """rowcount=0 (listing_id not found) -> returns False.""" from app.services.scrapers.yandex_detail import DetailEnrichment, save_detail_enrichment enrichment = DetailEnrichment( offer_id="404", source_url="https://realty.yandex.ru/offer/404/", ) db = MagicMock() result_mock = MagicMock() result_mock.rowcount = 0 db.execute.return_value = result_mock saved = save_detail_enrichment(db, listing_id=404, e=enrichment) assert saved is False