"""Tests for yandex_newbuilding_sweep task (#974). No real network, no real Postgres. BrowserFetcher, resolve_yandex_jk_slug and fetch_jk are mocked. The DB session is an in-memory fake that models market.yandex_jk_enrichment + houses.yandex_jk_slug persistence assertions. """ from __future__ import annotations import os import sys from unittest.mock import AsyncMock, MagicMock, patch # DATABASE_URL required by config before any app import. os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test") # WeasyPrint stub — not installed in CI without GTK. _wp_mock = MagicMock() sys.modules.setdefault("weasyprint", _wp_mock) import pytest # noqa: E402 from app.tasks.yandex_newbuilding_sweep import ( # noqa: E402 YandexNewbuildingSweepResult, count_yandex_newbuilding_houses, enrich_yandex_newbuilding_sweep, ) # ── In-memory fake DB ───────────────────────────────────────────────────────── class _FakeResult: def __init__(self, *, scalar=None, rows=None, fetchone=None): self._scalar = scalar self._rows = rows or [] self._fetchone = fetchone def scalar_one(self): return self._scalar def mappings(self): return self def all(self): return self._rows def fetchone(self): return self._fetchone class FakeDB: """Minimal SQLAlchemy-Session fake for yandex_newbuilding_sweep. Tracks: - enrichment: set of ext_id strings (inserted via UPSERT) - slug_updates: dict house_id → slug (persisted via UPDATE houses) - savepoints: int (# of begin_nested calls) """ def __init__(self, pending_rows: list[dict]): self._pending_rows = pending_rows # market.yandex_jk_enrichment keyed by ext_id self.enrichment: dict[str, dict] = {} # houses.yandex_jk_slug written during resolve self.slug_updates: dict[int, str] = {} self.commits = 0 self.rollbacks = 0 self._sp_stack: list[_FakeSavepoint] = [] def execute(self, statement, params=None): sql = str(statement) params = params or {} # COUNT queries if "COUNT(DISTINCT h.id)" in sql and "yandex_realty_nb" in sql: if "NOT EXISTS" in sql: # _COUNT_PENDING pending = sum( 1 for r in self._pending_rows if r.get("ext_id") not in self.enrichment ) return _FakeResult(scalar=pending) if "yandex_jk_slug IS NOT NULL" in sql: # _COUNT_FETCHABLE return _FakeResult(scalar=len(self._pending_rows)) # _COUNT_TOTAL return _FakeResult(scalar=len(self._pending_rows)) # SELECT pending houses if "SELECT DISTINCT ON (h.id)" in sql and "yandex_realty_nb" in sql: lim = params.get("lim", len(self._pending_rows)) force = params.get("force", False) out = [] for r in self._pending_rows[:lim]: ext_id = r.get("ext_id") if not force and ext_id in self.enrichment: continue # Reflect persisted slug slug = self.slug_updates.get(r["house_id"]) or r.get("yandex_jk_slug") out.append({**r, "yandex_jk_slug": slug}) return _FakeResult(rows=out) # UPDATE houses SET yandex_jk_slug if "UPDATE houses" in sql and "yandex_jk_slug" in sql: self.slug_updates[params["hid"]] = params["slug"] return _FakeResult() # Idempotency check: SELECT 1 FROM market.yandex_jk_enrichment if "SELECT 1 FROM market.yandex_jk_enrichment" in sql: ext_id = params.get("ext_id") if ext_id in self.enrichment: return _FakeResult(fetchone=object()) return _FakeResult(fetchone=None) # UPSERT market.yandex_jk_enrichment if "INSERT INTO market.yandex_jk_enrichment" in sql: self.enrichment[params["ext_id"]] = dict(params) return _FakeResult() return _FakeResult() def begin_nested(self): sp = _FakeSavepoint(self) self._sp_stack.append(sp) return sp def commit(self): self.commits += 1 def rollback(self): self.rollbacks += 1 class _FakeSavepoint: def __init__(self, db: FakeDB): self._db = db def commit(self): pass def rollback(self): pass # ── Tests: count_yandex_newbuilding_houses ─────────────────────────────────── def test_count_returns_sizing_dict(): db = FakeDB([{"house_id": 1, "ext_id": "100", "yandex_jk_slug": None}]) sizing = count_yandex_newbuilding_houses(db) assert "total" in sizing assert "fetchable" in sizing assert "pending" in sizing assert sizing["total"] >= 0 # ── Tests: enrich_yandex_newbuilding_sweep ─────────────────────────────────── @pytest.mark.asyncio async def test_dry_run_returns_counts_without_fetch(): """dry_run: популяция заполнена, fetch не вызван.""" db = FakeDB( [ {"house_id": 1, "ext_id": "111", "yandex_jk_slug": None, "yandex_jk_id": None}, ] ) result = await enrich_yandex_newbuilding_sweep(db, limit=5, dry_run=True) assert isinstance(result, YandexNewbuildingSweepResult) assert result.processed == 0 # dry_run = no processing assert result.rows_inserted == 0 assert result.total >= 0 @pytest.mark.asyncio async def test_sweep_with_existing_slug_upserts_enrichment(): """House с уже известным slug: fetch_jk вызван, UPSERT market + счётчик.""" db = FakeDB( [ { "house_id": 10, "ext_id": "999", "yandex_jk_slug": "tatlin", "yandex_jk_id": "999", } ] ) from app.services.scrapers.yandex_newbuilding import YandexNewbuildingInfo fake_info = YandexNewbuildingInfo( ext_id="999", ext_slug="tatlin", source_url="https://realty.yandex.ru/ekaterinburg/kupit/novostrojka/tatlin-999/", name="ЖК Татлин", developer_name="ПРИНЦИПЗ", lat=56.855, lon=60.576, rating=4.3, ratings_count=100, text_reviews_count=25, ) with patch("app.services.scrapers.yandex_newbuilding.YandexNewbuildingScraper") as mock_scraper: scraper_instance = MagicMock() scraper_instance.fetch_jk = AsyncMock(return_value=fake_info) mock_scraper.return_value = scraper_instance result = await enrich_yandex_newbuilding_sweep(db, limit=5, request_delay_sec=0) assert result.processed == 1 assert result.succeeded == 1 assert result.rows_inserted == 1 assert result.failed_fetch == 0 assert result.failed_resolve == 0 assert "999" in db.enrichment assert db.enrichment["999"]["name"] == "ЖК Татлин" @pytest.mark.asyncio async def test_sweep_resolves_slug_when_missing(): """House без slug: resolve_yandex_jk_slug вызван, slug сохранён, fetch_jk вызван.""" db = FakeDB( [ { "house_id": 20, "ext_id": "888", "yandex_jk_slug": None, "yandex_jk_id": None, } ] ) from app.services.scrapers.yandex_newbuilding import YandexNewbuildingInfo fake_info = YandexNewbuildingInfo( ext_id="888", ext_slug="testjk", source_url="https://realty.yandex.ru/ekaterinburg/kupit/novostrojka/testjk-888/", name="ЖК Тест", lat=56.8, lon=60.5, ) with ( patch( "app.services.scrapers.yandex_newbuilding.resolve_yandex_jk_slug", new_callable=AsyncMock, return_value="testjk", ) as mock_resolve, patch("app.services.scrapers.yandex_newbuilding.YandexNewbuildingScraper") as mock_scraper, ): scraper_instance = MagicMock() scraper_instance.fetch_jk = AsyncMock(return_value=fake_info) mock_scraper.return_value = scraper_instance result = await enrich_yandex_newbuilding_sweep(db, limit=5, request_delay_sec=0) mock_resolve.assert_called_once_with("888", city="ekaterinburg") assert result.resolved_slug == 1 assert result.succeeded == 1 assert result.rows_inserted == 1 # Slug persisted to houses assert db.slug_updates.get(20) == "testjk" assert "888" in db.enrichment @pytest.mark.asyncio async def test_sweep_failed_resolve_counts(): """resolve_yandex_jk_slug returns None → failed_resolve += 1, no upsert.""" db = FakeDB( [ { "house_id": 30, "ext_id": "777", "yandex_jk_slug": None, "yandex_jk_id": None, } ] ) with patch( "app.services.scrapers.yandex_newbuilding.resolve_yandex_jk_slug", new_callable=AsyncMock, return_value=None, ): result = await enrich_yandex_newbuilding_sweep(db, limit=5, request_delay_sec=0) assert result.failed_resolve == 1 assert result.succeeded == 0 assert result.rows_inserted == 0 assert "777" not in db.enrichment @pytest.mark.asyncio async def test_sweep_failed_fetch_counts(): """fetch_jk returns None → failed_fetch += 1, no upsert.""" db = FakeDB( [ { "house_id": 40, "ext_id": "666", "yandex_jk_slug": "existingslug", "yandex_jk_id": "666", } ] ) with patch("app.services.scrapers.yandex_newbuilding.YandexNewbuildingScraper") as mock_scraper: scraper_instance = MagicMock() scraper_instance.fetch_jk = AsyncMock(return_value=None) mock_scraper.return_value = scraper_instance result = await enrich_yandex_newbuilding_sweep(db, limit=5, request_delay_sec=0) assert result.failed_fetch == 1 assert result.succeeded == 0 assert "666" not in db.enrichment @pytest.mark.asyncio async def test_sweep_idempotent_skip_already_enriched(): """force=False: дом уже в enrichment → пропускается без fetch.""" db = FakeDB( [ { "house_id": 50, "ext_id": "555", "yandex_jk_slug": "existingslug", "yandex_jk_id": "555", } ] ) # Pre-populate enrichment db.enrichment["555"] = {"ext_id": "555", "name": "Already enriched"} with patch("app.services.scrapers.yandex_newbuilding.YandexNewbuildingScraper") as mock_scraper: scraper_instance = MagicMock() scraper_instance.fetch_jk = AsyncMock(return_value=None) mock_scraper.return_value = scraper_instance result = await enrich_yandex_newbuilding_sweep(db, limit=5, request_delay_sec=0) # The SELECT filters out already-enriched rows (force=False in SQL), # so processed=0 (SELECT returns empty) assert result.succeeded == 0 scraper_instance.fetch_jk.assert_not_called() @pytest.mark.asyncio async def test_sweep_no_ext_id_fails_resolve(): """House без ext_id и без slug → failed_resolve (нечего резолвить).""" db = FakeDB( [ { "house_id": 60, "ext_id": None, "yandex_jk_slug": None, "yandex_jk_id": None, } ] ) result = await enrich_yandex_newbuilding_sweep(db, limit=5, request_delay_sec=0) assert result.failed_resolve == 1 assert result.succeeded == 0 @pytest.mark.asyncio async def test_sweep_slug_present_but_ext_id_null_skips_fetch(): """House с jk_slug но ext_id=NULL → guard срабатывает, fetch_jk НЕ вызван. M1 review: пустой jk_id даёт URL /{slug}-/, который является битым. Ожидаем failed_fetch += 1 и scraper.fetch_jk не вызван. """ db = FakeDB( [ { "house_id": 70, "ext_id": None, "yandex_jk_slug": "tatlin", "yandex_jk_id": None, } ] ) with patch("app.services.scrapers.yandex_newbuilding.YandexNewbuildingScraper") as mock_scraper: scraper_instance = MagicMock() scraper_instance.fetch_jk = AsyncMock() mock_scraper.return_value = scraper_instance result = await enrich_yandex_newbuilding_sweep(db, limit=5, request_delay_sec=0) scraper_instance.fetch_jk.assert_not_called() assert result.failed_fetch == 1 assert result.succeeded == 0 assert result.rows_inserted == 0