gendesign/tradein-mvp/backend/tests/tasks/test_yandex_newbuilding_sweep.py

"""Tests for yandex_newbuilding_sweep task (#974).

No real network, no real Postgres. BrowserFetcher, resolve_yandex_jk_slug and
fetch_jk are mocked. The DB session is an in-memory fake that models
market.yandex_jk_enrichment + houses.yandex_jk_slug persistence assertions.
"""

from __future__ import annotations

import os
import sys
from unittest.mock import AsyncMock, MagicMock, patch

# DATABASE_URL required by config before any app import.
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")

# WeasyPrint stub — not installed in CI without GTK.
_wp_mock = MagicMock()
sys.modules.setdefault("weasyprint", _wp_mock)

import pytest  # noqa: E402

from app.tasks.yandex_newbuilding_sweep import (  # noqa: E402
    YandexNewbuildingSweepResult,
    count_yandex_newbuilding_houses,
    enrich_yandex_newbuilding_sweep,
)

# ── In-memory fake DB ─────────────────────────────────────────────────────────


class _FakeResult:
    def __init__(self, *, scalar=None, rows=None, fetchone=None):
        self._scalar = scalar
        self._rows = rows or []
        self._fetchone = fetchone

    def scalar_one(self):
        return self._scalar

    def mappings(self):
        return self

    def all(self):
        return self._rows

    def fetchone(self):
        return self._fetchone


class FakeDB:
    """Minimal SQLAlchemy-Session fake for yandex_newbuilding_sweep.

    Tracks:
      - enrichment: set of ext_id strings (inserted via UPSERT)
      - slug_updates: dict house_id → slug (persisted via UPDATE houses)
      - savepoints: int (# of begin_nested calls)
    """

    def __init__(self, pending_rows: list[dict]):
        self._pending_rows = pending_rows
        # market.yandex_jk_enrichment keyed by ext_id
        self.enrichment: dict[str, dict] = {}
        # houses.yandex_jk_slug written during resolve
        self.slug_updates: dict[int, str] = {}
        self.commits = 0
        self.rollbacks = 0
        self._sp_stack: list[_FakeSavepoint] = []

    def execute(self, statement, params=None):
        sql = str(statement)
        params = params or {}

        # COUNT queries
        if "COUNT(DISTINCT h.id)" in sql and "yandex_realty_nb" in sql:
            if "NOT EXISTS" in sql:
                # _COUNT_PENDING
                pending = sum(
                    1 for r in self._pending_rows if r.get("ext_id") not in self.enrichment
                )
                return _FakeResult(scalar=pending)
            if "yandex_jk_slug IS NOT NULL" in sql:
                # _COUNT_FETCHABLE
                return _FakeResult(scalar=len(self._pending_rows))
            # _COUNT_TOTAL
            return _FakeResult(scalar=len(self._pending_rows))

        # SELECT pending houses
        if "SELECT DISTINCT ON (h.id)" in sql and "yandex_realty_nb" in sql:
            lim = params.get("lim", len(self._pending_rows))
            force = params.get("force", False)
            out = []
            for r in self._pending_rows[:lim]:
                ext_id = r.get("ext_id")
                if not force and ext_id in self.enrichment:
                    continue
                # Reflect persisted slug
                slug = self.slug_updates.get(r["house_id"]) or r.get("yandex_jk_slug")
                out.append({**r, "yandex_jk_slug": slug})
            return _FakeResult(rows=out)

        # UPDATE houses SET yandex_jk_slug
        if "UPDATE houses" in sql and "yandex_jk_slug" in sql:
            self.slug_updates[params["hid"]] = params["slug"]
            return _FakeResult()

        # Idempotency check: SELECT 1 FROM market.yandex_jk_enrichment
        if "SELECT 1 FROM market.yandex_jk_enrichment" in sql:
            ext_id = params.get("ext_id")
            if ext_id in self.enrichment:
                return _FakeResult(fetchone=object())
            return _FakeResult(fetchone=None)

        # UPSERT market.yandex_jk_enrichment
        if "INSERT INTO market.yandex_jk_enrichment" in sql:
            self.enrichment[params["ext_id"]] = dict(params)
            return _FakeResult()

        return _FakeResult()

    def begin_nested(self):
        sp = _FakeSavepoint(self)
        self._sp_stack.append(sp)
        return sp

    def commit(self):
        self.commits += 1

    def rollback(self):
        self.rollbacks += 1


class _FakeSavepoint:
    def __init__(self, db: FakeDB):
        self._db = db

    def commit(self):
        pass

    def rollback(self):
        pass


# ── Tests: count_yandex_newbuilding_houses ───────────────────────────────────


def test_count_returns_sizing_dict():
    db = FakeDB([{"house_id": 1, "ext_id": "100", "yandex_jk_slug": None}])
    sizing = count_yandex_newbuilding_houses(db)
    assert "total" in sizing
    assert "fetchable" in sizing
    assert "pending" in sizing
    assert sizing["total"] >= 0


# ── Tests: enrich_yandex_newbuilding_sweep ───────────────────────────────────


@pytest.mark.asyncio
async def test_dry_run_returns_counts_without_fetch():
    """dry_run: популяция заполнена, fetch не вызван."""
    db = FakeDB(
        [
            {"house_id": 1, "ext_id": "111", "yandex_jk_slug": None, "yandex_jk_id": None},
        ]
    )
    result = await enrich_yandex_newbuilding_sweep(db, limit=5, dry_run=True)
    assert isinstance(result, YandexNewbuildingSweepResult)
    assert result.processed == 0  # dry_run = no processing
    assert result.rows_inserted == 0
    assert result.total >= 0


@pytest.mark.asyncio
async def test_sweep_with_existing_slug_upserts_enrichment():
    """House с уже известным slug: fetch_jk вызван, UPSERT market + счётчик."""
    db = FakeDB(
        [
            {
                "house_id": 10,
                "ext_id": "999",
                "yandex_jk_slug": "tatlin",
                "yandex_jk_id": "999",
            }
        ]
    )

    from app.services.scrapers.yandex_newbuilding import YandexNewbuildingInfo

    fake_info = YandexNewbuildingInfo(
        ext_id="999",
        ext_slug="tatlin",
        source_url="https://realty.yandex.ru/ekaterinburg/kupit/novostrojka/tatlin-999/",
        name="ЖК Татлин",
        developer_name="ПРИНЦИПЗ",
        lat=56.855,
        lon=60.576,
        rating=4.3,
        ratings_count=100,
        text_reviews_count=25,
    )

    with patch("app.services.scrapers.yandex_newbuilding.YandexNewbuildingScraper") as mock_scraper:
        scraper_instance = MagicMock()
        scraper_instance.fetch_jk = AsyncMock(return_value=fake_info)
        mock_scraper.return_value = scraper_instance

        result = await enrich_yandex_newbuilding_sweep(db, limit=5, request_delay_sec=0)

    assert result.processed == 1
    assert result.succeeded == 1
    assert result.rows_inserted == 1
    assert result.failed_fetch == 0
    assert result.failed_resolve == 0
    assert "999" in db.enrichment
    assert db.enrichment["999"]["name"] == "ЖК Татлин"


@pytest.mark.asyncio
async def test_sweep_resolves_slug_when_missing():
    """House без slug: resolve_yandex_jk_slug вызван, slug сохранён, fetch_jk вызван."""
    db = FakeDB(
        [
            {
                "house_id": 20,
                "ext_id": "888",
                "yandex_jk_slug": None,
                "yandex_jk_id": None,
            }
        ]
    )

    from app.services.scrapers.yandex_newbuilding import YandexNewbuildingInfo

    fake_info = YandexNewbuildingInfo(
        ext_id="888",
        ext_slug="testjk",
        source_url="https://realty.yandex.ru/ekaterinburg/kupit/novostrojka/testjk-888/",
        name="ЖК Тест",
        lat=56.8,
        lon=60.5,
    )

    with (
        patch(
            "app.services.scrapers.yandex_newbuilding.resolve_yandex_jk_slug",
            new_callable=AsyncMock,
            return_value="testjk",
        ) as mock_resolve,
        patch("app.services.scrapers.yandex_newbuilding.YandexNewbuildingScraper") as mock_scraper,
    ):
        scraper_instance = MagicMock()
        scraper_instance.fetch_jk = AsyncMock(return_value=fake_info)
        mock_scraper.return_value = scraper_instance

        result = await enrich_yandex_newbuilding_sweep(db, limit=5, request_delay_sec=0)

    mock_resolve.assert_called_once_with("888", city="ekaterinburg")
    assert result.resolved_slug == 1
    assert result.succeeded == 1
    assert result.rows_inserted == 1
    # Slug persisted to houses
    assert db.slug_updates.get(20) == "testjk"
    assert "888" in db.enrichment


@pytest.mark.asyncio
async def test_sweep_failed_resolve_counts():
    """resolve_yandex_jk_slug returns None → failed_resolve += 1, no upsert."""
    db = FakeDB(
        [
            {
                "house_id": 30,
                "ext_id": "777",
                "yandex_jk_slug": None,
                "yandex_jk_id": None,
            }
        ]
    )

    with patch(
        "app.services.scrapers.yandex_newbuilding.resolve_yandex_jk_slug",
        new_callable=AsyncMock,
        return_value=None,
    ):
        result = await enrich_yandex_newbuilding_sweep(db, limit=5, request_delay_sec=0)

    assert result.failed_resolve == 1
    assert result.succeeded == 0
    assert result.rows_inserted == 0
    assert "777" not in db.enrichment


@pytest.mark.asyncio
async def test_sweep_failed_fetch_counts():
    """fetch_jk returns None → failed_fetch += 1, no upsert."""
    db = FakeDB(
        [
            {
                "house_id": 40,
                "ext_id": "666",
                "yandex_jk_slug": "existingslug",
                "yandex_jk_id": "666",
            }
        ]
    )

    with patch("app.services.scrapers.yandex_newbuilding.YandexNewbuildingScraper") as mock_scraper:
        scraper_instance = MagicMock()
        scraper_instance.fetch_jk = AsyncMock(return_value=None)
        mock_scraper.return_value = scraper_instance

        result = await enrich_yandex_newbuilding_sweep(db, limit=5, request_delay_sec=0)

    assert result.failed_fetch == 1
    assert result.succeeded == 0
    assert "666" not in db.enrichment


@pytest.mark.asyncio
async def test_sweep_idempotent_skip_already_enriched():
    """force=False: дом уже в enrichment → пропускается без fetch."""
    db = FakeDB(
        [
            {
                "house_id": 50,
                "ext_id": "555",
                "yandex_jk_slug": "existingslug",
                "yandex_jk_id": "555",
            }
        ]
    )
    # Pre-populate enrichment
    db.enrichment["555"] = {"ext_id": "555", "name": "Already enriched"}

    with patch("app.services.scrapers.yandex_newbuilding.YandexNewbuildingScraper") as mock_scraper:
        scraper_instance = MagicMock()
        scraper_instance.fetch_jk = AsyncMock(return_value=None)
        mock_scraper.return_value = scraper_instance

        result = await enrich_yandex_newbuilding_sweep(db, limit=5, request_delay_sec=0)

    # The SELECT filters out already-enriched rows (force=False in SQL),
    # so processed=0 (SELECT returns empty)
    assert result.succeeded == 0
    scraper_instance.fetch_jk.assert_not_called()


@pytest.mark.asyncio
async def test_sweep_no_ext_id_fails_resolve():
    """House без ext_id и без slug → failed_resolve (нечего резолвить)."""
    db = FakeDB(
        [
            {
                "house_id": 60,
                "ext_id": None,
                "yandex_jk_slug": None,
                "yandex_jk_id": None,
            }
        ]
    )

    result = await enrich_yandex_newbuilding_sweep(db, limit=5, request_delay_sec=0)

    assert result.failed_resolve == 1
    assert result.succeeded == 0


@pytest.mark.asyncio
async def test_sweep_slug_present_but_ext_id_null_skips_fetch():
    """House с jk_slug но ext_id=NULL → guard срабатывает, fetch_jk НЕ вызван.

    M1 review: пустой jk_id даёт URL /{slug}-/, который является битым.
    Ожидаем failed_fetch += 1 и scraper.fetch_jk не вызван.
    """
    db = FakeDB(
        [
            {
                "house_id": 70,
                "ext_id": None,
                "yandex_jk_slug": "tatlin",
                "yandex_jk_id": None,
            }
        ]
    )

    with patch("app.services.scrapers.yandex_newbuilding.YandexNewbuildingScraper") as mock_scraper:
        scraper_instance = MagicMock()
        scraper_instance.fetch_jk = AsyncMock()
        mock_scraper.return_value = scraper_instance

        result = await enrich_yandex_newbuilding_sweep(db, limit=5, request_delay_sec=0)

    scraper_instance.fetch_jk.assert_not_called()
    assert result.failed_fetch == 1
    assert result.succeeded == 0
    assert result.rows_inserted == 0