"""Unit tests for PR J — backfill_listing_sources.py.

Coverage:
    - Happy path: one listing → upsert_listing_source called + house resolved.
    - Idempotency: re-run skips listings already in listing_sources via the
      NOT EXISTS predicate (verified by feeding the second SELECT a 0-row
      result and confirming no hooks fire).
    - house_source/house_ext_id linkage: when the listing carries them they
      are passed straight to match_or_create_house as ext_source/ext_id.
    - house match failure: upsert_listing_source still fires with
      source_data=None so the listing_sources row is created without a
      house linkage.
    - --dry-run: no writes at all (no db.execute writes, no match service
      calls).
    - --limit: caps total rows processed even when more would otherwise be
      returned by the next batch.
    - _ext_id_for fallback order — source_id → dedup_hash → computed sha.

No real Postgres. The DB session is a MagicMock that records execute() calls
and short-circuits matching service calls via patch().
"""

from __future__ import annotations

import os
from contextlib import contextmanager
from unittest.mock import MagicMock, patch

# Settings requires DATABASE_URL at import time — set dummy DSN before any
# `app.*` import (same pattern as test_audit_address_mismatch.py).
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")

import pytest

from scripts.backfill_listing_sources import (
    ListingRow,
    Stats,
    _build_select_sql,
    _ext_id_for,
    _link_listing_to_house,
    _process_row,
    main,
    run_backfill,
)

# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------


def _row(**overrides) -> ListingRow:
    """Minimal valid ListingRow with one source_id by default."""
    base = {
        "id": 1,
        "source": "avito",
        "source_id": "100",
        "source_url": "https://www.avito.ru/ekaterinburg/kvartiry/1-k_100",
        "dedup_hash": "a" * 64,
        "address": "Екатеринбург, Ленина, 1",
        "lat": 56.84,
        "lon": 60.60,
        "rooms": 2,
        "area_m2": 50.0,
        "floor": 5,
        "price_rub": 4_500_000,
        "year_built": 2010,
        "cadastral_number": None,
        "building_cadastral_number": None,
        "house_source": None,
        "house_ext_id": None,
        "house_url": None,
        "house_id_fk": None,
    }
    base.update(overrides)
    return ListingRow(**base)


def _make_db_mock(batches: list[list[dict]] | None = None) -> MagicMock:
    """MagicMock Session that returns the next batch on each SELECT.

    `batches` — list of row-dict lists; each subsequent call to db.execute()
    with a `FROM listings` query returns the next one. After exhaustion an
    empty list is returned so the driver stops.

    The mock honors the :limit bind value so test_run_backfill_limit_caps_*
    can rely on the driver's effective_size = min(batch_size, limit-processed)
    logic without depending on real SQL LIMIT behaviour.

    Coverage query (`GROUP BY l.source`) — separate side-effect for the final
    log line; we return a single-row aggregate so the summary code path runs.
    """
    batches = batches or []
    pending: list[list[dict]] = list(batches)

    db = MagicMock()

    @contextmanager
    def _nested():
        yield MagicMock()

    db.begin_nested.side_effect = _nested
    db.commit = MagicMock()
    db.rollback = MagicMock()
    db.close = MagicMock()

    def execute_side_effect(sql, params=None):
        sql_str = str(sql)
        result = MagicMock()
        if "GROUP BY l.source" in sql_str:
            # _coverage_summary
            result.mappings.return_value.all.return_value = [
                {"source": "avito", "total": 10, "linked": 10},
            ]
            return result
        if "FROM listings" in sql_str:
            batch = pending.pop(0) if pending else []
            # Honor SQL LIMIT — driver computes effective_size and binds it.
            if params is not None and "limit" in params:
                batch = batch[: params["limit"]]
            result.mappings.return_value.all.return_value = batch
            return result
        # UPDATE listings.house_id_fk — no-op
        if "UPDATE listings" in sql_str:
            return result
        return result

    db.execute.side_effect = execute_side_effect
    return db


def _row_dict(row: ListingRow) -> dict:
    """Convert ListingRow → dict form returned by db.execute(...).mappings()."""
    return {
        "id": row.id,
        "source": row.source,
        "source_id": row.source_id,
        "source_url": row.source_url,
        "dedup_hash": row.dedup_hash,
        "address": row.address,
        "lat": row.lat,
        "lon": row.lon,
        "rooms": row.rooms,
        "area_m2": row.area_m2,
        "floor": row.floor,
        "price_rub": row.price_rub,
        "year_built": row.year_built,
        "cadastral_number": row.cadastral_number,
        "building_cadastral_number": row.building_cadastral_number,
        "house_source": row.house_source,
        "house_ext_id": row.house_ext_id,
        "house_url": row.house_url,
        "house_id_fk": row.house_id_fk,
    }


@pytest.fixture
def _patch_matching():
    """Stub matching service for all tests."""
    with (
        patch("scripts.backfill_listing_sources.match_or_create_house") as m_house,
        patch("scripts.backfill_listing_sources.upsert_listing_source") as m_link,
    ):
        m_house.return_value = (501, 1.0, "new")
        m_link.return_value = None
        yield {"house": m_house, "link": m_link}


# ---------------------------------------------------------------------------
# _ext_id_for — fallback order
# ---------------------------------------------------------------------------


def test_ext_id_prefers_source_id():
    r = _row(source_id="abc", dedup_hash="d" * 64)
    assert _ext_id_for(r) == "abc"


def test_ext_id_falls_back_to_dedup_hash():
    r = _row(source_id=None, dedup_hash="d" * 64)
    assert _ext_id_for(r) == "d" * 64


def test_ext_id_computed_when_both_missing():
    """Defensive — listings.dedup_hash is NOT NULL UNIQUE, but if it's somehow
    blank we recompute a sha256 so we never write '' as ext_id."""
    r = _row(source_id=None, dedup_hash="")
    out = _ext_id_for(r)
    assert len(out) == 64  # sha256 hex


# ---------------------------------------------------------------------------
# _build_select_sql — NOT EXISTS predicate + optional source filter
# ---------------------------------------------------------------------------


def test_select_sql_excludes_listings_already_in_listing_sources():
    sql = _build_select_sql(source=None, batch_size=500)
    assert "NOT EXISTS" in sql
    assert "listing_sources" in sql
    # The NOT EXISTS body must match on COALESCE(source_id, dedup_hash) so
    # Yandex (no source_id) listings are still uniquely identifiable.
    assert "COALESCE(listings.source_id, listings.dedup_hash)" in sql
    # Default — no `source = :source` clause.
    assert ":source" not in sql


def test_select_sql_filters_by_source_when_provided():
    sql = _build_select_sql(source="avito", batch_size=500)
    assert "source = :source" in sql


def test_select_sql_has_no_dropped_kadastr_num_column():
    """`listings.kadastr_num` was dropped in 094_cadastral_unify.sql — the SELECT
    must not reference it or psycopg raises UndefinedColumn on every run."""
    sql = _build_select_sql(source=None, batch_size=500)
    assert "kadastr_num" not in sql
    # The canonical apartment-level cadastral columns remain.
    assert "cadastral_number" in sql
    assert "building_cadastral_number" in sql


# ---------------------------------------------------------------------------
# Happy path — one listing → 1 link + 1 house resolution
# ---------------------------------------------------------------------------


def test_link_listing_happy_path(_patch_matching):
    """Single listing with address → match_or_create_house + upsert called once."""
    db = _make_db_mock()
    row = _row(id=42, source_id="100")

    house_id, resolved = _link_listing_to_house(db, row, dry_run=False)

    assert house_id == 501
    assert resolved is True
    assert _patch_matching["house"].call_count == 1
    assert _patch_matching["link"].call_count == 1

    # upsert kwargs reflect the listing context
    kwargs = _patch_matching["link"].call_args.kwargs
    assert kwargs["listing_id"] == 42
    assert kwargs["ext_source"] == "avito"
    assert kwargs["ext_id"] == "100"
    assert kwargs["method"] == "backfill"
    assert kwargs["confidence"] == 0.9
    assert kwargs["price_rub"] == 4_500_000
    assert kwargs["area_m2"] == 50.0
    assert kwargs["floor"] == 5
    assert kwargs["rooms_count"] == 2
    assert kwargs["source_data"] == {"house_id": 501}


def test_link_listing_no_address_no_coords_skips_house_match(_patch_matching):
    """No address + no coords → no house resolution, but listing_sources still upserted."""
    db = _make_db_mock()
    row = _row(
        source_id="200",
        address=None,
        lat=None,
        lon=None,
    )

    house_id, resolved = _link_listing_to_house(db, row, dry_run=False)

    assert house_id is None
    assert resolved is False
    assert _patch_matching["house"].call_count == 0
    assert _patch_matching["link"].call_count == 1
    kwargs = _patch_matching["link"].call_args.kwargs
    assert kwargs["source_data"] is None


# ---------------------------------------------------------------------------
# house_source / house_ext_id direct linkage
# ---------------------------------------------------------------------------


def test_link_listing_uses_house_source_ext_id_when_present(_patch_matching):
    """Avito Houses Catalog row: house_source='avito', house_ext_id='3171365' →
    those are passed verbatim to match_or_create_house."""
    db = _make_db_mock()
    row = _row(
        source="avito",
        source_id="100",
        house_source="avito",
        house_ext_id="3171365",
        house_url="https://www.avito.ru/catalog/houses/foo/3171365",
    )

    _link_listing_to_house(db, row, dry_run=False)

    kwargs = _patch_matching["house"].call_args.kwargs
    assert kwargs["ext_source"] == "avito"
    assert kwargs["ext_id"] == "3171365"
    assert kwargs["source_url"] == "https://www.avito.ru/catalog/houses/foo/3171365"


def test_link_listing_falls_back_to_listing_source_when_house_missing(
    _patch_matching,
):
    """No house_source on listing → ext_source = listing.source, ext_id =
    source_id/dedup_hash. Same as PR I hook."""
    db = _make_db_mock()
    row = _row(source="cian", source_id="999", house_source=None, house_ext_id=None)

    _link_listing_to_house(db, row, dry_run=False)

    kwargs = _patch_matching["house"].call_args.kwargs
    assert kwargs["ext_source"] == "cian"
    assert kwargs["ext_id"] == "999"


# ---------------------------------------------------------------------------
# House match failure still upserts listing_sources
# ---------------------------------------------------------------------------


def test_link_listing_house_match_failure_still_upserts_source(_patch_matching):
    """match_or_create_house raises → listing_sources row created with house_id=None."""
    _patch_matching["house"].side_effect = RuntimeError("normalize boom")

    db = _make_db_mock()
    row = _row(source_id="bad", address="Bad addr")

    house_id, resolved = _link_listing_to_house(db, row, dry_run=False)

    assert house_id is None
    assert resolved is False
    assert _patch_matching["house"].call_count == 1
    assert _patch_matching["link"].call_count == 1
    kwargs = _patch_matching["link"].call_args.kwargs
    assert kwargs["source_data"] is None


# ---------------------------------------------------------------------------
# Dry run — no writes anywhere
# ---------------------------------------------------------------------------


def test_link_listing_dry_run_skips_all_writes(_patch_matching):
    """--dry-run path: no match_or_create_house call, no upsert_listing_source call."""
    db = _make_db_mock()
    row = _row(source_id="100")

    _link_listing_to_house(db, row, dry_run=True)

    assert _patch_matching["house"].call_count == 0
    assert _patch_matching["link"].call_count == 0


# ---------------------------------------------------------------------------
# _process_row — SAVEPOINT wrapping, stats updates
# ---------------------------------------------------------------------------


def test_process_row_updates_stats_on_success(_patch_matching):
    db = _make_db_mock()
    row = _row(source="cian", source_id="abc")

    stats = Stats()
    _process_row(db, row, dry_run=False, stats=stats)

    assert stats.processed == 1
    assert stats.matched == 1
    assert stats.house_resolved == 1
    assert stats.errors == 0
    assert stats.by_source["cian"]["matched"] == 1
    assert stats.by_source["cian"]["house_resolved"] == 1


def test_process_row_updates_stats_on_error(_patch_matching):
    """upsert raises inside the SAVEPOINT → counted in errors, not matched."""
    _patch_matching["link"].side_effect = RuntimeError("DB blip")

    db = _make_db_mock()
    row = _row(source="avito", source_id="abc")

    stats = Stats()
    _process_row(db, row, dry_run=False, stats=stats)

    assert stats.processed == 1
    assert stats.errors == 1
    assert stats.matched == 0
    assert stats.by_source["avito"]["errors"] == 1


# ---------------------------------------------------------------------------
# run_backfill — batch loop + idempotency + --limit
# ---------------------------------------------------------------------------


def test_run_backfill_processes_single_batch(_patch_matching):
    """One batch of 2 rows → 2 link calls + commit."""
    rows = [_row(id=1, source_id="a"), _row(id=2, source_id="b")]
    db = _make_db_mock(batches=[[_row_dict(r) for r in rows], []])

    stats = run_backfill(db, batch_size=500, limit=None, source=None, dry_run=False)

    assert stats.processed == 2
    assert _patch_matching["link"].call_count == 2
    # Exactly one batch committed before the empty-batch stops the loop.
    assert db.commit.call_count == 1


def test_run_backfill_idempotent_no_remaining(_patch_matching):
    """When SELECT returns 0 rows (re-run after full backfill) → 0 work done."""
    db = _make_db_mock(batches=[[]])

    stats = run_backfill(db, batch_size=500, limit=None, source=None, dry_run=False)

    assert stats.processed == 0
    assert _patch_matching["link"].call_count == 0
    assert _patch_matching["house"].call_count == 0


def test_run_backfill_dry_run_skips_writes(_patch_matching):
    """--dry-run: counters move, but no upsert/match calls and no commit."""
    rows = [_row(id=1, source_id="a")]
    db = _make_db_mock(batches=[[_row_dict(r) for r in rows], []])

    stats = run_backfill(db, batch_size=500, limit=None, source=None, dry_run=True)

    assert stats.processed == 1
    assert _patch_matching["link"].call_count == 0
    assert _patch_matching["house"].call_count == 0
    assert db.commit.call_count == 0


def test_run_backfill_limit_caps_processed(_patch_matching):
    """--limit 1 stops after the first row even when more would arrive."""
    rows = [
        _row(id=1, source_id="a"),
        _row(id=2, source_id="b"),
        _row(id=3, source_id="c"),
    ]
    db = _make_db_mock(batches=[[_row_dict(r) for r in rows], []])

    stats = run_backfill(db, batch_size=500, limit=1, source=None, dry_run=False)

    # Limit truncates the batch size, so only 1 row is fetched.
    assert stats.processed == 1
    assert _patch_matching["link"].call_count == 1


def test_run_backfill_advances_after_id(_patch_matching):
    """Cursor advances past max(id) in each batch so the next SELECT uses :after_id."""
    rows_batch1 = [_row(id=1, source_id="a"), _row(id=5, source_id="b")]
    rows_batch2 = [_row(id=12, source_id="c")]
    db = _make_db_mock(
        batches=[
            [_row_dict(r) for r in rows_batch1],
            [_row_dict(r) for r in rows_batch2],
            [],
        ]
    )

    stats = run_backfill(db, batch_size=2, limit=None, source=None, dry_run=False)

    assert stats.processed == 3
    # Third SELECT should have been called with after_id >= 12.
    # Find the SELECT calls and look at the last one's params.
    listings_calls = [c for c in db.execute.call_args_list if "FROM listings" in str(c.args[0])]
    assert len(listings_calls) >= 3
    last_params = listings_calls[-1].args[1]
    assert last_params["after_id"] >= 12


# ---------------------------------------------------------------------------
# main() CLI smoke — happy path with one batch
# ---------------------------------------------------------------------------


def test_main_cli_runs_one_batch(_patch_matching):
    """main(--limit 1 --dry-run) returns 1 row processed."""
    rows = [_row(id=1, source_id="a")]

    class FakeSessionLocal:
        def __call__(self):
            return _make_db_mock(batches=[[_row_dict(r) for r in rows], []])

    with patch("scripts.backfill_listing_sources.SessionLocal", FakeSessionLocal()):
        processed = main(["--limit", "1", "--dry-run"])

    assert processed == 1
