"""Tests for the external_valuations.house_id backfill job (issue #2236). Coverage: - happy path: resolvable rows get UPDATE-ed with house_id, counts add up. - unresolved rows (matcher → None) stay untouched, no UPDATE, counted. - per-row error isolation: one raising matcher call doesn't kill the batch. - batching: batch_size < total → multiple batches, one commit each. - idempotency: after a run marks rows resolved, a re-run finds nothing. - dry-run: no UPDATE / no commit. No real Postgres — DB is a stateful MagicMock modelling the `WHERE house_id IS NULL AND id > :after_id` source query and the guarded UPDATE (same convention as test_backfill_house_coords). """ from __future__ import annotations import os # Settings требует DATABASE_URL на import. os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db") from unittest.mock import MagicMock from scripts.backfill_external_valuations_house_id import ( ExtValRow, _fetch_batch, run_backfill, ) def _make_db(pool: list[dict]): """Stateful MagicMock DB. `pool` = list of {id, source, address, house_id}. SELECT returns rows with house_id IS NULL AND id > after_id (ordered, capped by limit). UPDATE sets house_id on the matching pool row (guarded on house_id IS NULL). Records commits. """ db = MagicMock() db.begin_nested.return_value.__enter__ = lambda self: self db.begin_nested.return_value.__exit__ = lambda self, *a: False commits: list[int] = [] def execute_side_effect(sql, params=None): sql_str = str(sql) result = MagicMock() if "SELECT id, source, address" in sql_str: after_id = params["after_id"] limit = params["limit"] src = params.get("source") rows = [ {"id": r["id"], "source": r["source"], "address": r["address"]} for r in sorted(pool, key=lambda x: x["id"]) if r["house_id"] is None and r["id"] > after_id and r["address"] is not None and (src is None or r["source"] == src) ][:limit] result.mappings.return_value.all.return_value = rows elif "UPDATE external_valuations" in sql_str: for r in pool: if r["id"] == params["eid"] and r["house_id"] is None: r["house_id"] = params["hid"] return result db.execute.side_effect = execute_side_effect db.commit.side_effect = lambda: commits.append(1) db.close = MagicMock() return db, commits def _pool(n: int) -> list[dict]: return [ { "id": i, "source": "cian_valuation", "address": f"Екатеринбург, ул Тест, {i}", "house_id": None, } for i in range(1, n + 1) ] # --------------------------------------------------------------------------- # _fetch_batch # --------------------------------------------------------------------------- def test_fetch_batch_maps_rows(): db, _ = _make_db(_pool(3)) rows = _fetch_batch(db, after_id=0, batch_size=10, source=None) assert [r.id for r in rows] == [1, 2, 3] assert all(isinstance(r, ExtValRow) for r in rows) def test_fetch_batch_respects_after_id_and_limit(): db, _ = _make_db(_pool(5)) rows = _fetch_batch(db, after_id=2, batch_size=2, source=None) assert [r.id for r in rows] == [3, 4] # --------------------------------------------------------------------------- # Happy path — resolvable rows updated # --------------------------------------------------------------------------- def test_run_resolves_and_updates(): pool = _pool(3) db, commits = _make_db(pool) matcher = MagicMock(return_value=555) stats = run_backfill(db, batch_size=10, limit=None, source=None, dry_run=False, matcher=matcher) assert stats.processed == 3 assert stats.resolved == 3 assert stats.unresolved == 0 assert stats.errors == 0 assert all(r["house_id"] == 555 for r in pool) assert len(commits) == 1 # one batch → one commit def test_run_unresolved_rows_untouched(): """matcher → None: house_id остаётся NULL, UPDATE не вызывается, счётчик растёт.""" pool = _pool(2) db, _ = _make_db(pool) matcher = MagicMock(return_value=None) stats = run_backfill(db, batch_size=10, limit=None, source=None, dry_run=False, matcher=matcher) assert stats.processed == 2 assert stats.resolved == 0 assert stats.unresolved == 2 assert all(r["house_id"] is None for r in pool) def test_run_row_error_isolated(): """Матчер кидает на одной строке — батч продолжается, ошибка посчитана.""" pool = _pool(3) db, _ = _make_db(pool) def matcher(_db, addr): if "2" in addr: raise RuntimeError("boom") return 999 stats = run_backfill(db, batch_size=10, limit=None, source=None, dry_run=False, matcher=matcher) assert stats.processed == 3 assert stats.resolved == 2 assert stats.errors == 1 # --------------------------------------------------------------------------- # Batching # --------------------------------------------------------------------------- def test_run_multiple_batches_commit_each(): pool = _pool(5) db, commits = _make_db(pool) matcher = MagicMock(return_value=7) stats = run_backfill(db, batch_size=2, limit=None, source=None, dry_run=False, matcher=matcher) # 5 rows / batch 2 → batches of 2, 2, 1 = 3 commits assert stats.processed == 5 assert stats.resolved == 5 assert len(commits) == 3 def test_run_limit_caps_total(): pool = _pool(10) db, _ = _make_db(pool) matcher = MagicMock(return_value=1) stats = run_backfill(db, batch_size=3, limit=4, source=None, dry_run=False, matcher=matcher) assert stats.processed == 4 # --------------------------------------------------------------------------- # Idempotency # --------------------------------------------------------------------------- def test_run_idempotent_second_pass_noop(): pool = _pool(4) db, _ = _make_db(pool) matcher = MagicMock(return_value=42) first = run_backfill(db, batch_size=10, limit=None, source=None, dry_run=False, matcher=matcher) assert first.processed == 4 and first.resolved == 4 # Все строки теперь house_id != NULL → source-query пуст на re-run. second = run_backfill( db, batch_size=10, limit=None, source=None, dry_run=False, matcher=matcher ) assert second.processed == 0 assert second.resolved == 0 # --------------------------------------------------------------------------- # Dry-run # --------------------------------------------------------------------------- def test_run_dry_run_no_writes(): pool = _pool(3) db, commits = _make_db(pool) matcher = MagicMock(return_value=123) stats = run_backfill(db, batch_size=10, limit=None, source=None, dry_run=True, matcher=matcher) assert stats.processed == 3 assert stats.resolved == 3 # matcher нашёл дом assert all(r["house_id"] is None for r in pool) # но НЕ записали assert commits == []