214 lines
7.2 KiB
Python
214 lines
7.2 KiB
Python
"""Tests for the external_valuations.house_id backfill job (issue #2236).
|
||
|
||
Coverage:
|
||
- happy path: resolvable rows get UPDATE-ed with house_id, counts add up.
|
||
- unresolved rows (matcher → None) stay untouched, no UPDATE, counted.
|
||
- per-row error isolation: one raising matcher call doesn't kill the batch.
|
||
- batching: batch_size < total → multiple batches, one commit each.
|
||
- idempotency: after a run marks rows resolved, a re-run finds nothing.
|
||
- dry-run: no UPDATE / no commit.
|
||
|
||
No real Postgres — DB is a stateful MagicMock modelling the
|
||
`WHERE house_id IS NULL AND id > :after_id` source query and the guarded
|
||
UPDATE (same convention as test_backfill_house_coords).
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import os
|
||
|
||
# Settings требует DATABASE_URL на import.
|
||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
|
||
|
||
from unittest.mock import MagicMock
|
||
|
||
from scripts.backfill_external_valuations_house_id import (
|
||
ExtValRow,
|
||
_fetch_batch,
|
||
run_backfill,
|
||
)
|
||
|
||
|
||
def _make_db(pool: list[dict]):
|
||
"""Stateful MagicMock DB.
|
||
|
||
`pool` = list of {id, source, address, house_id}. SELECT returns rows with
|
||
house_id IS NULL AND id > after_id (ordered, capped by limit). UPDATE sets
|
||
house_id on the matching pool row (guarded on house_id IS NULL).
|
||
Records commits.
|
||
"""
|
||
db = MagicMock()
|
||
db.begin_nested.return_value.__enter__ = lambda self: self
|
||
db.begin_nested.return_value.__exit__ = lambda self, *a: False
|
||
commits: list[int] = []
|
||
|
||
def execute_side_effect(sql, params=None):
|
||
sql_str = str(sql)
|
||
result = MagicMock()
|
||
if "SELECT id, source, address" in sql_str:
|
||
after_id = params["after_id"]
|
||
limit = params["limit"]
|
||
src = params.get("source")
|
||
rows = [
|
||
{"id": r["id"], "source": r["source"], "address": r["address"]}
|
||
for r in sorted(pool, key=lambda x: x["id"])
|
||
if r["house_id"] is None
|
||
and r["id"] > after_id
|
||
and r["address"] is not None
|
||
and (src is None or r["source"] == src)
|
||
][:limit]
|
||
result.mappings.return_value.all.return_value = rows
|
||
elif "UPDATE external_valuations" in sql_str:
|
||
for r in pool:
|
||
if r["id"] == params["eid"] and r["house_id"] is None:
|
||
r["house_id"] = params["hid"]
|
||
return result
|
||
|
||
db.execute.side_effect = execute_side_effect
|
||
db.commit.side_effect = lambda: commits.append(1)
|
||
db.close = MagicMock()
|
||
return db, commits
|
||
|
||
|
||
def _pool(n: int) -> list[dict]:
|
||
return [
|
||
{
|
||
"id": i,
|
||
"source": "cian_valuation",
|
||
"address": f"Екатеринбург, ул Тест, {i}",
|
||
"house_id": None,
|
||
}
|
||
for i in range(1, n + 1)
|
||
]
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# _fetch_batch
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_fetch_batch_maps_rows():
|
||
db, _ = _make_db(_pool(3))
|
||
rows = _fetch_batch(db, after_id=0, batch_size=10, source=None)
|
||
assert [r.id for r in rows] == [1, 2, 3]
|
||
assert all(isinstance(r, ExtValRow) for r in rows)
|
||
|
||
|
||
def test_fetch_batch_respects_after_id_and_limit():
|
||
db, _ = _make_db(_pool(5))
|
||
rows = _fetch_batch(db, after_id=2, batch_size=2, source=None)
|
||
assert [r.id for r in rows] == [3, 4]
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Happy path — resolvable rows updated
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_run_resolves_and_updates():
|
||
pool = _pool(3)
|
||
db, commits = _make_db(pool)
|
||
matcher = MagicMock(return_value=555)
|
||
|
||
stats = run_backfill(db, batch_size=10, limit=None, source=None, dry_run=False, matcher=matcher)
|
||
|
||
assert stats.processed == 3
|
||
assert stats.resolved == 3
|
||
assert stats.unresolved == 0
|
||
assert stats.errors == 0
|
||
assert all(r["house_id"] == 555 for r in pool)
|
||
assert len(commits) == 1 # one batch → one commit
|
||
|
||
|
||
def test_run_unresolved_rows_untouched():
|
||
"""matcher → None: house_id остаётся NULL, UPDATE не вызывается, счётчик растёт."""
|
||
pool = _pool(2)
|
||
db, _ = _make_db(pool)
|
||
matcher = MagicMock(return_value=None)
|
||
|
||
stats = run_backfill(db, batch_size=10, limit=None, source=None, dry_run=False, matcher=matcher)
|
||
|
||
assert stats.processed == 2
|
||
assert stats.resolved == 0
|
||
assert stats.unresolved == 2
|
||
assert all(r["house_id"] is None for r in pool)
|
||
|
||
|
||
def test_run_row_error_isolated():
|
||
"""Матчер кидает на одной строке — батч продолжается, ошибка посчитана."""
|
||
pool = _pool(3)
|
||
db, _ = _make_db(pool)
|
||
|
||
def matcher(_db, addr):
|
||
if "2" in addr:
|
||
raise RuntimeError("boom")
|
||
return 999
|
||
|
||
stats = run_backfill(db, batch_size=10, limit=None, source=None, dry_run=False, matcher=matcher)
|
||
assert stats.processed == 3
|
||
assert stats.resolved == 2
|
||
assert stats.errors == 1
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Batching
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_run_multiple_batches_commit_each():
|
||
pool = _pool(5)
|
||
db, commits = _make_db(pool)
|
||
matcher = MagicMock(return_value=7)
|
||
|
||
stats = run_backfill(db, batch_size=2, limit=None, source=None, dry_run=False, matcher=matcher)
|
||
# 5 rows / batch 2 → batches of 2, 2, 1 = 3 commits
|
||
assert stats.processed == 5
|
||
assert stats.resolved == 5
|
||
assert len(commits) == 3
|
||
|
||
|
||
def test_run_limit_caps_total():
|
||
pool = _pool(10)
|
||
db, _ = _make_db(pool)
|
||
matcher = MagicMock(return_value=1)
|
||
|
||
stats = run_backfill(db, batch_size=3, limit=4, source=None, dry_run=False, matcher=matcher)
|
||
assert stats.processed == 4
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Idempotency
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_run_idempotent_second_pass_noop():
|
||
pool = _pool(4)
|
||
db, _ = _make_db(pool)
|
||
matcher = MagicMock(return_value=42)
|
||
|
||
first = run_backfill(db, batch_size=10, limit=None, source=None, dry_run=False, matcher=matcher)
|
||
assert first.processed == 4 and first.resolved == 4
|
||
|
||
# Все строки теперь house_id != NULL → source-query пуст на re-run.
|
||
second = run_backfill(
|
||
db, batch_size=10, limit=None, source=None, dry_run=False, matcher=matcher
|
||
)
|
||
assert second.processed == 0
|
||
assert second.resolved == 0
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Dry-run
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_run_dry_run_no_writes():
|
||
pool = _pool(3)
|
||
db, commits = _make_db(pool)
|
||
matcher = MagicMock(return_value=123)
|
||
|
||
stats = run_backfill(db, batch_size=10, limit=None, source=None, dry_run=True, matcher=matcher)
|
||
assert stats.processed == 3
|
||
assert stats.resolved == 3 # matcher нашёл дом
|
||
assert all(r["house_id"] is None for r in pool) # но НЕ записали
|
||
assert commits == []
|