bot-backend
cc231946c2
All checks were successful
Deploy Trade-In / changes (push) Successful in 12s
Deploy Trade-In / build-browser (push) Has been skipped
Deploy Trade-In / build-frontend (push) Has been skipped
Deploy Trade-In / test (push) Successful in 45s
Deploy Trade-In / build-backend (push) Successful in 1m1s
Deploy Trade-In / deploy (push) Successful in 2m46s
389 lines
22 KiB
Python
389 lines
22 KiB
Python
"""Тесты ДОМ.РФ капремонт loader'а (app/services/domrf_kapremont_loader.py, мигр. 176, #2013).
|
||
|
||
Coverage:
|
||
- Чистые парс-хелперы: parse_decimal_comma / parse_int_field.
|
||
- parse_kr11_csv / parse_kr12_wall_materials — фикстуры собраны из РЕАЛЬНЫХ строк
|
||
региона 66 (export-kr1_1-66-20260701.csv / export-kr1_2-66-20260701.csv, дом
|
||
2170017185 «Ломоносова 3» и 2170025270 «Ломоносова 4»): КР1.2 long-format,
|
||
wall_material непустой только на строке construction_element_type='фасад'.
|
||
- build_domrf_houses — join КР1.1+КР1.2 по mkd_code.
|
||
- fetch_domrf_csvs — скачивание+распаковка zip, HTTP замокан через httpx.MockTransport
|
||
(никакой живой сети).
|
||
- Статические asserts по SQL: _UPSERT_SQL (ON CONFLICT, IS DISTINCT FROM, CAST),
|
||
backfill houses (COALESCE-порядок year_built/domrf/zhkh_year, only-fill-NULL),
|
||
propagate listings (только NULL listings.year_built).
|
||
- backfill_houses_from_domrf / propagate_listings_year_from_houses — dry_run вызывает
|
||
COUNT-SQL и не пишет, real-run суммирует rowcount обоих шагов (MagicMock db).
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import io
|
||
import os
|
||
import re
|
||
import zipfile
|
||
from pathlib import Path
|
||
from unittest.mock import MagicMock
|
||
|
||
# settings/SessionLocal не импортируем, но пакет app.* на импорте может тронуть DSN — как в
|
||
# sibling-тестах (test_zhkh_flats_loader) задаём заглушку (ядро статично/без реальной БД).
|
||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
|
||
|
||
import httpx
|
||
import pytest
|
||
|
||
from app.services import domrf_kapremont_loader as dkl
|
||
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# Фикстуры: РЕАЛЬНЫЕ строки региона 66 (export-kr1_1/kr1_2-66-20260701.csv)
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# КР1.1 house registry — 2 дома (Ломоносова 3 / Ломоносова 4, п. Арамиль).
|
||
_KR11_HEADER = (
|
||
"subject_rf;mun_obr_oktmo;mun_obr;mkd_code;houseguid;address;commission_year;"
|
||
"architectural_monument_category;total_sq;total_rooms_amount;living_rooms_amount;"
|
||
"total_rooms_sq;living_rooms_sq;total_ppl;number_floors_max;money_collecting_way;"
|
||
"money_ppl_collected;money_ppl_collected_debts;overhaul_funds_spent_all;"
|
||
"overhaul_funds_spent_subsidy;overhaul_fund_spent_other;overhaul_funds_balance;"
|
||
"update_date_of_information;money_ppl_collected_date;owners_payment;energy_efficiency;"
|
||
"alarm_document_date;exclude_date_from_program;inclusion_date_to_program;comment;"
|
||
"last_update;house_id"
|
||
)
|
||
_KR11_ROW_3 = (
|
||
'"Свердловская область";65729000;"Арамильский городской округ Свердловской области";'
|
||
"2170017185;04b98bcb-3232-4b26-b0fa-3c4a47adf394;"
|
||
'"Арамильский городской округ Свердловской области, пос. Арамиль, ул. Ломоносова, д. 3";'
|
||
'1964;;399,90;;;366,60;333,30;20;2;"Счет регионального оператора";486,84407;72,59638;'
|
||
"1580,29712;0,00000;0,00000;-1093,45305;2026-05-31;01.10.2014;18,81;"
|
||
'"не присвоен";;;22.04.2014;;15.05.2026;8857551'
|
||
)
|
||
_KR11_ROW_4 = (
|
||
'"Свердловская область";65729000;"Арамильский городской округ Свердловской области";'
|
||
"2170025270;15064d04-3aec-4266-985b-da26e9a5f392;"
|
||
'"Арамильский городской округ Свердловской области, пос. Арамиль, ул. Ломоносова, д. 4";'
|
||
'1984;;906,40;;;;;;2;"Счет регионального оператора";1046,79431;204,75296;0,00000;'
|
||
'0,00000;0,00000;1046,79431;2026-05-31;01.10.2014;18,81;"не присвоен";;;22.04.2014;;'
|
||
"15.05.2026;8857552"
|
||
)
|
||
KR11_CSV = "\n".join([_KR11_HEADER, _KR11_ROW_3, _KR11_ROW_4]) + "\n"
|
||
|
||
# КР1.2 constructive elements — long-format: обычная строка (wall_material пуст) +
|
||
# строка "фасад" (wall_material='кирпич') на каждый из тех же 2 домов.
|
||
_KR12_HEADER = (
|
||
"subject_rf;mun_obr_oktmo;mun_obr;mkd_code;address;construction_element_code;"
|
||
"construction_element_type;system_type;lift_type;stops_count;commissioning_date;"
|
||
"decommissioning_date;roof_type;roofing_area;basement_area;facade_type;facade_area;"
|
||
"foundation_type;wall_material;comment;last_update"
|
||
)
|
||
_KR12_ROW_3_OTHER = (
|
||
'"Свердловская область";65729000;"Арамильский городской округ Свердловской области";'
|
||
"2170017185;"
|
||
'"Арамильский городской округ Свердловской области, пос. Арамиль, ул. Ломоносова, д. 3";'
|
||
'580000;"система дымоудаления и (или) противопожарной автоматики";;;;;;;;;;;;;;15.05.2026'
|
||
)
|
||
_KR12_ROW_3_FACADE = (
|
||
'"Свердловская область";65729000;"Арамильский городской округ Свердловской области";'
|
||
"2170017185;"
|
||
'"Арамильский городской округ Свердловской области, пос. Арамиль, ул. Ломоносова, д. 3";'
|
||
'60000;фасад;;;;;;;;;"облицованный камнем";480,00;;кирпич;;15.05.2026'
|
||
)
|
||
_KR12_ROW_4_OTHER = (
|
||
'"Свердловская область";65729000;"Арамильский городской округ Свердловской области";'
|
||
"2170025270;"
|
||
'"Арамильский городской округ Свердловской области, пос. Арамиль, ул. Ломоносова, д. 4";'
|
||
'100000;"инженерная система водоснабжения (ХВС)";центральная;;;;;;;;;;;;;15.05.2026'
|
||
)
|
||
_KR12_ROW_4_FACADE = (
|
||
'"Свердловская область";65729000;"Арамильский городской округ Свердловской области";'
|
||
"2170025270;"
|
||
'"Арамильский городской округ Свердловской области, пос. Арамиль, ул. Ломоносова, д. 4";'
|
||
"60000;фасад;;;;;;;;;;;;кирпич;;15.05.2026"
|
||
)
|
||
KR12_CSV = (
|
||
"\n".join(
|
||
[_KR12_HEADER, _KR12_ROW_3_OTHER, _KR12_ROW_3_FACADE, _KR12_ROW_4_OTHER, _KR12_ROW_4_FACADE]
|
||
)
|
||
+ "\n"
|
||
)
|
||
|
||
|
||
@pytest.fixture
|
||
def kr11_path(tmp_path: Path) -> Path:
|
||
p = tmp_path / "kr11.csv"
|
||
p.write_text(KR11_CSV, encoding="utf-8-sig")
|
||
return p
|
||
|
||
|
||
@pytest.fixture
|
||
def kr12_path(tmp_path: Path) -> Path:
|
||
p = tmp_path / "kr12.csv"
|
||
p.write_text(KR12_CSV, encoding="utf-8-sig")
|
||
return p
|
||
|
||
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# parse_decimal_comma / parse_int_field
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
def test_parse_decimal_comma() -> None:
|
||
assert dkl.parse_decimal_comma("909,80") == pytest.approx(909.80)
|
||
assert dkl.parse_decimal_comma("399,90") == pytest.approx(399.90)
|
||
assert dkl.parse_decimal_comma("100.5") == pytest.approx(100.5)
|
||
assert dkl.parse_decimal_comma("") is None
|
||
assert dkl.parse_decimal_comma(None) is None
|
||
assert dkl.parse_decimal_comma("не число") is None
|
||
|
||
|
||
def test_parse_int_field() -> None:
|
||
assert dkl.parse_int_field("1964") == 1964
|
||
assert dkl.parse_int_field("2") == 2
|
||
assert dkl.parse_int_field("") is None
|
||
assert dkl.parse_int_field(None) is None
|
||
assert dkl.parse_int_field("x") is None
|
||
assert dkl.parse_int_field("5,0") == 5 # decimal-фоллбек
|
||
|
||
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# parse_kr11_csv — house registry
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
def test_parse_kr11_csv_real_rows(kr11_path: Path) -> None:
|
||
rows = dkl.parse_kr11_csv(kr11_path)
|
||
assert set(rows.keys()) == {"2170017185", "2170025270"}
|
||
|
||
house3 = rows["2170017185"]
|
||
assert house3.houseguid == "04b98bcb-3232-4b26-b0fa-3c4a47adf394"
|
||
assert house3.commission_year == 1964
|
||
assert house3.number_floors_max == 2
|
||
assert house3.total_sq == pytest.approx(399.90)
|
||
assert "Ломоносова, д. 3" in (house3.address or "")
|
||
|
||
house4 = rows["2170025270"]
|
||
assert house4.houseguid == "15064d04-3aec-4266-985b-da26e9a5f392"
|
||
assert house4.commission_year == 1984
|
||
assert house4.total_sq == pytest.approx(906.40)
|
||
|
||
|
||
def test_parse_kr11_csv_skips_blank_mkd_code(tmp_path: Path) -> None:
|
||
csv_text = _KR11_HEADER + "\n" + (";" * (_KR11_HEADER.count(";"))) + "\n"
|
||
p = tmp_path / "blank.csv"
|
||
p.write_text(csv_text, encoding="utf-8-sig")
|
||
assert dkl.parse_kr11_csv(p) == {}
|
||
|
||
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# parse_kr12_wall_materials — long-format, wall_material только на строке 'фасад'
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
def test_parse_kr12_wall_materials_extracts_facade_row_only(kr12_path: Path) -> None:
|
||
materials = dkl.parse_kr12_wall_materials(kr12_path)
|
||
# Обе non-facade строки (дымоудаление / ХВС) имели wall_material='' — исключены.
|
||
assert materials == {"2170017185": "кирпич", "2170025270": "кирпич"}
|
||
|
||
|
||
def test_parse_kr12_wall_materials_empty_when_no_facade_row(tmp_path: Path) -> None:
|
||
csv_text = "\n".join([_KR12_HEADER, _KR12_ROW_3_OTHER]) + "\n"
|
||
p = tmp_path / "no_facade.csv"
|
||
p.write_text(csv_text, encoding="utf-8-sig")
|
||
assert dkl.parse_kr12_wall_materials(p) == {}
|
||
|
||
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# build_domrf_houses — join КР1.1 + КР1.2(wall_material)
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
def test_build_domrf_houses_joins_wall_material(kr11_path: Path, kr12_path: Path) -> None:
|
||
kr11 = dkl.parse_kr11_csv(kr11_path)
|
||
walls = dkl.parse_kr12_wall_materials(kr12_path)
|
||
houses = dkl.build_domrf_houses(kr11, walls)
|
||
|
||
by_code = {h.mkd_code: h for h in houses}
|
||
assert len(houses) == 2
|
||
assert by_code["2170017185"].wall_material == "кирпич"
|
||
assert by_code["2170017185"].commission_year == 1964
|
||
assert by_code["2170025270"].wall_material == "кирпич"
|
||
|
||
|
||
# Дом без строки 'фасад' в КР1.2 → wall_material остаётся None (не KeyError/пустая строка).
|
||
def test_wall_material_none_when_house_missing_from_kr12(tmp_path: Path, kr11_path: Path) -> None:
|
||
kr11 = dkl.parse_kr11_csv(kr11_path)
|
||
houses = dkl.build_domrf_houses(kr11, {}) # пустой КР1.2 lookup
|
||
assert all(h.wall_material is None for h in houses)
|
||
|
||
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# fetch_domrf_csvs — скачивание+распаковка zip, HTTP замокан (MockTransport)
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
def _zip_bytes(inner_name: str, content: str) -> bytes:
|
||
buf = io.BytesIO()
|
||
with zipfile.ZipFile(buf, "w") as zf:
|
||
zf.writestr(inner_name, content.encode("utf-8-sig"))
|
||
return buf.getvalue()
|
||
|
||
|
||
def test_fetch_domrf_csvs_downloads_and_extracts(tmp_path: Path) -> None:
|
||
kr11_zip = _zip_bytes("export-kr1_1-66-20260701.csv", KR11_CSV)
|
||
kr12_zip = _zip_bytes("export-kr1_2-66-20260701.csv", KR12_CSV)
|
||
|
||
def handler(request: httpx.Request) -> httpx.Response:
|
||
url = str(request.url)
|
||
if url == dkl.KR11_URL:
|
||
return httpx.Response(200, content=kr11_zip)
|
||
if url == dkl.KR12_URL:
|
||
return httpx.Response(200, content=kr12_zip)
|
||
raise AssertionError(f"unexpected URL {url}")
|
||
|
||
transport = httpx.MockTransport(handler)
|
||
with httpx.Client(transport=transport) as client:
|
||
kr11_out, kr12_out = dkl.fetch_domrf_csvs(tmp_path, client=client)
|
||
|
||
assert kr11_out.exists() and kr11_out.name.endswith(".csv")
|
||
assert kr12_out.exists() and kr12_out.name.endswith(".csv")
|
||
|
||
# Круглый путь: распакованные файлы парсятся так же, как локальные фикстуры.
|
||
rows = dkl.parse_kr11_csv(kr11_out)
|
||
assert set(rows.keys()) == {"2170017185", "2170025270"}
|
||
materials = dkl.parse_kr12_wall_materials(kr12_out)
|
||
assert materials == {"2170017185": "кирпич", "2170025270": "кирпич"}
|
||
|
||
|
||
def test_extract_csv_from_zip_raises_on_no_csv_entry(tmp_path: Path) -> None:
|
||
buf = io.BytesIO()
|
||
with zipfile.ZipFile(buf, "w") as zf:
|
||
zf.writestr("readme.txt", "not a csv")
|
||
with pytest.raises(ValueError, match="csv"):
|
||
dkl._extract_csv_from_zip(buf.getvalue(), tmp_path)
|
||
|
||
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# Статические asserts по _UPSERT_SQL (psycopg v3, ON CONFLICT, идемпотентность)
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
_UPSERT_SQL = str(dkl._UPSERT_SQL.text)
|
||
|
||
|
||
def test_upsert_sql_uses_psycopg_v3_cast_not_double_colon() -> None:
|
||
assert not re.search(r":\w+::", _UPSERT_SQL)
|
||
assert "CAST(:mkd_code AS text)" in _UPSERT_SQL
|
||
assert "CAST(:commission_year AS smallint)" in _UPSERT_SQL
|
||
assert "CAST(:total_sq AS numeric)" in _UPSERT_SQL
|
||
|
||
|
||
def test_upsert_sql_idempotent_on_conflict() -> None:
|
||
flat = re.sub(r"\s+", " ", _UPSERT_SQL)
|
||
assert "INSERT INTO domrf_kapremont" in flat
|
||
assert "ON CONFLICT (mkd_code) DO UPDATE SET" in flat
|
||
assert "domrf_kapremont.houseguid IS DISTINCT FROM EXCLUDED.houseguid" in flat
|
||
assert "domrf_kapremont.wall_material IS DISTINCT FROM EXCLUDED.wall_material" in flat
|
||
|
||
|
||
def test_upsert_uses_savepoint_per_chunk() -> None:
|
||
import inspect
|
||
|
||
src = inspect.getsource(dkl.upsert_domrf_kapremont)
|
||
assert "with db.begin_nested():" in src
|
||
assert "db.commit()" not in src # не коммитит сам (коммитит caller)
|
||
|
||
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# Статические asserts: backfill houses — COALESCE(year_built, domrf, zhkh_year)
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
_BACKFILL_DOMRF_SQL = re.sub(r"\s+", " ", str(dkl._BACKFILL_HOUSES_FROM_DOMRF_SQL.text))
|
||
_BACKFILL_ZHKH_SQL = re.sub(r"\s+", " ", str(dkl._BACKFILL_HOUSES_ZHKH_FALLBACK_SQL.text))
|
||
|
||
|
||
def test_backfill_domrf_sql_coalesce_order_matches_spec() -> None:
|
||
# COALESCE(year_built, domrf.commission_year, zhkh_year) — ровно в этом порядке (#2013).
|
||
assert "COALESCE(h.year_built, d.commission_year, h.zhkh_year)" in _BACKFILL_DOMRF_SQL
|
||
assert "COALESCE(h.material_walls, d.wall_material)" in _BACKFILL_DOMRF_SQL
|
||
assert "COALESCE(h.total_floors, d.number_floors_max, h.zhkh_floors)" in _BACKFILL_DOMRF_SQL
|
||
|
||
|
||
def test_backfill_domrf_sql_matches_by_guid_coalesce_priority() -> None:
|
||
# gar_house_guid первый (78% coverage), затем house_fias_id, затем zhkh_house_guid.
|
||
assert (
|
||
"d.houseguid = COALESCE(h.gar_house_guid, h.house_fias_id, h.zhkh_house_guid)"
|
||
in _BACKFILL_DOMRF_SQL
|
||
)
|
||
|
||
|
||
def test_backfill_domrf_sql_only_fills_null_never_overwrites() -> None:
|
||
# Гейт: каждый SET-таргет требует h.<col> IS NULL — COALESCE никогда не перезапишет
|
||
# существующее non-null значение (SQL COALESCE тоже это гарантирует, но гейт защищает
|
||
# от no-op UPDATE на уже заполненных строках).
|
||
assert "h.year_built IS NULL" in _BACKFILL_DOMRF_SQL
|
||
assert "h.material_walls IS NULL" in _BACKFILL_DOMRF_SQL
|
||
assert "h.total_floors IS NULL" in _BACKFILL_DOMRF_SQL
|
||
|
||
|
||
def test_backfill_zhkh_fallback_sql_folds_zhkh_for_unmatched_houses() -> None:
|
||
# Шаг 2 — без JOIN на domrf_kapremont вообще: применяется к ЛЮБОМУ дому, включая
|
||
# те, что не заматчились в шаге 1 (issue: "houses without a ДОМ.РФ match still benefit").
|
||
assert "domrf_kapremont" not in _BACKFILL_ZHKH_SQL
|
||
assert "COALESCE(year_built, zhkh_year)" in _BACKFILL_ZHKH_SQL
|
||
assert "COALESCE(total_floors, zhkh_floors)" in _BACKFILL_ZHKH_SQL
|
||
assert "year_built IS NULL AND zhkh_year IS NOT NULL" in _BACKFILL_ZHKH_SQL
|
||
|
||
|
||
def test_backfill_sql_no_psycopg_v3_colon_colon_cast() -> None:
|
||
assert not re.search(r":\w+::", _BACKFILL_DOMRF_SQL)
|
||
assert not re.search(r":\w+::", _BACKFILL_ZHKH_SQL)
|
||
|
||
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# backfill_houses_from_domrf — dry_run vs real-run (MagicMock db)
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
def test_backfill_houses_real_run_sums_both_steps_rowcount() -> None:
|
||
db = MagicMock()
|
||
results = [MagicMock(rowcount=3), MagicMock(rowcount=2)]
|
||
db.execute.side_effect = results
|
||
|
||
out = dkl.backfill_houses_from_domrf(db, dry_run=False)
|
||
|
||
assert out == {"domrf_matched": 3, "zhkh_fallback": 2, "houses_updated": 5}
|
||
assert db.execute.call_count == 2
|
||
executed_sql = [str(call.args[0]) for call in db.execute.call_args_list]
|
||
assert "UPDATE houses" in executed_sql[0]
|
||
assert "UPDATE houses" in executed_sql[1]
|
||
|
||
|
||
def test_backfill_houses_dry_run_only_counts_no_update() -> None:
|
||
db = MagicMock()
|
||
db.execute.return_value.scalar_one.side_effect = [7, 4]
|
||
|
||
out = dkl.backfill_houses_from_domrf(db, dry_run=True)
|
||
|
||
assert out == {"domrf_matched": 7, "zhkh_fallback": 4, "houses_updated": 0}
|
||
for call in db.execute.call_args_list:
|
||
sql_text = str(call.args[0]).upper()
|
||
assert "UPDATE" not in sql_text
|
||
assert "SELECT COUNT(*)" in sql_text
|
||
|
||
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# Статические asserts + behavior: propagate_listings_year_from_houses
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
_PROPAGATE_SQL = re.sub(r"\s+", " ", str(dkl._PROPAGATE_LISTINGS_YEAR_SQL.text))
|
||
|
||
|
||
def test_propagate_sql_only_fills_null_listings_year() -> None:
|
||
assert "SET year_built = h.year_built" in _PROPAGATE_SQL
|
||
assert "l.year_built IS NULL" in _PROPAGATE_SQL
|
||
assert "h.year_built IS NOT NULL" in _PROPAGATE_SQL
|
||
assert "l.house_id_fk = h.id" in _PROPAGATE_SQL
|
||
|
||
|
||
def test_propagate_listings_year_real_run_returns_rowcount() -> None:
|
||
db = MagicMock()
|
||
db.execute.return_value.rowcount = 42
|
||
|
||
out = dkl.propagate_listings_year_from_houses(db, dry_run=False)
|
||
|
||
assert out == {"listings_updated": 42}
|
||
sql_text = str(db.execute.call_args[0][0])
|
||
assert "UPDATE listings" in sql_text
|
||
|
||
|
||
def test_propagate_listings_year_dry_run_uses_count_sql() -> None:
|
||
db = MagicMock()
|
||
db.execute.return_value.scalar_one.return_value = 99
|
||
|
||
out = dkl.propagate_listings_year_from_houses(db, dry_run=True)
|
||
|
||
assert out == {"listings_updated": 0, "would_update": 99}
|
||
sql_text = str(db.execute.call_args[0][0]).upper()
|
||
assert "UPDATE" not in sql_text
|