lekss361
50f0674977
All checks were successful
Deploy Trade-In / changes (push) Successful in 11s
Deploy Trade-In / build-frontend (push) Has been skipped
Deploy Trade-In / build-browser (push) Has been skipped
Deploy Trade-In / test (push) Successful in 4m9s
Deploy Trade-In / build-backend (push) Successful in 1m54s
Deploy Trade-In / deploy (push) Successful in 1m51s
Deploy Trade-In / deploy-status (push) Successful in 1s
Deploy Trade-In / perimeter-smoke (push) Successful in 11s
272 lines
14 KiB
Python
272 lines
14 KiB
Python
"""Тесты dtp-stat.ru loader'а (app/services/dtp_stat_loader.py, мигр. 294, #3410).
|
||
|
||
Coverage:
|
||
- Чистые парс-хелперы: _parse_datetime / _parse_str_list / _parse_int.
|
||
- _parse_feature на РЕАЛЬНОЙ первой записи sverdlovskaia-oblast.geojson (id=74, "ул
|
||
Победы, 31") — обязательный тест, что vehicles/participants (ПДн физлиц) НЕ
|
||
попадают в результат парсинга ни в каком виде.
|
||
- iter_incidents — потоковый парс (ijson) мини-FeatureCollection, пропуск невалидных
|
||
фич (нет geometry.Point, нет id).
|
||
- download_dtp_zip / load_dtp_incidents(src_path=...) — HTTP замокан httpx.MockTransport,
|
||
zip без .geojson entry -> ValueError.
|
||
- load_dtp_incidents dry_run: TRUNCATE/INSERT не вызываются, счётчик parsed верный.
|
||
- load_dtp_incidents real-run: TRUNCATE вызван, INSERT чанками, rowcount суммируется
|
||
(MagicMock db).
|
||
- Статические asserts по SQL: _INSERT_SQL (ON CONFLICT DO NOTHING, CAST(...), geom
|
||
строится явно ST_SetSRID/ST_MakePoint, нет bare :x:: кастов).
|
||
- Гейт: `load_dtp_incidents` не коммитит (коммитит caller — app/tasks/dtp_stat_refresh.py).
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import inspect
|
||
import io
|
||
import json
|
||
import os
|
||
import re
|
||
import zipfile
|
||
from pathlib import Path
|
||
from unittest.mock import MagicMock
|
||
|
||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
|
||
|
||
import httpx
|
||
import pytest
|
||
|
||
from app.services import dtp_stat_loader as dsl
|
||
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# Фикстура: РЕАЛЬНАЯ первая запись sverdlovskaia-oblast.geojson (проверено 08.09.2026,
|
||
# id=74, "г Екатеринбург, ул Победы, 31") — включает vehicles/participants (ПДн) НАРОЧНО,
|
||
# чтобы доказать, что парсер их не трогает.
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
_FEATURE_74 = json.loads(
|
||
'{"type": "Feature", "geometry": {"type": "Point", "coordinates": [60.603333, 56.896944]}, '
|
||
'"properties": {"id": 74, "tags": ["Дорожно-транспортные происшествия"], '
|
||
'"light": "В темное время суток, освещение включено", '
|
||
'"point": {"lat": 56.896944, "long": 60.603333}, '
|
||
'"nearby": ["Отсутствие в непосредственной близости объектов УДС и объектов притяжения", '
|
||
'"Перегон (нет объектов на месте ДТП)"], "region": "Свердловская область", '
|
||
'"scheme": "200", "address": "г Екатеринбург, ул Победы, 31", "weather": ["Снегопад"], '
|
||
'"category": "Столкновение", "datetime": "2015-01-01 03:00:00", "severity": "Тяжёлый", '
|
||
'"vehicles": [{"year": 2006, "brand": "ВАЗ", "color": "Иные цвета", '
|
||
'"model": "Нива ВАЗ-2121 и модификации", "category": "D-класс (средний) до 4,6 м", '
|
||
'"participants": [{"id": 1649862, "role": "Водитель", "gender": "Мужской", '
|
||
'"violations": ["Выезд на полосу встречного движения"], '
|
||
'"health_status": "Раненый, находящийся (находившийся) на стационарном лечении", '
|
||
'"years_of_driving_experience": 6}]}, {"year": 2013, "brand": "DAEWOO", "color": "Белый", '
|
||
'"model": "Pathfinder", "category": "D-класс (средний) до 4,6 м", "participants": '
|
||
'[{"id": 1649863, "role": "Водитель", "gender": "Мужской", "violations": ["Нет нарушений"], '
|
||
'"health_status": "Не пострадал", "years_of_driving_experience": 15}, '
|
||
'{"id": 1649864, "role": "Пассажир", "gender": "Женский", "violations": ["Нет нарушений"], '
|
||
'"health_status": "Раненый, находящийся (находившийся) на стационарном лечении", '
|
||
'"years_of_driving_experience": null}, {"id": 1649865, "role": "Пассажир", '
|
||
'"gender": "Мужской", "violations": ["Нет нарушений"], '
|
||
'"health_status": "Раненый, находящийся (находившийся) на стационарном лечении", '
|
||
'"years_of_driving_experience": null}]}], "dead_count": 0, "gibdd_number": "157467063", '
|
||
'"participants": [], "injured_count": 3, "parent_region": null, '
|
||
'"road_conditions": ["Не установлены"], "participants_count": 4, '
|
||
'"participant_categories": ["Все участники"]}}'
|
||
)
|
||
|
||
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# Чистые парс-хелперы
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
def test_parse_datetime_happy_and_garbage() -> None:
|
||
dt = dsl._parse_datetime("2015-01-01 03:00:00")
|
||
assert dt is not None
|
||
assert (dt.year, dt.month, dt.day, dt.hour) == (2015, 1, 1, 3)
|
||
assert dsl._parse_datetime(None) is None
|
||
assert dsl._parse_datetime("") is None
|
||
assert dsl._parse_datetime("не дата") is None
|
||
|
||
|
||
def test_parse_str_list() -> None:
|
||
assert dsl._parse_str_list(["a", "b"]) == ["a", "b"]
|
||
assert dsl._parse_str_list([]) is None
|
||
assert dsl._parse_str_list(None) is None
|
||
assert dsl._parse_str_list("не список") is None
|
||
|
||
|
||
def test_parse_int() -> None:
|
||
assert dsl._parse_int(3) == 3
|
||
assert dsl._parse_int(0) == 0
|
||
assert dsl._parse_int(None) is None
|
||
assert dsl._parse_int("3") is None
|
||
assert dsl._parse_int(True) is None
|
||
|
||
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# _parse_feature — ПДн-гейт
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
def test_parse_feature_real_record_no_pii() -> None:
|
||
row = dsl._parse_feature(_FEATURE_74)
|
||
assert row is not None
|
||
assert row.source_id == "74"
|
||
assert row.address == "г Екатеринбург, ул Победы, 31"
|
||
assert row.category == "Столкновение"
|
||
assert row.severity == "Тяжёлый"
|
||
assert row.dead == 0
|
||
assert row.injured == 3
|
||
assert row.weather == ["Снегопад"]
|
||
assert len(row.nearby) == 2
|
||
assert row.lat == pytest.approx(56.896944)
|
||
assert row.lon == pytest.approx(60.603333)
|
||
assert (row.dtp_at.year, row.dtp_at.month, row.dtp_at.day) == (2015, 1, 1)
|
||
|
||
# ПДн-гейт: dataclass не несёт ни vehicles, ни participants ни в каком поле.
|
||
field_names = {f for f in dsl.DtpIncidentRow.__dataclass_fields__}
|
||
assert "vehicles" not in field_names
|
||
assert "participants" not in field_names
|
||
row_values = repr(row)
|
||
assert "Водитель" not in row_values
|
||
assert "Мужской" not in row_values
|
||
assert "1649862" not in row_values
|
||
|
||
|
||
def test_parse_feature_skips_non_point_geometry() -> None:
|
||
feature = {"geometry": {"type": "Polygon", "coordinates": []}, "properties": {"id": 1}}
|
||
assert dsl._parse_feature(feature) is None
|
||
|
||
|
||
def test_parse_feature_skips_missing_id() -> None:
|
||
feature = {
|
||
"geometry": {"type": "Point", "coordinates": [60.0, 56.0]},
|
||
"properties": {},
|
||
}
|
||
assert dsl._parse_feature(feature) is None
|
||
|
||
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# iter_incidents — потоковый парс
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
def _feature_collection_bytes(features: list[dict]) -> bytes:
|
||
return json.dumps({"type": "FeatureCollection", "features": features}).encode("utf-8")
|
||
|
||
|
||
def test_iter_incidents_streams_and_skips_invalid() -> None:
|
||
garbage = {
|
||
"type": "Feature",
|
||
"geometry": {"type": "Point", "coordinates": []},
|
||
"properties": {},
|
||
}
|
||
data = _feature_collection_bytes([_FEATURE_74, garbage])
|
||
rows = list(dsl.iter_incidents(io.BytesIO(data)))
|
||
assert len(rows) == 1
|
||
assert rows[0].source_id == "74"
|
||
|
||
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# download_dtp_zip / load_dtp_incidents — HTTP замокан
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
def _zip_bytes(inner_name: str, payload: bytes) -> bytes:
|
||
buf = io.BytesIO()
|
||
with zipfile.ZipFile(buf, "w") as zf:
|
||
zf.writestr(inner_name, payload)
|
||
return buf.getvalue()
|
||
|
||
|
||
def test_load_dtp_incidents_dry_run_downloads_and_counts() -> None:
|
||
data = _feature_collection_bytes([_FEATURE_74])
|
||
zip_bytes = _zip_bytes("sverdlovskaia-oblast.geojson", data)
|
||
|
||
def handler(request: httpx.Request) -> httpx.Response:
|
||
assert str(request.url) == dsl.SRC_URL
|
||
return httpx.Response(200, content=zip_bytes)
|
||
|
||
client = httpx.Client(transport=httpx.MockTransport(handler))
|
||
db = MagicMock()
|
||
|
||
counts = dsl.load_dtp_incidents(db, client=client, dry_run=True)
|
||
|
||
assert counts == {"parsed": 1, "inserted": 0}
|
||
db.execute.assert_not_called()
|
||
|
||
|
||
def test_load_dtp_incidents_real_run_truncates_and_inserts(tmp_path: Path) -> None:
|
||
data = _feature_collection_bytes([_FEATURE_74])
|
||
zip_path = tmp_path / "local.zip"
|
||
zip_path.write_bytes(_zip_bytes("sverdlovskaia-oblast.geojson", data))
|
||
|
||
db = MagicMock()
|
||
db.execute.return_value.rowcount = 1
|
||
|
||
counts = dsl.load_dtp_incidents(db, src_path=zip_path, dry_run=False)
|
||
|
||
assert counts == {"parsed": 1, "inserted": 1}
|
||
calls = db.execute.call_args_list
|
||
assert calls[0].args[0] is dsl._TRUNCATE_SQL
|
||
assert calls[1].args[0] is dsl._INSERT_SQL
|
||
inserted_rows = calls[1].args[1]
|
||
assert len(inserted_rows) == 1
|
||
assert inserted_rows[0]["source_id"] == "74"
|
||
assert "vehicles" not in inserted_rows[0]
|
||
assert "participants" not in inserted_rows[0]
|
||
|
||
|
||
def test_zip_without_geojson_raises() -> None:
|
||
buf = io.BytesIO()
|
||
with zipfile.ZipFile(buf, "w") as zf:
|
||
zf.writestr("readme.txt", "not geojson")
|
||
with zipfile.ZipFile(buf) as zf, pytest.raises(ValueError, match="geojson"):
|
||
dsl._open_geojson_in_zip(zf)
|
||
|
||
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# Статические asserts по SQL + гейт "сервис не коммитит"
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
def test_insert_sql_static_shape() -> None:
|
||
sql = re.sub(r"\s+", " ", str(dsl._INSERT_SQL.text))
|
||
assert "ON CONFLICT (source_id) DO NOTHING" in sql
|
||
assert "CAST(:source_id AS text)" in sql
|
||
assert "CAST(:weather AS text[])" in sql
|
||
assert "CAST(:nearby AS text[])" in sql
|
||
assert "ST_SetSRID(ST_MakePoint(" in sql
|
||
# psycopg v3 trap: bind-параметр не должен иметь :name::type сразу за собой.
|
||
assert not re.search(r":[a-zA-Z_]+::[a-zA-Z]", sql)
|
||
|
||
|
||
def test_truncate_sql_static_shape() -> None:
|
||
assert str(dsl._TRUNCATE_SQL.text).strip() == "TRUNCATE dtp_incidents"
|
||
|
||
|
||
def test_load_dtp_incidents_does_not_commit() -> None:
|
||
src = inspect.getsource(dsl.load_dtp_incidents)
|
||
assert "db.commit()" not in src
|
||
|
||
|
||
def _feature(lon: float, lat: float, source_id: int = 1) -> dict:
|
||
return {
|
||
"type": "Feature",
|
||
"geometry": {"type": "Point", "coordinates": [lon, lat]},
|
||
"properties": {"id": source_id, "datetime": "2020-05-05 12:00:00"},
|
||
}
|
||
|
||
|
||
@pytest.mark.parametrize(
|
||
("lon", "lat"),
|
||
[
|
||
(0.0, 0.0), # null island — 10 таких в реальном дампе
|
||
(1.0, 1.0),
|
||
(47.0, 41.0),
|
||
(55.3, 22.183),
|
||
(59.852, 5.8),
|
||
],
|
||
)
|
||
def test_parse_feature_drops_broken_coordinates(lon: float, lat: float) -> None:
|
||
"""Битые координаты источника не должны попадать в geo-слой.
|
||
|
||
Реальный дамп по Свердловской области (2026-02-26): из 31 253 записей 86 имеют
|
||
координаты вне региона, в т.ч. 10 ровно в (0,0). Слой кормит радиусные агрегаты
|
||
локации, поэтому мусор режем на парсе.
|
||
"""
|
||
assert dsl._parse_feature(_feature(lon, lat)) is None
|
||
|
||
|
||
@pytest.mark.parametrize(("lon", "lat"), [(60.6055, 56.8386), (56.94, 63.283)])
|
||
def test_parse_feature_keeps_region_points_including_northern_tip(lon: float, lat: float) -> None:
|
||
"""Рамка шире официальных границ — север области (~63°) не срезается."""
|
||
row = dsl._parse_feature(_feature(lon, lat))
|
||
assert row is not None
|
||
assert row.lat == lat and row.lon == lon
|