fix(site-finder): true-counts вместо молча-усечённых len(list) (#2445-A1/A2)
All checks were successful
CI Trade-In / changes (pull_request) Successful in 7s
CI / changes (pull_request) Successful in 7s
CI Trade-In / backend-tests (pull_request) Has been skipped
CI Trade-In / frontend-checks (pull_request) Has been skipped
CI / frontend-tests (pull_request) Successful in 1m7s
CI / openapi-codegen-check (pull_request) Successful in 2m0s
CI / backend-tests (pull_request) Successful in 14m52s

quarter_dump_lookup (connection-points) и utility_infrastructure_loader
капали список через SQL LIMIT, но выдавали len(list) как полный итог —
молча скрывая structures/zones/features сверх капа. Паттерн permits_nearby.py:
отдельный COUNT(*) по полной выборке, LIMIT остаётся на display-списке,
*_truncated булевы флаги. Плюс добавлен детерминированный ORDER BY перед
zouit_engineering LIMIT 100 (раньше был неупорядоченный, недетерминированные 100).

Schema: +protection_zones_truncated/structures_truncated (ConnectionPointsSummary),
+features_truncated (UtilityInfrastructureSummary), аддитивно default False.
api-types.ts регенерирован (только 3 новых поля).

Refs #2445
This commit is contained in:
bot-backend 2026-07-05 23:38:19 +05:00
parent 5b3799a23a
commit c2306c78c4
6 changed files with 337 additions and 59 deletions

View file

@ -65,6 +65,11 @@ class ConnectionPointsSummary(BaseModel):
in_protection_zone: bool
protection_zones_intersecting: int
total_structures_in_radius: int
# Epic #2445 A1: честные флаги усечения — True, когда истинный count (COUNT(*) без
# LIMIT, по той же выборке) больше длины отдаваемого списка (LIMIT 100/50 в SQL).
# ADDITIVE (default False) — старые клиенты не ломаются.
protection_zones_truncated: bool = False
structures_truncated: bool = False
class ConnectionPointsResponse(BaseModel):
@ -100,6 +105,10 @@ class UtilityInfrastructureSummary(BaseModel):
nearest_distance_m: float | None
# Карта вид сети → расстояние до ближайшего объекта данного вида (м), либо null.
nearest_by_kind: dict[str, float | None]
# Epic #2445 A2: True, когда истинный count (COUNT(*) без LIMIT, по той же
# ST_DWithin-выборке) больше длины отдаваемого `features` (LIMIT :lim, default 200).
# ADDITIVE (default False) — старые клиенты не ломаются.
features_truncated: bool = False
class UtilityInfrastructureResponse(BaseModel):

View file

@ -1138,6 +1138,8 @@ def get_connection_points(db: Session, cad_num: str, radius_m: int = 500) -> dic
"in_protection_zone": False,
"protection_zones_intersecting": 0,
"total_structures_in_radius": 0,
"protection_zones_truncated": False,
"structures_truncated": False,
},
"dump_available": False,
"dump_fetched_at": fetched_at_iso,
@ -1166,20 +1168,25 @@ def get_connection_points(db: Session, cad_num: str, radius_m: int = 500) -> dic
_trigger_harvest(quarter)
return _empty_unavailable(dump_fetched_at)
structures = _get_engineering_structures_by_boundary(db, quarter, parcel_wkt, radius_m)
zouit_overlaps = _get_zouit_engineering_overlaps(db, quarter, parcel_wkt)
structures, structures_total = _get_engineering_structures_by_boundary(
db, quarter, parcel_wkt, radius_m
)
zouit_overlaps, zouit_total = _get_zouit_engineering_overlaps(db, quarter, parcel_wkt)
nearest_dist: float | None = structures[0]["distance_to_boundary_m"] if structures else None
protection_count = len(zouit_overlaps)
return {
"engineering_structures": structures,
"zouit_engineering_overlaps": zouit_overlaps,
"summary": {
"nearest_structure_distance_m": nearest_dist,
"in_protection_zone": protection_count > 0,
"protection_zones_intersecting": protection_count,
"total_structures_in_radius": len(structures),
"in_protection_zone": zouit_total > 0,
# Epic #2445 A1: истинные счётчики по ПОЛНОЙ выборке (COUNT(*) без LIMIT),
# НЕ len(list) — раньше тихо капались на 100/50 (LIMIT в SQL row-fetch).
"protection_zones_intersecting": zouit_total,
"total_structures_in_radius": structures_total,
"protection_zones_truncated": zouit_total > len(zouit_overlaps),
"structures_truncated": structures_total > len(structures),
},
"dump_available": True,
"dump_fetched_at": dump_fetched_at,
@ -1227,18 +1234,59 @@ def _get_parcel_wkt(db: Session, cad_num: str) -> str | None:
return None
_ENGINEERING_STRUCTURES_ITEMS_LIMIT = 50
# Условие радиус-выборки engineering_structures — используется И в row-fetch (с ORDER BY +
# LIMIT), И в честном COUNT(*) БЕЗ LIMIT (epic #2445 A1: список капается на 50, но
# summary.total_structures_in_radius обязан считаться по ВСЕЙ выборке, не по len(list)).
_ENGINEERING_STRUCTURES_WHERE = """
AND feat.value->>'layer' = 'engineering_structures'
AND (feat.value->'geometry') IS NOT NULL
AND feat.value->>'geometry' != 'null'
AND ST_DWithin(
ST_Transform(
ST_SetSRID(
ST_GeomFromGeoJSON(feat.value->>'geometry'),
3857
),
4326
)::geography,
ST_GeomFromText(:wkt, 4326)::geography,
:radius_m
)
"""
def _get_engineering_structures_by_boundary(
db: Session,
quarter: str,
parcel_wkt: str,
radius_m: int,
) -> list[dict[str, Any]]:
) -> tuple[list[dict[str, Any]], int]:
"""Engineering structures из dump в radius_m от boundary участка.
Использует ST_Distance к boundary (не centroid) для корректного расстояния.
Geometry трансформируется из EPSG:3857 (хранение dump) 4326.
Returns:
(items, total_count) items ограничен _ENGINEERING_STRUCTURES_ITEMS_LIMIT
(ближайшие), total_count ЧЕСТНЫЙ COUNT(*) по всей радиус-выборке (epic #2445
A1: раньше summary.total_structures_in_radius = len(items) тихо капался на 50).
"""
params = {"q": quarter, "wkt": parcel_wkt, "radius_m": radius_m}
try:
total_count = (
db.execute(
text(
"SELECT COUNT(*)"
" FROM nspd_quarter_dumps d,"
" jsonb_array_elements(d.features_json) AS feat(value)"
" WHERE d.quarter_cad = :q" + _ENGINEERING_STRUCTURES_WHERE
),
params,
).scalar()
or 0
)
rows = db.execute(
text(
"""
@ -1256,26 +1304,14 @@ def _get_engineering_structures_by_boundary(
) AS distance_m
FROM nspd_quarter_dumps d,
jsonb_array_elements(d.features_json) AS feat(value)
WHERE d.quarter_cad = :q
AND feat.value->>'layer' = 'engineering_structures'
AND (feat.value->'geometry') IS NOT NULL
AND feat.value->>'geometry' != 'null'
AND ST_DWithin(
ST_Transform(
ST_SetSRID(
ST_GeomFromGeoJSON(feat.value->>'geometry'),
3857
),
4326
)::geography,
ST_GeomFromText(:wkt, 4326)::geography,
:radius_m
)
WHERE d.quarter_cad = :q"""
+ _ENGINEERING_STRUCTURES_WHERE
+ """
ORDER BY distance_m ASC
LIMIT 50
LIMIT :items_limit
"""
),
{"q": quarter, "wkt": parcel_wkt, "radius_m": radius_m},
{**params, "items_limit": _ENGINEERING_STRUCTURES_ITEMS_LIMIT},
).fetchall()
except (OperationalError, ProgrammingError, DataError) as e:
# DataError — malformed WKT в ST_GeomFromText / ST_GeomFromGeoJSON (PostGIS ERROR).
@ -1284,7 +1320,7 @@ def _get_engineering_structures_by_boundary(
quarter,
e,
)
return []
return [], 0
result: list[dict[str, Any]] = []
for r in rows:
@ -1349,7 +1385,7 @@ def _get_engineering_structures_by_boundary(
"source": "nspd_36328",
}
)
return result
return result, int(total_count)
def _clean_struct_str(value: Any) -> str | None:
@ -1390,24 +1426,12 @@ def _build_structure_characteristics(opts: dict[str, Any]) -> str | None:
return " · ".join(parts) if parts else None
def _get_zouit_engineering_overlaps(
db: Session,
quarter: str,
parcel_wkt: str,
) -> list[dict[str, Any]]:
"""ZOUIT engineering (cat 37578) — охранные зоны, пересекающие участок.
_ZOUIT_ENGINEERING_ITEMS_LIMIT = 100
Использует слой 'zouit_engineering' из dump.
"""
try:
rows = db.execute(
text(
"""
SELECT feat.value->'properties' AS props,
feat.value->>'geometry' AS geom_json
FROM nspd_quarter_dumps d,
jsonb_array_elements(d.features_json) AS feat(value)
WHERE d.quarter_cad = :q
# Условие пересечения zouit_engineering с участком — используется И в row-fetch (с
# ORDER BY + LIMIT), И в честном COUNT(*) БЕЗ LIMIT (epic #2445 A1: список капается на
# 100, но summary.protection_zones_intersecting обязан считаться по ВСЕЙ выборке).
_ZOUIT_ENGINEERING_WHERE = """
AND feat.value->>'layer' = 'zouit_engineering'
AND (feat.value->'geometry') IS NOT NULL
AND feat.value->>'geometry' != 'null'
@ -1421,10 +1445,55 @@ def _get_zouit_engineering_overlaps(
),
ST_GeomFromText(:wkt, 4326)
)
LIMIT 100
"""
def _get_zouit_engineering_overlaps(
db: Session,
quarter: str,
parcel_wkt: str,
) -> tuple[list[dict[str, Any]], int]:
"""ZOUIT engineering (cat 37578) — охранные зоны, пересекающие участок.
Использует слой 'zouit_engineering' из dump.
Returns:
(items, total_count) items ограничен _ZOUIT_ENGINEERING_ITEMS_LIMIT,
детерминированно отсортирован (epic #2445 A1: раньше не было ORDER BY перед
LIMIT 100 недетерминированные 100 строк на каждый запрос). total_count
ЧЕСТНЫЙ COUNT(*) по всей intersects-выборке (раньше summary.
protection_zones_intersecting = len(items) тихо капался на 100).
"""
params = {"q": quarter, "wkt": parcel_wkt}
try:
total_count = (
db.execute(
text(
"SELECT COUNT(*)"
" FROM nspd_quarter_dumps d,"
" jsonb_array_elements(d.features_json) AS feat(value)"
" WHERE d.quarter_cad = :q" + _ZOUIT_ENGINEERING_WHERE
),
params,
).scalar()
or 0
)
rows = db.execute(
text(
"""
SELECT feat.value->'properties' AS props,
feat.value->>'geometry' AS geom_json
FROM nspd_quarter_dumps d,
jsonb_array_elements(d.features_json) AS feat(value)
WHERE d.quarter_cad = :q"""
+ _ZOUIT_ENGINEERING_WHERE
+ """
ORDER BY feat.value->'properties'->>'reg_numb_border' NULLS LAST,
feat.value->>'geometry'
LIMIT :items_limit
"""
),
{"q": quarter, "wkt": parcel_wkt},
{**params, "items_limit": _ZOUIT_ENGINEERING_ITEMS_LIMIT},
).fetchall()
except (OperationalError, ProgrammingError, DataError) as e:
# DataError — malformed WKT в ST_GeomFromText / ST_GeomFromGeoJSON (PostGIS ERROR).
@ -1433,7 +1502,7 @@ def _get_zouit_engineering_overlaps(
quarter,
e,
)
return []
return [], 0
result: list[dict[str, Any]] = []
for r in rows:
@ -1466,7 +1535,7 @@ def _get_zouit_engineering_overlaps(
"source": "nspd_37578",
}
)
return result
return result, int(total_count)
# ── Harvest trigger ───────────────────────────────────────────────────────────

View file

@ -657,9 +657,30 @@ def get_nearby_utility_infrastructure(
if parcel_wkt is None:
raise ValueError(f"Участок {cad_num!r} не найден в БД")
# Условие радиус-выборки — общее для честного COUNT(*) (без LIMIT) и row-fetch
# (с ORDER BY + LIMIT). Epic #2445 A2: summary.total_features раньше = len(features),
# тихо капался на `limit` (default 200) даже когда в радиусе фич больше.
where_clause = """
WHERE ST_DWithin(
u.geom::geography,
ST_Centroid(ST_GeomFromText(:wkt, 4326))::geography,
CAST(:radius_m AS float)
)
"""
params = {"wkt": parcel_wkt, "radius_m": radius_m}
total_features = (
db.execute(
text("SELECT COUNT(*) FROM osm_utility_infrastructure_ekb u" + where_clause),
params,
).scalar()
or 0
)
rows = (
db.execute(
text("""
text(
"""
SELECT osm_id, osm_type, infrastructure_kind, name, source_tag,
u.tags AS tags,
ST_Distance(
@ -667,16 +688,14 @@ def get_nearby_utility_infrastructure(
ST_Centroid(ST_GeomFromText(:wkt, 4326))::geography
) AS distance_m,
ST_AsGeoJSON(u.geom) AS geojson
FROM osm_utility_infrastructure_ekb u
WHERE ST_DWithin(
u.geom::geography,
ST_Centroid(ST_GeomFromText(:wkt, 4326))::geography,
CAST(:radius_m AS float)
)
FROM osm_utility_infrastructure_ekb u"""
+ where_clause
+ """
ORDER BY distance_m ASC
LIMIT :lim
"""),
{"wkt": parcel_wkt, "radius_m": radius_m, "lim": limit},
"""
),
{**params, "lim": limit},
)
.mappings()
.all()
@ -706,11 +725,15 @@ def get_nearby_utility_infrastructure(
}
)
total_features_int = int(total_features)
return {
"features": features,
"summary": {
"total_features": len(features),
# Epic #2445 A2: честный COUNT(*) по всей радиус-выборке — НЕ len(features)
# (которое капалось на `limit`, default 200).
"total_features": total_features_int,
"nearest_distance_m": nearest_overall,
"nearest_by_kind": nearest_by_kind,
"features_truncated": total_features_int > len(features),
},
}

View file

@ -27,6 +27,7 @@ from app.services.site_finder.quarter_dump_lookup import (
_get_opportunity_parcels,
_get_red_lines,
_get_risk_zones,
_get_zouit_engineering_overlaps,
_get_zouit_overlaps,
_zouit_subcategory_type,
derive_quarter_cad,
@ -103,10 +104,17 @@ def test_extract_features_by_layer_non_string_layer() -> None:
def _make_mock_db_with_rows(rows: list[tuple[Any, ...]]) -> MagicMock:
"""Мок Session где execute().fetchall() возвращает rows."""
"""Мок Session где execute().fetchall() возвращает rows.
execute().scalar() тоже замокан на len(rows) epic #2445 A1 добавил отдельный
honest-COUNT(*) запрос в _get_engineering_structures_by_boundary /
_get_zouit_engineering_overlaps ДО row-fetch; хелперы без true-count-запроса
(_get_risk_zones и т.п.) scalar() не вызывают, так что это безопасный default.
"""
db = MagicMock()
mock_result = MagicMock()
mock_result.fetchall.return_value = rows
mock_result.scalar.return_value = len(rows)
db.execute.return_value = mock_result
return db
@ -791,9 +799,10 @@ def _eng_structures(opts: dict[str, Any]) -> list[dict[str, Any]]:
"""Прогнать _get_engineering_structures_by_boundary на одной строке с данными opts."""
geom = '{"type":"Point","coordinates":[0,0]}'
db = _make_mock_db_with_rows([({"options": opts}, geom, 12.3)])
return _get_engineering_structures_by_boundary(
items, _total = _get_engineering_structures_by_boundary(
db, "66:41:0204016", "POLYGON((0 0,1 0,1 1,0 0))", 500
)
return items
def test_engineering_purpose_set_when_differs_from_type() -> None:
@ -831,3 +840,94 @@ def test_engineering_purpose_from_object_type_value_distinct() -> None:
def test_engineering_purpose_none_when_no_options() -> None:
result = _eng_structures({})
assert result[0]["purpose"] is None
# ── Epic #2445 A1: honest total_count vs capped items list ─────────────────────
def _make_mock_db_count_then_rows(total_count: int, rows: list[tuple[Any, ...]]) -> MagicMock:
"""Мок Session: 1-й execute() → COUNT(*).scalar(), 2-й execute() → .fetchall() capped rows."""
db = MagicMock()
count_result = MagicMock()
count_result.scalar.return_value = total_count
rows_result = MagicMock()
rows_result.fetchall.return_value = rows
db.execute.side_effect = [count_result, rows_result]
return db
def test_engineering_structures_total_count_exceeds_capped_list() -> None:
"""total_count (COUNT(*) без LIMIT) > len(items) (LIMIT 50 row-fetch) → structures_truncated."""
geom = '{"type":"Point","coordinates":[0,0]}'
# Мок отдаёт только 2 строки (симулирует LIMIT 50 сработавший в БД), но истинный
# count в радиусе — 137 (COUNT(*) без LIMIT видит больше, чем row-fetch отдал).
rows = [({"options": {}}, geom, 12.3), ({"options": {}}, geom, 45.6)]
db = _make_mock_db_count_then_rows(total_count=137, rows=rows)
items, total = _get_engineering_structures_by_boundary(
db, "66:41:0204016", "POLYGON((0 0,1 0,1 1,0 0))", 2000
)
assert len(items) == 2
assert total == 137
assert total > len(items)
def test_engineering_structures_not_truncated_when_under_cap() -> None:
"""total_count == len(items) (нет капа) → честно нет усечения."""
geom = '{"type":"Point","coordinates":[0,0]}'
rows = [({"options": {}}, geom, 12.3)]
db = _make_mock_db_count_then_rows(total_count=1, rows=rows)
items, total = _get_engineering_structures_by_boundary(
db, "66:41:0204016", "POLYGON((0 0,1 0,1 1,0 0))", 500
)
assert len(items) == 1
assert total == 1
def test_zouit_engineering_overlaps_total_count_exceeds_capped_list() -> None:
"""total_count (COUNT(*) без LIMIT) > len(items) (LIMIT 100) → protection_zones_truncated."""
geom = '{"type":"Polygon","coordinates":[[[0,0],[1,0],[1,1],[0,0]]]}'
rows = [
({"reg_numb_border": "RN-001", "subcategory": "17"}, geom),
({"reg_numb_border": "RN-002", "subcategory": "17"}, geom),
]
db = _make_mock_db_count_then_rows(total_count=243, rows=rows)
items, total = _get_zouit_engineering_overlaps(
db, "66:41:0204016", "POLYGON((0 0,1 0,1 1,0 0))"
)
assert len(items) == 2
assert total == 243
assert total > len(items)
def test_zouit_engineering_overlaps_order_by_is_present_in_sql() -> None:
"""Fix для non-determinism: ORDER BY обязателен перед LIMIT 100 (was missing)."""
geom = '{"type":"Polygon","coordinates":[[[0,0],[1,0],[1,1],[0,0]]]}'
db = _make_mock_db_count_then_rows(total_count=1, rows=[({"reg_numb_border": "RN-1"}, geom)])
_get_zouit_engineering_overlaps(db, "66:41:0204016", "POLYGON((0 0,1 0,1 1,0 0))")
# 2-й вызов execute() — row-fetch (1-й — COUNT). Проверяем что SQL содержит ORDER BY.
row_fetch_sql = str(db.execute.call_args_list[1][0][0])
assert "ORDER BY" in row_fetch_sql
assert "LIMIT" in row_fetch_sql
def test_zouit_engineering_overlaps_query_failure_returns_zero_total() -> None:
"""DB exception на COUNT/row-fetch → ([], 0), не крашится (existing graceful contract)."""
from sqlalchemy.exc import OperationalError
db = MagicMock()
db.execute.side_effect = OperationalError("stmt", {}, Exception("boom"))
items, total = _get_zouit_engineering_overlaps(
db, "66:41:0204016", "POLYGON((0 0,1 0,1 1,0 0))"
)
assert items == []
assert total == 0

View file

@ -5,6 +5,8 @@ Mock-based / pure — без реального Overpass и БД. Покрыва
- _passes_tag_filters: substance-фильтр (man_made=pipeline) + tower:type (man_made=tower)
- _upsert_elements: парсинг wayLineString / nodePoint, классификация kind,
per-element SAVEPOINT (битый элемент не валит батч), per-kind counts
- get_nearby_utility_infrastructure: honest total_features vs capped features list
(epic #2445 A2 — true COUNT(*) без LIMIT, НЕ len(features))
"""
from __future__ import annotations
@ -12,6 +14,7 @@ from __future__ import annotations
import json
from contextlib import contextmanager
from typing import Any
from unittest.mock import MagicMock, patch
import httpx
@ -593,3 +596,62 @@ async def test_fetch_one_failing_query_does_not_abort_others(monkeypatch: Any) -
# каждое зеркало получило ровно _MAX_RETRIES_PER_ENDPOINT попыток от power
for mirror in ul.OVERPASS_ENDPOINTS[1:]:
assert client.calls.count(mirror) == ul._MAX_RETRIES_PER_ENDPOINT
# ── get_nearby_utility_infrastructure: honest total_features (epic #2445 A2) ────
def _make_feature_row(osm_id: int, distance_m: float) -> dict[str, Any]:
return {
"osm_id": osm_id,
"osm_type": "node",
"infrastructure_kind": "power",
"name": None,
"source_tag": "power=tower",
"tags": {},
"distance_m": distance_m,
"geojson": json.dumps({"type": "Point", "coordinates": [60.6, 56.8]}),
}
def _make_mock_db(total_count: int, capped_rows: list[dict[str, Any]]) -> MagicMock:
"""Мок Session: 1-й execute() → COUNT(*).scalar(), 2-й execute() → capped .mappings().all()."""
db = MagicMock()
count_result = MagicMock()
count_result.scalar.return_value = total_count
rows_result = MagicMock()
rows_result.mappings.return_value.all.return_value = capped_rows
db.execute.side_effect = [count_result, rows_result]
return db
def test_get_nearby_utility_infrastructure_reports_true_total_when_capped() -> None:
"""total_features = честный COUNT(*) (350), НЕ len(features) (капнутый на limit=200)."""
capped_rows = [_make_feature_row(i, float(i)) for i in range(200)]
db = _make_mock_db(total_count=350, capped_rows=capped_rows)
with patch(
"app.services.site_finder.quarter_dump_lookup._get_parcel_wkt",
return_value="POLYGON((0 0,1 0,1 1,0 0))",
):
result = ul.get_nearby_utility_infrastructure(db, "66:41:0204016:10", radius_m=500)
assert len(result["features"]) == 200
assert result["summary"]["total_features"] == 350
assert result["summary"]["features_truncated"] is True
def test_get_nearby_utility_infrastructure_not_truncated_when_under_limit() -> None:
"""total_features == len(features) (нет капа) → features_truncated=False."""
capped_rows = [_make_feature_row(i, float(i)) for i in range(5)]
db = _make_mock_db(total_count=5, capped_rows=capped_rows)
with patch(
"app.services.site_finder.quarter_dump_lookup._get_parcel_wkt",
return_value="POLYGON((0 0,1 0,1 1,0 0))",
):
result = ul.get_nearby_utility_infrastructure(db, "66:41:0204016:10", radius_m=500)
assert len(result["features"]) == 5
assert result["summary"]["total_features"] == 5
assert result["summary"]["features_truncated"] is False

View file

@ -3794,6 +3794,16 @@ export interface components {
protection_zones_intersecting: number;
/** Total Structures In Radius */
total_structures_in_radius: number;
/**
* Protection Zones Truncated
* @default false
*/
protection_zones_truncated: boolean;
/**
* Structures Truncated
* @default false
*/
structures_truncated: boolean;
};
/**
* CreateCadastreJobRequest
@ -5653,6 +5663,11 @@ export interface components {
nearest_by_kind: {
[key: string]: number | null;
};
/**
* Features Truncated
* @default false
*/
features_truncated: boolean;
};
/** ValidationError */
ValidationError: {