Light1YT
86e9ea2937
Многоагентный аудит + имплементация: один воркер на файл, точечные правки. Верификация: py_compile (47/47 .py) + tsc --noEmit (0 ошибок). Unit-тесты не прогонялись (окружение не поднято: rollup native dep / нет pytest-venv). Полностью исправлено (169): #1336, #1337, #1339, #1340, #1341, #1342, #1343, #1345, #1346, #1348, #1349, #1350, #1351, #1354, #1356, #1358, #1359, #1360, #1362, #1364, #1365, #1366, #1367, #1368, #1369, #1370, #1371, #1372, #1373, #1374, #1375, #1376, #1377, #1378, #1379, #1380, #1381, #1382, #1384, #1385, #1386, #1387, #1388, #1389, #1390, #1391, #1392, #1394, #1395, #1396, #1397, #1399, #1400, #1401, #1402, #1403, #1404, #1408, #1409, #1410, #1411, #1412, #1413, #1414, #1415, #1416, #1417, #1418, #1420, #1423, #1425, #1426, #1427, #1428, #1429, #1430, #1431, #1432, #1433, #1434, #1435, #1437, #1438, #1439, #1440, #1441, #1442, #1443, #1444, #1445, #1446, #1447, #1448, #1449, #1450, #1451, #1452, #1453, #1454, #1455, #1456, #1457, #1458, #1459, #1460, #1461, #1462, #1463, #1464, #1465, #1466, #1467, #1468, #1469, #1471, #1472, #1473, #1474, #1476, #1478, #1479, #1481, #1482, #1483, #1484, #1485, #1487, #1488, #1489, #1490, #1491, #1492, #1493, #1494, #1495, #1496, #1497, #1499, #1500, #1501, #1502, #1504, #1505, #1506, #1507, #1510, #1514, #1515, #1516, #1517, #1518, #1519, #1521, #1522, #1523, #1524, #1525, #1526, #1527, #1528, #1529, #1531, #1532, #1533, #1534, #1535, #1536, #1537, #1538 Частично (9, in-file часть, остаток cross-file): #1361, #1419, #1422, #1424, #1470, #1475, #1477, #1480, #1498 Требуют cross-file (3, не тронуты): #1338, #1363, #1421 Пропущено (1): #1539 Не входило в партию: 22 needs-Leha issue (нужны решения владельца). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
307 lines
14 KiB
Python
307 lines
14 KiB
Python
"""Загрузчик OSM источников шума из Overpass API для site-finder.
|
||
|
||
Запускается раз в неделю через Celery beat. Хранит дороги (way), ж/д пути,
|
||
промзоны и аэродромы в таблице osm_noise_sources_ekb.
|
||
"""
|
||
|
||
import asyncio
|
||
import json
|
||
import logging
|
||
|
||
import httpx
|
||
from sqlalchemy import text
|
||
|
||
from app.core.db import SessionLocal
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
OVERPASS_URL = "https://overpass-api.de/api/interpreter"
|
||
EKB_BBOX = (56.7, 60.5, 56.95, 60.75) # (south, west, north, east)
|
||
|
||
# Overpass блокирует дефолтный User-Agent (406) — явный UA с контактом.
|
||
_HEADERS = {
|
||
"User-Agent": "GenDesign-SiteFinder/1.0 (+https://gendsgn.ru)",
|
||
"Accept": "application/json",
|
||
}
|
||
|
||
# Описание запросов: (overpass_key, overpass_value, source_type, road_class_from_value)
|
||
_NOISE_QUERIES: list[tuple[str, str, str, str | None]] = [
|
||
("highway", "motorway", "highway", "motorway"),
|
||
("highway", "trunk", "highway", "trunk"),
|
||
("highway", "primary", "highway", "primary"),
|
||
("highway", "secondary", "highway", "secondary"),
|
||
("highway", "tertiary", "highway", "tertiary"),
|
||
("highway", "residential", "highway", "residential"),
|
||
("railway", "rail", "railway", None),
|
||
("landuse", "industrial", "industrial", None),
|
||
("aeroway", "aerodrome", "aerodrome", None),
|
||
# Гидрология — реки, ручьи, каналы (way-линии) и водоёмы (way-полигоны)
|
||
("waterway", "river", "water", "river"),
|
||
("waterway", "stream", "water", "stream"),
|
||
("waterway", "canal", "water", "canal"),
|
||
("natural", "water", "water", "lake_or_pond"),
|
||
# Инженерные сети — магистральные линии (для proximity-проверки доступности
|
||
# подключения / распознавания ЛЭП-охранных зон через ЗОУИТ типа 5/9)
|
||
("power", "line", "utility", "power_line"),
|
||
("power", "minor_line", "utility", "power_line"),
|
||
("man_made", "pipeline", "utility", "pipeline"),
|
||
# Магистральные трубопроводы по substance (way) — газ / вода / канализация (#44).
|
||
# OSM tag `pipeline` (не `man_made=pipeline`): pipeline=substance.
|
||
("pipeline", "gas", "utility", "gas_pipeline"),
|
||
("pipeline", "water", "utility", "water_main"),
|
||
("pipeline", "sewerage", "utility", "sewerage"),
|
||
# Сети ВКХ (водоснабжение / канализация) — узлы, не магистрали
|
||
("man_made", "water_works", "utility", "water_works"),
|
||
("man_made", "wastewater_plant", "utility", "wastewater_plant"),
|
||
("man_made", "water_tower", "utility", "water_tower"),
|
||
("man_made", "storage_tank", "utility", "storage_tank"),
|
||
# Электроподстанции / трансформаторы — точки подключения 10/35/110 кВ (#44)
|
||
("power", "substation", "utility", "substation"),
|
||
("power", "transformer", "utility", "transformer"),
|
||
# ЦТП (центральный тепловой пункт) — точка подключения теплосети (#44)
|
||
("man_made", "heat_substation", "utility", "heat_substation"),
|
||
]
|
||
|
||
|
||
# Инж.-сетевые точки подключения (#44), которые в OSM маппят и как node, и как
|
||
# area (closed way): подстанции / трансформаторы / ЦТП / водозабор и т.п.
|
||
# Для них спрашиваем nwr (node + way) с `out geom`, чтобы поймать оба варианта
|
||
# маппинга и получить координаты узлов для way-площадок.
|
||
_UTILITY_POINT_VALUES: frozenset[str] = frozenset(
|
||
{
|
||
"substation",
|
||
"transformer",
|
||
"heat_substation",
|
||
"water_works",
|
||
"wastewater_plant",
|
||
"water_tower",
|
||
"storage_tank",
|
||
}
|
||
)
|
||
|
||
|
||
def _build_overpass_query(key: str, value: str, el_type: str) -> str:
|
||
"""Запрос Overpass для одного типа элемента.
|
||
|
||
Для way / nwr используем `out geom` — получаем полные координаты узлов
|
||
(нужно для построения LineString / Polygon геометрии way-площадок).
|
||
Для node — `out body`.
|
||
"""
|
||
south, west, north, east = EKB_BBOX
|
||
bbox = f"({south},{west},{north},{east})"
|
||
if el_type == "nwr":
|
||
# node + way: точки подключения бывают и точкой, и площадкой
|
||
return f"[out:json][timeout:30];" f'nwr["{key}"="{value}"]{bbox};' f"out geom;"
|
||
if el_type == "way":
|
||
return f"[out:json][timeout:30];" f'way["{key}"="{value}"]{bbox};' f"out geom;"
|
||
return f"[out:json][timeout:30];" f'node["{key}"="{value}"]{bbox};' f"out body;"
|
||
|
||
|
||
async def fetch_overpass_noise() -> list[dict]:
|
||
"""Запрашивает Overpass API для всех шумовых источников ЕКБ.
|
||
|
||
Возвращает enriched list dict'ов с дополнительными полями:
|
||
_source_type, _road_class — для использования при UPSERT.
|
||
|
||
Между запросами sleep 1с (Overpass usage policy).
|
||
"""
|
||
all_elements: list[dict] = []
|
||
async with httpx.AsyncClient(timeout=60, headers=_HEADERS) as client:
|
||
for key, value, source_type, road_class in _NOISE_QUERIES:
|
||
# aerodrome — nwr (node+way): в OSM аэродромы/аэропорты (Кольцово,
|
||
# IATA SVX) почти всегда замаплены как way-полигон, а не одиночный
|
||
# node. Запрос только по node возвращал ~0 элементов → авиашум молча
|
||
# терялся в скоринге (#1350). nwr ловит оба варианта; way-полигон
|
||
# граница уходит в _way_to_linestring_wkt и корректно участвует в
|
||
# ST_Distance-проверках. Точки подключения инж.сетей — тоже nwr;
|
||
# остальное (дороги/ж-д/трубы/ЛЭП-линии) — way.
|
||
if key == "aeroway":
|
||
el_type = "nwr"
|
||
elif value in _UTILITY_POINT_VALUES:
|
||
el_type = "nwr"
|
||
else:
|
||
el_type = "way"
|
||
query = _build_overpass_query(key, value, el_type)
|
||
try:
|
||
r = await client.post(OVERPASS_URL, data={"data": query})
|
||
r.raise_for_status()
|
||
elements: list[dict] = r.json().get("elements", [])
|
||
logger.info(
|
||
"Overpass noise: %s=%s (%s) → %d elements",
|
||
key,
|
||
value,
|
||
source_type,
|
||
len(elements),
|
||
)
|
||
for el in elements:
|
||
el["_source_type"] = source_type
|
||
el["_road_class"] = road_class
|
||
all_elements.extend(elements)
|
||
except Exception as e:
|
||
logger.warning("Overpass noise failed for %s=%s: %s", key, value, e)
|
||
await asyncio.sleep(1.0)
|
||
logger.info("Overpass noise: total %d elements", len(all_elements))
|
||
return all_elements
|
||
|
||
|
||
def _way_to_linestring_wkt(nodes: list[dict]) -> str | None:
|
||
"""Построить WKT LineString из списка узлов Overpass (out geom).
|
||
|
||
Каждый узел — {'lat': ..., 'lon': ...}. Минимум 2 точки для валидной линии.
|
||
"""
|
||
if len(nodes) < 2:
|
||
return None
|
||
# WKT LINESTRING требует запятую между парами точек: "lon1 lat1, lon2 lat2".
|
||
# Раньше joined через пробел → "lon1 lat1 lon2 lat2" → invalid geometry.
|
||
points = [f"{n['lon']} {n['lat']}" for n in nodes if "lon" in n and "lat" in n]
|
||
if len(points) < 2:
|
||
return None
|
||
return f"LINESTRING({', '.join(points)})"
|
||
|
||
|
||
def _way_to_polygon_wkt(nodes: list[dict]) -> str | None:
|
||
"""Построить WKT Polygon из замкнутого way Overpass (out geom).
|
||
|
||
Используется для natural=water (озёра, пруды). Если кольцо не замкнуто
|
||
автоматически — добавляем первую точку в конец.
|
||
|
||
PostGIS отвергает linear ring < 4 точек. Замкнутый way из 3 точек
|
||
[A, B, A] (points[0]==points[-1]) даёт POLYGON((A,B,A)) — "geometry
|
||
requires more points". Возвращаем None, чтобы caller сделал fallback
|
||
на LineString или пропустил элемент.
|
||
"""
|
||
points = [f"{n['lon']} {n['lat']}" for n in nodes if "lon" in n and "lat" in n]
|
||
if len(points) < 3:
|
||
return None
|
||
# Замкнуть кольцо если нужно
|
||
if points[0] != points[-1]:
|
||
points.append(points[0])
|
||
# Итоговое кольцо должно содержать ≥4 точек (PostGIS требование).
|
||
if len(points) < 4:
|
||
return None
|
||
return f"POLYGON(({', '.join(points)}))"
|
||
|
||
|
||
def sync_noise_sources_to_db() -> dict[str, int]:
|
||
"""UPSERT шумовых источников из Overpass в osm_noise_sources_ekb.
|
||
|
||
UPSERT по UNIQUE(osm_type, osm_id).
|
||
Геометрия:
|
||
- way с nodes → LineString WKT через ST_GeomFromText(..., 4326)
|
||
- node → ST_SetSRID(ST_MakePoint(lon, lat), 4326)
|
||
Записывает breadcrumb в nspd_geo_log (stage='noise_sync').
|
||
"""
|
||
elements = asyncio.run(fetch_overpass_noise())
|
||
|
||
inserted = 0
|
||
updated = 0
|
||
skipped = 0
|
||
fetched = len(elements)
|
||
|
||
db = SessionLocal()
|
||
try:
|
||
for el in elements:
|
||
osm_id: int = el["id"]
|
||
osm_type: str = el["type"] # 'way' | 'node'
|
||
source_type: str = el.get("_source_type", "unknown")
|
||
road_class: str | None = el.get("_road_class")
|
||
tags: dict = el.get("tags") or {}
|
||
name: str | None = tags.get("name")
|
||
|
||
# Геометрия зависит от типа элемента
|
||
geom_sql: str
|
||
geom_params: dict
|
||
if osm_type == "way":
|
||
nodes: list[dict] = el.get("geometry") or []
|
||
# natural=water → замкнутый полигон; waterway → линия
|
||
if source_type == "water" and road_class == "lake_or_pond":
|
||
wkt = _way_to_polygon_wkt(nodes)
|
||
if wkt is None:
|
||
# Fallback: попробовать как линию (частично загруженный bbox-обрезок)
|
||
wkt = _way_to_linestring_wkt(nodes)
|
||
else:
|
||
wkt = _way_to_linestring_wkt(nodes)
|
||
if wkt is None:
|
||
skipped += 1
|
||
continue
|
||
geom_sql = "ST_GeomFromText(:wkt, 4326)"
|
||
geom_params = {"wkt": wkt}
|
||
elif osm_type == "node":
|
||
lat = el.get("lat")
|
||
lon = el.get("lon")
|
||
if lat is None or lon is None:
|
||
skipped += 1
|
||
continue
|
||
geom_sql = "ST_SetSRID(ST_MakePoint(:lon, :lat), 4326)"
|
||
geom_params = {"lat": lat, "lon": lon}
|
||
else:
|
||
# relation — skip (сложная геометрия, не нужна сейчас)
|
||
skipped += 1
|
||
continue
|
||
|
||
params: dict = {
|
||
"osm_id": osm_id,
|
||
"osm_type": osm_type,
|
||
"source_type": source_type,
|
||
"road_class": road_class,
|
||
"name": name,
|
||
"tags": json.dumps(tags, ensure_ascii=False),
|
||
**geom_params,
|
||
}
|
||
|
||
try:
|
||
with db.begin_nested(): # SAVEPOINT — откат только этой записи
|
||
result = db.execute(
|
||
text(f"""
|
||
INSERT INTO osm_noise_sources_ekb
|
||
(osm_type, osm_id, source_type, road_class, name, geom, tags,
|
||
fetched_at)
|
||
VALUES (
|
||
:osm_type, :osm_id, :source_type, :road_class, :name,
|
||
{geom_sql},
|
||
CAST(:tags AS jsonb), NOW()
|
||
)
|
||
ON CONFLICT (osm_type, osm_id) DO UPDATE
|
||
SET source_type = EXCLUDED.source_type,
|
||
road_class = EXCLUDED.road_class,
|
||
name = EXCLUDED.name,
|
||
geom = EXCLUDED.geom,
|
||
tags = EXCLUDED.tags,
|
||
fetched_at = NOW()
|
||
RETURNING (xmax = 0) AS is_insert
|
||
"""),
|
||
params,
|
||
).scalar()
|
||
if result:
|
||
inserted += 1
|
||
else:
|
||
updated += 1
|
||
except Exception as e:
|
||
# Дефектный way (например, 3-точечное замкнутое кольцо POLYGON((A,B,A))
|
||
# отвергается PostGIS) не должен валить весь weekly-sync. SAVEPOINT
|
||
# откатывает только эту строку, продолжаем с остальными.
|
||
logger.warning(
|
||
"noise_sync insert failed for %s/%s (source_type=%s): %s",
|
||
osm_type,
|
||
osm_id,
|
||
source_type,
|
||
e,
|
||
)
|
||
skipped += 1
|
||
|
||
db.commit()
|
||
except Exception as e:
|
||
db.rollback()
|
||
logger.exception("noise_sync: unexpected error, outer tx rolled back: %s", e)
|
||
raise
|
||
finally:
|
||
db.close()
|
||
|
||
result_dict = {
|
||
"fetched": fetched,
|
||
"inserted": inserted,
|
||
"updated": updated,
|
||
"skipped": skipped,
|
||
}
|
||
logger.info("noise_sync done: %s", result_dict)
|
||
return result_dict
|