Light1YT
86e9ea2937
Многоагентный аудит + имплементация: один воркер на файл, точечные правки. Верификация: py_compile (47/47 .py) + tsc --noEmit (0 ошибок). Unit-тесты не прогонялись (окружение не поднято: rollup native dep / нет pytest-venv). Полностью исправлено (169): #1336, #1337, #1339, #1340, #1341, #1342, #1343, #1345, #1346, #1348, #1349, #1350, #1351, #1354, #1356, #1358, #1359, #1360, #1362, #1364, #1365, #1366, #1367, #1368, #1369, #1370, #1371, #1372, #1373, #1374, #1375, #1376, #1377, #1378, #1379, #1380, #1381, #1382, #1384, #1385, #1386, #1387, #1388, #1389, #1390, #1391, #1392, #1394, #1395, #1396, #1397, #1399, #1400, #1401, #1402, #1403, #1404, #1408, #1409, #1410, #1411, #1412, #1413, #1414, #1415, #1416, #1417, #1418, #1420, #1423, #1425, #1426, #1427, #1428, #1429, #1430, #1431, #1432, #1433, #1434, #1435, #1437, #1438, #1439, #1440, #1441, #1442, #1443, #1444, #1445, #1446, #1447, #1448, #1449, #1450, #1451, #1452, #1453, #1454, #1455, #1456, #1457, #1458, #1459, #1460, #1461, #1462, #1463, #1464, #1465, #1466, #1467, #1468, #1469, #1471, #1472, #1473, #1474, #1476, #1478, #1479, #1481, #1482, #1483, #1484, #1485, #1487, #1488, #1489, #1490, #1491, #1492, #1493, #1494, #1495, #1496, #1497, #1499, #1500, #1501, #1502, #1504, #1505, #1506, #1507, #1510, #1514, #1515, #1516, #1517, #1518, #1519, #1521, #1522, #1523, #1524, #1525, #1526, #1527, #1528, #1529, #1531, #1532, #1533, #1534, #1535, #1536, #1537, #1538 Частично (9, in-file часть, остаток cross-file): #1361, #1419, #1422, #1424, #1470, #1475, #1477, #1480, #1498 Требуют cross-file (3, не тронуты): #1338, #1363, #1421 Пропущено (1): #1539 Не входило в партию: 22 needs-Leha issue (нужны решения владельца). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
93 lines
4.3 KiB
Python
93 lines
4.3 KiB
Python
"""Address normalization and fingerprint utilities.
|
||
|
||
Used by 3-tier matching to produce stable, source-independent keys.
|
||
Algorithm reference: decisions/Cross_Source_Matching_Strategy.md sec 3.2
|
||
"""
|
||
|
||
import hashlib
|
||
import re
|
||
import unicodedata
|
||
|
||
# Strip punctuation EXCEPT hyphens — keep hyphens so abbreviation rules like
|
||
# 'пр-кт', 'б-р', 'пр-д' can match before they are collapsed to spaces.
|
||
_PUNCT = re.compile(r'[^\w\s\-]', flags=re.UNICODE)
|
||
_WS = re.compile(r'\s+')
|
||
|
||
# House/building markers ('дом'/'д'/'корпус'/'корп'/'к') are dropped to a single
|
||
# canonical form: only the number is kept. This makes the output source-independent —
|
||
# SERP writes a bare number ('Ленина 5') while detail/БТИ write a marker ('д. 5' -> 'д 5')
|
||
# and both must yield the same normalized_address / fingerprint (bug #1534).
|
||
# The marker is removed ONLY when immediately followed by a number, so street-name
|
||
# initials like 'К. Либкнехта' / 'Д. Зверева' (marker followed by a word) are left
|
||
# untouched and not mis-expanded into 'корпус'/'дом' (bug #1535).
|
||
_HOUSE_MARKER = re.compile(r'\b(?:дом|корпус|корп|д|к)\s+(?=\d)', flags=re.UNICODE)
|
||
|
||
# Abbreviation expansion — applied after lowercasing, bounded by spaces.
|
||
# Order matters: longer/more-specific patterns first to avoid partial overlap.
|
||
# Hyphenated forms ('пр-кт', 'б-р', 'пр-д') come first so they match before
|
||
# the shorter variants ('пр', 'б') would erroneously consume the prefix.
|
||
_ABBREV = [
|
||
(' пр-кт ', ' проспект '),
|
||
(' пр-кт. ', ' проспект '),
|
||
(' б-р ', ' бульвар '),
|
||
(' пр-д ', ' проезд '),
|
||
(' ш ', ' шоссе '),
|
||
(' ул ', ' улица '),
|
||
(' ул. ', ' улица '),
|
||
(' пр ', ' проспект '),
|
||
(' пр. ', ' проспект '),
|
||
(' пер ', ' переулок '),
|
||
(' пл ', ' площадь '),
|
||
(' наб ', ' набережная '),
|
||
(' бул ', ' бульвар '),
|
||
(' туп ', ' тупик '),
|
||
(' стр ', ' строение '),
|
||
(' корп ', ' корпус '),
|
||
# NOTE: house/building markers ('д'/'дом'/'к'/'корп'/'корпус') are NOT expanded here.
|
||
# They are handled separately by _HOUSE_MARKER below — stripped to a single canonical
|
||
# form (number only) and only when followed by a number — to avoid asymmetric keys
|
||
# (#1534) and mis-expanding street-name initials like 'К. Либкнехта' (#1535).
|
||
]
|
||
|
||
|
||
def normalize_address(text: str | None) -> str:
|
||
"""Normalize address string for cross-source comparison.
|
||
|
||
Steps:
|
||
1. Unicode NFC normalization
|
||
2. Lowercase
|
||
3. Strip punctuation except hyphens (preserve 'пр-кт', 'б-р', 'пр-д' for expansion)
|
||
4. Collapse whitespace
|
||
5. Expand common Russian address abbreviations (hyphenated forms first)
|
||
6. Collapse remaining hyphens to spaces
|
||
7. Drop house/building markers ('дом'/'д'/'корпус'/'корп'/'к') that precede a
|
||
number, keeping only the number — a single canonical form across sources
|
||
"""
|
||
if not text:
|
||
return ""
|
||
s = unicodedata.normalize('NFC', text).lower()
|
||
s = _PUNCT.sub(' ', s) # strip punctuation EXCEPT hyphens
|
||
s = _WS.sub(' ', s).strip()
|
||
# Wrap with spaces for clean boundary matching
|
||
s = f' {s} '
|
||
for short, full in _ABBREV:
|
||
s = s.replace(short, full)
|
||
# Collapse remaining hyphens (e.g. standalone '-' separators or numeric ranges)
|
||
s = s.replace('-', ' ')
|
||
s = _WS.sub(' ', s).strip()
|
||
# Drop house/building markers that precede a number ('д 5'/'дом 5'/'корпус 5' -> '5'),
|
||
# leaving street-name initials ('к либкнехта') untouched. Re-collapse whitespace after.
|
||
s = _HOUSE_MARKER.sub('', s)
|
||
return _WS.sub(' ', s).strip()
|
||
|
||
|
||
def address_fingerprint(address: str | None, lat: float | None, lon: float | None) -> str:
|
||
"""SHA-256 fingerprint of normalized address + rounded coordinates (4 dp ≈ 11 m).
|
||
|
||
Returns first 32 hex chars of the digest (128 bits — collision-safe for millions of rows).
|
||
"""
|
||
norm_addr = normalize_address(address or '')
|
||
lat_r = f'{lat:.4f}' if lat is not None else ''
|
||
lon_r = f'{lon:.4f}' if lon is not None else ''
|
||
key = f'{norm_addr}|{lat_r}|{lon_r}'
|
||
return hashlib.sha256(key.encode('utf-8')).hexdigest()[:32]
|