"""Address normalization and fingerprint utilities. Used by 3-tier matching to produce stable, source-independent keys. Algorithm reference: decisions/Cross_Source_Matching_Strategy.md sec 3.2 """ import hashlib import re import unicodedata # Strip punctuation EXCEPT hyphens — keep hyphens so abbreviation rules like # 'пр-кт', 'б-р', 'пр-д' can match before they are collapsed to spaces. _PUNCT = re.compile(r'[^\w\s\-]', flags=re.UNICODE) _WS = re.compile(r'\s+') # House/building markers ('дом'/'д'/'корпус'/'корп'/'к') are dropped to a single # canonical form: only the number is kept. This makes the output source-independent — # SERP writes a bare number ('Ленина 5') while detail/БТИ write a marker ('д. 5' -> 'д 5') # and both must yield the same normalized_address / fingerprint (bug #1534). # The marker is removed ONLY when immediately followed by a number, so street-name # initials like 'К. Либкнехта' / 'Д. Зверева' (marker followed by a word) are left # untouched and not mis-expanded into 'корпус'/'дом' (bug #1535). _HOUSE_MARKER = re.compile(r'\b(?:дом|корпус|корп|д|к)\s+(?=\d)', flags=re.UNICODE) # Abbreviation expansion — applied after lowercasing, bounded by spaces. # Order matters: longer/more-specific patterns first to avoid partial overlap. # Hyphenated forms ('пр-кт', 'б-р', 'пр-д') come first so they match before # the shorter variants ('пр', 'б') would erroneously consume the prefix. _ABBREV = [ (' пр-кт ', ' проспект '), (' пр-кт. ', ' проспект '), (' б-р ', ' бульвар '), (' пр-д ', ' проезд '), (' ш ', ' шоссе '), (' ул ', ' улица '), (' ул. ', ' улица '), (' пр ', ' проспект '), (' пр. ', ' проспект '), (' пер ', ' переулок '), (' пл ', ' площадь '), (' наб ', ' набережная '), (' бул ', ' бульвар '), (' туп ', ' тупик '), (' стр ', ' строение '), (' корп ', ' корпус '), # NOTE: house/building markers ('д'/'дом'/'к'/'корп'/'корпус') are NOT expanded here. # They are handled separately by _HOUSE_MARKER below — stripped to a single canonical # form (number only) and only when followed by a number — to avoid asymmetric keys # (#1534) and mis-expanding street-name initials like 'К. Либкнехта' (#1535). ] def normalize_address(text: str | None) -> str: """Normalize address string for cross-source comparison. Steps: 1. Unicode NFC normalization 2. Lowercase 3. Strip punctuation except hyphens (preserve 'пр-кт', 'б-р', 'пр-д' for expansion) 4. Collapse whitespace 5. Expand common Russian address abbreviations (hyphenated forms first) 6. Collapse remaining hyphens to spaces 7. Drop house/building markers ('дом'/'д'/'корпус'/'корп'/'к') that precede a number, keeping only the number — a single canonical form across sources """ if not text: return "" s = unicodedata.normalize('NFC', text).lower() s = _PUNCT.sub(' ', s) # strip punctuation EXCEPT hyphens s = _WS.sub(' ', s).strip() # Wrap with spaces for clean boundary matching s = f' {s} ' for short, full in _ABBREV: s = s.replace(short, full) # Collapse remaining hyphens (e.g. standalone '-' separators or numeric ranges) s = s.replace('-', ' ') s = _WS.sub(' ', s).strip() # Drop house/building markers that precede a number ('д 5'/'дом 5'/'корпус 5' -> '5'), # leaving street-name initials ('к либкнехта') untouched. Re-collapse whitespace after. s = _HOUSE_MARKER.sub('', s) return _WS.sub(' ', s).strip() def address_fingerprint(address: str | None, lat: float | None, lon: float | None) -> str: """SHA-256 fingerprint of normalized address + rounded coordinates (4 dp ≈ 11 m). Returns first 32 hex chars of the digest (128 bits — collision-safe for millions of rows). """ norm_addr = normalize_address(address or '') lat_r = f'{lat:.4f}' if lat is not None else '' lon_r = f'{lon:.4f}' if lon is not None else '' key = f'{norm_addr}|{lat_r}|{lon_r}' return hashlib.sha256(key.encode('utf-8')).hexdigest()[:32]