"""Address normalization and fingerprint utilities. Used by 3-tier matching to produce stable, source-independent keys. Algorithm reference: decisions/Cross_Source_Matching_Strategy.md sec 3.2 """ import hashlib import re import unicodedata # Strip punctuation EXCEPT hyphens — keep hyphens so abbreviation rules like # 'пр-кт', 'б-р', 'пр-д' can match before they are collapsed to spaces. _PUNCT = re.compile(r"[^\w\s\-]", flags=re.UNICODE) _WS = re.compile(r"\s+") # House/building markers ('дом'/'д'/'корпус'/'корп'/'к') are dropped to a single # canonical form: only the number is kept. This makes the output source-independent — # SERP writes a bare number ('Ленина 5') while detail/БТИ write a marker ('д. 5' -> 'д 5') # and both must yield the same normalized_address / fingerprint (bug #1534). # The marker is removed ONLY when immediately followed by a number, so street-name # initials like 'К. Либкнехта' / 'Д. Зверева' (marker followed by a word) are left # untouched and not mis-expanded into 'корпус'/'дом' (bug #1535). _HOUSE_MARKER = re.compile(r"\b(?:дом|корпус|корп|д|к)\s+(?=\d)", flags=re.UNICODE) # Abbreviation expansion — applied after lowercasing, bounded by spaces. # Order matters: longer/more-specific patterns first to avoid partial overlap. # Hyphenated forms ('пр-кт', 'б-р', 'пр-д') come first so they match before # the shorter variants ('пр', 'б') would erroneously consume the prefix. _ABBREV = [ (" пр-кт ", " проспект "), (" пр-кт. ", " проспект "), (" б-р ", " бульвар "), (" пр-д ", " проезд "), (" ш ", " шоссе "), (" ул ", " улица "), (" ул. ", " улица "), (" пр ", " проспект "), (" пр. ", " проспект "), (" пер ", " переулок "), (" пл ", " площадь "), (" наб ", " набережная "), (" бул ", " бульвар "), (" туп ", " тупик "), (" стр ", " строение "), (" корп ", " корпус "), # NOTE: house/building markers ('д'/'дом'/'к'/'корп'/'корпус') are NOT expanded here. # They are handled separately by _HOUSE_MARKER below — stripped to a single canonical # form (number only) and only when followed by a number — to avoid asymmetric keys # (#1534) and mis-expanding street-name initials like 'К. Либкнехта' (#1535). ] def normalize_address(text: str | None) -> str: """Normalize address string for cross-source comparison. Steps: 1. Unicode NFC normalization 2. Lowercase 3. Strip punctuation except hyphens (preserve 'пр-кт', 'б-р', 'пр-д' for expansion) 4. Collapse whitespace 5. Expand common Russian address abbreviations (hyphenated forms first) 6. Collapse remaining hyphens to spaces 7. Drop house/building markers ('дом'/'д'/'корпус'/'корп'/'к') that precede a number, keeping only the number — a single canonical form across sources """ if not text: return "" s = unicodedata.normalize("NFC", text).lower() s = _PUNCT.sub(" ", s) # strip punctuation EXCEPT hyphens s = _WS.sub(" ", s).strip() # Wrap with spaces for clean boundary matching s = f" {s} " for short, full in _ABBREV: s = s.replace(short, full) # Collapse remaining hyphens (e.g. standalone '-' separators or numeric ranges) s = s.replace("-", " ") s = _WS.sub(" ", s).strip() # Drop house/building markers that precede a number ('д 5'/'дом 5'/'корпус 5' -> '5'), # leaving street-name initials ('к либкнехта') untouched. Re-collapse whitespace after. s = _HOUSE_MARKER.sub("", s) return _WS.sub(" ", s).strip() def has_house_number(normalized: str | None) -> bool: """True if the normalized address ends with a house-number token. The trailing token must start with a digit: 'улица мамина сибиряка 126а' -> True 'улица 8 марта' -> False (street-name digit, no house no.) 'улица 8 марта 5' -> True ('5' is the house number) 'улица мамина сибиряка' -> False (bare street, no number) ''/None -> False Detection uses the TRAILING token only — a digit inside the street name ('8 марта', '50 лет октября') is NOT a house number. """ if not normalized: return False return re.search(r"(?:^|\s)\d\S*$", normalized) is not None def house_number_token(normalized: str | None) -> str | None: """The trailing house-number token (e.g. '126а', '32', '108и'), or None. Returns None when the normalized address has no trailing house number. Used to compare whether two addresses refer to the same building number (geo-proximity consistency guard in match_or_create_house). """ if not normalized: return None m = re.search(r"(?:^|\s)(\d\S*)$", normalized) return m.group(1) if m else None def address_fingerprint(address: str | None, lat: float | None, lon: float | None) -> str: """SHA-256 fingerprint of normalized address + rounded coordinates (4 dp ≈ 11 m). Returns first 32 hex chars of the digest (128 bits — collision-safe for millions of rows). """ norm_addr = normalize_address(address or "") lat_r = f"{lat:.4f}" if lat is not None else "" lon_r = f"{lon:.4f}" if lon is not None else "" key = f"{norm_addr}|{lat_r}|{lon_r}" return hashlib.sha256(key.encode("utf-8")).hexdigest()[:32]