bot-backend
384c4e7ac3
Три гарда в match_or_create_house (+ P2 в match_house_readonly): - P1: не матчить/алиасить голую улицу (normalized_address без номера дома) - P2: отклонять Tier-3 гео-кандидата, если номера домов различаются - P3: не цементировать слабый гео-матч (0.7) как alias-ключ здания Хелперы has_house_number/house_number_token (по хвостовому токену — «улица 8 марта» это улица, не дом). Корень mis-bucketing: house 131237 стянул 8 разных Мамина-номеров + голую улицу. Юнит + поведенческие тесты, 603 matching/normalize/house зелёных. Аффектит только НОВЫЕ матчи; чистка cemented-данных — отдельной миграцией (P4).
124 lines
5.6 KiB
Python
124 lines
5.6 KiB
Python
"""Address normalization and fingerprint utilities.
|
||
|
||
Used by 3-tier matching to produce stable, source-independent keys.
|
||
Algorithm reference: decisions/Cross_Source_Matching_Strategy.md sec 3.2
|
||
"""
|
||
|
||
import hashlib
|
||
import re
|
||
import unicodedata
|
||
|
||
# Strip punctuation EXCEPT hyphens — keep hyphens so abbreviation rules like
|
||
# 'пр-кт', 'б-р', 'пр-д' can match before they are collapsed to spaces.
|
||
_PUNCT = re.compile(r"[^\w\s\-]", flags=re.UNICODE)
|
||
_WS = re.compile(r"\s+")
|
||
|
||
# House/building markers ('дом'/'д'/'корпус'/'корп'/'к') are dropped to a single
|
||
# canonical form: only the number is kept. This makes the output source-independent —
|
||
# SERP writes a bare number ('Ленина 5') while detail/БТИ write a marker ('д. 5' -> 'д 5')
|
||
# and both must yield the same normalized_address / fingerprint (bug #1534).
|
||
# The marker is removed ONLY when immediately followed by a number, so street-name
|
||
# initials like 'К. Либкнехта' / 'Д. Зверева' (marker followed by a word) are left
|
||
# untouched and not mis-expanded into 'корпус'/'дом' (bug #1535).
|
||
_HOUSE_MARKER = re.compile(r"\b(?:дом|корпус|корп|д|к)\s+(?=\d)", flags=re.UNICODE)
|
||
|
||
# Abbreviation expansion — applied after lowercasing, bounded by spaces.
|
||
# Order matters: longer/more-specific patterns first to avoid partial overlap.
|
||
# Hyphenated forms ('пр-кт', 'б-р', 'пр-д') come first so they match before
|
||
# the shorter variants ('пр', 'б') would erroneously consume the prefix.
|
||
_ABBREV = [
|
||
(" пр-кт ", " проспект "),
|
||
(" пр-кт. ", " проспект "),
|
||
(" б-р ", " бульвар "),
|
||
(" пр-д ", " проезд "),
|
||
(" ш ", " шоссе "),
|
||
(" ул ", " улица "),
|
||
(" ул. ", " улица "),
|
||
(" пр ", " проспект "),
|
||
(" пр. ", " проспект "),
|
||
(" пер ", " переулок "),
|
||
(" пл ", " площадь "),
|
||
(" наб ", " набережная "),
|
||
(" бул ", " бульвар "),
|
||
(" туп ", " тупик "),
|
||
(" стр ", " строение "),
|
||
(" корп ", " корпус "),
|
||
# NOTE: house/building markers ('д'/'дом'/'к'/'корп'/'корпус') are NOT expanded here.
|
||
# They are handled separately by _HOUSE_MARKER below — stripped to a single canonical
|
||
# form (number only) and only when followed by a number — to avoid asymmetric keys
|
||
# (#1534) and mis-expanding street-name initials like 'К. Либкнехта' (#1535).
|
||
]
|
||
|
||
|
||
def normalize_address(text: str | None) -> str:
|
||
"""Normalize address string for cross-source comparison.
|
||
|
||
Steps:
|
||
1. Unicode NFC normalization
|
||
2. Lowercase
|
||
3. Strip punctuation except hyphens (preserve 'пр-кт', 'б-р', 'пр-д' for expansion)
|
||
4. Collapse whitespace
|
||
5. Expand common Russian address abbreviations (hyphenated forms first)
|
||
6. Collapse remaining hyphens to spaces
|
||
7. Drop house/building markers ('дом'/'д'/'корпус'/'корп'/'к') that precede a
|
||
number, keeping only the number — a single canonical form across sources
|
||
"""
|
||
if not text:
|
||
return ""
|
||
s = unicodedata.normalize("NFC", text).lower()
|
||
s = _PUNCT.sub(" ", s) # strip punctuation EXCEPT hyphens
|
||
s = _WS.sub(" ", s).strip()
|
||
# Wrap with spaces for clean boundary matching
|
||
s = f" {s} "
|
||
for short, full in _ABBREV:
|
||
s = s.replace(short, full)
|
||
# Collapse remaining hyphens (e.g. standalone '-' separators or numeric ranges)
|
||
s = s.replace("-", " ")
|
||
s = _WS.sub(" ", s).strip()
|
||
# Drop house/building markers that precede a number ('д 5'/'дом 5'/'корпус 5' -> '5'),
|
||
# leaving street-name initials ('к либкнехта') untouched. Re-collapse whitespace after.
|
||
s = _HOUSE_MARKER.sub("", s)
|
||
return _WS.sub(" ", s).strip()
|
||
|
||
|
||
def has_house_number(normalized: str | None) -> bool:
|
||
"""True if the normalized address ends with a house-number token.
|
||
|
||
The trailing token must start with a digit:
|
||
'улица мамина сибиряка 126а' -> True
|
||
'улица 8 марта' -> False (street-name digit, no house no.)
|
||
'улица 8 марта 5' -> True ('5' is the house number)
|
||
'улица мамина сибиряка' -> False (bare street, no number)
|
||
''/None -> False
|
||
|
||
Detection uses the TRAILING token only — a digit inside the street name
|
||
('8 марта', '50 лет октября') is NOT a house number.
|
||
"""
|
||
if not normalized:
|
||
return False
|
||
return re.search(r"(?:^|\s)\d\S*$", normalized) is not None
|
||
|
||
|
||
def house_number_token(normalized: str | None) -> str | None:
|
||
"""The trailing house-number token (e.g. '126а', '32', '108и'), or None.
|
||
|
||
Returns None when the normalized address has no trailing house number.
|
||
Used to compare whether two addresses refer to the same building number
|
||
(geo-proximity consistency guard in match_or_create_house).
|
||
"""
|
||
if not normalized:
|
||
return None
|
||
m = re.search(r"(?:^|\s)(\d\S*)$", normalized)
|
||
return m.group(1) if m else None
|
||
|
||
|
||
def address_fingerprint(address: str | None, lat: float | None, lon: float | None) -> str:
|
||
"""SHA-256 fingerprint of normalized address + rounded coordinates (4 dp ≈ 11 m).
|
||
|
||
Returns first 32 hex chars of the digest (128 bits — collision-safe for millions of rows).
|
||
"""
|
||
norm_addr = normalize_address(address or "")
|
||
lat_r = f"{lat:.4f}" if lat is not None else ""
|
||
lon_r = f"{lon:.4f}" if lon is not None else ""
|
||
key = f"{norm_addr}|{lat_r}|{lon_r}"
|
||
return hashlib.sha256(key.encode("utf-8")).hexdigest()[:32]
|