4470b1a764
CRITICAL #1: houses INSERT now populates source, ext_house_id, url (NOT NULL cols) with ON CONFLICT (source, ext_house_id) upsert; source_url param added with synthetic fallback. CRITICAL #2: geom removed from INSERT col list — houses_set_geom_trg trigger auto-populates. CRITICAL #3: Tier 3 ST_DWithin/ST_Distance now cast geom::geography on both sides to avoid mixed geometry/geography function lookup failure. CRITICAL #4: normalize.py _PUNCT now keeps hyphens ([^\w\s\-]); abbreviation expansion for пр-кт/б-р/пр-д runs before hyphen collapse, eliminating leftover кт token. HIGH #5: Tier 0 cadastral queries add ORDER BY id ASC for deterministic results (houses + listings). HIGH #6: match_or_create_house docstring documents race-condition known limitation + Stage 8.x plan. Medium #7: no-op ('литер', 'литер') entry removed from _ABBREV. Medium #9: update_canonical_fields raises NotImplementedError instead of silent pass. Medium #11: price_rub bigint truncation comment added to _upsert_listing_source. Medium #14: upsert_listing_source exported from __init__.py __all__. Tests: 29/29 pass; prkT test asserts no leftover кт; update_canonical_fields test updated.
76 lines
2.8 KiB
Python
76 lines
2.8 KiB
Python
"""Address normalization and fingerprint utilities.
|
||
|
||
Used by 3-tier matching to produce stable, source-independent keys.
|
||
Algorithm reference: decisions/Cross_Source_Matching_Strategy.md sec 3.2
|
||
"""
|
||
|
||
import hashlib
|
||
import re
|
||
import unicodedata
|
||
|
||
# Strip punctuation EXCEPT hyphens — keep hyphens so abbreviation rules like
|
||
# 'пр-кт', 'б-р', 'пр-д' can match before they are collapsed to spaces.
|
||
_PUNCT = re.compile(r'[^\w\s\-]', flags=re.UNICODE)
|
||
_WS = re.compile(r'\s+')
|
||
|
||
# Abbreviation expansion — applied after lowercasing, bounded by spaces.
|
||
# Order matters: longer/more-specific patterns first to avoid partial overlap.
|
||
# Hyphenated forms ('пр-кт', 'б-р', 'пр-д') come first so they match before
|
||
# the shorter variants ('пр', 'б') would erroneously consume the prefix.
|
||
_ABBREV = [
|
||
(' пр-кт ', ' проспект '),
|
||
(' пр-кт. ', ' проспект '),
|
||
(' б-р ', ' бульвар '),
|
||
(' пр-д ', ' проезд '),
|
||
(' ш ', ' шоссе '),
|
||
(' ул ', ' улица '),
|
||
(' ул. ', ' улица '),
|
||
(' пр ', ' проспект '),
|
||
(' пр. ', ' проспект '),
|
||
(' пер ', ' переулок '),
|
||
(' пл ', ' площадь '),
|
||
(' наб ', ' набережная '),
|
||
(' бул ', ' бульвар '),
|
||
(' туп ', ' тупик '),
|
||
(' стр ', ' строение '),
|
||
(' корп ', ' корпус '),
|
||
(' д ', ' дом '),
|
||
(' к ', ' корпус '),
|
||
]
|
||
|
||
|
||
def normalize_address(text: str | None) -> str:
|
||
"""Normalize address string for cross-source comparison.
|
||
|
||
Steps:
|
||
1. Unicode NFC normalization
|
||
2. Lowercase
|
||
3. Strip punctuation except hyphens (preserve 'пр-кт', 'б-р', 'пр-д' for expansion)
|
||
4. Collapse whitespace
|
||
5. Expand common Russian address abbreviations (hyphenated forms first)
|
||
6. Collapse remaining hyphens to spaces
|
||
"""
|
||
if not text:
|
||
return ""
|
||
s = unicodedata.normalize('NFC', text).lower()
|
||
s = _PUNCT.sub(' ', s) # strip punctuation EXCEPT hyphens
|
||
s = _WS.sub(' ', s).strip()
|
||
# Wrap with spaces for clean boundary matching
|
||
s = f' {s} '
|
||
for short, full in _ABBREV:
|
||
s = s.replace(short, full)
|
||
# Collapse remaining hyphens (e.g. standalone '-' separators or numeric ranges)
|
||
s = s.replace('-', ' ')
|
||
return _WS.sub(' ', s).strip()
|
||
|
||
|
||
def address_fingerprint(address: str | None, lat: float | None, lon: float | None) -> str:
|
||
"""SHA-256 fingerprint of normalized address + rounded coordinates (4 dp ≈ 11 m).
|
||
|
||
Returns first 32 hex chars of the digest (128 bits — collision-safe for millions of rows).
|
||
"""
|
||
norm_addr = normalize_address(address or '')
|
||
lat_r = f'{lat:.4f}' if lat is not None else ''
|
||
lon_r = f'{lon:.4f}' if lon is not None else ''
|
||
key = f'{norm_addr}|{lat_r}|{lon_r}'
|
||
return hashlib.sha256(key.encode('utf-8')).hexdigest()[:32]
|