185e7c2e26
- normalize.py: normalize_address() + address_fingerprint() (SHA-256, 4dp coords) - houses.py: match_or_create_house() — tiers: cadastr(1.0) / source_exact(1.0) / fingerprint(0.9) / geo_proximity 30m(0.7) / new - listings.py: match_or_create_listing() — tiers: cadastr(1.0) / source_exact(1.0) / minhash(0.85) / composite floor+area±2%+rooms(0.75) / new - conflict_resolution.py: HOUSE/LISTING_FIELD_PRIORITY dicts + update_canonical_fields() stub (Stage 8 v1) - tests/test_matching.py: 28 unit tests (all pass), mock-DB tier routing coverage
64 lines
2.4 KiB
Python
64 lines
2.4 KiB
Python
"""Field priority rules for canonical record merging.
|
|
|
|
When multiple sources provide conflicting values for the same field,
|
|
these dicts define which source to prefer (highest-priority first).
|
|
|
|
Algorithm reference: decisions/Cross_Source_Matching_Strategy.md sec 4.5
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from sqlalchemy.orm import Session
|
|
|
|
# Higher-priority source listed first.
|
|
# For each column, prefer the first source in the list that has a non-NULL value.
|
|
|
|
HOUSE_FIELD_PRIORITY: dict[str, list[str]] = {
|
|
'year_built': ['cian', 'avito_houses', 'rosreestr'],
|
|
'address': ['avito', 'cian', 'rosreestr'],
|
|
'cadastral_number': ['rosreestr', 'cian', 'avito'],
|
|
'building_class': ['cian', 'avito_houses'],
|
|
'floors_count': ['cian', 'avito_houses'],
|
|
'series_name': ['cian'],
|
|
'entrances': ['cian'],
|
|
'flat_count': ['cian'],
|
|
}
|
|
|
|
LISTING_FIELD_PRIORITY: dict[str, list[str]] = {
|
|
# Price: always from latest-seen source wins among equal-priority sources.
|
|
'price_rub': ['avito', 'cian', 'yandex'],
|
|
'area_m2': ['rosreestr', 'cian', 'avito'],
|
|
'living_area_m2': ['cian', 'avito'],
|
|
'kitchen_area_m2': ['cian', 'avito'],
|
|
'floor': ['rosreestr', 'cian', 'avito'],
|
|
'rooms': ['rosreestr', 'cian', 'avito'],
|
|
'ceiling_height': ['cian', 'avito'],
|
|
'description': ['avito', 'cian'],
|
|
'phones': ['avito', 'cian'],
|
|
'photos': ['avito', 'cian'],
|
|
'cadastral_number': ['rosreestr', 'cian', 'avito'],
|
|
}
|
|
|
|
|
|
def update_canonical_fields(
|
|
db: Session,
|
|
listing_id: int,
|
|
ext_source: str,
|
|
lot_data: object,
|
|
) -> None:
|
|
"""Merge source fields into the canonical listings row.
|
|
|
|
Stage 8 v1: populate-NULL strategy only — write source value only when
|
|
the canonical column is currently NULL. Full priority arbitration with
|
|
conflict logging is planned for Stage 8.x when ≥2 sources are live.
|
|
|
|
Args:
|
|
db: Active SQLAlchemy session.
|
|
listing_id: Canonical listings.id.
|
|
ext_source: Source identifier (e.g. 'cian', 'avito').
|
|
lot_data: ScrapedLot or similar object with attribute access.
|
|
Unknown attributes are silently skipped.
|
|
"""
|
|
# Stage 8 v1 — callers handle their own field persistence via save_listings.
|
|
# This function is the integration point for Stage 8.x full arbitration.
|
|
pass
|