bot-backend
fa2bd20ac6
Some checks failed
Deploy Trade-In / build-backend (push) Blocked by required conditions
Deploy Trade-In / deploy (push) Blocked by required conditions
Deploy Trade-In / changes (push) Successful in 4s
Deploy Trade-In / build-frontend (push) Has been skipped
Deploy Trade-In / test (push) Has been cancelled
Co-authored-by: bot-backend <bot-backend@gendsgn.local> Co-committed-by: bot-backend <bot-backend@gendsgn.local>
191 lines
7.7 KiB
Python
191 lines
7.7 KiB
Python
"""Per-field source priority for cross-source canonical merge.
|
|
|
|
Direct port of Cross_Source_Matching_Strategy.md sec 3.5 (houses) + 4.5 (listings).
|
|
Source-of-truth dicts read by merge logic in match_or_create_house/listing.
|
|
|
|
Sources covered: avito (serp/detail/houses_catalog/domoteka/imv),
|
|
cian (serp/bti/detail/stats/valuation), yandex (serp/detail/realty_nb/valuation).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import Any
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# HOUSE_FIELD_PRIORITY — vault sec 3.5
|
|
# ---------------------------------------------------------------------------
|
|
HOUSE_FIELD_PRIORITY: dict[str, list[str] | str] = {
|
|
"address": ["cian", "avito", "yandex"],
|
|
"lat": ["cian_serp", "avito_houses_catalog", "yandex_realty_nb"],
|
|
"lon": ["cian_serp", "avito_houses_catalog", "yandex_realty_nb"],
|
|
"year_built": [
|
|
"cian_bti",
|
|
"cian_serp",
|
|
"avito_houses_catalog",
|
|
"yandex_valuation",
|
|
"yandex_realty_nb",
|
|
],
|
|
"house_type": ["cian_bti", "cian_serp", "avito", "yandex_valuation", "yandex_realty_nb"],
|
|
"series_name": ["cian_bti"],
|
|
"passenger_lifts_count": ["cian", "avito"],
|
|
"cargo_lifts_count": ["cian", "avito"],
|
|
"has_concierge": ["cian", "avito"],
|
|
"closed_yard": ["cian", "avito"],
|
|
"parking_type": ["cian"],
|
|
"flat_count": ["cian_bti"],
|
|
"entrances": ["cian_bti"],
|
|
"is_emergency": ["cian_bti"],
|
|
"management_company_id": ["cian_valuation"],
|
|
"house_class": ["avito_houses_catalog", "cian", "yandex_realty_nb"],
|
|
"rating_score": ["avito_houses_catalog", "cian", "yandex_realty_nb"],
|
|
"reviews_count": ["avito_houses_catalog", "cian", "yandex_realty_nb"],
|
|
# Yandex unique fields (newbuilding landing only)
|
|
"text_reviews_count": ["yandex_realty_nb"], # 353 text reviews — Yandex strongpoint
|
|
"corpus_count": ["yandex_realty_nb"], # "три башни" → 3
|
|
"total_area_ha": ["yandex_realty_nb"], # ЖК footprint
|
|
"commission_year": ["cian_serp", "yandex_realty_nb"],
|
|
"commission_month": ["yandex_realty_nb"], # raw RU month name
|
|
"developer_name": ["cian", "yandex_realty_nb"],
|
|
"has_panorama": ["yandex_valuation"], # Yandex 3D panorama flag
|
|
"yandex_total_listings": ["yandex_valuation"], # "N объектов" в истории
|
|
# Yandex Valuation enrichment (existing house attrs)
|
|
"has_lift": ["cian_bti", "cian_detail", "yandex_valuation"],
|
|
"ceiling_height": ["cian_detail", "yandex_valuation"],
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# LISTING_FIELD_PRIORITY — vault sec 4.5
|
|
# ---------------------------------------------------------------------------
|
|
LISTING_FIELD_PRIORITY: dict[str, list[str] | str] = {
|
|
"address": ["cian_serp", "avito_detail", "avito_serp"],
|
|
"lat": ["cian_serp", "avito_detail"],
|
|
"lon": ["cian_serp", "avito_detail"],
|
|
"area_m2": ["cian_serp", "avito_detail"],
|
|
"living_area_m2": ["cian_serp"],
|
|
"kitchen_area_m2": ["cian_serp", "avito_detail"],
|
|
"ceiling_height": ["cian_detail"],
|
|
"floor": ["cian_serp", "avito_detail"],
|
|
"total_floors": ["cian_serp", "avito_detail"],
|
|
"year_built": ["cian_serp"],
|
|
"house_type": ["cian_serp", "avito_detail", "yandex_detail"],
|
|
"repair_state": ["cian_detail", "avito_detail"],
|
|
"has_balcony": ["cian_serp", "avito_detail"],
|
|
"balconies_count": ["cian_serp"],
|
|
"loggias_count": ["cian_serp"],
|
|
"windows_view_type": ["cian_detail", "avito_detail"],
|
|
"separate_wcs_count": ["cian_detail"],
|
|
"combined_wcs_count": ["cian_detail"],
|
|
"room_type": ["cian_detail", "avito_detail"],
|
|
"has_furniture": ["cian_serp", "avito_detail"],
|
|
"phones": ["cian_serp"],
|
|
"description": ["cian_serp", "avito_detail", "yandex_detail"],
|
|
"photo_urls": "union",
|
|
# Avito Domoteka unique
|
|
"owners_count": ["avito_domoteka"],
|
|
"owners_at_least": ["avito_domoteka"],
|
|
"last_owner_change_date": ["avito_domoteka"],
|
|
"encumbrances_clean": ["avito_domoteka"],
|
|
"registry_match": ["avito_domoteka"],
|
|
# Cian-only
|
|
"is_rosreestr_checked": ["cian_serp"],
|
|
"is_layout_approved": ["cian_serp"],
|
|
"is_commercial_ownership_verified": ["cian_serp"],
|
|
# Cross-validation
|
|
"price_rub": "cross_validate",
|
|
"kadastr_num": "first_non_null",
|
|
# NOTE: task description claims price_rub=['cian','avito','yandex']; vault sec 4.5
|
|
# says 'cross_validate' (flag if diff > 10%). Following vault — change to list if
|
|
# cross_validate semantics aren't desired at caller.
|
|
# Yandex unique (agency block — OfferCardAuthorInfo)
|
|
"agency_name": ["yandex_detail"],
|
|
"agency_founded_year": ["yandex_detail"],
|
|
"agency_objects_count": ["yandex_detail"],
|
|
# Yandex parallel views column (NOT existing `views_total` which is Cian's)
|
|
"views_total_yandex": ["yandex_detail"],
|
|
# Yandex raw publish-date text (relative form)
|
|
"publish_date_relative": ["yandex_detail"],
|
|
}
|
|
|
|
|
|
def _resolve(
|
|
priority: dict[str, list[str] | str],
|
|
field: str,
|
|
candidates: dict[str, Any],
|
|
) -> Any:
|
|
"""Pick value from candidates per priority dict semantics."""
|
|
if not candidates:
|
|
return None
|
|
rule = priority.get(field, "first_non_null")
|
|
|
|
if isinstance(rule, str):
|
|
if rule == "union":
|
|
out: list[Any] = []
|
|
for v in candidates.values():
|
|
if v is None:
|
|
continue
|
|
if isinstance(v, list | tuple | set):
|
|
out.extend(v)
|
|
else:
|
|
out.append(v)
|
|
# Preserve order, dedup
|
|
seen: set[Any] = set()
|
|
uniq: list[Any] = []
|
|
for x in out:
|
|
key = repr(x)
|
|
if key in seen:
|
|
continue
|
|
seen.add(key)
|
|
uniq.append(x)
|
|
return uniq
|
|
if rule == "first_non_null":
|
|
for v in candidates.values():
|
|
if v is not None:
|
|
return v
|
|
return None
|
|
if rule == "max":
|
|
non_null = [v for v in candidates.values() if v is not None]
|
|
return max(non_null) if non_null else None
|
|
if rule == "min":
|
|
non_null = [v for v in candidates.values() if v is not None]
|
|
return min(non_null) if non_null else None
|
|
if rule == "cross_validate":
|
|
# Caller decides — return median by default
|
|
nums = [v for v in candidates.values() if isinstance(v, int | float)]
|
|
if not nums:
|
|
return None
|
|
nums.sort()
|
|
return nums[len(nums) // 2]
|
|
# Unknown rule
|
|
return next((v for v in candidates.values() if v is not None), None)
|
|
|
|
# Priority list: pick value from highest-ranked source present (non-null)
|
|
for src in rule:
|
|
if src in candidates and candidates[src] is not None:
|
|
return candidates[src]
|
|
# Fallback: any non-null
|
|
return next((v for v in candidates.values() if v is not None), None)
|
|
|
|
|
|
def resolve_house_field(field: str, candidates: dict[str, Any]) -> Any:
|
|
"""Pick canonical house field value per HOUSE_FIELD_PRIORITY."""
|
|
return _resolve(HOUSE_FIELD_PRIORITY, field, candidates)
|
|
|
|
|
|
def resolve_listing_field(field: str, candidates: dict[str, Any]) -> Any:
|
|
"""Pick canonical listing field value per LISTING_FIELD_PRIORITY."""
|
|
return _resolve(LISTING_FIELD_PRIORITY, field, candidates)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Legacy stub — kept for backward compat with existing __init__.py and tests
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def update_canonical_fields(
|
|
db: Any,
|
|
listing_id: int,
|
|
ext_source: str,
|
|
lot_data: object,
|
|
) -> None:
|
|
"""Legacy Stage 8 v1 stub — full arbitration deferred to Stage 8.x."""
|
|
pass
|