Light1YT
86e9ea2937
Многоагентный аудит + имплементация: один воркер на файл, точечные правки. Верификация: py_compile (47/47 .py) + tsc --noEmit (0 ошибок). Unit-тесты не прогонялись (окружение не поднято: rollup native dep / нет pytest-venv). Полностью исправлено (169): #1336, #1337, #1339, #1340, #1341, #1342, #1343, #1345, #1346, #1348, #1349, #1350, #1351, #1354, #1356, #1358, #1359, #1360, #1362, #1364, #1365, #1366, #1367, #1368, #1369, #1370, #1371, #1372, #1373, #1374, #1375, #1376, #1377, #1378, #1379, #1380, #1381, #1382, #1384, #1385, #1386, #1387, #1388, #1389, #1390, #1391, #1392, #1394, #1395, #1396, #1397, #1399, #1400, #1401, #1402, #1403, #1404, #1408, #1409, #1410, #1411, #1412, #1413, #1414, #1415, #1416, #1417, #1418, #1420, #1423, #1425, #1426, #1427, #1428, #1429, #1430, #1431, #1432, #1433, #1434, #1435, #1437, #1438, #1439, #1440, #1441, #1442, #1443, #1444, #1445, #1446, #1447, #1448, #1449, #1450, #1451, #1452, #1453, #1454, #1455, #1456, #1457, #1458, #1459, #1460, #1461, #1462, #1463, #1464, #1465, #1466, #1467, #1468, #1469, #1471, #1472, #1473, #1474, #1476, #1478, #1479, #1481, #1482, #1483, #1484, #1485, #1487, #1488, #1489, #1490, #1491, #1492, #1493, #1494, #1495, #1496, #1497, #1499, #1500, #1501, #1502, #1504, #1505, #1506, #1507, #1510, #1514, #1515, #1516, #1517, #1518, #1519, #1521, #1522, #1523, #1524, #1525, #1526, #1527, #1528, #1529, #1531, #1532, #1533, #1534, #1535, #1536, #1537, #1538 Частично (9, in-file часть, остаток cross-file): #1361, #1419, #1422, #1424, #1470, #1475, #1477, #1480, #1498 Требуют cross-file (3, не тронуты): #1338, #1363, #1421 Пропущено (1): #1539 Не входило в партию: 22 needs-Leha issue (нужны решения владельца). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
216 lines
8.7 KiB
Python
216 lines
8.7 KiB
Python
"""Per-field source priority for cross-source canonical merge.
|
|
|
|
Direct port of Cross_Source_Matching_Strategy.md sec 3.5 (houses) + 4.5 (listings).
|
|
Source-of-truth dicts read by merge logic in match_or_create_house/listing.
|
|
|
|
Sources covered: avito (serp/detail/houses_catalog/domoteka/imv),
|
|
cian (serp/bti/detail/stats/valuation), yandex (serp/detail/realty_nb/valuation).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
from typing import Any
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# cross_validate: warn if spread between sources exceeds this fraction of the median.
|
|
_CROSS_VALIDATE_DIVERGENCE_THRESHOLD = 0.10
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# HOUSE_FIELD_PRIORITY — vault sec 3.5
|
|
# ---------------------------------------------------------------------------
|
|
HOUSE_FIELD_PRIORITY: dict[str, list[str] | str] = {
|
|
"address": ["cian", "avito", "yandex"],
|
|
"lat": ["cian_serp", "avito_houses_catalog", "yandex_realty_nb"],
|
|
"lon": ["cian_serp", "avito_houses_catalog", "yandex_realty_nb"],
|
|
"year_built": [
|
|
"cian_bti",
|
|
"cian_serp",
|
|
"avito_houses_catalog",
|
|
"yandex_valuation",
|
|
"yandex_realty_nb",
|
|
],
|
|
"house_type": ["cian_bti", "cian_serp", "avito", "yandex_valuation", "yandex_realty_nb"],
|
|
"series_name": ["cian_bti"],
|
|
"passenger_lifts_count": ["cian", "avito"],
|
|
"cargo_lifts_count": ["cian", "avito"],
|
|
"has_concierge": ["cian", "avito"],
|
|
"closed_yard": ["cian", "avito"],
|
|
"parking_type": ["cian"],
|
|
"flat_count": ["cian_bti"],
|
|
"entrances": ["cian_bti"],
|
|
"is_emergency": ["cian_bti"],
|
|
"management_company_id": ["cian_valuation"],
|
|
"house_class": ["avito_houses_catalog", "cian", "yandex_realty_nb"],
|
|
"rating_score": ["avito_houses_catalog", "cian", "yandex_realty_nb"],
|
|
"reviews_count": ["avito_houses_catalog", "cian", "yandex_realty_nb"],
|
|
# Yandex unique fields (newbuilding landing only)
|
|
"text_reviews_count": ["yandex_realty_nb"], # 353 text reviews — Yandex strongpoint
|
|
"corpus_count": ["yandex_realty_nb"], # "три башни" → 3
|
|
"total_area_ha": ["yandex_realty_nb"], # ЖК footprint
|
|
"commission_year": ["cian_serp", "yandex_realty_nb"],
|
|
"commission_month": ["yandex_realty_nb"], # raw RU month name
|
|
"developer_name": ["cian", "yandex_realty_nb"],
|
|
"has_panorama": ["yandex_valuation"], # Yandex 3D panorama flag
|
|
"yandex_total_listings": ["yandex_valuation"], # "N объектов" в истории
|
|
# Yandex Valuation enrichment (existing house attrs)
|
|
"has_lift": ["cian_bti", "cian_detail", "yandex_valuation"],
|
|
"ceiling_height": ["cian_detail", "yandex_valuation"],
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# LISTING_FIELD_PRIORITY — vault sec 4.5
|
|
# ---------------------------------------------------------------------------
|
|
LISTING_FIELD_PRIORITY: dict[str, list[str] | str] = {
|
|
"address": ["cian_serp", "avito_detail", "avito_serp"],
|
|
"lat": ["cian_serp", "avito_detail"],
|
|
"lon": ["cian_serp", "avito_detail"],
|
|
"area_m2": ["cian_serp", "avito_detail"],
|
|
"living_area_m2": ["cian_serp"],
|
|
"kitchen_area_m2": ["cian_serp", "avito_detail"],
|
|
"ceiling_height": ["cian_detail"],
|
|
"floor": ["cian_serp", "avito_detail"],
|
|
"total_floors": ["cian_serp", "avito_detail"],
|
|
"year_built": ["cian_serp"],
|
|
"house_type": ["cian_serp", "avito_detail", "yandex_detail"],
|
|
"repair_state": ["cian_detail", "avito_detail"],
|
|
"has_balcony": ["cian_serp", "avito_detail"],
|
|
"balconies_count": ["cian_serp"],
|
|
"loggias_count": ["cian_serp"],
|
|
"windows_view_type": ["cian_detail", "avito_detail"],
|
|
"separate_wcs_count": ["cian_detail"],
|
|
"combined_wcs_count": ["cian_detail"],
|
|
"room_type": ["cian_detail", "avito_detail"],
|
|
"has_furniture": ["cian_serp", "avito_detail"],
|
|
"phones": ["cian_serp"],
|
|
"description": ["cian_serp", "avito_detail", "yandex_detail"],
|
|
"photo_urls": "union",
|
|
# Avito Domoteka unique
|
|
"owners_count": ["avito_domoteka"],
|
|
"owners_at_least": ["avito_domoteka"],
|
|
"last_owner_change_date": ["avito_domoteka"],
|
|
"encumbrances_clean": ["avito_domoteka"],
|
|
"registry_match": ["avito_domoteka"],
|
|
# Cian-only
|
|
"is_rosreestr_checked": ["cian_serp"],
|
|
"is_layout_approved": ["cian_serp"],
|
|
"is_commercial_ownership_verified": ["cian_serp"],
|
|
# Cross-validation
|
|
"price_rub": "cross_validate",
|
|
"kadastr_num": "first_non_null",
|
|
# NOTE: task description claims price_rub=['cian','avito','yandex']; vault sec 4.5
|
|
# says 'cross_validate' (flag if diff > 10%). Following vault — change to list if
|
|
# cross_validate semantics aren't desired at caller.
|
|
# Yandex unique (agency block — OfferCardAuthorInfo)
|
|
"agency_name": ["yandex_detail"],
|
|
"agency_founded_year": ["yandex_detail"],
|
|
"agency_objects_count": ["yandex_detail"],
|
|
# Yandex parallel views column (NOT existing `views_total` which is Cian's)
|
|
"views_total_yandex": ["yandex_detail"],
|
|
# Yandex raw publish-date text (relative form)
|
|
"publish_date_relative": ["yandex_detail"],
|
|
}
|
|
|
|
|
|
def _resolve(
|
|
priority: dict[str, list[str] | str],
|
|
field: str,
|
|
candidates: dict[str, Any],
|
|
) -> Any:
|
|
"""Pick value from candidates per priority dict semantics."""
|
|
if not candidates:
|
|
return None
|
|
rule = priority.get(field, "first_non_null")
|
|
|
|
if isinstance(rule, str):
|
|
if rule == "union":
|
|
out: list[Any] = []
|
|
for v in candidates.values():
|
|
if v is None:
|
|
continue
|
|
if isinstance(v, list | tuple | set):
|
|
out.extend(v)
|
|
else:
|
|
out.append(v)
|
|
# Preserve order, dedup
|
|
seen: set[Any] = set()
|
|
uniq: list[Any] = []
|
|
for x in out:
|
|
key = repr(x)
|
|
if key in seen:
|
|
continue
|
|
seen.add(key)
|
|
uniq.append(x)
|
|
return uniq
|
|
if rule == "first_non_null":
|
|
for v in candidates.values():
|
|
if v is not None:
|
|
return v
|
|
return None
|
|
if rule == "max":
|
|
non_null = [v for v in candidates.values() if v is not None]
|
|
return max(non_null) if non_null else None
|
|
if rule == "min":
|
|
non_null = [v for v in candidates.values() if v is not None]
|
|
return min(non_null) if non_null else None
|
|
if rule == "cross_validate":
|
|
# Return the median; flag divergence (vault sec 4.5: warn if diff > 10%).
|
|
nums = [v for v in candidates.values() if isinstance(v, int | float)]
|
|
if not nums:
|
|
return None
|
|
nums.sort()
|
|
n = len(nums)
|
|
if n % 2:
|
|
median = nums[n // 2]
|
|
else:
|
|
# True median for even n: mean of the two central values.
|
|
median = (nums[n // 2 - 1] + nums[n // 2]) / 2
|
|
if median and (max(nums) - min(nums)) > _CROSS_VALIDATE_DIVERGENCE_THRESHOLD * abs(
|
|
median
|
|
):
|
|
logger.warning(
|
|
"cross_validate divergence for field %r: min=%s max=%s median=%s "
|
|
"(spread exceeds %.0f%% of median); candidates=%r",
|
|
field,
|
|
min(nums),
|
|
max(nums),
|
|
median,
|
|
_CROSS_VALIDATE_DIVERGENCE_THRESHOLD * 100,
|
|
candidates,
|
|
)
|
|
return median
|
|
# Unknown rule
|
|
return next((v for v in candidates.values() if v is not None), None)
|
|
|
|
# Priority list: pick value from highest-ranked source present (non-null)
|
|
for src in rule:
|
|
if src in candidates and candidates[src] is not None:
|
|
return candidates[src]
|
|
# Fallback: any non-null
|
|
return next((v for v in candidates.values() if v is not None), None)
|
|
|
|
|
|
def resolve_house_field(field: str, candidates: dict[str, Any]) -> Any:
|
|
"""Pick canonical house field value per HOUSE_FIELD_PRIORITY."""
|
|
return _resolve(HOUSE_FIELD_PRIORITY, field, candidates)
|
|
|
|
|
|
def resolve_listing_field(field: str, candidates: dict[str, Any]) -> Any:
|
|
"""Pick canonical listing field value per LISTING_FIELD_PRIORITY."""
|
|
return _resolve(LISTING_FIELD_PRIORITY, field, candidates)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Legacy stub — kept for backward compat with existing __init__.py and tests
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def update_canonical_fields(
|
|
db: Any,
|
|
listing_id: int,
|
|
ext_source: str,
|
|
lot_data: object,
|
|
) -> None:
|
|
"""Legacy Stage 8 v1 stub — full arbitration deferred to Stage 8.x."""
|
|
pass
|