Stage 7 of YandexRealtyScraper v1 (Wave 5). Integrates the 4 Yandex scrapers
(yandex / yandex_detail / yandex_realty_nb / yandex_valuation) into the
cross-source field-priority arbitration.
HOUSE_FIELD_PRIORITY:
- Renamed legacy 'yandex' → 'yandex_realty_nb' for lat/lon (house geo comes
from ЖК landing, not SERP)
- Added yandex_valuation + yandex_realty_nb to year_built, house_type
- Added yandex_realty_nb to house_class, rating_score, reviews_count
- NEW fields:
* text_reviews_count: ['yandex_realty_nb'] (the 353 — Yandex unique)
* corpus_count: ['yandex_realty_nb'] ('три башни' → 3)
* total_area_ha: ['yandex_realty_nb']
* commission_year: ['cian_serp', 'yandex_realty_nb']
* commission_month: ['yandex_realty_nb']
* developer_name: ['cian', 'yandex_realty_nb']
* has_panorama: ['yandex_valuation']
* yandex_total_listings: ['yandex_valuation']
* has_lift: ['cian_bti', 'cian_detail', 'yandex_valuation']
* ceiling_height: ['cian_detail', 'yandex_valuation']
LISTING_FIELD_PRIORITY:
- Added yandex_detail to description, house_type (NLP fallback)
- NEW fields:
* agency_name / agency_founded_year / agency_objects_count: ['yandex_detail']
(OfferCardAuthorInfo — Yandex unique structured agency block)
* views_total_yandex: ['yandex_detail'] (parallel to Cian's views_total)
* publish_date_relative: ['yandex_detail']
* sale_type_text: ['yandex_detail'] (raw RU phrase, vs Cian enum)
Tests: 30 new cases (37 total), all pre-existing tests preserved. Ruff clean.
Resolver engine unchanged (existing list / first_non_null / union / cross_validate
semantics still apply).
230 lines
8.7 KiB
Python
230 lines
8.7 KiB
Python
"""Per-field priority resolution tests."""
|
|
from app.services.matching.conflict_resolution import (
|
|
HOUSE_FIELD_PRIORITY,
|
|
LISTING_FIELD_PRIORITY,
|
|
resolve_house_field,
|
|
resolve_listing_field,
|
|
)
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Pre-existing tests (preserved)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def test_house_year_built_prefers_cian_bti() -> None:
|
|
out = resolve_house_field(
|
|
"year_built",
|
|
{"avito_houses_catalog": 2020, "cian_bti": 2021, "cian_serp": 2022},
|
|
)
|
|
assert out == 2021
|
|
|
|
|
|
def test_house_unknown_field_first_non_null() -> None:
|
|
assert resolve_house_field("totally_unknown", {"x": None, "y": 5}) == 5
|
|
|
|
|
|
def test_listing_owners_count_avito_domoteka_only() -> None:
|
|
out = resolve_listing_field("owners_count", {"avito_domoteka": 2, "cian_serp": 99})
|
|
assert out == 2
|
|
|
|
|
|
def test_listing_photo_urls_union() -> None:
|
|
out = resolve_listing_field(
|
|
"photo_urls", {"avito": ["a.jpg", "b.jpg"], "cian": ["b.jpg", "c.jpg"]},
|
|
)
|
|
assert set(out) == {"a.jpg", "b.jpg", "c.jpg"}
|
|
|
|
|
|
def test_listing_kadastr_first_non_null() -> None:
|
|
out = resolve_listing_field("kadastr_num", {"avito": None, "cian": "66:1:1:1"})
|
|
assert out == "66:1:1:1"
|
|
|
|
|
|
def test_house_priority_dict_has_year_built() -> None:
|
|
assert "cian_bti" in HOUSE_FIELD_PRIORITY["year_built"]
|
|
|
|
|
|
def test_listing_priority_dict_has_owners() -> None:
|
|
assert "avito_domoteka" in LISTING_FIELD_PRIORITY["owners_count"]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Yandex house priority tests
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestYandexHousePriority:
|
|
def test_house_lat_yandex_realty_nb_when_cian_missing(self) -> None:
|
|
out = resolve_house_field("lat", {"yandex_realty_nb": 56.85})
|
|
assert out == 56.85
|
|
|
|
def test_house_lat_cian_preferred_over_yandex(self) -> None:
|
|
out = resolve_house_field(
|
|
"lat", {"cian_serp": 56.83, "yandex_realty_nb": 56.85}
|
|
)
|
|
assert out == 56.83
|
|
|
|
def test_house_year_built_yandex_valuation_picked(self) -> None:
|
|
out = resolve_house_field("year_built", {"yandex_valuation": 1981})
|
|
assert out == 1981
|
|
|
|
def test_house_year_built_cian_bti_preferred(self) -> None:
|
|
out = resolve_house_field(
|
|
"year_built", {"cian_bti": 1980, "yandex_valuation": 1981}
|
|
)
|
|
assert out == 1980
|
|
|
|
def test_house_text_reviews_count_yandex_only(self) -> None:
|
|
out = resolve_house_field("text_reviews_count", {"yandex_realty_nb": 353})
|
|
assert out == 353
|
|
|
|
def test_house_corpus_count_yandex_only(self) -> None:
|
|
out = resolve_house_field("corpus_count", {"yandex_realty_nb": 3})
|
|
assert out == 3
|
|
|
|
def test_house_commission_year_cian_serp_preferred(self) -> None:
|
|
out = resolve_house_field(
|
|
"commission_year", {"cian_serp": 2022, "yandex_realty_nb": 2023}
|
|
)
|
|
assert out == 2022
|
|
|
|
def test_house_commission_month_yandex_only(self) -> None:
|
|
out = resolve_house_field("commission_month", {"yandex_realty_nb": "июнь"})
|
|
assert out == "июнь"
|
|
|
|
def test_house_developer_name_cian_preferred(self) -> None:
|
|
out = resolve_house_field(
|
|
"developer_name",
|
|
{"cian": "PRINZIP", "yandex_realty_nb": "PRINZIP недвижимость"},
|
|
)
|
|
assert out == "PRINZIP"
|
|
|
|
def test_house_has_lift_cian_bti_preferred_over_yandex_valuation(self) -> None:
|
|
out = resolve_house_field(
|
|
"has_lift", {"cian_bti": True, "yandex_valuation": True}
|
|
)
|
|
assert out is True
|
|
|
|
def test_house_ceiling_height_cian_detail_preferred(self) -> None:
|
|
out = resolve_house_field(
|
|
"ceiling_height", {"cian_detail": 2.7, "yandex_valuation": 2.5}
|
|
)
|
|
assert out == 2.7
|
|
|
|
def test_house_has_panorama_yandex_valuation_only(self) -> None:
|
|
out = resolve_house_field("has_panorama", {"yandex_valuation": True})
|
|
assert out is True
|
|
|
|
def test_house_yandex_total_listings_yandex_valuation_only(self) -> None:
|
|
out = resolve_house_field("yandex_total_listings", {"yandex_valuation": 42})
|
|
assert out == 42
|
|
|
|
def test_house_house_class_yandex_realty_nb_fallback(self) -> None:
|
|
out = resolve_house_field("house_class", {"yandex_realty_nb": "бизнес"})
|
|
assert out == "бизнес"
|
|
|
|
def test_house_house_class_avito_preferred_over_yandex(self) -> None:
|
|
out = resolve_house_field(
|
|
"house_class",
|
|
{"avito_houses_catalog": "комфорт", "yandex_realty_nb": "бизнес"},
|
|
)
|
|
assert out == "комфорт"
|
|
|
|
def test_house_total_area_ha_yandex_only(self) -> None:
|
|
out = resolve_house_field("total_area_ha", {"yandex_realty_nb": 12.5})
|
|
assert out == 12.5
|
|
|
|
def test_house_has_lift_yandex_valuation_fallback_when_cian_missing(self) -> None:
|
|
out = resolve_house_field("has_lift", {"yandex_valuation": False})
|
|
assert out is False
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Yandex listing priority tests
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestYandexListingPriority:
|
|
def test_listing_description_cian_preferred_over_yandex_detail(self) -> None:
|
|
out = resolve_listing_field(
|
|
"description",
|
|
{"cian_serp": "cian text", "yandex_detail": "yandex text"},
|
|
)
|
|
assert out == "cian text"
|
|
|
|
def test_listing_house_type_yandex_detail_used_when_cian_avito_missing(self) -> None:
|
|
out = resolve_listing_field("house_type", {"yandex_detail": "brick"})
|
|
assert out == "brick"
|
|
|
|
def test_listing_agency_name_yandex_only(self) -> None:
|
|
out = resolve_listing_field(
|
|
"agency_name", {"yandex_detail": "Агентство «Диал»"}
|
|
)
|
|
assert out == "Агентство «Диал»"
|
|
|
|
def test_listing_agency_founded_year_yandex_only(self) -> None:
|
|
out = resolve_listing_field("agency_founded_year", {"yandex_detail": 2005})
|
|
assert out == 2005
|
|
|
|
def test_listing_agency_objects_count_yandex_only(self) -> None:
|
|
out = resolve_listing_field("agency_objects_count", {"yandex_detail": 120})
|
|
assert out == 120
|
|
|
|
def test_listing_views_total_yandex_only(self) -> None:
|
|
out = resolve_listing_field("views_total_yandex", {"yandex_detail": 874})
|
|
assert out == 874
|
|
|
|
def test_listing_publish_date_relative_yandex_only(self) -> None:
|
|
out = resolve_listing_field(
|
|
"publish_date_relative", {"yandex_detail": "3 дня назад"}
|
|
)
|
|
assert out == "3 дня назад"
|
|
|
|
def test_listing_sale_type_text_yandex_only(self) -> None:
|
|
out = resolve_listing_field(
|
|
"sale_type_text", {"yandex_detail": "Прямая продажа"}
|
|
)
|
|
assert out == "Прямая продажа"
|
|
|
|
def test_listing_agency_name_none_when_no_yandex(self) -> None:
|
|
out = resolve_listing_field("agency_name", {"cian_serp": None})
|
|
assert out is None
|
|
|
|
def test_listing_description_avito_detail_preferred_over_yandex(self) -> None:
|
|
out = resolve_listing_field(
|
|
"description",
|
|
{"avito_detail": "avito desc", "yandex_detail": "yandex desc"},
|
|
)
|
|
assert out == "avito desc"
|
|
|
|
def test_listing_house_type_cian_preferred_over_yandex_detail(self) -> None:
|
|
out = resolve_listing_field(
|
|
"house_type",
|
|
{"cian_serp": "панель", "yandex_detail": "кирпич"},
|
|
)
|
|
assert out == "панель"
|
|
|
|
def test_listing_yandex_detail_keys_registered(self) -> None:
|
|
"""Ensure all Yandex-unique listing keys are in priority dict."""
|
|
yandex_keys = [
|
|
"agency_name",
|
|
"agency_founded_year",
|
|
"agency_objects_count",
|
|
"views_total_yandex",
|
|
"publish_date_relative",
|
|
"sale_type_text",
|
|
]
|
|
for key in yandex_keys:
|
|
assert key in LISTING_FIELD_PRIORITY, f"{key!r} missing from LISTING_FIELD_PRIORITY"
|
|
rule = LISTING_FIELD_PRIORITY[key]
|
|
assert rule == ["yandex_detail"], f"{key!r} rule mismatch: {rule!r}"
|
|
|
|
def test_house_yandex_keys_registered(self) -> None:
|
|
"""Ensure all Yandex-unique house keys are in priority dict."""
|
|
yandex_keys = [
|
|
"text_reviews_count",
|
|
"corpus_count",
|
|
"total_area_ha",
|
|
"commission_month",
|
|
"has_panorama",
|
|
"yandex_total_listings",
|
|
]
|
|
for key in yandex_keys:
|
|
assert key in HOUSE_FIELD_PRIORITY, f"{key!r} missing from HOUSE_FIELD_PRIORITY"
|