diff --git a/tradein-mvp/backend/app/services/scrapers/yandex_valuation.py b/tradein-mvp/backend/app/services/scrapers/yandex_valuation.py index 425a01f0..34ce69f5 100644 --- a/tradein-mvp/backend/app/services/scrapers/yandex_valuation.py +++ b/tradein-mvp/backend/app/services/scrapers/yandex_valuation.py @@ -52,6 +52,35 @@ class ValuationHouseMeta(BaseModel): total_objects: int | None = None # 'N объектов' (full archive count) has_panorama: bool = False # 'Панорама' label present + def validate_match( + self, + expected_year_built: int | None = None, + expected_total_floors: int | None = None, + ) -> float: + """Return confidence 0..1 that this house meta matches expected values. + + Used after fetching valuation by address to detect when Yandex returned a + different house (ambiguous address geocoding). Tolerance ±1 year, ±1 floor. + + Both expected=None → 1.0 (no check). Mismatch on any dimension → 0.0 for it. + """ + score = 0.0 + checks = 0 + if expected_year_built is not None: + checks += 1 + if self.year_built is not None and abs(self.year_built - expected_year_built) <= 1: + score += 1.0 + if expected_total_floors is not None: + checks += 1 + if ( + self.total_floors is not None + and abs(self.total_floors - expected_total_floors) <= 1 + ): + score += 1.0 + if checks == 0: + return 1.0 + return score / checks + class ValuationHistoryItem(BaseModel): """One historical offer entry from the valuation page.""" @@ -64,6 +93,7 @@ class ValuationHistoryItem(BaseModel): last_price: int | None = None last_price_per_m2: int | None = None publish_date: date | None = None + removed_date: date | None = None # ← NEW exposure_days: int | None = None status: str | None = None # 'В продаже' / 'Снято' @@ -86,7 +116,7 @@ class YandexValuationResult(BaseModel): # --------------------------------------------------------------------------- RE_YEAR_BUILT = re.compile(r"Дом\s+(\d{4})\s+года", re.IGNORECASE) -RE_FLOORS = re.compile(r"(\d+)\s+этаж", re.IGNORECASE) +RE_FLOORS = re.compile(r"(\d+)\s+этажей", re.IGNORECASE) RE_CEILING = re.compile(r"([\d,]+)\s*м\s+потолки", re.IGNORECASE) RE_TOTAL_OBJECTS = re.compile(r"(\d+)\s+объект", re.IGNORECASE) @@ -98,9 +128,7 @@ RE_ITEM_EXPOSURE = re.compile(r"экспозиции\s+(\d+)\s+дн", re.IGNOREC RE_ITEM_STATUS = re.compile(r"(В\s+продаже|Снят[оа])", re.IGNORECASE) # Matches total-price tokens (rubles) — excludes per-m2 tokens by negative lookahead -_RE_PRICE_TOKEN = re.compile( - r"(?:\d[\d\s]*\d|\d)(?:[.,]\d+)?\s*(?:млн)?\s*₽(?!\s*за\s*м²)" -) +_RE_PRICE_TOKEN = re.compile(r"(?:\d[\d\s]*\d|\d)(?:[.,]\d+)?\s*(?:млн)?\s*₽(?!\s*за\s*м²)") _RE_PPM2_TOKEN = re.compile(r"\d[\d\s]*\s*₽\s*за\s*м²") @@ -126,9 +154,7 @@ class YandexValuationScraper(BaseScraper): super().__init__() self.request_delay_sec = get_scraper_delay(self.name) - async def fetch_around( - self, lat: float, lon: float, radius_m: int = 1000 - ) -> list: # type: ignore[override] + async def fetch_around(self, lat: float, lon: float, radius_m: int = 1000) -> list: # type: ignore[override] raise NotImplementedError( "YandexValuationScraper is address-based; use fetch_house_history() instead." ) @@ -164,9 +190,7 @@ class YandexValuationScraper(BaseScraper): logger.exception("yandex valuation fetch failed: %s", url) return None if response.status_code != 200: - logger.warning( - "yandex valuation returned %d for %s", response.status_code, url - ) + logger.warning("yandex valuation returned %d for %s", response.status_code, url) return None result = self.parse( response.text, @@ -189,9 +213,10 @@ class YandexValuationScraper(BaseScraper): source_url: str, ) -> YandexValuationResult: """Parse raw HTML into YandexValuationResult. Pure function — usable in unit tests.""" - tree = HTMLParser(html) + html_normalized = html.replace("\xa0", " ") + tree = HTMLParser(html_normalized) body = tree.body - body_text = body.text(strip=True) if body else "" + body_text = (body.text(strip=True) if body else "").replace("\xa0", " ") house = self._parse_house_meta(body_text) history_items = self._parse_history_items(tree, body_text) @@ -221,17 +246,13 @@ class YandexValuationScraper(BaseScraper): year_built=int(year_m.group(1)) if year_m else None, total_floors=int(floors_m.group(1)) if floors_m else None, house_type=parse_house_type(body_text), - ceiling_height=( - float(ceiling_m.group(1).replace(",", ".")) if ceiling_m else None - ), + ceiling_height=(float(ceiling_m.group(1).replace(",", ".")) if ceiling_m else None), has_lift="Лифт" in body_text, total_objects=int(objects_m.group(1)) if objects_m else None, has_panorama="Панорама" in body_text, ) - def _parse_history_items( - self, tree: HTMLParser, body_text: str - ) -> list[ValuationHistoryItem]: + def _parse_history_items(self, tree: HTMLParser, body_text: str) -> list[ValuationHistoryItem]: """Extract list of historical offer items using best available strategy. Strategy 1: CSS data-test containers (if Yandex exposes them). @@ -239,9 +260,7 @@ class YandexValuationScraper(BaseScraper): """ items: list[ValuationHistoryItem] = [] # Strategy 1: explicit data-test container - for container in tree.css( - '[data-test*="HistoryItem"], [data-test*="ValuationItem"]' - ): + for container in tree.css('[data-test*="HistoryItem"], [data-test*="ValuationItem"]'): item = self._parse_item_text(container.text(strip=True)) if item: items.append(item) @@ -284,7 +303,15 @@ class YandexValuationScraper(BaseScraper): start_ppm2 = parse_rub(ppm2_tokens[0]) if len(ppm2_tokens) >= 1 else None last_ppm2 = parse_rub(ppm2_tokens[1]) if len(ppm2_tokens) >= 2 else None - publish_date = parse_dmy(text) + # Extract ALL DD.MM.YYYY dates: first → publish_date, second → removed_date + date_matches = list(re.finditer(r"\d{2}\.\d{2}\.\d{4}", text)) + dates_parsed: list[date] = [] + for m in date_matches: + d = parse_dmy(m.group(0)) + if d is not None: + dates_parsed.append(d) + publish_date = dates_parsed[0] if dates_parsed else None + removed_date = dates_parsed[1] if len(dates_parsed) >= 2 else None expo_m = RE_ITEM_EXPOSURE.search(text) exposure_days = int(expo_m.group(1)) if expo_m else None @@ -304,6 +331,7 @@ class YandexValuationScraper(BaseScraper): last_price=last_price, last_price_per_m2=last_ppm2, publish_date=publish_date, + removed_date=removed_date, exposure_days=exposure_days, status=status, ) diff --git a/tradein-mvp/backend/tests/test_yandex_valuation.py b/tradein-mvp/backend/tests/test_yandex_valuation.py index 2de8afd8..8d878997 100644 --- a/tradein-mvp/backend/tests/test_yandex_valuation.py +++ b/tradein-mvp/backend/tests/test_yandex_valuation.py @@ -20,9 +20,7 @@ from app.services.scrapers.yandex_valuation import ( # Fixture helpers # --------------------------------------------------------------------------- -_HOUSE_META_BLOCK = ( - "12 объектов Дом 1981 года 9 этажей Панельное здание 2,50 м потолки Лифт" -) +_HOUSE_META_BLOCK = "12 объектов Дом 1981 года 9 этажей Панельное здание 2,50 м потолки Лифт" _ITEM_1 = ( "2-комнатная квартира 45,3 м² 4 этаж " @@ -52,9 +50,7 @@ def _make_full_html(body_content: str) -> str: return f"{body_content}" -FULL_FIXTURE_HTML = _make_full_html( - f"{_HOUSE_META_BLOCK}\n{_ITEM_1}\n{_ITEM_2}\n{_ITEM_3}" -) +FULL_FIXTURE_HTML = _make_full_html(f"{_HOUSE_META_BLOCK}\n{_ITEM_1}\n{_ITEM_2}\n{_ITEM_3}") # --------------------------------------------------------------------------- @@ -155,9 +151,7 @@ def test_parse_items_chunked_dedup(): duplicate_block = f"{_ITEM_1}\n{_ITEM_1}" items = YandexValuationScraper._parse_items_from_chunked_text(duplicate_block) # Both items reference 15.03.2023 + 45.3 m2 + floor 4 — must dedup to 1 - matching = [ - i for i in items if i.area_m2 == 45.3 and i.publish_date == date(2023, 3, 15) - ] + matching = [i for i in items if i.area_m2 == 45.3 and i.publish_date == date(2023, 3, 15)] assert len(matching) == 1 @@ -240,3 +234,106 @@ def test_parse_ceiling_height_comma(): text = "Дом 1999 года 5 этажей Кирпичное здание 2,70 м потолки Лифт" meta = YandexValuationScraper._parse_house_meta(text) assert meta.ceiling_height == 2.7 + + +# --------------------------------------------------------------------------- +# Regression tests — parser fixes (2026-05-24) +# --------------------------------------------------------------------------- + + +def test_parse_item_extracts_removed_date(): + """Two dates in chunk → first = publish_date, second = removed_date.""" + text = "2-комнатная квартира 50,0 м² 3 этаж 10.05.2024 В экспозиции 30 дней 09.06.2024" + item = YandexValuationScraper._parse_item_text(text) + assert item is not None + assert item.publish_date == date(2024, 5, 10) + assert item.removed_date == date(2024, 6, 9) + + +def test_parse_item_in_sale_has_no_removed_date(): + """Single date + 'В продаже' → removed_date is None.""" + text = "1-комнатная квартира 40 м² 5 этаж 10.05.2024 В экспозиции 30 дней В продаже" + item = YandexValuationScraper._parse_item_text(text) + assert item is not None + assert item.publish_date == date(2024, 5, 10) + assert item.removed_date is None + + +def test_total_floors_extracted_from_dom_meta_not_items(): + """Regression: 'N этаж' inside each item must NOT be matched as dom total_floors. + Real Yandex page has 'M этажей' (plural) in dom-meta and 'N этаж' (singular) per item. + """ + text = ( + "Дом 2025 года Панорама 25 этажей Монолитное здание 2,7 м потолки Лифт " + "1-комнатная 40 м² 3 этаж 10.01.2026 В экспозиции 5 дней В продаже " + "2-комнатная 55 м² 17 этаж 05.01.2026 В экспозиции 10 дней В продаже" + ) + meta = YandexValuationScraper._parse_house_meta(text) + assert meta.year_built == 2025 + assert meta.total_floors == 25 + assert meta.house_type == "monolith" + + +def test_parse_normalizes_nbsp_for_price_per_m2(): + """NBSP (\\xa0) in price tokens must not break price_per_m2 regex. + Real Yandex SSR HTML uses NBSP between digit groups and before ₽. + """ + html = ( + "" + "Дом 2020 года 16 этажей Монолитное здание " + "2-комнатная 50,0 м² 3 этаж " + "10.05.2024 В экспозиции 30 дней В продаже " + "5\xa0000\xa0000 ₽ 100\xa0000 ₽ за м² " + "4\xa0900\xa0000 ₽ 98\xa0000 ₽ за м² " + "" + ) + scraper = YandexValuationScraper() + result = scraper.parse( + html, + address="test", + offer_category="APARTMENT", + offer_type="SELL", + page=1, + source_url="http://test/", + ) + assert len(result.history_items) >= 1 + item = result.history_items[0] + assert item.start_price == 5_000_000 + assert item.start_price_per_m2 == 100_000 + + +def test_validate_match_full_match(): + from app.services.scrapers.yandex_valuation import ValuationHouseMeta + + meta = ValuationHouseMeta(year_built=2025, total_floors=25) + assert meta.validate_match(2025, 25) == 1.0 + + +def test_validate_match_year_mismatch(): + from app.services.scrapers.yandex_valuation import ValuationHouseMeta + + meta = ValuationHouseMeta(year_built=1984, total_floors=9) + assert meta.validate_match(2025, 25) == 0.0 + + +def test_validate_match_partial(): + from app.services.scrapers.yandex_valuation import ValuationHouseMeta + + meta = ValuationHouseMeta(year_built=2025, total_floors=10) + # year matches, floors mismatch (>1 tolerance) → 0.5 + assert meta.validate_match(2025, 25) == 0.5 + + +def test_validate_match_tolerance_one_year(): + from app.services.scrapers.yandex_valuation import ValuationHouseMeta + + meta = ValuationHouseMeta(year_built=2024, total_floors=25) + # ±1 year tolerance + assert meta.validate_match(2025, 25) == 1.0 + + +def test_validate_match_no_expected_returns_one(): + from app.services.scrapers.yandex_valuation import ValuationHouseMeta + + meta = ValuationHouseMeta(year_built=None, total_floors=None) + assert meta.validate_match(None, None) == 1.0