From e070f06c263c6e1cc48f7dcabe18aea583c637e1 Mon Sep 17 00:00:00 2001 From: lekss361 Date: Sun, 24 May 2026 19:20:20 +0300 Subject: [PATCH] refactor(yandex-valuation): DOM-based parser via .OffersArchiveSearchOffers__row MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Live HTML recon discovered stable BEM-style classes. Each history item is a .OffersArchiveSearchOffers__row with 6 positional cells: area+rooms, floor, start_price+ppm2, last_price+ppm2, publish_date+exposure, status/removed_date. Drops chunked-text fallback as primary strategy (kept as last-resort with warning log). Eliminates 3 PRs of regex whack-a-mole (#538/#541/#542): - area = year+area concat → impossible with cell isolation - area = sub-fragment '2,2' → impossible - date inversion → cell [4] is always publish, cell [5] is always removed - 1-комн/2-комн/3-комн all parsed identically Re-sweep Базовый 52 should now hit ~90%+ plausible area_filled vs 8.7% from PR #542. --- .../app/services/scrapers/yandex_valuation.py | 122 ++++++++++++++- .../backend/tests/test_yandex_valuation.py | 139 ++++++++++++++++++ 2 files changed, 254 insertions(+), 7 deletions(-) diff --git a/tradein-mvp/backend/app/services/scrapers/yandex_valuation.py b/tradein-mvp/backend/app/services/scrapers/yandex_valuation.py index 09b89f19..467dc8bd 100644 --- a/tradein-mvp/backend/app/services/scrapers/yandex_valuation.py +++ b/tradein-mvp/backend/app/services/scrapers/yandex_valuation.py @@ -258,22 +258,130 @@ class YandexValuationScraper(BaseScraper): ) def _parse_history_items(self, tree: HTMLParser, body_text: str) -> list[ValuationHistoryItem]: - """Extract list of historical offer items using best available strategy. + """Extract historical offer items. - Strategy 1: CSS data-test containers (if Yandex exposes them). - Strategy 2: Fallback — split body text into chunks around DD.MM.YYYY dates. + Strategy 1 (preferred, structured): each row at .OffersArchiveSearchOffers__row, + with 6 cells (.OffersArchiveSearchOffers__cell) in fixed order: + [0] area+rooms, [1] floor, [2] start_price+ppm2, [3] last_price+ppm2, + [4] publish_date+exposure, [5] status/removed_date. + Yields per-cell parsing — no chunked-text ambiguity. + + Strategy 2 (last-resort fallback): legacy chunked-text scan around + DD.MM.YYYY anchors. Kept for the rare case Yandex changes class names; + triggers a logger.warning so we notice. """ items: list[ValuationHistoryItem] = [] - # Strategy 1: explicit data-test container - for container in tree.css('[data-test*="HistoryItem"], [data-test*="ValuationItem"]'): - item = self._parse_item_text(container.text(strip=True)) + for row in tree.css(".OffersArchiveSearchOffers__row"): + item = self._parse_row_cells(row) if item: items.append(item) if items: return items - # Strategy 2: text-chunk fallback + # Fallback (only if structured selector failed) + logger.warning( + "yandex_valuation: .OffersArchiveSearchOffers__row matched 0 items — " + "falling back to chunked-text scan" + ) return self._parse_items_from_chunked_text(body_text) + @classmethod + def _parse_row_cells(cls, row) -> ValuationHistoryItem | None: + """Parse one .OffersArchiveSearchOffers__row into a ValuationHistoryItem. + + Cell layout is positional (no labels). Returns None if the row doesn't + match the expected 6-cell structure (defensive — Yandex may A/B-test). + """ + cells = row.css(".OffersArchiveSearchOffers__cell") + if len(cells) < 5: + return None + cell_texts = [(c.text(strip=True) or "").replace("\xa0", " ") for c in cells] + + # [0] area + rooms + area_text = cell_texts[0] + area_m = RE_ITEM_AREA.search(area_text) + area_m2: float | None = float(area_m.group(1).replace(",", ".")) if area_m else None + # Sanity drop (belt-and-suspenders from PR #538) + if area_m2 is not None and (area_m2 > 10_000 or area_m2 <= 0): + area_m2 = None + + rooms: int | None = None + if RE_ITEM_STUDIO.search(area_text): + rooms = 0 + else: + rooms_m = RE_ITEM_ROOMS.search(area_text) + if rooms_m: + rooms = int(rooms_m.group(1)) + + # [1] floor + floor: int | None = None + if len(cell_texts) > 1: + floor_m = RE_ITEM_FLOOR.search(cell_texts[1]) + if floor_m: + floor = int(floor_m.group(1)) + + # [2] start price + ppm2 — "5,1 млн ₽105 155 ₽ за м²" (no separator) + start_price: int | None = None + start_ppm2: int | None = None + if len(cell_texts) > 2: + tokens = _RE_PRICE_TOKEN.findall(cell_texts[2]) + if tokens: + start_price = parse_rub(tokens[0]) + ppm2_tokens = _RE_PPM2_TOKEN.findall(cell_texts[2]) + if ppm2_tokens: + start_ppm2 = parse_rub(ppm2_tokens[0]) + + # [3] last price + ppm2 + last_price: int | None = None + last_ppm2: int | None = None + if len(cell_texts) > 3: + tokens = _RE_PRICE_TOKEN.findall(cell_texts[3]) + if tokens: + last_price = parse_rub(tokens[0]) + ppm2_tokens = _RE_PPM2_TOKEN.findall(cell_texts[3]) + if ppm2_tokens: + last_ppm2 = parse_rub(ppm2_tokens[0]) + + # [4] publish_date + exposure ("23.10.2023В экспозиции 945 дней") + publish_date = None + exposure_days: int | None = None + if len(cell_texts) > 4: + publish_date = parse_dmy(cell_texts[4]) + expo_m = RE_ITEM_EXPOSURE.search(cell_texts[4]) + if expo_m: + exposure_days = int(expo_m.group(1)) + + # [5] status or removed_date + removed_date = None + status: str | None = None + if len(cell_texts) > 5: + last_cell = cell_texts[5] + status_m = RE_ITEM_STATUS.search(last_cell) + if status_m: + status = status_m.group(1) + removed_date = parse_dmy(last_cell) + + # Sort dates chronologically (in case Yandex flips them — same fix as PR #541) + if publish_date and removed_date and removed_date < publish_date: + publish_date, removed_date = removed_date, publish_date + + # Row is valid only if we got area OR start_price + if area_m2 is None and start_price is None: + return None + + return ValuationHistoryItem( + area_m2=area_m2, + rooms=rooms, + floor=floor, + start_price=start_price, + start_price_per_m2=start_ppm2, + last_price=last_price, + last_price_per_m2=last_ppm2, + publish_date=publish_date, + removed_date=removed_date, + exposure_days=exposure_days, + status=status, + ) + @staticmethod def _parse_item_text(text: str) -> ValuationHistoryItem | None: """Parse a single text block into a ValuationHistoryItem. diff --git a/tradein-mvp/backend/tests/test_yandex_valuation.py b/tradein-mvp/backend/tests/test_yandex_valuation.py index a727d62b..4b1988ff 100644 --- a/tradein-mvp/backend/tests/test_yandex_valuation.py +++ b/tradein-mvp/backend/tests/test_yandex_valuation.py @@ -482,3 +482,142 @@ def test_area_regex_still_blocks_subfragment(): # '2,2' would need (? +
+
48,5 м², 1-комнатная
+
19 этаж
+
5,1 млн ₽105 155 ₽ за м²
+
5,1 млн ₽105 155 ₽ за м²
+
23.10.2023В экспозиции 945 дней
+
В продаже
+
+
+
58,2 м², 2-комнатная
+
3 этаж
+
9,3 млн ₽159 932 ₽ за м²
+
8,9 млн ₽152 062 ₽ за м²
+
12.02.2026В экспозиции 156 дней
+
17.07.2026
+
+ +""" + + +def test_dom_parser_extracts_two_rows(): + scraper = YandexValuationScraper() + house_meta = "Дом 2020 года 25 этажей Монолитное здание " + result = scraper.parse( + f"{house_meta}{_DOM_FIXTURE_ROW_HTML}", + address="test", + offer_category="APARTMENT", + offer_type="SELL", + page=1, + source_url="http://test/", + ) + assert len(result.history_items) == 2 + + # Row 1 — 1-комн in sale + r1 = result.history_items[0] + assert r1.area_m2 == 48.5 + assert r1.rooms == 1 + assert r1.floor == 19 + assert r1.start_price == 5_100_000 + assert r1.start_price_per_m2 == 105_155 + assert r1.last_price == 5_100_000 + assert r1.publish_date == date(2023, 10, 23) + assert r1.exposure_days == 945 + assert r1.removed_date is None + assert r1.status == "В продаже" + + # Row 2 — 2-комн sold with both dates + r2 = result.history_items[1] + assert r2.area_m2 == 58.2 + assert r2.rooms == 2 + assert r2.floor == 3 + assert r2.start_price == 9_300_000 + assert r2.last_price == 8_900_000 + assert r2.publish_date == date(2026, 2, 12) + assert r2.removed_date == date(2026, 7, 17) + assert r2.exposure_days == 156 + + +def test_dom_parser_studio_row(): + html = ( + "" + '
' + '
Студия 25 м²
' + '
5 этаж
' + '
4 млн ₽160 000 ₽ за м²
' + '
4 млн ₽160 000 ₽ за м²
' + '
01.03.2024В экспозиции 30 дней
' + '
В продаже
' + "
" + "" + ) + scraper = YandexValuationScraper() + result = scraper.parse( + html, + address="t", + offer_category="APARTMENT", + offer_type="SELL", + page=1, + source_url="http://test/", + ) + assert len(result.history_items) == 1 + r = result.history_items[0] + assert r.area_m2 == 25.0 + assert r.rooms == 0 # studio + + +def test_dom_parser_fallback_when_no_rows(): + """If .OffersArchiveSearchOffers__row matches nothing, fall back to chunked text.""" + # No rows — but body_text has a single item with all info + html = ( + "" + "Дом 2020 года 16 этажей " + "2-комнатная 58,0 м² 3 этаж " + "10.05.2024 9 000 000 ₽ В продаже" + "" + ) + scraper = YandexValuationScraper() + result = scraper.parse( + html, + address="t", + offer_category="APARTMENT", + offer_type="SELL", + page=1, + source_url="http://test/", + ) + # Fallback path runs — should still find at least price + # (Don't over-assert: chunked fallback is imperfect; we just want NOT to crash) + assert isinstance(result.history_items, list) + + +def test_dom_parser_skips_malformed_row(): + """Row with <5 cells is skipped (defensive against Yandex A/B variants).""" + html = ( + "" + '
' + '
48,5 м²
' + '
19 этаж
' + "
" + "" + ) + scraper = YandexValuationScraper() + result = scraper.parse( + html, + address="t", + offer_category="APARTMENT", + offer_type="SELL", + page=1, + source_url="http://test/", + ) + # 2-cell row rejected by _parse_row_cells; fallback also doesn't find a date → 0 items + assert len(result.history_items) == 0 -- 2.45.3