diff --git a/tradein-mvp/backend/app/services/scrapers/yandex_valuation.py b/tradein-mvp/backend/app/services/scrapers/yandex_valuation.py index 09b89f19..467dc8bd 100644 --- a/tradein-mvp/backend/app/services/scrapers/yandex_valuation.py +++ b/tradein-mvp/backend/app/services/scrapers/yandex_valuation.py @@ -258,22 +258,130 @@ class YandexValuationScraper(BaseScraper): ) def _parse_history_items(self, tree: HTMLParser, body_text: str) -> list[ValuationHistoryItem]: - """Extract list of historical offer items using best available strategy. + """Extract historical offer items. - Strategy 1: CSS data-test containers (if Yandex exposes them). - Strategy 2: Fallback — split body text into chunks around DD.MM.YYYY dates. + Strategy 1 (preferred, structured): each row at .OffersArchiveSearchOffers__row, + with 6 cells (.OffersArchiveSearchOffers__cell) in fixed order: + [0] area+rooms, [1] floor, [2] start_price+ppm2, [3] last_price+ppm2, + [4] publish_date+exposure, [5] status/removed_date. + Yields per-cell parsing — no chunked-text ambiguity. + + Strategy 2 (last-resort fallback): legacy chunked-text scan around + DD.MM.YYYY anchors. Kept for the rare case Yandex changes class names; + triggers a logger.warning so we notice. """ items: list[ValuationHistoryItem] = [] - # Strategy 1: explicit data-test container - for container in tree.css('[data-test*="HistoryItem"], [data-test*="ValuationItem"]'): - item = self._parse_item_text(container.text(strip=True)) + for row in tree.css(".OffersArchiveSearchOffers__row"): + item = self._parse_row_cells(row) if item: items.append(item) if items: return items - # Strategy 2: text-chunk fallback + # Fallback (only if structured selector failed) + logger.warning( + "yandex_valuation: .OffersArchiveSearchOffers__row matched 0 items — " + "falling back to chunked-text scan" + ) return self._parse_items_from_chunked_text(body_text) + @classmethod + def _parse_row_cells(cls, row) -> ValuationHistoryItem | None: + """Parse one .OffersArchiveSearchOffers__row into a ValuationHistoryItem. + + Cell layout is positional (no labels). Returns None if the row doesn't + match the expected 6-cell structure (defensive — Yandex may A/B-test). + """ + cells = row.css(".OffersArchiveSearchOffers__cell") + if len(cells) < 5: + return None + cell_texts = [(c.text(strip=True) or "").replace("\xa0", " ") for c in cells] + + # [0] area + rooms + area_text = cell_texts[0] + area_m = RE_ITEM_AREA.search(area_text) + area_m2: float | None = float(area_m.group(1).replace(",", ".")) if area_m else None + # Sanity drop (belt-and-suspenders from PR #538) + if area_m2 is not None and (area_m2 > 10_000 or area_m2 <= 0): + area_m2 = None + + rooms: int | None = None + if RE_ITEM_STUDIO.search(area_text): + rooms = 0 + else: + rooms_m = RE_ITEM_ROOMS.search(area_text) + if rooms_m: + rooms = int(rooms_m.group(1)) + + # [1] floor + floor: int | None = None + if len(cell_texts) > 1: + floor_m = RE_ITEM_FLOOR.search(cell_texts[1]) + if floor_m: + floor = int(floor_m.group(1)) + + # [2] start price + ppm2 — "5,1 млн ₽105 155 ₽ за м²" (no separator) + start_price: int | None = None + start_ppm2: int | None = None + if len(cell_texts) > 2: + tokens = _RE_PRICE_TOKEN.findall(cell_texts[2]) + if tokens: + start_price = parse_rub(tokens[0]) + ppm2_tokens = _RE_PPM2_TOKEN.findall(cell_texts[2]) + if ppm2_tokens: + start_ppm2 = parse_rub(ppm2_tokens[0]) + + # [3] last price + ppm2 + last_price: int | None = None + last_ppm2: int | None = None + if len(cell_texts) > 3: + tokens = _RE_PRICE_TOKEN.findall(cell_texts[3]) + if tokens: + last_price = parse_rub(tokens[0]) + ppm2_tokens = _RE_PPM2_TOKEN.findall(cell_texts[3]) + if ppm2_tokens: + last_ppm2 = parse_rub(ppm2_tokens[0]) + + # [4] publish_date + exposure ("23.10.2023В экспозиции 945 дней") + publish_date = None + exposure_days: int | None = None + if len(cell_texts) > 4: + publish_date = parse_dmy(cell_texts[4]) + expo_m = RE_ITEM_EXPOSURE.search(cell_texts[4]) + if expo_m: + exposure_days = int(expo_m.group(1)) + + # [5] status or removed_date + removed_date = None + status: str | None = None + if len(cell_texts) > 5: + last_cell = cell_texts[5] + status_m = RE_ITEM_STATUS.search(last_cell) + if status_m: + status = status_m.group(1) + removed_date = parse_dmy(last_cell) + + # Sort dates chronologically (in case Yandex flips them — same fix as PR #541) + if publish_date and removed_date and removed_date < publish_date: + publish_date, removed_date = removed_date, publish_date + + # Row is valid only if we got area OR start_price + if area_m2 is None and start_price is None: + return None + + return ValuationHistoryItem( + area_m2=area_m2, + rooms=rooms, + floor=floor, + start_price=start_price, + start_price_per_m2=start_ppm2, + last_price=last_price, + last_price_per_m2=last_ppm2, + publish_date=publish_date, + removed_date=removed_date, + exposure_days=exposure_days, + status=status, + ) + @staticmethod def _parse_item_text(text: str) -> ValuationHistoryItem | None: """Parse a single text block into a ValuationHistoryItem. diff --git a/tradein-mvp/backend/tests/test_yandex_valuation.py b/tradein-mvp/backend/tests/test_yandex_valuation.py index a727d62b..4b1988ff 100644 --- a/tradein-mvp/backend/tests/test_yandex_valuation.py +++ b/tradein-mvp/backend/tests/test_yandex_valuation.py @@ -482,3 +482,142 @@ def test_area_regex_still_blocks_subfragment(): # '2,2' would need (? +