refactor(yandex-valuation): DOM-based parser via .OffersArchiveSearchOffers__row
Live HTML recon discovered stable BEM-style classes. Each history item is a .OffersArchiveSearchOffers__row with 6 positional cells: area+rooms, floor, start_price+ppm2, last_price+ppm2, publish_date+exposure, status/removed_date. Drops chunked-text fallback as primary strategy (kept as last-resort with warning log). Eliminates 3 PRs of regex whack-a-mole (#538/#541/#542): - area = year+area concat → impossible with cell isolation - area = sub-fragment '2,2' → impossible - date inversion → cell [4] is always publish, cell [5] is always removed - 1-комн/2-комн/3-комн all parsed identically Re-sweep Базовый 52 should now hit ~90%+ plausible area_filled vs 8.7% from PR #542.
This commit is contained in:
lekss361
parent
cbe81b7fb7
commit
e070f06c26
2 changed files with 254 additions and 7 deletions
|
|
@ -258,22 +258,130 @@ class YandexValuationScraper(BaseScraper):
|
|||
)
|
||||
|
||||
def _parse_history_items(self, tree: HTMLParser, body_text: str) -> list[ValuationHistoryItem]:
|
||||
"""Extract list of historical offer items using best available strategy.
|
||||
"""Extract historical offer items.
|
||||
|
||||
Strategy 1: CSS data-test containers (if Yandex exposes them).
|
||||
Strategy 2: Fallback — split body text into chunks around DD.MM.YYYY dates.
|
||||
Strategy 1 (preferred, structured): each row at .OffersArchiveSearchOffers__row,
|
||||
with 6 cells (.OffersArchiveSearchOffers__cell) in fixed order:
|
||||
[0] area+rooms, [1] floor, [2] start_price+ppm2, [3] last_price+ppm2,
|
||||
[4] publish_date+exposure, [5] status/removed_date.
|
||||
Yields per-cell parsing — no chunked-text ambiguity.
|
||||
|
||||
Strategy 2 (last-resort fallback): legacy chunked-text scan around
|
||||
DD.MM.YYYY anchors. Kept for the rare case Yandex changes class names;
|
||||
triggers a logger.warning so we notice.
|
||||
"""
|
||||
items: list[ValuationHistoryItem] = []
|
||||
# Strategy 1: explicit data-test container
|
||||
for container in tree.css('[data-test*="HistoryItem"], [data-test*="ValuationItem"]'):
|
||||
item = self._parse_item_text(container.text(strip=True))
|
||||
for row in tree.css(".OffersArchiveSearchOffers__row"):
|
||||
item = self._parse_row_cells(row)
|
||||
if item:
|
||||
items.append(item)
|
||||
if items:
|
||||
return items
|
||||
# Strategy 2: text-chunk fallback
|
||||
# Fallback (only if structured selector failed)
|
||||
logger.warning(
|
||||
"yandex_valuation: .OffersArchiveSearchOffers__row matched 0 items — "
|
||||
"falling back to chunked-text scan"
|
||||
)
|
||||
return self._parse_items_from_chunked_text(body_text)
|
||||
|
||||
@classmethod
|
||||
def _parse_row_cells(cls, row) -> ValuationHistoryItem | None:
|
||||
"""Parse one .OffersArchiveSearchOffers__row into a ValuationHistoryItem.
|
||||
|
||||
Cell layout is positional (no labels). Returns None if the row doesn't
|
||||
match the expected 6-cell structure (defensive — Yandex may A/B-test).
|
||||
"""
|
||||
cells = row.css(".OffersArchiveSearchOffers__cell")
|
||||
if len(cells) < 5:
|
||||
return None
|
||||
cell_texts = [(c.text(strip=True) or "").replace("\xa0", " ") for c in cells]
|
||||
|
||||
# [0] area + rooms
|
||||
area_text = cell_texts[0]
|
||||
area_m = RE_ITEM_AREA.search(area_text)
|
||||
area_m2: float | None = float(area_m.group(1).replace(",", ".")) if area_m else None
|
||||
# Sanity drop (belt-and-suspenders from PR #538)
|
||||
if area_m2 is not None and (area_m2 > 10_000 or area_m2 <= 0):
|
||||
area_m2 = None
|
||||
|
||||
rooms: int | None = None
|
||||
if RE_ITEM_STUDIO.search(area_text):
|
||||
rooms = 0
|
||||
else:
|
||||
rooms_m = RE_ITEM_ROOMS.search(area_text)
|
||||
if rooms_m:
|
||||
rooms = int(rooms_m.group(1))
|
||||
|
||||
# [1] floor
|
||||
floor: int | None = None
|
||||
if len(cell_texts) > 1:
|
||||
floor_m = RE_ITEM_FLOOR.search(cell_texts[1])
|
||||
if floor_m:
|
||||
floor = int(floor_m.group(1))
|
||||
|
||||
# [2] start price + ppm2 — "5,1 млн ₽105 155 ₽ за м²" (no separator)
|
||||
start_price: int | None = None
|
||||
start_ppm2: int | None = None
|
||||
if len(cell_texts) > 2:
|
||||
tokens = _RE_PRICE_TOKEN.findall(cell_texts[2])
|
||||
if tokens:
|
||||
start_price = parse_rub(tokens[0])
|
||||
ppm2_tokens = _RE_PPM2_TOKEN.findall(cell_texts[2])
|
||||
if ppm2_tokens:
|
||||
start_ppm2 = parse_rub(ppm2_tokens[0])
|
||||
|
||||
# [3] last price + ppm2
|
||||
last_price: int | None = None
|
||||
last_ppm2: int | None = None
|
||||
if len(cell_texts) > 3:
|
||||
tokens = _RE_PRICE_TOKEN.findall(cell_texts[3])
|
||||
if tokens:
|
||||
last_price = parse_rub(tokens[0])
|
||||
ppm2_tokens = _RE_PPM2_TOKEN.findall(cell_texts[3])
|
||||
if ppm2_tokens:
|
||||
last_ppm2 = parse_rub(ppm2_tokens[0])
|
||||
|
||||
# [4] publish_date + exposure ("23.10.2023В экспозиции 945 дней")
|
||||
publish_date = None
|
||||
exposure_days: int | None = None
|
||||
if len(cell_texts) > 4:
|
||||
publish_date = parse_dmy(cell_texts[4])
|
||||
expo_m = RE_ITEM_EXPOSURE.search(cell_texts[4])
|
||||
if expo_m:
|
||||
exposure_days = int(expo_m.group(1))
|
||||
|
||||
# [5] status or removed_date
|
||||
removed_date = None
|
||||
status: str | None = None
|
||||
if len(cell_texts) > 5:
|
||||
last_cell = cell_texts[5]
|
||||
status_m = RE_ITEM_STATUS.search(last_cell)
|
||||
if status_m:
|
||||
status = status_m.group(1)
|
||||
removed_date = parse_dmy(last_cell)
|
||||
|
||||
# Sort dates chronologically (in case Yandex flips them — same fix as PR #541)
|
||||
if publish_date and removed_date and removed_date < publish_date:
|
||||
publish_date, removed_date = removed_date, publish_date
|
||||
|
||||
# Row is valid only if we got area OR start_price
|
||||
if area_m2 is None and start_price is None:
|
||||
return None
|
||||
|
||||
return ValuationHistoryItem(
|
||||
area_m2=area_m2,
|
||||
rooms=rooms,
|
||||
floor=floor,
|
||||
start_price=start_price,
|
||||
start_price_per_m2=start_ppm2,
|
||||
last_price=last_price,
|
||||
last_price_per_m2=last_ppm2,
|
||||
publish_date=publish_date,
|
||||
removed_date=removed_date,
|
||||
exposure_days=exposure_days,
|
||||
status=status,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _parse_item_text(text: str) -> ValuationHistoryItem | None:
|
||||
"""Parse a single text block into a ValuationHistoryItem.
|
||||
|
|
|
|||
|
|
@ -482,3 +482,142 @@ def test_area_regex_still_blocks_subfragment():
|
|||
# '2,2' would need (?<!\\d) — `,` is not digit, so lookbehind allows start.
|
||||
# But `\\d{2,4}` needs ≥2 leading digits — '2,' only has one. No match.
|
||||
assert item is None or item.area_m2 is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# DOM-based parser (PR refactor — replaces chunked-text fallback)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_DOM_FIXTURE_ROW_HTML = """
|
||||
<div class="OffersArchiveSearchOffers__body">
|
||||
<div class="OffersArchiveSearchOffers__row">
|
||||
<div class="OffersArchiveSearchOffers__cell">48,5 м², 1-комнатная</div>
|
||||
<div class="OffersArchiveSearchOffers__cell">19 этаж</div>
|
||||
<div class="OffersArchiveSearchOffers__cell">5,1 млн ₽105 155 ₽ за м²</div>
|
||||
<div class="OffersArchiveSearchOffers__cell">5,1 млн ₽105 155 ₽ за м²</div>
|
||||
<div class="OffersArchiveSearchOffers__cell">23.10.2023В экспозиции 945 дней</div>
|
||||
<div class="OffersArchiveSearchOffers__cell">В продаже</div>
|
||||
</div>
|
||||
<div class="OffersArchiveSearchOffers__row">
|
||||
<div class="OffersArchiveSearchOffers__cell">58,2 м², 2-комнатная</div>
|
||||
<div class="OffersArchiveSearchOffers__cell">3 этаж</div>
|
||||
<div class="OffersArchiveSearchOffers__cell">9,3 млн ₽159 932 ₽ за м²</div>
|
||||
<div class="OffersArchiveSearchOffers__cell">8,9 млн ₽152 062 ₽ за м²</div>
|
||||
<div class="OffersArchiveSearchOffers__cell">12.02.2026В экспозиции 156 дней</div>
|
||||
<div class="OffersArchiveSearchOffers__cell">17.07.2026</div>
|
||||
</div>
|
||||
</div>
|
||||
"""
|
||||
|
||||
|
||||
def test_dom_parser_extracts_two_rows():
|
||||
scraper = YandexValuationScraper()
|
||||
house_meta = "Дом 2020 года 25 этажей Монолитное здание "
|
||||
result = scraper.parse(
|
||||
f"<html><body>{house_meta}{_DOM_FIXTURE_ROW_HTML}</body></html>",
|
||||
address="test",
|
||||
offer_category="APARTMENT",
|
||||
offer_type="SELL",
|
||||
page=1,
|
||||
source_url="http://test/",
|
||||
)
|
||||
assert len(result.history_items) == 2
|
||||
|
||||
# Row 1 — 1-комн in sale
|
||||
r1 = result.history_items[0]
|
||||
assert r1.area_m2 == 48.5
|
||||
assert r1.rooms == 1
|
||||
assert r1.floor == 19
|
||||
assert r1.start_price == 5_100_000
|
||||
assert r1.start_price_per_m2 == 105_155
|
||||
assert r1.last_price == 5_100_000
|
||||
assert r1.publish_date == date(2023, 10, 23)
|
||||
assert r1.exposure_days == 945
|
||||
assert r1.removed_date is None
|
||||
assert r1.status == "В продаже"
|
||||
|
||||
# Row 2 — 2-комн sold with both dates
|
||||
r2 = result.history_items[1]
|
||||
assert r2.area_m2 == 58.2
|
||||
assert r2.rooms == 2
|
||||
assert r2.floor == 3
|
||||
assert r2.start_price == 9_300_000
|
||||
assert r2.last_price == 8_900_000
|
||||
assert r2.publish_date == date(2026, 2, 12)
|
||||
assert r2.removed_date == date(2026, 7, 17)
|
||||
assert r2.exposure_days == 156
|
||||
|
||||
|
||||
def test_dom_parser_studio_row():
|
||||
html = (
|
||||
"<html><body>"
|
||||
'<div class="OffersArchiveSearchOffers__row">'
|
||||
' <div class="OffersArchiveSearchOffers__cell">Студия 25 м²</div>'
|
||||
' <div class="OffersArchiveSearchOffers__cell">5 этаж</div>'
|
||||
' <div class="OffersArchiveSearchOffers__cell">4 млн ₽160 000 ₽ за м²</div>'
|
||||
' <div class="OffersArchiveSearchOffers__cell">4 млн ₽160 000 ₽ за м²</div>'
|
||||
' <div class="OffersArchiveSearchOffers__cell">01.03.2024В экспозиции 30 дней</div>'
|
||||
' <div class="OffersArchiveSearchOffers__cell">В продаже</div>'
|
||||
"</div>"
|
||||
"</body></html>"
|
||||
)
|
||||
scraper = YandexValuationScraper()
|
||||
result = scraper.parse(
|
||||
html,
|
||||
address="t",
|
||||
offer_category="APARTMENT",
|
||||
offer_type="SELL",
|
||||
page=1,
|
||||
source_url="http://test/",
|
||||
)
|
||||
assert len(result.history_items) == 1
|
||||
r = result.history_items[0]
|
||||
assert r.area_m2 == 25.0
|
||||
assert r.rooms == 0 # studio
|
||||
|
||||
|
||||
def test_dom_parser_fallback_when_no_rows():
|
||||
"""If .OffersArchiveSearchOffers__row matches nothing, fall back to chunked text."""
|
||||
# No rows — but body_text has a single item with all info
|
||||
html = (
|
||||
"<html><body>"
|
||||
"Дом 2020 года 16 этажей "
|
||||
"2-комнатная 58,0 м² 3 этаж "
|
||||
"10.05.2024 9 000 000 ₽ В продаже"
|
||||
"</body></html>"
|
||||
)
|
||||
scraper = YandexValuationScraper()
|
||||
result = scraper.parse(
|
||||
html,
|
||||
address="t",
|
||||
offer_category="APARTMENT",
|
||||
offer_type="SELL",
|
||||
page=1,
|
||||
source_url="http://test/",
|
||||
)
|
||||
# Fallback path runs — should still find at least price
|
||||
# (Don't over-assert: chunked fallback is imperfect; we just want NOT to crash)
|
||||
assert isinstance(result.history_items, list)
|
||||
|
||||
|
||||
def test_dom_parser_skips_malformed_row():
|
||||
"""Row with <5 cells is skipped (defensive against Yandex A/B variants)."""
|
||||
html = (
|
||||
"<html><body>"
|
||||
'<div class="OffersArchiveSearchOffers__row">'
|
||||
' <div class="OffersArchiveSearchOffers__cell">48,5 м²</div>'
|
||||
' <div class="OffersArchiveSearchOffers__cell">19 этаж</div>'
|
||||
"</div>"
|
||||
"</body></html>"
|
||||
)
|
||||
scraper = YandexValuationScraper()
|
||||
result = scraper.parse(
|
||||
html,
|
||||
address="t",
|
||||
offer_category="APARTMENT",
|
||||
offer_type="SELL",
|
||||
page=1,
|
||||
source_url="http://test/",
|
||||
)
|
||||
# 2-cell row rejected by _parse_row_cells; fallback also doesn't find a date → 0 items
|
||||
assert len(result.history_items) == 0
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue