Merge pull request 'fix(scrapers): avito detail parser — захват Жилая площадь / Высота потолков / Отделка' (#1550) from fix/avito-detail-apartment-fields into main

Reviewed-on: #1550

tradein-mvp/backend/app/services/scrapers/avito_detail.py

@@ -89,6 +89,9 @@ RUS_FIELD_MAP: dict[str, tuple[str, str | None]] = {
     "Количество комнат": ("rooms", "int"),
     "Общая площадь": ("area_m2", "float_m2"),
     "Площадь кухни": ("kitchen_area_m2", "float_m2"),
+    "Жилая площадь": ("living_area_m2", "float_m2"),
+    "Высота потолков": ("ceiling_height_m", "height_m"),
+    "Отделка": ("finishing", "text_lower"),
     "Этаж": ("floor_split", None),
     "Балкон или лоджия": ("balcony_loggia", "balcony"),
     "Тип комнат": ("room_layout", "layout"),
@@ -184,6 +187,9 @@ class DetailEnrichment:
     rooms: int | None = None
     area_m2: float | None = None
     kitchen_area_m2: float | None = None
+    living_area_m2: float | None = None
+    ceiling_height_m: float | None = None
+    finishing: str | None = None
     floor: int | None = None
     total_floors: int | None = None
     balcony_loggia: str | None = None
@@ -370,6 +376,9 @@ def parse_detail_html(html: str, source_url: str) -> DetailEnrichment:
     rooms: int | None = None
     area_m2: float | None = None
     kitchen_area_m2: float | None = None
+    living_area_m2: float | None = None
+    ceiling_height_m: float | None = None
+    finishing: str | None = None
     floor: int | None = None
     total_floors: int | None = None
     balcony_loggia: str | None = None
@@ -398,6 +407,12 @@ def parse_detail_html(html: str, source_url: str) -> DetailEnrichment:
             area_m2 = _parse_float_m2(val)
         elif field_name == "kitchen_area_m2":
             kitchen_area_m2 = _parse_float_m2(val)
+        elif field_name == "living_area_m2":
+            living_area_m2 = _parse_float_m2(val)
+        elif field_name == "ceiling_height_m":
+            ceiling_height_m = _parse_height_m(val)
+        elif field_name == "finishing":
+            finishing = val.strip().lower() or None
         elif field_name == "floor_split":
             floor, total_floors = _parse_floor_split(val)
         elif field_name == "balcony_loggia":
@@ -487,6 +502,9 @@ def parse_detail_html(html: str, source_url: str) -> DetailEnrichment:
         rooms=rooms,
         area_m2=area_m2,
         kitchen_area_m2=kitchen_area_m2,
+        living_area_m2=living_area_m2,
+        ceiling_height_m=ceiling_height_m,
+        finishing=finishing,
         floor=floor,
         total_floors=total_floors,
         balcony_loggia=balcony_loggia,
@@ -531,6 +549,9 @@ def save_detail_enrichment(db: Session, e: DetailEnrichment) -> bool:
         text("""
             UPDATE listings SET
                 kitchen_area_m2 = COALESCE(:kitchen_area_m2, kitchen_area_m2),
+                living_area_m2 = COALESCE(:living_area_m2, living_area_m2),
+                ceiling_height_m = COALESCE(:ceiling_height_m, ceiling_height_m),
+                finishing = COALESCE(:finishing, finishing),
                 balcony_loggia = COALESCE(:balcony_loggia, balcony_loggia),
                 room_layout = COALESCE(:room_layout, room_layout),
                 bathroom_type = COALESCE(:bathroom_type, bathroom_type),
@@ -563,6 +584,9 @@ def save_detail_enrichment(db: Session, e: DetailEnrichment) -> bool:
         {
             "item_id": e.item_id,
             "kitchen_area_m2": e.kitchen_area_m2,
+            "living_area_m2": e.living_area_m2,
+            "ceiling_height_m": e.ceiling_height_m,
+            "finishing": e.finishing,
             "balcony_loggia": e.balcony_loggia,
             "room_layout": e.room_layout,
             "bathroom_type": e.bathroom_type,
@@ -645,6 +669,17 @@ def _parse_float_m2(val: str) -> float | None:
     return None
 
 
+def _parse_height_m(val: str) -> float | None:
+    """'2.7 м' / '2,7\xa0м' / '3 м' → float метры. Junk → None."""
+    m = _FLOAT_RE.search(val)
+    if m:
+        try:
+            return float(m.group().replace(",", "."))
+        except ValueError:
+            return None
+    return None
+
+
 def _parse_floor_split(val: str) -> tuple[int | None, int | None]:
     """'20 из 26' → (20, 26)."""
     m = _FLOOR_SPLIT_RE.search(val)

tradein-mvp/backend/data/sql/111_listings_avito_detail_fields.sql

@@ -0,0 +1,17 @@
+-- Migration 111: добавить поля avito detail-page, которые парсер молча дропал.
+--
+-- living_area_m2 уже существует в listings (колонка была создана ранее),
+-- но никогда не записывалась — RUS_FIELD_MAP не содержал "Жилая площадь".
+-- После этой миграции + исправления парсера (avito_detail.py) колонка наполняется.
+--
+-- ceiling_height_m и finishing — новые колонки (avito detail-страница содержит
+-- "Высота потолков" и "Отделка"; живая проверка 2026-06-16 подтвердила оба поля).
+-- Для repair_cost_rub метка "Стоимость ремонта" не обнаружена на живых страницах
+-- → колонка не добавляется.
+
+BEGIN;
+
+ALTER TABLE listings ADD COLUMN IF NOT EXISTS ceiling_height_m numeric(5,2);
+ALTER TABLE listings ADD COLUMN IF NOT EXISTS finishing text;
+
+COMMIT;

tradein-mvp/backend/tests/test_avito_detail_parse.py

@@ -8,7 +8,11 @@
 
 import pytest
 
-from app.services.scrapers.avito_detail import DetailEnrichment, parse_detail_html
+from app.services.scrapers.avito_detail import (
+    DetailEnrichment,
+    _parse_height_m,
+    parse_detail_html,
+)
 
 MINIMAL_HTML = """
 <html><body>
@@ -241,3 +245,90 @@ def test_repair_state_none_when_no_signal() -> None:
     """
     result = parse_detail_html(html, "https://www.avito.ru/test_5555555555")
     assert result.repair_state is None
+
+
+# ── Новые поля: living_area_m2, ceiling_height_m, finishing ──────────────────
+
+
+_NEW_FIELDS_HTML = """
+<html><body>
+<div data-marker="item-view/item-id">№ 6666666666</div>
+<span itemprop="price" content="8500000">8 500 000 ₽</span>
+<div data-marker="item-view/item-params">
+  <ul>
+    <li>Количество комнат: 1</li>
+    <li>Общая площадь: 43.7\xa0м²</li>
+    <li>Площадь кухни: 17.1\xa0м²</li>
+    <li>Жилая площадь: 12.6\xa0м²</li>
+    <li>Высота потолков: 2.7\xa0м</li>
+    <li>Отделка: чистовая</li>
+    <li>Этаж: 6 из 16</li>
+  </ul>
+</div>
+</body></html>
+"""
+
+
+def test_new_apartment_fields_populated() -> None:
+    """living_area_m2 / ceiling_height_m / finishing парсятся из params-блока."""
+    result = parse_detail_html(_NEW_FIELDS_HTML, "https://www.avito.ru/test_6666666666")
+    assert result.living_area_m2 == 12.6
+    assert result.ceiling_height_m == 2.7
+    assert result.finishing == "чистовая"
+
+
+def test_new_fields_absent_when_not_in_params() -> None:
+    """Если полей нет в params — None, не дефолт."""
+    result = parse_detail_html(
+        """
+        <html><body>
+        <div data-marker="item-view/item-id">№ 7777777777</div>
+        <span itemprop="price" content="4000000">4 000 000 ₽</span>
+        <div data-marker="item-view/item-params">
+          <ul><li>Количество комнат: 2</li></ul>
+        </div>
+        </body></html>
+        """,
+        "https://www.avito.ru/test_7777777777",
+    )
+    assert result.living_area_m2 is None
+    assert result.ceiling_height_m is None
+    assert result.finishing is None
+
+
+def test_finishing_stored_lowercase_stripped() -> None:
+    """finishing → lowercase trimmed ('  Чистовая  ' → 'чистовая')."""
+    html = """
+    <html><body>
+    <div data-marker="item-view/item-id">№ 8888888888</div>
+    <span itemprop="price" content="5000000">5 000 000 ₽</span>
+    <div data-marker="item-view/item-params">
+      <ul><li>Отделка:   Чистовая  </li></ul>
+    </div>
+    </body></html>
+    """
+    result = parse_detail_html(html, "https://www.avito.ru/test_8888888888")
+    assert result.finishing == "чистовая"
+
+
+# ── _parse_height_m unit tests ────────────────────────────────────────────────
+
+
+def test_parse_height_m_dot() -> None:
+    assert _parse_height_m("2.7\xa0м") == 2.7
+
+
+def test_parse_height_m_comma() -> None:
+    assert _parse_height_m("2,7\xa0м") == 2.7
+
+
+def test_parse_height_m_integer() -> None:
+    assert _parse_height_m("3 м") == 3.0
+
+
+def test_parse_height_m_junk() -> None:
+    assert _parse_height_m("нет данных") is None
+
+
+def test_parse_height_m_empty() -> None:
+    assert _parse_height_m("") is None