diff --git a/tradein-mvp/backend/app/services/scrapers/avito_detail.py b/tradein-mvp/backend/app/services/scrapers/avito_detail.py index 35a7f52c..c0e246a3 100644 --- a/tradein-mvp/backend/app/services/scrapers/avito_detail.py +++ b/tradein-mvp/backend/app/services/scrapers/avito_detail.py @@ -89,6 +89,9 @@ RUS_FIELD_MAP: dict[str, tuple[str, str | None]] = { "Количество комнат": ("rooms", "int"), "Общая площадь": ("area_m2", "float_m2"), "Площадь кухни": ("kitchen_area_m2", "float_m2"), + "Жилая площадь": ("living_area_m2", "float_m2"), + "Высота потолков": ("ceiling_height_m", "height_m"), + "Отделка": ("finishing", "text_lower"), "Этаж": ("floor_split", None), "Балкон или лоджия": ("balcony_loggia", "balcony"), "Тип комнат": ("room_layout", "layout"), @@ -184,6 +187,9 @@ class DetailEnrichment: rooms: int | None = None area_m2: float | None = None kitchen_area_m2: float | None = None + living_area_m2: float | None = None + ceiling_height_m: float | None = None + finishing: str | None = None floor: int | None = None total_floors: int | None = None balcony_loggia: str | None = None @@ -370,6 +376,9 @@ def parse_detail_html(html: str, source_url: str) -> DetailEnrichment: rooms: int | None = None area_m2: float | None = None kitchen_area_m2: float | None = None + living_area_m2: float | None = None + ceiling_height_m: float | None = None + finishing: str | None = None floor: int | None = None total_floors: int | None = None balcony_loggia: str | None = None @@ -398,6 +407,12 @@ def parse_detail_html(html: str, source_url: str) -> DetailEnrichment: area_m2 = _parse_float_m2(val) elif field_name == "kitchen_area_m2": kitchen_area_m2 = _parse_float_m2(val) + elif field_name == "living_area_m2": + living_area_m2 = _parse_float_m2(val) + elif field_name == "ceiling_height_m": + ceiling_height_m = _parse_height_m(val) + elif field_name == "finishing": + finishing = val.strip().lower() or None elif field_name == "floor_split": floor, total_floors = _parse_floor_split(val) elif field_name == "balcony_loggia": @@ -487,6 +502,9 @@ def parse_detail_html(html: str, source_url: str) -> DetailEnrichment: rooms=rooms, area_m2=area_m2, kitchen_area_m2=kitchen_area_m2, + living_area_m2=living_area_m2, + ceiling_height_m=ceiling_height_m, + finishing=finishing, floor=floor, total_floors=total_floors, balcony_loggia=balcony_loggia, @@ -531,6 +549,9 @@ def save_detail_enrichment(db: Session, e: DetailEnrichment) -> bool: text(""" UPDATE listings SET kitchen_area_m2 = COALESCE(:kitchen_area_m2, kitchen_area_m2), + living_area_m2 = COALESCE(:living_area_m2, living_area_m2), + ceiling_height_m = COALESCE(:ceiling_height_m, ceiling_height_m), + finishing = COALESCE(:finishing, finishing), balcony_loggia = COALESCE(:balcony_loggia, balcony_loggia), room_layout = COALESCE(:room_layout, room_layout), bathroom_type = COALESCE(:bathroom_type, bathroom_type), @@ -563,6 +584,9 @@ def save_detail_enrichment(db: Session, e: DetailEnrichment) -> bool: { "item_id": e.item_id, "kitchen_area_m2": e.kitchen_area_m2, + "living_area_m2": e.living_area_m2, + "ceiling_height_m": e.ceiling_height_m, + "finishing": e.finishing, "balcony_loggia": e.balcony_loggia, "room_layout": e.room_layout, "bathroom_type": e.bathroom_type, @@ -645,6 +669,17 @@ def _parse_float_m2(val: str) -> float | None: return None +def _parse_height_m(val: str) -> float | None: + """'2.7 м' / '2,7\xa0м' / '3 м' → float метры. Junk → None.""" + m = _FLOAT_RE.search(val) + if m: + try: + return float(m.group().replace(",", ".")) + except ValueError: + return None + return None + + def _parse_floor_split(val: str) -> tuple[int | None, int | None]: """'20 из 26' → (20, 26).""" m = _FLOOR_SPLIT_RE.search(val) diff --git a/tradein-mvp/backend/data/sql/111_listings_avito_detail_fields.sql b/tradein-mvp/backend/data/sql/111_listings_avito_detail_fields.sql new file mode 100644 index 00000000..536ec406 --- /dev/null +++ b/tradein-mvp/backend/data/sql/111_listings_avito_detail_fields.sql @@ -0,0 +1,17 @@ +-- Migration 111: добавить поля avito detail-page, которые парсер молча дропал. +-- +-- living_area_m2 уже существует в listings (колонка была создана ранее), +-- но никогда не записывалась — RUS_FIELD_MAP не содержал "Жилая площадь". +-- После этой миграции + исправления парсера (avito_detail.py) колонка наполняется. +-- +-- ceiling_height_m и finishing — новые колонки (avito detail-страница содержит +-- "Высота потолков" и "Отделка"; живая проверка 2026-06-16 подтвердила оба поля). +-- Для repair_cost_rub метка "Стоимость ремонта" не обнаружена на живых страницах +-- → колонка не добавляется. + +BEGIN; + +ALTER TABLE listings ADD COLUMN IF NOT EXISTS ceiling_height_m numeric(5,2); +ALTER TABLE listings ADD COLUMN IF NOT EXISTS finishing text; + +COMMIT; diff --git a/tradein-mvp/backend/tests/test_avito_detail_parse.py b/tradein-mvp/backend/tests/test_avito_detail_parse.py index 14abe8f3..e9e419fb 100644 --- a/tradein-mvp/backend/tests/test_avito_detail_parse.py +++ b/tradein-mvp/backend/tests/test_avito_detail_parse.py @@ -8,7 +8,11 @@ import pytest -from app.services.scrapers.avito_detail import DetailEnrichment, parse_detail_html +from app.services.scrapers.avito_detail import ( + DetailEnrichment, + _parse_height_m, + parse_detail_html, +) MINIMAL_HTML = """ @@ -241,3 +245,90 @@ def test_repair_state_none_when_no_signal() -> None: """ result = parse_detail_html(html, "https://www.avito.ru/test_5555555555") assert result.repair_state is None + + +# ── Новые поля: living_area_m2, ceiling_height_m, finishing ────────────────── + + +_NEW_FIELDS_HTML = """ + +
№ 6666666666
+8 500 000 ₽ +
+ +
+ +""" + + +def test_new_apartment_fields_populated() -> None: + """living_area_m2 / ceiling_height_m / finishing парсятся из params-блока.""" + result = parse_detail_html(_NEW_FIELDS_HTML, "https://www.avito.ru/test_6666666666") + assert result.living_area_m2 == 12.6 + assert result.ceiling_height_m == 2.7 + assert result.finishing == "чистовая" + + +def test_new_fields_absent_when_not_in_params() -> None: + """Если полей нет в params — None, не дефолт.""" + result = parse_detail_html( + """ + +
№ 7777777777
+ 4 000 000 ₽ +
+ +
+ + """, + "https://www.avito.ru/test_7777777777", + ) + assert result.living_area_m2 is None + assert result.ceiling_height_m is None + assert result.finishing is None + + +def test_finishing_stored_lowercase_stripped() -> None: + """finishing → lowercase trimmed (' Чистовая ' → 'чистовая').""" + html = """ + +
№ 8888888888
+ 5 000 000 ₽ +
+ +
+ + """ + result = parse_detail_html(html, "https://www.avito.ru/test_8888888888") + assert result.finishing == "чистовая" + + +# ── _parse_height_m unit tests ──────────────────────────────────────────────── + + +def test_parse_height_m_dot() -> None: + assert _parse_height_m("2.7\xa0м") == 2.7 + + +def test_parse_height_m_comma() -> None: + assert _parse_height_m("2,7\xa0м") == 2.7 + + +def test_parse_height_m_integer() -> None: + assert _parse_height_m("3 м") == 3.0 + + +def test_parse_height_m_junk() -> None: + assert _parse_height_m("нет данных") is None + + +def test_parse_height_m_empty() -> None: + assert _parse_height_m("") is None