From 7eb4d6d2550019751e99c3fc3cc6e574a25fc787 Mon Sep 17 00:00:00 2001 From: bot-backend Date: Sat, 30 May 2026 20:27:59 +0300 Subject: [PATCH] =?UTF-8?q?feat(scrapers/n1):=20=D0=B8=D0=B7=D0=B2=D0=BB?= =?UTF-8?q?=D0=B5=D0=BA=D0=B0=D1=82=D1=8C=20floor/total=5Ffloors=20=D0=B8?= =?UTF-8?q?=D0=B7=20=D0=B2=D0=B5=D0=B4=D1=83=D1=89=D0=B5=D0=B3=D0=BE=20N/M?= =?UTF-8?q?-=D1=82=D0=BE=D0=BA=D0=B5=D0=BD=D0=B0=20(Refs=20#795)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit n1.py не извлекал floor/total_floors. Этаж в ведущем токене N/M карточки ('1/11 ...'→этаж 1 из 11). _extract_floor_from_title + ^-anchored regex (дом '75/2' не матчится), валидация диапазонов. listing_date не трогался. 19 тестов, 24 pass, ruff clean. --- .../backend/app/services/scrapers/n1.py | 71 ++++++-- .../backend/tests/test_n1_floor_parse.py | 169 ++++++++++++++++++ 2 files changed, 224 insertions(+), 16 deletions(-) create mode 100644 tradein-mvp/backend/tests/test_n1_floor_parse.py diff --git a/tradein-mvp/backend/app/services/scrapers/n1.py b/tradein-mvp/backend/app/services/scrapers/n1.py index 907756f6..be2dfe51 100644 --- a/tradein-mvp/backend/app/services/scrapers/n1.py +++ b/tradein-mvp/backend/app/services/scrapers/n1.py @@ -7,7 +7,8 @@ SSR HTML с DOM-карточками `data-test="offers-list-item"`. Карточка: - a[href^="/view/NNN/"] — ссылка на объявление + offer_id - текст ссылки = "3-к, Репина, 75/2 стр." (комнаты + адрес) + текст ссылки = "1/11 2-к, Репина, 75/2 стр." (этаж/этажность + комнаты + адрес) + Этаж/этажность — leading-токен «N/M» перед сегментом комнат. - [data-test="offers-list-item-param-total-area"] — "79 м2" - [data-test="offers-list-item-param-price"] — "15 700 000" - img.src — фото @@ -66,8 +67,12 @@ class N1Scraper(BaseScraper): await super().__aexit__(*args) async def fetch_around( - self, lat: float, lon: float, radius_m: int = 1000, - rooms: int | None = None, page: int = 1, + self, + lat: float, + lon: float, + radius_m: int = 1000, + rooms: int | None = None, + page: int = 1, ) -> list[ScrapedLot]: self._anchor_lat = lat self._anchor_lon = lon @@ -87,13 +92,21 @@ class N1Scraper(BaseScraper): lots = self._parse_html(response.text) logger.info( "n1: %d lots around (%.4f, %.4f) rooms=%s page=%d", - len(lots), lat, lon, rooms, page, + len(lots), + lat, + lon, + rooms, + page, ) await self.sleep_between_requests() return lots async def fetch_around_multi_room( - self, lat: float, lon: float, radius_m: int = 1000, pages: int = 8, + self, + lat: float, + lon: float, + radius_m: int = 1000, + pages: int = 8, ) -> list[ScrapedLot]: """Скрейп N1 по сегментам комнат 1/2/3/4 × N страниц — расширяет выборку.""" seen: dict[str, ScrapedLot] = {} @@ -102,9 +115,7 @@ class N1Scraper(BaseScraper): try: lots = await self.fetch_around(lat, lon, radius_m, rooms=rooms, page=page) except Exception: - logger.exception( - "n1 multi-room fetch failed rooms=%s page=%d", rooms, page - ) + logger.exception("n1 multi-room fetch failed rooms=%s page=%d", rooms, page) continue if not lots: break # пустая страница → дальше нет смысла @@ -112,9 +123,7 @@ class N1Scraper(BaseScraper): key = lot.source_id or lot.source_url if key and key not in seen: seen[key] = lot - logger.info( - "n1 multi-room: %d unique lots around (%.4f, %.4f)", len(seen), lat, lon - ) + logger.info("n1 multi-room: %d unique lots around (%.4f, %.4f)", len(seen), lat, lon) return list(seen.values()) def _build_url(self, rooms: int | None = None, page: int = 1) -> str: @@ -137,9 +146,7 @@ class N1Scraper(BaseScraper): def _parse_html(self, html: str) -> list[ScrapedLot]: tree = HTMLParser(html) cards = tree.css('[data-test="offers-list-item"]') - return [ - lot for lot in (self._card_to_lot(c) for c in cards) if lot is not None - ] + return [lot for lot in (self._card_to_lot(c) for c in cards) if lot is not None] def _card_to_lot(self, card: Any) -> ScrapedLot | None: try: @@ -152,10 +159,11 @@ class N1Scraper(BaseScraper): offer_id = m_id.group(1) if m_id else None url = href if href.startswith("http") else f"{self.base_url}{href}" - # Заголовок ссылки: "3-к, Репина, 75/2 стр." + # Заголовок ссылки: "1/11 2-к, Репина, 75/2 стр." (этаж/этажность + комнаты + адрес) title = link.text(strip=True) or "" + floor, total_floors = _extract_floor_from_title(title) rooms = _extract_rooms(title) - # Адрес = title без префикса комнат + # Адрес = title без ведущего «N/M» и без префикса комнат address = re.sub( r"^\s*(\d+-к|студи[яи])\s*,?\s*", "", title, flags=re.IGNORECASE ).strip() @@ -193,6 +201,8 @@ class N1Scraper(BaseScraper): lon=lon, rooms=rooms, area_m2=area, + floor=floor, + total_floors=total_floors, price_rub=price, photo_urls=photos, raw_payload={"title": title}, @@ -209,6 +219,11 @@ _RE_STUDIO = re.compile(r"студи", re.IGNORECASE) _RE_AREA = re.compile(r"([\d]+(?:[.,]\d+)?)") _RE_DIGITS = re.compile(r"\d") +# Ведущий токен «N/M» в тексте карточки N1: «1/11 2-к, Репина, 75/2 стр.» +# Якорь ^ гарантирует: только если N/M стоит В НАЧАЛЕ строки (до сегмента комнат). +# Дробь в адресе вида «75/2» в середине строки НЕ совпадёт. +_RE_FLOOR_LEADING = re.compile(r"^\s*(\d{1,3})\s*/\s*(\d{1,3})\b") + def _extract_rooms(text: str) -> int | None: if _RE_STUDIO.search(text): @@ -230,3 +245,27 @@ def _extract_area(text: str) -> float | None: def _extract_price(text: str) -> int | None: digits = "".join(_RE_DIGITS.findall(text)) return int(digits) if digits else None + + +def _extract_floor_from_title(text: str) -> tuple[int | None, int | None]: + """Извлекает этаж и этажность из ведущего токена «N/M» в тексте карточки N1. + + N1 SERP отдаёт текст вида «1/11 2-к, Репина, 75/2 стр.» — этаж/этажность + идут первыми, до сегмента комнат. Регулярка привязана к началу строки (^), + поэтому дробь в номере дома («75/2») в середине строки не захватывается. + + Sanity-проверки: + - floor >= 1 + - total_floors >= floor + - total_floors <= 100 + + Возвращает (None, None) при отсутствии совпадения или нарушении диапазона. + """ + m = _RE_FLOOR_LEADING.match(text) + if m is None: + return None, None + floor = int(m.group(1)) + total_floors = int(m.group(2)) + if floor < 1 or total_floors < floor or total_floors > 100: + return None, None + return floor, total_floors diff --git a/tradein-mvp/backend/tests/test_n1_floor_parse.py b/tradein-mvp/backend/tests/test_n1_floor_parse.py new file mode 100644 index 00000000..df89d9f4 --- /dev/null +++ b/tradein-mvp/backend/tests/test_n1_floor_parse.py @@ -0,0 +1,169 @@ +"""Unit tests for N1 floor/total_floors extraction (#795). + +Tests _extract_floor_from_title() directly (pure function, no network/DB). +Also covers _card_to_lot integration via a minimal fake card HTML node. +""" + +from __future__ import annotations + +import pytest + +from app.services.scrapers.n1 import _extract_floor_from_title + +# ── _extract_floor_from_title: direct unit tests ──────────────────────────── + + +@pytest.mark.parametrize( + "text, expected_floor, expected_total", + [ + # Typical N1 card: floor 1 of 11 + ("1/11 2-к, Репина, 75/2 стр.", 1, 11), + # Floor in middle of building + ("5/9 3-к, Ленина, 10 стр.", 5, 9), + # Top floor + ("17/17 1-к, Малышева, 12", 17, 17), + # Leading whitespace tolerated + (" 3/16 студия, Вайнера, 5", 3, 16), + # Three-digit total floors (edge of range) + ("2/100 2-к, Тверитина, 8", 2, 100), + ], +) +def test_extract_floor_happy_path(text: str, expected_floor: int, expected_total: int) -> None: + floor, total = _extract_floor_from_title(text) + assert floor == expected_floor + assert total == expected_total + + +@pytest.mark.parametrize( + "text", + [ + # No leading N/M at all — classic old format without floor token + ("2-к, Репина, 75/2 стр."), + # House number fraction in middle — must NOT be parsed as floor + ("Репина, 75/2 стр."), + # N/M not at string start (preceded by other text) + ("квартира 3/9, Ленина, 1"), + # floor=0 is invalid + ("0/9 1-к, Тестовая, 1"), + # total_floors < floor — sanity fail + ("9/5 2-к, Тестовая, 1"), + # total_floors > 100 — sanity fail + ("2/101 1-к, Тестовая, 1"), + # Empty string + (""), + # Only digits, no slash + ("5 2-к, Тестовая, 1"), + ], +) +def test_extract_floor_returns_none(text: str) -> None: + floor, total = _extract_floor_from_title(text) + assert floor is None + assert total is None + + +def test_house_number_fraction_not_misparsed() -> None: + """«75/2 стр.» appearing after rooms segment must not match floor regex.""" + # The fraction 75/2 comes AFTER the rooms prefix, not at string start. + floor, total = _extract_floor_from_title("3-к, Репина, 75/2 стр.") + assert floor is None + assert total is None + + +def test_non_leading_fraction_after_comma() -> None: + """Fraction embedded mid-string via comma-separated address token.""" + floor, total = _extract_floor_from_title("ул. 8 Марта, 51/3, 2-к") + assert floor is None + assert total is None + + +# ── Integration: _card_to_lot via fake selectolax node ────────────────────── + + +class _FakeAttr(dict): + """Minimal attributes dict for a fake HTML node.""" + + +class _FakeNode: + """Minimal selectolax node stub for _card_to_lot tests.""" + + def __init__( + self, + *, + href: str = "/view/12345/", + title_text: str = "1/11 2-к, Репина, 75/2 стр.", + area_text: str = "52 м2", + price_text: str = "8 500 000", + img_src: str = "https://img.n1.ru/photo.jpg", + ) -> None: + self._href = href + self._title_text = title_text + self._area_text = area_text + self._price_text = price_text + self._img_src = img_src + + def css_first(self, selector: str) -> _FakeNode | None: + if 'a[href*="/view/"]' in selector: + return self + if "total-area" in selector: + return _FakeTextNode(self._area_text) + if "param-price" in selector: + return _FakeTextNode(self._price_text) + return None + + def css(self, selector: str) -> list[_FakeNode]: + if selector == "img": + return [_FakeImgNode(self._img_src)] + return [] + + @property + def attributes(self) -> dict: + return {"href": self._href} + + def text(self, strip: bool = False) -> str: + t = self._title_text + return t.strip() if strip else t + + +class _FakeTextNode: + def __init__(self, text: str) -> None: + self._text = text + + def text(self, strip: bool = False) -> str: + return self._text.strip() if strip else self._text + + +class _FakeImgNode: + def __init__(self, src: str) -> None: + self.attributes = {"src": src} + + +def _call_card_to_lot(card: _FakeNode): # type: ignore[return] + from app.services.scrapers.n1 import N1Scraper + + scraper = object.__new__(N1Scraper) + scraper.base_url = "https://ekaterinburg.n1.ru" # type: ignore[attr-defined] + return scraper._card_to_lot(card) # type: ignore[attr-defined] + + +def test_card_to_lot_parses_floor() -> None: + card = _FakeNode(title_text="1/11 2-к, Репина, 75/2 стр.") + lot = _call_card_to_lot(card) + assert lot is not None + assert lot.floor == 1 + assert lot.total_floors == 11 + + +def test_card_to_lot_floor_none_when_no_token() -> None: + card = _FakeNode(title_text="2-к, Репина, 75/2 стр.") + lot = _call_card_to_lot(card) + assert lot is not None + assert lot.floor is None + assert lot.total_floors is None + + +def test_card_to_lot_listing_date_is_none() -> None: + """N1 SERP has no date — listing_date must remain None.""" + card = _FakeNode(title_text="3/9 1-к, Малышева, 12") + lot = _call_card_to_lot(card) + assert lot is not None + assert lot.listing_date is None