feat(scrapers/n1): извлекать floor/total_floors из ведущего N/M-токена (Refs #795)
n1.py не извлекал floor/total_floors. Этаж в ведущем токене N/M карточки
('1/11 ...'→этаж 1 из 11). _extract_floor_from_title + ^-anchored regex
(дом '75/2' не матчится), валидация диапазонов. listing_date не трогался.
19 тестов, 24 pass, ruff clean.
This commit is contained in:
parent
4c275e04ea
commit
7eb4d6d255
2 changed files with 224 additions and 16 deletions
|
|
@ -7,7 +7,8 @@ SSR HTML с DOM-карточками `data-test="offers-list-item"`.
|
|||
|
||||
Карточка:
|
||||
- a[href^="/view/NNN/"] — ссылка на объявление + offer_id
|
||||
текст ссылки = "3-к, Репина, 75/2 стр." (комнаты + адрес)
|
||||
текст ссылки = "1/11 2-к, Репина, 75/2 стр." (этаж/этажность + комнаты + адрес)
|
||||
Этаж/этажность — leading-токен «N/M» перед сегментом комнат.
|
||||
- [data-test="offers-list-item-param-total-area"] — "79 м2"
|
||||
- [data-test="offers-list-item-param-price"] — "15 700 000"
|
||||
- img.src — фото
|
||||
|
|
@ -66,8 +67,12 @@ class N1Scraper(BaseScraper):
|
|||
await super().__aexit__(*args)
|
||||
|
||||
async def fetch_around(
|
||||
self, lat: float, lon: float, radius_m: int = 1000,
|
||||
rooms: int | None = None, page: int = 1,
|
||||
self,
|
||||
lat: float,
|
||||
lon: float,
|
||||
radius_m: int = 1000,
|
||||
rooms: int | None = None,
|
||||
page: int = 1,
|
||||
) -> list[ScrapedLot]:
|
||||
self._anchor_lat = lat
|
||||
self._anchor_lon = lon
|
||||
|
|
@ -87,13 +92,21 @@ class N1Scraper(BaseScraper):
|
|||
lots = self._parse_html(response.text)
|
||||
logger.info(
|
||||
"n1: %d lots around (%.4f, %.4f) rooms=%s page=%d",
|
||||
len(lots), lat, lon, rooms, page,
|
||||
len(lots),
|
||||
lat,
|
||||
lon,
|
||||
rooms,
|
||||
page,
|
||||
)
|
||||
await self.sleep_between_requests()
|
||||
return lots
|
||||
|
||||
async def fetch_around_multi_room(
|
||||
self, lat: float, lon: float, radius_m: int = 1000, pages: int = 8,
|
||||
self,
|
||||
lat: float,
|
||||
lon: float,
|
||||
radius_m: int = 1000,
|
||||
pages: int = 8,
|
||||
) -> list[ScrapedLot]:
|
||||
"""Скрейп N1 по сегментам комнат 1/2/3/4 × N страниц — расширяет выборку."""
|
||||
seen: dict[str, ScrapedLot] = {}
|
||||
|
|
@ -102,9 +115,7 @@ class N1Scraper(BaseScraper):
|
|||
try:
|
||||
lots = await self.fetch_around(lat, lon, radius_m, rooms=rooms, page=page)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"n1 multi-room fetch failed rooms=%s page=%d", rooms, page
|
||||
)
|
||||
logger.exception("n1 multi-room fetch failed rooms=%s page=%d", rooms, page)
|
||||
continue
|
||||
if not lots:
|
||||
break # пустая страница → дальше нет смысла
|
||||
|
|
@ -112,9 +123,7 @@ class N1Scraper(BaseScraper):
|
|||
key = lot.source_id or lot.source_url
|
||||
if key and key not in seen:
|
||||
seen[key] = lot
|
||||
logger.info(
|
||||
"n1 multi-room: %d unique lots around (%.4f, %.4f)", len(seen), lat, lon
|
||||
)
|
||||
logger.info("n1 multi-room: %d unique lots around (%.4f, %.4f)", len(seen), lat, lon)
|
||||
return list(seen.values())
|
||||
|
||||
def _build_url(self, rooms: int | None = None, page: int = 1) -> str:
|
||||
|
|
@ -137,9 +146,7 @@ class N1Scraper(BaseScraper):
|
|||
def _parse_html(self, html: str) -> list[ScrapedLot]:
|
||||
tree = HTMLParser(html)
|
||||
cards = tree.css('[data-test="offers-list-item"]')
|
||||
return [
|
||||
lot for lot in (self._card_to_lot(c) for c in cards) if lot is not None
|
||||
]
|
||||
return [lot for lot in (self._card_to_lot(c) for c in cards) if lot is not None]
|
||||
|
||||
def _card_to_lot(self, card: Any) -> ScrapedLot | None:
|
||||
try:
|
||||
|
|
@ -152,10 +159,11 @@ class N1Scraper(BaseScraper):
|
|||
offer_id = m_id.group(1) if m_id else None
|
||||
url = href if href.startswith("http") else f"{self.base_url}{href}"
|
||||
|
||||
# Заголовок ссылки: "3-к, Репина, 75/2 стр."
|
||||
# Заголовок ссылки: "1/11 2-к, Репина, 75/2 стр." (этаж/этажность + комнаты + адрес)
|
||||
title = link.text(strip=True) or ""
|
||||
floor, total_floors = _extract_floor_from_title(title)
|
||||
rooms = _extract_rooms(title)
|
||||
# Адрес = title без префикса комнат
|
||||
# Адрес = title без ведущего «N/M» и без префикса комнат
|
||||
address = re.sub(
|
||||
r"^\s*(\d+-к|студи[яи])\s*,?\s*", "", title, flags=re.IGNORECASE
|
||||
).strip()
|
||||
|
|
@ -193,6 +201,8 @@ class N1Scraper(BaseScraper):
|
|||
lon=lon,
|
||||
rooms=rooms,
|
||||
area_m2=area,
|
||||
floor=floor,
|
||||
total_floors=total_floors,
|
||||
price_rub=price,
|
||||
photo_urls=photos,
|
||||
raw_payload={"title": title},
|
||||
|
|
@ -209,6 +219,11 @@ _RE_STUDIO = re.compile(r"студи", re.IGNORECASE)
|
|||
_RE_AREA = re.compile(r"([\d]+(?:[.,]\d+)?)")
|
||||
_RE_DIGITS = re.compile(r"\d")
|
||||
|
||||
# Ведущий токен «N/M» в тексте карточки N1: «1/11 2-к, Репина, 75/2 стр.»
|
||||
# Якорь ^ гарантирует: только если N/M стоит В НАЧАЛЕ строки (до сегмента комнат).
|
||||
# Дробь в адресе вида «75/2» в середине строки НЕ совпадёт.
|
||||
_RE_FLOOR_LEADING = re.compile(r"^\s*(\d{1,3})\s*/\s*(\d{1,3})\b")
|
||||
|
||||
|
||||
def _extract_rooms(text: str) -> int | None:
|
||||
if _RE_STUDIO.search(text):
|
||||
|
|
@ -230,3 +245,27 @@ def _extract_area(text: str) -> float | None:
|
|||
def _extract_price(text: str) -> int | None:
|
||||
digits = "".join(_RE_DIGITS.findall(text))
|
||||
return int(digits) if digits else None
|
||||
|
||||
|
||||
def _extract_floor_from_title(text: str) -> tuple[int | None, int | None]:
|
||||
"""Извлекает этаж и этажность из ведущего токена «N/M» в тексте карточки N1.
|
||||
|
||||
N1 SERP отдаёт текст вида «1/11 2-к, Репина, 75/2 стр.» — этаж/этажность
|
||||
идут первыми, до сегмента комнат. Регулярка привязана к началу строки (^),
|
||||
поэтому дробь в номере дома («75/2») в середине строки не захватывается.
|
||||
|
||||
Sanity-проверки:
|
||||
- floor >= 1
|
||||
- total_floors >= floor
|
||||
- total_floors <= 100
|
||||
|
||||
Возвращает (None, None) при отсутствии совпадения или нарушении диапазона.
|
||||
"""
|
||||
m = _RE_FLOOR_LEADING.match(text)
|
||||
if m is None:
|
||||
return None, None
|
||||
floor = int(m.group(1))
|
||||
total_floors = int(m.group(2))
|
||||
if floor < 1 or total_floors < floor or total_floors > 100:
|
||||
return None, None
|
||||
return floor, total_floors
|
||||
|
|
|
|||
169
tradein-mvp/backend/tests/test_n1_floor_parse.py
Normal file
169
tradein-mvp/backend/tests/test_n1_floor_parse.py
Normal file
|
|
@ -0,0 +1,169 @@
|
|||
"""Unit tests for N1 floor/total_floors extraction (#795).
|
||||
|
||||
Tests _extract_floor_from_title() directly (pure function, no network/DB).
|
||||
Also covers _card_to_lot integration via a minimal fake card HTML node.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from app.services.scrapers.n1 import _extract_floor_from_title
|
||||
|
||||
# ── _extract_floor_from_title: direct unit tests ────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"text, expected_floor, expected_total",
|
||||
[
|
||||
# Typical N1 card: floor 1 of 11
|
||||
("1/11 2-к, Репина, 75/2 стр.", 1, 11),
|
||||
# Floor in middle of building
|
||||
("5/9 3-к, Ленина, 10 стр.", 5, 9),
|
||||
# Top floor
|
||||
("17/17 1-к, Малышева, 12", 17, 17),
|
||||
# Leading whitespace tolerated
|
||||
(" 3/16 студия, Вайнера, 5", 3, 16),
|
||||
# Three-digit total floors (edge of range)
|
||||
("2/100 2-к, Тверитина, 8", 2, 100),
|
||||
],
|
||||
)
|
||||
def test_extract_floor_happy_path(text: str, expected_floor: int, expected_total: int) -> None:
|
||||
floor, total = _extract_floor_from_title(text)
|
||||
assert floor == expected_floor
|
||||
assert total == expected_total
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"text",
|
||||
[
|
||||
# No leading N/M at all — classic old format without floor token
|
||||
("2-к, Репина, 75/2 стр."),
|
||||
# House number fraction in middle — must NOT be parsed as floor
|
||||
("Репина, 75/2 стр."),
|
||||
# N/M not at string start (preceded by other text)
|
||||
("квартира 3/9, Ленина, 1"),
|
||||
# floor=0 is invalid
|
||||
("0/9 1-к, Тестовая, 1"),
|
||||
# total_floors < floor — sanity fail
|
||||
("9/5 2-к, Тестовая, 1"),
|
||||
# total_floors > 100 — sanity fail
|
||||
("2/101 1-к, Тестовая, 1"),
|
||||
# Empty string
|
||||
(""),
|
||||
# Only digits, no slash
|
||||
("5 2-к, Тестовая, 1"),
|
||||
],
|
||||
)
|
||||
def test_extract_floor_returns_none(text: str) -> None:
|
||||
floor, total = _extract_floor_from_title(text)
|
||||
assert floor is None
|
||||
assert total is None
|
||||
|
||||
|
||||
def test_house_number_fraction_not_misparsed() -> None:
|
||||
"""«75/2 стр.» appearing after rooms segment must not match floor regex."""
|
||||
# The fraction 75/2 comes AFTER the rooms prefix, not at string start.
|
||||
floor, total = _extract_floor_from_title("3-к, Репина, 75/2 стр.")
|
||||
assert floor is None
|
||||
assert total is None
|
||||
|
||||
|
||||
def test_non_leading_fraction_after_comma() -> None:
|
||||
"""Fraction embedded mid-string via comma-separated address token."""
|
||||
floor, total = _extract_floor_from_title("ул. 8 Марта, 51/3, 2-к")
|
||||
assert floor is None
|
||||
assert total is None
|
||||
|
||||
|
||||
# ── Integration: _card_to_lot via fake selectolax node ──────────────────────
|
||||
|
||||
|
||||
class _FakeAttr(dict):
|
||||
"""Minimal attributes dict for a fake HTML node."""
|
||||
|
||||
|
||||
class _FakeNode:
|
||||
"""Minimal selectolax node stub for _card_to_lot tests."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
href: str = "/view/12345/",
|
||||
title_text: str = "1/11 2-к, Репина, 75/2 стр.",
|
||||
area_text: str = "52 м2",
|
||||
price_text: str = "8 500 000",
|
||||
img_src: str = "https://img.n1.ru/photo.jpg",
|
||||
) -> None:
|
||||
self._href = href
|
||||
self._title_text = title_text
|
||||
self._area_text = area_text
|
||||
self._price_text = price_text
|
||||
self._img_src = img_src
|
||||
|
||||
def css_first(self, selector: str) -> _FakeNode | None:
|
||||
if 'a[href*="/view/"]' in selector:
|
||||
return self
|
||||
if "total-area" in selector:
|
||||
return _FakeTextNode(self._area_text)
|
||||
if "param-price" in selector:
|
||||
return _FakeTextNode(self._price_text)
|
||||
return None
|
||||
|
||||
def css(self, selector: str) -> list[_FakeNode]:
|
||||
if selector == "img":
|
||||
return [_FakeImgNode(self._img_src)]
|
||||
return []
|
||||
|
||||
@property
|
||||
def attributes(self) -> dict:
|
||||
return {"href": self._href}
|
||||
|
||||
def text(self, strip: bool = False) -> str:
|
||||
t = self._title_text
|
||||
return t.strip() if strip else t
|
||||
|
||||
|
||||
class _FakeTextNode:
|
||||
def __init__(self, text: str) -> None:
|
||||
self._text = text
|
||||
|
||||
def text(self, strip: bool = False) -> str:
|
||||
return self._text.strip() if strip else self._text
|
||||
|
||||
|
||||
class _FakeImgNode:
|
||||
def __init__(self, src: str) -> None:
|
||||
self.attributes = {"src": src}
|
||||
|
||||
|
||||
def _call_card_to_lot(card: _FakeNode): # type: ignore[return]
|
||||
from app.services.scrapers.n1 import N1Scraper
|
||||
|
||||
scraper = object.__new__(N1Scraper)
|
||||
scraper.base_url = "https://ekaterinburg.n1.ru" # type: ignore[attr-defined]
|
||||
return scraper._card_to_lot(card) # type: ignore[attr-defined]
|
||||
|
||||
|
||||
def test_card_to_lot_parses_floor() -> None:
|
||||
card = _FakeNode(title_text="1/11 2-к, Репина, 75/2 стр.")
|
||||
lot = _call_card_to_lot(card)
|
||||
assert lot is not None
|
||||
assert lot.floor == 1
|
||||
assert lot.total_floors == 11
|
||||
|
||||
|
||||
def test_card_to_lot_floor_none_when_no_token() -> None:
|
||||
card = _FakeNode(title_text="2-к, Репина, 75/2 стр.")
|
||||
lot = _call_card_to_lot(card)
|
||||
assert lot is not None
|
||||
assert lot.floor is None
|
||||
assert lot.total_floors is None
|
||||
|
||||
|
||||
def test_card_to_lot_listing_date_is_none() -> None:
|
||||
"""N1 SERP has no date — listing_date must remain None."""
|
||||
card = _FakeNode(title_text="3/9 1-к, Малышева, 12")
|
||||
lot = _call_card_to_lot(card)
|
||||
assert lot is not None
|
||||
assert lot.listing_date is None
|
||||
Loading…
Add table
Reference in a new issue