fix(tradein-avito): strip Emotion CSS from listings.address
Avito DOM leaks inline <style>.css-XXX{...}</style> inside the location <p>,
which gets pulled by .text(strip=True). Add _clean_address() helper that strips
CSS rules + post-address noise (Площадь N года, от N мин.), apply at extraction
site. Also applied to address_full in avito_detail.py via import from avito.py.
Backfill migration cleans the ~hundreds of historical rows.
Source: estimate a0a0b820-e8a8-4eee-aa73-0ab3b98ac233 analog #4.
This commit is contained in:
parent
02b48b8d04
commit
9e1c7d5147
4 changed files with 65 additions and 2 deletions
|
|
@ -205,7 +205,7 @@ class AvitoScraper(BaseScraper):
|
|||
if loc_el is not None:
|
||||
street_el = loc_el.css_first("p")
|
||||
if street_el is not None:
|
||||
address = street_el.text(strip=True) or None
|
||||
address = _clean_address(street_el.text(strip=True))
|
||||
|
||||
# Фото
|
||||
photo_urls: list[str] = []
|
||||
|
|
@ -267,6 +267,29 @@ _RE_STUDIO = re.compile(r"студи[яиюей]", re.IGNORECASE)
|
|||
_RE_AREA = re.compile(r"(\d+[.,]?\d*)\s*м[²2]", re.IGNORECASE)
|
||||
_RE_FLOOR = re.compile(r"(\d+)\s*/\s*(\d+)\s*эт\.?", re.IGNORECASE)
|
||||
|
||||
_CSS_NOISE_RE = re.compile(r"\.?css-[a-z0-9_-]+\s*\{[^}]*\}", flags=re.I)
|
||||
_NOT_ADDRESS_TAIL_RE = re.compile(
|
||||
r"\s*(Площадь \d|от \d+\s?мин\.|css-[a-z0-9_-]+)",
|
||||
flags=re.I,
|
||||
)
|
||||
|
||||
|
||||
def _clean_address(raw: str | None) -> str | None:
|
||||
"""Strip Emotion CSS and post-address noise that Avito DOM leaks via .text().
|
||||
|
||||
Examples:
|
||||
"ул. Токарей, 56к1Площадь 1905 года.css-39hgr0{fill:..."
|
||||
-> "ул. Токарей, 56к1"
|
||||
"ул. Малышева, 1.css-xxx{...}"
|
||||
-> "ул. Малышева, 1"
|
||||
"""
|
||||
if not raw:
|
||||
return None
|
||||
cleaned = _CSS_NOISE_RE.sub("", raw)
|
||||
cleaned = _NOT_ADDRESS_TAIL_RE.split(cleaned, maxsplit=1)[0]
|
||||
cleaned = cleaned.strip(" ,.\n\t")
|
||||
return cleaned or None
|
||||
|
||||
|
||||
def _extract_rooms_from_title(title: str) -> int | None:
|
||||
if _RE_STUDIO.search(title):
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@ from selectolax.parser import HTMLParser, Node
|
|||
from sqlalchemy import text
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.services.scrapers.avito import _clean_address
|
||||
from app.services.scrapers.avito_exceptions import AvitoBlockedError, AvitoRateLimitedError
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
@ -297,7 +298,7 @@ def parse_detail_html(html: str, source_url: str) -> DetailEnrichment:
|
|||
lon = _try_float(map_el.attributes.get("data-map-lon"))
|
||||
avito_location_id = _try_int(map_el.attributes.get("data-location-id"))
|
||||
|
||||
address_full = _text(tree, "[itemprop='address']")
|
||||
address_full = _clean_address(_text(tree, "[itemprop='address']"))
|
||||
|
||||
# ── Description ─────────────────────────────────────────────────────────
|
||||
description = _text(tree, "[data-marker='item-view/item-description']")
|
||||
|
|
|
|||
15
tradein-mvp/backend/data/sql/062_clean_avito_addresses.sql
Normal file
15
tradein-mvp/backend/data/sql/062_clean_avito_addresses.sql
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
-- 062_clean_avito_addresses.sql
|
||||
-- Strip CSS noise from listings.address for Avito source (one-shot backfill).
|
||||
-- Idempotent: re-run is safe (regexp_replace on already-clean string is a no-op).
|
||||
BEGIN;
|
||||
|
||||
UPDATE listings
|
||||
SET address = trim(both ' ,.' FROM
|
||||
regexp_replace(
|
||||
regexp_replace(address, '\.?css-[a-z0-9_-]+\s*\{[^}]*\}', '', 'gi'),
|
||||
'\s*(Площадь \d|от \d+\s?мин\.|css-[a-z0-9_-]+).*$', '', 'i'
|
||||
))
|
||||
WHERE source = 'avito'
|
||||
AND (address ~* 'css-[a-z0-9_-]+' OR address ~* 'Площадь \d|от \d+\s?мин\.');
|
||||
|
||||
COMMIT;
|
||||
24
tradein-mvp/backend/tests/test_avito_address_clean.py
Normal file
24
tradein-mvp/backend/tests/test_avito_address_clean.py
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
from app.services.scrapers.avito import _clean_address
|
||||
|
||||
|
||||
def test_strips_css_noise():
|
||||
raw = (
|
||||
"ул. Токарей, 56к1Площадь 1905 года.css-39hgr0{fill:currentColor;"
|
||||
"height:1em;box-sizing:border-box;}.css-39hgr0:focus{outline:none;}"
|
||||
"от 31 мин.Геологическая.css-39hgr0{...}"
|
||||
)
|
||||
assert _clean_address(raw) == "ул. Токарей, 56к1"
|
||||
|
||||
|
||||
def test_clean_address_passthrough():
|
||||
assert _clean_address("ул. Малышева, 1") == "ул. Малышева, 1"
|
||||
|
||||
|
||||
def test_clean_handles_none():
|
||||
assert _clean_address(None) is None
|
||||
assert _clean_address("") is None
|
||||
assert _clean_address(" ") is None
|
||||
|
||||
|
||||
def test_clean_strips_trailing_punctuation():
|
||||
assert _clean_address("ул. Ленина, 5,") == "ул. Ленина, 5"
|
||||
Loading…
Add table
Reference in a new issue