fix(tradein-avito): strip Emotion CSS from listings.address

Avito DOM leaks inline <style>.css-XXX{...}</style> inside the location <p>,
which gets pulled by .text(strip=True). Add _clean_address() helper that strips
CSS rules + post-address noise (Площадь N года, от N мин.), apply at extraction
site. Also applied to address_full in avito_detail.py via import from avito.py.
Backfill migration cleans the ~hundreds of historical rows.

Source: estimate a0a0b820-e8a8-4eee-aa73-0ab3b98ac233 analog #4.
This commit is contained in:
lekss361 2026-05-24 13:28:41 +03:00
parent 02b48b8d04
commit 9e1c7d5147
4 changed files with 65 additions and 2 deletions

View file

@ -205,7 +205,7 @@ class AvitoScraper(BaseScraper):
if loc_el is not None: if loc_el is not None:
street_el = loc_el.css_first("p") street_el = loc_el.css_first("p")
if street_el is not None: if street_el is not None:
address = street_el.text(strip=True) or None address = _clean_address(street_el.text(strip=True))
# Фото # Фото
photo_urls: list[str] = [] photo_urls: list[str] = []
@ -267,6 +267,29 @@ _RE_STUDIO = re.compile(r"студи[яиюей]", re.IGNORECASE)
_RE_AREA = re.compile(r"(\d+[.,]?\d*)\s*м[²2]", re.IGNORECASE) _RE_AREA = re.compile(r"(\d+[.,]?\d*)\s*м[²2]", re.IGNORECASE)
_RE_FLOOR = re.compile(r"(\d+)\s*/\s*(\d+)\s*эт\.?", re.IGNORECASE) _RE_FLOOR = re.compile(r"(\d+)\s*/\s*(\d+)\s*эт\.?", re.IGNORECASE)
_CSS_NOISE_RE = re.compile(r"\.?css-[a-z0-9_-]+\s*\{[^}]*\}", flags=re.I)
_NOT_ADDRESS_TAIL_RE = re.compile(
r"\s*(Площадь \d|от \d+\s?мин\.|css-[a-z0-9_-]+)",
flags=re.I,
)
def _clean_address(raw: str | None) -> str | None:
"""Strip Emotion CSS and post-address noise that Avito DOM leaks via .text().
Examples:
"ул. Токарей, 56к1Площадь 1905 года.css-39hgr0{fill:..."
-> "ул. Токарей, 56к1"
"ул. Малышева, 1.css-xxx{...}"
-> "ул. Малышева, 1"
"""
if not raw:
return None
cleaned = _CSS_NOISE_RE.sub("", raw)
cleaned = _NOT_ADDRESS_TAIL_RE.split(cleaned, maxsplit=1)[0]
cleaned = cleaned.strip(" ,.\n\t")
return cleaned or None
def _extract_rooms_from_title(title: str) -> int | None: def _extract_rooms_from_title(title: str) -> int | None:
if _RE_STUDIO.search(title): if _RE_STUDIO.search(title):

View file

@ -32,6 +32,7 @@ from selectolax.parser import HTMLParser, Node
from sqlalchemy import text from sqlalchemy import text
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from app.services.scrapers.avito import _clean_address
from app.services.scrapers.avito_exceptions import AvitoBlockedError, AvitoRateLimitedError from app.services.scrapers.avito_exceptions import AvitoBlockedError, AvitoRateLimitedError
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -297,7 +298,7 @@ def parse_detail_html(html: str, source_url: str) -> DetailEnrichment:
lon = _try_float(map_el.attributes.get("data-map-lon")) lon = _try_float(map_el.attributes.get("data-map-lon"))
avito_location_id = _try_int(map_el.attributes.get("data-location-id")) avito_location_id = _try_int(map_el.attributes.get("data-location-id"))
address_full = _text(tree, "[itemprop='address']") address_full = _clean_address(_text(tree, "[itemprop='address']"))
# ── Description ───────────────────────────────────────────────────────── # ── Description ─────────────────────────────────────────────────────────
description = _text(tree, "[data-marker='item-view/item-description']") description = _text(tree, "[data-marker='item-view/item-description']")

View file

@ -0,0 +1,15 @@
-- 062_clean_avito_addresses.sql
-- Strip CSS noise from listings.address for Avito source (one-shot backfill).
-- Idempotent: re-run is safe (regexp_replace on already-clean string is a no-op).
BEGIN;
UPDATE listings
SET address = trim(both ' ,.' FROM
regexp_replace(
regexp_replace(address, '\.?css-[a-z0-9_-]+\s*\{[^}]*\}', '', 'gi'),
'\s*(Площадь \d|от \d+\s?мин\.|css-[a-z0-9_-]+).*$', '', 'i'
))
WHERE source = 'avito'
AND (address ~* 'css-[a-z0-9_-]+' OR address ~* 'Площадь \d|от \d+\s?мин\.');
COMMIT;

View file

@ -0,0 +1,24 @@
from app.services.scrapers.avito import _clean_address
def test_strips_css_noise():
raw = (
"ул. Токарей, 56к1Площадь 1905 года.css-39hgr0{fill:currentColor;"
"height:1em;box-sizing:border-box;}.css-39hgr0:focus{outline:none;}"
"от 31 мин.Геологическая.css-39hgr0{...}"
)
assert _clean_address(raw) == "ул. Токарей, 56к1"
def test_clean_address_passthrough():
assert _clean_address("ул. Малышева, 1") == "ул. Малышева, 1"
def test_clean_handles_none():
assert _clean_address(None) is None
assert _clean_address("") is None
assert _clean_address(" ") is None
def test_clean_strips_trailing_punctuation():
assert _clean_address("ул. Ленина, 5,") == "ул. Ленина, 5"