From 9e1c7d5147ff481ac3b47b1d57883d55ba7fa317 Mon Sep 17 00:00:00 2001 From: lekss361 Date: Sun, 24 May 2026 13:28:41 +0300 Subject: [PATCH] fix(tradein-avito): strip Emotion CSS from listings.address MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Avito DOM leaks inline inside the location

, which gets pulled by .text(strip=True). Add _clean_address() helper that strips CSS rules + post-address noise (Площадь N года, от N мин.), apply at extraction site. Also applied to address_full in avito_detail.py via import from avito.py. Backfill migration cleans the ~hundreds of historical rows. Source: estimate a0a0b820-e8a8-4eee-aa73-0ab3b98ac233 analog #4. --- .../backend/app/services/scrapers/avito.py | 25 ++++++++++++++++++- .../app/services/scrapers/avito_detail.py | 3 ++- .../data/sql/062_clean_avito_addresses.sql | 15 +++++++++++ .../backend/tests/test_avito_address_clean.py | 24 ++++++++++++++++++ 4 files changed, 65 insertions(+), 2 deletions(-) create mode 100644 tradein-mvp/backend/data/sql/062_clean_avito_addresses.sql create mode 100644 tradein-mvp/backend/tests/test_avito_address_clean.py diff --git a/tradein-mvp/backend/app/services/scrapers/avito.py b/tradein-mvp/backend/app/services/scrapers/avito.py index c6e19f0e..6a52ee15 100644 --- a/tradein-mvp/backend/app/services/scrapers/avito.py +++ b/tradein-mvp/backend/app/services/scrapers/avito.py @@ -205,7 +205,7 @@ class AvitoScraper(BaseScraper): if loc_el is not None: street_el = loc_el.css_first("p") if street_el is not None: - address = street_el.text(strip=True) or None + address = _clean_address(street_el.text(strip=True)) # Фото photo_urls: list[str] = [] @@ -267,6 +267,29 @@ _RE_STUDIO = re.compile(r"студи[яиюей]", re.IGNORECASE) _RE_AREA = re.compile(r"(\d+[.,]?\d*)\s*м[²2]", re.IGNORECASE) _RE_FLOOR = re.compile(r"(\d+)\s*/\s*(\d+)\s*эт\.?", re.IGNORECASE) +_CSS_NOISE_RE = re.compile(r"\.?css-[a-z0-9_-]+\s*\{[^}]*\}", flags=re.I) +_NOT_ADDRESS_TAIL_RE = re.compile( + r"\s*(Площадь \d|от \d+\s?мин\.|css-[a-z0-9_-]+)", + flags=re.I, +) + + +def _clean_address(raw: str | None) -> str | None: + """Strip Emotion CSS and post-address noise that Avito DOM leaks via .text(). + + Examples: + "ул. Токарей, 56к1Площадь 1905 года.css-39hgr0{fill:..." + -> "ул. Токарей, 56к1" + "ул. Малышева, 1.css-xxx{...}" + -> "ул. Малышева, 1" + """ + if not raw: + return None + cleaned = _CSS_NOISE_RE.sub("", raw) + cleaned = _NOT_ADDRESS_TAIL_RE.split(cleaned, maxsplit=1)[0] + cleaned = cleaned.strip(" ,.\n\t") + return cleaned or None + def _extract_rooms_from_title(title: str) -> int | None: if _RE_STUDIO.search(title): diff --git a/tradein-mvp/backend/app/services/scrapers/avito_detail.py b/tradein-mvp/backend/app/services/scrapers/avito_detail.py index 3ac657bd..49d7d614 100644 --- a/tradein-mvp/backend/app/services/scrapers/avito_detail.py +++ b/tradein-mvp/backend/app/services/scrapers/avito_detail.py @@ -32,6 +32,7 @@ from selectolax.parser import HTMLParser, Node from sqlalchemy import text from sqlalchemy.orm import Session +from app.services.scrapers.avito import _clean_address from app.services.scrapers.avito_exceptions import AvitoBlockedError, AvitoRateLimitedError logger = logging.getLogger(__name__) @@ -297,7 +298,7 @@ def parse_detail_html(html: str, source_url: str) -> DetailEnrichment: lon = _try_float(map_el.attributes.get("data-map-lon")) avito_location_id = _try_int(map_el.attributes.get("data-location-id")) - address_full = _text(tree, "[itemprop='address']") + address_full = _clean_address(_text(tree, "[itemprop='address']")) # ── Description ───────────────────────────────────────────────────────── description = _text(tree, "[data-marker='item-view/item-description']") diff --git a/tradein-mvp/backend/data/sql/062_clean_avito_addresses.sql b/tradein-mvp/backend/data/sql/062_clean_avito_addresses.sql new file mode 100644 index 00000000..2f1819e0 --- /dev/null +++ b/tradein-mvp/backend/data/sql/062_clean_avito_addresses.sql @@ -0,0 +1,15 @@ +-- 062_clean_avito_addresses.sql +-- Strip CSS noise from listings.address for Avito source (one-shot backfill). +-- Idempotent: re-run is safe (regexp_replace on already-clean string is a no-op). +BEGIN; + +UPDATE listings + SET address = trim(both ' ,.' FROM + regexp_replace( + regexp_replace(address, '\.?css-[a-z0-9_-]+\s*\{[^}]*\}', '', 'gi'), + '\s*(Площадь \d|от \d+\s?мин\.|css-[a-z0-9_-]+).*$', '', 'i' + )) + WHERE source = 'avito' + AND (address ~* 'css-[a-z0-9_-]+' OR address ~* 'Площадь \d|от \d+\s?мин\.'); + +COMMIT; diff --git a/tradein-mvp/backend/tests/test_avito_address_clean.py b/tradein-mvp/backend/tests/test_avito_address_clean.py new file mode 100644 index 00000000..e3b14a20 --- /dev/null +++ b/tradein-mvp/backend/tests/test_avito_address_clean.py @@ -0,0 +1,24 @@ +from app.services.scrapers.avito import _clean_address + + +def test_strips_css_noise(): + raw = ( + "ул. Токарей, 56к1Площадь 1905 года.css-39hgr0{fill:currentColor;" + "height:1em;box-sizing:border-box;}.css-39hgr0:focus{outline:none;}" + "от 31 мин.Геологическая.css-39hgr0{...}" + ) + assert _clean_address(raw) == "ул. Токарей, 56к1" + + +def test_clean_address_passthrough(): + assert _clean_address("ул. Малышева, 1") == "ул. Малышева, 1" + + +def test_clean_handles_none(): + assert _clean_address(None) is None + assert _clean_address("") is None + assert _clean_address(" ") is None + + +def test_clean_strips_trailing_punctuation(): + assert _clean_address("ул. Ленина, 5,") == "ул. Ленина, 5"