From f8818be9c7fc7949b859cf1eeee9f0d65d78deb1 Mon Sep 17 00:00:00 2001 From: bot-backend Date: Sun, 31 May 2026 11:59:47 +0300 Subject: [PATCH] =?UTF-8?q?fix(tradein):=20yandex=20address-enrich=20?= =?UTF-8?q?=E2=80=94=20city-agnostic=20title=20regex=20(=D1=81=D0=BF=D1=83?= =?UTF-8?q?=D1=82=D0=BD=D0=B8=D0=BA=D0=B8=20=D0=95=D0=9A=D0=91:=20=D0=92?= =?UTF-8?q?=D0=B5=D1=80=D1=85=D0=BD=D1=8F=D1=8F=20=D0=9F=D1=8B=D1=88=D0=BC?= =?UTF-8?q?=D0=B0/=D0=91=D0=B5=D1=80=D1=91=D0=B7=D0=BE=D0=B2=D1=81=D0=BA?= =?UTF-8?q?=D0=B8=D0=B9/...)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _RE_TITLE_ADDRESS жёстко требовал «Екатеринбург,» → sweep #28 возвращал 0 enriched при address_attempted=11 (все объявления из спутников ЕКБ). Заменил захват города на city-agnostic «[А-ЯЁ][^—<]+?,\s*[^<—]+?» + уточнил optional ЖК-prefix через negative lookahead (?![А-ЯЁ][а-яё]), чтобы prefix не «съедал» сам город (Екатеринбург, Верхняя Пышма, и т.д.). _RE_HAS_HOUSE_NUMBER guard не менялся. Tests: добавлены кейсы для реальных тайтлов Верхняя Пышма/Берёзовский/ЕКБ и проверка guard без номера дома. --- .../app/services/yandex_address_backfill.py | 21 +++-- .../tasks/test_yandex_address_backfill.py | 77 +++++++++++++++---- 2 files changed, 76 insertions(+), 22 deletions(-) diff --git a/tradein-mvp/backend/app/services/yandex_address_backfill.py b/tradein-mvp/backend/app/services/yandex_address_backfill.py index ba9586aa..1b5984f3 100644 --- a/tradein-mvp/backend/app/services/yandex_address_backfill.py +++ b/tradein-mvp/backend/app/services/yandex_address_backfill.py @@ -26,15 +26,20 @@ from sqlalchemy.orm import Session logger = logging.getLogger(__name__) -# Extract 'Екатеринбург, ' from — matches: +# Extract '<City>, <full address>' from <title> — matches any city, not just Ekaterinburg. +# Handles EKB metro-area satellite towns: Верхняя Пышма, Берёзовский, Среднеуральск, Арамиль, etc. +# Examples: # «Продажа квартиры — Екатеринбург, улица Горького, 36 — id 7654321 на Яндекс.Недвижимости» -# Group 1 = «Екатеринбург, улица Горького, 36» (stripped of trailing punctuation/spaces) +# «Купить квартиру … — ЖК «Дуэт», Верхняя Пышма, улица Мальцева, 14 — id 75332…» +# «Купить квартиру … — ЖК «Уют-Сити», Берёзовский, Александровский проспект, 3 — id 3988…» +# Group 1 = «<City>, <street/house>» (stripped of trailing punctuation/spaces) +# The optional ЖК-prefix (?:[^—,]*?,\s*) consumes «ЖК «Дуэт», » before the city name. +# City capture starts with [А-ЯЁ] to ensure the prefix never swallows the city itself. _RE_TITLE_ADDRESS = re.compile( r"<title>[^<]*?—\s*" - r"(?:[^—,]*?,\s*)?" # optional ЖК/жилой комплекс prefix - r"(Екатеринбург,\s*[^<—]+?)" + r"(?:(?![А-ЯЁ][а-яё])[^—,]*?,\s*)?" # optional ЖК-prefix — fires only when NOT a city name + r"([А-ЯЁ][^—<]+?,\s*[^<—]+?)" # <City>, <street/house> — any city (uppercase Cyrillic start) r"\s*—\s*id\s*\d+[^<]*", # allow trailing text after id NNN (e.g. 'на Яндекс...') - re.IGNORECASE, ) # Listings without a house number: address contains no digit after comma-space @@ -53,7 +58,11 @@ class YandexAddressBackfillResult: def _extract_address_from_title(html: str) -> str | None: - """Extract full address from Yandex offer detail page .""" + """Extract full address from Yandex offer detail page <title>. + + City-agnostic: works for EKB and satellite towns (Верхняя Пышма, Берёзовский, etc.). + Returns «<City>, <street>, <house>» or None if the title format is not recognised. + """ html_clean = html.replace("\xa0", " ") m = _RE_TITLE_ADDRESS.search(html_clean) if not m: diff --git a/tradein-mvp/backend/tests/tasks/test_yandex_address_backfill.py b/tradein-mvp/backend/tests/tasks/test_yandex_address_backfill.py index 3340fa86..2e59eb54 100644 --- a/tradein-mvp/backend/tests/tasks/test_yandex_address_backfill.py +++ b/tradein-mvp/backend/tests/tasks/test_yandex_address_backfill.py @@ -60,6 +60,67 @@ def test_extract_address_with_zhk_prefix() -> None: assert "Екатеринбург" in addr +# --------------------------------------------------------------------------- +# City-agnostic tests — satellite towns (T10 live-bug fix) +# --------------------------------------------------------------------------- + + +def test_extract_address_verkhnaya_pyshma_with_zhk() -> None: + """Real title from sweep #28: ЖК «Дуэт», Верхняя Пышма — was returning None before fix.""" + html = ( + "<title>Купить квартиру 102.4 м²" + " — ЖК «Дуэт», Верхняя Пышма, улица Мальцева, 14" + " — id 7533242317391171630" + ) + addr = _extract_address_from_title(html) + assert addr is not None, "Верхняя Пышма title should produce an address" + assert _RE_HAS_HOUSE_NUMBER.search(addr), f"Should contain house number: {addr!r}" + assert "Верхняя Пышма" in addr + assert "Мальцева" in addr + assert "14" in addr + + +def test_extract_address_beryozovsky_with_zhk() -> None: + """Real title from sweep #28: ЖК «Уют-Сити», Берёзовский — was returning None before fix.""" + html = ( + "Купить квартиру 139.2 м²" + " — ЖК «Уют-Сити», Берёзовский, Александровский проспект, 3" + " — id 3988368535316036933" + ) + addr = _extract_address_from_title(html) + assert addr is not None, "Берёзовский title should produce an address" + assert _RE_HAS_HOUSE_NUMBER.search(addr), f"Should contain house number: {addr!r}" + assert "Берёзовский" in addr + assert "Александровский" in addr + assert "3" in addr + + +def test_extract_address_ekb_no_zhk_still_matches() -> None: + """EKB variant from docstring (no ЖК prefix) still matches after city-agnostic change.""" + html = "Продажа квартиры — Екатеринбург, улица Горького, 36" " — id 7654321" + addr = _extract_address_from_title(html) + assert addr is not None, "EKB no-ЖК title should still match" + assert _RE_HAS_HOUSE_NUMBER.search(addr), f"Should contain house number: {addr!r}" + assert "Екатеринбург" in addr + assert "Горького" in addr + assert "36" in addr + + +def test_extract_address_no_house_number_guard() -> None: + """Street-only title (no house number): extractor may return value, but guard rejects it. + + _RE_HAS_HOUSE_NUMBER must NOT match → backfill skips the listing to avoid overwriting + with another street-only address. + """ + html = "Продажа квартиры — Екатеринбург, улица Мира — id 999888" + addr = _extract_address_from_title(html) + # If the regex matches at all, the extracted value must not contain a house number. + if addr is not None: + assert not _RE_HAS_HOUSE_NUMBER.search( + addr + ), f"Street-only addr must fail house-number guard: {addr!r}" + + def test_extract_address_nbsp_replaced() -> None: """\\xa0 (non-breaking space) in title should be normalised before regex match.""" html = ( @@ -87,22 +148,6 @@ def test_extract_address_title_without_id_returns_none() -> None: assert addr is None -def test_extract_address_street_only_still_extracted() -> None: - """Title with street-only (no house number) is extracted but fails the house-number check. - - The extractor itself does not gate on house number presence — the caller - (backfill_yandex_addresses) checks _RE_HAS_HOUSE_NUMBER separately and skips. - """ - html = "Продажа квартиры — Екатеринбург, улица Горького — id 7654321" - addr = _extract_address_from_title(html) - # May or may not match depending on whether «улица Горького» satisfies the regex — - # the key assertion is that IF it does, it does NOT contain a house number. - if addr is not None: - assert not _RE_HAS_HOUSE_NUMBER.search( - addr - ), "Street-only extracted address should NOT contain a house number" - - # --------------------------------------------------------------------------- # Task-level smoke tests (no real DB, no real HTTP) # ---------------------------------------------------------------------------