Merge pull request 'fix(tradein): oblast avito-sweep city-slug + Tier-2b geo-guard (R2, latent)' (#2500) from fix/tradein-scrapers-oblast into main
All checks were successful
Deploy Trade-In / changes (push) Successful in 13s
Deploy Trade-In / build-frontend (push) Successful in 2m32s
Deploy Trade-In / deploy (push) Successful in 1m37s
Deploy Trade-In / build-browser (push) Has been skipped
Deploy Trade-In / test (push) Successful in 59s
Deploy Trade-In / build-backend (push) Successful in 1m47s

This commit is contained in:
bot-backend 2026-07-12 19:42:29 +00:00
commit 51473fc841
7 changed files with 366 additions and 20 deletions

View file

@ -17,6 +17,7 @@ from sqlalchemy.orm import Session
from app.services.matching.normalize import (
address_fingerprint,
has_city_token,
has_house_number,
house_number_token,
normalize_address,
@ -24,6 +25,14 @@ from app.services.matching.normalize import (
logger = logging.getLogger(__name__)
# Tier-2b oblast guard radius (metres). A bare street+number normalized_address carries
# NO city token yet is globally unique in house_address_aliases, so 'улица ленина 100'
# physically exists in ЕКБ, Н.Тагил, Каменск… When the incoming card HAS coords we accept
# the alias only if its house geom is within this radius. Cities in обл.66 are ≥ ~9-15 km
# apart, so 3 km cleanly separates them while absorbing the intra-city geocoder drift that
# Tier-2b exists to bridge (same building, different provider coords).
_TIER2B_GUARD_M = 3000
def match_or_create_house(
db: Session,
@ -193,6 +202,35 @@ def match_or_create_house(
# Gated on has_num (P1): never match a bare-street normalized_address — any
# numberless listing would otherwise collapse into whichever house first
# registered that street.
#
# Oblast guard (bug #2, обл.66 per-city rollout): normalized_address has NO city
# token, so a bare street+number is globally unique in the alias table yet exists
# physically in several cities. Accept the match only when the incoming listing is
# confirmably the same place:
# • coords present → aliased house geom within _TIER2B_GUARD_M metres;
# • coords absent → norm_addr carries a city token (self-disambiguating —
# 'екатеринбург улица ленина 5' can't collide with a Н.Тагил address);
# • neither → skip. A bare common-street with no geo signal is too
# ambiguous — matching would mis-bucket an oblast card into a same-named ЕКБ
# house. Conservative: fall through to a New house rather than mis-match.
if lat is not None and lon is not None:
row = (
db.execute(
text(
"SELECT a.house_id FROM house_address_aliases a "
"JOIN houses h ON h.id = a.house_id "
"WHERE a.normalized_address = :na "
" AND h.geom IS NOT NULL "
" AND ST_DWithin(h.geom::geography, "
" ST_MakePoint(:lon, :lat)::geography, :thr) "
"LIMIT 1"
),
{"na": norm_addr, "lon": lon, "lat": lat, "thr": _TIER2B_GUARD_M},
)
.mappings()
.first()
)
elif has_city_token(norm_addr):
row = (
db.execute(
text(
@ -204,6 +242,12 @@ def match_or_create_house(
.mappings()
.first()
)
else:
logger.info(
"house tier2b skip: bare common-street, no coords/city na=%r src=%s",
norm_addr,
ext_source,
)
if row:
house_id = int(row["house_id"])
_upsert_house_source(
@ -520,6 +564,12 @@ def _insert_alias(
row with the latest fingerprint, which is then found by Tier 2a on the next scrape.
house_id is not updated on conflict: the first writer wins canonical ownership.
Oblast guard (bug #2): the fingerprint/source are refreshed ONLY when the conflicting
row belongs to the SAME house. A DIFFERENT house colliding on a bare common-street
normalized_address (an oblast card that fell through Tier-2b's guard to a New INSERT)
must NOT rewrite the owner's fingerprint — doing so would downgrade a safe coord-bearing
alias to a coord-less one and let the next no-coord card mis-bucket via Tier 2a.
P1: a bare-street normalized_address (no house number) is NOT registered as an
alias it is too ambiguous to serve as a building key. Any later numberless
listing would otherwise hit that alias via Tier 2b and be mass-dumped into the
@ -533,8 +583,12 @@ def _insert_alias(
INSERT INTO house_address_aliases (house_id, normalized_address, fingerprint, source)
VALUES (CAST(:hid AS bigint), :na, :fp, :src)
ON CONFLICT (normalized_address) DO UPDATE SET
fingerprint = EXCLUDED.fingerprint,
source = EXCLUDED.source
fingerprint = CASE
WHEN house_address_aliases.house_id = EXCLUDED.house_id
THEN EXCLUDED.fingerprint ELSE house_address_aliases.fingerprint END,
source = CASE
WHEN house_address_aliases.house_id = EXCLUDED.house_id
THEN EXCLUDED.source ELSE house_address_aliases.source END
"""),
{
"hid": house_id,

View file

@ -112,6 +112,40 @@ def house_number_token(normalized: str | None) -> str | None:
return m.group(1) if m else None
# Cities covered by the обл.66 sweep rollout (ЕКБ + oblast per-city schedules).
# Normalized form: lowercase, hyphens collapsed to spaces (mirrors normalize_address,
# e.g. 'каменск-уральский' -> 'каменск уральский'). Mirrors CITY_ANCHORS in
# scraper_kit.orchestration.pipeline — extend together as the sweep adds cities.
_CITY_TOKENS: tuple[str, ...] = (
"екатеринбург",
"нижний тагил",
"каменск уральский",
"первоуральск",
"верхняя пышма",
"серов",
)
# Token-bounded (not substring) so a street named after a city does NOT false-positive:
# 'улица серова 5' has token 'серова' (trailing 'а'), never bare 'серов'.
_CITY_TOKEN_RE = re.compile(
r"(?:^|\s)(?:" + "|".join(re.escape(c) for c in _CITY_TOKENS) + r")(?:\s|$)"
)
def has_city_token(normalized: str | None) -> bool:
"""True if the normalized address carries a known обл.66 city token.
A city token makes the normalized_address city-scoped two addresses in different
cities can then never collapse to the same key, so the alias is a safe cross-source
building key. Used by the Tier-2b oblast guard in match_or_create_house: a bare
common-street address with NO city token AND no coords is too ambiguous to match a
globally-unique normalized_address alias (it would mis-bucket an oblast card into a
same-named ЕКБ house bug #2 oblast rollout).
"""
if not normalized:
return False
return _CITY_TOKEN_RE.search(normalized) is not None
def address_fingerprint(address: str | None, lat: float | None, lon: float | None) -> str:
"""SHA-256 fingerprint of normalized address + rounded coordinates (4 dp ≈ 11 m).

View file

@ -0,0 +1,84 @@
"""Avito SERP city-slug filter (#2487 oblast rollout).
`_parse_html` drops padding cards ("по всей России") when `avito_serp_ekb_only`
is on. The kept city-slug used to be hardcoded `/ekaterinburg/`, so an oblast
per-city sweep (`_job_avito_city_sweep` with a `city` param) discarded 100% of the
target city's cards. The slug is now parameterized via `AvitoScraper.target_city_slug`
(default "ekaterinburg", i.e. ЕКБ behavior unchanged when no target is set).
No network / no DB `_parse_html` runs on inline HTML with a stub config.
"""
from __future__ import annotations
import os
from types import SimpleNamespace
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
from scraper_kit.providers.avito.serp import AvitoScraper
_BASE_URL = "https://www.avito.ru/ekaterinburg/kvartiry/prodam-ASgBAgICAUSSA8YQ?p=1"
def _card(slug: str, item_id: str, price: int = 5_000_000) -> str:
"""Minimal Avito SERP card whose per-card href carries `slug` as the city segment."""
href = f"/{slug}/kvartiry/2k_kvartira_50_m_5_5et_{item_id}"
return (
f'<div data-marker="item" data-item-id="{item_id}">'
f'<a data-marker="item-title" href="{href}">2-к. квартира, 50 м², 5/9 эт.</a>'
f'<meta itemprop="price" content="{price}">'
f"</div>"
)
def _html(*cards: str) -> str:
return "<html><body>" + "".join(cards) + "</body></html>"
def _ekb_only_cfg() -> SimpleNamespace:
# _parse_html only reads config.avito_serp_ekb_only; a stub keeps the test
# deterministic regardless of AVITO_SERP_EKB_ONLY env in the runner.
return SimpleNamespace(avito_serp_ekb_only=True)
def test_parse_keeps_target_oblast_city_drops_off_target_padding() -> None:
"""Sweep targeting Н.Тагил keeps its cards; ЕКБ/Москва padding is dropped."""
s = AvitoScraper(_ekb_only_cfg(), target_city_slug="nizhniy_tagil") # type: ignore[arg-type]
html = _html(
_card("nizhniy_tagil", "nt1"), # target city → KEPT
_card("ekaterinburg", "ekb1"), # off-target padding → DROPPED
_card("moskva", "msk1"), # off-target padding → DROPPED
)
lots = s._parse_html(html, source_url_base=_BASE_URL)
urls = [lot.source_url or "" for lot in lots]
assert len(lots) == 1, urls
assert "/nizhniy_tagil/" in urls[0]
assert all("/ekaterinburg/" not in u for u in urls)
assert all("/moskva/" not in u for u in urls)
def test_parse_default_target_is_ekb_and_drops_oblast_padding() -> None:
"""No target (ЕКБ sweep/full-load): kept slug defaults to 'ekaterinburg' — behavior
identical to the previous hardcode. ЕКБ card kept, oblast padding dropped."""
s = AvitoScraper(_ekb_only_cfg(), target_city_slug=None) # type: ignore[arg-type]
html = _html(
_card("ekaterinburg", "e1"), # ЕКБ → KEPT
_card("nizhniy_tagil", "nt1"), # off-target padding → DROPPED
)
lots = s._parse_html(html, source_url_base=_BASE_URL)
urls = [lot.source_url or "" for lot in lots]
assert len(lots) == 1, urls
assert "/ekaterinburg/" in urls[0]
assert all("/nizhniy_tagil/" not in u for u in urls)
def test_parse_ekb_only_disabled_keeps_all_cities() -> None:
"""avito_serp_ekb_only=False → no city filter at all, every card is kept."""
cfg = SimpleNamespace(avito_serp_ekb_only=False)
s = AvitoScraper(cfg, target_city_slug="nizhniy_tagil") # type: ignore[arg-type]
html = _html(_card("nizhniy_tagil", "nt1"), _card("ekaterinburg", "e1"))
lots = s._parse_html(html, source_url_base=_BASE_URL)
assert len(lots) == 2

View file

@ -17,6 +17,7 @@ from app.services.matching.conflict_resolution import (
)
from app.services.matching.normalize import (
address_fingerprint,
has_city_token,
has_house_number,
house_number_token,
normalize_address,
@ -168,6 +169,37 @@ def test_fingerprint_all_none():
assert len(fp) == 32
# ---------------------------------------------------------------------------
# has_city_token — обл.66 city-token detection (Tier-2b oblast guard)
# ---------------------------------------------------------------------------
def test_has_city_token_detects_ekb():
assert has_city_token("екатеринбург улица ленина 5") is True
def test_has_city_token_detects_multiword_oblast_city():
assert has_city_token("нижний тагил улица ленина 100") is True
assert has_city_token("каменск уральский улица мира 7") is True
def test_has_city_token_bare_street_is_false():
# The collision-prone case: a bare street+number carries no city token.
assert has_city_token("улица ленина 100") is False
assert has_city_token("уральская улица 62к1") is False
def test_has_city_token_no_false_positive_on_street_named_after_city():
# 'улица серова' must NOT match the city token 'серов' (word-bounded, not substring).
assert has_city_token("улица серова 5") is False
assert has_city_token("первоуральская улица 3") is False
def test_has_city_token_handles_empty():
assert has_city_token("") is False
assert has_city_token(None) is False
# ---------------------------------------------------------------------------
# match_or_create_house — mock DB tier routing
# ---------------------------------------------------------------------------
@ -622,6 +654,125 @@ def test_insert_alias_executes_when_house_number_present():
db.execute.assert_called_once()
def test_insert_alias_conflict_only_rebinds_same_house():
"""Oblast guard (#2): ON CONFLICT refreshes fingerprint/source ONLY for the same
house. A different house colliding on a bare common-street normalized_address must
not rewrite the owner's fingerprint (that would downgrade a coord-bearing alias to a
coord-less one and let the next no-coord card mis-bucket via Tier 2a)."""
from app.services.matching.houses import _insert_alias
db = MagicMock()
_insert_alias(db, house_id=1, address="улица Ленина 100", fp="c" * 32, source="avito")
sql = " ".join(str(db.execute.call_args.args[0]).split())
assert "ON CONFLICT (normalized_address) DO UPDATE" in sql
# fingerprint/source updated only when the existing owner == the incoming house.
assert "house_address_aliases.house_id = EXCLUDED.house_id" in sql
# ---------------------------------------------------------------------------
# match_or_create_house — Tier-2b oblast geo/city guard (bug #2)
# ---------------------------------------------------------------------------
def test_tier2b_coord_present_uses_geo_guard_and_matches_within_radius():
"""Coords present → Tier-2b runs the ST_DWithin JOIN guard; a house within the
radius is accepted as a fingerprint match (0.9)."""
from app.services.matching.houses import match_or_create_house
db = _make_db(
[
None, # pg_advisory_xact_lock
None, # house_sources miss (Tier 1)
None, # fingerprint miss (Tier 2a)
{"house_id": 77}, # Tier 2b JOIN+ST_DWithin hit (same place, within radius)
None, # _upsert_house_source
None, # _insert_alias
]
)
house_id, conf, method = match_or_create_house(
db, "avito", "ext-2b-geo", address="улица Ленина 100", lat=57.910, lon=59.980
)
assert (house_id, conf, method) == (77, 0.9, "fingerprint")
# The Tier-2b lookup must be the geo-guarded JOIN, not a bare normalized_address SELECT.
tier2b_sql = _executed_sqls(db)[3]
assert "ST_DWithin" in tier2b_sql
assert "JOIN houses" in tier2b_sql
def test_tier2b_coord_present_cross_city_falls_through_to_new():
"""A bare street+number oblast card WITH coords whose only same-street alias is a
far-away (cross-city) house: the ST_DWithin guard filters it out (JOIN miss) the
card falls through to a New house instead of mis-bucketing into the ЕКБ house."""
from app.services.matching.houses import match_or_create_house
db = _make_db(
[
None, # pg_advisory_xact_lock
None, # house_sources miss
None, # fingerprint miss (Tier 2a)
None, # Tier 2b JOIN+ST_DWithin miss (candidate house is >3km away)
None, # geo miss (Tier 3 — coords present)
{"id": 909}, # INSERT RETURNING id (New house)
None, # _upsert_house_source
None, # _insert_alias
]
)
house_id, conf, method = match_or_create_house(
db, "avito", "ext-2b-xcity", address="улица Ленина 100", lat=57.910, lon=59.980
)
assert (house_id, conf, method) == (909, 1.0, "new")
def test_tier2b_no_coords_no_city_token_skips_and_creates_new():
"""No coords AND a bare common-street (no city token) → Tier-2b is SKIPPED entirely
(no normalized_address lookup). The card falls through to a New house conservative:
do not mis-bucket a bare 'улица ленина 100' into a same-named ЕКБ alias."""
from app.services.matching.houses import match_or_create_house
db = _make_db(
[
None, # pg_advisory_xact_lock
None, # house_sources miss
None, # fingerprint miss (Tier 2a); Tier 2b SKIPPED (no coords, no city token)
{"id": 910}, # INSERT RETURNING id (New house) — no Tier 3 (no coords)
None, # _upsert_house_source
None, # _insert_alias
]
)
house_id, conf, method = match_or_create_house(
db, "avito", "ext-2b-bare", address="улица Ленина 100"
)
assert (house_id, conf, method) == (910, 1.0, "new")
assert not any(
"normalized_address = :na" in s for s in _executed_sqls(db)
), "bare common-street with no coords/city must not run a Tier-2b lookup"
def test_tier2b_no_coords_with_city_token_matches():
"""No coords but the normalized_address carries a city token → self-disambiguating,
so Tier-2b matches on normalized_address alone (0.9). ЕКБ cross-source dedup for
city-qualified addresses is preserved."""
from app.services.matching.houses import match_or_create_house
db = _make_db(
[
None, # pg_advisory_xact_lock
None, # house_sources miss
None, # fingerprint miss (Tier 2a)
{"house_id": 33}, # Tier 2b normalized_address hit (city token present)
None, # _upsert_house_source
None, # _insert_alias
]
)
house_id, conf, method = match_or_create_house(
db, "avito", "ext-2b-city", address="Екатеринбург, улица Ленина, 100"
)
assert (house_id, conf, method) == (33, 0.9, "fingerprint")
tier2b_sql = _executed_sqls(db)[3]
assert "normalized_address = :na" in tier2b_sql
assert "ST_DWithin" not in tier2b_sql
# ---------------------------------------------------------------------------
# match_or_create_listing — mock DB tier routing
# ---------------------------------------------------------------------------

View file

@ -829,6 +829,7 @@ async def run_avito_city_sweep(
shutdown_requested: Callable[[], bool] = lambda: False,
proxy_provider: ProxyProvider | None = None,
anchors: list[tuple[float, float, str]] | None = None,
city_slug: str | None = None,
radius_m: int = 1500,
pages_per_anchor: int = 3,
enrich_houses: bool = True,
@ -853,6 +854,9 @@ async def run_avito_city_sweep(
вместо прямых импортов app.* (см. scraper_kit.contracts). process_houses_imv_batch
зовётся через enrichment (EnrichmentJobs Protocol).
"""
# city_slug (#2487): слаг города-цели (напр. "nizhniy_tagil") прокидывается в
# AvitoScraper.target_city_slug → SERP-фильтр оставляет карточки этого города.
# None → ЕКБ-дефолт (совпадает с anchors=EKB_ANCHORS fallback ниже).
_anchors = anchors if anchors is not None else EKB_ANCHORS
counters = CitySweepCounters(anchors_total=len(_anchors))
all_touched_house_ids: set[int] = set()
@ -959,7 +963,9 @@ async def run_avito_city_sweep(
nonlocal all_touched_house_ids, sweep_rotations_done
# ── Phase 1+2: SERP + save ─────────────────────────────────
scraper = AvitoScraper(config)
# #2487: target_city_slug → _parse_html оставляет карточки
# города-цели (oblast slug), а не хардкод /ekaterinburg/. None → ЕКБ.
scraper = AvitoScraper(config, target_city_slug=city_slug)
if browser_mode:
scraper._browser = shared_bf
# Shared-browser режим: _cffi=None → curl_cffi-fallback на

View file

@ -455,6 +455,9 @@ async def _job_avito_city_sweep(
shutdown_requested=ctx.shutdown_requested,
proxy_provider=ctx.proxy_provider,
anchors=anchors,
# #2487: слаг города-цели → SERP-фильтр _parse_html оставляет карточки
# этого города (иначе oblast-sweep дропал 100% из-за хардкода /ekaterinburg/).
city_slug=city,
pages_per_anchor=int(params.get("pages_per_anchor", 3)),
detail_top_n=int(params.get("detail_top_n", 20)),
request_delay_sec=float(params.get("request_delay_sec", 7.0)),

View file

@ -350,6 +350,7 @@ class AvitoScraper(BaseScraper):
config: ScraperConfig,
*,
delay_provider: Callable[[str], float] | None = None,
target_city_slug: str | None = None,
) -> None:
super().__init__()
# Strangler-инжекция (#2133): конфиг и провайдер задержки приходят снаружи
@ -363,6 +364,10 @@ class AvitoScraper(BaseScraper):
self._browser: BrowserFetcher | None = None
# #823: счётчик карточек, которые не удалось распарсить из-за неожиданной структуры DOM.
self.parse_failures: int = 0
# #2487 oblast rollout: slug города-цели sweep'а (напр. "nizhniy_tagil").
# None → ЕКБ (дефолт). Используется _parse_html-фильтром avito_serp_ekb_only,
# чтобы оставлять карточки TARGET-города, а не хардкодить /ekaterinburg/.
self._target_city_slug = target_city_slug
async def __aenter__(self) -> AvitoScraper:
# Проактивная ротация IP в начале sweep (#1731): apw-IP «протухает» по
@ -1714,8 +1719,11 @@ class AvitoScraper(BaseScraper):
лениво только у верхних карточек (#726); DOM-marker остаётся fallback.
Если self._config.avito_serp_ekb_only=True (дефолт), отбрасываем карточки,
чей per-card source_url не содержит /ekaterinburg/ Avito добивает выдачу
«по всей России» при малом числе ЕКБ-результатов (4+ комн., дорогие сегменты).
чей per-card source_url не содержит слаг ГОРОДА-ЦЕЛИ sweep'а — Avito добивает
выдачу «по всей России» при малом числе результатов (4+ комн., дорогие сегменты).
Город-цель = self._target_city_slug (напр. "nizhniy_tagil"), по умолчанию
"ekaterinburg". Без параметризации (#2487) oblast-sweep по городу вне ЕКБ терял
100% карточек: их per-card slug (/nizhniy_tagil/) не содержит /ekaterinburg/.
Карточки с нераспознанным city-slug (href без ведущего /city/) пропускаются
консервативно (не дропаются по ошибке).
"""
@ -1724,6 +1732,10 @@ class AvitoScraper(BaseScraper):
cards = tree.css('[data-marker="item"]')
lots: list[ScrapedLot] = []
ekb_only = self._config.avito_serp_ekb_only
# #2487: слаг города, карточки которого оставляем. Дефолт "ekaterinburg" —
# ЕКБ-поведение неизменно, когда target не задан (обычный ЕКБ-sweep/full-load).
kept_slug = self._target_city_slug or "ekaterinburg"
kept_seg = f"/{kept_slug}/"
dropped = 0
no_city = 0
for card in cards:
@ -1736,11 +1748,11 @@ class AvitoScraper(BaseScraper):
# urljoin всегда возвращает абсолютный URL (https://www.avito.ru/...).
# Avito city-slug — первый path-сегмент после netloc.
# Пример: https://www.avito.ru/ekaterinburg/... → /ekaterinburg/ присутствует.
# https://www.avito.ru/moskva/... → нет /ekaterinburg/.
# https://www.avito.ru/moskva/... → нет слага города-цели.
# Для href без явного city (/kvartiry/... — теоретически не должно быть,
# но консервативно пропускаем).
if "/ekaterinburg/" in card_url:
# ЕКБ-карточка — берём
if kept_seg in card_url:
# Карточка города-цели — берём
pass
else:
# Проверяем, есть ли вообще city-slug (хотя бы один path-сегмент
@ -1758,13 +1770,15 @@ class AvitoScraper(BaseScraper):
# Нет city-slug → консервативно берём карточку
lots.append(lot)
continue
# Есть city-slug, но не /ekaterinburg/ → padding по России → дропаем
# Есть city-slug, но не города-цели → padding по России → дропаем
dropped += 1
continue
lots.append(lot)
if ekb_only and (dropped or no_city):
logger.info(
"avito SERP: dropped %d non-EKB cards (padding), %d cards without city-slug kept",
"avito SERP (kept=%s): dropped %d off-target cards (padding), "
"%d cards without city-slug kept",
kept_slug,
dropped,
no_city,
)