0817178efa
All checks were successful
CI Trade-In / changes (pull_request) Successful in 7s
CI / changes (pull_request) Successful in 8s
CI Trade-In / frontend-checks (pull_request) Has been skipped
CI / backend-tests (pull_request) Has been skipped
CI / frontend-tests (pull_request) Has been skipped
CI / openapi-codegen-check (pull_request) Has been skipped
CI Trade-In / backend-tests (pull_request) Successful in 4m55s
avito/cian/yandex хардкодили ЕКБ (path /ekaterinburg/, region 4743, rgid 559132)
мимо anchor'а oblast-города → city-sweep целевого города скрейпил ЕКБ, n_analogs=0.
- pipeline: CityLocation + CITY_LOCATIONS (5 городов Свердловской обл.) + get_city_location;
прокинуто в run_{avito,cian,yandex}_city_sweep
- avito: target_city_slug (avito_slug) теперь в ПУТЬ URL (4 билдера), не только фильтр #2487
- cian: city_region_id → region= (subdomain ekb.cian.ru общий для области, не меняется)
- yandex: city_rgid → rgid=
- scheduler: city_slug прокинут в cian/yandex jobs (avito уже был)
Все новые параметры default None → ЕКБ-поведение byte-identical (back-compat).
Tests: +9 URL-builder тестов; full backend suite 2514 passed.
104 lines
4.6 KiB
Python
104 lines
4.6 KiB
Python
"""Per-city SERP location URL builders (#12 fix EKB-leak в oblast city-sweep).
|
||
|
||
Проблема: avito/cian/yandex провайдеры хардкодили ЕКБ-локацию (путь /ekaterinburg/,
|
||
CIAN_EKB_REGION_ID, _EKB_RGID) мимо anchor'а oblast-города → city-sweep целевого
|
||
города (напр. Н.Тагил) фактически скрейпил ЕКБ, n_analogs=0. Фикс — новые
|
||
kw-параметры (avito: target_city_slug уже был #2487, теперь и в ПУТИ URL;
|
||
cian: city_region_id; yandex: city_rgid), все дефолт None → ЕКБ-поведение
|
||
byte-identical прежнему (back-compat critical).
|
||
|
||
No network / no DB — чистые URL-билдеры на stub-конфиге.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import os
|
||
from types import SimpleNamespace
|
||
from urllib.parse import parse_qs, urlparse
|
||
|
||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
|
||
|
||
from scraper_kit.providers.avito.serp import NOVOSTROYKA_SLUG, AvitoScraper
|
||
from scraper_kit.providers.cian.serp import CIAN_EKB_REGION_ID, CianScraper
|
||
from scraper_kit.providers.yandex.serp import _EKB_RGID, YandexRealtyScraper
|
||
|
||
_STUB_CFG = SimpleNamespace()
|
||
|
||
|
||
def _query(url: str) -> dict[str, list[str]]:
|
||
return parse_qs(urlparse(url).query)
|
||
|
||
|
||
# ── Avito: city-slug в ПУТЬ URL (не только SERP-фильтр) ────────────────────────
|
||
|
||
|
||
def test_avito_web_url_uses_target_city_slug_in_path() -> None:
|
||
s = AvitoScraper(_STUB_CFG, target_city_slug="nizhniy_tagil") # type: ignore[arg-type]
|
||
url = s._build_web_url(57.910, 59.980, radius_km=2, page=1)
|
||
assert "/nizhniy_tagil/" in url
|
||
assert "/ekaterinburg/" not in url
|
||
|
||
|
||
def test_avito_citywide_url_uses_target_city_slug_in_path() -> None:
|
||
s = AvitoScraper(_STUB_CFG, target_city_slug="kamensk-uralskiy") # type: ignore[arg-type]
|
||
url = s._build_citywide_url(page=2)
|
||
assert "/kamensk-uralskiy/" in url
|
||
assert "/ekaterinburg/" not in url
|
||
|
||
|
||
def test_avito_newbuilding_url_uses_target_city_slug_in_path() -> None:
|
||
s = AvitoScraper(_STUB_CFG, target_city_slug="verhnyaya_pyshma") # type: ignore[arg-type]
|
||
url = s._build_newbuilding_url(page=1)
|
||
assert "/verhnyaya_pyshma/" in url
|
||
assert "/ekaterinburg/" not in url
|
||
assert NOVOSTROYKA_SLUG in url
|
||
|
||
|
||
def test_avito_rooms_url_uses_target_city_slug_in_path() -> None:
|
||
s = AvitoScraper(_STUB_CFG, target_city_slug="serov") # type: ignore[arg-type]
|
||
url = s._build_rooms_url("2k", page=1)
|
||
assert "/serov/" in url
|
||
assert "/ekaterinburg/" not in url
|
||
|
||
|
||
def test_avito_urls_default_to_ekaterinburg_when_no_target_city_slug() -> None:
|
||
"""Back-compat: target_city_slug=None → путь /ekaterinburg/ (ЕКБ full_load/pipeline)."""
|
||
s = AvitoScraper(_STUB_CFG, target_city_slug=None) # type: ignore[arg-type]
|
||
assert "/ekaterinburg/" in s._build_web_url(56.84, 60.60, radius_km=1, page=1)
|
||
assert "/ekaterinburg/" in s._build_citywide_url(page=1)
|
||
assert "/ekaterinburg/" in s._build_newbuilding_url(page=1)
|
||
assert "/ekaterinburg/" in s._build_rooms_url("2k", page=1)
|
||
|
||
|
||
# ── Cian: city_region_id → region= (subdomain НЕ трогается) ────────────────────
|
||
|
||
|
||
def test_cian_build_url_uses_city_region_id() -> None:
|
||
s = CianScraper(_STUB_CFG, city_region_id=4886) # type: ignore[arg-type]
|
||
url = s._build_url()
|
||
assert _query(url)["region"] == ["4886"]
|
||
assert urlparse(url).hostname == "ekb.cian.ru"
|
||
|
||
|
||
def test_cian_build_url_defaults_to_ekb_region_when_no_city_region_id() -> None:
|
||
"""Back-compat: city_region_id=None → region=CIAN_EKB_REGION_ID (4743)."""
|
||
s = CianScraper(_STUB_CFG, city_region_id=None) # type: ignore[arg-type]
|
||
url = s._build_url()
|
||
assert _query(url)["region"] == [str(CIAN_EKB_REGION_ID)]
|
||
assert urlparse(url).hostname == "ekb.cian.ru"
|
||
|
||
|
||
# ── Yandex: city_rgid → rgid= ───────────────────────────────────────────────────
|
||
|
||
|
||
def test_yandex_build_url_uses_city_rgid() -> None:
|
||
s = YandexRealtyScraper(_STUB_CFG, city_rgid=328286) # type: ignore[arg-type]
|
||
url = s._build_url()
|
||
assert _query(url)["rgid"] == ["328286"]
|
||
|
||
|
||
def test_yandex_build_url_defaults_to_ekb_rgid_when_no_city_rgid() -> None:
|
||
"""Back-compat: city_rgid=None → rgid=_EKB_RGID (559132)."""
|
||
s = YandexRealtyScraper(_STUB_CFG, city_rgid=None) # type: ignore[arg-type]
|
||
url = s._build_url()
|
||
assert _query(url)["rgid"] == [str(_EKB_RGID)]
|