fix(tradein): yandex sweep — guard ext_id=NULL перед fetch_jk + cleanup httpx (#974 review M1/M3)
All checks were successful
CI / backend-tests (push) Has been skipped
CI / frontend-tests (push) Has been skipped
CI / openapi-codegen-check (push) Has been skipped
CI / backend-tests (pull_request) Has been skipped
CI / changes (push) Successful in 7s
CI / changes (pull_request) Successful in 8s
CI / frontend-tests (pull_request) Has been skipped
CI / openapi-codegen-check (pull_request) Has been skipped

M1: добавлен guard перед вызовом fetch_jk — если ext_id пустой/None при наличии
jk_slug, логируем warning, incrementим failed_fetch и пропускаем (continue).
Пустой jk_id формирует битый URL /{slug}-/ и коллизию на UNIQUE ext_id=''.

M3: в admin.py /scrape/yandex-newbuilding заменён `async with YandexNewbuildingScraper()`
на прямое `scraper = YandexNewbuildingScraper()` — fetch_jk использует внутренний
BrowserFetcher, httpx-клиент BaseScraper открывался вхолостую.

Тесты: добавлен test_sweep_slug_present_but_ext_id_null_skips_fetch.
This commit is contained in:
bot-backend 2026-06-15 18:41:08 +03:00
parent feab585ad5
commit c9daf8e268
3 changed files with 46 additions and 3 deletions

View file

@ -1468,8 +1468,9 @@ async def scrape_yandex_newbuilding(
URL: /{city}/kupit/novostrojka/<slug>-<id>/
"""
async with YandexNewbuildingScraper() as scraper:
result = await scraper.fetch_jk(jk_slug=slug, jk_id=id, city=city)
# fetch_jk использует внутренний BrowserFetcher, httpx-клиент BaseScraper не нужен
scraper = YandexNewbuildingScraper()
result = await scraper.fetch_jk(jk_slug=slug, jk_id=id, city=city)
if result is None:
raise HTTPException(404, f"Could not parse Yandex JK: {slug}-{id} in {city}")
return YandexNewbuildingTriggerResp(

View file

@ -312,11 +312,22 @@ async def enrich_yandex_newbuilding_sweep(
jk_slug = resolved
result.resolved_slug += 1
# ── Guard: ext_id обязателен для fetch_jk (формирует URL /{slug}-{id}/) ──
if not ext_id:
logger.warning(
"house_id=%s: jk_slug=%s известен, но ext_id пустой — "
"fetch_jk пропущен (пустой jk_id даёт битый URL)",
house_id,
jk_slug,
)
result.failed_fetch += 1
continue
# ── Fetch через BrowserFetcher ────────────────────────────────────
info = None
try:
scraper = YandexNewbuildingScraper()
info = await scraper.fetch_jk(jk_slug=jk_slug, jk_id=ext_id or "", city=city)
info = await scraper.fetch_jk(jk_slug=jk_slug, jk_id=ext_id, city=city)
except Exception as exc:
logger.warning(
"fetch_jk failed house_id=%s jk_slug=%s ext_id=%s: %s",

View file

@ -364,3 +364,34 @@ async def test_sweep_no_ext_id_fails_resolve():
assert result.failed_resolve == 1
assert result.succeeded == 0
@pytest.mark.asyncio
async def test_sweep_slug_present_but_ext_id_null_skips_fetch():
"""House с jk_slug но ext_id=NULL → guard срабатывает, fetch_jk НЕ вызван.
M1 review: пустой jk_id даёт URL /{slug}-/, который является битым.
Ожидаем failed_fetch += 1 и scraper.fetch_jk не вызван.
"""
db = FakeDB(
[
{
"house_id": 70,
"ext_id": None,
"yandex_jk_slug": "tatlin",
"yandex_jk_id": None,
}
]
)
with patch("app.services.scrapers.yandex_newbuilding.YandexNewbuildingScraper") as mock_scraper:
scraper_instance = MagicMock()
scraper_instance.fetch_jk = AsyncMock()
mock_scraper.return_value = scraper_instance
result = await enrich_yandex_newbuilding_sweep(db, limit=5, request_delay_sec=0)
scraper_instance.fetch_jk.assert_not_called()
assert result.failed_fetch == 1
assert result.succeeded == 0
assert result.rows_inserted == 0