fix(tradein): yandex sweep — guard ext_id=NULL перед fetch_jk + cleanup httpx (#974 review M1/M3)
All checks were successful
CI / backend-tests (push) Has been skipped
CI / frontend-tests (push) Has been skipped
CI / openapi-codegen-check (push) Has been skipped
CI / backend-tests (pull_request) Has been skipped
CI / changes (push) Successful in 7s
CI / changes (pull_request) Successful in 8s
CI / frontend-tests (pull_request) Has been skipped
CI / openapi-codegen-check (pull_request) Has been skipped
All checks were successful
CI / backend-tests (push) Has been skipped
CI / frontend-tests (push) Has been skipped
CI / openapi-codegen-check (push) Has been skipped
CI / backend-tests (pull_request) Has been skipped
CI / changes (push) Successful in 7s
CI / changes (pull_request) Successful in 8s
CI / frontend-tests (pull_request) Has been skipped
CI / openapi-codegen-check (pull_request) Has been skipped
M1: добавлен guard перед вызовом fetch_jk — если ext_id пустой/None при наличии
jk_slug, логируем warning, incrementим failed_fetch и пропускаем (continue).
Пустой jk_id формирует битый URL /{slug}-/ и коллизию на UNIQUE ext_id=''.
M3: в admin.py /scrape/yandex-newbuilding заменён `async with YandexNewbuildingScraper()`
на прямое `scraper = YandexNewbuildingScraper()` — fetch_jk использует внутренний
BrowserFetcher, httpx-клиент BaseScraper открывался вхолостую.
Тесты: добавлен test_sweep_slug_present_but_ext_id_null_skips_fetch.
This commit is contained in:
parent
feab585ad5
commit
c9daf8e268
3 changed files with 46 additions and 3 deletions
|
|
@ -1468,8 +1468,9 @@ async def scrape_yandex_newbuilding(
|
|||
|
||||
URL: /{city}/kupit/novostrojka/<slug>-<id>/
|
||||
"""
|
||||
async with YandexNewbuildingScraper() as scraper:
|
||||
result = await scraper.fetch_jk(jk_slug=slug, jk_id=id, city=city)
|
||||
# fetch_jk использует внутренний BrowserFetcher, httpx-клиент BaseScraper не нужен
|
||||
scraper = YandexNewbuildingScraper()
|
||||
result = await scraper.fetch_jk(jk_slug=slug, jk_id=id, city=city)
|
||||
if result is None:
|
||||
raise HTTPException(404, f"Could not parse Yandex JK: {slug}-{id} in {city}")
|
||||
return YandexNewbuildingTriggerResp(
|
||||
|
|
|
|||
|
|
@ -312,11 +312,22 @@ async def enrich_yandex_newbuilding_sweep(
|
|||
jk_slug = resolved
|
||||
result.resolved_slug += 1
|
||||
|
||||
# ── Guard: ext_id обязателен для fetch_jk (формирует URL /{slug}-{id}/) ──
|
||||
if not ext_id:
|
||||
logger.warning(
|
||||
"house_id=%s: jk_slug=%s известен, но ext_id пустой — "
|
||||
"fetch_jk пропущен (пустой jk_id даёт битый URL)",
|
||||
house_id,
|
||||
jk_slug,
|
||||
)
|
||||
result.failed_fetch += 1
|
||||
continue
|
||||
|
||||
# ── Fetch через BrowserFetcher ────────────────────────────────────
|
||||
info = None
|
||||
try:
|
||||
scraper = YandexNewbuildingScraper()
|
||||
info = await scraper.fetch_jk(jk_slug=jk_slug, jk_id=ext_id or "", city=city)
|
||||
info = await scraper.fetch_jk(jk_slug=jk_slug, jk_id=ext_id, city=city)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"fetch_jk failed house_id=%s jk_slug=%s ext_id=%s: %s",
|
||||
|
|
|
|||
|
|
@ -364,3 +364,34 @@ async def test_sweep_no_ext_id_fails_resolve():
|
|||
|
||||
assert result.failed_resolve == 1
|
||||
assert result.succeeded == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sweep_slug_present_but_ext_id_null_skips_fetch():
|
||||
"""House с jk_slug но ext_id=NULL → guard срабатывает, fetch_jk НЕ вызван.
|
||||
|
||||
M1 review: пустой jk_id даёт URL /{slug}-/, который является битым.
|
||||
Ожидаем failed_fetch += 1 и scraper.fetch_jk не вызван.
|
||||
"""
|
||||
db = FakeDB(
|
||||
[
|
||||
{
|
||||
"house_id": 70,
|
||||
"ext_id": None,
|
||||
"yandex_jk_slug": "tatlin",
|
||||
"yandex_jk_id": None,
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
with patch("app.services.scrapers.yandex_newbuilding.YandexNewbuildingScraper") as mock_scraper:
|
||||
scraper_instance = MagicMock()
|
||||
scraper_instance.fetch_jk = AsyncMock()
|
||||
mock_scraper.return_value = scraper_instance
|
||||
|
||||
result = await enrich_yandex_newbuilding_sweep(db, limit=5, request_delay_sec=0)
|
||||
|
||||
scraper_instance.fetch_jk.assert_not_called()
|
||||
assert result.failed_fetch == 1
|
||||
assert result.succeeded == 0
|
||||
assert result.rows_inserted == 0
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue