feat(tradein): yandex_newbuilding sweep-entrypoint + market-схема enrichment, seed dormant (#974) #1541
3 changed files with 46 additions and 3 deletions
|
|
@ -1468,8 +1468,9 @@ async def scrape_yandex_newbuilding(
|
|||
|
||||
URL: /{city}/kupit/novostrojka/<slug>-<id>/
|
||||
"""
|
||||
async with YandexNewbuildingScraper() as scraper:
|
||||
result = await scraper.fetch_jk(jk_slug=slug, jk_id=id, city=city)
|
||||
# fetch_jk использует внутренний BrowserFetcher, httpx-клиент BaseScraper не нужен
|
||||
scraper = YandexNewbuildingScraper()
|
||||
result = await scraper.fetch_jk(jk_slug=slug, jk_id=id, city=city)
|
||||
if result is None:
|
||||
raise HTTPException(404, f"Could not parse Yandex JK: {slug}-{id} in {city}")
|
||||
return YandexNewbuildingTriggerResp(
|
||||
|
|
|
|||
|
|
@ -312,11 +312,22 @@ async def enrich_yandex_newbuilding_sweep(
|
|||
jk_slug = resolved
|
||||
result.resolved_slug += 1
|
||||
|
||||
# ── Guard: ext_id обязателен для fetch_jk (формирует URL /{slug}-{id}/) ──
|
||||
if not ext_id:
|
||||
logger.warning(
|
||||
"house_id=%s: jk_slug=%s известен, но ext_id пустой — "
|
||||
"fetch_jk пропущен (пустой jk_id даёт битый URL)",
|
||||
house_id,
|
||||
jk_slug,
|
||||
)
|
||||
result.failed_fetch += 1
|
||||
continue
|
||||
|
||||
# ── Fetch через BrowserFetcher ────────────────────────────────────
|
||||
info = None
|
||||
try:
|
||||
scraper = YandexNewbuildingScraper()
|
||||
info = await scraper.fetch_jk(jk_slug=jk_slug, jk_id=ext_id or "", city=city)
|
||||
info = await scraper.fetch_jk(jk_slug=jk_slug, jk_id=ext_id, city=city)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"fetch_jk failed house_id=%s jk_slug=%s ext_id=%s: %s",
|
||||
|
|
|
|||
|
|
@ -364,3 +364,34 @@ async def test_sweep_no_ext_id_fails_resolve():
|
|||
|
||||
assert result.failed_resolve == 1
|
||||
assert result.succeeded == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sweep_slug_present_but_ext_id_null_skips_fetch():
|
||||
"""House с jk_slug но ext_id=NULL → guard срабатывает, fetch_jk НЕ вызван.
|
||||
|
||||
M1 review: пустой jk_id даёт URL /{slug}-/, который является битым.
|
||||
Ожидаем failed_fetch += 1 и scraper.fetch_jk не вызван.
|
||||
"""
|
||||
db = FakeDB(
|
||||
[
|
||||
{
|
||||
"house_id": 70,
|
||||
"ext_id": None,
|
||||
"yandex_jk_slug": "tatlin",
|
||||
"yandex_jk_id": None,
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
with patch("app.services.scrapers.yandex_newbuilding.YandexNewbuildingScraper") as mock_scraper:
|
||||
scraper_instance = MagicMock()
|
||||
scraper_instance.fetch_jk = AsyncMock()
|
||||
mock_scraper.return_value = scraper_instance
|
||||
|
||||
result = await enrich_yandex_newbuilding_sweep(db, limit=5, request_delay_sec=0)
|
||||
|
||||
scraper_instance.fetch_jk.assert_not_called()
|
||||
assert result.failed_fetch == 1
|
||||
assert result.succeeded == 0
|
||||
assert result.rows_inserted == 0
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue