diff --git a/tradein-mvp/backend/app/tasks/yandex_newbuilding_sweep.py b/tradein-mvp/backend/app/tasks/yandex_newbuilding_sweep.py index 5a967840..ba1c8d73 100644 --- a/tradein-mvp/backend/app/tasks/yandex_newbuilding_sweep.py +++ b/tradein-mvp/backend/app/tasks/yandex_newbuilding_sweep.py @@ -46,6 +46,7 @@ from dataclasses import dataclass, field, fields from sqlalchemy import text from sqlalchemy.orm import Session +from app.services.scraper_adapters import RealScraperConfig from app.services.scraper_settings import get_scraper_delay logger = logging.getLogger(__name__) @@ -221,16 +222,17 @@ async def enrich_yandex_newbuilding_sweep( Returns: YandexNewbuildingSweepResult со счётчиками. """ - # NOT migrated to scraper_kit (issue #2310, Group C): both - # scraper_kit.providers.yandex.newbuilding.YandexNewbuildingScraper.fetch_jk() - # AND resolve_yandex_jk_slug() construct BrowserFetcher(source="yandex") - # WITHOUT the now-mandatory endpoint= kwarg — every call raises TypeError, - # with no caller-side fix possible (the kit provider functions don't expose - # a config/endpoint hook at all). This is issue #2322, verified STILL OPEN - # by reading the actual provider source (not just the issue's open/closed - # status) at the time of this migration. Left on legacy entirely, exactly - # like Group A (#2305) did for the same bug in admin.py's debug endpoints. - from app.services.scrapers.yandex_newbuilding import ( + # Migrated to scraper_kit (#2397 Part D5, issue #2310 Group C): #2322 + # (BrowserFetcher(source="yandex") constructed без endpoint= kwarg) was + # fixed by PR #2356 — both YandexNewbuildingScraper.fetch_jk() и + # resolve_yandex_jk_slug() now accept config=ScraperConfig и пробрасывают + # config.browser_http_endpoint в BrowserFetcher. RealScraperConfig() + # (см. import выше) обязателен на ОБОИХ call site — без него + # BrowserFetcher.__aenter__ падает AssertionError (endpoint не задан). + # Lazy import (не module top) — как и раньше, чтобы тяжёлый browser/parse + # стек грузился только когда sweep реально запускается (unit-тесты + # патчат эти имена cheap-ly через mocker без импорта selectolax и т.п.). + from scraper_kit.providers.yandex.newbuilding import ( YandexNewbuildingScraper, resolve_yandex_jk_slug, ) @@ -301,7 +303,9 @@ async def enrich_yandex_newbuilding_sweep( continue try: - resolved = await resolve_yandex_jk_slug(ext_id, city=city) + resolved = await resolve_yandex_jk_slug( + ext_id, city=city, config=RealScraperConfig() + ) except Exception as exc: logger.warning( "resolve_yandex_jk_slug house_id=%s ext_id=%s raised: %s", @@ -350,7 +354,7 @@ async def enrich_yandex_newbuilding_sweep( # ── Fetch через BrowserFetcher ──────────────────────────────────── info = None try: - scraper = YandexNewbuildingScraper() + scraper = YandexNewbuildingScraper(config=RealScraperConfig()) info = await scraper.fetch_jk(jk_slug=jk_slug, jk_id=ext_id, city=city) except Exception as exc: logger.warning( diff --git a/tradein-mvp/backend/tests/tasks/test_yandex_newbuilding_sweep.py b/tradein-mvp/backend/tests/tasks/test_yandex_newbuilding_sweep.py index e6b2eddf..3849f944 100644 --- a/tradein-mvp/backend/tests/tasks/test_yandex_newbuilding_sweep.py +++ b/tradein-mvp/backend/tests/tasks/test_yandex_newbuilding_sweep.py @@ -192,7 +192,7 @@ async def test_sweep_with_existing_slug_upserts_enrichment(): ] ) - from app.services.scrapers.yandex_newbuilding import YandexNewbuildingInfo + from scraper_kit.providers.yandex.newbuilding import YandexNewbuildingInfo fake_info = YandexNewbuildingInfo( ext_id="999", @@ -207,7 +207,9 @@ async def test_sweep_with_existing_slug_upserts_enrichment(): text_reviews_count=25, ) - with patch("app.services.scrapers.yandex_newbuilding.YandexNewbuildingScraper") as mock_scraper: + from app.services.scraper_adapters import RealScraperConfig + + with patch("scraper_kit.providers.yandex.newbuilding.YandexNewbuildingScraper") as mock_scraper: scraper_instance = MagicMock() scraper_instance.fetch_jk = AsyncMock(return_value=fake_info) mock_scraper.return_value = scraper_instance @@ -224,6 +226,12 @@ async def test_sweep_with_existing_slug_upserts_enrichment(): # house_class не распарсен в этой фикстуре (None) — UPDATE houses.house_class не должен # выполниться вовсе (Bug 1 guard: лишний UPDATE с NULL пропускается). assert 10 not in db.house_class_updates + # #2397 Part D5 regression guard: kit YandexNewbuildingScraper.fetch_jk builds + # BrowserFetcher(endpoint=self._config.browser_http_endpoint) — without + # config=RealScraperConfig() threaded at the call site, BrowserFetcher.__aenter__ + # raises AssertionError (missing endpoint). call_count alone wouldn't catch a + # future drop of that kwarg (mirrors #2306/#2310's isinstance-call_kwargs pattern). + assert isinstance(mock_scraper.call_args.kwargs.get("config"), RealScraperConfig) @pytest.mark.asyncio @@ -244,7 +252,7 @@ async def test_sweep_persists_house_class_when_parsed(): ] ) - from app.services.scrapers.yandex_newbuilding import YandexNewbuildingInfo + from scraper_kit.providers.yandex.newbuilding import YandexNewbuildingInfo fake_info = YandexNewbuildingInfo( ext_id="1000", @@ -254,7 +262,7 @@ async def test_sweep_persists_house_class_when_parsed(): house_class="comfort_plus", ) - with patch("app.services.scrapers.yandex_newbuilding.YandexNewbuildingScraper") as mock_scraper: + with patch("scraper_kit.providers.yandex.newbuilding.YandexNewbuildingScraper") as mock_scraper: scraper_instance = MagicMock() scraper_instance.fetch_jk = AsyncMock(return_value=fake_info) mock_scraper.return_value = scraper_instance @@ -279,7 +287,9 @@ async def test_sweep_resolves_slug_when_missing(): ] ) - from app.services.scrapers.yandex_newbuilding import YandexNewbuildingInfo + from scraper_kit.providers.yandex.newbuilding import YandexNewbuildingInfo + + from app.services.scraper_adapters import RealScraperConfig fake_info = YandexNewbuildingInfo( ext_id="888", @@ -292,11 +302,11 @@ async def test_sweep_resolves_slug_when_missing(): with ( patch( - "app.services.scrapers.yandex_newbuilding.resolve_yandex_jk_slug", + "scraper_kit.providers.yandex.newbuilding.resolve_yandex_jk_slug", new_callable=AsyncMock, return_value="testjk", ) as mock_resolve, - patch("app.services.scrapers.yandex_newbuilding.YandexNewbuildingScraper") as mock_scraper, + patch("scraper_kit.providers.yandex.newbuilding.YandexNewbuildingScraper") as mock_scraper, ): scraper_instance = MagicMock() scraper_instance.fetch_jk = AsyncMock(return_value=fake_info) @@ -304,7 +314,14 @@ async def test_sweep_resolves_slug_when_missing(): result = await enrich_yandex_newbuilding_sweep(db, limit=5, request_delay_sec=0) - mock_resolve.assert_called_once_with("888", city="ekaterinburg") + # resolve_yandex_jk_slug теперь принимает config= (#2322 fix) — сравниваем + # позиционный arg + city= отдельно от config, т.к. RealScraperConfig + # instance-equality не поддерживает (нет __eq__); проверяем isinstance, + # тот же regression-guard паттерн что и для YandexNewbuildingScraper выше. + resolve_args, resolve_kwargs = mock_resolve.call_args + assert resolve_args == ("888",) + assert resolve_kwargs.get("city") == "ekaterinburg" + assert isinstance(resolve_kwargs.get("config"), RealScraperConfig) assert result.resolved_slug == 1 assert result.succeeded == 1 assert result.rows_inserted == 1 @@ -328,7 +345,7 @@ async def test_sweep_failed_resolve_counts(): ) with patch( - "app.services.scrapers.yandex_newbuilding.resolve_yandex_jk_slug", + "scraper_kit.providers.yandex.newbuilding.resolve_yandex_jk_slug", new_callable=AsyncMock, return_value=None, ): @@ -354,7 +371,7 @@ async def test_sweep_failed_fetch_counts(): ] ) - with patch("app.services.scrapers.yandex_newbuilding.YandexNewbuildingScraper") as mock_scraper: + with patch("scraper_kit.providers.yandex.newbuilding.YandexNewbuildingScraper") as mock_scraper: scraper_instance = MagicMock() scraper_instance.fetch_jk = AsyncMock(return_value=None) mock_scraper.return_value = scraper_instance @@ -382,7 +399,7 @@ async def test_sweep_idempotent_skip_already_enriched(): # Pre-populate enrichment db.enrichment["555"] = {"ext_id": "555", "name": "Already enriched"} - with patch("app.services.scrapers.yandex_newbuilding.YandexNewbuildingScraper") as mock_scraper: + with patch("scraper_kit.providers.yandex.newbuilding.YandexNewbuildingScraper") as mock_scraper: scraper_instance = MagicMock() scraper_instance.fetch_jk = AsyncMock(return_value=None) mock_scraper.return_value = scraper_instance @@ -433,7 +450,7 @@ async def test_sweep_slug_present_but_ext_id_null_skips_fetch(): ] ) - with patch("app.services.scrapers.yandex_newbuilding.YandexNewbuildingScraper") as mock_scraper: + with patch("scraper_kit.providers.yandex.newbuilding.YandexNewbuildingScraper") as mock_scraper: scraper_instance = MagicMock() scraper_instance.fetch_jk = AsyncMock() mock_scraper.return_value = scraper_instance