feat(tradein/scraper-kit): migrate backfill-task imports to kit, Group C (#2310)
Migrates legacy app.services.scrapers.* imports to scraper_kit equivalents for house_imv_backfill.py, avito_detail_backfill.py, cian_history_backfill.py, ekb_geoportal_ingest.py, and yandex_detail_backfill.py, proving parity via tests/support/parity.assert_parity per the epic's gate (#2304). newbuilding_enrich_backfill.py and yandex_newbuilding_sweep.py are left fully on legacy imports: both call into scraper_kit.providers.{cian,yandex}.newbuilding, which construct BrowserFetcher(source=...) without the now-mandatory endpoint= kwarg (issue #2322, verified still open against the actual provider source, not just issue status) -- no caller-side fix is possible, matching Group A's (#2305) precedent for the same bug in admin.py. Config-gated kit function footguns found and fixed (Group B #2306 pattern): - avito fetch_detail's backconnect-on-403 retry silently drops when config= is omitted -- now passes config=RealScraperConfig() explicitly, with a regression test proving the gate. - kit AvitoScraper's constructor now requires ScraperConfig positionally -- wired via RealScraperConfig(), which _rotate_ip() reads for avito_proxy_rotate_url. - BrowserFetcher(source=...) call sites (house_imv_backfill, avito/cian detail backfills) now pass the mandatory endpoint=settings.browser_http_endpoint. New footgun discovered (NOT #2322, flagged for follow-up): kit's build_warmed_session() builds its curl_cffi session via _build_detail_session() with no config parameter at all, unlike fetch_detail -- migrating it would silently drop the sticky MGTS-proxy egress on avito_detail_backfill's warm-batch path (the prod default). Left build_warmed_session/ _AVITO_WARM_SEARCH_URL on legacy imports, documented inline. Refs #2310
This commit is contained in:
bot-backend
parent
793234cff1
commit
eb564cfc01
9 changed files with 400 additions and 43 deletions
|
|
@ -25,10 +25,18 @@ from collections.abc import Callable
|
|||
from dataclasses import dataclass, field
|
||||
from typing import Literal
|
||||
|
||||
from scraper_kit.browser_fetcher import BrowserFetcher
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.core.config import settings
|
||||
|
||||
# app.services.scrapers.avito_imv остаётся legacy — миграция валюационных
|
||||
# зависимостей (avito_imv/cian_valuation/yandex_valuation) выделена в отдельный,
|
||||
# более рискованный issue #2308 (см. epic #2277, issue #2310 Group C). Мигрируем
|
||||
# здесь только BrowserFetcher (#2310, выше) — avito_imv.evaluate_via_imv принимает
|
||||
# его исключительно как TYPE_CHECKING-аннотацию (duck-typed на fetch_json), поэтому
|
||||
# смена конкретного класса не требует правок avito_imv.py.
|
||||
from app.services.scrapers.avito_imv import (
|
||||
IMVAddressNotFoundError,
|
||||
IMVAuthError,
|
||||
|
|
@ -36,7 +44,6 @@ from app.services.scrapers.avito_imv import (
|
|||
IMVTransientError,
|
||||
evaluate_via_imv,
|
||||
)
|
||||
from app.services.scrapers.browser_fetcher import BrowserFetcher
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -484,7 +491,7 @@ async def backfill_house_imv(
|
|||
# Флаг OFF → _bf=None → evaluate_via_imv делает свою curl-сессию как раньше
|
||||
# (поведение байт-в-байт идентично доспринтовому).
|
||||
if settings.avito_imv_use_browser_fetcher:
|
||||
async with BrowserFetcher(source="avito") as _bf:
|
||||
async with BrowserFetcher(source="avito", endpoint=settings.browser_http_endpoint) as _bf:
|
||||
await _run_loop(_bf)
|
||||
else:
|
||||
await _run_loop(None)
|
||||
|
|
|
|||
|
|
@ -23,6 +23,18 @@ from dataclasses import dataclass, field
|
|||
from urllib.parse import urlparse
|
||||
|
||||
from curl_cffi.requests import AsyncSession
|
||||
from scraper_kit.avito_exceptions import (
|
||||
AvitoBlockedError,
|
||||
AvitoListingGoneError,
|
||||
AvitoRateLimitedError,
|
||||
)
|
||||
from scraper_kit.browser_fetcher import BrowserFetcher
|
||||
from scraper_kit.providers.avito.detail import (
|
||||
fetch_detail,
|
||||
research_in_session,
|
||||
save_detail_enrichment,
|
||||
)
|
||||
from scraper_kit.providers.avito.serp import AvitoScraper
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
|
|
@ -30,20 +42,22 @@ from app.core.config import settings
|
|||
from app.core.shutdown import shutdown_requested
|
||||
from app.services import scrape_runs as runs_mod
|
||||
from app.services.scrape_pipeline import _CHROME_HEADERS, _avito_proxies
|
||||
from app.services.scrapers.avito import AvitoScraper
|
||||
from app.services.scraper_adapters import RealScraperConfig
|
||||
|
||||
# _AVITO_WARM_SEARCH_URL/build_warmed_session остаются legacy (#2310): kit's
|
||||
# scraper_kit.providers.avito.detail.build_warmed_session() строит сессию через
|
||||
# _build_detail_session() БЕЗ config-параметра (сигнатура вообще не принимает
|
||||
# ScraperConfig — в отличие от fetch_detail, здесь нет caller-side способа
|
||||
# прокинуть прокси). Мигрировав этот импорт, warm-batch curl-путь (use_curl=True,
|
||||
# ДЕФОЛТ прод-режима, avito_detail_backfill_use_curl=True) молча перестал бы
|
||||
# использовать settings.scraper_proxy_url (sticky МГТС-прокси) — прямое
|
||||
# datacenter-подключение вместо backconnect. Тот же класс бага, что #2322
|
||||
# (недостающая инъекция конфига в scraper_kit provider-функции), но другая
|
||||
# функция — заведён как отдельный follow-up, НЕ #2322. См. отчёт issue #2310.
|
||||
from app.services.scrapers.avito_detail import (
|
||||
_AVITO_WARM_SEARCH_URL,
|
||||
build_warmed_session,
|
||||
fetch_detail,
|
||||
research_in_session,
|
||||
save_detail_enrichment,
|
||||
)
|
||||
from app.services.scrapers.avito_exceptions import (
|
||||
AvitoBlockedError,
|
||||
AvitoListingGoneError,
|
||||
AvitoRateLimitedError,
|
||||
)
|
||||
from app.services.scrapers.browser_fetcher import BrowserFetcher
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -121,7 +135,10 @@ async def run_avito_detail_backfill(
|
|||
own_session = False
|
||||
own_browser = False
|
||||
|
||||
scraper = AvitoScraper()
|
||||
# kit AvitoScraper требует ScraperConfig позиционно (Strangler-инжекция #2133) —
|
||||
# RealScraperConfig проксирует settings.* так же, как читал legacy-конструктор без
|
||||
# аргументов (avito_proxy_rotate_url и т.д. для _rotate_ip()).
|
||||
scraper = AvitoScraper(RealScraperConfig())
|
||||
start = time.monotonic()
|
||||
|
||||
logger.info(
|
||||
|
|
@ -136,7 +153,9 @@ async def run_avito_detail_backfill(
|
|||
# эфемерную _build_detail_session() через settings.scraper_proxy_url (backconnect).
|
||||
# use_curl=False (legacy): browser_mode=True → BrowserFetcher/auv, как раньше.
|
||||
if browser_mode:
|
||||
browser_fetcher = BrowserFetcher(source="avito")
|
||||
browser_fetcher = BrowserFetcher(
|
||||
source="avito", endpoint=settings.browser_http_endpoint
|
||||
)
|
||||
await browser_fetcher.__aenter__()
|
||||
own_browser = True
|
||||
scraper._browser = browser_fetcher
|
||||
|
|
@ -314,6 +333,13 @@ async def run_avito_detail_backfill(
|
|||
browser_fetcher=browser_fetcher,
|
||||
referer=_AVITO_WARM_SEARCH_URL if use_curl else None,
|
||||
reconnect_on_block=not use_curl,
|
||||
# config= обязателен: kit fetch_detail без него читает
|
||||
# config.scraper_proxy_url=None для backconnect-gate'а
|
||||
# (elif not use_curl own-session path), теряя reconnect-
|
||||
# on-403 поведение legacy (settings.scraper_proxy_url
|
||||
# напрямую). Не влияет на use_curl=True (прод-дефолт) —
|
||||
# там reconnect_on_block=False уже гасит backconnect.
|
||||
config=RealScraperConfig(),
|
||||
),
|
||||
timeout=fetch_timeout_s,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -27,12 +27,17 @@ import logging
|
|||
import time
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from scraper_kit.browser_fetcher import BrowserFetcher
|
||||
from scraper_kit.providers.cian.detail import fetch_detail, save_detail_enrichment
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.core.config import settings
|
||||
from app.services.scraper_settings import get_scraper_delay
|
||||
from app.services.scrapers.browser_fetcher import BrowserFetcher
|
||||
from app.services.scrapers.cian_detail import fetch_detail, save_detail_enrichment
|
||||
|
||||
# cian_valuation остаётся legacy — валюационные зависимости (avito_imv/
|
||||
# cian_valuation/yandex_valuation) выделены в отдельный, более рискованный
|
||||
# issue #2308 (см. epic #2277, issue #2310 Group C), не в этой миграции.
|
||||
from app.services.scrapers.cian_valuation import estimate_via_cian_valuation
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
@ -113,7 +118,7 @@ async def backfill_cian_history(
|
|||
else:
|
||||
# One BrowserFetcher instance shared across all listings in this batch.
|
||||
# priceChanges requires JS rendering — curl_cffi returns empty list (#1574).
|
||||
async with BrowserFetcher(source="cian") as bf:
|
||||
async with BrowserFetcher(source="cian", endpoint=settings.browser_http_endpoint) as bf:
|
||||
for row in rows:
|
||||
listing_id: int = row["id"]
|
||||
source_url: str = row["source_url"]
|
||||
|
|
@ -157,6 +162,11 @@ async def backfill_cian_history(
|
|||
|
||||
# ── 2. Houses: missing houses_price_dynamics ──────────────────────────────
|
||||
if do_houses:
|
||||
# cian_newbuilding остаётся legacy (issue #2310, NOT migrated): kit's
|
||||
# scraper_kit.providers.cian.newbuilding.fetch_newbuilding() constructs
|
||||
# BrowserFetcher(source="cian") WITHOUT the now-mandatory endpoint= kwarg
|
||||
# → TypeError on every call, no caller-side fix possible (issue #2322,
|
||||
# still open — verified against provider source, not just issue status).
|
||||
from app.services.scrapers.cian_newbuilding import (
|
||||
fetch_newbuilding,
|
||||
save_newbuilding_enrichment,
|
||||
|
|
|
|||
|
|
@ -24,14 +24,14 @@ import logging
|
|||
import os
|
||||
import re
|
||||
|
||||
from scraper_kit.providers.ekb_geoportal.client import (
|
||||
EkbGeoportalClient,
|
||||
TopoBuilding,
|
||||
)
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.core.db import SessionLocal
|
||||
from app.services.scrapers.ekb_geoportal_client import (
|
||||
EkbGeoportalClient,
|
||||
TopoBuilding,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
|
|||
|
|
@ -355,6 +355,16 @@ async def backfill_newbuilding_enrichment(
|
|||
# Import here (not module top) so the heavy curl_cffi scraper stack is only loaded
|
||||
# when the task actually runs — mirrors cian_history_backfill's lazy import and keeps
|
||||
# unit tests able to patch fetch_newbuilding cheaply.
|
||||
#
|
||||
# NOT migrated to scraper_kit (issue #2310, Group C): kit's
|
||||
# scraper_kit.providers.cian.newbuilding.fetch_newbuilding() constructs
|
||||
# BrowserFetcher(source="cian") WITHOUT the now-mandatory endpoint= kwarg —
|
||||
# every call raises TypeError, with no caller-side fix possible (the kit
|
||||
# provider function doesn't expose a config/endpoint hook at all). This is
|
||||
# issue #2322, verified STILL OPEN by reading the actual provider source
|
||||
# (not just the issue's open/closed status) at the time of this migration.
|
||||
# Left on legacy entirely, exactly like Group A (#2305) did for the same
|
||||
# bug in admin.py's debug endpoints.
|
||||
from app.services.scrapers.cian_newbuilding import (
|
||||
fetch_newbuilding,
|
||||
resolve_cian_zhk_url_via_search,
|
||||
|
|
@ -371,11 +381,7 @@ async def backfill_newbuilding_enrichment(
|
|||
result.cian_houses_fetchable = sizing["fetchable"]
|
||||
result.cian_houses_pending = sizing["pending"]
|
||||
|
||||
rows = (
|
||||
db.execute(text(_SELECT_PENDING_HOUSES), {"force": force, "lim": limit})
|
||||
.mappings()
|
||||
.all()
|
||||
)
|
||||
rows = db.execute(text(_SELECT_PENDING_HOUSES), {"force": force, "lim": limit}).mappings().all()
|
||||
|
||||
logger.info(
|
||||
"newbuilding-enrich backfill: cian_houses total=%d fetchable=%d pending=%d; "
|
||||
|
|
@ -547,9 +553,7 @@ async def backfill_newbuilding_enrichment(
|
|||
db.commit()
|
||||
except Exception as dexc:
|
||||
sp.rollback()
|
||||
logger.warning(
|
||||
"reliability dedup failed house_id=%s: %s", house_id, dexc
|
||||
)
|
||||
logger.warning("reliability dedup failed house_id=%s: %s", house_id, dexc)
|
||||
|
||||
pd_after, rc_after, rv_after = _house_enrichment_counts(db, house_id)
|
||||
result.price_dynamics_rows += max(0, pd_after - pd_before)
|
||||
|
|
|
|||
|
|
@ -31,12 +31,12 @@ import time
|
|||
from dataclasses import dataclass, field
|
||||
|
||||
from curl_cffi.requests import AsyncSession
|
||||
from scraper_kit.providers.yandex.detail import YandexDetailScraper, save_detail_enrichment
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.core.config import settings
|
||||
from app.services import scrape_runs as runs_mod
|
||||
from app.services.scrapers.yandex_detail import YandexDetailScraper, save_detail_enrichment
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
|
|||
|
|
@ -206,6 +206,15 @@ async def enrich_yandex_newbuilding_sweep(
|
|||
Returns:
|
||||
YandexNewbuildingSweepResult со счётчиками.
|
||||
"""
|
||||
# NOT migrated to scraper_kit (issue #2310, Group C): both
|
||||
# scraper_kit.providers.yandex.newbuilding.YandexNewbuildingScraper.fetch_jk()
|
||||
# AND resolve_yandex_jk_slug() construct BrowserFetcher(source="yandex")
|
||||
# WITHOUT the now-mandatory endpoint= kwarg — every call raises TypeError,
|
||||
# with no caller-side fix possible (the kit provider functions don't expose
|
||||
# a config/endpoint hook at all). This is issue #2322, verified STILL OPEN
|
||||
# by reading the actual provider source (not just the issue's open/closed
|
||||
# status) at the time of this migration. Left on legacy entirely, exactly
|
||||
# like Group A (#2305) did for the same bug in admin.py's debug endpoints.
|
||||
from app.services.scrapers.yandex_newbuilding import (
|
||||
YandexNewbuildingScraper,
|
||||
resolve_yandex_jk_slug,
|
||||
|
|
|
|||
|
|
@ -145,7 +145,7 @@ async def test_backfill_blocked_abort_after_max_consecutive() -> None:
|
|||
#1950 abort-reorder: abort-check ПЕРЕД recovery → на 5-м (аборт-)блоке rotate_ip
|
||||
НЕ дёргается (не тратим recovery на финальном блоке). rotate_ip x4 (блоки 1-4).
|
||||
"""
|
||||
from app.services.scrapers.avito_exceptions import AvitoBlockedError
|
||||
from scraper_kit.avito_exceptions import AvitoBlockedError
|
||||
|
||||
snapshot = _make_snapshot(10)
|
||||
db = _mock_db(snapshot)
|
||||
|
|
@ -267,7 +267,7 @@ async def test_backfill_top_level_exception_marks_failed() -> None:
|
|||
@pytest.mark.asyncio
|
||||
async def test_backfill_rotate_ip_called_on_each_block() -> None:
|
||||
"""1 block + 1 success -> rotate_ip called once, enriched=1."""
|
||||
from app.services.scrapers.avito_exceptions import AvitoBlockedError
|
||||
from scraper_kit.avito_exceptions import AvitoBlockedError
|
||||
|
||||
snapshot = _make_snapshot(2)
|
||||
db = _mock_db(snapshot)
|
||||
|
|
@ -379,6 +379,7 @@ async def test_backfill_fetch_timeout_skips_and_continues() -> None:
|
|||
browser_fetcher: object = None,
|
||||
referer: object = None,
|
||||
reconnect_on_block: bool = True,
|
||||
config: object = None,
|
||||
):
|
||||
call_urls.append(url)
|
||||
if len(call_urls) == 1:
|
||||
|
|
@ -423,7 +424,7 @@ async def test_backfill_listing_gone_marks_inactive_no_breaker() -> None:
|
|||
состоит из dead-листингов; раньше 404 ловился как soft-block → breaker абортил run
|
||||
до live-листингов. Теперь 404 нейтрален к breaker'у и метит листинг inactive.
|
||||
"""
|
||||
from app.services.scrapers.avito_exceptions import AvitoListingGoneError
|
||||
from scraper_kit.avito_exceptions import AvitoListingGoneError
|
||||
|
||||
snapshot = _make_snapshot(1)
|
||||
db = _mock_db(snapshot)
|
||||
|
|
@ -541,8 +542,11 @@ async def test_backfill_use_curl_false_creates_browser_fetcher() -> None:
|
|||
db, run_id=10, params={"batch_size": 10, "budget_sec": 3600}
|
||||
)
|
||||
|
||||
# BrowserFetcher должен быть создан (source="avito")
|
||||
mock_bf_cls.assert_called_once_with(source="avito")
|
||||
# BrowserFetcher должен быть создан (source="avito", endpoint из settings — #2310
|
||||
# kit BrowserFetcher требует endpoint= обязательным keyword-only параметром)
|
||||
mock_bf_cls.assert_called_once_with(
|
||||
source="avito", endpoint=fake_settings.browser_http_endpoint
|
||||
)
|
||||
# fetch_detail вызван с browser_fetcher установленным (не None)
|
||||
assert mock_fetch.call_count == 1
|
||||
_, kwargs = mock_fetch.call_args
|
||||
|
|
@ -560,7 +564,7 @@ async def test_backfill_use_curl_block_cooldown_research_no_rebuild() -> None:
|
|||
нельзя. Блок = rate-limit текущего IP → cooldown + re-search той же сессией.
|
||||
fetch: 1-й вызов BLOCKED, 2-й — успех.
|
||||
"""
|
||||
from app.services.scrapers.avito_exceptions import AvitoBlockedError
|
||||
from scraper_kit.avito_exceptions import AvitoBlockedError
|
||||
|
||||
snapshot = _make_snapshot(2)
|
||||
db = _mock_db(snapshot)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,297 @@
|
|||
"""Parity tests for issue #2310 (Group C of scraper_kit migration epic #2277).
|
||||
|
||||
Migrates legacy scraper imports of these files to their `scraper_kit` equivalents
|
||||
(behavior of the enclosing task/service functions is unchanged — see vault audit
|
||||
`Scraper_Kit_Legacy_Dependency_Audit_0703`):
|
||||
|
||||
- app/services/house_imv_backfill.py → BrowserFetcher now from
|
||||
`scraper_kit.browser_fetcher` (was `app.services.scrapers.browser_fetcher`).
|
||||
avito_imv itself stays legacy (valuation dependency, issue #2308 — separate).
|
||||
- app/tasks/avito_detail_backfill.py → AvitoScraper/BrowserFetcher/avito
|
||||
exceptions/fetch_detail/research_in_session/save_detail_enrichment now from
|
||||
`scraper_kit.*`. `build_warmed_session`/`_AVITO_WARM_SEARCH_URL` stay legacy —
|
||||
kit's `build_warmed_session()` builds its curl_cffi session via
|
||||
`_build_detail_session()` with NO config parameter at all (unlike
|
||||
`fetch_detail`, which does), so there is no caller-side way to inject
|
||||
`settings.scraper_proxy_url` — migrating it would silently drop the sticky
|
||||
MGTS-proxy egress on the warm-batch path (avito_detail_backfill_use_curl=True,
|
||||
the PROD DEFAULT). Same bug category as issue #2322 (scraper_kit provider
|
||||
missing a config hook), different function — flagged as a new follow-up.
|
||||
- app/tasks/cian_history_backfill.py → fetch_detail/save_detail_enrichment
|
||||
(listings block) + BrowserFetcher now from `scraper_kit.*`. The houses block
|
||||
(cian_newbuilding.fetch_newbuilding/save_newbuilding_enrichment) and the
|
||||
valuations block (cian_valuation.estimate_via_cian_valuation) stay legacy —
|
||||
issue #2322 (newbuilding BrowserFetcher missing endpoint=) resp. issue #2308
|
||||
(valuation dependencies), both out of scope here.
|
||||
- app/tasks/ekb_geoportal_ingest.py → EkbGeoportalClient/TopoBuilding now
|
||||
from `scraper_kit.providers.ekb_geoportal.client` (already proven byte-
|
||||
identical by tests/scrapers/test_ekb_geoportal_client_kit_parity.py, #2307).
|
||||
|
||||
This file proves the DELTA introduced by the #2310 migration specifically —
|
||||
general avito/cian-provider parsing golden-parity is already covered by
|
||||
`tests/test_scraper_kit_avito_golden_parity.py` and
|
||||
`tests/test_scraper_kit_cian_golden_parity.py`. Uses `tests/support/parity.assert_parity`
|
||||
per the epic's parity-gate convention (issue #2304).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import dataclasses
|
||||
import json
|
||||
import os
|
||||
from typing import Any
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
# Legacy app.services.scrapers.* / app.core.config.settings require DATABASE_URL at
|
||||
# import time (same pattern as tests/scrapers/test_avito_detail_kit_parity.py).
|
||||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
|
||||
|
||||
from scraper_kit.avito_exceptions import AvitoBlockedError as KitAvitoBlockedError
|
||||
from scraper_kit.providers.avito.detail import fetch_detail as kit_avito_fetch_detail
|
||||
from scraper_kit.providers.avito.serp import AvitoScraper as KitAvitoScraper
|
||||
from scraper_kit.providers.cian.detail import fetch_detail as kit_cian_fetch_detail
|
||||
|
||||
from app.core.config import settings
|
||||
from app.services.scraper_adapters import RealScraperConfig
|
||||
from app.services.scrapers.avito_detail import fetch_detail as legacy_avito_fetch_detail
|
||||
from app.services.scrapers.cian_detail import fetch_detail as legacy_cian_fetch_detail
|
||||
from tests.support.parity import assert_parity
|
||||
|
||||
# ── avito fetch_detail: backconnect config-gate (avito_detail_backfill.py delta) ──
|
||||
|
||||
|
||||
def _mock_session_403_then_ok(html: str) -> MagicMock:
|
||||
"""curl_cffi-like session: first .get() -> 403, second -> 200 with html."""
|
||||
resp_403 = MagicMock(status_code=403, text="")
|
||||
resp_ok = MagicMock(status_code=200, text=html)
|
||||
session = MagicMock()
|
||||
session.get = AsyncMock(side_effect=[resp_403, resp_ok])
|
||||
session.close = AsyncMock()
|
||||
return session
|
||||
|
||||
|
||||
def _detail_html() -> str:
|
||||
return (
|
||||
'<html><body><div data-marker="item-view/item-id">'
|
||||
"№ 1 · 1 мая в 10:00 · 1 просмотр</div>"
|
||||
'<span itemprop="price" content="1000000">1 000 000 ₽</span>'
|
||||
"</body></html>"
|
||||
)
|
||||
|
||||
|
||||
async def test_avito_fetch_detail_without_config_raises_on_403_no_reconnect() -> None:
|
||||
"""kit fetch_detail with config=None never retries on 403 — proves the config-gate
|
||||
is real: avito_detail_backfill's own-session call site MUST pass config= or it loses
|
||||
the backconnect-reconnect-on-block behavior that legacy always had via settings.*."""
|
||||
session = _mock_session_403_then_ok(_detail_html())
|
||||
with patch("scraper_kit.providers.avito.detail.asyncio.sleep", new_callable=AsyncMock):
|
||||
try:
|
||||
await kit_avito_fetch_detail(
|
||||
"https://www.avito.ru/x", cffi_session=session, reconnect_on_block=True
|
||||
)
|
||||
except KitAvitoBlockedError:
|
||||
pass
|
||||
else:
|
||||
raise AssertionError("expected AvitoBlockedError when config= is omitted")
|
||||
# Only ONE .get() happened — no reconnect retry attempted (backconnect gate closed).
|
||||
assert session.get.call_count == 1
|
||||
|
||||
|
||||
async def test_avito_fetch_detail_with_config_reconnects_on_403_like_legacy() -> None:
|
||||
"""With config=RealScraperConfig() (our actual avito_detail_backfill call site) and
|
||||
a configured proxy, a 403 triggers a fresh-session reconnect retry — matching legacy,
|
||||
which always read settings.scraper_proxy_url directly (no injection needed)."""
|
||||
session = _mock_session_403_then_ok(_detail_html())
|
||||
retry_session = MagicMock()
|
||||
retry_session.get = AsyncMock(return_value=MagicMock(status_code=200, text=_detail_html()))
|
||||
retry_session.close = AsyncMock()
|
||||
|
||||
with (
|
||||
patch.object(settings, "scraper_proxy_url_env", "http://test-proxy.local:8080"),
|
||||
patch("scraper_kit.providers.avito.detail.asyncio.sleep", new_callable=AsyncMock),
|
||||
patch(
|
||||
"scraper_kit.providers.avito.detail._build_detail_session",
|
||||
return_value=retry_session,
|
||||
),
|
||||
):
|
||||
result = await kit_avito_fetch_detail(
|
||||
"https://www.avito.ru/x",
|
||||
cffi_session=session,
|
||||
reconnect_on_block=True,
|
||||
config=RealScraperConfig(),
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
# First (caller-provided) session saw the 403; retry_session served the 200 that
|
||||
# produced the parsed result.
|
||||
assert session.get.call_count == 1
|
||||
retry_session.get.assert_awaited_once()
|
||||
|
||||
|
||||
# ── AvitoScraper: kit constructor now requires ScraperConfig (positional) ────────
|
||||
|
||||
|
||||
def test_avito_scraper_rotate_ip_reads_injected_config() -> None:
|
||||
"""kit AvitoScraper(config) is a required positional arg (legacy AvitoScraper()
|
||||
took none, reading settings.avito_proxy_rotate_url directly at call time inside
|
||||
_rotate_ip). avito_detail_backfill.py now constructs AvitoScraper(RealScraperConfig())
|
||||
— this proves the injected config is what _rotate_ip actually reads."""
|
||||
scraper = KitAvitoScraper(RealScraperConfig())
|
||||
assert scraper._config.avito_proxy_rotate_url == settings.avito_proxy_rotate_url
|
||||
|
||||
|
||||
# ── cian fetch_detail via browser_fetcher (cian_history_backfill.py call shape) ──
|
||||
|
||||
|
||||
def _cian_detail_html() -> str:
|
||||
default_state = {
|
||||
"offerData": {
|
||||
"offer": {
|
||||
"cianId": 777,
|
||||
"priceChanges": [
|
||||
{"changeTime": "2026-06-01T00:00:00Z", "price": 5900000, "diffPercent": -0.5}
|
||||
],
|
||||
},
|
||||
"stats": {},
|
||||
}
|
||||
}
|
||||
ds_json = json.dumps(default_state, ensure_ascii=False)
|
||||
return (
|
||||
"<html><body><script>\n"
|
||||
"window._cianConfig['frontend-offer-card'] = "
|
||||
"window._cianConfig['frontend-offer-card'] || [];\n"
|
||||
"window._cianConfig['frontend-offer-card'].push({"
|
||||
f"key: 'defaultState', value: {ds_json}, priority: 10, filter: null"
|
||||
"});\n</script></body></html>"
|
||||
)
|
||||
|
||||
|
||||
def _fake_browser_fetcher(html: str) -> Any:
|
||||
bf = MagicMock()
|
||||
bf.fetch = AsyncMock(return_value=html)
|
||||
return bf
|
||||
|
||||
|
||||
def test_cian_fetch_detail_via_browser_fetcher_parity() -> None:
|
||||
"""cian_history_backfill.py's listings block ALWAYS calls fetch_detail(url,
|
||||
browser_fetcher=bf) — never the curl_cffi/config own-session path exercised by
|
||||
tests/test_scraper_kit_pricehistory_session_parity.py (cian_price_history.py's
|
||||
call shape). Proves parity holds for THIS call shape too."""
|
||||
html = _cian_detail_html()
|
||||
url = "https://ekb.cian.ru/sale/flat/777/"
|
||||
|
||||
def _run_legacy() -> Any:
|
||||
return dataclasses.asdict(
|
||||
asyncio.run(legacy_cian_fetch_detail(url, browser_fetcher=_fake_browser_fetcher(html)))
|
||||
)
|
||||
|
||||
def _run_kit() -> Any:
|
||||
return dataclasses.asdict(
|
||||
asyncio.run(kit_cian_fetch_detail(url, browser_fetcher=_fake_browser_fetcher(html)))
|
||||
)
|
||||
|
||||
assert_parity(legacy_fn=_run_legacy, kit_fn=_run_kit, fixtures=[()])
|
||||
|
||||
|
||||
# ── avito fetch_detail via browser_fetcher (avito_detail_backfill.py browser_mode) ──
|
||||
|
||||
|
||||
def test_avito_fetch_detail_via_browser_fetcher_parity() -> None:
|
||||
"""browser_mode branch of avito_detail_backfill.py calls fetch_detail(url,
|
||||
browser_fetcher=browser_fetcher) — config is never read on this path (early
|
||||
return before the config-gated backconnect logic). Confirms legacy/kit agree."""
|
||||
html = _detail_html()
|
||||
url = "https://www.avito.ru/ekaterinburg/kvartiry/test_1"
|
||||
|
||||
def _run_legacy() -> Any:
|
||||
return dataclasses.asdict(
|
||||
asyncio.run(legacy_avito_fetch_detail(url, browser_fetcher=_fake_browser_fetcher(html)))
|
||||
)
|
||||
|
||||
def _run_kit() -> Any:
|
||||
return dataclasses.asdict(
|
||||
asyncio.run(kit_avito_fetch_detail(url, browser_fetcher=_fake_browser_fetcher(html)))
|
||||
)
|
||||
|
||||
assert_parity(legacy_fn=_run_legacy, kit_fn=_run_kit, fixtures=[()])
|
||||
|
||||
|
||||
# ── BrowserFetcher(endpoint=) wiring — house_imv_backfill.py / cian_history_backfill.py ──
|
||||
|
||||
|
||||
async def test_house_imv_backfill_browser_fetcher_uses_settings_endpoint() -> None:
|
||||
"""house_imv_backfill.py's BrowserFetcher(source="avito") now needs endpoint=
|
||||
(kit's mandatory keyword-only param) — proves the call site passes
|
||||
settings.browser_http_endpoint, matching legacy's internal settings read."""
|
||||
from app.services import house_imv_backfill
|
||||
|
||||
captured: dict[str, Any] = {}
|
||||
|
||||
class _FakeBrowserFetcher:
|
||||
def __init__(self, **kwargs: Any) -> None:
|
||||
captured.update(kwargs)
|
||||
|
||||
async def __aenter__(self) -> _FakeBrowserFetcher:
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *_: object) -> None:
|
||||
return None
|
||||
|
||||
with (
|
||||
patch.object(house_imv_backfill.settings, "avito_imv_use_browser_fetcher", True),
|
||||
patch.object(house_imv_backfill, "BrowserFetcher", _FakeBrowserFetcher),
|
||||
patch.object(house_imv_backfill, "_process_one_house", AsyncMock(return_value="ok")),
|
||||
):
|
||||
db = MagicMock()
|
||||
db.execute.return_value.mappings.return_value.all.return_value = [
|
||||
{"id": 1, "address": "ЕКБ, ул. X, 1", "full_address": None, "lat": 56.8, "lon": 60.6}
|
||||
]
|
||||
await house_imv_backfill.backfill_house_imv(db, batch_size=1)
|
||||
|
||||
assert captured == {"source": "avito", "endpoint": settings.browser_http_endpoint}
|
||||
|
||||
|
||||
async def test_cian_history_backfill_browser_fetcher_uses_settings_endpoint() -> None:
|
||||
"""cian_history_backfill.py's BrowserFetcher(source="cian") also needs endpoint=."""
|
||||
from app.tasks import cian_history_backfill
|
||||
|
||||
captured: dict[str, Any] = {}
|
||||
|
||||
class _FakeBrowserFetcher:
|
||||
def __init__(self, **kwargs: Any) -> None:
|
||||
captured.update(kwargs)
|
||||
|
||||
async def __aenter__(self) -> _FakeBrowserFetcher:
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *_: object) -> None:
|
||||
return None
|
||||
|
||||
db = MagicMock()
|
||||
db.execute.return_value.mappings.return_value.all.return_value = []
|
||||
|
||||
with patch.object(cian_history_backfill, "BrowserFetcher", _FakeBrowserFetcher):
|
||||
await cian_history_backfill.backfill_cian_history(
|
||||
db, do_listings=True, do_houses=False, do_valuations=False
|
||||
)
|
||||
|
||||
assert captured == {"source": "cian", "endpoint": settings.browser_http_endpoint}
|
||||
|
||||
|
||||
# ── ekb_geoportal_ingest.py: task module resolves the kit client import ──────────
|
||||
|
||||
|
||||
def test_ekb_geoportal_ingest_imports_kit_client() -> None:
|
||||
"""app/tasks/ekb_geoportal_ingest.py now imports EkbGeoportalClient/TopoBuilding
|
||||
from scraper_kit.providers.ekb_geoportal.client (issue #2310) — proves the task
|
||||
module resolves to the SAME classes as the kit package (not a stale/duplicate
|
||||
import), and that construction still works with no arguments."""
|
||||
from scraper_kit.providers.ekb_geoportal.client import (
|
||||
EkbGeoportalClient as KitClient,
|
||||
)
|
||||
|
||||
from app.tasks.ekb_geoportal_ingest import EkbGeoportalClient as TaskClient
|
||||
|
||||
assert TaskClient is KitClient
|
||||
assert isinstance(TaskClient(), KitClient)
|
||||
Loading…
Add table
Reference in a new issue