feat(tradein/scraper-kit): migrate backfill-task imports to kit, Group C (#2310)

Migrates legacy app.services.scrapers.* imports to scraper_kit equivalents for
house_imv_backfill.py, avito_detail_backfill.py, cian_history_backfill.py,
ekb_geoportal_ingest.py, and yandex_detail_backfill.py, proving parity via
tests/support/parity.assert_parity per the epic's gate (#2304).

newbuilding_enrich_backfill.py and yandex_newbuilding_sweep.py are left fully
on legacy imports: both call into scraper_kit.providers.{cian,yandex}.newbuilding,
which construct BrowserFetcher(source=...) without the now-mandatory endpoint=
kwarg (issue #2322, verified still open against the actual provider source, not
just issue status) -- no caller-side fix is possible, matching Group A's (#2305)
precedent for the same bug in admin.py.

Config-gated kit function footguns found and fixed (Group B #2306 pattern):
  - avito fetch_detail's backconnect-on-403 retry silently drops when config=
    is omitted -- now passes config=RealScraperConfig() explicitly, with a
    regression test proving the gate.
  - kit AvitoScraper's constructor now requires ScraperConfig positionally --
    wired via RealScraperConfig(), which _rotate_ip() reads for
    avito_proxy_rotate_url.
  - BrowserFetcher(source=...) call sites (house_imv_backfill, avito/cian
    detail backfills) now pass the mandatory endpoint=settings.browser_http_endpoint.

New footgun discovered (NOT #2322, flagged for follow-up): kit's
build_warmed_session() builds its curl_cffi session via _build_detail_session()
with no config parameter at all, unlike fetch_detail -- migrating it would
silently drop the sticky MGTS-proxy egress on avito_detail_backfill's
warm-batch path (the prod default). Left build_warmed_session/
_AVITO_WARM_SEARCH_URL on legacy imports, documented inline.

Refs #2310
This commit is contained in:
bot-backend 2026-07-04 01:04:50 +03:00
parent 793234cff1
commit eb564cfc01
9 changed files with 400 additions and 43 deletions

View file

@ -25,10 +25,18 @@ from collections.abc import Callable
from dataclasses import dataclass, field from dataclasses import dataclass, field
from typing import Literal from typing import Literal
from scraper_kit.browser_fetcher import BrowserFetcher
from sqlalchemy import text from sqlalchemy import text
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from app.core.config import settings from app.core.config import settings
# app.services.scrapers.avito_imv остаётся legacy — миграция валюационных
# зависимостей (avito_imv/cian_valuation/yandex_valuation) выделена в отдельный,
# более рискованный issue #2308 (см. epic #2277, issue #2310 Group C). Мигрируем
# здесь только BrowserFetcher (#2310, выше) — avito_imv.evaluate_via_imv принимает
# его исключительно как TYPE_CHECKING-аннотацию (duck-typed на fetch_json), поэтому
# смена конкретного класса не требует правок avito_imv.py.
from app.services.scrapers.avito_imv import ( from app.services.scrapers.avito_imv import (
IMVAddressNotFoundError, IMVAddressNotFoundError,
IMVAuthError, IMVAuthError,
@ -36,7 +44,6 @@ from app.services.scrapers.avito_imv import (
IMVTransientError, IMVTransientError,
evaluate_via_imv, evaluate_via_imv,
) )
from app.services.scrapers.browser_fetcher import BrowserFetcher
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -484,7 +491,7 @@ async def backfill_house_imv(
# Флаг OFF → _bf=None → evaluate_via_imv делает свою curl-сессию как раньше # Флаг OFF → _bf=None → evaluate_via_imv делает свою curl-сессию как раньше
# (поведение байт-в-байт идентично доспринтовому). # (поведение байт-в-байт идентично доспринтовому).
if settings.avito_imv_use_browser_fetcher: if settings.avito_imv_use_browser_fetcher:
async with BrowserFetcher(source="avito") as _bf: async with BrowserFetcher(source="avito", endpoint=settings.browser_http_endpoint) as _bf:
await _run_loop(_bf) await _run_loop(_bf)
else: else:
await _run_loop(None) await _run_loop(None)

View file

@ -23,6 +23,18 @@ from dataclasses import dataclass, field
from urllib.parse import urlparse from urllib.parse import urlparse
from curl_cffi.requests import AsyncSession from curl_cffi.requests import AsyncSession
from scraper_kit.avito_exceptions import (
AvitoBlockedError,
AvitoListingGoneError,
AvitoRateLimitedError,
)
from scraper_kit.browser_fetcher import BrowserFetcher
from scraper_kit.providers.avito.detail import (
fetch_detail,
research_in_session,
save_detail_enrichment,
)
from scraper_kit.providers.avito.serp import AvitoScraper
from sqlalchemy import text from sqlalchemy import text
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
@ -30,20 +42,22 @@ from app.core.config import settings
from app.core.shutdown import shutdown_requested from app.core.shutdown import shutdown_requested
from app.services import scrape_runs as runs_mod from app.services import scrape_runs as runs_mod
from app.services.scrape_pipeline import _CHROME_HEADERS, _avito_proxies from app.services.scrape_pipeline import _CHROME_HEADERS, _avito_proxies
from app.services.scrapers.avito import AvitoScraper from app.services.scraper_adapters import RealScraperConfig
# _AVITO_WARM_SEARCH_URL/build_warmed_session остаются legacy (#2310): kit's
# scraper_kit.providers.avito.detail.build_warmed_session() строит сессию через
# _build_detail_session() БЕЗ config-параметра (сигнатура вообще не принимает
# ScraperConfig — в отличие от fetch_detail, здесь нет caller-side способа
# прокинуть прокси). Мигрировав этот импорт, warm-batch curl-путь (use_curl=True,
# ДЕФОЛТ прод-режима, avito_detail_backfill_use_curl=True) молча перестал бы
# использовать settings.scraper_proxy_url (sticky МГТС-прокси) — прямое
# datacenter-подключение вместо backconnect. Тот же класс бага, что #2322
# (недостающая инъекция конфига в scraper_kit provider-функции), но другая
# функция — заведён как отдельный follow-up, НЕ #2322. См. отчёт issue #2310.
from app.services.scrapers.avito_detail import ( from app.services.scrapers.avito_detail import (
_AVITO_WARM_SEARCH_URL, _AVITO_WARM_SEARCH_URL,
build_warmed_session, build_warmed_session,
fetch_detail,
research_in_session,
save_detail_enrichment,
) )
from app.services.scrapers.avito_exceptions import (
AvitoBlockedError,
AvitoListingGoneError,
AvitoRateLimitedError,
)
from app.services.scrapers.browser_fetcher import BrowserFetcher
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -121,7 +135,10 @@ async def run_avito_detail_backfill(
own_session = False own_session = False
own_browser = False own_browser = False
scraper = AvitoScraper() # kit AvitoScraper требует ScraperConfig позиционно (Strangler-инжекция #2133) —
# RealScraperConfig проксирует settings.* так же, как читал legacy-конструктор без
# аргументов (avito_proxy_rotate_url и т.д. для _rotate_ip()).
scraper = AvitoScraper(RealScraperConfig())
start = time.monotonic() start = time.monotonic()
logger.info( logger.info(
@ -136,7 +153,9 @@ async def run_avito_detail_backfill(
# эфемерную _build_detail_session() через settings.scraper_proxy_url (backconnect). # эфемерную _build_detail_session() через settings.scraper_proxy_url (backconnect).
# use_curl=False (legacy): browser_mode=True → BrowserFetcher/auv, как раньше. # use_curl=False (legacy): browser_mode=True → BrowserFetcher/auv, как раньше.
if browser_mode: if browser_mode:
browser_fetcher = BrowserFetcher(source="avito") browser_fetcher = BrowserFetcher(
source="avito", endpoint=settings.browser_http_endpoint
)
await browser_fetcher.__aenter__() await browser_fetcher.__aenter__()
own_browser = True own_browser = True
scraper._browser = browser_fetcher scraper._browser = browser_fetcher
@ -314,6 +333,13 @@ async def run_avito_detail_backfill(
browser_fetcher=browser_fetcher, browser_fetcher=browser_fetcher,
referer=_AVITO_WARM_SEARCH_URL if use_curl else None, referer=_AVITO_WARM_SEARCH_URL if use_curl else None,
reconnect_on_block=not use_curl, reconnect_on_block=not use_curl,
# config= обязателен: kit fetch_detail без него читает
# config.scraper_proxy_url=None для backconnect-gate'а
# (elif not use_curl own-session path), теряя reconnect-
# on-403 поведение legacy (settings.scraper_proxy_url
# напрямую). Не влияет на use_curl=True (прод-дефолт) —
# там reconnect_on_block=False уже гасит backconnect.
config=RealScraperConfig(),
), ),
timeout=fetch_timeout_s, timeout=fetch_timeout_s,
) )

View file

@ -27,12 +27,17 @@ import logging
import time import time
from dataclasses import dataclass, field from dataclasses import dataclass, field
from scraper_kit.browser_fetcher import BrowserFetcher
from scraper_kit.providers.cian.detail import fetch_detail, save_detail_enrichment
from sqlalchemy import text from sqlalchemy import text
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from app.core.config import settings
from app.services.scraper_settings import get_scraper_delay from app.services.scraper_settings import get_scraper_delay
from app.services.scrapers.browser_fetcher import BrowserFetcher
from app.services.scrapers.cian_detail import fetch_detail, save_detail_enrichment # cian_valuation остаётся legacy — валюационные зависимости (avito_imv/
# cian_valuation/yandex_valuation) выделены в отдельный, более рискованный
# issue #2308 (см. epic #2277, issue #2310 Group C), не в этой миграции.
from app.services.scrapers.cian_valuation import estimate_via_cian_valuation from app.services.scrapers.cian_valuation import estimate_via_cian_valuation
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -113,7 +118,7 @@ async def backfill_cian_history(
else: else:
# One BrowserFetcher instance shared across all listings in this batch. # One BrowserFetcher instance shared across all listings in this batch.
# priceChanges requires JS rendering — curl_cffi returns empty list (#1574). # priceChanges requires JS rendering — curl_cffi returns empty list (#1574).
async with BrowserFetcher(source="cian") as bf: async with BrowserFetcher(source="cian", endpoint=settings.browser_http_endpoint) as bf:
for row in rows: for row in rows:
listing_id: int = row["id"] listing_id: int = row["id"]
source_url: str = row["source_url"] source_url: str = row["source_url"]
@ -157,6 +162,11 @@ async def backfill_cian_history(
# ── 2. Houses: missing houses_price_dynamics ────────────────────────────── # ── 2. Houses: missing houses_price_dynamics ──────────────────────────────
if do_houses: if do_houses:
# cian_newbuilding остаётся legacy (issue #2310, NOT migrated): kit's
# scraper_kit.providers.cian.newbuilding.fetch_newbuilding() constructs
# BrowserFetcher(source="cian") WITHOUT the now-mandatory endpoint= kwarg
# → TypeError on every call, no caller-side fix possible (issue #2322,
# still open — verified against provider source, not just issue status).
from app.services.scrapers.cian_newbuilding import ( from app.services.scrapers.cian_newbuilding import (
fetch_newbuilding, fetch_newbuilding,
save_newbuilding_enrichment, save_newbuilding_enrichment,

View file

@ -24,14 +24,14 @@ import logging
import os import os
import re import re
from scraper_kit.providers.ekb_geoportal.client import (
EkbGeoportalClient,
TopoBuilding,
)
from sqlalchemy import text from sqlalchemy import text
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from app.core.db import SessionLocal from app.core.db import SessionLocal
from app.services.scrapers.ekb_geoportal_client import (
EkbGeoportalClient,
TopoBuilding,
)
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)

View file

@ -95,18 +95,18 @@ class NewbuildingEnrichBackfillResult:
"""Per-run counters for the newbuilding-enrichment backfill.""" """Per-run counters for the newbuilding-enrichment backfill."""
# Population sizing (independent of `limit`). # Population sizing (independent of `limit`).
cian_houses_total: int = 0 # houses linked to ext_source='cian_newbuilding' cian_houses_total: int = 0 # houses linked to ext_source='cian_newbuilding'
cian_houses_fetchable: int = 0 # of those, with cian_zhk_url OR resolvable ext_id cian_houses_fetchable: int = 0 # of those, with cian_zhk_url OR resolvable ext_id
cian_houses_pending: int = 0 # fetchable AND not yet fully enriched (force=False) cian_houses_pending: int = 0 # fetchable AND not yet fully enriched (force=False)
# Processing (bounded by `limit`). # Processing (bounded by `limit`).
processed: int = 0 processed: int = 0
skipped_already_enriched: int = 0 skipped_already_enriched: int = 0
succeeded: int = 0 succeeded: int = 0
resolved_zhk_url: int = 0 # ext_id → cian_zhk_url resolved + persisted (#972) resolved_zhk_url: int = 0 # ext_id → cian_zhk_url resolved + persisted (#972)
failed_resolve: int = 0 # had only ext_id, resolver returned None failed_resolve: int = 0 # had only ext_id, resolver returned None
failed_fetch: int = 0 # fetch returned None / raised failed_fetch: int = 0 # fetch returned None / raised
failed_save: int = 0 # save raised after a good fetch failed_save: int = 0 # save raised after a good fetch
# Row-level deltas (how much actually landed). # Row-level deltas (how much actually landed).
price_dynamics_rows: int = 0 price_dynamics_rows: int = 0
@ -355,6 +355,16 @@ async def backfill_newbuilding_enrichment(
# Import here (not module top) so the heavy curl_cffi scraper stack is only loaded # Import here (not module top) so the heavy curl_cffi scraper stack is only loaded
# when the task actually runs — mirrors cian_history_backfill's lazy import and keeps # when the task actually runs — mirrors cian_history_backfill's lazy import and keeps
# unit tests able to patch fetch_newbuilding cheaply. # unit tests able to patch fetch_newbuilding cheaply.
#
# NOT migrated to scraper_kit (issue #2310, Group C): kit's
# scraper_kit.providers.cian.newbuilding.fetch_newbuilding() constructs
# BrowserFetcher(source="cian") WITHOUT the now-mandatory endpoint= kwarg —
# every call raises TypeError, with no caller-side fix possible (the kit
# provider function doesn't expose a config/endpoint hook at all). This is
# issue #2322, verified STILL OPEN by reading the actual provider source
# (not just the issue's open/closed status) at the time of this migration.
# Left on legacy entirely, exactly like Group A (#2305) did for the same
# bug in admin.py's debug endpoints.
from app.services.scrapers.cian_newbuilding import ( from app.services.scrapers.cian_newbuilding import (
fetch_newbuilding, fetch_newbuilding,
resolve_cian_zhk_url_via_search, resolve_cian_zhk_url_via_search,
@ -371,11 +381,7 @@ async def backfill_newbuilding_enrichment(
result.cian_houses_fetchable = sizing["fetchable"] result.cian_houses_fetchable = sizing["fetchable"]
result.cian_houses_pending = sizing["pending"] result.cian_houses_pending = sizing["pending"]
rows = ( rows = db.execute(text(_SELECT_PENDING_HOUSES), {"force": force, "lim": limit}).mappings().all()
db.execute(text(_SELECT_PENDING_HOUSES), {"force": force, "lim": limit})
.mappings()
.all()
)
logger.info( logger.info(
"newbuilding-enrich backfill: cian_houses total=%d fetchable=%d pending=%d; " "newbuilding-enrich backfill: cian_houses total=%d fetchable=%d pending=%d; "
@ -547,9 +553,7 @@ async def backfill_newbuilding_enrichment(
db.commit() db.commit()
except Exception as dexc: except Exception as dexc:
sp.rollback() sp.rollback()
logger.warning( logger.warning("reliability dedup failed house_id=%s: %s", house_id, dexc)
"reliability dedup failed house_id=%s: %s", house_id, dexc
)
pd_after, rc_after, rv_after = _house_enrichment_counts(db, house_id) pd_after, rc_after, rv_after = _house_enrichment_counts(db, house_id)
result.price_dynamics_rows += max(0, pd_after - pd_before) result.price_dynamics_rows += max(0, pd_after - pd_before)

View file

@ -31,12 +31,12 @@ import time
from dataclasses import dataclass, field from dataclasses import dataclass, field
from curl_cffi.requests import AsyncSession from curl_cffi.requests import AsyncSession
from scraper_kit.providers.yandex.detail import YandexDetailScraper, save_detail_enrichment
from sqlalchemy import text from sqlalchemy import text
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from app.core.config import settings from app.core.config import settings
from app.services import scrape_runs as runs_mod from app.services import scrape_runs as runs_mod
from app.services.scrapers.yandex_detail import YandexDetailScraper, save_detail_enrichment
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)

View file

@ -206,6 +206,15 @@ async def enrich_yandex_newbuilding_sweep(
Returns: Returns:
YandexNewbuildingSweepResult со счётчиками. YandexNewbuildingSweepResult со счётчиками.
""" """
# NOT migrated to scraper_kit (issue #2310, Group C): both
# scraper_kit.providers.yandex.newbuilding.YandexNewbuildingScraper.fetch_jk()
# AND resolve_yandex_jk_slug() construct BrowserFetcher(source="yandex")
# WITHOUT the now-mandatory endpoint= kwarg — every call raises TypeError,
# with no caller-side fix possible (the kit provider functions don't expose
# a config/endpoint hook at all). This is issue #2322, verified STILL OPEN
# by reading the actual provider source (not just the issue's open/closed
# status) at the time of this migration. Left on legacy entirely, exactly
# like Group A (#2305) did for the same bug in admin.py's debug endpoints.
from app.services.scrapers.yandex_newbuilding import ( from app.services.scrapers.yandex_newbuilding import (
YandexNewbuildingScraper, YandexNewbuildingScraper,
resolve_yandex_jk_slug, resolve_yandex_jk_slug,

View file

@ -145,7 +145,7 @@ async def test_backfill_blocked_abort_after_max_consecutive() -> None:
#1950 abort-reorder: abort-check ПЕРЕД recovery → на 5-м (аборт-)блоке rotate_ip #1950 abort-reorder: abort-check ПЕРЕД recovery → на 5-м (аборт-)блоке rotate_ip
НЕ дёргается (не тратим recovery на финальном блоке). rotate_ip x4 (блоки 1-4). НЕ дёргается (не тратим recovery на финальном блоке). rotate_ip x4 (блоки 1-4).
""" """
from app.services.scrapers.avito_exceptions import AvitoBlockedError from scraper_kit.avito_exceptions import AvitoBlockedError
snapshot = _make_snapshot(10) snapshot = _make_snapshot(10)
db = _mock_db(snapshot) db = _mock_db(snapshot)
@ -267,7 +267,7 @@ async def test_backfill_top_level_exception_marks_failed() -> None:
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_backfill_rotate_ip_called_on_each_block() -> None: async def test_backfill_rotate_ip_called_on_each_block() -> None:
"""1 block + 1 success -> rotate_ip called once, enriched=1.""" """1 block + 1 success -> rotate_ip called once, enriched=1."""
from app.services.scrapers.avito_exceptions import AvitoBlockedError from scraper_kit.avito_exceptions import AvitoBlockedError
snapshot = _make_snapshot(2) snapshot = _make_snapshot(2)
db = _mock_db(snapshot) db = _mock_db(snapshot)
@ -379,6 +379,7 @@ async def test_backfill_fetch_timeout_skips_and_continues() -> None:
browser_fetcher: object = None, browser_fetcher: object = None,
referer: object = None, referer: object = None,
reconnect_on_block: bool = True, reconnect_on_block: bool = True,
config: object = None,
): ):
call_urls.append(url) call_urls.append(url)
if len(call_urls) == 1: if len(call_urls) == 1:
@ -423,7 +424,7 @@ async def test_backfill_listing_gone_marks_inactive_no_breaker() -> None:
состоит из dead-листингов; раньше 404 ловился как soft-block breaker абортил run состоит из dead-листингов; раньше 404 ловился как soft-block breaker абортил run
до live-листингов. Теперь 404 нейтрален к breaker'у и метит листинг inactive. до live-листингов. Теперь 404 нейтрален к breaker'у и метит листинг inactive.
""" """
from app.services.scrapers.avito_exceptions import AvitoListingGoneError from scraper_kit.avito_exceptions import AvitoListingGoneError
snapshot = _make_snapshot(1) snapshot = _make_snapshot(1)
db = _mock_db(snapshot) db = _mock_db(snapshot)
@ -541,8 +542,11 @@ async def test_backfill_use_curl_false_creates_browser_fetcher() -> None:
db, run_id=10, params={"batch_size": 10, "budget_sec": 3600} db, run_id=10, params={"batch_size": 10, "budget_sec": 3600}
) )
# BrowserFetcher должен быть создан (source="avito") # BrowserFetcher должен быть создан (source="avito", endpoint из settings — #2310
mock_bf_cls.assert_called_once_with(source="avito") # kit BrowserFetcher требует endpoint= обязательным keyword-only параметром)
mock_bf_cls.assert_called_once_with(
source="avito", endpoint=fake_settings.browser_http_endpoint
)
# fetch_detail вызван с browser_fetcher установленным (не None) # fetch_detail вызван с browser_fetcher установленным (не None)
assert mock_fetch.call_count == 1 assert mock_fetch.call_count == 1
_, kwargs = mock_fetch.call_args _, kwargs = mock_fetch.call_args
@ -560,7 +564,7 @@ async def test_backfill_use_curl_block_cooldown_research_no_rebuild() -> None:
нельзя. Блок = rate-limit текущего IP cooldown + re-search той же сессией. нельзя. Блок = rate-limit текущего IP cooldown + re-search той же сессией.
fetch: 1-й вызов BLOCKED, 2-й успех. fetch: 1-й вызов BLOCKED, 2-й успех.
""" """
from app.services.scrapers.avito_exceptions import AvitoBlockedError from scraper_kit.avito_exceptions import AvitoBlockedError
snapshot = _make_snapshot(2) snapshot = _make_snapshot(2)
db = _mock_db(snapshot) db = _mock_db(snapshot)

View file

@ -0,0 +1,297 @@
"""Parity tests for issue #2310 (Group C of scraper_kit migration epic #2277).
Migrates legacy scraper imports of these files to their `scraper_kit` equivalents
(behavior of the enclosing task/service functions is unchanged see vault audit
`Scraper_Kit_Legacy_Dependency_Audit_0703`):
- app/services/house_imv_backfill.py BrowserFetcher now from
`scraper_kit.browser_fetcher` (was `app.services.scrapers.browser_fetcher`).
avito_imv itself stays legacy (valuation dependency, issue #2308 — separate).
- app/tasks/avito_detail_backfill.py AvitoScraper/BrowserFetcher/avito
exceptions/fetch_detail/research_in_session/save_detail_enrichment now from
`scraper_kit.*`. `build_warmed_session`/`_AVITO_WARM_SEARCH_URL` stay legacy
kit's `build_warmed_session()` builds its curl_cffi session via
`_build_detail_session()` with NO config parameter at all (unlike
`fetch_detail`, which does), so there is no caller-side way to inject
`settings.scraper_proxy_url` migrating it would silently drop the sticky
MGTS-proxy egress on the warm-batch path (avito_detail_backfill_use_curl=True,
the PROD DEFAULT). Same bug category as issue #2322 (scraper_kit provider
missing a config hook), different function flagged as a new follow-up.
- app/tasks/cian_history_backfill.py fetch_detail/save_detail_enrichment
(listings block) + BrowserFetcher now from `scraper_kit.*`. The houses block
(cian_newbuilding.fetch_newbuilding/save_newbuilding_enrichment) and the
valuations block (cian_valuation.estimate_via_cian_valuation) stay legacy
issue #2322 (newbuilding BrowserFetcher missing endpoint=) resp. issue #2308
(valuation dependencies), both out of scope here.
- app/tasks/ekb_geoportal_ingest.py EkbGeoportalClient/TopoBuilding now
from `scraper_kit.providers.ekb_geoportal.client` (already proven byte-
identical by tests/scrapers/test_ekb_geoportal_client_kit_parity.py, #2307).
This file proves the DELTA introduced by the #2310 migration specifically —
general avito/cian-provider parsing golden-parity is already covered by
`tests/test_scraper_kit_avito_golden_parity.py` and
`tests/test_scraper_kit_cian_golden_parity.py`. Uses `tests/support/parity.assert_parity`
per the epic's parity-gate convention (issue #2304).
"""
from __future__ import annotations
import asyncio
import dataclasses
import json
import os
from typing import Any
from unittest.mock import AsyncMock, MagicMock, patch
# Legacy app.services.scrapers.* / app.core.config.settings require DATABASE_URL at
# import time (same pattern as tests/scrapers/test_avito_detail_kit_parity.py).
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
from scraper_kit.avito_exceptions import AvitoBlockedError as KitAvitoBlockedError
from scraper_kit.providers.avito.detail import fetch_detail as kit_avito_fetch_detail
from scraper_kit.providers.avito.serp import AvitoScraper as KitAvitoScraper
from scraper_kit.providers.cian.detail import fetch_detail as kit_cian_fetch_detail
from app.core.config import settings
from app.services.scraper_adapters import RealScraperConfig
from app.services.scrapers.avito_detail import fetch_detail as legacy_avito_fetch_detail
from app.services.scrapers.cian_detail import fetch_detail as legacy_cian_fetch_detail
from tests.support.parity import assert_parity
# ── avito fetch_detail: backconnect config-gate (avito_detail_backfill.py delta) ──
def _mock_session_403_then_ok(html: str) -> MagicMock:
"""curl_cffi-like session: first .get() -> 403, second -> 200 with html."""
resp_403 = MagicMock(status_code=403, text="")
resp_ok = MagicMock(status_code=200, text=html)
session = MagicMock()
session.get = AsyncMock(side_effect=[resp_403, resp_ok])
session.close = AsyncMock()
return session
def _detail_html() -> str:
return (
'<html><body><div data-marker="item-view/item-id">'
"№ 1 · 1 мая в 10:00 · 1 просмотр</div>"
'<span itemprop="price" content="1000000">1 000 000 ₽</span>'
"</body></html>"
)
async def test_avito_fetch_detail_without_config_raises_on_403_no_reconnect() -> None:
"""kit fetch_detail with config=None never retries on 403 — proves the config-gate
is real: avito_detail_backfill's own-session call site MUST pass config= or it loses
the backconnect-reconnect-on-block behavior that legacy always had via settings.*."""
session = _mock_session_403_then_ok(_detail_html())
with patch("scraper_kit.providers.avito.detail.asyncio.sleep", new_callable=AsyncMock):
try:
await kit_avito_fetch_detail(
"https://www.avito.ru/x", cffi_session=session, reconnect_on_block=True
)
except KitAvitoBlockedError:
pass
else:
raise AssertionError("expected AvitoBlockedError when config= is omitted")
# Only ONE .get() happened — no reconnect retry attempted (backconnect gate closed).
assert session.get.call_count == 1
async def test_avito_fetch_detail_with_config_reconnects_on_403_like_legacy() -> None:
"""With config=RealScraperConfig() (our actual avito_detail_backfill call site) and
a configured proxy, a 403 triggers a fresh-session reconnect retry matching legacy,
which always read settings.scraper_proxy_url directly (no injection needed)."""
session = _mock_session_403_then_ok(_detail_html())
retry_session = MagicMock()
retry_session.get = AsyncMock(return_value=MagicMock(status_code=200, text=_detail_html()))
retry_session.close = AsyncMock()
with (
patch.object(settings, "scraper_proxy_url_env", "http://test-proxy.local:8080"),
patch("scraper_kit.providers.avito.detail.asyncio.sleep", new_callable=AsyncMock),
patch(
"scraper_kit.providers.avito.detail._build_detail_session",
return_value=retry_session,
),
):
result = await kit_avito_fetch_detail(
"https://www.avito.ru/x",
cffi_session=session,
reconnect_on_block=True,
config=RealScraperConfig(),
)
assert result is not None
# First (caller-provided) session saw the 403; retry_session served the 200 that
# produced the parsed result.
assert session.get.call_count == 1
retry_session.get.assert_awaited_once()
# ── AvitoScraper: kit constructor now requires ScraperConfig (positional) ────────
def test_avito_scraper_rotate_ip_reads_injected_config() -> None:
"""kit AvitoScraper(config) is a required positional arg (legacy AvitoScraper()
took none, reading settings.avito_proxy_rotate_url directly at call time inside
_rotate_ip). avito_detail_backfill.py now constructs AvitoScraper(RealScraperConfig())
this proves the injected config is what _rotate_ip actually reads."""
scraper = KitAvitoScraper(RealScraperConfig())
assert scraper._config.avito_proxy_rotate_url == settings.avito_proxy_rotate_url
# ── cian fetch_detail via browser_fetcher (cian_history_backfill.py call shape) ──
def _cian_detail_html() -> str:
default_state = {
"offerData": {
"offer": {
"cianId": 777,
"priceChanges": [
{"changeTime": "2026-06-01T00:00:00Z", "price": 5900000, "diffPercent": -0.5}
],
},
"stats": {},
}
}
ds_json = json.dumps(default_state, ensure_ascii=False)
return (
"<html><body><script>\n"
"window._cianConfig['frontend-offer-card'] = "
"window._cianConfig['frontend-offer-card'] || [];\n"
"window._cianConfig['frontend-offer-card'].push({"
f"key: 'defaultState', value: {ds_json}, priority: 10, filter: null"
"});\n</script></body></html>"
)
def _fake_browser_fetcher(html: str) -> Any:
bf = MagicMock()
bf.fetch = AsyncMock(return_value=html)
return bf
def test_cian_fetch_detail_via_browser_fetcher_parity() -> None:
"""cian_history_backfill.py's listings block ALWAYS calls fetch_detail(url,
browser_fetcher=bf) never the curl_cffi/config own-session path exercised by
tests/test_scraper_kit_pricehistory_session_parity.py (cian_price_history.py's
call shape). Proves parity holds for THIS call shape too."""
html = _cian_detail_html()
url = "https://ekb.cian.ru/sale/flat/777/"
def _run_legacy() -> Any:
return dataclasses.asdict(
asyncio.run(legacy_cian_fetch_detail(url, browser_fetcher=_fake_browser_fetcher(html)))
)
def _run_kit() -> Any:
return dataclasses.asdict(
asyncio.run(kit_cian_fetch_detail(url, browser_fetcher=_fake_browser_fetcher(html)))
)
assert_parity(legacy_fn=_run_legacy, kit_fn=_run_kit, fixtures=[()])
# ── avito fetch_detail via browser_fetcher (avito_detail_backfill.py browser_mode) ──
def test_avito_fetch_detail_via_browser_fetcher_parity() -> None:
"""browser_mode branch of avito_detail_backfill.py calls fetch_detail(url,
browser_fetcher=browser_fetcher) config is never read on this path (early
return before the config-gated backconnect logic). Confirms legacy/kit agree."""
html = _detail_html()
url = "https://www.avito.ru/ekaterinburg/kvartiry/test_1"
def _run_legacy() -> Any:
return dataclasses.asdict(
asyncio.run(legacy_avito_fetch_detail(url, browser_fetcher=_fake_browser_fetcher(html)))
)
def _run_kit() -> Any:
return dataclasses.asdict(
asyncio.run(kit_avito_fetch_detail(url, browser_fetcher=_fake_browser_fetcher(html)))
)
assert_parity(legacy_fn=_run_legacy, kit_fn=_run_kit, fixtures=[()])
# ── BrowserFetcher(endpoint=) wiring — house_imv_backfill.py / cian_history_backfill.py ──
async def test_house_imv_backfill_browser_fetcher_uses_settings_endpoint() -> None:
"""house_imv_backfill.py's BrowserFetcher(source="avito") now needs endpoint=
(kit's mandatory keyword-only param) — proves the call site passes
settings.browser_http_endpoint, matching legacy's internal settings read."""
from app.services import house_imv_backfill
captured: dict[str, Any] = {}
class _FakeBrowserFetcher:
def __init__(self, **kwargs: Any) -> None:
captured.update(kwargs)
async def __aenter__(self) -> _FakeBrowserFetcher:
return self
async def __aexit__(self, *_: object) -> None:
return None
with (
patch.object(house_imv_backfill.settings, "avito_imv_use_browser_fetcher", True),
patch.object(house_imv_backfill, "BrowserFetcher", _FakeBrowserFetcher),
patch.object(house_imv_backfill, "_process_one_house", AsyncMock(return_value="ok")),
):
db = MagicMock()
db.execute.return_value.mappings.return_value.all.return_value = [
{"id": 1, "address": "ЕКБ, ул. X, 1", "full_address": None, "lat": 56.8, "lon": 60.6}
]
await house_imv_backfill.backfill_house_imv(db, batch_size=1)
assert captured == {"source": "avito", "endpoint": settings.browser_http_endpoint}
async def test_cian_history_backfill_browser_fetcher_uses_settings_endpoint() -> None:
"""cian_history_backfill.py's BrowserFetcher(source="cian") also needs endpoint=."""
from app.tasks import cian_history_backfill
captured: dict[str, Any] = {}
class _FakeBrowserFetcher:
def __init__(self, **kwargs: Any) -> None:
captured.update(kwargs)
async def __aenter__(self) -> _FakeBrowserFetcher:
return self
async def __aexit__(self, *_: object) -> None:
return None
db = MagicMock()
db.execute.return_value.mappings.return_value.all.return_value = []
with patch.object(cian_history_backfill, "BrowserFetcher", _FakeBrowserFetcher):
await cian_history_backfill.backfill_cian_history(
db, do_listings=True, do_houses=False, do_valuations=False
)
assert captured == {"source": "cian", "endpoint": settings.browser_http_endpoint}
# ── ekb_geoportal_ingest.py: task module resolves the kit client import ──────────
def test_ekb_geoportal_ingest_imports_kit_client() -> None:
"""app/tasks/ekb_geoportal_ingest.py now imports EkbGeoportalClient/TopoBuilding
from scraper_kit.providers.ekb_geoportal.client (issue #2310) — proves the task
module resolves to the SAME classes as the kit package (not a stale/duplicate
import), and that construction still works with no arguments."""
from scraper_kit.providers.ekb_geoportal.client import (
EkbGeoportalClient as KitClient,
)
from app.tasks.ekb_geoportal_ingest import EkbGeoportalClient as TaskClient
assert TaskClient is KitClient
assert isinstance(TaskClient(), KitClient)