a9b87aa311
All checks were successful
CI Trade-In / changes (pull_request) Successful in 8s
CI / changes (pull_request) Successful in 8s
CI Trade-In / frontend-checks (pull_request) Has been skipped
CI / backend-tests (pull_request) Has been skipped
CI / frontend-tests (pull_request) Has been skipped
CI / openapi-codegen-check (pull_request) Has been skipped
CI Trade-In / backend-tests (pull_request) Successful in 1m36s
Удалены (0 runtime importers в app/, scripts/, packages/; проверено grep'ом, включая intra-legacy импорты в оставшихся 21 legacy-модуле scrapers/): - backend/app/services/scrapers/avito_imv.py - backend/app/services/scrapers/domclick_detail.py - backend/app/services/scrapers/ekb_geoportal_client.py - backend/app/services/scrapers/yandex_detail.py - backend/app/services/scrapers/yandex_valuation.py Тесты — удалены (pure legacy-vs-kit parity, kit-only coverage уже есть в другом месте): - tests/scrapers/test_ekb_geoportal_client_kit_parity.py - tests/scrapers/test_domclick_detail_kit_parity.py Тесты — конвертированы в kit-only (импорт легаси заменён на scraper_kit.*, покрытие сохранено без потерь): - tests/scrapers/test_avito_imv_kit_parity.py (parity-тесты убраны, config-footgun regression на kit-стороне оставлен) - tests/scrapers/test_avito_imv_browser_transport.py - tests/scrapers/test_yandex_valuation_kit_migration.py (parity убрана, footgun- регрессия mandatory config/delay_provider оставлена) - tests/scrapers/test_domclick_detail.py (+ exceptions переведены на scraper_kit.domclick_exceptions, т.к. kit parse_detail_html поднимает их) - tests/test_avito_imv_parse.py - tests/test_ekb_geoportal_ingest.py (client-часть; ingest/geocoder-часть не трогали — не зависят от удалённого модуля) - tests/test_yandex_detail.py - tests/test_yandex_detail_structural.py - tests/test_yandex_valuation.py (YandexValuationScraper() → config=_KIT_CONFIG, kit конструктор требует обязательный config) - tests/test_yandex_valuation_save.py - tests/test_extval_house_id_write_path.py (только yandex_valuation-часть; cian_valuation остался нетронутым — живой легаси-модуль) - tests/test_yandex_history_area_filter.py Тесты — частично отредактированы (убраны только части про удалённые модули, живые легаси-модули/их parity не тронуты): - tests/scrapers/test_admin_domclick_ingest_kit_parity.py (base.py/ScrapedLot parity остался) - tests/scrapers/test_admin_yandex_kit_parity.py (yandex_realty parity остался) - tests/scrapers/test_admin_avito_kit_parity.py (avito.py/avito_houses.py parity остался; _parse_price parity убран) - tests/scrapers/test_avito_unix_date_tz_consistency.py (IMV-ветка переведена на kit avito.imv, легаси avito/avito_houses/avito_shared/avito_detail не тронуты) - tests/tasks/test_yandex_detail_backfill.py (subject — kit-задача, уже вызывает scraper_kit.providers.yandex.detail; save_detail_enrichment coverage-тесты внизу файла переведены на kit) - tests/test_scraper_kit_yandex_golden_parity.py (detail/valuation parity убраны; serp/newbuilding/house_type_normalizer/build_url для yandex_realty/ yandex_newbuilding не тронуты) - tests/test_scraper_proxy.py (YandexValuation proxy-тесты переведены на kit config= DI; test_avito_imv_own_session_receives_proxies удалён — дублирует test_avito_imv_kit_parity.py footgun-тесты) - tests/test_yandex_scrapers_delay_wiring.py (yandex_detail/yandex_valuation → kit delay_provider=; yandex_realty/yandex_newbuilding не тронуты) - tests/test_scraper_kit_group_c_backfill_kit_parity.py (docstring уточнение, без функциональных изменений) Полный backend-suite зелёный (3253 passed, 6 skipped) кроме известного pre-existing flake test_search_api.py::test_search_cache_hit (#2208, не регрессия этого PR). ruff check + ruff format — чисто на всех изменённых файлах.
446 lines
15 KiB
Python
446 lines
15 KiB
Python
"""Tests for app.tasks.yandex_detail_backfill (the KIT task — already imports
|
|
`scraper_kit.providers.yandex.detail.{YandexDetailScraper,save_detail_enrichment}`).
|
|
|
|
Fetch mechanism changed: curl_cffi AsyncSession(chrome120+proxy) + YandexDetailScraper.parse
|
|
(instead of old fetch_detail path). Mocks target session.get and scraper.parse.
|
|
|
|
Легаси `app.services.scrapers.yandex_detail` удалён (#2277 финальный шаг
|
|
scraper_kit-миграции) — `save_detail_enrichment` coverage-тесты внизу файла
|
|
переведены на kit-эквивалент (тот же, что реально вызывает эта задача).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import sys
|
|
from unittest.mock import AsyncMock, MagicMock, patch
|
|
|
|
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
|
|
|
|
_wp_mock = MagicMock()
|
|
sys.modules.setdefault("weasyprint", _wp_mock)
|
|
|
|
import pytest # noqa: E402
|
|
|
|
from app.tasks.yandex_detail_backfill import ( # noqa: E402
|
|
YandexDetailBackfillResult,
|
|
run_yandex_detail_backfill,
|
|
)
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Path constants for patching
|
|
# ---------------------------------------------------------------------------
|
|
|
|
_ASYNC_SESSION = "app.tasks.yandex_detail_backfill.AsyncSession"
|
|
_PARSE = "app.tasks.yandex_detail_backfill.YandexDetailScraper.parse"
|
|
_SAVE = "app.tasks.yandex_detail_backfill.save_detail_enrichment"
|
|
_RUNS = "app.tasks.yandex_detail_backfill.runs_mod"
|
|
_SLEEP = "app.tasks.yandex_detail_backfill.asyncio.sleep"
|
|
_SETTINGS = "app.tasks.yandex_detail_backfill.settings"
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _make_snapshot(n: int) -> list[dict]:
|
|
return [
|
|
{"id": i + 1, "source_url": f"https://realty.yandex.ru/offer/{i + 1}/"} for i in range(n)
|
|
]
|
|
|
|
|
|
def _mock_db(snapshot: list[dict]) -> MagicMock:
|
|
"""Fake Session: first execute() returns snapshot via .mappings().all()."""
|
|
db = MagicMock()
|
|
sel = MagicMock()
|
|
sel.mappings.return_value.all.return_value = snapshot
|
|
db.execute.return_value = sel
|
|
return db
|
|
|
|
|
|
def _make_resp(status: int = 200, text: str = "<html>ok</html>") -> MagicMock:
|
|
"""Fake curl_cffi response."""
|
|
resp = MagicMock()
|
|
resp.status_code = status
|
|
resp.text = text
|
|
return resp
|
|
|
|
|
|
def _make_session_ctx(get_side_effect) -> MagicMock:
|
|
"""Build AsyncSession context-manager mock with session.get side_effect."""
|
|
session = AsyncMock()
|
|
session.get = AsyncMock(side_effect=get_side_effect)
|
|
|
|
ctx = MagicMock()
|
|
ctx.__aenter__ = AsyncMock(return_value=session)
|
|
ctx.__aexit__ = AsyncMock(return_value=None)
|
|
|
|
session_cls = MagicMock(return_value=ctx)
|
|
return session_cls, session
|
|
|
|
|
|
def _mock_settings(proxy: str | None = "http://proxy:3128") -> MagicMock:
|
|
s = MagicMock()
|
|
s.scraper_proxy_url = proxy
|
|
return s
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Tests: run_yandex_detail_backfill
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_backfill_empty_snapshot_marks_done() -> None:
|
|
"""Empty snapshot -> mark_done immediately, no session.get calls."""
|
|
db = _mock_db([])
|
|
runs = MagicMock()
|
|
session_cls, session = _make_session_ctx([])
|
|
|
|
with (
|
|
patch(_ASYNC_SESSION, session_cls),
|
|
patch(_RUNS, runs),
|
|
patch(_SETTINGS, _mock_settings()),
|
|
):
|
|
result = await run_yandex_detail_backfill(
|
|
db, run_id=1, params={"batch_size": 10, "budget_sec": 60}
|
|
)
|
|
|
|
assert isinstance(result, YandexDetailBackfillResult)
|
|
assert result.attempted == 0
|
|
assert result.enriched == 0
|
|
session.get.assert_not_called()
|
|
runs.mark_done.assert_called_once()
|
|
runs.mark_failed.assert_not_called()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_backfill_processes_snapshot_to_completion() -> None:
|
|
"""3 listings -> all fetched (200) + parse success -> enriched=3, mark_done."""
|
|
snapshot = _make_snapshot(3)
|
|
db = _mock_db(snapshot)
|
|
runs = MagicMock()
|
|
mock_enrichment = MagicMock()
|
|
|
|
responses = [_make_resp(200), _make_resp(200), _make_resp(200)]
|
|
session_cls, session = _make_session_ctx(responses)
|
|
|
|
with (
|
|
patch(_ASYNC_SESSION, session_cls),
|
|
patch(_PARSE, return_value=mock_enrichment),
|
|
patch(_RUNS, runs),
|
|
patch(_SAVE, return_value=True),
|
|
patch(_SLEEP, new_callable=AsyncMock),
|
|
patch(_SETTINGS, _mock_settings()),
|
|
):
|
|
result = await run_yandex_detail_backfill(
|
|
db, run_id=2, params={"batch_size": 10, "budget_sec": 3600}
|
|
)
|
|
|
|
assert result.attempted == 3
|
|
assert result.enriched == 3
|
|
assert result.failed == 0
|
|
assert session.get.call_count == 3
|
|
runs.mark_done.assert_called_once()
|
|
runs.mark_failed.assert_not_called()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_backfill_parse_none_abort_after_max_consecutive() -> None:
|
|
"""5 consecutive parse→None results -> abort, mark_done (NOT mark_failed)."""
|
|
snapshot = _make_snapshot(10)
|
|
db = _mock_db(snapshot)
|
|
runs = MagicMock()
|
|
|
|
responses = [_make_resp(200)] * 10
|
|
session_cls, _session = _make_session_ctx(responses)
|
|
|
|
with (
|
|
patch(_ASYNC_SESSION, session_cls),
|
|
patch(_PARSE, return_value=None),
|
|
patch(_RUNS, runs),
|
|
patch(_SLEEP, new_callable=AsyncMock),
|
|
patch(_SETTINGS, _mock_settings()),
|
|
):
|
|
result = await run_yandex_detail_backfill(
|
|
db,
|
|
run_id=3,
|
|
params={"batch_size": 10, "budget_sec": 3600, "max_consecutive_blocks": 5},
|
|
)
|
|
|
|
assert result.attempted == 5
|
|
assert result.failed == 5
|
|
assert result.enriched == 0
|
|
runs.mark_done.assert_called_once()
|
|
runs.mark_failed.assert_not_called()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_backfill_parse_none_resets_on_success() -> None:
|
|
"""1 parse→None + 1 success + 1 parse→None -> consecutive resets, no abort."""
|
|
snapshot = _make_snapshot(3)
|
|
db = _mock_db(snapshot)
|
|
runs = MagicMock()
|
|
mock_enrichment = MagicMock()
|
|
|
|
responses = [_make_resp(200)] * 3
|
|
session_cls, _session = _make_session_ctx(responses)
|
|
parse_results = [None, mock_enrichment, None]
|
|
|
|
with (
|
|
patch(_ASYNC_SESSION, session_cls),
|
|
patch(_PARSE, side_effect=parse_results),
|
|
patch(_RUNS, runs),
|
|
patch(_SAVE, return_value=True),
|
|
patch(_SLEEP, new_callable=AsyncMock),
|
|
patch(_SETTINGS, _mock_settings()),
|
|
):
|
|
result = await run_yandex_detail_backfill(
|
|
db,
|
|
run_id=4,
|
|
params={"batch_size": 10, "budget_sec": 3600, "max_consecutive_blocks": 5},
|
|
)
|
|
|
|
assert result.attempted == 3
|
|
assert result.enriched == 1
|
|
assert result.failed == 2
|
|
runs.mark_done.assert_called_once()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_backfill_non200_counts_as_fail_and_aborts() -> None:
|
|
"""5 consecutive HTTP 403 responses -> abort after max_consecutive_blocks."""
|
|
snapshot = _make_snapshot(10)
|
|
db = _mock_db(snapshot)
|
|
runs = MagicMock()
|
|
|
|
responses = [_make_resp(403)] * 10
|
|
session_cls, _session = _make_session_ctx(responses)
|
|
|
|
with (
|
|
patch(_ASYNC_SESSION, session_cls),
|
|
patch(_PARSE, return_value=MagicMock()),
|
|
patch(_RUNS, runs),
|
|
patch(_SLEEP, new_callable=AsyncMock),
|
|
patch(_SETTINGS, _mock_settings()),
|
|
):
|
|
result = await run_yandex_detail_backfill(
|
|
db,
|
|
run_id=5,
|
|
params={"batch_size": 10, "budget_sec": 3600, "max_consecutive_blocks": 5},
|
|
)
|
|
|
|
assert result.failed == 5
|
|
assert result.enriched == 0
|
|
runs.mark_done.assert_called_once()
|
|
runs.mark_failed.assert_not_called()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_backfill_budget_guard_stops_loop() -> None:
|
|
"""Budget expired before first listing -> session.get not called."""
|
|
snapshot = _make_snapshot(5)
|
|
db = _mock_db(snapshot)
|
|
runs = MagicMock()
|
|
|
|
responses = [_make_resp(200)] * 5
|
|
session_cls, session = _make_session_ctx(responses)
|
|
|
|
mono_values = iter([0.0, 999.0, 999.0])
|
|
with (
|
|
patch(_ASYNC_SESSION, session_cls),
|
|
patch(_RUNS, runs),
|
|
patch("app.tasks.yandex_detail_backfill.time.monotonic", side_effect=mono_values),
|
|
patch(_SETTINGS, _mock_settings()),
|
|
):
|
|
await run_yandex_detail_backfill(db, run_id=6, params={"batch_size": 5, "budget_sec": 1})
|
|
|
|
session.get.assert_not_called()
|
|
runs.mark_done.assert_called_once()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_backfill_top_level_exception_marks_failed() -> None:
|
|
"""db.execute raises -> mark_failed called, exception re-raised."""
|
|
db = MagicMock()
|
|
db.execute.side_effect = RuntimeError("DB connection lost")
|
|
runs = MagicMock()
|
|
|
|
with (
|
|
patch(_RUNS, runs),
|
|
patch(_SETTINGS, _mock_settings()),
|
|
):
|
|
with pytest.raises(RuntimeError, match="DB connection lost"):
|
|
await run_yandex_detail_backfill(
|
|
db, run_id=7, params={"batch_size": 5, "budget_sec": 60}
|
|
)
|
|
|
|
runs.mark_failed.assert_called_once()
|
|
runs.mark_done.assert_not_called()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_backfill_fetch_exception_continues() -> None:
|
|
"""session.get raises on first listing -> failed++, loop continues for second."""
|
|
snapshot = _make_snapshot(2)
|
|
db = _mock_db(snapshot)
|
|
runs = MagicMock()
|
|
mock_enrichment = MagicMock()
|
|
|
|
# First call raises, second returns 200 OK
|
|
responses = [RuntimeError("connection reset"), _make_resp(200)]
|
|
session_cls, _session = _make_session_ctx(responses)
|
|
|
|
with (
|
|
patch(_ASYNC_SESSION, session_cls),
|
|
patch(_PARSE, return_value=mock_enrichment),
|
|
patch(_RUNS, runs),
|
|
patch(_SAVE, return_value=True),
|
|
patch(_SLEEP, new_callable=AsyncMock),
|
|
patch(_SETTINGS, _mock_settings()),
|
|
):
|
|
result = await run_yandex_detail_backfill(
|
|
db, run_id=8, params={"batch_size": 10, "budget_sec": 3600}
|
|
)
|
|
|
|
assert result.failed == 1
|
|
assert result.enriched == 1
|
|
assert result.attempted == 2
|
|
# fetch exception is caught by inner try/except (not DB) — no rollback needed
|
|
db.rollback.assert_not_called()
|
|
runs.mark_done.assert_called_once()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_backfill_no_proxy_when_settings_none() -> None:
|
|
"""scraper_proxy_url=None -> AsyncSession called with proxies=None."""
|
|
snapshot = _make_snapshot(1)
|
|
db = _mock_db(snapshot)
|
|
runs = MagicMock()
|
|
mock_enrichment = MagicMock()
|
|
|
|
responses = [_make_resp(200)]
|
|
session_cls, _session = _make_session_ctx(responses)
|
|
|
|
with (
|
|
patch(_ASYNC_SESSION, session_cls),
|
|
patch(_PARSE, return_value=mock_enrichment),
|
|
patch(_RUNS, runs),
|
|
patch(_SAVE, return_value=True),
|
|
patch(_SLEEP, new_callable=AsyncMock),
|
|
patch(_SETTINGS, _mock_settings(proxy=None)),
|
|
):
|
|
result = await run_yandex_detail_backfill(
|
|
db, run_id=9, params={"batch_size": 10, "budget_sec": 3600}
|
|
)
|
|
|
|
assert result.enriched == 1
|
|
# Verify AsyncSession was constructed with proxies=None
|
|
call_kwargs = session_cls.call_args[1]
|
|
assert call_kwargs.get("proxies") is None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Tests: save_detail_enrichment (kit — same function the task above actually calls;
|
|
# легаси `app.services.scrapers.yandex_detail` удалён #2277 финальный шаг миграции)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_save_detail_enrichment_maps_fields_to_update() -> None:
|
|
"""save_detail_enrichment calls db.execute with UPDATE and commits."""
|
|
from scraper_kit.providers.yandex.detail import (
|
|
DetailEnrichment,
|
|
MetroStation,
|
|
save_detail_enrichment,
|
|
)
|
|
|
|
enrichment = DetailEnrichment(
|
|
offer_id="12345",
|
|
source_url="https://realty.yandex.ru/offer/12345/",
|
|
rooms=2,
|
|
area_m2=54.5,
|
|
floor=3,
|
|
total_floors=9,
|
|
address="Екатеринбург, ул. Ленина, 10",
|
|
description="Хорошая квартира",
|
|
repair_state="standard",
|
|
publish_date=None,
|
|
views_total=42,
|
|
publish_date_relative="вчера",
|
|
agency_name="Агентство «Тест»",
|
|
agency_founded_year=2005,
|
|
agency_objects_count=150,
|
|
metro_stations=[MetroStation(name="Чкаловская", walk_min=11)],
|
|
photo_urls=["https://example.com/1.jpg", "https://example.com/2.jpg"],
|
|
newbuilding_url="https://realty.yandex.ru/kupit/novostrojka/test-12345/",
|
|
newbuilding_id="12345",
|
|
)
|
|
|
|
db = MagicMock()
|
|
result_mock = MagicMock()
|
|
result_mock.rowcount = 1
|
|
db.execute.return_value = result_mock
|
|
|
|
saved = save_detail_enrichment(db, listing_id=99, e=enrichment)
|
|
|
|
assert saved is True
|
|
db.execute.assert_called_once()
|
|
db.commit.assert_called_once()
|
|
|
|
call_args = db.execute.call_args
|
|
params = call_args[0][1]
|
|
assert params["rooms"] == 2
|
|
assert params["area_m2"] == 54.5
|
|
assert params["agency_name"] == "Агентство «Тест»"
|
|
|
|
metro_val = params["metro_stations"]
|
|
assert metro_val is not None
|
|
parsed_metro = json.loads(metro_val)
|
|
assert parsed_metro[0]["name"] == "Чкаловская"
|
|
assert parsed_metro[0]["walk_min"] == 11
|
|
|
|
photo_val = params["photo_urls"]
|
|
assert photo_val is not None
|
|
parsed_photos = json.loads(photo_val)
|
|
assert len(parsed_photos) == 2
|
|
assert parsed_photos[0] == "https://example.com/1.jpg"
|
|
|
|
|
|
def test_save_detail_enrichment_empty_metro_and_photos() -> None:
|
|
"""Empty metro_stations and photo_urls -> NULL passed for both jsonb columns."""
|
|
from scraper_kit.providers.yandex.detail import DetailEnrichment, save_detail_enrichment
|
|
|
|
enrichment = DetailEnrichment(
|
|
offer_id="99",
|
|
source_url="https://realty.yandex.ru/offer/99/",
|
|
)
|
|
|
|
db = MagicMock()
|
|
result_mock = MagicMock()
|
|
result_mock.rowcount = 1
|
|
db.execute.return_value = result_mock
|
|
|
|
save_detail_enrichment(db, listing_id=1, e=enrichment)
|
|
|
|
params = db.execute.call_args[0][1]
|
|
assert params["metro_stations"] is None
|
|
assert params["photo_urls"] is None
|
|
|
|
|
|
def test_save_detail_enrichment_rowcount_zero_returns_false() -> None:
|
|
"""rowcount=0 (listing_id not found) -> returns False."""
|
|
from scraper_kit.providers.yandex.detail import DetailEnrichment, save_detail_enrichment
|
|
|
|
enrichment = DetailEnrichment(
|
|
offer_id="404",
|
|
source_url="https://realty.yandex.ru/offer/404/",
|
|
)
|
|
|
|
db = MagicMock()
|
|
result_mock = MagicMock()
|
|
result_mock.rowcount = 0
|
|
db.execute.return_value = result_mock
|
|
|
|
saved = save_detail_enrichment(db, listing_id=404, e=enrichment)
|
|
assert saved is False
|