27bbd579c3
All checks were successful
CI Trade-In / changes (pull_request) Successful in 9s
CI / changes (pull_request) Successful in 9s
CI Trade-In / frontend-checks (pull_request) Has been skipped
CI / backend-tests (pull_request) Has been skipped
CI / frontend-tests (pull_request) Has been skipped
CI / openapi-codegen-check (pull_request) Has been skipped
CI Trade-In / backend-tests (pull_request) Successful in 47s
Удаляет весь `app/services/scrapers/` (16 файлов, ~7100 строк) — Part D (D1-D5) убрал всех внешних вызывающих, 0 runtime importers подтверждено grep'ом на main. Заодно: - удалены 7 осиротевших локальных probe/sweep-скриптов (tradein-mvp/scripts/), импортировавших уже-удалённые или удаляемые сейчас legacy-модули - тест-хирургия по 65+ файлам: DELETE прямых legacy-юнит-тестов, RETARGET тестов, тестирующих ещё живую бизнес-логику (переключены на scraper_kit.* эквиваленты, включая quality-gate #781/#753/#754/#755/#773/#740), partial-delete golden-parity тестов, потерявших legacy-оракл - kit save_listings/AvitoScraper/etc. требуют инжектируемые matcher/config — ретаргетированные тесты обновлены под новую сигнатуру (RealScraperConfig(), MagicMock HouseMatcher, region_code=66) Полный pytest suite зелёный (2255 passed, 6 skipped) кроме известного флейка #2208 (test_search_cache_hit, не связан со scrapers).
168 lines
6.4 KiB
Python
168 lines
6.4 KiB
Python
"""Avito newbuilding (novostroyka) citywide sweep.
|
||
|
||
Тесты (без сети, без реального DB):
|
||
1. _build_newbuilding_url — novostroyka-slug в пути, s=104, page param, без geo.
|
||
2. _parse_html на novostroyka-SERP fixture → listing_segment='novostroyki' +
|
||
newbuilding_id заполнен (DOM-маркер застройщика + zhk-якорь).
|
||
3. fetch_newbuildings break-on-empty + dedup (mirror fetch_city_wide).
|
||
4. NewbuildingSweepCounters — defaults + to_dict.
|
||
|
||
Legacy `run_avito_newbuilding_sweep` orchestration wiring (fetch_newbuildings +
|
||
save_listings + mark_done) удалён вместе с `app.services.scrape_pipeline` (#2397
|
||
Part E1) — эквивалентная regression-coverage теперь в
|
||
`test_scraper_kit_pipeline_parity2.py::test_avito_newbuilding_sweep` (kit
|
||
`run_avito_newbuilding_sweep`).
|
||
|
||
Легаси `app.services.scrapers.avito`/`base` удалены (#2397 финальный шаг E, 0
|
||
runtime-импортёров) — AvitoScraper/ScrapedLot переведены на kit-эквиваленты
|
||
`scraper_kit.providers.avito.serp`/`scraper_kit.base` (byte-identical
|
||
strangler-копия).
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import os
|
||
from pathlib import Path
|
||
from unittest.mock import AsyncMock, patch
|
||
|
||
import pytest
|
||
|
||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
|
||
|
||
from scraper_kit.base import ScrapedLot
|
||
from scraper_kit.providers.avito.serp import NOVOSTROYKA_SLUG, AvitoScraper
|
||
|
||
from app.services.scraper_adapters import RealScraperConfig
|
||
|
||
FIXTURE = Path(__file__).parent / "fixtures" / "avito_serp_novostroyka.html"
|
||
|
||
|
||
def _make_lot(source_id: str) -> ScrapedLot:
|
||
return ScrapedLot(
|
||
source="avito",
|
||
source_url=f"https://www.avito.ru/ekaterinburg/kvartiry/{source_id}",
|
||
source_id=source_id,
|
||
price_rub=6_000_000,
|
||
listing_segment="novostroyki",
|
||
)
|
||
|
||
|
||
# ── 1. _build_newbuilding_url ───────────────────────────────────────────────
|
||
|
||
|
||
def test_build_newbuilding_url_contains_slug_and_sort() -> None:
|
||
s = AvitoScraper(RealScraperConfig())
|
||
url = s._build_newbuilding_url(page=1)
|
||
assert NOVOSTROYKA_SLUG in url
|
||
assert "novostroyka-" in url
|
||
assert "geoCoords" not in url
|
||
assert "radius" not in url
|
||
assert "s=104" in url
|
||
assert "p=1" in url
|
||
assert "ekaterinburg/kvartiry/prodam/" in url
|
||
|
||
|
||
def test_build_newbuilding_url_page_param() -> None:
|
||
s = AvitoScraper(RealScraperConfig())
|
||
url = s._build_newbuilding_url(page=2)
|
||
assert "novostroyka-" in url
|
||
assert "p=2" in url
|
||
|
||
|
||
def test_build_newbuilding_url_differs_from_citywide() -> None:
|
||
s = AvitoScraper(RealScraperConfig())
|
||
assert s._build_newbuilding_url(page=1) != s._build_citywide_url(page=1)
|
||
|
||
|
||
# ── 2. _parse_html classifies novostroyka cards ─────────────────────────────
|
||
|
||
|
||
def test_parse_novostroyka_fixture_segments_and_newbuilding_id() -> None:
|
||
html = FIXTURE.read_text(encoding="utf-8")
|
||
s = AvitoScraper(RealScraperConfig())
|
||
lots = s._parse_html(html, source_url_base=s._build_newbuilding_url(page=1))
|
||
|
||
assert len(lots) == 2
|
||
for lot in lots:
|
||
assert lot.listing_segment == "novostroyki"
|
||
assert lot.newbuilding_id is not None
|
||
assert lot.newbuilding_url is not None
|
||
|
||
ids = {lot.newbuilding_id for lot in lots}
|
||
assert "meridian-ekaterinburg" in ids
|
||
assert "severnyy-kvartal-ekaterinburg" in ids
|
||
|
||
|
||
# ── 3. fetch_newbuildings break-on-empty + dedup ────────────────────────────
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_newbuildings_break_on_empty() -> None:
|
||
s = AvitoScraper(RealScraperConfig())
|
||
pages_data: list[list[ScrapedLot]] = [[_make_lot("A"), _make_lot("B")], []]
|
||
call_n = 0
|
||
|
||
async def mock_html(url: str, page: int) -> str:
|
||
assert "novostroyka-" in url
|
||
return f"<html>{page}</html>"
|
||
|
||
def mock_parse(html: str, source_url_base: str) -> list[ScrapedLot]:
|
||
nonlocal call_n
|
||
idx = call_n
|
||
call_n += 1
|
||
return pages_data[idx] if idx < len(pages_data) else []
|
||
|
||
with patch.object(s, "_fetch_serp_html", side_effect=mock_html):
|
||
with patch.object(s, "_parse_html", side_effect=mock_parse):
|
||
with patch.object(s, "sleep_between_requests", new=AsyncMock()):
|
||
result = await s.fetch_newbuildings(pages=10, delay_override_sec=0)
|
||
|
||
assert len(result) == 2
|
||
assert call_n == 2 # page1 + page2(empty) → stop
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_newbuildings_dedup_by_source_id() -> None:
|
||
s = AvitoScraper(RealScraperConfig())
|
||
dup = _make_lot("SAME")
|
||
pages_data: list[list[ScrapedLot]] = [[dup], [dup], []]
|
||
call_n = 0
|
||
|
||
async def mock_html(url: str, page: int) -> str:
|
||
return f"<html>{page}</html>"
|
||
|
||
def mock_parse(html: str, source_url_base: str) -> list[ScrapedLot]:
|
||
nonlocal call_n
|
||
idx = call_n
|
||
call_n += 1
|
||
return pages_data[idx] if idx < len(pages_data) else []
|
||
|
||
with patch.object(s, "_fetch_serp_html", side_effect=mock_html):
|
||
with patch.object(s, "_parse_html", side_effect=mock_parse):
|
||
with patch.object(s, "sleep_between_requests", new=AsyncMock()):
|
||
result = await s.fetch_newbuildings(pages=10, delay_override_sec=0)
|
||
|
||
assert len(result) == 1
|
||
assert result[0].source_id == "SAME"
|
||
|
||
|
||
def test_fetch_city_wide_url_unchanged() -> None:
|
||
"""fetch_city_wide остаётся citywide-URL — рефактор не сменил его поведение."""
|
||
s = AvitoScraper(RealScraperConfig())
|
||
assert "novostroyka-" not in s._build_citywide_url(page=1)
|
||
assert "prodam-ASgBAgICAUSSA8YQ" in s._build_citywide_url(page=1)
|
||
|
||
|
||
# ── 4. NewbuildingSweepCounters ─────────────────────────────────────────────
|
||
|
||
|
||
def test_newbuilding_counters_defaults_and_to_dict() -> None:
|
||
from scraper_kit.orchestration.pipeline import NewbuildingSweepCounters
|
||
|
||
c = NewbuildingSweepCounters()
|
||
assert c.lots_fetched == 0
|
||
assert c.lots_inserted == 0
|
||
assert c.lots_updated == 0
|
||
assert c.errors_count == 0
|
||
d = c.to_dict()
|
||
assert set(d.keys()) == {"lots_fetched", "lots_inserted", "lots_updated", "errors_count"}
|