lekss361
5c17d21f6c
All checks were successful
Deploy Trade-In / changes (push) Successful in 11s
Deploy Trade-In / build-frontend (push) Has been skipped
Deploy Trade-In / build-browser (push) Has been skipped
Deploy Trade-In / test (push) Successful in 1m27s
Deploy Trade-In / build-backend (push) Successful in 1m22s
Deploy Trade-In / deploy (push) Successful in 1m8s
363 lines
16 KiB
Python
363 lines
16 KiB
Python
"""Golden-parity: `scraper_kit.base.save_listings` ≡ `app.services.scrapers.base.save_listings`.
|
||
|
||
Strangler-инвариант (#2132): новая scraper_kit-копия базового слоя должна давать
|
||
БАЙТ-ИДЕНТИЧНЫЙ эффект старому боевому коду на одинаковом входе. Единственное
|
||
намеренное расхождение — развязка `region_code`: старый хардкодит литерал `66` в
|
||
INSERT SQL, новый получает его параметром `:region_code` (kit не знает про регион).
|
||
При region_code=66 итоговый INSERT идентичен — что этот тест и доказывает,
|
||
нормализуя `:region_code` → `66` на новой стороне перед сравнением.
|
||
|
||
Метод: гоняем ОБА модуля на одном списке `ScrapedLot`, перехватывая через
|
||
recording-mock всю последовательность `db.execute(sql, params)` + аргументы hook'а
|
||
матчинга. Сравниваем:
|
||
- последовательность (SQL, params) вызовов execute (INSERT / SELECT / reconcile
|
||
UPDATE / house_id_fk UPDATE) — после нормализации region_code;
|
||
- kwargs каждого вызова matcher.match_or_create_house / upsert_listing_source;
|
||
- compute_dedup_hash / compute_card_hash / compute_price_per_m2 старого и нового
|
||
класса ScrapedLot на идентичном входе.
|
||
|
||
Фикстуры покрывают: avito (без адреса), cian (адрес+coords → house resolve → FK
|
||
UPDATE), студию rooms=0, yandex без source_id (ext_id := dedup_hash fallback) и
|
||
dedup-коллизию (INSERT → UniqueViolation → reconcile UPDATE by source,source_id).
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import contextlib
|
||
from typing import Any
|
||
from unittest.mock import MagicMock, patch
|
||
|
||
import psycopg.errors
|
||
from scraper_kit.base import ScrapedLot as NewLot
|
||
from scraper_kit.base import save_listings as new_save_listings
|
||
from sqlalchemy.exc import IntegrityError
|
||
|
||
# СТАРЫЙ (боевой) и НОВЫЙ (scraper_kit) модули — импортируем оба в одном тесте.
|
||
from app.services.scrapers.base import ScrapedLot as OldLot
|
||
from app.services.scrapers.base import save_listings as old_save_listings
|
||
|
||
_REGION_CODE = 66
|
||
|
||
|
||
# ── recording session ────────────────────────────────────────────────────────
|
||
|
||
|
||
class _Row:
|
||
def __init__(self, **kw: Any) -> None:
|
||
self.__dict__.update(kw)
|
||
|
||
|
||
class _Result:
|
||
def __init__(self, row: _Row | None) -> None:
|
||
self._row = row
|
||
|
||
def fetchone(self) -> _Row | None:
|
||
return self._row
|
||
|
||
|
||
@contextlib.contextmanager
|
||
def _nested_cm() -> Any:
|
||
yield MagicMock()
|
||
|
||
|
||
class RecordingSession:
|
||
"""Мок Session: пишет (sql, params) всех execute, детерминированно отвечает.
|
||
|
||
- SELECT card_hash … WHERE dedup_hash → None (prior_row отсутствует, fresh)
|
||
- INSERT INTO listings … → row(id, inserted); если source_id в
|
||
drift_ids → raise IntegrityError(UniqueViolation) (dedup-коллизия)
|
||
- reconcile UPDATE listings SET dedup_hash … → row(id=reconcile_id)
|
||
- UPDATE listings … house_id_fk / snapshot EXISTS → None
|
||
"""
|
||
|
||
def __init__(
|
||
self,
|
||
*,
|
||
drift_ids: frozenset[str] = frozenset(),
|
||
update_ids: frozenset[str] = frozenset(),
|
||
insert_id: int = 42,
|
||
reconcile_id: int = 99,
|
||
) -> None:
|
||
self.calls: list[tuple[str, dict[str, Any]]] = []
|
||
self._drift = drift_ids
|
||
self._update = update_ids
|
||
self._insert_id = insert_id
|
||
self._reconcile_id = reconcile_id
|
||
|
||
def execute(self, sql: Any, params: dict[str, Any] | None = None) -> _Result:
|
||
s = str(sql)
|
||
p = dict(params) if params else {}
|
||
self.calls.append((s, p))
|
||
if "INSERT INTO listings (" in s:
|
||
sid = p.get("source_id")
|
||
if sid in self._drift:
|
||
raise IntegrityError("INSERT", p, psycopg.errors.UniqueViolation("dup"))
|
||
inserted = sid not in self._update
|
||
return _Result(_Row(id=self._insert_id, inserted=inserted))
|
||
if "UPDATE listings" in s and "SET dedup_hash" in s:
|
||
return _Result(_Row(id=self._reconcile_id))
|
||
return _Result(None)
|
||
|
||
def begin_nested(self) -> Any:
|
||
return _nested_cm()
|
||
|
||
def commit(self) -> None:
|
||
pass
|
||
|
||
|
||
# ── fixtures ─────────────────────────────────────────────────────────────────
|
||
|
||
|
||
def _fixture_data() -> list[dict[str, Any]]:
|
||
"""Реалистичные входные dict'ы (общие для old/new ScrapedLot)."""
|
||
return [
|
||
# 1. avito без адреса → house resolve пропускается (нет FK UPDATE)
|
||
{
|
||
"source": "avito",
|
||
"source_url": "https://www.avito.ru/ekaterinburg/kvartiry/1-k_987654?context=AB",
|
||
"source_id": "987654",
|
||
"price_rub": 4_500_000,
|
||
"area_m2": 38.0,
|
||
"rooms": 1,
|
||
"floor": 5,
|
||
"total_floors": 9,
|
||
},
|
||
# 2. cian full: адрес + coords → house resolve → house_id_fk UPDATE
|
||
{
|
||
"source": "cian",
|
||
"source_url": "https://ekb.cian.ru/sale/flat/123456/",
|
||
"source_id": "123456",
|
||
"price_rub": 6_200_000,
|
||
"address": "Екатеринбург, улица Постовского, 17А",
|
||
"lat": 56.8385,
|
||
"lon": 60.5940,
|
||
"rooms": 2,
|
||
"area_m2": 52.0,
|
||
"floor": 7,
|
||
"total_floors": 16,
|
||
"living_area_m2": 35.0,
|
||
"bedrooms_count": 2,
|
||
"cadastral_number": "66:41:0204016:1234",
|
||
"building_cadastral_number": "66:41:0204016:100",
|
||
"phones": [{"countryCode": "7", "number": "9012345678"}],
|
||
"is_homeowner": True,
|
||
"metro_stations": [{"name": "Геологическая", "time": 10}],
|
||
"house_source": "cian",
|
||
"listing_segment": "vtorichka",
|
||
},
|
||
# 3. студия rooms=0 (avito) с адресом → house resolve
|
||
{
|
||
"source": "avito",
|
||
"source_url": "https://www.avito.ru/ekaterinburg/kvartiry/studiya_555",
|
||
"source_id": "555",
|
||
"price_rub": 3_100_000,
|
||
"address": "Екатеринбург, Малышева 51",
|
||
"lat": 56.84,
|
||
"lon": 60.60,
|
||
"rooms": 0,
|
||
"area_m2": 24.5,
|
||
"floor": 12,
|
||
"total_floors": 25,
|
||
},
|
||
# 4. yandex без source_id → ext_id := dedup_hash fallback
|
||
{
|
||
"source": "yandex",
|
||
"source_url": "https://realty.yandex.ru/offer/5500000000000000001/?from=serp",
|
||
"source_id": None,
|
||
"price_rub": 7_200_000,
|
||
"address": "Екатеринбург, улица Мира, 10",
|
||
"lat": 56.83,
|
||
"lon": 60.65,
|
||
"rooms": 2,
|
||
"area_m2": 48.0,
|
||
"yandex_offer_id": "5500000000000000001",
|
||
"predicted_price_rub": 7_554_000,
|
||
"price_trend": "DECREASED",
|
||
"price_previous_rub": 7_500_000,
|
||
},
|
||
# 5. dedup-коллизия: INSERT → UniqueViolation → reconcile UPDATE
|
||
{
|
||
"source": "cian",
|
||
"source_url": "https://ekb.cian.ru/sale/flat/DRIFT-1/",
|
||
"source_id": "DRIFT-1",
|
||
"price_rub": 5_900_000,
|
||
"address": "Екатеринбург, Ленина 1",
|
||
"lat": 56.838,
|
||
"lon": 60.597,
|
||
"rooms": 3,
|
||
"area_m2": 72.0,
|
||
},
|
||
]
|
||
|
||
|
||
_DRIFT_IDS = frozenset({"DRIFT-1"})
|
||
|
||
|
||
# ── run helpers ──────────────────────────────────────────────────────────────
|
||
|
||
|
||
def _run_old(lots: list[Any]) -> tuple[RecordingSession, MagicMock, MagicMock]:
|
||
db = RecordingSession(drift_ids=_DRIFT_IDS)
|
||
with (
|
||
patch("app.services.scrapers.base.match_or_create_house") as m_house,
|
||
patch("app.services.scrapers.base.upsert_listing_source") as m_link,
|
||
patch("app.services.scrapers.base.upsert_listing_snapshot"),
|
||
):
|
||
m_house.return_value = (101, 1.0, "new")
|
||
m_link.return_value = None
|
||
old_save_listings(db, lots)
|
||
return db, m_house, m_link
|
||
|
||
|
||
def _run_new(lots: list[Any]) -> tuple[RecordingSession, MagicMock]:
|
||
db = RecordingSession(drift_ids=_DRIFT_IDS)
|
||
matcher = MagicMock()
|
||
matcher.match_or_create_house.return_value = (101, 1.0, "new")
|
||
matcher.upsert_listing_source.return_value = None
|
||
with patch("scraper_kit.base.upsert_listing_snapshot"):
|
||
new_save_listings(db, lots, matcher=matcher, region_code=_REGION_CODE)
|
||
return db, matcher
|
||
|
||
|
||
def _normalize_new_calls(
|
||
calls: list[tuple[str, dict[str, Any]]],
|
||
) -> list[tuple[str, dict[str, Any]]]:
|
||
"""Нормализуем расхождение region_code: `:region_code` → `66`, убираем ключ.
|
||
|
||
Это единственное намеренное отличие новой копии; после нормализации при
|
||
region_code=66 SQL и params должны быть байт-идентичны старому боевому коду.
|
||
"""
|
||
out: list[tuple[str, dict[str, Any]]] = []
|
||
for sql, params in calls:
|
||
norm_sql = sql.replace(":region_code", str(_REGION_CODE))
|
||
norm_params = {k: v for k, v in params.items() if k != "region_code"}
|
||
out.append((norm_sql, norm_params))
|
||
return out
|
||
|
||
|
||
def _kwargs_list(mock_or_method: Any) -> list[dict[str, Any]]:
|
||
return [c.kwargs for c in mock_or_method.call_args_list]
|
||
|
||
|
||
# ── tests ────────────────────────────────────────────────────────────────────
|
||
|
||
|
||
def test_save_listings_execute_sequence_parity() -> None:
|
||
"""Последовательность (SQL, params) execute идентична (после region_code-нормализации)."""
|
||
lots = [NewLot(**d) for d in _fixture_data()]
|
||
|
||
old_db, _oh, _ol = _run_old(lots)
|
||
new_db, _nm = _run_new(lots)
|
||
|
||
old_calls = old_db.calls
|
||
new_calls = _normalize_new_calls(new_db.calls)
|
||
|
||
assert len(new_calls) == len(
|
||
old_calls
|
||
), f"разное число execute: old={len(old_calls)} new={len(new_calls)}"
|
||
for i, (old_c, new_c) in enumerate(zip(old_calls, new_calls, strict=True)):
|
||
assert old_c[0] == new_c[0], f"SQL расходится на вызове #{i}"
|
||
assert old_c[1] == new_c[1], f"params расходятся на вызове #{i}"
|
||
|
||
|
||
def test_insert_sql_region_code_injection() -> None:
|
||
"""Старый хардкодит `66` в VALUES; новый — параметр `:region_code`, params[region_code]=66."""
|
||
lots = [NewLot(**_fixture_data()[0])]
|
||
|
||
old_db, _oh, _ol = _run_old(lots)
|
||
new_db, _nm = _run_new(lots)
|
||
|
||
old_insert = next(s for s, _ in old_db.calls if "INSERT INTO listings (" in s)
|
||
new_insert, new_params = next((s, p) for s, p in new_db.calls if "INSERT INTO listings (" in s)
|
||
|
||
assert ", 66," in old_insert and ":region_code" not in old_insert
|
||
assert ":region_code," in new_insert and ", 66," not in new_insert
|
||
assert new_params["region_code"] == 66
|
||
# После нормализации → идентичны
|
||
assert new_insert.replace(":region_code", "66") == old_insert
|
||
|
||
|
||
def test_matching_hook_kwargs_parity() -> None:
|
||
"""kwargs match_or_create_house / upsert_listing_source идентичны old ↔ new."""
|
||
lots = [NewLot(**d) for d in _fixture_data()]
|
||
|
||
_old_db, m_house, m_link = _run_old(lots)
|
||
_new_db, matcher = _run_new(lots)
|
||
|
||
assert _kwargs_list(m_house) == _kwargs_list(matcher.match_or_create_house)
|
||
assert _kwargs_list(m_link) == _kwargs_list(matcher.upsert_listing_source)
|
||
|
||
# sanity: hook действительно сработал (не пустой прогон)
|
||
assert m_link.call_count == len(lots)
|
||
# house resolve только когда есть address/coords (4 из 5 фикстур)
|
||
assert m_house.call_count == 4
|
||
assert matcher.match_or_create_house.call_count == 4
|
||
|
||
|
||
def test_dedup_collision_triggers_reconcile_update_in_both() -> None:
|
||
"""Drift-лот → INSERT UniqueViolation → reconcile UPDATE by (source, source_id) в обоих."""
|
||
lots = [NewLot(**_fixture_data()[4])] # DRIFT-1
|
||
|
||
old_db, _oh, _ol = _run_old(lots)
|
||
new_db, _nm = _run_new(lots)
|
||
|
||
old_reconcile = [s for s, _ in old_db.calls if "UPDATE listings" in s and "SET dedup_hash" in s]
|
||
new_reconcile = [s for s, _ in new_db.calls if "UPDATE listings" in s and "SET dedup_hash" in s]
|
||
assert len(old_reconcile) == 1
|
||
assert len(new_reconcile) == 1
|
||
assert old_reconcile[0] == new_reconcile[0]
|
||
|
||
|
||
def test_return_counts_parity() -> None:
|
||
"""(inserted, updated) идентичны old ↔ new на одном входе."""
|
||
lots = [NewLot(**d) for d in _fixture_data()]
|
||
|
||
old_db = RecordingSession(drift_ids=_DRIFT_IDS)
|
||
with (
|
||
patch("app.services.scrapers.base.match_or_create_house", return_value=(101, 1.0, "new")),
|
||
patch("app.services.scrapers.base.upsert_listing_source", return_value=None),
|
||
patch("app.services.scrapers.base.upsert_listing_snapshot"),
|
||
):
|
||
old_result = old_save_listings(old_db, lots)
|
||
|
||
new_db = RecordingSession(drift_ids=_DRIFT_IDS)
|
||
matcher = MagicMock()
|
||
matcher.match_or_create_house.return_value = (101, 1.0, "new")
|
||
matcher.upsert_listing_source.return_value = None
|
||
with patch("scraper_kit.base.upsert_listing_snapshot"):
|
||
new_result = new_save_listings(new_db, lots, matcher=matcher, region_code=_REGION_CODE)
|
||
|
||
assert old_result == new_result
|
||
|
||
|
||
def test_hash_parity_old_vs_new_scraped_lot() -> None:
|
||
"""compute_dedup_hash / compute_card_hash / compute_price_per_m2 идентичны у обоих классов."""
|
||
for data in _fixture_data():
|
||
old_lot = OldLot(**data)
|
||
new_lot = NewLot(**data)
|
||
assert old_lot.compute_dedup_hash() == new_lot.compute_dedup_hash()
|
||
assert old_lot.compute_card_hash() == new_lot.compute_card_hash()
|
||
assert old_lot.compute_price_per_m2() == new_lot.compute_price_per_m2()
|
||
|
||
|
||
def test_kit_base_public_api_importable() -> None:
|
||
"""Acceptance: from scraper_kit.base import ScrapedLot, save_listings, BaseScraper."""
|
||
from scraper_kit.base import BaseScraper, ScrapedLot, save_listings
|
||
|
||
assert ScrapedLot is not None
|
||
assert callable(save_listings)
|
||
assert BaseScraper is not None
|
||
|
||
|
||
def test_kit_base_has_no_app_imports() -> None:
|
||
"""scraper_kit.base не должен импортировать app.* (strangler-развязка)."""
|
||
import inspect
|
||
|
||
import scraper_kit.base as kit_base
|
||
|
||
source = inspect.getsource(kit_base)
|
||
# Игнорируем docstring-упоминания: проверяем именно import-стейтменты.
|
||
for line in source.splitlines():
|
||
stripped = line.strip()
|
||
assert not stripped.startswith("from app"), f"app import найден: {line!r}"
|
||
assert not stripped.startswith("import app"), f"app import найден: {line!r}"
|