475 lines
17 KiB
Python
475 lines
17 KiB
Python
"""Тесты для base.py save_listings — проверяем что Cian-специфичные колонки
|
||
передаются в INSERT и ON CONFLICT DO UPDATE SET.
|
||
|
||
Используем unittest.mock для замены db.execute — без реальной БД.
|
||
"""
|
||
|
||
import json
|
||
from contextlib import contextmanager
|
||
from unittest.mock import MagicMock, patch
|
||
|
||
import pytest
|
||
|
||
from app.services.scrapers.base import ScrapedLot, save_listings
|
||
|
||
# ── Autouse fixture: patch matching service so all save_listings tests are deterministic ──
|
||
|
||
|
||
@pytest.fixture(autouse=True)
|
||
def _patch_matching():
|
||
"""Stub matching service for all tests in this module.
|
||
|
||
save_listings() now calls match_or_create_house + upsert_listing_source after
|
||
each INSERT. Without patching, those would issue extra db.execute() calls with
|
||
real SQL (and MagicMock returns truthy mappings() that would simulate Tier 0
|
||
matches with fake house_ids). Tests assert via call_args_list[0] (the listings
|
||
INSERT) and call_count on the patched stubs to verify the hook fires.
|
||
"""
|
||
with (
|
||
patch("app.services.scrapers.base.match_or_create_house") as m_house,
|
||
patch("app.services.scrapers.base.upsert_listing_source") as m_link,
|
||
):
|
||
m_house.return_value = (101, 1.0, "new") # (house_id, conf, method)
|
||
m_link.return_value = None
|
||
yield {"house": m_house, "link": m_link}
|
||
|
||
|
||
# ── Helpers ──────────────────────────────────────────────────────────────────
|
||
|
||
|
||
def _mock_db(inserted: bool = True, listing_id: int = 42) -> MagicMock:
|
||
"""Mock Session: INSERT listings RETURNING id, inserted=<bool>; SAVEPOINTs no-op."""
|
||
row = MagicMock()
|
||
row.id = listing_id
|
||
row.inserted = inserted
|
||
|
||
result = MagicMock()
|
||
result.fetchone.return_value = row
|
||
|
||
db = MagicMock()
|
||
db.execute.return_value = result
|
||
|
||
# begin_nested() returns a context manager; SAVEPOINT semantics — no-op here
|
||
@contextmanager
|
||
def _nested():
|
||
yield MagicMock()
|
||
|
||
db.begin_nested.side_effect = _nested
|
||
return db
|
||
|
||
|
||
def _base_lot(**overrides) -> ScrapedLot:
|
||
"""Минимальный валидный ScrapedLot — только обязательные поля."""
|
||
defaults = {
|
||
"source": "avito",
|
||
"source_url": "https://www.avito.ru/ekaterinburg/kvartiry/1-k_987654",
|
||
"source_id": "987654",
|
||
"price_rub": 4_500_000,
|
||
}
|
||
defaults.update(overrides)
|
||
return ScrapedLot(**defaults)
|
||
|
||
|
||
def _cian_lot(**overrides) -> ScrapedLot:
|
||
"""ScrapedLot с полным набором Cian-специфичных полей."""
|
||
base = {
|
||
"source": "cian",
|
||
"source_url": "https://ekb.cian.ru/sale/flat/123456/",
|
||
"source_id": "123456",
|
||
"price_rub": 6_200_000,
|
||
"address": "Екатеринбург, улица Постовского, 17А",
|
||
"lat": 56.8385,
|
||
"lon": 60.5940,
|
||
"rooms": 2,
|
||
"area_m2": 52.0,
|
||
"floor": 7,
|
||
"total_floors": 16,
|
||
# Cian-specific
|
||
"living_area_m2": 35.0,
|
||
"bedrooms_count": 2,
|
||
"balconies_count": 1,
|
||
"loggias_count": 0,
|
||
"description_minhash": "abc123def456abc1",
|
||
"cadastral_number": "66:41:0204016:1234",
|
||
"building_cadastral_number": "66:41:0204016:100",
|
||
"phones": [{"countryCode": "7", "number": "9012345678"}],
|
||
"is_homeowner": True,
|
||
"is_pro_seller": False,
|
||
"bargain_allowed": True,
|
||
"sale_type": "free",
|
||
"metro_stations": [{"name": "Геологическая", "time": 10, "mode": "walk"}],
|
||
"house_source": "cian",
|
||
"house_ext_id": None,
|
||
"listing_segment": "vtorichka",
|
||
}
|
||
base.update(overrides)
|
||
return ScrapedLot(**base)
|
||
|
||
|
||
# ── Utility: extract params from mock db call ────────────────────────────────
|
||
|
||
|
||
def _get_execute_params(db: MagicMock) -> dict:
|
||
"""Получить params dict из первого db.execute() вызова (INSERT listings)."""
|
||
assert db.execute.called, "db.execute was not called"
|
||
# First call is INSERT INTO listings ... RETURNING id, inserted.
|
||
# Subsequent calls are matching hook (match_or_create_house, listing_sources).
|
||
_sql, params = db.execute.call_args_list[0].args
|
||
return params
|
||
|
||
|
||
def _get_listings_insert_sql(db: MagicMock) -> str:
|
||
"""Получить SQL-текст INSERT INTO listings (первый db.execute() вызов)."""
|
||
assert db.execute.called, "db.execute was not called"
|
||
return str(db.execute.call_args_list[0].args[0])
|
||
|
||
|
||
# ── Test 1: Cian lot → все новые колонки попадают в params ──────────────────
|
||
|
||
|
||
def test_save_listings_persists_cian_specific_fields():
|
||
"""Cian lot с заполненными полями → все Cian колонки в params INSERT."""
|
||
db = _mock_db(inserted=True)
|
||
lot = _cian_lot()
|
||
|
||
inserted, updated = save_listings(db, [lot])
|
||
|
||
assert inserted == 1
|
||
assert updated == 0
|
||
|
||
params = _get_execute_params(db)
|
||
|
||
assert params["living_area_m2"] == 35.0
|
||
assert params["bedrooms_count"] == 2
|
||
assert params["balconies_count"] == 1
|
||
assert params["loggias_count"] == 0
|
||
assert params["description_minhash"] == "abc123def456abc1"
|
||
assert params["cadastral_number"] == "66:41:0204016:1234"
|
||
assert params["building_cadastral_number"] == "66:41:0204016:100"
|
||
assert params["is_homeowner"] is True
|
||
assert params["is_pro_seller"] is False
|
||
assert params["bargain_allowed"] is True
|
||
assert params["sale_type"] == "free"
|
||
|
||
# phones и metro_stations — JSON-строки, проверяем десериализацией
|
||
phones = json.loads(params["phones"])
|
||
assert phones == [{"countryCode": "7", "number": "9012345678"}]
|
||
|
||
metro = json.loads(params["metro_stations"])
|
||
assert metro[0]["name"] == "Геологическая"
|
||
assert metro[0]["time"] == 10
|
||
|
||
|
||
# ── Test 2: Avito lot → Cian-специфичные params = None (NULL) ───────────────
|
||
|
||
|
||
def test_save_listings_avito_compat_cian_cols_are_null():
|
||
"""Avito lot не имеет Cian-полей → все Cian params = None (→ SQL NULL).
|
||
|
||
Проверяет backward compat — Avito / DomRF scrapers не ломаются.
|
||
"""
|
||
db = _mock_db(inserted=True)
|
||
lot = _base_lot(source="avito", source_id="987654")
|
||
|
||
inserted, _updated = save_listings(db, [lot])
|
||
|
||
assert inserted == 1
|
||
|
||
params = _get_execute_params(db)
|
||
|
||
# ScrapedLot.living_area_m2 defaults to None
|
||
assert params["living_area_m2"] is None
|
||
assert params["bedrooms_count"] is None
|
||
assert params["balconies_count"] is None
|
||
assert params["loggias_count"] is None
|
||
assert params["description_minhash"] is None
|
||
assert params["cadastral_number"] is None
|
||
assert params["building_cadastral_number"] is None
|
||
assert params["is_homeowner"] is None
|
||
assert params["is_pro_seller"] is None
|
||
assert params["bargain_allowed"] is None
|
||
assert params["sale_type"] is None
|
||
# phones=[] (default_factory=list) → None (пустой список → None в helper)
|
||
assert params["phones"] is None
|
||
# metro_stations=[] → None
|
||
assert params["metro_stations"] is None
|
||
|
||
|
||
# ── Test 3: ON CONFLICT — при updated=False все новые поля в UPDATE SET ──────
|
||
|
||
|
||
def test_save_listings_on_conflict_updates_cian_cols():
|
||
"""ON CONFLICT path: xmax != 0 → updated counter. Params верны для UPDATE.
|
||
|
||
Проверяем что SQL-текст содержит все новые колонки в DO UPDATE SET.
|
||
"""
|
||
db = _mock_db(inserted=False) # xmax != 0 → updated
|
||
lot = _cian_lot(
|
||
source_id="111",
|
||
cadastral_number="66:41:0000000:NEW",
|
||
description_minhash="newhash",
|
||
)
|
||
|
||
inserted, updated = save_listings(db, [lot])
|
||
|
||
assert inserted == 0
|
||
assert updated == 1
|
||
|
||
# Проверяем SQL-текст на наличие DO UPDATE SET с новыми колонками
|
||
sql_text = _get_listings_insert_sql(db)
|
||
|
||
assert "cadastral_number = EXCLUDED.cadastral_number" in sql_text
|
||
assert "description_minhash = EXCLUDED.description_minhash" in sql_text
|
||
assert "building_cadastral_number = EXCLUDED.building_cadastral_number" in sql_text
|
||
assert "phones = EXCLUDED.phones" in sql_text
|
||
assert "is_homeowner = EXCLUDED.is_homeowner" in sql_text
|
||
assert "is_pro_seller = EXCLUDED.is_pro_seller" in sql_text
|
||
assert "bargain_allowed = EXCLUDED.bargain_allowed" in sql_text
|
||
assert "sale_type = EXCLUDED.sale_type" in sql_text
|
||
assert "metro_stations = EXCLUDED.metro_stations" in sql_text
|
||
assert "living_area_m2 = EXCLUDED.living_area_m2" in sql_text
|
||
assert "bedrooms_count = EXCLUDED.bedrooms_count" in sql_text
|
||
assert "balconies_count = EXCLUDED.balconies_count" in sql_text
|
||
assert "loggias_count = EXCLUDED.loggias_count" in sql_text
|
||
|
||
# Params с новым значением cadastral_number
|
||
params = _get_execute_params(db)
|
||
assert params["cadastral_number"] == "66:41:0000000:NEW"
|
||
assert params["description_minhash"] == "newhash"
|
||
|
||
|
||
# ── Test 4: empty list → early return, no db.execute call ───────────────────
|
||
|
||
|
||
def test_save_listings_empty_lots():
|
||
"""Пустой список → (0, 0), db.execute не вызывается."""
|
||
db = _mock_db()
|
||
|
||
inserted, updated = save_listings(db, [])
|
||
|
||
assert inserted == 0
|
||
assert updated == 0
|
||
db.execute.assert_not_called()
|
||
|
||
|
||
# ── Test 5: phones=[] и metro_stations=[] → None в params ───────────────────
|
||
|
||
|
||
def test_save_listings_empty_phones_and_metro_become_null():
|
||
"""Если phones=[] / metro_stations=[] — передаём None (не '[]' строку)."""
|
||
db = _mock_db(inserted=True)
|
||
lot = _cian_lot(phones=[], metro_stations=[])
|
||
|
||
save_listings(db, [lot])
|
||
|
||
params = _get_execute_params(db)
|
||
assert params["phones"] is None
|
||
assert params["metro_stations"] is None
|
||
|
||
|
||
# ── Test 6: INSERT SQL содержит все новые колонки ────────────────────────────
|
||
|
||
|
||
def test_save_listings_sql_contains_all_new_columns():
|
||
"""SQL INSERT перечисляет все Cian-специфичные колонки."""
|
||
db = _mock_db(inserted=True)
|
||
lot = _base_lot()
|
||
|
||
save_listings(db, [lot])
|
||
|
||
sql_text = _get_listings_insert_sql(db)
|
||
|
||
for col in (
|
||
"living_area_m2",
|
||
"bedrooms_count",
|
||
"balconies_count",
|
||
"loggias_count",
|
||
"description_minhash",
|
||
"cadastral_number",
|
||
"building_cadastral_number",
|
||
"phones",
|
||
"is_homeowner",
|
||
"is_pro_seller",
|
||
"bargain_allowed",
|
||
"sale_type",
|
||
"metro_stations",
|
||
):
|
||
assert col in sql_text, f"Column '{col}' missing from INSERT SQL"
|
||
|
||
|
||
# ── Test 7: INSERT RETURNING id — used downstream by matching hook ──────────
|
||
|
||
|
||
def test_save_listings_returning_id_for_matching_hook():
|
||
"""INSERT … RETURNING id, (xmax = 0) — needs both columns for the hook."""
|
||
db = _mock_db(inserted=True, listing_id=12345)
|
||
lot = _base_lot()
|
||
|
||
save_listings(db, [lot])
|
||
|
||
sql_text = _get_listings_insert_sql(db)
|
||
assert "RETURNING id, (xmax = 0) AS inserted" in sql_text
|
||
|
||
|
||
# ── Test 8: matching hook fires exactly once per saved listing ──────────────
|
||
|
||
|
||
def test_save_listings_invokes_matching_hook_once_per_lot(_patch_matching):
|
||
"""Each saved lot triggers exactly one upsert_listing_source call.
|
||
|
||
match_or_create_house may or may not be called (depends on address/lat/lon
|
||
presence). upsert_listing_source MUST always be called.
|
||
"""
|
||
db = _mock_db(inserted=True, listing_id=777)
|
||
lots = [
|
||
_base_lot(source_id="a1"),
|
||
_base_lot(source_id="a2", address="Екатеринбург, Ленина 1", lat=56.84, lon=60.6),
|
||
_cian_lot(source_id="c1"),
|
||
]
|
||
|
||
save_listings(db, lots)
|
||
|
||
# Hook fires once per lot
|
||
assert _patch_matching["link"].call_count == 3
|
||
|
||
# House resolution attempted only when address/coords present (2 of 3)
|
||
assert _patch_matching["house"].call_count == 2
|
||
|
||
# All upsert_listing_source calls use ext_source = listing.source
|
||
for idx, call_args in enumerate(_patch_matching["link"].call_args_list):
|
||
kwargs = call_args.kwargs
|
||
assert kwargs["listing_id"] == 777
|
||
assert kwargs["ext_source"] == lots[idx].source
|
||
assert kwargs["ext_id"] == lots[idx].source_id
|
||
assert kwargs["method"] == "source_link"
|
||
assert kwargs["confidence"] == 1.0
|
||
|
||
|
||
# ── Test 9: matching hook receives full listing context ─────────────────────
|
||
|
||
|
||
def test_save_listings_matching_hook_passes_listing_context(_patch_matching):
|
||
"""upsert_listing_source receives price/area/floor/rooms/url from the lot."""
|
||
db = _mock_db(inserted=True, listing_id=555)
|
||
lot = _cian_lot(
|
||
source_id="cian-999",
|
||
price_rub=7_500_000,
|
||
area_m2=54.5,
|
||
floor=3,
|
||
rooms=2,
|
||
)
|
||
|
||
save_listings(db, [lot])
|
||
|
||
kwargs = _patch_matching["link"].call_args.kwargs
|
||
assert kwargs["listing_id"] == 555
|
||
assert kwargs["ext_source"] == "cian"
|
||
assert kwargs["ext_id"] == "cian-999"
|
||
assert kwargs["price_rub"] == 7_500_000
|
||
assert kwargs["area_m2"] == 54.5
|
||
assert kwargs["floor"] == 3
|
||
assert kwargs["rooms_count"] == 2
|
||
assert kwargs["source_url"] == lot.source_url
|
||
|
||
|
||
# ── Test 10: matching hook idempotent on re-scrape (ON CONFLICT updated path) ─
|
||
|
||
|
||
def test_save_listings_matching_hook_fires_on_update_path(_patch_matching):
|
||
"""ON CONFLICT DO UPDATE (re-scrape) — hook still fires so last_seen_at refreshes."""
|
||
db = _mock_db(inserted=False, listing_id=12) # updated, not new
|
||
lot = _base_lot()
|
||
|
||
save_listings(db, [lot])
|
||
|
||
assert _patch_matching["link"].call_count == 1
|
||
|
||
|
||
# ── Test 11: matching failure logs warning but does not abort the batch ─────
|
||
|
||
|
||
def test_save_listings_matching_failure_does_not_abort_batch(_patch_matching, caplog):
|
||
"""When upsert_listing_source raises, the loop logs and continues."""
|
||
import logging
|
||
|
||
_patch_matching["link"].side_effect = RuntimeError("simulated DB blip")
|
||
|
||
db = _mock_db(inserted=True, listing_id=99)
|
||
lots = [_base_lot(source_id="a1"), _base_lot(source_id="a2")]
|
||
|
||
with caplog.at_level(logging.WARNING, logger="app.services.scrapers.base"):
|
||
inserted, updated = save_listings(db, lots)
|
||
|
||
# Both listings still INSERTed; only the hook failed
|
||
assert inserted == 2
|
||
assert updated == 0
|
||
|
||
# 2 warnings — one per lot
|
||
warns = [r for r in caplog.records if "match_failed" in r.getMessage()]
|
||
assert len(warns) == 2
|
||
|
||
# Hook attempted on both
|
||
assert _patch_matching["link"].call_count == 2
|
||
|
||
|
||
# ── Test 12: lot without source_id falls back to dedup_hash as ext_id ───────
|
||
|
||
|
||
def test_save_listings_matching_hook_dedup_hash_fallback(_patch_matching):
|
||
"""Yandex-style lot without stable source_id → ext_id := dedup_hash."""
|
||
db = _mock_db(inserted=True, listing_id=44)
|
||
lot = _base_lot(
|
||
source="yandex",
|
||
source_url="https://realty.yandex.ru/offer/some-offer/",
|
||
source_id=None,
|
||
address="Екатеринбург, Малышева 51",
|
||
lat=56.84,
|
||
lon=60.6,
|
||
)
|
||
|
||
save_listings(db, [lot])
|
||
|
||
kwargs = _patch_matching["link"].call_args.kwargs
|
||
assert kwargs["ext_source"] == "yandex"
|
||
# ext_id should be the dedup_hash (sha256 hex), not None
|
||
assert kwargs["ext_id"]
|
||
assert len(kwargs["ext_id"]) == 64 # sha256 hex
|
||
assert kwargs["ext_id"] == lot.compute_dedup_hash()
|
||
|
||
|
||
# ── Test 13: house_match failure (e.g. address normalization) still upserts source ─
|
||
|
||
|
||
def test_save_listings_house_match_failure_still_upserts_listing_source(_patch_matching):
|
||
"""If match_or_create_house raises, listing_sources still gets upserted."""
|
||
_patch_matching["house"].side_effect = RuntimeError("address parse boom")
|
||
|
||
db = _mock_db(inserted=True, listing_id=88)
|
||
lot = _base_lot(
|
||
source_id="x1",
|
||
address="Bad addr",
|
||
lat=56.84,
|
||
lon=60.6,
|
||
)
|
||
|
||
inserted, _ = save_listings(db, [lot])
|
||
|
||
assert inserted == 1
|
||
# House match attempted but failed — listing_source still recorded
|
||
assert _patch_matching["house"].call_count == 1
|
||
assert _patch_matching["link"].call_count == 1
|
||
kwargs = _patch_matching["link"].call_args.kwargs
|
||
# source_data should not carry a house_id since match failed
|
||
assert kwargs["source_data"] is None
|
||
|
||
|
||
# ── Test 14: empty list → matching service not called ───────────────────────
|
||
|
||
|
||
def test_save_listings_empty_lots_skips_matching(_patch_matching):
|
||
"""Empty input → no hook calls (verifies early return preserves behaviour)."""
|
||
db = _mock_db()
|
||
save_listings(db, [])
|
||
|
||
assert _patch_matching["house"].call_count == 0
|
||
assert _patch_matching["link"].call_count == 0
|