260 lines
9.3 KiB
Python
260 lines
9.3 KiB
Python
"""Тесты для base.py save_listings — проверяем что Cian-специфичные колонки
|
||
передаются в INSERT и ON CONFLICT DO UPDATE SET.
|
||
|
||
Используем unittest.mock для замены db.execute — без реальной БД.
|
||
"""
|
||
|
||
import json
|
||
from unittest.mock import MagicMock, call, patch
|
||
|
||
import pytest
|
||
|
||
from app.services.scrapers.base import ScrapedLot, save_listings
|
||
|
||
|
||
# ── Helpers ──────────────────────────────────────────────────────────────────
|
||
|
||
|
||
def _mock_db(inserted: bool = True) -> MagicMock:
|
||
"""Создать mock Session с fetchone() → (inserted=<bool>,)."""
|
||
row = MagicMock()
|
||
row.inserted = inserted
|
||
|
||
result = MagicMock()
|
||
result.fetchone.return_value = row
|
||
|
||
db = MagicMock()
|
||
db.execute.return_value = result
|
||
return db
|
||
|
||
|
||
def _base_lot(**overrides) -> ScrapedLot:
|
||
"""Минимальный валидный ScrapedLot — только обязательные поля."""
|
||
defaults = {
|
||
"source": "avito",
|
||
"source_url": "https://www.avito.ru/ekaterinburg/kvartiry/1-k_987654",
|
||
"source_id": "987654",
|
||
"price_rub": 4_500_000,
|
||
}
|
||
defaults.update(overrides)
|
||
return ScrapedLot(**defaults)
|
||
|
||
|
||
def _cian_lot(**overrides) -> ScrapedLot:
|
||
"""ScrapedLot с полным набором Cian-специфичных полей."""
|
||
base = {
|
||
"source": "cian",
|
||
"source_url": "https://ekb.cian.ru/sale/flat/123456/",
|
||
"source_id": "123456",
|
||
"price_rub": 6_200_000,
|
||
"address": "Екатеринбург, улица Постовского, 17А",
|
||
"lat": 56.8385,
|
||
"lon": 60.5940,
|
||
"rooms": 2,
|
||
"area_m2": 52.0,
|
||
"floor": 7,
|
||
"total_floors": 16,
|
||
# Cian-specific
|
||
"living_area_m2": 35.0,
|
||
"bedrooms_count": 2,
|
||
"balconies_count": 1,
|
||
"loggias_count": 0,
|
||
"description_minhash": "abc123def456abc1",
|
||
"cadastral_number": "66:41:0204016:1234",
|
||
"building_cadastral_number": "66:41:0204016:100",
|
||
"phones": [{"countryCode": "7", "number": "9012345678"}],
|
||
"is_homeowner": True,
|
||
"is_pro_seller": False,
|
||
"bargain_allowed": True,
|
||
"sale_type": "free",
|
||
"metro_stations": [{"name": "Геологическая", "time": 10, "mode": "walk"}],
|
||
"house_source": "cian",
|
||
"house_ext_id": None,
|
||
"listing_segment": "vtorichka",
|
||
}
|
||
base.update(overrides)
|
||
return ScrapedLot(**base)
|
||
|
||
|
||
# ── Utility: extract params from mock db call ────────────────────────────────
|
||
|
||
|
||
def _get_execute_params(db: MagicMock) -> dict:
|
||
"""Получить params dict из первого db.execute() вызова."""
|
||
assert db.execute.called, "db.execute was not called"
|
||
_sql, params = db.execute.call_args.args
|
||
return params
|
||
|
||
|
||
# ── Test 1: Cian lot → все новые колонки попадают в params ──────────────────
|
||
|
||
|
||
def test_save_listings_persists_cian_specific_fields():
|
||
"""Cian lot с заполненными полями → все Cian колонки в params INSERT."""
|
||
db = _mock_db(inserted=True)
|
||
lot = _cian_lot()
|
||
|
||
inserted, updated = save_listings(db, [lot])
|
||
|
||
assert inserted == 1
|
||
assert updated == 0
|
||
|
||
params = _get_execute_params(db)
|
||
|
||
assert params["living_area_m2"] == 35.0
|
||
assert params["bedrooms_count"] == 2
|
||
assert params["balconies_count"] == 1
|
||
assert params["loggias_count"] == 0
|
||
assert params["description_minhash"] == "abc123def456abc1"
|
||
assert params["cadastral_number"] == "66:41:0204016:1234"
|
||
assert params["building_cadastral_number"] == "66:41:0204016:100"
|
||
assert params["is_homeowner"] is True
|
||
assert params["is_pro_seller"] is False
|
||
assert params["bargain_allowed"] is True
|
||
assert params["sale_type"] == "free"
|
||
|
||
# phones и metro_stations — JSON-строки, проверяем десериализацией
|
||
phones = json.loads(params["phones"])
|
||
assert phones == [{"countryCode": "7", "number": "9012345678"}]
|
||
|
||
metro = json.loads(params["metro_stations"])
|
||
assert metro[0]["name"] == "Геологическая"
|
||
assert metro[0]["time"] == 10
|
||
|
||
|
||
# ── Test 2: Avito lot → Cian-специфичные params = None (NULL) ───────────────
|
||
|
||
|
||
def test_save_listings_avito_compat_cian_cols_are_null():
|
||
"""Avito lot не имеет Cian-полей → все Cian params = None (→ SQL NULL).
|
||
|
||
Проверяет backward compat — Avito / DomRF scrapers не ломаются.
|
||
"""
|
||
db = _mock_db(inserted=True)
|
||
lot = _base_lot(source="avito", source_id="987654")
|
||
|
||
inserted, updated = save_listings(db, [lot])
|
||
|
||
assert inserted == 1
|
||
|
||
params = _get_execute_params(db)
|
||
|
||
# ScrapedLot.living_area_m2 defaults to None
|
||
assert params["living_area_m2"] is None
|
||
assert params["bedrooms_count"] is None
|
||
assert params["balconies_count"] is None
|
||
assert params["loggias_count"] is None
|
||
assert params["description_minhash"] is None
|
||
assert params["cadastral_number"] is None
|
||
assert params["building_cadastral_number"] is None
|
||
assert params["is_homeowner"] is None
|
||
assert params["is_pro_seller"] is None
|
||
assert params["bargain_allowed"] is None
|
||
assert params["sale_type"] is None
|
||
# phones=[] (default_factory=list) → None (пустой список → None в helper)
|
||
assert params["phones"] is None
|
||
# metro_stations=[] → None
|
||
assert params["metro_stations"] is None
|
||
|
||
|
||
# ── Test 3: ON CONFLICT — при updated=False все новые поля в UPDATE SET ──────
|
||
|
||
|
||
def test_save_listings_on_conflict_updates_cian_cols():
|
||
"""ON CONFLICT path: xmax != 0 → updated counter. Params верны для UPDATE.
|
||
|
||
Проверяем что SQL-текст содержит все новые колонки в DO UPDATE SET.
|
||
"""
|
||
db = _mock_db(inserted=False) # xmax != 0 → updated
|
||
lot = _cian_lot(
|
||
source_id="111",
|
||
cadastral_number="66:41:0000000:NEW",
|
||
description_minhash="newhash",
|
||
)
|
||
|
||
inserted, updated = save_listings(db, [lot])
|
||
|
||
assert inserted == 0
|
||
assert updated == 1
|
||
|
||
# Проверяем SQL-текст на наличие DO UPDATE SET с новыми колонками
|
||
sql_obj = db.execute.call_args.args[0]
|
||
sql_text = str(sql_obj)
|
||
|
||
assert "cadastral_number = EXCLUDED.cadastral_number" in sql_text
|
||
assert "description_minhash = EXCLUDED.description_minhash" in sql_text
|
||
assert "building_cadastral_number = EXCLUDED.building_cadastral_number" in sql_text
|
||
assert "phones = EXCLUDED.phones" in sql_text
|
||
assert "is_homeowner = EXCLUDED.is_homeowner" in sql_text
|
||
assert "is_pro_seller = EXCLUDED.is_pro_seller" in sql_text
|
||
assert "bargain_allowed = EXCLUDED.bargain_allowed" in sql_text
|
||
assert "sale_type = EXCLUDED.sale_type" in sql_text
|
||
assert "metro_stations = EXCLUDED.metro_stations" in sql_text
|
||
assert "living_area_m2 = EXCLUDED.living_area_m2" in sql_text
|
||
assert "bedrooms_count = EXCLUDED.bedrooms_count" in sql_text
|
||
assert "balconies_count = EXCLUDED.balconies_count" in sql_text
|
||
assert "loggias_count = EXCLUDED.loggias_count" in sql_text
|
||
|
||
# Params с новым значением cadastral_number
|
||
params = _get_execute_params(db)
|
||
assert params["cadastral_number"] == "66:41:0000000:NEW"
|
||
assert params["description_minhash"] == "newhash"
|
||
|
||
|
||
# ── Test 4: empty list → early return, no db.execute call ───────────────────
|
||
|
||
|
||
def test_save_listings_empty_lots():
|
||
"""Пустой список → (0, 0), db.execute не вызывается."""
|
||
db = _mock_db()
|
||
|
||
inserted, updated = save_listings(db, [])
|
||
|
||
assert inserted == 0
|
||
assert updated == 0
|
||
db.execute.assert_not_called()
|
||
|
||
|
||
# ── Test 5: phones=[] и metro_stations=[] → None в params ───────────────────
|
||
|
||
|
||
def test_save_listings_empty_phones_and_metro_become_null():
|
||
"""Если phones=[] / metro_stations=[] — передаём None (не '[]' строку)."""
|
||
db = _mock_db(inserted=True)
|
||
lot = _cian_lot(phones=[], metro_stations=[])
|
||
|
||
save_listings(db, [lot])
|
||
|
||
params = _get_execute_params(db)
|
||
assert params["phones"] is None
|
||
assert params["metro_stations"] is None
|
||
|
||
|
||
# ── Test 6: INSERT SQL содержит все новые колонки ────────────────────────────
|
||
|
||
|
||
def test_save_listings_sql_contains_all_new_columns():
|
||
"""SQL INSERT перечисляет все Cian-специфичные колонки."""
|
||
db = _mock_db(inserted=True)
|
||
lot = _base_lot()
|
||
|
||
save_listings(db, [lot])
|
||
|
||
sql_text = str(db.execute.call_args.args[0])
|
||
|
||
for col in (
|
||
"living_area_m2",
|
||
"bedrooms_count",
|
||
"balconies_count",
|
||
"loggias_count",
|
||
"description_minhash",
|
||
"cadastral_number",
|
||
"building_cadastral_number",
|
||
"phones",
|
||
"is_homeowner",
|
||
"is_pro_seller",
|
||
"bargain_allowed",
|
||
"sale_type",
|
||
"metro_stations",
|
||
):
|
||
assert col in sql_text, f"Column '{col}' missing from INSERT SQL"
|