feat(tradein): extend save_listings INSERT to persist Cian-specific columns (PR #450 follow-up)

Extends base.py save_listings() to persist all 13 Cian-specific columns added to ScrapedLot in PR #447 and listings DDL in migration 019: living_area_m2, bedrooms_count, balconies_count, loggias_count, description_minhash, cadastral_number, building_cadastral_number, phones, is_homeowner, is_pro_seller, bargain_allowed, sale_type, metro_stations. ON CONFLICT DO UPDATE SET extended to refresh all new columns on re-scrape. Avito/DomRF backward compat: optional fields default None → SQL NULL. Adds 6 unit tests (test_base_save_listings.py) covering Cian INSERT, Avito null compat, ON CONFLICT UPDATE SQL text, empty phones/metro → NULL.
2026-05-23 15:58:02 +03:00 · 2026-05-23 15:58:02 +03:00 · 702cb2b7d0
commit 702cb2b7d0
parent 4f0319297e
2 changed files with 297 additions and 1 deletions
--- a/tradein-mvp/backend/app/services/scrapers/base.py
+++ b/tradein-mvp/backend/app/services/scrapers/base.py
@ -223,6 +223,10 @@ def save_listings(db: Session, lots: list[ScrapedLot]) -> tuple[int, int]:
                    house_source, house_ext_id, house_url, listing_segment,
                    price_rub, price_per_m2,
                    listing_date, days_on_market, photo_urls, raw_payload,
+                    living_area_m2, bedrooms_count, balconies_count, loggias_count,
+                    description_minhash, cadastral_number, building_cadastral_number,
+                    phones, is_homeowner, is_pro_seller,
+                    bargain_allowed, sale_type, metro_stations,
                    scraped_at, last_seen_at
                ) VALUES (
                    :source, :source_url, :source_id, :dedup,
@ -234,6 +238,10 @@ def save_listings(db: Session, lots: list[ScrapedLot]) -> tuple[int, int]:
                    :listing_date, :days_on_market,
                    CAST(:photos AS jsonb),
                    CAST(:raw AS jsonb),
+                    :living_area_m2, :bedrooms_count, :balconies_count, :loggias_count,
+                    :description_minhash, :cadastral_number, :building_cadastral_number,
+                    CAST(:phones AS jsonb), :is_homeowner, :is_pro_seller,
+                    :bargain_allowed, :sale_type, CAST(:metro_stations AS jsonb),
                    NOW(), NOW()
                )
                ON CONFLICT (dedup_hash) DO UPDATE
@ -241,7 +249,21 @@ def save_listings(db: Session, lots: list[ScrapedLot]) -> tuple[int, int]:
                        is_active = true,
                        -- если цена изменилась — обновляем
                        price_rub = EXCLUDED.price_rub,
-                        price_per_m2 = EXCLUDED.price_per_m2
+                        price_per_m2 = EXCLUDED.price_per_m2,
+                        -- Cian-specific: обновляем при каждом re-scrape
+                        living_area_m2 = EXCLUDED.living_area_m2,
+                        bedrooms_count = EXCLUDED.bedrooms_count,
+                        balconies_count = EXCLUDED.balconies_count,
+                        loggias_count = EXCLUDED.loggias_count,
+                        description_minhash = EXCLUDED.description_minhash,
+                        cadastral_number = EXCLUDED.cadastral_number,
+                        building_cadastral_number = EXCLUDED.building_cadastral_number,
+                        phones = EXCLUDED.phones,
+                        is_homeowner = EXCLUDED.is_homeowner,
+                        is_pro_seller = EXCLUDED.is_pro_seller,
+                        bargain_allowed = EXCLUDED.bargain_allowed,
+                        sale_type = EXCLUDED.sale_type,
+                        metro_stations = EXCLUDED.metro_stations
                RETURNING (xmax = 0) AS inserted
                """
            ),
@ -272,6 +294,20 @@ def save_listings(db: Session, lots: list[ScrapedLot]) -> tuple[int, int]:
                "days_on_market": lot.days_on_market,
                "photos": _to_json(lot.photo_urls),
                "raw": _to_json(lot.raw_payload) if lot.raw_payload else None,
+                # Cian-specific columns — None for non-Cian scrapers (→ SQL NULL)
+                "living_area_m2": lot.living_area_m2,
+                "bedrooms_count": lot.bedrooms_count,
+                "balconies_count": lot.balconies_count,
+                "loggias_count": lot.loggias_count,
+                "description_minhash": lot.description_minhash,
+                "cadastral_number": lot.cadastral_number,
+                "building_cadastral_number": lot.building_cadastral_number,
+                "phones": _to_json(lot.phones) if lot.phones else None,
+                "is_homeowner": lot.is_homeowner,
+                "is_pro_seller": lot.is_pro_seller,
+                "bargain_allowed": lot.bargain_allowed,
+                "sale_type": lot.sale_type,
+                "metro_stations": _to_json(lot.metro_stations) if lot.metro_stations else None,
            },
        ).fetchone()

--- a/tradein-mvp/backend/tests/test_base_save_listings.py
+++ b/tradein-mvp/backend/tests/test_base_save_listings.py
@ -0,0 +1,260 @@
+"""Тесты для base.py save_listings — проверяем что Cian-специфичные колонки
+передаются в INSERT и ON CONFLICT DO UPDATE SET.
+
+Используем unittest.mock для замены db.execute — без реальной БД.
+"""
+
+import json
+from unittest.mock import MagicMock, call, patch
+
+import pytest
+
+from app.services.scrapers.base import ScrapedLot, save_listings
+
+
+# ── Helpers ──────────────────────────────────────────────────────────────────
+
+
+def _mock_db(inserted: bool = True) -> MagicMock:
+    """Создать mock Session с fetchone() → (inserted=<bool>,)."""
+    row = MagicMock()
+    row.inserted = inserted
+
+    result = MagicMock()
+    result.fetchone.return_value = row
+
+    db = MagicMock()
+    db.execute.return_value = result
+    return db
+
+
+def _base_lot(**overrides) -> ScrapedLot:
+    """Минимальный валидный ScrapedLot — только обязательные поля."""
+    defaults = {
+        "source": "avito",
+        "source_url": "https://www.avito.ru/ekaterinburg/kvartiry/1-k_987654",
+        "source_id": "987654",
+        "price_rub": 4_500_000,
+    }
+    defaults.update(overrides)
+    return ScrapedLot(**defaults)
+
+
+def _cian_lot(**overrides) -> ScrapedLot:
+    """ScrapedLot с полным набором Cian-специфичных полей."""
+    base = {
+        "source": "cian",
+        "source_url": "https://ekb.cian.ru/sale/flat/123456/",
+        "source_id": "123456",
+        "price_rub": 6_200_000,
+        "address": "Екатеринбург, улица Постовского, 17А",
+        "lat": 56.8385,
+        "lon": 60.5940,
+        "rooms": 2,
+        "area_m2": 52.0,
+        "floor": 7,
+        "total_floors": 16,
+        # Cian-specific
+        "living_area_m2": 35.0,
+        "bedrooms_count": 2,
+        "balconies_count": 1,
+        "loggias_count": 0,
+        "description_minhash": "abc123def456abc1",
+        "cadastral_number": "66:41:0204016:1234",
+        "building_cadastral_number": "66:41:0204016:100",
+        "phones": [{"countryCode": "7", "number": "9012345678"}],
+        "is_homeowner": True,
+        "is_pro_seller": False,
+        "bargain_allowed": True,
+        "sale_type": "free",
+        "metro_stations": [{"name": "Геологическая", "time": 10, "mode": "walk"}],
+        "house_source": "cian",
+        "house_ext_id": None,
+        "listing_segment": "vtorichka",
+    }
+    base.update(overrides)
+    return ScrapedLot(**base)
+
+
+# ── Utility: extract params from mock db call ────────────────────────────────
+
+
+def _get_execute_params(db: MagicMock) -> dict:
+    """Получить params dict из первого db.execute() вызова."""
+    assert db.execute.called, "db.execute was not called"
+    _sql, params = db.execute.call_args.args
+    return params
+
+
+# ── Test 1: Cian lot → все новые колонки попадают в params ──────────────────
+
+
+def test_save_listings_persists_cian_specific_fields():
+    """Cian lot с заполненными полями → все Cian колонки в params INSERT."""
+    db = _mock_db(inserted=True)
+    lot = _cian_lot()
+
+    inserted, updated = save_listings(db, [lot])
+
+    assert inserted == 1
+    assert updated == 0
+
+    params = _get_execute_params(db)
+
+    assert params["living_area_m2"] == 35.0
+    assert params["bedrooms_count"] == 2
+    assert params["balconies_count"] == 1
+    assert params["loggias_count"] == 0
+    assert params["description_minhash"] == "abc123def456abc1"
+    assert params["cadastral_number"] == "66:41:0204016:1234"
+    assert params["building_cadastral_number"] == "66:41:0204016:100"
+    assert params["is_homeowner"] is True
+    assert params["is_pro_seller"] is False
+    assert params["bargain_allowed"] is True
+    assert params["sale_type"] == "free"
+
+    # phones и metro_stations — JSON-строки, проверяем десериализацией
+    phones = json.loads(params["phones"])
+    assert phones == [{"countryCode": "7", "number": "9012345678"}]
+
+    metro = json.loads(params["metro_stations"])
+    assert metro[0]["name"] == "Геологическая"
+    assert metro[0]["time"] == 10
+
+
+# ── Test 2: Avito lot → Cian-специфичные params = None (NULL) ───────────────
+
+
+def test_save_listings_avito_compat_cian_cols_are_null():
+    """Avito lot не имеет Cian-полей → все Cian params = None (→ SQL NULL).
+
+    Проверяет backward compat — Avito / DomRF scrapers не ломаются.
+    """
+    db = _mock_db(inserted=True)
+    lot = _base_lot(source="avito", source_id="987654")
+
+    inserted, updated = save_listings(db, [lot])
+
+    assert inserted == 1
+
+    params = _get_execute_params(db)
+
+    # ScrapedLot.living_area_m2 defaults to None
+    assert params["living_area_m2"] is None
+    assert params["bedrooms_count"] is None
+    assert params["balconies_count"] is None
+    assert params["loggias_count"] is None
+    assert params["description_minhash"] is None
+    assert params["cadastral_number"] is None
+    assert params["building_cadastral_number"] is None
+    assert params["is_homeowner"] is None
+    assert params["is_pro_seller"] is None
+    assert params["bargain_allowed"] is None
+    assert params["sale_type"] is None
+    # phones=[] (default_factory=list) → None (пустой список → None в helper)
+    assert params["phones"] is None
+    # metro_stations=[] → None
+    assert params["metro_stations"] is None
+
+
+# ── Test 3: ON CONFLICT — при updated=False все новые поля в UPDATE SET ──────
+
+
+def test_save_listings_on_conflict_updates_cian_cols():
+    """ON CONFLICT path: xmax != 0 → updated counter. Params верны для UPDATE.
+
+    Проверяем что SQL-текст содержит все новые колонки в DO UPDATE SET.
+    """
+    db = _mock_db(inserted=False)  # xmax != 0 → updated
+    lot = _cian_lot(
+        source_id="111",
+        cadastral_number="66:41:0000000:NEW",
+        description_minhash="newhash",
+    )
+
+    inserted, updated = save_listings(db, [lot])
+
+    assert inserted == 0
+    assert updated == 1
+
+    # Проверяем SQL-текст на наличие DO UPDATE SET с новыми колонками
+    sql_obj = db.execute.call_args.args[0]
+    sql_text = str(sql_obj)
+
+    assert "cadastral_number = EXCLUDED.cadastral_number" in sql_text
+    assert "description_minhash = EXCLUDED.description_minhash" in sql_text
+    assert "building_cadastral_number = EXCLUDED.building_cadastral_number" in sql_text
+    assert "phones = EXCLUDED.phones" in sql_text
+    assert "is_homeowner = EXCLUDED.is_homeowner" in sql_text
+    assert "is_pro_seller = EXCLUDED.is_pro_seller" in sql_text
+    assert "bargain_allowed = EXCLUDED.bargain_allowed" in sql_text
+    assert "sale_type = EXCLUDED.sale_type" in sql_text
+    assert "metro_stations = EXCLUDED.metro_stations" in sql_text
+    assert "living_area_m2 = EXCLUDED.living_area_m2" in sql_text
+    assert "bedrooms_count = EXCLUDED.bedrooms_count" in sql_text
+    assert "balconies_count = EXCLUDED.balconies_count" in sql_text
+    assert "loggias_count = EXCLUDED.loggias_count" in sql_text
+
+    # Params с новым значением cadastral_number
+    params = _get_execute_params(db)
+    assert params["cadastral_number"] == "66:41:0000000:NEW"
+    assert params["description_minhash"] == "newhash"
+
+
+# ── Test 4: empty list → early return, no db.execute call ───────────────────
+
+
+def test_save_listings_empty_lots():
+    """Пустой список → (0, 0), db.execute не вызывается."""
+    db = _mock_db()
+
+    inserted, updated = save_listings(db, [])
+
+    assert inserted == 0
+    assert updated == 0
+    db.execute.assert_not_called()
+
+
+# ── Test 5: phones=[] и metro_stations=[] → None в params ───────────────────
+
+
+def test_save_listings_empty_phones_and_metro_become_null():
+    """Если phones=[] / metro_stations=[] — передаём None (не '[]' строку)."""
+    db = _mock_db(inserted=True)
+    lot = _cian_lot(phones=[], metro_stations=[])
+
+    save_listings(db, [lot])
+
+    params = _get_execute_params(db)
+    assert params["phones"] is None
+    assert params["metro_stations"] is None
+
+
+# ── Test 6: INSERT SQL содержит все новые колонки ────────────────────────────
+
+
+def test_save_listings_sql_contains_all_new_columns():
+    """SQL INSERT перечисляет все Cian-специфичные колонки."""
+    db = _mock_db(inserted=True)
+    lot = _base_lot()
+
+    save_listings(db, [lot])
+
+    sql_text = str(db.execute.call_args.args[0])
+
+    for col in (
+        "living_area_m2",
+        "bedrooms_count",
+        "balconies_count",
+        "loggias_count",
+        "description_minhash",
+        "cadastral_number",
+        "building_cadastral_number",
+        "phones",
+        "is_homeowner",
+        "is_pro_seller",
+        "bargain_allowed",
+        "sale_type",
+        "metro_stations",
+    ):
+        assert col in sql_text, f"Column '{col}' missing from INSERT SQL"