From 702cb2b7d0b545826717336c5bbfd1862d437e3a Mon Sep 17 00:00:00 2001 From: lekss361 Date: Sat, 23 May 2026 15:58:02 +0300 Subject: [PATCH] feat(tradein): extend save_listings INSERT to persist Cian-specific columns (PR #450 follow-up) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extends base.py save_listings() to persist all 13 Cian-specific columns added to ScrapedLot in PR #447 and listings DDL in migration 019: living_area_m2, bedrooms_count, balconies_count, loggias_count, description_minhash, cadastral_number, building_cadastral_number, phones, is_homeowner, is_pro_seller, bargain_allowed, sale_type, metro_stations. ON CONFLICT DO UPDATE SET extended to refresh all new columns on re-scrape. Avito/DomRF backward compat: optional fields default None → SQL NULL. Adds 6 unit tests (test_base_save_listings.py) covering Cian INSERT, Avito null compat, ON CONFLICT UPDATE SQL text, empty phones/metro → NULL. --- .../backend/app/services/scrapers/base.py | 38 ++- .../backend/tests/test_base_save_listings.py | 260 ++++++++++++++++++ 2 files changed, 297 insertions(+), 1 deletion(-) create mode 100644 tradein-mvp/backend/tests/test_base_save_listings.py diff --git a/tradein-mvp/backend/app/services/scrapers/base.py b/tradein-mvp/backend/app/services/scrapers/base.py index 5f3285df..de649493 100644 --- a/tradein-mvp/backend/app/services/scrapers/base.py +++ b/tradein-mvp/backend/app/services/scrapers/base.py @@ -223,6 +223,10 @@ def save_listings(db: Session, lots: list[ScrapedLot]) -> tuple[int, int]: house_source, house_ext_id, house_url, listing_segment, price_rub, price_per_m2, listing_date, days_on_market, photo_urls, raw_payload, + living_area_m2, bedrooms_count, balconies_count, loggias_count, + description_minhash, cadastral_number, building_cadastral_number, + phones, is_homeowner, is_pro_seller, + bargain_allowed, sale_type, metro_stations, scraped_at, last_seen_at ) VALUES ( :source, :source_url, :source_id, :dedup, @@ -234,6 +238,10 @@ def save_listings(db: Session, lots: list[ScrapedLot]) -> tuple[int, int]: :listing_date, :days_on_market, CAST(:photos AS jsonb), CAST(:raw AS jsonb), + :living_area_m2, :bedrooms_count, :balconies_count, :loggias_count, + :description_minhash, :cadastral_number, :building_cadastral_number, + CAST(:phones AS jsonb), :is_homeowner, :is_pro_seller, + :bargain_allowed, :sale_type, CAST(:metro_stations AS jsonb), NOW(), NOW() ) ON CONFLICT (dedup_hash) DO UPDATE @@ -241,7 +249,21 @@ def save_listings(db: Session, lots: list[ScrapedLot]) -> tuple[int, int]: is_active = true, -- если цена изменилась — обновляем price_rub = EXCLUDED.price_rub, - price_per_m2 = EXCLUDED.price_per_m2 + price_per_m2 = EXCLUDED.price_per_m2, + -- Cian-specific: обновляем при каждом re-scrape + living_area_m2 = EXCLUDED.living_area_m2, + bedrooms_count = EXCLUDED.bedrooms_count, + balconies_count = EXCLUDED.balconies_count, + loggias_count = EXCLUDED.loggias_count, + description_minhash = EXCLUDED.description_minhash, + cadastral_number = EXCLUDED.cadastral_number, + building_cadastral_number = EXCLUDED.building_cadastral_number, + phones = EXCLUDED.phones, + is_homeowner = EXCLUDED.is_homeowner, + is_pro_seller = EXCLUDED.is_pro_seller, + bargain_allowed = EXCLUDED.bargain_allowed, + sale_type = EXCLUDED.sale_type, + metro_stations = EXCLUDED.metro_stations RETURNING (xmax = 0) AS inserted """ ), @@ -272,6 +294,20 @@ def save_listings(db: Session, lots: list[ScrapedLot]) -> tuple[int, int]: "days_on_market": lot.days_on_market, "photos": _to_json(lot.photo_urls), "raw": _to_json(lot.raw_payload) if lot.raw_payload else None, + # Cian-specific columns — None for non-Cian scrapers (→ SQL NULL) + "living_area_m2": lot.living_area_m2, + "bedrooms_count": lot.bedrooms_count, + "balconies_count": lot.balconies_count, + "loggias_count": lot.loggias_count, + "description_minhash": lot.description_minhash, + "cadastral_number": lot.cadastral_number, + "building_cadastral_number": lot.building_cadastral_number, + "phones": _to_json(lot.phones) if lot.phones else None, + "is_homeowner": lot.is_homeowner, + "is_pro_seller": lot.is_pro_seller, + "bargain_allowed": lot.bargain_allowed, + "sale_type": lot.sale_type, + "metro_stations": _to_json(lot.metro_stations) if lot.metro_stations else None, }, ).fetchone() diff --git a/tradein-mvp/backend/tests/test_base_save_listings.py b/tradein-mvp/backend/tests/test_base_save_listings.py new file mode 100644 index 00000000..ab026e60 --- /dev/null +++ b/tradein-mvp/backend/tests/test_base_save_listings.py @@ -0,0 +1,260 @@ +"""Тесты для base.py save_listings — проверяем что Cian-специфичные колонки +передаются в INSERT и ON CONFLICT DO UPDATE SET. + +Используем unittest.mock для замены db.execute — без реальной БД. +""" + +import json +from unittest.mock import MagicMock, call, patch + +import pytest + +from app.services.scrapers.base import ScrapedLot, save_listings + + +# ── Helpers ────────────────────────────────────────────────────────────────── + + +def _mock_db(inserted: bool = True) -> MagicMock: + """Создать mock Session с fetchone() → (inserted=,).""" + row = MagicMock() + row.inserted = inserted + + result = MagicMock() + result.fetchone.return_value = row + + db = MagicMock() + db.execute.return_value = result + return db + + +def _base_lot(**overrides) -> ScrapedLot: + """Минимальный валидный ScrapedLot — только обязательные поля.""" + defaults = { + "source": "avito", + "source_url": "https://www.avito.ru/ekaterinburg/kvartiry/1-k_987654", + "source_id": "987654", + "price_rub": 4_500_000, + } + defaults.update(overrides) + return ScrapedLot(**defaults) + + +def _cian_lot(**overrides) -> ScrapedLot: + """ScrapedLot с полным набором Cian-специфичных полей.""" + base = { + "source": "cian", + "source_url": "https://ekb.cian.ru/sale/flat/123456/", + "source_id": "123456", + "price_rub": 6_200_000, + "address": "Екатеринбург, улица Постовского, 17А", + "lat": 56.8385, + "lon": 60.5940, + "rooms": 2, + "area_m2": 52.0, + "floor": 7, + "total_floors": 16, + # Cian-specific + "living_area_m2": 35.0, + "bedrooms_count": 2, + "balconies_count": 1, + "loggias_count": 0, + "description_minhash": "abc123def456abc1", + "cadastral_number": "66:41:0204016:1234", + "building_cadastral_number": "66:41:0204016:100", + "phones": [{"countryCode": "7", "number": "9012345678"}], + "is_homeowner": True, + "is_pro_seller": False, + "bargain_allowed": True, + "sale_type": "free", + "metro_stations": [{"name": "Геологическая", "time": 10, "mode": "walk"}], + "house_source": "cian", + "house_ext_id": None, + "listing_segment": "vtorichka", + } + base.update(overrides) + return ScrapedLot(**base) + + +# ── Utility: extract params from mock db call ──────────────────────────────── + + +def _get_execute_params(db: MagicMock) -> dict: + """Получить params dict из первого db.execute() вызова.""" + assert db.execute.called, "db.execute was not called" + _sql, params = db.execute.call_args.args + return params + + +# ── Test 1: Cian lot → все новые колонки попадают в params ────────────────── + + +def test_save_listings_persists_cian_specific_fields(): + """Cian lot с заполненными полями → все Cian колонки в params INSERT.""" + db = _mock_db(inserted=True) + lot = _cian_lot() + + inserted, updated = save_listings(db, [lot]) + + assert inserted == 1 + assert updated == 0 + + params = _get_execute_params(db) + + assert params["living_area_m2"] == 35.0 + assert params["bedrooms_count"] == 2 + assert params["balconies_count"] == 1 + assert params["loggias_count"] == 0 + assert params["description_minhash"] == "abc123def456abc1" + assert params["cadastral_number"] == "66:41:0204016:1234" + assert params["building_cadastral_number"] == "66:41:0204016:100" + assert params["is_homeowner"] is True + assert params["is_pro_seller"] is False + assert params["bargain_allowed"] is True + assert params["sale_type"] == "free" + + # phones и metro_stations — JSON-строки, проверяем десериализацией + phones = json.loads(params["phones"]) + assert phones == [{"countryCode": "7", "number": "9012345678"}] + + metro = json.loads(params["metro_stations"]) + assert metro[0]["name"] == "Геологическая" + assert metro[0]["time"] == 10 + + +# ── Test 2: Avito lot → Cian-специфичные params = None (NULL) ─────────────── + + +def test_save_listings_avito_compat_cian_cols_are_null(): + """Avito lot не имеет Cian-полей → все Cian params = None (→ SQL NULL). + + Проверяет backward compat — Avito / DomRF scrapers не ломаются. + """ + db = _mock_db(inserted=True) + lot = _base_lot(source="avito", source_id="987654") + + inserted, updated = save_listings(db, [lot]) + + assert inserted == 1 + + params = _get_execute_params(db) + + # ScrapedLot.living_area_m2 defaults to None + assert params["living_area_m2"] is None + assert params["bedrooms_count"] is None + assert params["balconies_count"] is None + assert params["loggias_count"] is None + assert params["description_minhash"] is None + assert params["cadastral_number"] is None + assert params["building_cadastral_number"] is None + assert params["is_homeowner"] is None + assert params["is_pro_seller"] is None + assert params["bargain_allowed"] is None + assert params["sale_type"] is None + # phones=[] (default_factory=list) → None (пустой список → None в helper) + assert params["phones"] is None + # metro_stations=[] → None + assert params["metro_stations"] is None + + +# ── Test 3: ON CONFLICT — при updated=False все новые поля в UPDATE SET ────── + + +def test_save_listings_on_conflict_updates_cian_cols(): + """ON CONFLICT path: xmax != 0 → updated counter. Params верны для UPDATE. + + Проверяем что SQL-текст содержит все новые колонки в DO UPDATE SET. + """ + db = _mock_db(inserted=False) # xmax != 0 → updated + lot = _cian_lot( + source_id="111", + cadastral_number="66:41:0000000:NEW", + description_minhash="newhash", + ) + + inserted, updated = save_listings(db, [lot]) + + assert inserted == 0 + assert updated == 1 + + # Проверяем SQL-текст на наличие DO UPDATE SET с новыми колонками + sql_obj = db.execute.call_args.args[0] + sql_text = str(sql_obj) + + assert "cadastral_number = EXCLUDED.cadastral_number" in sql_text + assert "description_minhash = EXCLUDED.description_minhash" in sql_text + assert "building_cadastral_number = EXCLUDED.building_cadastral_number" in sql_text + assert "phones = EXCLUDED.phones" in sql_text + assert "is_homeowner = EXCLUDED.is_homeowner" in sql_text + assert "is_pro_seller = EXCLUDED.is_pro_seller" in sql_text + assert "bargain_allowed = EXCLUDED.bargain_allowed" in sql_text + assert "sale_type = EXCLUDED.sale_type" in sql_text + assert "metro_stations = EXCLUDED.metro_stations" in sql_text + assert "living_area_m2 = EXCLUDED.living_area_m2" in sql_text + assert "bedrooms_count = EXCLUDED.bedrooms_count" in sql_text + assert "balconies_count = EXCLUDED.balconies_count" in sql_text + assert "loggias_count = EXCLUDED.loggias_count" in sql_text + + # Params с новым значением cadastral_number + params = _get_execute_params(db) + assert params["cadastral_number"] == "66:41:0000000:NEW" + assert params["description_minhash"] == "newhash" + + +# ── Test 4: empty list → early return, no db.execute call ─────────────────── + + +def test_save_listings_empty_lots(): + """Пустой список → (0, 0), db.execute не вызывается.""" + db = _mock_db() + + inserted, updated = save_listings(db, []) + + assert inserted == 0 + assert updated == 0 + db.execute.assert_not_called() + + +# ── Test 5: phones=[] и metro_stations=[] → None в params ─────────────────── + + +def test_save_listings_empty_phones_and_metro_become_null(): + """Если phones=[] / metro_stations=[] — передаём None (не '[]' строку).""" + db = _mock_db(inserted=True) + lot = _cian_lot(phones=[], metro_stations=[]) + + save_listings(db, [lot]) + + params = _get_execute_params(db) + assert params["phones"] is None + assert params["metro_stations"] is None + + +# ── Test 6: INSERT SQL содержит все новые колонки ──────────────────────────── + + +def test_save_listings_sql_contains_all_new_columns(): + """SQL INSERT перечисляет все Cian-специфичные колонки.""" + db = _mock_db(inserted=True) + lot = _base_lot() + + save_listings(db, [lot]) + + sql_text = str(db.execute.call_args.args[0]) + + for col in ( + "living_area_m2", + "bedrooms_count", + "balconies_count", + "loggias_count", + "description_minhash", + "cadastral_number", + "building_cadastral_number", + "phones", + "is_homeowner", + "is_pro_seller", + "bargain_allowed", + "sale_type", + "metro_stations", + ): + assert col in sql_text, f"Column '{col}' missing from INSERT SQL" -- 2.45.3