"""Smoke тесты для cian.py SERP scraper (Stage 3 рефактор). Проверяют парсинг 137 полей из синтетического Redux state без сетевых запросов. Fixture формирует минимальный JS push() conforming to _cianConfig pattern. """ import json import pytest from app.services.scrapers.cian import CianScraper, _format_address # ── Fixture helpers ────────────────────────────────────────────────────────── def _make_cian_serp_html(offers: list[dict]) -> str: """Обернуть offers в валидный _cianConfig['frontend-serp'].push() HTML.""" state = { "results": { "totalOffers": len(offers), "offers": offers, } } state_json = json.dumps(state, ensure_ascii=False) return f""" """ def _make_full_offer( cian_id: int = 328739308, price_rur: int = 6500000, rooms: int = 2, total_area: str = "52.0", living_area: str = "35.0", kitchen_area: str = "11.5", floor: int = 7, floors_count: int = 16, year_built: int = 2010, material_type: str = "panel", lat: float = 56.8385, lng: float = 60.5940, cadastral_number: str = "66:41:0204016:1234", building_cadastral_number: str = "66:41:0204016:100", description: str = "Продаётся отличная квартира", description_minhash: str = "abc123def456abc1", newbuilding_id: int = 0, has_phones: bool = True, is_homeowner: bool = True, is_pro: bool = False, bargain_allowed: bool = True, sale_type: str = "free", ) -> dict: """Синтетический Cian offer с полным набором полей (137 полей subset).""" offer: dict = { "cianId": cian_id, "id": cian_id, "fullUrl": f"https://ekb.cian.ru/sale/flat/{cian_id}/", "category": "flatSale", "dealType": "sale", "offerType": "flat", "flatType": "rooms", "isApartments": False, "roomsCount": rooms, "status": "published", # Area "totalArea": total_area, "livingArea": living_area, "kitchenArea": kitchen_area, "balconiesCount": 1, "loggiasCount": 0, "bedroomsCount": None, "hasFurniture": False, "isSoldFurnished": False, "decoration": None, # Floor / building "floorNumber": floor, "building": { "floorsCount": floors_count, "materialType": material_type, "buildYear": year_built, "deadline": None, "parking": None, }, # Cadastral "cadastralNumber": cadastral_number, "buildingCadastralNumber": building_cadastral_number, # Price "bargainTerms": { "priceRur": price_rur, "price": price_rur, "currency": "rur", "saleType": sale_type, "mortgageAllowed": True, "bargainAllowed": bargain_allowed, "priceType": "all", }, "formattedShortPrice": f"{price_rur // 1000000} млн ₽", # Geo "geo": { "coordinates": {"lat": lat, "lng": lng}, "address": [ {"type": "location", "name": "Свердловская область"}, {"type": "city", "name": "Екатеринбург"}, {"type": "street", "name": "улица Постовского"}, {"type": "house", "name": "17А"}, ], "undergrounds": [ { "name": "Геологическая", "time": 10, "transportType": "walk", "lineId": 10, "lineColor": "FF0000", "isDefault": True, "id": 100, } ], "districts": [], }, # Newbuilding "newbuilding": {"id": newbuilding_id, "name": ""}, "buildersIds": [], # Seller "phones": [{"number": "9012345678", "countryCode": "7"}] if has_phones else [], "userId": 13332612, "cianUserId": 13332612, "publishedUserId": 13332612, "isByHomeowner": is_homeowner, "isByCommercialOwner": False, "isCianPartner": False, "isPro": is_pro, "isCalltrackingEnabled": False, # Description "description": description, "descriptionMinhash": description_minhash, "isDuplicatedDescription": False, # Time "addedTimestamp": 1716451200, # 2024-05-23 00:00:00 UTC "added": "вчера", # Photos "photos": [ { "id": 1001, "fullUrl": f"https://images.cdn-cian.ru/images/{cian_id}-1.jpg", "thumbnailUrl": f"https://images.cdn-cian.ru/images/{cian_id}-2.jpg", } ], # Verification "isRosreestrChecked": True, "isLayoutApproved": None, # Misc "isNew": False, "isTop3": False, "isPremium": False, } return offer # ── Tests ──────────────────────────────────────────────────────────────────── def test_cian_serp_parses_full_offer(): """Базовый тест: полный offer → ScrapedLot с ключевыми полями.""" offer = _make_full_offer() html = _make_cian_serp_html([offer]) scraper = CianScraper() lots = scraper._parse_serp_html(html) assert len(lots) == 1 lot = lots[0] assert lot.source == "cian" assert lot.source_id == "328739308" assert lot.source_url == "https://ekb.cian.ru/sale/flat/328739308/" def test_cian_serp_coords_no_jitter(): """Координаты берутся точно из state — NO anchor jitter.""" offer = _make_full_offer(lat=56.8385, lng=60.5940) html = _make_cian_serp_html([offer]) scraper = CianScraper() lots = scraper._parse_serp_html(html) assert len(lots) == 1 lot = lots[0] assert lot.lat == pytest.approx(56.8385, abs=1e-9) assert lot.lon == pytest.approx(60.5940, abs=1e-9) def test_cian_serp_kitchen_area_in_raw_payload(): """kitchen_area_m2 не в ScrapedLot, но сохраняется в raw_payload.""" offer = _make_full_offer(kitchen_area="11.5") html = _make_cian_serp_html([offer]) scraper = CianScraper() lots = scraper._parse_serp_html(html) lot = lots[0] assert lot.raw_payload is not None assert lot.raw_payload["kitchen_area_m2"] == "11.5" def test_cian_serp_description_minhash(): """description_minhash берётся из Cian state, если есть.""" offer = _make_full_offer( description_minhash="cian_minhash_abc123", description="Тест описания", ) html = _make_cian_serp_html([offer]) scraper = CianScraper() lots = scraper._parse_serp_html(html) lot = lots[0] assert lot.description_minhash == "cian_minhash_abc123" def test_cian_serp_description_minhash_fallback(): """Если Cian не дал minhash — вычисляем SHA1 из description.""" offer = _make_full_offer(description_minhash="", description="Уникальное описание квартиры") offer["descriptionMinhash"] = "" # явно пусто html = _make_cian_serp_html([offer]) scraper = CianScraper() lots = scraper._parse_serp_html(html) lot = lots[0] assert lot.description_minhash is not None assert len(lot.description_minhash) == 32 # SHA1 hex [:32] def test_cian_serp_phones(): """phones — список dicts из state.""" offer = _make_full_offer(has_phones=True) html = _make_cian_serp_html([offer]) scraper = CianScraper() lots = scraper._parse_serp_html(html) lot = lots[0] assert isinstance(lot.phones, list) assert len(lot.phones) == 1 assert lot.phones[0]["countryCode"] == "7" assert lot.phones[0]["number"] == "9012345678" def test_cian_serp_house_ext_id_newbuilding(): """house_ext_id заполнен если offer.newbuilding.id > 0.""" offer = _make_full_offer(newbuilding_id=54016) html = _make_cian_serp_html([offer]) scraper = CianScraper() lots = scraper._parse_serp_html(html) lot = lots[0] assert lot.house_source == "cian_newbuilding" assert lot.house_ext_id == "54016" assert lot.listing_segment == "novostroyki" def test_cian_serp_vtorichka_no_house_ext_id(): """Вторичка (newbuilding.id == 0) — house_ext_id None.""" offer = _make_full_offer(newbuilding_id=0) html = _make_cian_serp_html([offer]) scraper = CianScraper() lots = scraper._parse_serp_html(html) lot = lots[0] assert lot.house_source == "cian" assert lot.house_ext_id is None assert lot.listing_segment == "vtorichka" def test_cian_serp_metro_stations(): """metro_stations — список dicts с name/time/mode/line_color.""" offer = _make_full_offer() html = _make_cian_serp_html([offer]) scraper = CianScraper() lots = scraper._parse_serp_html(html) lot = lots[0] assert isinstance(lot.metro_stations, list) assert len(lot.metro_stations) == 1 station = lot.metro_stations[0] assert station["name"] == "Геологическая" assert station["time"] == 10 assert station["mode"] == "walk" assert station["line_color"] == "FF0000" def test_cian_serp_cadastral_numbers(): """Кадастровые номера квартиры и здания.""" offer = _make_full_offer( cadastral_number="66:41:0204016:1234", building_cadastral_number="66:41:0204016:100", ) html = _make_cian_serp_html([offer]) scraper = CianScraper() lots = scraper._parse_serp_html(html) lot = lots[0] assert lot.cadastral_number == "66:41:0204016:1234" assert lot.building_cadastral_number == "66:41:0204016:100" # kadastr_num (base field) тоже заполнен assert lot.kadastr_num == "66:41:0204016:1234" def test_cian_serp_is_homeowner_seller(): """is_homeowner и is_pro_seller из offer.""" offer = _make_full_offer(is_homeowner=True, is_pro=False) html = _make_cian_serp_html([offer]) scraper = CianScraper() lots = scraper._parse_serp_html(html) lot = lots[0] assert lot.is_homeowner is True assert lot.is_pro_seller is False def test_cian_serp_bargain_and_sale_type(): """bargain_allowed и sale_type из bargainTerms.""" offer = _make_full_offer(bargain_allowed=True, sale_type="alternative") html = _make_cian_serp_html([offer]) scraper = CianScraper() lots = scraper._parse_serp_html(html) lot = lots[0] assert lot.bargain_allowed is True assert lot.sale_type == "alternative" def test_cian_serp_building_fields(): """Поля здания: year_built, house_type, total_floors.""" offer = _make_full_offer(year_built=1988, material_type="brick", floors_count=9) html = _make_cian_serp_html([offer]) scraper = CianScraper() lots = scraper._parse_serp_html(html) lot = lots[0] assert lot.year_built == 1988 assert lot.house_type == "brick" assert lot.total_floors == 9 def test_cian_serp_living_area(): """living_area_m2 из offer.livingArea.""" offer = _make_full_offer(living_area="35.0") html = _make_cian_serp_html([offer]) scraper = CianScraper() lots = scraper._parse_serp_html(html) lot = lots[0] assert lot.living_area_m2 == pytest.approx(35.0) def test_cian_serp_state_missing_returns_empty(): """Если state нет в HTML — возвращаем пустой список.""" html = "

no cian state here

" scraper = CianScraper() lots = scraper._parse_serp_html(html) assert lots == [] def test_cian_serp_skips_offer_without_price(): """Offer без цены — пропускаем.""" offer = _make_full_offer() del offer["bargainTerms"]["priceRur"] del offer["bargainTerms"]["price"] html = _make_cian_serp_html([offer]) scraper = CianScraper() lots = scraper._parse_serp_html(html) assert lots == [] def test_cian_serp_multiple_offers(): """Несколько offers → список ScrapedLot.""" offers = [_make_full_offer(cian_id=100 + i) for i in range(5)] html = _make_cian_serp_html(offers) scraper = CianScraper() lots = scraper._parse_serp_html(html) assert len(lots) == 5 ids = [lot.source_id for lot in lots] assert sorted(ids) == ["100", "101", "102", "103", "104"] def test_cian_serp_photo_urls(): """photo_urls — полные URL из offer.photos[].fullUrl.""" offer = _make_full_offer(cian_id=999) html = _make_cian_serp_html([offer]) scraper = CianScraper() lots = scraper._parse_serp_html(html) lot = lots[0] assert len(lot.photo_urls) == 1 assert "cdn-cian.ru" in lot.photo_urls[0] assert "999-1.jpg" in lot.photo_urls[0] def test_cian_serp_no_coords_does_not_crash(): """Если координаты отсутствуют — lot.lat/lon = None, не краш.""" offer = _make_full_offer() del offer["geo"]["coordinates"] html = _make_cian_serp_html([offer]) scraper = CianScraper() lots = scraper._parse_serp_html(html) assert len(lots) == 1 assert lots[0].lat is None assert lots[0].lon is None # ── Address formatter tests ────────────────────────────────────────────────── def test_format_address_standard(): """Стандартный geo.address[].""" parts = [ {"type": "location", "name": "Свердловская область"}, {"type": "city", "name": "Екатеринбург"}, {"type": "district", "name": "Чкаловский"}, {"type": "street", "name": "улица Постовского"}, {"type": "house", "name": "17А"}, {"type": "metro", "name": "Геологическая"}, ] result = _format_address(parts) # location и metro пропускаем assert "Свердловская" not in result assert "Геологическая" not in result assert "Екатеринбург" in result assert "17А" in result def test_format_address_empty(): """Пустой address[] → fallback string.""" assert _format_address([]) == "Екатеринбург (Cian)"