"""Tests for cian_detail.py — Stage 5 of CianScraper v1.""" from __future__ import annotations from unittest.mock import AsyncMock, MagicMock import pytest from app.services.scrapers.cian_detail import ( DetailEnrichment, _extract_agent_profile, _extract_price_changes, _extract_repair_state, _parse_views, fetch_detail, save_detail_enrichment, ) # ── _parse_views ────────────────────────────────────────────────────────────── def test_parse_views_space_formatted(): assert _parse_views("1 234") == 1234 def test_parse_views_plain(): assert _parse_views("567") == 567 def test_parse_views_none(): assert _parse_views(None) is None def test_parse_views_invalid(): assert _parse_views("invalid") is None def test_parse_views_comma_formatted(): assert _parse_views("1,234") == 1234 # ── _extract_repair_state — структурное поле в приоритете, fallback на текст (#622) ── def test_repair_state_structured_wins_over_description(): """repairType присутствует → используется он, описание игнорируется.""" offer = { "repairType": "cosmetic", # → standard "description": "Шикарный дизайнерский ремонт", # инференс дал бы excellent } assert _extract_repair_state(offer) == "standard" def test_repair_state_falls_back_to_description_when_structured_missing(): """repairType отсутствует → инференс из описания.""" offer = { "repairType": None, "description": "Сделан свежий евроремонт, заезжай и живи", } assert _extract_repair_state(offer) == "good" def test_repair_state_none_when_both_missing(): """Нет structured и нет сигнала в тексте → None (не фабрикуем).""" offer = {"description": "Светлая квартира рядом с парком"} assert _extract_repair_state(offer) is None # ── _extract_price_changes ──────────────────────────────────────────────────── def test_extract_price_changes_normalizes(): """curl_cffi flat format: {changeTime, price, diffPercent}.""" offer = { "priceChanges": [ {"changeTime": "2026-04-01T00:00:00Z", "price": 5000000, "diffPercent": -2.5}, {"changeTime": "2026-04-10T00:00:00Z", "price": 4900000, "diffPercent": -2.0}, ] } result = _extract_price_changes(offer, {}, {}) assert len(result) == 2 assert result[0]["change_time"] == "2026-04-01T00:00:00Z" assert result[0]["price_rub"] == 5000000 assert result[0]["diff_percent"] == -2.5 def test_extract_price_changes_handles_empty_offer(): assert _extract_price_changes({}, {}, {}) == [] def test_extract_price_changes_handles_none_value(): assert _extract_price_changes({"priceChanges": None}, {}, {}) == [] def test_extract_price_changes_falls_back_to_state(): state = { "priceChanges": [ {"changeTime": "2026-03-01T00:00:00Z", "price": 6000000, "diffPercent": 0}, ] } result = _extract_price_changes({}, {}, state) assert len(result) == 1 assert result[0]["price_rub"] == 6000000 def test_extract_price_changes_skips_non_dict_entries(): offer = {"priceChanges": [{"changeTime": "2026-04-01T00:00:00Z", "price": 5000000}, "bad"]} result = _extract_price_changes(offer, {}, {}) assert len(result) == 1 def test_extract_price_changes_offer_data_browser_format(): """Browser-rendered HTML: offerData.priceChanges with nested priceData.price.""" offer_data = { "priceChanges": [ { "changeTime": "2026-06-05T15:32:41Z", "priceData": {"currency": "rur", "price": 13300000}, }, { "changeTime": "2026-06-03T08:05:04Z", "priceData": {"currency": "rur", "price": 13600000}, }, ] } result = _extract_price_changes({}, offer_data, {}) assert len(result) == 2 assert result[0]["change_time"] == "2026-06-05T15:32:41Z" assert result[0]["price_rub"] == 13300000 assert result[1]["price_rub"] == 13600000 def test_extract_price_changes_offer_wins_over_offer_data(): """offer.priceChanges takes priority over offerData.priceChanges (flat format wins).""" offer = { "priceChanges": [ {"changeTime": "2026-04-01T00:00:00Z", "price": 5000000}, ] } offer_data = { "priceChanges": [ {"changeTime": "2026-06-01T00:00:00Z", "priceData": {"price": 9000000}}, ] } result = _extract_price_changes(offer, offer_data, {}) assert len(result) == 1 assert result[0]["price_rub"] == 5000000 # ── _extract_agent_profile ──────────────────────────────────────────────────── def test_extract_agent_profile_from_agent_key(): offer = { "agent": { "id": 12345, "companyName": "Новосёл", "isPro": True, "skills": [{"id": 1, "name": "Продажа"}], "accountType": "specialist", } } profile = _extract_agent_profile(offer) assert profile is not None assert profile["ext_agent_id"] == 12345 assert profile["company_name"] == "Новосёл" assert profile["is_pro"] is True assert profile["account_type"] == "specialist" def test_extract_agent_profile_from_user_key(): offer = { "user": { "cianUserId": 99999, "companyName": "Агентство Икс", "isPro": False, "accountType": "agency", } } profile = _extract_agent_profile(offer) assert profile is not None assert profile["ext_agent_id"] == 99999 assert profile["company_name"] == "Агентство Икс" def test_extract_agent_profile_returns_none_when_no_agent(): assert _extract_agent_profile({}) is None assert _extract_agent_profile({"agent": {}, "user": {}}) is None # ── DetailEnrichment defaults ───────────────────────────────────────────────── def test_dataclass_defaults(): enrich = DetailEnrichment() assert enrich.cian_id is None assert enrich.price_changes == [] assert enrich.raw_sister_states == {} assert enrich.bti_data is None assert enrich.agent_profile is None assert enrich.newbuilding_data is None # ── fetch_detail integration (mocked) ──────────────────────────────────────── @pytest.mark.asyncio async def test_fetch_detail_returns_none_on_no_state(monkeypatch): """If state extraction returns None → fetch_detail returns None gracefully.""" monkeypatch.setattr( "app.services.scrapers.cian_detail.extract_state", lambda html, mfe, key: None, ) response = MagicMock() response.status_code = 200 response.text = "" session = MagicMock() session.get = AsyncMock(return_value=response) session.close = AsyncMock() result = await fetch_detail("https://ekb.cian.ru/sale/flat/123/", session=session) assert result is None @pytest.mark.asyncio async def test_fetch_detail_returns_none_on_non_200(monkeypatch): """Non-200 HTTP response → returns None, no exception raised.""" response = MagicMock() response.status_code = 403 response.text = "" session = MagicMock() session.get = AsyncMock(return_value=response) session.close = AsyncMock() result = await fetch_detail("https://ekb.cian.ru/sale/flat/456/", session=session) assert result is None @pytest.mark.asyncio async def test_fetch_detail_extracts_core_fields(monkeypatch): """Verify core fields are extracted correctly from mocked state.""" fake_state = { "offerData": { "offer": { "cianId": 327830237, "windowsViewType": "yardAndStreet", "separateWcsCount": 1, "combinedWcsCount": 0, "ceilingHeight": 2.65, "repairType": "cosmetic", "priceChanges": [ { "changeTime": "2026-04-01T00:00:00Z", "price": 5500000, "diffPercent": -1.8, } ], "agent": { "id": 7654321, "companyName": "Первичка Плюс", "isPro": True, "skills": [], "accountType": "specialist", }, }, "stats": { "totalViewsFormattedString": "1 042", "todayViewsFormattedString": "7", }, } } monkeypatch.setattr( "app.services.scrapers.cian_detail.extract_state", lambda html, mfe, key: fake_state if key == "defaultState" else None, ) monkeypatch.setattr( "app.services.scrapers.cian_detail.extract_all_states", lambda html: {}, ) response = MagicMock() response.status_code = 200 response.text = "" session = MagicMock() session.get = AsyncMock(return_value=response) session.close = AsyncMock() result = await fetch_detail("https://ekb.cian.ru/sale/flat/327830237/", session=session) assert result is not None assert result.cian_id == 327830237 assert result.windows_view_type == "yardAndStreet" assert result.separate_wcs_count == 1 assert result.combined_wcs_count == 0 assert result.ceiling_height == 2.65 assert result.repair_type == "cosmetic" assert result.views_total == 1042 assert result.views_today == 7 assert len(result.price_changes) == 1 assert result.price_changes[0]["price_rub"] == 5500000 assert result.agent_profile is not None assert result.agent_profile["ext_agent_id"] == 7654321 assert result.agent_profile["company_name"] == "Первичка Плюс" @pytest.mark.asyncio async def test_fetch_detail_extracts_bti_from_sister_states(monkeypatch): """BTI sister state is captured into bti_data.""" fake_state = { "offerData": { "offer": { "cianId": 111111, } } } fake_all_states = { "frontend-offer-card": { "bti": { "houseData": { "seriesName": "П-44Т", "floorsCount": 17, "flatCount": 120, } } } } monkeypatch.setattr( "app.services.scrapers.cian_detail.extract_state", lambda html, mfe, key: fake_state if key == "defaultState" else None, ) monkeypatch.setattr( "app.services.scrapers.cian_detail.extract_all_states", lambda html: fake_all_states, ) response = MagicMock() response.status_code = 200 response.text = "" session = MagicMock() session.get = AsyncMock(return_value=response) session.close = AsyncMock() result = await fetch_detail("https://ekb.cian.ru/sale/flat/111111/", session=session) assert result is not None assert result.bti_data is not None assert result.bti_data["seriesName"] == "П-44Т" assert "bti" in result.raw_sister_states @pytest.mark.asyncio async def test_fetch_detail_extracts_newbuilding_link(monkeypatch): """Newbuilding reference is captured from offer.newbuilding.""" fake_state = { "offerData": { "offer": { "cianId": 222222, "newbuilding": { "id": 54016, "name": "Екатерининский парк", "url": "https://zhk-ekaterininskiy-park-ekb-i.cian.ru/", }, } } } monkeypatch.setattr( "app.services.scrapers.cian_detail.extract_state", lambda html, mfe, key: fake_state if key == "defaultState" else None, ) monkeypatch.setattr( "app.services.scrapers.cian_detail.extract_all_states", lambda html: {}, ) response = MagicMock() response.status_code = 200 response.text = "" session = MagicMock() session.get = AsyncMock(return_value=response) session.close = AsyncMock() result = await fetch_detail("https://ekb.cian.ru/sale/flat/222222/", session=session) assert result is not None assert result.newbuilding_data is not None assert result.newbuilding_data["id"] == 54016 # ── kitchen_area_m2 extraction ──────────────────────────────────────────────── @pytest.mark.asyncio async def test_fetch_detail_extracts_kitchen_area(monkeypatch): """kitchenArea из offer → kitchen_area_m2 в DetailEnrichment.""" fake_state = { "offerData": { "offer": { "cianId": 333333, "kitchenArea": 9.5, } } } monkeypatch.setattr( "app.services.scrapers.cian_detail.extract_state", lambda html, mfe, key: fake_state if key == "defaultState" else None, ) monkeypatch.setattr( "app.services.scrapers.cian_detail.extract_all_states", lambda html: {}, ) response = MagicMock() response.status_code = 200 response.text = "" session = MagicMock() session.get = AsyncMock(return_value=response) session.close = AsyncMock() result = await fetch_detail("https://ekb.cian.ru/sale/flat/333333/", session=session) assert result is not None assert result.kitchen_area_m2 == 9.5 @pytest.mark.asyncio async def test_fetch_detail_kitchen_area_none_when_missing(monkeypatch): """Если kitchenArea отсутствует в offer — kitchen_area_m2 остаётся None.""" fake_state = { "offerData": { "offer": { "cianId": 444444, } } } monkeypatch.setattr( "app.services.scrapers.cian_detail.extract_state", lambda html, mfe, key: fake_state if key == "defaultState" else None, ) monkeypatch.setattr( "app.services.scrapers.cian_detail.extract_all_states", lambda html: {}, ) response = MagicMock() response.status_code = 200 response.text = "" session = MagicMock() session.get = AsyncMock(return_value=response) session.close = AsyncMock() result = await fetch_detail("https://ekb.cian.ru/sale/flat/444444/", session=session) assert result is not None assert result.kitchen_area_m2 is None # ── save_detail_enrichment — kitchen_area_m2 persisted ─────────────────────── def test_save_detail_enrichment_writes_kitchen_area(): """save_detail_enrichment передаёт kitchen_area_m2 в UPDATE listings.""" db = MagicMock() # Мокаем fetchone для snapshot-запроса — price_rub=None → snapshot не пишется db.execute.return_value.fetchone.return_value = None enrichment = DetailEnrichment( windows_view_type=None, separate_wcs_count=None, combined_wcs_count=None, ceiling_height=None, repair_type=None, repair_state=None, kitchen_area_m2=8.3, views_total=None, views_today=None, ) save_detail_enrichment(db, listing_id=999, enrichment=enrichment) # Проверяем что UPDATE listings вызван с параметром ka=8.3 update_call_params = db.execute.call_args_list[0][0][1] assert update_call_params["ka"] == 8.3 def test_save_detail_enrichment_kitchen_area_none_passthrough(): """Если kitchen_area_m2 = None — параметр ka=None, COALESCE сохраняет существующее.""" db = MagicMock() db.execute.return_value.fetchone.return_value = None enrichment = DetailEnrichment(kitchen_area_m2=None) save_detail_enrichment(db, listing_id=1001, enrichment=enrichment) update_call_params = db.execute.call_args_list[0][0][1] assert update_call_params["ka"] is None # ── fetch_detail browser_fetcher param ─────────────────────────────────────── @pytest.mark.asyncio async def test_fetch_detail_browser_fetcher_uses_browser_html(monkeypatch): """browser_fetcher provided → HTML comes from BrowserFetcher.fetch, not curl_cffi. Simulates the browser-rendered format where priceChanges is at offerData level with nested priceData.price (verified on prod 2026-06-16). """ fake_state = { "offerData": { "offer": { "cianId": 328647045, }, # Browser-rendered: priceChanges at offerData level, priceData.price format "priceChanges": [ { "changeTime": "2026-06-05T15:32:41Z", "priceData": {"currency": "rur", "price": 13300000}, }, { "changeTime": "2026-06-03T08:05:04Z", "priceData": {"currency": "rur", "price": 13600000}, }, ], } } monkeypatch.setattr( "app.services.scrapers.cian_detail.extract_state", lambda html, mfe, key: fake_state if key == "defaultState" else None, ) monkeypatch.setattr( "app.services.scrapers.cian_detail.extract_all_states", lambda html: {}, ) # Mock BrowserFetcher: fetch() returns HTML tagged so we can verify it was called. mock_bf = MagicMock() mock_bf.fetch = AsyncMock(return_value="browser-rendered") result = await fetch_detail( "https://ekb.cian.ru/sale/flat/328647045/", browser_fetcher=mock_bf, ) assert result is not None assert result.cian_id == 328647045 # price_changes populated from browser-rendered HTML via offerData.priceChanges assert len(result.price_changes) == 2 assert result.price_changes[0]["price_rub"] == 13300000 assert result.price_changes[0]["change_time"] == "2026-06-05T15:32:41Z" assert result.price_changes[1]["price_rub"] == 13600000 # BrowserFetcher.fetch was called with the offer URL mock_bf.fetch.assert_called_once_with("https://ekb.cian.ru/sale/flat/328647045/") @pytest.mark.asyncio async def test_fetch_detail_browser_fetcher_no_curl_session_used(monkeypatch): """When browser_fetcher is given, curl_cffi AsyncSession is NOT instantiated.""" fake_state = {"offerData": {"offer": {"cianId": 555555}}} monkeypatch.setattr( "app.services.scrapers.cian_detail.extract_state", lambda html, mfe, key: fake_state if key == "defaultState" else None, ) monkeypatch.setattr( "app.services.scrapers.cian_detail.extract_all_states", lambda html: {}, ) # Patch AsyncSession so we can assert it was NOT called. mock_session_cls = MagicMock() monkeypatch.setattr( "app.services.scrapers.cian_detail.AsyncSession", mock_session_cls, ) mock_bf = MagicMock() mock_bf.fetch = AsyncMock(return_value="") result = await fetch_detail( "https://ekb.cian.ru/sale/flat/555555/", browser_fetcher=mock_bf, ) assert result is not None mock_session_cls.assert_not_called() @pytest.mark.asyncio async def test_fetch_detail_browser_fetcher_returns_none_on_exception(monkeypatch): """If BrowserFetcher.fetch raises, fetch_detail returns None (no crash).""" mock_bf = MagicMock() mock_bf.fetch = AsyncMock(side_effect=RuntimeError("browser service unavailable")) result = await fetch_detail( "https://ekb.cian.ru/sale/flat/999999/", browser_fetcher=mock_bf, ) assert result is None @pytest.mark.asyncio async def test_fetch_detail_browser_fetcher_state_missing_returns_none(monkeypatch): """browser HTML parsed but no defaultState → fetch_detail returns None.""" monkeypatch.setattr( "app.services.scrapers.cian_detail.extract_state", lambda html, mfe, key: None, ) mock_bf = MagicMock() mock_bf.fetch = AsyncMock(return_value="no state here") result = await fetch_detail( "https://ekb.cian.ru/sale/flat/777777/", browser_fetcher=mock_bf, ) assert result is None