lekss361
19c0be5d5a
Some checks failed
Deploy Trade-In / test (push) Failing after 1m30s
Deploy Trade-In / build-backend (push) Has been skipped
Deploy Trade-In / deploy (push) Has been skipped
Deploy Trade-In / changes (push) Successful in 12s
Deploy Trade-In / build-browser (push) Has been skipped
Deploy Trade-In / build-frontend (push) Has been skipped
492 lines
19 KiB
Python
492 lines
19 KiB
Python
"""Golden-parity: `scraper_kit.providers.cian.*` парсинг ≡ `app.services.scrapers.cian*`.
|
||
|
||
Strangler-инвариант (#2133 cian): новая scraper_kit-копия cian-провайдера должна давать
|
||
БАЙТ-ИДЕНТИЧНЫЙ результат парсинга старому боевому коду на одинаковом входе. Развязка
|
||
(settings→ScraperConfig, get_scraper_delay→delay_provider, cian_session→локальная копия)
|
||
НЕ меняет распарсенные данные.
|
||
|
||
Гоняем ОБА модуля на одних фикстурах и сравниваем результат:
|
||
- SERP `_parse_serp_html` (Redux state → list[ScrapedLot]) на синтетическом _cianConfig
|
||
push-HTML (вторичка + новостройка + студия), сравнение model_dump();
|
||
- SERP `_offer_to_lot` пооферно + `_format_address` (pure, восстановление дома ЖК);
|
||
- `_extract_total_offers` (totalOffers из state);
|
||
- `extract_state` — app-версия ≡ kit-версия на одном HTML (общий парсер state);
|
||
- detail `fetch_detail` (defaultState + bti sister → DetailEnrichment, dataclasses.asdict);
|
||
- valuation `_parse_valuation_state` (estimation/chart/houseInfo → CianValuationResult);
|
||
- newbuilding pure-экстракторы (_extract_chart / _extract_reliability_checks /
|
||
_extract_nested_offers / _extract_zhk_url_from_serp).
|
||
|
||
Идентичность результата = kit-парсер верно скопирован (развязка не задела логику).
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import dataclasses
|
||
import json
|
||
import os
|
||
from types import SimpleNamespace
|
||
from typing import Any
|
||
from unittest.mock import AsyncMock, MagicMock, patch
|
||
|
||
import pytest
|
||
|
||
# Старый app.services.scrapers.cian* импортирует app.core.config.settings=Settings(),
|
||
# которому нужен DATABASE_URL. Офлайн-парсинг БД не трогает — фиктивный DSN достаточен.
|
||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
|
||
|
||
# НОВЫЙ (scraper_kit) cian-провайдер.
|
||
from scraper_kit.cian_state_parser import extract_state as new_extract_state
|
||
from scraper_kit.providers.cian.detail import fetch_detail as new_fetch_detail
|
||
from scraper_kit.providers.cian.newbuilding import _extract_chart as new_extract_chart
|
||
from scraper_kit.providers.cian.newbuilding import (
|
||
_extract_nested_offers as new_extract_nested_offers,
|
||
)
|
||
from scraper_kit.providers.cian.newbuilding import (
|
||
_extract_reliability_checks as new_extract_reliability,
|
||
)
|
||
from scraper_kit.providers.cian.newbuilding import (
|
||
_extract_zhk_url_from_serp as new_extract_zhk_url,
|
||
)
|
||
from scraper_kit.providers.cian.serp import CianScraper as NewCianScraper
|
||
from scraper_kit.providers.cian.serp import _format_address as new_format_address
|
||
from scraper_kit.providers.cian.valuation import (
|
||
_parse_valuation_state as new_parse_valuation,
|
||
)
|
||
|
||
# СТАРЫЙ (app.services.scrapers) cian-провайдер.
|
||
from app.services.scrapers.cian import CianScraper as OldCianScraper
|
||
from app.services.scrapers.cian import _format_address as old_format_address
|
||
from app.services.scrapers.cian_detail import fetch_detail as old_fetch_detail
|
||
from app.services.scrapers.cian_newbuilding import _extract_chart as old_extract_chart
|
||
from app.services.scrapers.cian_newbuilding import (
|
||
_extract_nested_offers as old_extract_nested_offers,
|
||
)
|
||
from app.services.scrapers.cian_newbuilding import (
|
||
_extract_reliability_checks as old_extract_reliability,
|
||
)
|
||
from app.services.scrapers.cian_newbuilding import (
|
||
_extract_zhk_url_from_serp as old_extract_zhk_url,
|
||
)
|
||
from app.services.scrapers.cian_state_parser import extract_state as old_extract_state
|
||
from app.services.scrapers.cian_valuation import (
|
||
_parse_valuation_state as old_parse_valuation,
|
||
)
|
||
|
||
# ── Fixtures: SERP offers → _cianConfig push HTML ────────────────────────────
|
||
|
||
_CONFIG = SimpleNamespace(glitchtip_dsn=None)
|
||
|
||
|
||
def _serp_html(offers: list[dict[str, Any]]) -> str:
|
||
"""Обернуть offers в валидный _cianConfig['frontend-serp'].push() HTML."""
|
||
state = {"results": {"totalOffers": len(offers), "offers": offers}}
|
||
state_json = json.dumps(state, ensure_ascii=False)
|
||
return (
|
||
"<html><body><script>\n"
|
||
"window._cianConfig['frontend-serp'] = window._cianConfig['frontend-serp'] || [];\n"
|
||
"window._cianConfig['frontend-serp'].push({"
|
||
f"key: 'initialState', value: {state_json}, priority: 10, filter: null"
|
||
"});\n</script></body></html>"
|
||
)
|
||
|
||
|
||
def _vtorichka_offer() -> dict[str, Any]:
|
||
return {
|
||
"cianId": 328739308,
|
||
"id": 328739308,
|
||
"fullUrl": "https://ekb.cian.ru/sale/flat/328739308/",
|
||
"offerType": "flat",
|
||
"flatType": "rooms",
|
||
"category": "flatSale",
|
||
"isApartments": False,
|
||
"roomsCount": 2,
|
||
"totalArea": "52.0",
|
||
"livingArea": "35.0",
|
||
"kitchenArea": "11.5",
|
||
"balconiesCount": 1,
|
||
"loggiasCount": 0,
|
||
"bedroomsCount": None,
|
||
"hasFurniture": False,
|
||
"isSoldFurnished": False,
|
||
"decoration": "euro",
|
||
"floorNumber": 7,
|
||
"building": {
|
||
"floorsCount": 16,
|
||
"materialType": "monolithBrick",
|
||
"buildYear": 2010,
|
||
"parking": {"type": "underground"},
|
||
"totalArea": "3400",
|
||
},
|
||
"cadastralNumber": "66:41:0204016:1234",
|
||
"buildingCadastralNumber": "66:41:0204016:100",
|
||
"bargainTerms": {
|
||
"priceRur": 6500000,
|
||
"price": 6500000,
|
||
"saleType": "free",
|
||
"mortgageAllowed": True,
|
||
"bargainAllowed": True,
|
||
},
|
||
"geo": {
|
||
"coordinates": {"lat": 56.8385, "lng": 60.5940},
|
||
"address": [
|
||
{"type": "location", "fullName": "Свердловская область"},
|
||
{"type": "raion", "fullName": "Кировский"},
|
||
{"type": "street", "fullName": "улица Постовского"},
|
||
{"type": "house", "fullName": "17А"},
|
||
{"type": "metro", "fullName": "Геологическая"},
|
||
],
|
||
"undergrounds": [
|
||
{
|
||
"name": "Геологическая",
|
||
"time": 10,
|
||
"transportType": "walk",
|
||
"lineId": 10,
|
||
"lineColor": "FF0000",
|
||
"isDefault": True,
|
||
}
|
||
],
|
||
},
|
||
"newbuilding": {"id": 0, "name": ""},
|
||
"phones": [{"number": "9012345678", "countryCode": "7"}],
|
||
"userId": 13332612,
|
||
"publishedUserId": 13332612,
|
||
"isByHomeowner": True,
|
||
"isPro": False,
|
||
"isRosreestrChecked": True,
|
||
"description": "Продаётся отличная квартира с евроремонтом",
|
||
"descriptionMinhash": "abc123def456abc1",
|
||
"addedTimestamp": 1716451200,
|
||
"photos": [{"fullUrl": "https://images.cdn-cian.ru/images/328739308-1.jpg"}],
|
||
}
|
||
|
||
|
||
def _newbuilding_offer() -> dict[str, Any]:
|
||
"""Новостройка без части house в geo.address[] — жмёт _recover_newbuilding_house."""
|
||
return {
|
||
"cianId": 411112222,
|
||
"id": 411112222,
|
||
"fullUrl": "https://ekb.cian.ru/sale/flat/411112222/",
|
||
"offerType": "flat",
|
||
"flatType": "rooms",
|
||
"isApartments": False,
|
||
"roomsCount": 1,
|
||
"totalArea": "38.0",
|
||
"floorNumber": 12,
|
||
"building": {
|
||
"floorsCount": 25,
|
||
"materialType": "monolith",
|
||
"buildYear": None,
|
||
"deadline": {"year": 2026},
|
||
},
|
||
"buildingCadastralNumber": "66:41:0204016:11396",
|
||
"bargainTerms": {"priceRur": 5200000, "price": 5200000, "mortgageAllowed": True},
|
||
"geo": {
|
||
"coordinates": {"lat": 56.8000, "lng": 60.6000},
|
||
"address": [
|
||
{"type": "location", "fullName": "Свердловская область"},
|
||
{"type": "street", "fullName": "улица Новая"},
|
||
],
|
||
},
|
||
"newbuilding": {"id": 48853, "name": "ЖК Парковый квартал 29А"},
|
||
"isFromDeveloper": True,
|
||
"photos": [],
|
||
}
|
||
|
||
|
||
def _studio_offer() -> dict[str, Any]:
|
||
return {
|
||
"cianId": 500011111,
|
||
"id": 500011111,
|
||
"offerType": "flat",
|
||
"flatType": "studio",
|
||
"roomsCount": None,
|
||
"totalArea": "24.5",
|
||
"floorNumber": 3,
|
||
"building": {"floorsCount": 9, "materialType": "panel", "buildYear": 1980},
|
||
"bargainTerms": {"priceRur": 3100000, "price": 3100000},
|
||
"geo": {
|
||
"coordinates": {"lat": 56.81, "lng": 60.61},
|
||
"address": [
|
||
{"type": "street", "fullName": "улица Ленина"},
|
||
{"type": "house", "fullName": "5"},
|
||
],
|
||
},
|
||
"newbuilding": {"id": 0},
|
||
"photos": [],
|
||
}
|
||
|
||
|
||
_OFFERS = [_vtorichka_offer(), _newbuilding_offer(), _studio_offer()]
|
||
|
||
|
||
def _make_old() -> OldCianScraper:
|
||
with patch("app.services.scrapers.cian.get_scraper_delay", return_value=5.0):
|
||
return OldCianScraper()
|
||
|
||
|
||
def _make_new() -> NewCianScraper:
|
||
return NewCianScraper(config=_CONFIG)
|
||
|
||
|
||
def _dump(lots: list[Any]) -> list[dict[str, Any]]:
|
||
return [lot.model_dump() for lot in lots]
|
||
|
||
|
||
# ── SERP _parse_serp_html parity ─────────────────────────────────────────────
|
||
|
||
|
||
def test_serp_parse_serp_html_parity() -> None:
|
||
"""Redux offers → list[ScrapedLot] идентичны (вторичка + новостройка + студия)."""
|
||
html = _serp_html(_OFFERS)
|
||
old_lots = _make_old()._parse_serp_html(html)
|
||
new_lots = _make_new()._parse_serp_html(html)
|
||
|
||
assert len(old_lots) == len(new_lots) == 3
|
||
assert _dump(old_lots) == _dump(new_lots)
|
||
|
||
|
||
def test_serp_offer_to_lot_parity_per_offer() -> None:
|
||
"""Пооферный _offer_to_lot: model_dump идентичен для каждого offer."""
|
||
old, new = _make_old(), _make_new()
|
||
for offer in _OFFERS:
|
||
old_lot = old._offer_to_lot(offer)
|
||
new_lot = new._offer_to_lot(offer)
|
||
assert (old_lot is None) == (new_lot is None)
|
||
if old_lot is not None and new_lot is not None:
|
||
assert old_lot.model_dump() == new_lot.model_dump()
|
||
|
||
|
||
def test_serp_extract_total_offers_parity() -> None:
|
||
html = _serp_html(_OFFERS)
|
||
assert _make_old()._extract_total_offers(html) == _make_new()._extract_total_offers(html)
|
||
|
||
|
||
def test_format_address_parity() -> None:
|
||
"""_format_address (pure) — вкл. восстановление дома для ЖК без части house."""
|
||
building = {"houseNumber": None}
|
||
nb = {"name": "ЖК Парковый квартал 29А к1"}
|
||
cases = [
|
||
# (address_parts, is_newbuilding kwargs)
|
||
([], {}),
|
||
(
|
||
[
|
||
{"type": "location", "fullName": "Свердловская область"},
|
||
{"type": "street", "fullName": "улица Мира"},
|
||
{"type": "house", "fullName": "12"},
|
||
],
|
||
{},
|
||
),
|
||
(
|
||
[
|
||
{"type": "location", "fullName": "Свердловская область"},
|
||
{"type": "street", "fullName": "улица Новая"},
|
||
],
|
||
{
|
||
"building": building,
|
||
"building_cadastral_number": "66:41:0204016:11396",
|
||
"newbuilding": nb,
|
||
"is_newbuilding": True,
|
||
},
|
||
),
|
||
]
|
||
for parts, kwargs in cases:
|
||
assert old_format_address(parts, **kwargs) == new_format_address(parts, **kwargs)
|
||
|
||
|
||
def test_extract_state_parity() -> None:
|
||
"""app.extract_state ≡ scraper_kit.extract_state на SERP-фикстуре."""
|
||
html = _serp_html(_OFFERS)
|
||
old_state = old_extract_state(html, mfe="frontend-serp", key="initialState")
|
||
new_state = new_extract_state(html, mfe="frontend-serp", key="initialState")
|
||
assert old_state == new_state
|
||
assert new_state is not None
|
||
|
||
|
||
# ── detail fetch_detail parity ───────────────────────────────────────────────
|
||
|
||
|
||
def _detail_html() -> str:
|
||
"""defaultState + bti sister → _cianConfig['frontend-offer-card'] push HTML."""
|
||
default_state = {
|
||
"offerData": {
|
||
"offer": {
|
||
"cianId": 327830237,
|
||
"windowsViewType": "yardAndStreet",
|
||
"separateWcsCount": 1,
|
||
"combinedWcsCount": 0,
|
||
"building": {"ceilingHeight": 2.65},
|
||
"repairType": "cosmetic",
|
||
"kitchenArea": "11.5",
|
||
"description": "Отличная квартира",
|
||
"priceChanges": [
|
||
{"changeTime": "2026-04-01T00:00:00Z", "price": 5500000, "diffPercent": -1.8}
|
||
],
|
||
"agent": {
|
||
"id": 7654321,
|
||
"companyName": "Первичка Плюс",
|
||
"isPro": True,
|
||
"skills": [],
|
||
"accountType": "specialist",
|
||
},
|
||
"newbuilding": {"id": 0},
|
||
},
|
||
"stats": {
|
||
"totalViewsFormattedString": "1 042",
|
||
"todayViewsFormattedString": "7",
|
||
},
|
||
}
|
||
}
|
||
bti_state = {"houseData": {"series": "1-464", "wallMaterial": "brick"}}
|
||
ds_json = json.dumps(default_state, ensure_ascii=False)
|
||
bti_json = json.dumps(bti_state, ensure_ascii=False)
|
||
return (
|
||
"<html><body><script>\n"
|
||
"window._cianConfig['frontend-offer-card'] = "
|
||
"window._cianConfig['frontend-offer-card'] || [];\n"
|
||
"window._cianConfig['frontend-offer-card'].push({"
|
||
f"key: 'defaultState', value: {ds_json}, priority: 10, filter: null"
|
||
"});\n"
|
||
"window._cianConfig['frontend-offer-card'].push({"
|
||
f"key: 'bti', value: {bti_json}, priority: 5, filter: null"
|
||
"});\n</script></body></html>"
|
||
)
|
||
|
||
|
||
def _mock_session(html: str) -> MagicMock:
|
||
response = MagicMock()
|
||
response.status_code = 200
|
||
response.text = html
|
||
session = MagicMock()
|
||
session.get = AsyncMock(return_value=response)
|
||
session.close = AsyncMock()
|
||
return session
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_detail_fetch_parity() -> None:
|
||
"""fetch_detail (defaultState + bti) → DetailEnrichment идентичен (все поля)."""
|
||
html = _detail_html()
|
||
url = "https://ekb.cian.ru/sale/flat/327830237/"
|
||
|
||
old_res = await old_fetch_detail(url, session=_mock_session(html))
|
||
new_res = await new_fetch_detail(url, session=_mock_session(html))
|
||
|
||
assert old_res is not None
|
||
assert new_res is not None
|
||
assert dataclasses.asdict(old_res) == dataclasses.asdict(new_res)
|
||
|
||
|
||
# ── valuation _parse_valuation_state parity ──────────────────────────────────
|
||
|
||
|
||
def _valuation_state() -> dict[str, Any]:
|
||
return {
|
||
"user": {"isAuthenticated": True, "userId": 42},
|
||
"estimation": {
|
||
"sale": {
|
||
"data": {
|
||
"price": 6800000,
|
||
"accuracy": 87,
|
||
"priceFrom": 6400000,
|
||
"priceTo": 7200000,
|
||
"priceSqm": 130000,
|
||
"filtersHash": "hash-abc",
|
||
}
|
||
},
|
||
"rent": {
|
||
"data": {
|
||
"price": 42000,
|
||
"accuracy": 80,
|
||
"priceFrom": 39000,
|
||
"priceTo": 45000,
|
||
"taxPrice": 5460,
|
||
}
|
||
},
|
||
},
|
||
"estimationChart": {
|
||
"data": {
|
||
"title": {"change": "increase", "changeValue": "8,3%"},
|
||
"chartData": {
|
||
"data": [
|
||
{"date": 1714521600000, "price": 6700000, "priceFormatted": "6,7 млн"},
|
||
{"date": 1717200000000, "price": 6800000, "priceFormatted": "6,8 млн"},
|
||
]
|
||
},
|
||
}
|
||
},
|
||
"houseInfo": {
|
||
"data": {
|
||
"items": [
|
||
{"title": "Год постройки", "value": "2010"},
|
||
{"title": "Тип дома", "value": "монолит"},
|
||
]
|
||
}
|
||
},
|
||
"filters": {"houseId": 998877},
|
||
"managementCompany": {"data": {"name": "УК Ромашка", "phones": ["+73430000000"]}},
|
||
}
|
||
|
||
|
||
def test_valuation_parse_state_parity() -> None:
|
||
"""_parse_valuation_state → CianValuationResult идентичен (все поля)."""
|
||
state = _valuation_state()
|
||
old_res = old_parse_valuation(state)
|
||
new_res = new_parse_valuation(state)
|
||
assert dataclasses.asdict(old_res) == dataclasses.asdict(new_res)
|
||
|
||
|
||
# ── newbuilding pure-экстракторы parity ──────────────────────────────────────
|
||
|
||
|
||
def test_newbuilding_extractors_parity() -> None:
|
||
rv_state = {
|
||
"data": {
|
||
"priceDynamics": {
|
||
"chart": {
|
||
"data": {"labels": ["2026-01-01", "2026-02-01"], "values": [120000, 125000]}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
assert old_extract_chart(rv_state) == new_extract_chart(rv_state)
|
||
|
||
rel_state = {
|
||
"checkStatus": {"status": "reliable", "title": "Надёжный застройщик"},
|
||
"details": [{"title": "Проектная декларация", "type": "doc"}],
|
||
}
|
||
assert old_extract_reliability(rel_state) == new_extract_reliability(rel_state)
|
||
|
||
offers_state = {
|
||
"data": {
|
||
"total": {
|
||
"fromDeveloperRooms": [
|
||
{"roomType": "1", "layouts": [{"id": 1, "area": 38}]},
|
||
{"roomType": "2", "layouts": [{"id": 2, "area": 55}]},
|
||
]
|
||
}
|
||
}
|
||
}
|
||
assert old_extract_nested_offers(offers_state) == new_extract_nested_offers(offers_state)
|
||
|
||
serp_html = (
|
||
'<h1 data-name="Title">Купить квартиру в '
|
||
'<a href="https://zhk-parkovyy-kvartal-ekb-i.cian.ru/">ЖК «Парковый квартал»</a></h1>'
|
||
)
|
||
assert old_extract_zhk_url(serp_html) == new_extract_zhk_url(serp_html)
|
||
|
||
|
||
# ── strangler-guard: kit-провайдер не импортирует app.* ──────────────────────
|
||
|
||
|
||
def test_kit_cian_has_no_app_imports() -> None:
|
||
"""Ни один модуль scraper_kit.providers.cian.* не импортирует app.* (развязка)."""
|
||
import inspect
|
||
|
||
from scraper_kit.providers.cian import detail, newbuilding, serp, session, valuation
|
||
|
||
for mod in (serp, detail, newbuilding, valuation, session):
|
||
source = inspect.getsource(mod)
|
||
for line in source.splitlines():
|
||
stripped = line.strip()
|
||
assert not stripped.startswith("from app"), f"{mod.__name__}: {line!r}"
|
||
assert not stripped.startswith("import app"), f"{mod.__name__}: {line!r}"
|