diff --git a/tradein-mvp/backend/tests/fixtures/cian_serp_ekb_2room_trimmed.html b/tradein-mvp/backend/tests/fixtures/cian_serp_ekb_2room_trimmed.html new file mode 100644 index 00000000..2a488670 --- /dev/null +++ b/tradein-mvp/backend/tests/fixtures/cian_serp_ekb_2room_trimmed.html @@ -0,0 +1,911 @@ + + +
+ + diff --git a/tradein-mvp/backend/tests/fixtures/yandex_gate_ekb_novostroyki_trimmed.json b/tradein-mvp/backend/tests/fixtures/yandex_gate_ekb_novostroyki_trimmed.json new file mode 100644 index 00000000..c509a138 --- /dev/null +++ b/tradein-mvp/backend/tests/fixtures/yandex_gate_ekb_novostroyki_trimmed.json @@ -0,0 +1,120 @@ +{ + "_comment": "#2422: живой ответ gate-API Яндекс Недвижимости, снят 2026-09-17 (curl_cffi impersonate=chrome, HTTP 200): rgid=559132 type=SELL category=APARTMENT newFlat=YES page=1. Урезано: 1 из 23 entities, у entity только читаемые _entity_to_lot ключи, author без agentName (PII), mainImages до 6. Значения полей не менялись.", + "response": { + "search": { + "offers": { + "entities": [ + { + "offerId": "6216493807134609867", + "url": "//realty.yandex.ru/offer/6216493807134609867", + "price": { + "currency": "RUR", + "value": 27620850, + "period": "WHOLE_LIFE", + "unit": "WHOLE_OFFER", + "trend": "UNCHANGED", + "hasPriceHistory": false, + "valuePerPart": 366859, + "unitPerPart": "SQUARE_METER", + "valueForWhole": 27620850, + "unitForWhole": "WHOLE_OFFER", + "price": { + "value": 27620850, + "currency": "RUB", + "priceType": "PER_OFFER", + "pricingPeriod": "WHOLE_LIFE" + }, + "pricePerPart": { + "value": 366859, + "currency": "RUB", + "priceType": "PER_METER", + "pricingPeriod": "WHOLE_LIFE" + }, + "priceForWhole": { + "value": 27620850, + "currency": "RUB", + "priceType": "PER_OFFER", + "pricingPeriod": "WHOLE_LIFE" + }, + "minMonthlyMortgagePayment": 282584, + "mortgageInfo": { + "bestMortgage": { + "id": "2974521", + "payment": "282584" + }, + "minPreferentialMortgageRate": 6 + } + }, + "area": { + "value": 75.29, + "unit": "SQUARE_METER" + }, + "livingSpace": { + "value": 29.03, + "unit": "SQUARE_METER" + }, + "roomsTotal": 2, + "floorsOffered": [ + 12 + ], + "floorsTotal": 16, + "building": { + "builtYear": 2026, + "siteId": 2006795, + "siteName": "Александровский сад" + }, + "location": { + "point": { + "latitude": 56.821068, + "longitude": 60.609924, + "precision": "EXACT" + }, + "geocoderAddress": "Россия, Свердловская область, Екатеринбург, жилой район Центральный, ЖК Александровский Сад", + "streetAddress": "Свердловская область, Екатеринбург, Декабристов", + "metro": { + "metroGeoId": 101986, + "metroCityRgid": 559132, + "name": "Геологическая", + "metroTransport": "ON_FOOT", + "timeToMetro": 9, + "latitude": 56.827534, + "longitude": 60.602486, + "minTimeToMetro": 8, + "lineColors": [ + "f03d2f" + ], + "rgbColor": "f03d2f" + }, + "metroList": [ + { + "name": "Геологическая", + "timeToMetro": 9 + } + ] + }, + "creationDate": "2026-03-17T11:04:44Z", + "description": "Александровский сад настоящий бизнес-класс в центре Екатеринбурга от девелопера Forum. Забудьте о суете города. Ваш двор это пейзажный сад, где царят умиротворение и гармония. Прогулочные дорожки с мягкой вечерней подсветкой проведут по уединенным", + "author": { + "category": "DEVELOPER" + }, + "mainImages": [ + "//avatars.mds.yandex.net/get-realty-offers/20177950/9476e26d-8cbc-4b69-a7bd-03fefa0a960a/main", + "//avatars.mds.yandex.net/get-realty-offers/16258130/d185b4dc-8386-4fa1-bdf9-0865da1f8e5d/main", + "//avatars.mds.yandex.net/get-realty-offers/13970442/da0f5415-d014-466b-b5e9-0fe70d5e8680/main", + "//avatars.mds.yandex.net/get-realty-offers/14824781/f882caeb-4d40-4775-9f0a-acb01177ff0e/main", + "//avatars.mds.yandex.net/get-realty-offers/20657783/1359e480-6c21-4c07-b1de-9e36149f83fd/main", + "//avatars.mds.yandex.net/get-realty-offers/14671802/38d6dc24-8366-4460-9e99-06a1605e1919/main" + ] + } + ], + "pager": { + "page": 0, + "pageSize": 20, + "sitesPageSize": 20, + "totalItems": 32234, + "totalPages": 25 + } + } + } + } +} diff --git a/tradein-mvp/backend/tests/fixtures/yandex_gate_ekb_vtorichka_trimmed.json b/tradein-mvp/backend/tests/fixtures/yandex_gate_ekb_vtorichka_trimmed.json new file mode 100644 index 00000000..e3123148 --- /dev/null +++ b/tradein-mvp/backend/tests/fixtures/yandex_gate_ekb_vtorichka_trimmed.json @@ -0,0 +1,355 @@ +{ + "_comment": "#2422: живой ответ gate-API Яндекс Недвижимости, снят 2026-09-17 (curl_cffi impersonate=chrome, HTTP 200): rgid=559132 type=SELL category=APARTMENT newFlat=NO page=1. Урезано: 3 из 23 entities, у entity только читаемые _entity_to_lot ключи, author без agentName (PII), mainImages до 6. Значения полей не менялись.", + "response": { + "search": { + "offers": { + "entities": [ + { + "offerId": "5228381943217465492", + "url": "https://novostroi-ekb.ru/catalog/sale/vtorichka/object/1064", + "price": { + "currency": "RUR", + "value": 4400000, + "period": "WHOLE_LIFE", + "unit": "WHOLE_OFFER", + "trend": "DECREASED", + "previous": 4450000, + "hasPriceHistory": true, + "valuePerPart": 129412, + "unitPerPart": "SQUARE_METER", + "valueForWhole": 4400000, + "unitForWhole": "WHOLE_OFFER", + "price": { + "value": 4400000, + "currency": "RUB", + "priceType": "PER_OFFER", + "pricingPeriod": "WHOLE_LIFE" + }, + "pricePerPart": { + "value": 129412, + "currency": "RUB", + "priceType": "PER_METER", + "pricingPeriod": "WHOLE_LIFE" + }, + "priceForWhole": { + "value": 4400000, + "currency": "RUB", + "priceType": "PER_OFFER", + "pricingPeriod": "WHOLE_LIFE" + } + }, + "area": { + "value": 34, + "unit": "SQUARE_METER" + }, + "kitchenSpace": { + "value": 7, + "unit": "SQUARE_METER" + }, + "roomsTotal": 1, + "floorsOffered": [ + 8 + ], + "floorsTotal": 9, + "ceilingHeight": 2.5, + "building": { + "builtYear": 1978, + "buildingType": "BRICK" + }, + "location": { + "point": { + "latitude": 56.903275, + "longitude": 60.6229, + "precision": "EXACT" + }, + "geocoderAddress": "Россия, Свердловская область, Екатеринбург, Шефская улица, 85", + "streetAddress": "Шефская улица, 85", + "metro": { + "metroGeoId": 101947, + "metroCityRgid": 559132, + "name": "Проспект Космонавтов", + "metroTransport": "ON_FOOT", + "timeToMetro": 7, + "latitude": 56.901844, + "longitude": 60.61394, + "minTimeToMetro": 5, + "lineColors": [ + "f03d2f" + ], + "rgbColor": "f03d2f" + }, + "metroList": [ + { + "name": "Проспект Космонавтов", + "timeToMetro": 7 + }, + { + "name": "Машиностроителей", + "timeToMetro": 14 + } + ] + }, + "creationDate": "2026-08-19T06:23:51Z", + "description": "1 комнатная квартира 34 кв .м с развитой инфраструктурой , станция метро «КОСМАНАВТОВ» в шаговой доступности . Фукциональная планировка правильной формы, дополнительное место хранения гардеробная , в квартире сделана качественная предчистовая отделка", + "author": { + "category": "AGENCY", + "organization": "САМОЛЕТ ПЛЮС" + }, + "predictions": { + "predictedPrice": { + "min": "3795000", + "max": "4638000", + "value": "4217000" + } + }, + "mainImages": [ + "//avatars.mds.yandex.net/get-realty-offers/14333116/89f721ff-ad4b-49f1-8cec-432ad11baeab/main", + "//avatars.mds.yandex.net/get-realty-offers/13853239/caf4740f-dca8-47a3-8005-56968d20a3b1/main", + "//avatars.mds.yandex.net/get-realty-offers/14244677/f1943334-731e-4539-98f6-9d38d4a725ce/main", + "//avatars.mds.yandex.net/get-realty-offers/12726755/346626db-ba69-4a49-8940-76f08d6aa924/main", + "//avatars.mds.yandex.net/get-realty-offers/20074534/021cc849-6554-4059-9423-19193e3006b9/main", + "//avatars.mds.yandex.net/get-realty-offers/13912669/b240dbf0-1adc-4dbd-9ce8-c0ca6c478fc8/main" + ] + }, + { + "offerId": "6282460993454547968", + "url": "//realty.yandex.ru/offer/6282460993454547968", + "price": { + "currency": "RUR", + "value": 10290000, + "period": "WHOLE_LIFE", + "unit": "WHOLE_OFFER", + "trend": "UNCHANGED", + "hasPriceHistory": false, + "valuePerPart": 294000, + "unitPerPart": "SQUARE_METER", + "valueForWhole": 10290000, + "unitForWhole": "WHOLE_OFFER", + "price": { + "value": 10290000, + "currency": "RUB", + "priceType": "PER_OFFER", + "pricingPeriod": "WHOLE_LIFE" + }, + "pricePerPart": { + "value": 294000, + "currency": "RUB", + "priceType": "PER_METER", + "pricingPeriod": "WHOLE_LIFE" + }, + "priceForWhole": { + "value": 10290000, + "currency": "RUB", + "priceType": "PER_OFFER", + "pricingPeriod": "WHOLE_LIFE" + } + }, + "area": { + "value": 35, + "unit": "SQUARE_METER" + }, + "livingSpace": { + "value": 11, + "unit": "SQUARE_METER" + }, + "kitchenSpace": { + "value": 14, + "unit": "SQUARE_METER" + }, + "roomsTotal": 1, + "floorsOffered": [ + 27 + ], + "floorsTotal": 27, + "ceilingHeight": 2.75, + "building": { + "builtYear": 2023, + "buildingType": "MONOLIT", + "siteId": 1668651, + "siteName": "Парковый квартал" + }, + "location": { + "point": { + "latitude": 56.815243, + "longitude": 60.62593, + "precision": "EXACT" + }, + "geocoderAddress": "Россия, Свердловская область, Екатеринбург, Машинная улица, 4", + "streetAddress": "Машинная улица, 4", + "metro": { + "metroGeoId": 116399, + "metroCityRgid": 559132, + "name": "Чкаловская", + "metroTransport": "ON_FOOT", + "timeToMetro": 17, + "latitude": 56.807995, + "longitude": 60.610184, + "minTimeToMetro": 12, + "lineColors": [ + "f03d2f" + ], + "rgbColor": "f03d2f" + }, + "metroList": [ + { + "name": "Чкаловская", + "timeToMetro": 17 + }, + { + "name": "Ботаническая", + "timeToMetro": 13 + }, + { + "name": "Площадь 1905 Года", + "timeToMetro": 16 + }, + { + "name": "Геологическая", + "timeToMetro": 20 + } + ] + }, + "creationDate": "2026-08-02T14:04:19Z", + "description": "Продам квартиру в 10 минутах от центра города. Рядом Парк Маяковского. Хорошая транспортная доступность: до метро можно дойти пешком. Великолепный вид из окна.Хороший обзор. Высокие потолки, 2,75.Нет соседей сверху. Качественный", + "author": { + "category": "OWNER" + }, + "predictions": { + "predictedPrice": { + "min": "8741000", + "max": "10683000", + "value": "9712000" + } + }, + "mainImages": [ + "//avatars.mds.yandex.net/get-realty-offers/10211679/add.69def35c26379a08babb742069d310dc.realty-api-vos/main", + "//avatars.mds.yandex.net/get-realty-offers/14373170/add.e2e2b76847d9b69e3abe0c64efee75c4.realty-api-vos/main", + "//avatars.mds.yandex.net/get-realty-offers/14717586/add.c65fd15bb15736448bac2a597ffd0bae.realty-api-vos/main", + "//avatars.mds.yandex.net/get-realty-offers/11512705/add.b894c4e448f696251deaed148ee9c30e.realty-api-vos/main", + "//avatars.mds.yandex.net/get-realty-offers/13847384/add.f13072669ebf54647b1682fc70fa9e99.realty-api-vos/main", + "//avatars.mds.yandex.net/get-realty-offers/14527622/add.1708ed4ada10c2d407f64b0e744d8047.realty-api-vos/main" + ] + }, + { + "offerId": "8530722463088927619", + "url": "//realty.yandex.ru/offer/8530722463088927619", + "price": { + "currency": "RUR", + "value": 5600000, + "period": "WHOLE_LIFE", + "unit": "WHOLE_OFFER", + "trend": "UNCHANGED", + "hasPriceHistory": false, + "valuePerPart": 269231, + "unitPerPart": "SQUARE_METER", + "valueForWhole": 5600000, + "unitForWhole": "WHOLE_OFFER", + "price": { + "value": 5600000, + "currency": "RUB", + "priceType": "PER_OFFER", + "pricingPeriod": "WHOLE_LIFE" + }, + "pricePerPart": { + "value": 269231, + "currency": "RUB", + "priceType": "PER_METER", + "pricingPeriod": "WHOLE_LIFE" + }, + "priceForWhole": { + "value": 5600000, + "currency": "RUB", + "priceType": "PER_OFFER", + "pricingPeriod": "WHOLE_LIFE" + } + }, + "area": { + "value": 20.8, + "unit": "SQUARE_METER" + }, + "livingSpace": { + "value": 12, + "unit": "SQUARE_METER" + }, + "floorsOffered": [ + 8 + ], + "floorsTotal": 11, + "ceilingHeight": 2.7, + "building": { + "builtYear": 2021, + "buildingType": "MONOLIT_BRICK", + "siteId": 1948687, + "siteName": "Северный квартал" + }, + "location": { + "point": { + "latitude": 56.89804, + "longitude": 60.6199, + "precision": "EXACT" + }, + "geocoderAddress": "Россия, Свердловская область, Екатеринбург, переулок Ритслянда, 15", + "streetAddress": "переулок Ритслянда, 15", + "metro": { + "metroGeoId": 101947, + "metroCityRgid": 559132, + "name": "Проспект Космонавтов", + "metroTransport": "ON_FOOT", + "timeToMetro": 5, + "latitude": 56.901844, + "longitude": 60.61394, + "minTimeToMetro": 5, + "lineColors": [ + "f03d2f" + ], + "rgbColor": "f03d2f" + }, + "metroList": [ + { + "name": "Проспект Космонавтов", + "timeToMetro": 5 + }, + { + "name": "Уралмаш", + "timeToMetro": 13 + }, + { + "name": "Машиностроителей", + "timeToMetro": 12 + } + ] + }, + "creationDate": "2026-09-15T06:17:51Z", + "description": "Продается прекрасная студия в ЖК Северные кварталы, застройщик Брусника. Дом комфорт класса Площадь студии 20,8 плюс теплая лоджия 4кв.м (убрана перегородка) -благодаря этому площадь студии увеличена до 24,8кв.м - компактно и функционально для одного", + "author": { + "category": "AGENCY", + "organization": "DOMOS CLUB" + }, + "predictions": { + "predictedPrice": { + "min": "4562000", + "max": "5576000", + "value": "5069000" + } + }, + "mainImages": [ + "//avatars.mds.yandex.net/get-realty-offers/14727395/1635cddb-a57c-4cce-a6c1-c70b9aef13cd/main", + "//avatars.mds.yandex.net/get-realty-offers/14621926/3ae41eaf-5b30-46ec-b844-b40af2bb697a/main", + "//avatars.mds.yandex.net/get-realty-offers/12719313/6921877e-51f6-4d02-93ef-0395b99776bb/main", + "//avatars.mds.yandex.net/get-realty-offers/13297592/8a8d583b-d205-460e-aec8-c9cb827eae73/main", + "//avatars.mds.yandex.net/get-realty-offers/10246351/8eee4902-aa24-42dc-aaa2-9259c82146d8/main", + "//avatars.mds.yandex.net/get-realty-offers/14621926/116dc1df-3288-4236-acfe-2f660fb2ac8b/main" + ] + } + ], + "pager": { + "page": 0, + "pageSize": 20, + "sitesPageSize": 20, + "totalItems": 4077, + "totalPages": 25 + } + } + } + } +} diff --git a/tradein-mvp/backend/tests/scrapers/test_2422_kit_no_app_imports.py b/tradein-mvp/backend/tests/scrapers/test_2422_kit_no_app_imports.py new file mode 100644 index 00000000..be520a23 --- /dev/null +++ b/tradein-mvp/backend/tests/scrapers/test_2422_kit_no_app_imports.py @@ -0,0 +1,45 @@ +"""scraper_kit не импортирует app.* (#2422 п.2). + +Инвариант держал test_kit_avito_has_no_app_imports, удалённый вместе с golden-parity +(#2419). Проверка обходит весь пакет, а не только providers: импорт app внутри kit — +цикл зависимостей, где бы он ни стоял. Смотрит AST, поэтому находит и ленивый импорт +внутри функции, и не срабатывает на слово «app» в комментарии. +""" + +from __future__ import annotations + +import ast +import os +from pathlib import Path + +import scraper_kit + +# conftest.py (autouse) тянет app.core.config, которому нужен DATABASE_URL; БД не нужна. +os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test") + +KIT_ROOT = Path(scraper_kit.__file__).parent + + +def _app_imports(path: Path) -> list[str]: + found: list[str] = [] + for node in ast.walk(ast.parse(path.read_text(encoding="utf-8"), filename=str(path))): + if isinstance(node, ast.Import): + names = [alias.name for alias in node.names] + elif isinstance(node, ast.ImportFrom) and node.level == 0: + names = [node.module or ""] + else: + continue + found += [f"{n} (строка {node.lineno})" for n in names if n.split(".")[0] == "app"] + return found + + +def test_kit_has_no_app_imports() -> None: + files = sorted(KIT_ROOT.rglob("*.py")) + # Обход не пустой и дошёл до провайдеров — иначе тест зелёный по построению. + assert KIT_ROOT / "providers" / "avito" / "houses.py" in files + assert len(files) > 40 + + offenders = { + str(p.relative_to(KIT_ROOT)): imports for p in files if (imports := _app_imports(p)) + } + assert offenders == {} diff --git a/tradein-mvp/backend/tests/scrapers/test_2422_kit_parsers_on_fixtures.py b/tradein-mvp/backend/tests/scrapers/test_2422_kit_parsers_on_fixtures.py new file mode 100644 index 00000000..384674ed --- /dev/null +++ b/tradein-mvp/backend/tests/scrapers/test_2422_kit_parsers_on_fixtures.py @@ -0,0 +1,401 @@ +"""Разбор kit-парсеров на сохранённых страницах площадок: фикстура → лот (#2422, #2408). + +Golden-parity тесты сверяли kit с legacy-копией и ушли вместе с ней (#2419). После +этого разбор avito.houses, cian.serp и yandex.serp не проверял никто: правка +извлечения поля в kit проходила бы зелёной. Здесь ожидания записаны прямо значениями +из сохранённых страниц, без второй реализации в роли эталона. + +Фикстуры: + - avito_house_572538.html, avito_houses_sample.html — карточки домов Авито + (вторичка), рейтинг только в reviews.filterEntries (#1789); + - avito_novostroyka_12879380.html — карточка ЖК (developmentPage); + - avito_serp_sample.html — выдача Авито, собранная из живой страницы (#726); + - cian_serp_ekb_2room_trimmed.html, yandex_gate_ekb_*_trimmed.json — живые выдачи + 2026-09-17, урезанные до читаемых парсером полей (что и как урезано — в шапке + каждого файла; разбор урезанной и полной выдачи совпал поле в поле, кроме числа + фото). + +Не покрыто здесь, потому что уже покрыто на фикстурах в своих файлах: avito.detail +(test_avito_detail_fields_3047.py), cian.detail (test_cian_views_parse.py, +test_cian_bti_house_persist.py), domclick (test_domclick_sweep.py, +scrapers/test_domclick_detail.py), base (scrapers/test_base_clamp_date.py). +""" + +from __future__ import annotations + +import json +import os +from datetime import date +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import pytest + +# conftest.py (autouse) тянет app.core.config, которому нужен DATABASE_URL; БД не нужна. +os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test") + +from scraper_kit.providers.avito.houses import ( + HouseCatalogEnrichment, + _extract_preloaded_state, + parse_houses_state, +) +from scraper_kit.providers.avito.serp import AvitoScraper +from scraper_kit.providers.cian.serp import CianScraper +from scraper_kit.providers.yandex.serp import _extract_gate_data, _parse_gate_json + +FIXTURES = Path(__file__).parent.parent / "fixtures" + + +def _read(name: str) -> str: + return (FIXTURES / name).read_text(encoding="utf-8") + + +# ── avito.houses: карточка дома (вторичка) ───────────────────────────────────── + + +def _house(name: str, url: str) -> HouseCatalogEnrichment: + state = _extract_preloaded_state(_read(name)) + assert state is not None, name + return parse_houses_state(state, url) + + +@pytest.fixture(scope="module") +def house_572538() -> HouseCatalogEnrichment: + return _house("avito_house_572538.html", "https://www.avito.ru/catalog/houses/572538") + + +@pytest.fixture(scope="module") +def house_115188() -> HouseCatalogEnrichment: + return _house("avito_houses_sample.html", "https://www.avito.ru/catalog/houses/115188") + + +def test_avito_house_characteristics(house_572538: HouseCatalogEnrichment) -> None: + h = house_572538.house + assert h.ext_id == 572538 + assert h.ext_id_hash == "MTE3LjQ1Nzk0Mg" + assert h.full_address == "Свердловская область, Екатеринбург, ул. Сони Морозовой, 190" + assert (h.lat, h.lon) == (56.832359, 60.625583) + assert h.year_built == 1994 + assert h.total_floors == 10 + assert h.house_type == "brick" + assert h.material_floors == "Железобетонные" + assert h.hot_water == "Центральное" + assert h.passenger_elevators == 1 + assert h.cargo_elevators is None # параметра нет на странице — не 0 + assert h.has_concierge is True + assert h.has_playground is True + assert h.closed_yard is True + + +def test_avito_house_rating_from_reviews_filter_entries_1789( + house_572538: HouseCatalogEnrichment, house_115188: HouseCatalogEnrichment +) -> None: + """В обеих карточках housePage отдаёт бейдж без оценки (scoreString «—»), оценка + есть только в reviews.filterEntries[type=score]. Без запасного пути #1789 + рейтинг и число отзывов были бы None.""" + h = house_572538.house + assert h.rating_score == pytest.approx(14 / 3) + assert h.rating_string == "4,7" + assert h.reviews_count == 3 + assert [(r["score"], r["count"]) for r in h.rating_distribution] == [ + (5, 2), + (4, 1), + (3, 0), + (2, 0), + (1, 0), + ] + + h2 = house_115188.house + assert (h2.rating_score, h2.rating_string, h2.reviews_count) == (4.0, "4,0", 4) + + +def test_avito_house_reviews(house_572538: HouseCatalogEnrichment) -> None: + reviews = {r.ext_review_id: r for r in house_572538.reviews} + assert list(reviews) == [2360701, 2018777, 1757422] + + full = reviews[2018777] + assert full.author_name == "Иван" + assert full.score == 5 + assert full.rated_date == date(2025, 9, 14) + assert full.text_main is not None and full.text_main.startswith("Желательно добавить") + assert full.text_pros is not None and full.text_pros.startswith("Удобное расположение") + assert full.text_cons is not None and full.text_cons.startswith("Возможно, проблемы") + + # Секции без заголовка нет — основной текст пуст, плюсы/минусы на месте. + no_main = reviews[1757422] + assert no_main.rated_date == date(2025, 7, 9) + assert no_main.text_main is None + assert no_main.text_pros is not None and no_main.text_pros.startswith("Двор закрытый") + + # Отзыв 2360701 датирован «16 апреля» без года — такую дату разбор сейчас не + # читает (rated_date=None). Это дефект, а не контракт, поэтому здесь не закреплён. + + +def test_avito_house_reviews_next_page( + house_572538: HouseCatalogEnrichment, house_115188: HouseCatalogEnrichment +) -> None: + assert house_572538.reviews_next_page_url is None # запись pages без nextPageUrl + assert house_115188.reviews_next_page_url == ( + "/web/3/rating/model?categoryId=24&catalogId=283&paths[]=115188&limit=10&cursor=3" + "&sortBy[]=modelTypeId&sortBy[]=publishedAt&modelTypeId=14&developmentId=115188" + ) + + +def test_avito_house_placement_history(house_572538: HouseCatalogEnrichment) -> None: + history = house_572538.placement_history + assert len(history) == 8 + first = history[0] + assert first.ext_item_id == "2304054124" + assert first.title == "3-к. квартира, 108\xa0м², 5/10\xa0эт." + assert (first.start_price, first.last_price) == (11_000_000, 14_000_000) + assert first.start_price_date == date(2022, 1, 8) + assert first.last_price_date == date(2026, 1, 17) + assert first.exposure_days == 677 + assert first.removed_date is None # виджет даты снятия не отдаёт + assert first.raw_payload is not None and "itemImage" not in first.raw_payload + + +def test_avito_house_mini_serp_and_seller(house_572538: HouseCatalogEnrichment) -> None: + # Цена в виджете — строка «21 700 000 ₽» с неразрывными пробелами. + assert [(m.ext_item_id, m.price_rub) for m in house_572538.mini_serp] == [ + (8007602481, 21_700_000), + (8029678809, 21_600_000), + ] + first = house_572538.mini_serp[0] + assert first.metro_text == "Геологическая, 1,5 км" + assert first.metro_color == "#0A6F20" + assert first.seller is not None + assert first.seller.ext_seller_id == "i251003466" + assert first.seller.name == "Самолет Плюс Центральный" + assert first.seller.seller_type == "Компания" + + +def test_avito_house_recommendations(house_115188: HouseCatalogEnrichment) -> None: + recs = house_115188.recommendations + assert len(recs) == 14 + assert recs[0].title == "1-к. квартира, 30,3\xa0м², 3/5\xa0эт." + assert recs[0].price_text == "4\xa0000\xa0000 ₽" + assert recs[0].address == "Свердловская область, Екатеринбург, улица 40-летия Октября, 32А" + assert recs[0].image_url is not None and recs[0].image_url.startswith( + "https://50.img.avito.st/" + ) + + +# ── avito.houses: карточка ЖК (developmentPage) ──────────────────────────────── + + +def test_avito_novostroyka_development_page() -> None: + e = _house( + "avito_novostroyka_12879380.html", + "https://www.avito.ru/catalog/novostroyki/ekaterinburg/semeynyy_kvartal_beregovoy/12879380", + ) + h = e.house + assert h.ext_id == 12879380 + assert h.ext_id_hash == "MGDKHN70PQ" + assert h.title == "Семейный квартал «Береговой»" + assert h.short_address == "ул. Щербакова, жилые дома" + assert h.full_address is None # на странице пустая строка + assert (h.lat, h.lon) == (56.77005, 60.67244) + assert h.house_type == "monolith" + assert h.house_class == "comfort" + assert (h.passenger_elevators, h.cargo_elevators) == (42, 60) + assert (h.developer_name, h.developer_key) == ("Талан", "6749804656-3468412387") + assert h.year_built is None # дом строится: «Срок сдачи», а не год постройки + assert h.rating_score is None + assert e.reviews == [] and e.placement_history == [] + + +# ── avito.serp ───────────────────────────────────────────────────────────────── + + +def test_avito_serp_cards_to_lots() -> None: + scraper = AvitoScraper(SimpleNamespace(avito_serp_ekb_only=True)) # type: ignore[arg-type] + lots = scraper._parse_html( + _read("avito_serp_sample.html"), + source_url_base="https://www.avito.ru/ekaterinburg/kvartiry/prodam?p=1", + ) + got = [ + (lot.source_id, lot.price_rub, lot.rooms, lot.area_m2, lot.floor, lot.total_floors) + for lot in lots + ] + assert got == [ + ("8043936560", 5_500_000, 2, 54.0, 5, 9), + ("8163084615", 4_200_000, 1, 38.0, 3, 16), + ] + assert lots[0].source_url == ( + "https://www.avito.ru/ekaterinburg/kvartiry/2-k._kvartira_54m_59et._8043936560" + ) + # sortTimeStamp в миллисекундах, дата по Москве (#726). + assert [lot.listing_date for lot in lots] == [date(2023, 11, 15), date(2023, 11, 16)] + assert {lot.listing_segment for lot in lots} == {"vtorichka"} + assert scraper.parse_failures == 0 + + +# ── cian.serp ────────────────────────────────────────────────────────────────── + + +@pytest.fixture(scope="module") +def cian_lots() -> dict[str, Any]: + scraper = CianScraper(SimpleNamespace(glitchtip_dsn=None)) # type: ignore[arg-type] + html = _read("cian_serp_ekb_2room_trimmed.html") + lots = scraper._parse_serp_html(html) + assert scraper._extract_total_offers(html) == 13_895 + assert (scraper.state_extraction_attempts, scraper.state_extraction_failures) == (2, 0) + return {lot.source_id: lot for lot in lots} + + +def test_cian_serp_all_offers_parsed(cian_lots: dict[str, Any]) -> None: + assert list(cian_lots) == ["331661795", "333599038", "333279363", "321628098"] + + +def test_cian_serp_secondary_offer(cian_lots: dict[str, Any]) -> None: + lot = cian_lots["331661795"] + assert lot.source == "cian" + assert lot.source_url.startswith("https://ekb.cian.ru/sale/flat/331661795/") + assert lot.listing_segment == "vtorichka" + assert (lot.house_source, lot.house_ext_id) == ("cian", None) + assert lot.price_rub == 7_990_000 + assert (lot.rooms, lot.area_m2, lot.living_area_m2, lot.kitchen_area_m2) == ( + 2, + 47.2, + 28.0, + 7.5, + ) + assert (lot.floor, lot.total_floors, lot.year_built) == (2, 17, 1980) + assert lot.house_type == "panel" + assert lot.address == "р-н Октябрьский, мкр. Парковый, улица Большакова, 13" + assert (lot.lat, lot.lon) == (56.820035, 60.634287) + assert lot.listing_date == date(2026, 9, 14) + assert (lot.mortgage_available, lot.sale_type) == (True, "alternative") + assert (lot.is_homeowner, lot.is_pro_seller) == (False, True) + assert lot.phones == [] # #2217: телефоны не сохраняем + assert lot.metro_stations[0] == { + "name": "Геологическая", + "time": 5, + "mode": "transport", + "line_color": "00701A", + "line_id": 26, + "is_default": True, + } + # descriptionMinhash приходит списком — склеен через запятую в исходном порядке. + assert lot.description_minhash.startswith("16147461,5781654,4548911,") + assert len(lot.photo_urls) == 3 + + +def test_cian_serp_newbuilding_with_house(cian_lots: dict[str, Any]) -> None: + lot = cian_lots["333599038"] + assert lot.listing_segment == "novostroyki" + assert (lot.house_source, lot.house_ext_id) == ("cian_newbuilding", "5779142") + assert lot.address == "р-н Железнодорожный, мкр. Старая Сортировка, переулок Проходной, 1Б" + assert lot.year_built == 2028 # buildYear нет, берётся год сдачи из deadline + assert lot.house_type == "monolith" + assert (lot.price_rub, lot.area_m2, lot.floor, lot.total_floors) == (4_834_000, 36.04, 22, 32) + assert lot.sale_type == "fz214" + + +def test_cian_serp_newbuilding_decoration_and_balcony(cian_lots: dict[str, Any]) -> None: + lot = cian_lots["333279363"] + assert lot.repair_state == "good" # decoration=fine + assert lot.has_balcony is True # balconiesCount=1 + assert lot.house_type is None # materialType на странице null + assert lot.year_built == 2026 + + +def test_cian_serp_newbuilding_without_street_keeps_address_houseless( + cian_lots: dict[str, Any], +) -> None: + """В geo.address нет ни улицы, ни дома; в названии ЖК номера нет, кадастра нет. + Номер дома не выдумывается (#1773).""" + lot = cian_lots["321628098"] + assert lot.address == "р-н Чкаловский, мкр. Вторчермет" + assert (lot.house_source, lot.house_ext_id) == ("cian_newbuilding", "5705732") + + +# ── yandex.serp ──────────────────────────────────────────────────────────────── + + +def _gate(name: str) -> dict[str, Any]: + return json.loads(_read(name)) + + +@pytest.fixture(scope="module") +def yandex_secondary() -> dict[str, Any]: + lots = _parse_gate_json(_gate("yandex_gate_ekb_vtorichka_trimmed.json"), 1, "NO") + return {lot.source_id: lot for lot in lots} + + +def test_yandex_gate_pager() -> None: + extracted = _extract_gate_data(_gate("yandex_gate_ekb_vtorichka_trimmed.json")) + assert extracted is not None + entities, pager = extracted + assert len(entities) == 3 + assert (pager["totalItems"], pager["totalPages"]) == (4077, 25) + + +def test_yandex_agency_offer_with_partner_url(yandex_secondary: dict[str, Any]) -> None: + lot = yandex_secondary["5228381943217465492"] + # url в выдаче — сайт агентства; source_url строится из offerId (#2235). + assert lot.source_url == "https://realty.yandex.ru/offer/5228381943217465492/" + assert lot.yandex_offer_id == "5228381943217465492" + assert lot.listing_segment == "vtorichka" + assert (lot.price_rub, lot.price_per_m2) == (4_400_000, 129_412) + assert (lot.rooms, lot.area_m2, lot.living_area_m2, lot.kitchen_area_m2) == (1, 34.0, None, 7.0) + assert (lot.floor, lot.total_floors, lot.year_built) == (8, 9, 1978) + assert lot.house_type == "brick" # BRICK + assert lot.ceiling_height_m == 2.5 + assert lot.address == "Россия, Свердловская область, Екатеринбург, Шефская улица, 85" + assert (lot.lat, lot.lon) == (56.903275, 60.6229) + assert (lot.house_source, lot.house_ext_id) == (None, None) + assert lot.listing_date == lot.publish_date == date(2026, 8, 19) + assert (lot.is_homeowner, lot.is_pro_seller, lot.agency_name) == (None, True, "САМОЛЕТ ПЛЮС") + assert (lot.predicted_price_rub, lot.predicted_price_min, lot.predicted_price_max) == ( + 4_217_000, + 3_795_000, + 4_638_000, + ) + assert (lot.price_trend, lot.price_previous_rub) == ("DECREASED", 4_450_000) + assert lot.metro_stations[0] == {"name": "Проспект Космонавтов", "min": 7} + # В фикстуре 6 фото с «//»: берутся первые 5 со схемой. + assert len(lot.photo_urls) == 5 + assert all(u.startswith("https://avatars.mds.yandex.net/") for u in lot.photo_urls) + + +def test_yandex_owner_offer_in_complex(yandex_secondary: dict[str, Any]) -> None: + lot = yandex_secondary["6282460993454547968"] + assert lot.source_url == "https://realty.yandex.ru/offer/6282460993454547968" + assert (lot.is_homeowner, lot.is_pro_seller, lot.agency_name) == (True, None, None) + assert (lot.house_source, lot.house_ext_id) == ("yandex_realty_nb", "1668651") + assert lot.house_type == "monolith" # MONOLIT + assert (lot.floor, lot.total_floors) == (27, 27) + assert (lot.living_area_m2, lot.kitchen_area_m2) == (11.0, 14.0) + + +def test_yandex_studio_without_rooms_total(yandex_secondary: dict[str, Any]) -> None: + lot = yandex_secondary["8530722463088927619"] + assert lot.rooms == 0 # roomsTotal отсутствует — студия + assert lot.area_m2 == 20.8 + assert lot.kitchen_area_m2 is None + assert lot.house_type == "monolith_brick" # MONOLIT_BRICK + + +def test_yandex_newbuilding_without_valuation() -> None: + lots = _parse_gate_json(_gate("yandex_gate_ekb_novostroyki_trimmed.json"), 1, "YES") + assert len(lots) == 1 + lot = lots[0] + assert lot.source_id == "6216493807134609867" + assert lot.listing_segment == "novostroyki" + assert lot.price_rub == 27_620_850 + assert (lot.predicted_price_rub, lot.predicted_price_min, lot.predicted_price_max) == ( + None, + None, + None, + ) + assert lot.ceiling_height_m is None + assert lot.house_type is None + assert (lot.house_source, lot.house_ext_id) == ("yandex_realty_nb", "2006795") + assert (lot.is_homeowner, lot.is_pro_seller, lot.agency_name) == (None, None, None) + assert lot.address == ( + "Россия, Свердловская область, Екатеринбург, жилой район Центральный, " + "ЖК Александровский Сад" + )