lekss361
35f5c3426b
All checks were successful
Deploy Trade-In / changes (push) Successful in 12s
Deploy Trade-In / build-frontend (push) Has been skipped
Deploy Trade-In / build-browser (push) Has been skipped
Deploy Trade-In / test (push) Successful in 4m50s
Deploy Trade-In / build-backend (push) Successful in 1m34s
Deploy Trade-In / deploy (push) Successful in 1m34s
106 lines
5.3 KiB
Python
106 lines
5.3 KiB
Python
"""Audit-scrapers finding 2: Cian totalOffers vs results.offers length mismatch.
|
||
|
||
`_parse_serp_html` извлекает `totalOffers` и `results.offers` из ОДНОГО Redux
|
||
state-блоба (одна SSR-выдача). Раньше `results.offers` пустой при `totalOffers>0`
|
||
логировался WARNING'ом и тихо возвращался `[]` — не считался schema-regression, не
|
||
попадал в мониторинг (`_report_schema_regression`/Glitchtip).
|
||
|
||
Порог: 0 vs >0 — единственный позиционно-независимый сигнал, который можно
|
||
проверить без номера страницы внутри `_parse_serp_html` (эта функция не знает,
|
||
какая это страница пагинации — дробный порог типа "< 50% от totalOffers" ложно
|
||
сработал бы на легитимной последней частичной странице exhaustive-пагинации,
|
||
которую эта функция не различает). totalOffers=0 (реально пустой поиск) НЕ
|
||
считается регрессией.
|
||
|
||
Refs: audit-scrapers 2026-07-26, finding 2 (low).
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import os
|
||
from unittest.mock import MagicMock, patch
|
||
|
||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
|
||
|
||
from scraper_kit.providers.cian.serp import CianScraper
|
||
|
||
from app.services.scraper_adapters import RealScraperConfig
|
||
|
||
|
||
def _scraper() -> CianScraper:
|
||
return CianScraper(RealScraperConfig())
|
||
|
||
|
||
def test_total_offers_positive_but_offers_empty_reports_regression() -> None:
|
||
"""totalOffers=5, results.offers=[] — internal contradiction, must report."""
|
||
s = _scraper()
|
||
state = {"results": {"totalOffers": 5, "offers": []}}
|
||
with patch("scraper_kit.providers.cian.serp.extract_state", return_value=state):
|
||
with patch.object(s, "_report_schema_regression") as mock_report:
|
||
lots = s._parse_serp_html("<html>irrelevant</html>")
|
||
|
||
assert lots == []
|
||
mock_report.assert_called_once()
|
||
(msg,), _ = mock_report.call_args
|
||
assert "totalOffers=5" in msg
|
||
|
||
|
||
def test_total_offers_zero_and_offers_empty_is_valid_empty_search() -> None:
|
||
"""totalOffers=0, offers=[] — легитимная пустая выборка, НЕ регрессия."""
|
||
s = _scraper()
|
||
state = {"results": {"totalOffers": 0, "offers": []}}
|
||
with patch("scraper_kit.providers.cian.serp.extract_state", return_value=state):
|
||
with patch.object(s, "_report_schema_regression") as mock_report:
|
||
lots = s._parse_serp_html("<html>irrelevant</html>")
|
||
|
||
assert lots == []
|
||
mock_report.assert_not_called()
|
||
|
||
|
||
def test_total_offers_none_and_offers_empty_is_not_reported_as_regression() -> None:
|
||
"""totalOffers отсутствует/None в state — недостаточно сигнала для regression-репорта
|
||
(могла быть частично битая state-структура без явного totalOffers>0 контр-сигнала)."""
|
||
s = _scraper()
|
||
state = {"results": {"offers": []}}
|
||
with patch("scraper_kit.providers.cian.serp.extract_state", return_value=state):
|
||
with patch.object(s, "_report_schema_regression") as mock_report:
|
||
lots = s._parse_serp_html("<html>irrelevant</html>")
|
||
|
||
assert lots == []
|
||
mock_report.assert_not_called()
|
||
|
||
|
||
def test_offers_present_normal_path_unaffected() -> None:
|
||
"""totalOffers=1, offers содержит 1 запись без cianId/id — не проходит
|
||
_offer_to_lot, но это уже существующая (0/N offer-level) охрана, не finding 2."""
|
||
s = _scraper()
|
||
state = {"results": {"totalOffers": 1, "offers": [{"noId": True}]}}
|
||
with patch("scraper_kit.providers.cian.serp.extract_state", return_value=state):
|
||
with patch.object(s, "_report_schema_regression") as mock_report:
|
||
lots = s._parse_serp_html("<html>irrelevant</html>")
|
||
|
||
# offers_data непустой → finding 2 guard не участвует; существующая offer-level
|
||
# охрана (raw_count>0 and saved_count==0) должна отработать вместо неё.
|
||
assert lots == []
|
||
mock_report.assert_called_once()
|
||
(msg,), _ = mock_report.call_args
|
||
assert "_offer_to_lot" in msg
|
||
|
||
|
||
def test_state_none_extraction_failed_no_regression_report() -> None:
|
||
"""extract_state вернул None (captcha/структура целиком не найдена) — уже
|
||
существующая ветка, НЕ должна триггерить finding-2 regression report."""
|
||
s = _scraper()
|
||
with patch("scraper_kit.providers.cian.serp.extract_state", return_value=None):
|
||
with patch.object(s, "_report_schema_regression") as mock_report:
|
||
lots = s._parse_serp_html("<html>irrelevant</html>")
|
||
|
||
assert lots == []
|
||
mock_report.assert_not_called()
|
||
|
||
|
||
def test_report_schema_regression_swallows_missing_glitchtip_dsn() -> None:
|
||
"""_report_schema_regression не должен падать, если glitchtip_dsn не настроен."""
|
||
s = _scraper()
|
||
s._config = MagicMock(glitchtip_dsn=None)
|
||
s._report_schema_regression("test message") # не должно бросить исключение
|