gendesign/tradein-mvp/backend/tests/scrapers/test_domclick_detail.py

"""Unit-тесты для DomClick Layer B detail-scraper (scraper_kit.providers.domclick.detail).

Offline: parse/SSR-тесты без сети/БД; save-тест через MagicMock (зеркало
test_cian_detail.py — в проекте нет live-DB фикстуры, идемпотентность
гарантируется ON CONFLICT-клаузой, которая проверяется на уровне SQL-строки).

Покрытие:
- _extract_ssr_state: bare `undefined` → null; in-string "undefined" выживает;
  скобки/эскейпы внутри строк не ломают balanced-scan; missing marker →
  DomClickParseError; challenge → DomClickBlockedError; unbalanced/невалидный
  JSON → DomClickParseError.
- parse_detail_html (REAL __SSR_STATE__ paths, live card 2075729321):
  productCard.objectInfo.{renovation→repair_state+repair_type,livingArea,kitchenArea};
  balconies absent→None; productCard.legalOptions.saleType; egrnData snake_case
  owners_count + encumbrances инверсия (collateral); houseInfo.info.{wallType,floorType}
  → raw_extra (НЕ house_type); productCard.viewsCount; priceInfo.priceHistory
  (ISO8601+offset kept tz-aware / bad-date + no-price skipped); top-level
  pricePrediction → raw_extra.avm (плоский + nested .result); raw_extra без None;
  item_id из URL.
- fetch_detail: browser-branch + проброс ошибок.
- save_detail_enrichment: UPDATE-параметры, detail_enriched_at, price-history INSERT.

Легаси `app.services.scrapers.domclick_detail` удалён (#2277 финальный шаг
scraper_kit-миграции, 0 runtime importers) — тесты переведены на kit-эквивалент
`scraper_kit.providers.domclick.detail` без изменения покрытия. Exceptions тоже
переведены на `scraper_kit.domclick_exceptions` (kit держит свою независимую копию,
`parse_detail_html`(kit) поднимает именно её). Legacy `app.services.scrapers.
domclick`/`domclick_exceptions` удалены целиком (#2397 Part E2, 0 runtime
importers) — `scraper_kit.domclick_exceptions` остаётся единственным живым классом
для `pytest.raises`.
"""

from __future__ import annotations

import json
from datetime import UTC, datetime, timedelta
from unittest.mock import AsyncMock, MagicMock

import pytest
from scraper_kit.domclick_exceptions import DomClickBlockedError, DomClickParseError
from scraper_kit.offer_price_history import clamp_diff_percent
from scraper_kit.providers.domclick.detail import (
    DomClickDetailEnrichment,
    _extract_ssr_state,
    fetch_detail,
    parse_detail_html,
    save_detail_enrichment,
)

_CARD_URL = "https://ekaterinburg.domclick.ru/card/sale__flat__2075729321"

# Литерал с bare `undefined` (value-позиция), in-string "undefined", скобкой
# внутри строки и эскейп-кавычками — написан вручную, json.dumps использовать
# нельзя (нужен именно НЕвалидный-для-json undefined).
# Real __SSR_STATE__ shape (ground truth, live card 2075729321, 2026-06-27).
# bare `undefined` (value-pos), in-string "undefined", '{' внутри строки и
# эскейп-кавычки оставлены для проверки санитайзера/balanced-scan; json.dumps
# использовать нельзя (нужен именно НЕвалидный-для-json undefined).
# priceHistory[0].date намеренно flush-left (длинная ISO+offset строка) — отступ
# в литерале не влияет на парсер, но держит физическую строку ≤100 для ruff.
_SSR_LITERAL = """{
  "productCard": {
    "objectInfo": {
      "area": 38.0,
      "rooms": 1,
      "floor": 5,
      "isApartment": false,
      "renovation": "евроремонт",
      "livingArea": 22.37,
      "kitchenArea": 10.13,
      "description": "Состояние пока undefined.",
      "note": "угол {da}",
      "quote": "сказал \\"привет\\" сосед"
    },
    "legalOptions": {"saleType": "Свободная продажа"},
    "egrnData": {
      "area": 38.2,
      "floor": 5,
      "owners_count": 1,
      "collateral": true,
      "collateral_sber": false
    },
    "priceInfo": {
      "priceHistory": [
{"date": "2026-04-15T09:02:26.548728+03:00", "price": 13400000, "diff": -400000, "state": "less"},
        {"date": "not-a-date", "price": 4800000, "diff": -1.0, "state": "less"},
        {"date": "2026-05-01T10:00:00Z", "price": null, "diff": 0, "state": "more"}
      ]
    },
    "viewsCount": 338,
    "callsCount": 5,
    "favoriteOfferUsersCount": 12,
    "duplicatesOfferCount": 0,
    "address": {"guid": "abc-guid-123"},
    "unknownField": undefined
  },
  "houseInfo": {
    "info": {
      "buildYear": 2015,
      "wallType": "Кирпично-монолитный",
      "floorType": "Железобетонный",
      "entranceCount": 3,
      "quartersCount": 120,
      "energyEfficiency": "B",
      "buildingSeries": "индивидуальный"
    }
  },
  "pricePrediction": {
    "market_price": 13000000,
    "min_market_price": 12000000,
    "max_market_price": 14000000,
    "rent_short": 50000,
    "rent_long": 35000,
    "repair_quality": "good"
  }
}"""

_HTML = (
    "<html><head></head><body><script>"
    f"window.__SSR_STATE__ = {_SSR_LITERAL};"
    "</script></body></html>"
)


def _wrap(state: dict) -> str:
    """Заворачивает валидный dict в SSR-HTML (для variant-фикстур без undefined)."""
    literal = json.dumps(state, ensure_ascii=False)
    return f"<html><body><script>window.__SSR_STATE__ = {literal};</script></body></html>"


# ── _extract_ssr_state ────────────────────────────────────────────────────────


def test_extract_ssr_state_parses_despite_undefined() -> None:
    state = _extract_ssr_state(_HTML)
    # bare undefined в value-позиции → null → None
    assert state["productCard"]["unknownField"] is None
    # in-string "undefined" НЕ испорчен санитайзером
    assert state["productCard"]["objectInfo"]["description"] == "Состояние пока undefined."


def test_extract_ssr_state_brace_and_escape_in_string() -> None:
    state = _extract_ssr_state(_HTML)
    oi = state["productCard"]["objectInfo"]
    # '{' внутри строки не ломает balanced-brace scan
    assert oi["note"] == "угол {da}"
    # эскейп-кавычки внутри строки обработаны
    assert oi["quote"] == 'сказал "привет" сосед'


def test_extract_ssr_state_missing_marker_raises_parse() -> None:
    with pytest.raises(DomClickParseError):
        _extract_ssr_state("<html><body>no ssr state here</body></html>")


def test_extract_ssr_state_challenge_raises_blocked() -> None:
    html = "<html><body>Access denied by DataDome captcha</body></html>"
    with pytest.raises(DomClickBlockedError):
        _extract_ssr_state(html)


def test_extract_ssr_state_unbalanced_raises_parse() -> None:
    with pytest.raises(DomClickParseError):
        _extract_ssr_state('window.__SSR_STATE__ = {"a": 1')


def test_extract_ssr_state_invalid_json_raises_parse() -> None:
    with pytest.raises(DomClickParseError):
        _extract_ssr_state("window.__SSR_STATE__ = {this: is, not: json}")


# ── parse_detail_html — полная карточка ───────────────────────────────────────


def test_parse_detail_html_full() -> None:
    e = parse_detail_html(_HTML, _CARD_URL)

    assert e.item_id == "2075729321"
    assert e.source_url == _CARD_URL

    assert e.repair_state == "good"  # евроремонт → good (LOCAL map)
    assert e.repair_type == "евроремонт"  # сырое значение сохранено

    assert e.living_area_m2 == 22.37
    assert e.kitchen_area_m2 == 10.13

    # На реальной карточке нет ключа balconies в objectInfo → optional/None.
    assert e.balconies_count is None
    assert e.has_balcony is None

    assert e.sale_type == "Свободная продажа"
    assert e.owners_count == 1
    assert e.encumbrances_clean is False  # collateral=true → есть обременение
    assert e.year_built == 2015
    assert e.views_total == 338


def test_parse_detail_html_price_changes() -> None:
    e = parse_detail_html(_HTML, _CARD_URL)
    # entry1 ок; entry2 bad-date skip; entry3 price=null skip → 1 запись
    assert len(e.price_changes) == 1
    change = e.price_changes[0]
    assert change["price_rub"] == 13400000
    assert change["diff_percent"] == -400000
    assert isinstance(change["change_time"], datetime)
    # ISO8601-with-offset: tz-aware, offset СОХРАНЁН (+03:00), НЕ сконвертирован в UTC.
    assert change["change_time"].tzinfo is not None
    assert change["change_time"].utcoffset() == timedelta(hours=3)


def test_parse_detail_html_raw_extra() -> None:
    e = parse_detail_html(_HTML, _CARD_URL)
    raw = e.raw_extra
    assert raw["wall_type"] == "Кирпично-монолитный"
    assert raw["floor_type"] == "Железобетонный"
    # wall/floor type живут ТОЛЬКО в raw_extra, НЕ в listings.house_type.
    assert "house_type" not in raw
    assert raw["domclick_building_guid"] == "abc-guid-123"
    assert raw["egrn_area"] == 38.2
    assert raw["demand"]["calls"] == 5
    assert raw["demand"]["favorites"] == 12
    # AVM (Layer C, top-level pricePrediction) → raw_extra.avm
    assert raw["avm"]["market_price"] == 13000000
    assert raw["avm"]["min"] == 12000000
    assert raw["avm"]["max"] == 14000000
    assert raw["avm"]["repair_quality"] == "good"
    # нет ключа balconies → raw_extra НЕ содержит balconies_raw
    assert "balconies_raw" not in raw
    # никаких None-значений на верхнем уровне (компактность)
    assert all(v is not None for v in raw.values())


# ── parse_detail_html — частные ветки ─────────────────────────────────────────


def test_parse_balconies_string_goes_to_raw_extra() -> None:
    html = _wrap({"productCard": {"objectInfo": {"balconies": "есть"}}})
    e = parse_detail_html(html, _CARD_URL)
    assert e.balconies_count is None
    assert e.has_balcony is None
    assert e.raw_extra["balconies_raw"] == "есть"


def test_parse_encumbrances_clean_true_when_no_collateral() -> None:
    html = _wrap({"productCard": {"egrnData": {"collateral": False, "collateral_sber": False}}})
    e = parse_detail_html(html, _CARD_URL)
    assert e.encumbrances_clean is True


def test_parse_encumbrances_none_when_egrn_missing() -> None:
    html = _wrap({"productCard": {"objectInfo": {"livingArea": 30}}})
    e = parse_detail_html(html, _CARD_URL)
    assert e.encumbrances_clean is None


def test_parse_repair_map_hit() -> None:
    html = _wrap({"productCard": {"objectInfo": {"renovation": "Косметический"}}})
    e = parse_detail_html(html, _CARD_URL)
    assert e.repair_state == "standard"
    assert e.repair_type == "Косметический"


def test_parse_repair_fallback_to_inference() -> None:
    # Не точное совпадение в карте → инференс из текста (евроремонт → good)
    html = _wrap({"productCard": {"objectInfo": {"renovation": "сделан свежий евроремонт"}}})
    e = parse_detail_html(html, _CARD_URL)
    assert e.repair_state == "good"
    assert e.repair_type == "сделан свежий евроремонт"


def test_parse_owners_count_camelcase_key() -> None:
    html = _wrap({"productCard": {"egrnData": {"ownersCount": 3}}})
    e = parse_detail_html(html, _CARD_URL)
    assert e.owners_count == 3


def test_parse_avm_nested_under_result() -> None:
    # pricePrediction может приходить обёрнутым в .result — unwrap должен сработать.
    html = _wrap(
        {
            "productCard": {},
            "pricePrediction": {"result": {"marketPrice": 9000000, "repairQuality": "fine"}},
        }
    )
    e = parse_detail_html(html, _CARD_URL)
    assert e.raw_extra["avm"]["market_price"] == 9000000
    assert e.raw_extra["avm"]["repair_quality"] == "fine"


def test_parse_avm_absent_no_key() -> None:
    # Нет pricePrediction → raw_extra без ключа avm (компактность).
    e = parse_detail_html(_wrap({"productCard": {}}), _CARD_URL)
    assert "avm" not in e.raw_extra


def test_parse_item_id_none_for_bad_url() -> None:
    e = parse_detail_html(_wrap({"productCard": {}}), "https://domclick.ru/card/rent__room__1")
    assert e.item_id is None


def test_dataclass_defaults() -> None:
    e = DomClickDetailEnrichment()
    assert e.item_id is None
    assert e.price_changes == []
    assert e.raw_extra == {}


# ── fetch_detail (browser-branch) ─────────────────────────────────────────────


@pytest.mark.asyncio
async def test_fetch_detail_parses_via_browser() -> None:
    bf = MagicMock()
    bf.fetch = AsyncMock(return_value=_HTML)
    e = await fetch_detail(_CARD_URL, browser_fetcher=bf)
    assert e.item_id == "2075729321"
    assert e.repair_state == "good"
    bf.fetch.assert_called_once_with(
        _CARD_URL, origin="https://ekaterinburg.domclick.ru/pokupka/kvartiry/vtorichka"
    )


@pytest.mark.asyncio
async def test_fetch_detail_passes_derived_vtorichka_origin() -> None:
    """fetch_detail выводит origin из scheme+netloc card_url → vtorichka-SERP root.

    Организм-навигация (SERP до карточки) снижает подозрительность холодного
    захода для QRATOR — см. комментарий в fetch_detail. Проверяем сам вывод origin
    для другого хоста/пути, не завязываясь на _CARD_URL из happy-path теста выше.
    """
    bf = MagicMock()
    bf.fetch = AsyncMock(return_value=_HTML)
    other_card_url = "https://spb.domclick.ru/card/sale__flat__999888777"

    await fetch_detail(other_card_url, browser_fetcher=bf)

    bf.fetch.assert_called_once_with(
        other_card_url, origin="https://spb.domclick.ru/pokupka/kvartiry/vtorichka"
    )


@pytest.mark.asyncio
async def test_fetch_detail_browser_failure_raises_blocked() -> None:
    bf = MagicMock()
    bf.fetch = AsyncMock(side_effect=RuntimeError("502 bad gateway"))
    with pytest.raises(DomClickBlockedError):
        await fetch_detail(_CARD_URL, browser_fetcher=bf)


@pytest.mark.asyncio
async def test_fetch_detail_propagates_parse_error() -> None:
    bf = MagicMock()
    bf.fetch = AsyncMock(return_value="<html><body>no state</body></html>")
    with pytest.raises(DomClickParseError):
        await fetch_detail(_CARD_URL, browser_fetcher=bf)


@pytest.mark.asyncio
async def test_fetch_detail_propagates_blocked_from_html() -> None:
    bf = MagicMock()
    bf.fetch = AsyncMock(return_value="<html><body>Access denied datadome</body></html>")
    with pytest.raises(DomClickBlockedError):
        await fetch_detail(_CARD_URL, browser_fetcher=bf)


# ── save_detail_enrichment (MagicMock — зеркало test_cian_detail) ─────────────


def test_save_detail_enrichment_writes_columns() -> None:
    db = MagicMock()
    db.execute.return_value.rowcount = 1
    ct = datetime(2026, 4, 1, tzinfo=UTC)
    e = DomClickDetailEnrichment(
        item_id="2075729321",
        source_url=_CARD_URL,
        repair_state="good",
        repair_type="Евроремонт",
        living_area_m2=45.5,
        kitchen_area_m2=12.3,
        balconies_count=2,
        has_balcony=True,
        sale_type="свободная",
        owners_count=1,
        encumbrances_clean=False,
        year_built=2015,
        views_total=1234,
        price_changes=[{"change_time": ct, "price_rub": 5000000, "diff_percent": -2.5}],
        raw_extra={"wall_type": "монолит"},
    )

    ok = save_detail_enrichment(db, 999, e)
    assert ok is True

    # UPDATE — первый execute
    update_sql = str(db.execute.call_args_list[0][0][0])
    update_params = db.execute.call_args_list[0][0][1]
    assert "detail_enriched_at = NOW()" in update_sql
    assert update_params["lid"] == 999
    assert update_params["repair_state"] == "good"
    assert update_params["living_area_m2"] == 45.5
    assert update_params["balconies_count"] == 2
    assert update_params["has_balcony"] is True
    assert update_params["encumbrances_clean"] is False
    assert "монолит" in update_params["raw_extra"]

    # price-history INSERT с ON CONFLICT (идемпотентность)
    insert_calls = [c for c in db.execute.call_args_list if "offer_price_history" in str(c[0][0])]
    assert len(insert_calls) == 1
    assert "ON CONFLICT ON CONSTRAINT offer_price_history_listing_change_uq" in str(
        insert_calls[0][0][0]
    )
    ins_params = insert_calls[0][0][1]
    assert ins_params["price"] == 5000000
    assert ins_params["ct"] == ct
    db.commit.assert_called_once()


def test_save_detail_enrichment_listing_not_found() -> None:
    db = MagicMock()
    db.execute.return_value.rowcount = 0
    e = DomClickDetailEnrichment(item_id="x", source_url=_CARD_URL, repair_state="good")
    assert save_detail_enrichment(db, 123, e) is False


def test_save_detail_enrichment_none_passthrough() -> None:
    db = MagicMock()
    db.execute.return_value.rowcount = 1
    e = DomClickDetailEnrichment(item_id="x", source_url=_CARD_URL)  # все поля None
    save_detail_enrichment(db, 1, e)
    params = db.execute.call_args_list[0][0][1]
    assert params["repair_state"] is None
    assert params["living_area_m2"] is None
    # пустой raw_extra → "{}" (merge no-op, не затирает)
    assert params["raw_extra"] == "{}"


# ── save_detail_enrichment — diff_percent клампится перед INSERT (#2277) ──────
# offer_price_history.diff_percent — NUMERIC(8,2) (миграция 131). Прямой INSERT
# здесь не проходит через БД-триггер (тот срабатывает только на UPDATE listings) —
# garbage/экстремальный diff_percent из скрапленного HTML иначе переполняет колонку.


def _insert_diff_param(db: MagicMock) -> float | None:
    insert_calls = [c for c in db.execute.call_args_list if "offer_price_history" in str(c[0][0])]
    assert len(insert_calls) == 1
    return insert_calls[0][0][1]["diff"]


def test_save_detail_enrichment_clamps_extreme_positive_diff() -> None:
    db = MagicMock()
    db.execute.return_value.rowcount = 1
    ct = datetime(2026, 4, 1, tzinfo=UTC)
    e = DomClickDetailEnrichment(
        item_id="x",
        source_url=_CARD_URL,
        price_changes=[{"change_time": ct, "price_rub": 5000000, "diff_percent": 5000000}],
    )
    save_detail_enrichment(db, 1, e)
    assert _insert_diff_param(db) == 999999.99


def test_save_detail_enrichment_clamps_extreme_negative_diff() -> None:
    db = MagicMock()
    db.execute.return_value.rowcount = 1
    ct = datetime(2026, 4, 1, tzinfo=UTC)
    e = DomClickDetailEnrichment(
        item_id="x",
        source_url=_CARD_URL,
        price_changes=[{"change_time": ct, "price_rub": 5000000, "diff_percent": -5000000}],
    )
    save_detail_enrichment(db, 1, e)
    assert _insert_diff_param(db) == -999999.99


def test_save_detail_enrichment_normal_diff_unchanged() -> None:
    db = MagicMock()
    db.execute.return_value.rowcount = 1
    ct = datetime(2026, 4, 1, tzinfo=UTC)
    e = DomClickDetailEnrichment(
        item_id="x",
        source_url=_CARD_URL,
        price_changes=[{"change_time": ct, "price_rub": 5000000, "diff_percent": -2.5}],
    )
    save_detail_enrichment(db, 1, e)
    assert _insert_diff_param(db) == -2.5


def test_save_detail_enrichment_none_diff_unchanged() -> None:
    db = MagicMock()
    db.execute.return_value.rowcount = 1
    ct = datetime(2026, 4, 1, tzinfo=UTC)
    e = DomClickDetailEnrichment(
        item_id="x",
        source_url=_CARD_URL,
        price_changes=[{"change_time": ct, "price_rub": 5000000, "diff_percent": None}],
    )
    save_detail_enrichment(db, 1, e)
    assert _insert_diff_param(db) is None


# ── clamp_diff_percent (scraper_kit.offer_price_history) — standalone unit ────
# Нет отдельной tests/ директории для scraper-kit-пакета (см. audit перед фиксом) —
# прямые unit-тесты хелпера живут здесь, рядом с save_detail_enrichment-тестами,
# которые его же и используют.


def test_clamp_diff_percent_extreme_positive() -> None:
    assert clamp_diff_percent(5000000) == 999999.99


def test_clamp_diff_percent_extreme_negative() -> None:
    assert clamp_diff_percent(-5000000) == -999999.99


def test_clamp_diff_percent_in_range_unchanged() -> None:
    assert clamp_diff_percent(-2.5) == -2.5


def test_clamp_diff_percent_none() -> None:
    assert clamp_diff_percent(None) is None


def test_clamp_diff_percent_bool_treated_as_none() -> None:
    assert clamp_diff_percent(True) is None
    assert clamp_diff_percent(False) is None