gendesign/backend/app/services/scrapers/obj_checks.py

"""Extractor for 6 «Проверено на наш.дом.рф» checks per object.

Issue #297, sub-task 22f. Table created in PR #303 (data/sql/111_22f_domrf_obj_checks.sql).

Check types (canonical):
    no_problems  / docs  / timing  / photos  / bankruptcy  / declaration

Source endpoint: /сервисы/api/object/{obj_id}/checks
    (pattern analogичен /infrastructure и /photos — те же сервисы/api/object/{obj_id}/ prefix)
    URL not verified via devtools — структура payload выведена из аудита страницы ЖК.
    Если endpoint не существует — scraper получит HTTP-ошибку, которую _classify_and_log
    запишет в kn_scrape_failures, данные в domrf_obj_checks не поступят, scrape не упадёт.

Expected payload shape (предположительно):
    {
      "data": {
        "noProblemObjects":   true,   # no_problems
        "hasDocuments":       true,   # docs
        "meetsDeadlines":     true,   # timing
        "hasPhotos":          true,   # photos
        "notBankrupt":        true,   # bankruptcy
        "hasDeclaration":     true    # declaration
      }
    }

    OR possibly an array:
    [{"checkType": "no_problems", "passed": true}, ...]

Поддерживаются оба варианта: dict-payload (поля маппятся через CHECK_FIELD_MAP)
и list-payload (поля check_type + passed/value).
"""

from __future__ import annotations

import logging
from typing import Any

from sqlalchemy import text
from sqlalchemy.orm import Session

logger = logging.getLogger(__name__)

CHECK_TYPES = ["no_problems", "docs", "timing", "photos", "bankruptcy", "declaration"]

# Mapping of possible API field names → canonical check_type.
# Best-guess from DOM.РФ API field naming conventions (кейс camelCase).
_CHECK_FIELD_MAP: dict[str, str] = {
    "noProblemObjects": "no_problems",
    "noProblemFlg": "no_problems",
    "hasDocuments": "docs",
    "documentsFlg": "docs",
    "meetsDeadlines": "timing",
    "deadlinesFlg": "timing",
    "hasPhotos": "photos",
    "photosFlg": "photos",
    "notBankrupt": "bankruptcy",
    "bankruptcyFlg": "bankruptcy",
    "hasDeclaration": "declaration",
    "declarationFlg": "declaration",
}

# Canonical check_type name → possible API field name aliases
_CHECK_TYPE_ALIASES: dict[str, list[str]] = {
    "no_problems": ["no_problems", "noProblemObjects", "noProblemFlg"],
    "docs": ["docs", "hasDocuments", "documentsFlg"],
    "timing": ["timing", "meetsDeadlines", "deadlinesFlg"],
    "photos": ["photos", "hasPhotos", "photosFlg"],
    "bankruptcy": ["bankruptcy", "notBankrupt", "bankruptcyFlg"],
    "declaration": ["declaration", "hasDeclaration", "declarationFlg"],
}

_UPSERT_CHECKS_SQL = text(
    """
    INSERT INTO domrf_obj_checks (obj_id, check_type, passed, checked_at, scraped_at)
    VALUES (:obj_id, :check_type, :passed, NOW(), NOW())
    ON CONFLICT (obj_id, check_type) DO UPDATE SET
        passed     = EXCLUDED.passed,
        checked_at = NOW(),
        scraped_at = NOW()
    """
)


def extract_obj_checks(raw_payload: Any) -> list[dict[str, Any]]:
    """Извлечь 6 чекбоксов из payload endpoint /object/{obj_id}/checks.

    Поддерживает два варианта payload:
    1. dict с полями (ожидаемый API-формат): {"data": {"noProblemObjects": true, ...}}
    2. list объектов: [{"checkType": "no_problems", "passed": true}, ...]

    Для неизвестных полей и неподдерживаемых форматов — WARNING + пустой список.
    """
    if not raw_payload:
        return []

    # Вариант 1: dict с data-обёрткой
    data: Any = raw_payload
    if isinstance(raw_payload, dict):
        data = raw_payload.get("data") or raw_payload

    results: list[dict[str, Any]] = []

    if isinstance(data, dict):
        # Map известных полей к canonical check_type
        found: dict[str, bool] = {}
        for field, value in data.items():
            ct = _CHECK_FIELD_MAP.get(field)
            if ct and ct not in found:
                found[ct] = bool(value)
        # Также проверить canonical names напрямую
        for ct in CHECK_TYPES:
            if ct not in found and ct in data:
                found[ct] = bool(data[ct])
        if found:
            for ct in CHECK_TYPES:
                results.append({"check_type": ct, "passed": found.get(ct, False)})
            return results
        # dict не содержит известных полей — попробуем как list-формат ниже
        logger.warning(
            "domrf obj_checks: dict payload has no known check fields: %s", list(data)[:10]
        )
        return []

    if isinstance(data, list):
        found_list: dict[str, bool] = {}
        for item in data:
            if not isinstance(item, dict):
                continue
            ct_raw = item.get("checkType") or item.get("check_type") or item.get("type")
            if ct_raw and str(ct_raw) in CHECK_TYPES:
                passed_raw = item.get("passed") or item.get("value") or item.get("status")
                found_list[str(ct_raw)] = bool(passed_raw)
        if found_list:
            for ct in CHECK_TYPES:
                results.append({"check_type": ct, "passed": found_list.get(ct, False)})
            return results
        logger.warning(
            "domrf obj_checks: list payload has no recognisable check items: %s", data[:3]
        )
        return []

    logger.warning("domrf obj_checks: unexpected payload type %s", type(raw_payload))
    return []


def upsert_obj_checks(db: Session, obj_id: int, checks: list[dict[str, Any]]) -> int:
    """UPSERT 6 чек-строк в domrf_obj_checks. Returns count of inserted/updated rows.

    Использует SAVEPOINT (begin_nested) per-row — одна битая строка не откатывает
    всю транзакцию.
    """
    if not checks:
        return 0
    ok = 0
    for c in checks:
        try:
            with db.begin_nested():
                db.execute(
                    _UPSERT_CHECKS_SQL,
                    {
                        "obj_id": obj_id,
                        "check_type": c["check_type"],
                        "passed": c["passed"],
                    },
                )
            ok += 1
        except Exception as exc:
            logger.warning(
                "upsert obj_checks obj=%s check_type=%s failed: %s",
                obj_id,
                c.get("check_type"),
                exc,
            )
    db.commit()
    return ok