feat(22d): domrf_catalog_object scraper — fill ~25 NULL kn_objects cols from SSR __NEXT_DATA__ (#335)

backend/app/services/scrapers/domrf_catalog_object.py

@@ -0,0 +1,459 @@
+"""DOM.РФ catalog-OBJECT scraper (issue #297 sub-task 22d).
+
+Fills ~25 NULL columns in domrf_kn_objects from public SSR catalog page:
+  https://наш.дом.рф/сервисы/каталог-новостроек/объект/{obj_id}
+
+kn-API не возвращает: wall_type, energy_eff, ceiling_height_m, parking_*,
+playground_*, finishing_variants_count, etc. — все эти поля есть в
+__NEXT_DATA__ JSON блоке на странице каталога (Next.js SSR).
+
+Uses BrowserSession from app.services.scrapers.stealth (Playwright + WAF bypass).
+Fetches HTML, extracts __NEXT_DATA__ JSON, maps to DB columns,
+UPDATE domrf_kn_objects WHERE obj_id = :id (не перетирает kn-API данные).
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+import re
+from datetime import date
+from typing import Any
+
+from sqlalchemy import text
+from sqlalchemy.orm import Session
+
+from app.services.scrapers.stealth import BASE_URL, BrowserSession, WafBlockedError, jitter_sleep
+
+logger = logging.getLogger(__name__)
+
+# URL шаблон страницы объекта в каталоге DOM.РФ.
+# Человекочитаемый вид: https://наш.дом.рф/сервисы/каталог-новостроек/объект/{obj_id}
+CATALOG_OBJECT_PATH = "/сервисы/каталог-новостроек/объект/{obj_id}"
+
+# JS snippet — аналог _FETCH_HTML_JS из domrf_catalog.py.
+# Выполняется внутри живой Playwright-страницы, возвращает HTML текст.
+_FETCH_HTML_JS = """
+async ({url}) => {
+  try {
+    const r = await fetch(url, {credentials: 'include'});
+    const ctype = r.headers.get('content-type') || '';
+    const body = await r.text();
+    return {ok: r.ok, status: r.status, body, contentType: ctype};
+  } catch (e) {
+    return {ok: false, status: 0, body: String(e), contentType: ''};
+  }
+}
+"""
+
+# UPDATE SQL — обновляет только catalog-derived поля.
+# COALESCE гарантирует что NULL-значение не перетирает уже заполненное поле.
+UPDATE_OBJECT_CATALOG_SQL = text(
+    """
+    UPDATE domrf_kn_objects SET
+        obj_class                  = COALESCE(:obj_class, obj_class),
+        wall_type                  = COALESCE(:wall_type, wall_type),
+        energy_eff                 = COALESCE(:energy_eff, energy_eff),
+        section_count              = COALESCE(:section_count, section_count),
+        parking_total_slots        = COALESCE(:parking_total_slots, parking_total_slots),
+        guest_parking_inside_count = COALESCE(
+            :guest_parking_inside_count, guest_parking_inside_count
+        ),
+        guest_parking_outside_count = COALESCE(
+            :guest_parking_outside_count, guest_parking_outside_count
+        ),
+        ceiling_height_m           = COALESCE(:ceiling_height_m, ceiling_height_m),
+        finishing_variants_count   = COALESCE(:finishing_variants_count, finishing_variants_count),
+        has_free_planning          = COALESCE(:has_free_planning, has_free_planning),
+        avg_flat_area_m2           = COALESCE(:avg_flat_area_m2, avg_flat_area_m2),
+        elevators_passenger_count  = COALESCE(
+            :elevators_passenger_count, elevators_passenger_count
+        ),
+        elevators_cargo_count      = COALESCE(:elevators_cargo_count, elevators_cargo_count),
+        playground_kids_count      = COALESCE(:playground_kids_count, playground_kids_count),
+        playground_sport_count     = COALESCE(:playground_sport_count, playground_sport_count),
+        has_bike_paths             = COALESCE(:has_bike_paths, has_bike_paths),
+        trash_areas_count          = COALESCE(:trash_areas_count, trash_areas_count),
+        has_ramp                   = COALESCE(:has_ramp, has_ramp),
+        has_low_platforms          = COALESCE(:has_low_platforms, has_low_platforms),
+        has_wheelchair_lift        = COALESCE(:has_wheelchair_lift, has_wheelchair_lift),
+        first_floor_type           = COALESCE(:first_floor_type, first_floor_type),
+        parking_provision_pct      = COALESCE(:parking_provision_pct, parking_provision_pct),
+        project_published_at       = COALESCE(:project_published_at, project_published_at),
+        project_declaration_num    = COALESCE(:project_declaration_num, project_declaration_num),
+        domrf_score_infrastructure = COALESCE(
+            :domrf_score_infrastructure, domrf_score_infrastructure
+        ),
+        domrf_score_transport      = COALESCE(:domrf_score_transport, domrf_score_transport),
+        catalog_scraped_at         = NOW()
+    WHERE obj_id = :obj_id
+      AND snapshot_date = :snapshot_date
+    """
+)
+
+
+# ── Value helpers ─────────────────────────────────────────────────────────────
+
+
+def _to_numeric_comma(s: Any) -> float | None:
+    """Конвертировать строку с запятой-десятичным разделителем в float.
+
+    Примеры: "2,7" → 2.7; "2.7" → 2.7; "" → None; None → None.
+    """
+    if s is None:
+        return None
+    raw = str(s).strip().replace(",", ".")
+    if not raw:
+        return None
+    try:
+        return float(raw)
+    except ValueError:
+        return None
+
+
+def _to_date_ddmmyyyy(s: Any) -> date | None:
+    """Конвертировать строку "DD.MM.YYYY" в date.
+
+    Примеры: "31.03.2025" → date(2025, 3, 31); "" → None; invalid → None.
+    """
+    if not s:
+        return None
+    raw = str(s).strip()
+    if not raw:
+        return None
+    try:
+        parts = raw.split(".")
+        if len(parts) == 3:
+            return date(int(parts[2]), int(parts[1]), int(parts[0]))
+    except (ValueError, IndexError):
+        pass
+    return None
+
+
+def _to_bool_int(v: Any) -> bool | None:
+    """Конвертировать 0/1 (или любое int-like) в bool.
+
+    Примеры: 1 → True; 0 → False; None → None; 3 → True (>0).
+    """
+    if v is None:
+        return None
+    try:
+        return int(v) > 0
+    except (ValueError, TypeError):
+        return None
+
+
+def _to_bool_da_net(s: Any) -> bool | None:
+    """Конвертировать "Да"/"Нет" строку в bool.
+
+    Примеры: "Да" → True; "Нет" → False; "" → None; None → None.
+    """
+    if s is None:
+        return None
+    raw = str(s).strip().lower()
+    if raw == "да":
+        return True
+    if raw == "нет":
+        return False
+    return None
+
+
+def _safe_int(v: Any) -> int | None:
+    """Безопасная конвертация в int, None при ошибке."""
+    if v is None:
+        return None
+    try:
+        return int(v)
+    except (ValueError, TypeError):
+        return None
+
+
+# ── HTML fetching ─────────────────────────────────────────────────────────────
+
+
+async def fetch_catalog_object_html(session: BrowserSession, obj_id: int) -> str:
+    """Получить SSR-HTML страницы объекта в каталоге DOM.РФ.
+
+    Использует тот же паттерн что fetch_catalog_html из domrf_catalog.py:
+    fetch() внутри живой Playwright-страницы — WAF-fingerprint идентичен браузеру.
+
+    Raises:
+        WafBlockedError: если вернулся не-HTML (JS-challenge или JSON).
+        RuntimeError: при 404 или исчерпании попыток.
+    """
+    if session._page is None:
+        raise RuntimeError("BrowserSession not bootstrapped")
+
+    url = BASE_URL + CATALOG_OBJECT_PATH.format(obj_id=obj_id)
+    last_err: Exception | None = None
+
+    for attempt in range(5):
+        async with session._sem:
+            await jitter_sleep(800, 1500)
+            try:
+                session._request_count += 1
+                result = await session._page.evaluate(_FETCH_HTML_JS, {"url": url})
+            except Exception as exc:
+                last_err = exc
+                logger.warning(
+                    "catalog_object html evaluate err attempt=%d obj_id=%d: %r",
+                    attempt,
+                    obj_id,
+                    exc,
+                )
+                await asyncio.sleep(2**attempt)
+                continue
+
+        status: int = result.get("status", 0)
+        body: str = result.get("body", "")
+        ctype: str = result.get("contentType", "")
+
+        if status in (429,) or status >= 500 or status == 0:
+            last_err = RuntimeError(f"transient status={status}")
+            logger.warning(
+                "catalog_object transient status=%d attempt=%d obj_id=%d, backing off",
+                status,
+                attempt,
+                obj_id,
+            )
+            await asyncio.sleep(2**attempt)
+            continue
+
+        if status == 404:
+            raise RuntimeError(f"catalog_object 404 for obj_id={obj_id}")
+
+        if status != 200:
+            raise RuntimeError(f"catalog_object http {status}: {body[:200]} obj_id={obj_id}")
+
+        # Проверяем что вернулся HTML, а не WAF JS-challenge.
+        is_html = "text/html" in ctype or "<!doctype" in body[:100].lower()
+        if body and not is_html:
+            raise WafBlockedError(
+                f"non-HTML response for obj_id={obj_id}: status={status} ctype={ctype!r}"
+                f" body[:120]={body[:120]!r}"
+            )
+
+        if not body:
+            raise RuntimeError(f"catalog_object empty body for obj_id={obj_id}")
+
+        return body
+
+    raise RuntimeError(f"catalog_object html max retries exhausted obj_id={obj_id}: {last_err!r}")
+
+
+# ── __NEXT_DATA__ extraction ──────────────────────────────────────────────────
+
+
+def extract_next_data(html: str) -> dict[str, Any]:
+    """Извлечь JSON из тега <script id="__NEXT_DATA__"> в SSR HTML.
+
+    Raises:
+        ValueError: если тег не найден или JSON не парсится.
+    """
+    match = re.search(
+        r'<script\s+id=["\']__NEXT_DATA__["\'][^>]*>(.+?)</script>',
+        html,
+        re.DOTALL,
+    )
+    if not match:
+        raise ValueError("__NEXT_DATA__ script tag not found in HTML")
+
+    raw_json = match.group(1).strip()
+    try:
+        return json.loads(raw_json)  # type: ignore[no-any-return]
+    except json.JSONDecodeError as exc:
+        raise ValueError(f"__NEXT_DATA__ JSON parse error: {exc}") from exc
+
+
+# ── Field mapping ─────────────────────────────────────────────────────────────
+
+
+def parse_catalog_object(next_data: dict[str, Any]) -> dict[str, Any]:
+    """Извлечь поля объекта из __NEXT_DATA__ и вернуть dict для UPDATE.
+
+    Все .get() безопасны — partial responses OK, отсутствующие поля = None.
+    Возвращает dict с bind-параметрами для UPDATE_OBJECT_CATALOG_SQL.
+    """
+    pp: dict[str, Any] = next_data.get("props", {}).get("pageProps", {})
+    ai: dict[str, Any] = pp.get("additionalInfo") or {}
+    quart: dict[str, Any] = pp.get("quartography") or {}
+    indexes: dict[str, Any] = pp.get("indexes") or {}
+    decl: dict[str, Any] = pp.get("projectDeclaration") or {}
+
+    # first_floor_type: 1 = нежилой, 0 = жилой
+    first_floor_raw = quart.get("nonLivFirstFloor")
+    first_floor_type: str | None = None
+    if first_floor_raw is not None:
+        try:
+            first_floor_type = "нежилой" if int(first_floor_raw) == 1 else "жилой"
+        except (ValueError, TypeError):
+            pass
+
+    # elevators_cargo_count = cargoElevatorsCount + cargoPassengerElevatorCount
+    cargo = _safe_int(ai.get("cargoElevatorsCount"))
+    cargo_pass = _safe_int(ai.get("cargoPassengerElevatorCount"))
+    if cargo is not None or cargo_pass is not None:
+        elevators_cargo_count: int | None = (cargo or 0) + (cargo_pass or 0)
+    else:
+        elevators_cargo_count = None
+
+    return {
+        "obj_class": pp.get("buildingClass"),
+        "wall_type": pp.get("wallMaterial"),
+        "energy_eff": pp.get("objEnergyEfficiency"),
+        "section_count": _safe_int(quart.get("objLivElemEntrCnt")),
+        "parking_total_slots": _safe_int(pp.get("parkingCount")),
+        "guest_parking_inside_count": _safe_int(ai.get("objectParkingPlaces")),
+        "guest_parking_outside_count": _safe_int(ai.get("nearbyParkingPlaces")),
+        "ceiling_height_m": _to_numeric_comma(ai.get("ceilingHeight")),
+        "finishing_variants_count": _safe_int(pp.get("finishTypeCount")),
+        "has_free_planning": _to_bool_da_net(pp.get("freePlan")),
+        "avg_flat_area_m2": _to_numeric_comma(quart.get("objLivElemSqAvg")),
+        "elevators_passenger_count": _safe_int(ai.get("passengerElevatorsCount")),
+        "elevators_cargo_count": elevators_cargo_count,
+        "playground_kids_count": _safe_int(ai.get("playgroundsCount")),
+        "playground_sport_count": _safe_int(ai.get("sportsgroundCount")),
+        "has_bike_paths": _to_bool_int(ai.get("bicycleLane")),
+        "trash_areas_count": _safe_int(ai.get("trashAreaCount")),
+        "has_ramp": _to_bool_int(ai.get("ramp")),
+        "has_low_platforms": _to_bool_int(ai.get("curbLowering")),
+        "has_wheelchair_lift": _to_bool_int(ai.get("wheelchairElevatorsCount")),
+        "first_floor_type": first_floor_type,
+        "parking_provision_pct": _to_numeric_comma(ai.get("parkingAvailabilityPerc")),
+        "project_published_at": _to_date_ddmmyyyy(pp.get("publicationDate")),
+        "project_declaration_num": decl.get("number"),
+        "domrf_score_infrastructure": _safe_int(indexes.get("infrastructure")),
+        "domrf_score_transport": _safe_int(indexes.get("transport")),
+        # TODO: obj_checks (6 detailed checks) — separate investigation (task #21).
+        # pageProps.isChecked (bool), verificationId, verificationFlg available here
+        # but detailed per-check breakdown requires separate API investigation.
+    }
+
+
+# ── DB write ──────────────────────────────────────────────────────────────────
+
+
+async def scrape_catalog_object(
+    db: Session,
+    session: BrowserSession,
+    obj_id: int,
+    snapshot_date: date,
+) -> bool:
+    """Scrape одного объекта: fetch HTML → extract __NEXT_DATA__ → parse → UPDATE.
+
+    Использует SAVEPOINT (begin_nested) для изоляции per-row ошибок.
+    Логирует результат через logger.info.
+
+    Returns:
+        True если UPDATE затронул строку, False при ошибке или 0 rows.
+    """
+    logger.info("catalog_object scrape start obj_id=%d snapshot_date=%s", obj_id, snapshot_date)
+
+    try:
+        html = await fetch_catalog_object_html(session, obj_id)
+    except WafBlockedError as exc:
+        logger.warning("catalog_object WAF blocked obj_id=%d: %s", obj_id, exc)
+        return False
+    except Exception as exc:
+        logger.warning("catalog_object fetch failed obj_id=%d: %s", obj_id, exc)
+        return False
+
+    try:
+        next_data = extract_next_data(html)
+    except ValueError as exc:
+        logger.warning("catalog_object extract_next_data failed obj_id=%d: %s", obj_id, exc)
+        return False
+
+    try:
+        data = parse_catalog_object(next_data)
+    except Exception as exc:
+        logger.warning("catalog_object parse failed obj_id=%d: %s", obj_id, exc)
+        return False
+
+    fields_extracted = len([v for v in data.values() if v is not None])
+
+    params: dict[str, Any] = {
+        "obj_id": obj_id,
+        "snapshot_date": snapshot_date,
+        **data,
+    }
+
+    try:
+        with db.begin_nested():
+            result = db.execute(UPDATE_OBJECT_CATALOG_SQL, params)
+        rows_affected: int = result.rowcount or 0
+    except Exception as exc:
+        logger.warning("catalog_object UPDATE failed obj_id=%d: %s", obj_id, exc)
+        return False
+
+    if rows_affected == 0:
+        logger.warning(
+            "catalog_object UPDATE 0 rows obj_id=%d snapshot_date=%s — not in DB?",
+            obj_id,
+            snapshot_date,
+        )
+        return False
+
+    logger.info(
+        "catalog_object scraped obj_id=%d fields=%d rows_updated=%d",
+        obj_id,
+        fields_extracted,
+        rows_affected,
+    )
+    return True
+
+
+async def scrape_catalog_objects(
+    db: Session,
+    obj_ids: list[int],
+    snapshot_date: date,
+    region_code: int = 66,
+) -> dict[str, int]:
+    """Scrape списка объектов через один BrowserSession.
+
+    Запускает один BrowserSession на весь batch; jitter_sleep (800–1500 мс)
+    встроен в fetch_catalog_object_html для защиты от rate-limit.
+
+    Returns:
+        {"processed": N, "succeeded": N, "failed": N, "skipped": N}
+    """
+    stats: dict[str, int] = {
+        "processed": 0,
+        "succeeded": 0,
+        "failed": 0,
+        "skipped": 0,
+    }
+
+    if not obj_ids:
+        logger.info("scrape_catalog_objects: empty list, nothing to do")
+        return stats
+
+    logger.info(
+        "scrape_catalog_objects: starting %d objects region=%d snapshot_date=%s",
+        len(obj_ids),
+        region_code,
+        snapshot_date,
+    )
+
+    async with BrowserSession(
+        region_code=region_code,
+        # Страницы каталога публичные — Basic auth не нужен
+        auth=None,
+    ) as session:
+        for obj_id in obj_ids:
+            stats["processed"] += 1
+            ok = await scrape_catalog_object(db, session, obj_id, snapshot_date)
+            if ok:
+                stats["succeeded"] += 1
+            else:
+                stats["failed"] += 1
+
+    logger.info(
+        "scrape_catalog_objects done: processed=%d succeeded=%d failed=%d skipped=%d",
+        stats["processed"],
+        stats["succeeded"],
+        stats["failed"],
+        stats["skipped"],
+    )
+    return stats

backend/app/workers/beat_schedule.py

@@ -237,6 +237,16 @@ def build_beat_schedule() -> dict:
         "options": {"queue": "celery"},
     }
 
+    # Catalog-object scrape — наполняет ~25 NULL колонок domrf_kn_objects из SSR-страниц.
+    # kn-API не отдаёт wall_type, energy_eff, ceiling_height_m, parking_* и т.д.
+    # Вторник 04:00 UTC. batch 300/run → 1532 объекта за ~5 недель полного обновления.
+    schedule["scrape-kn-catalog-objects-weekly"] = {
+        "task": "tasks.scrape_kn_catalog_objects.scrape_kn_catalog_objects",
+        "schedule": _parse_cron("0 4 * * 2"),  # Tuesday 04:00 UTC
+        "kwargs": {"region_code": 66, "max_objects": 300},
+        "options": {"queue": "celery"},
+    }
+
     # NSPD quarter dump refresh — re-enabled 2026-05-17 после Sub-PR B (#260)
     # переключения search_by_quarter на grid-walk. Foundation (#247) + integration
     # (#260) теперь возвращают полноценные dumps (territorial_zones, ЗОУИТ, risk

backend/app/workers/tasks/scrape_kn_catalog_objects.py

@@ -0,0 +1,157 @@
+"""Celery task: periodic catalog-object scrape для DOM.РФ.
+
+Дополняет ~25 NULL колонок в domrf_kn_objects из SSR-страниц каталога.
+kn-API эти поля не возвращает — они только на публичных страницах объектов.
+
+Выбирает объекты где catalog_scraped_at IS NULL или устарело (>30 дней).
+Ограничивает batch per run чтобы не нагружать сайт.
+
+Beat schedule: вторник 04:00 UTC (в beat_schedule.py).
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from datetime import date
+from typing import Any
+
+from sqlalchemy import text
+
+from app.core.db import SessionLocal
+from app.workers.celery_app import celery_app
+
+logger = logging.getLogger(__name__)
+
+# Запрос для выбора устаревших / ненаполненных объектов.
+# Возвращает obj_id + snapshot_date одним запросом, чтобы избежать race condition:
+# если между двумя запросами kn-scraper запишет новый snapshot — UPDATE по старой
+# snapshot_date не затронет ни одной строки.  MAX subquery ограничена тем же
+# region_cd чтобы не захватить snapshot другого региона.
+_SELECT_STALE_SQL = text(
+    """
+    SELECT obj_id, snapshot_date
+      FROM domrf_kn_objects
+     WHERE region_cd = :region_code
+       AND (
+           catalog_scraped_at IS NULL
+           OR catalog_scraped_at < NOW() - INTERVAL '30 days'
+       )
+       AND snapshot_date = (
+           SELECT MAX(snapshot_date)
+             FROM domrf_kn_objects
+            WHERE region_cd = :region_code
+       )
+     ORDER BY catalog_scraped_at NULLS FIRST
+     LIMIT :max_objects
+    """
+)
+
+# Лимит по умолчанию если max_objects не задан явно.
+_DEFAULT_MAX_OBJECTS = 300
+
+
+@celery_app.task(
+    bind=True,
+    name="tasks.scrape_kn_catalog_objects.scrape_kn_catalog_objects",
+    time_limit=3600,
+)
+def scrape_kn_catalog_objects(
+    self: Any,
+    region_code: int = 66,
+    max_objects: int | None = None,
+) -> dict[str, Any]:
+    """Periodic catalog-object scrape.
+
+    Picks objects from domrf_kn_objects where catalog data missing OR stale
+    (catalog_scraped_at < NOW() - INTERVAL '30 days'). Limits per run.
+
+    Args:
+        region_code: Код региона (ОКАТО prefix). Default 66 = Свердловская обл.
+        max_objects: Максимум объектов за один run. Default 300.
+
+    Returns:
+        dict с ключами: region_code, snapshot_date, obj_ids_count,
+        processed, succeeded, failed, skipped.
+
+    Concurrency:
+        No Redis lock — consistent with sibling tasks (scrape_kn_region etc.).
+        Beat is configured for non-overlapping fire (Tuesday 04:00 UTC, ~5min run),
+        so concurrent execution is extremely rare. If it occurs:
+        - UPDATE is idempotent (COALESCE, catalog_scraped_at = NOW())
+        - Max risk: 2x WAF load on DOM.РФ for the same batch
+        - Both tasks complete; second update is no-op (catalog_scraped_at расхождение)
+
+        Add Redis lock if WAF blocks observed or beat schedule changes to overlap.
+    """
+    from app.services.scrapers.domrf_catalog_object import scrape_catalog_objects
+
+    limit = max_objects if max_objects is not None else _DEFAULT_MAX_OBJECTS
+
+    db = SessionLocal()
+    try:
+        rows = (
+            db.execute(
+                _SELECT_STALE_SQL,
+                {"region_code": region_code, "max_objects": limit},
+            )
+            .mappings()
+            .all()
+        )
+        obj_ids: list[int] = [int(r["obj_id"]) for r in rows]
+    except Exception as exc:
+        logger.error("scrape_kn_catalog_objects: failed to fetch obj_ids: %s", exc)
+        db.close()
+        raise
+
+    if not obj_ids:
+        logger.info(
+            "scrape_kn_catalog_objects: no stale objects for region=%d, nothing to do",
+            region_code,
+        )
+        db.close()
+        return {
+            "region_code": region_code,
+            "obj_ids_count": 0,
+            "processed": 0,
+            "succeeded": 0,
+            "failed": 0,
+            "skipped": 0,
+        }
+
+    # snapshot_date берётся из первой строки результата — все строки одинаковые
+    # (WHERE snapshot_date = MAX(snapshot_date)).  Это атомарно: один SELECT вместо двух,
+    # что устраняет race condition с kn-scraper.
+    snapshot_date_val: date = rows[0]["snapshot_date"]
+
+    logger.info(
+        "scrape_kn_catalog_objects: region=%d snapshot_date=%s obj_ids=%d (limit=%d)",
+        region_code,
+        snapshot_date_val,
+        len(obj_ids),
+        limit,
+    )
+
+    try:
+        stats = asyncio.run(
+            scrape_catalog_objects(
+                db=db,
+                obj_ids=obj_ids,
+                snapshot_date=snapshot_date_val,
+                region_code=region_code,
+            )
+        )
+    except Exception as exc:
+        logger.error("scrape_kn_catalog_objects: scrape failed: %s", exc)
+        raise
+    finally:
+        db.close()
+
+    result: dict[str, Any] = {
+        "region_code": region_code,
+        "snapshot_date": str(snapshot_date_val),
+        "obj_ids_count": len(obj_ids),
+        **stats,
+    }
+    logger.info("scrape_kn_catalog_objects done: %s", result)
+    return result

backend/tests/services/scrapers/test_domrf_catalog_object.py

@@ -0,0 +1,290 @@
+"""Тесты для domrf_catalog_object.py (issue #297 sub-task 22d).
+
+Покрывает:
+  - extract_next_data — парсинг HTML с __NEXT_DATA__
+  - parse_catalog_object — маппинг pageProps → DB columns
+  - value helpers (_to_numeric_comma, _to_bool_da_net, _to_date_ddmmyyyy)
+  - partial responses (partial pageProps → all other fields = None, no crash)
+"""
+
+from __future__ import annotations
+
+from datetime import date
+from typing import Any
+
+import pytest
+
+from app.services.scrapers.domrf_catalog_object import (
+    _to_bool_da_net,
+    _to_bool_int,
+    _to_date_ddmmyyyy,
+    _to_numeric_comma,
+    extract_next_data,
+    parse_catalog_object,
+)
+
+# ── extract_next_data ─────────────────────────────────────────────────────────
+
+
+def test_extract_next_data_from_html() -> None:
+    """Базовый case: тег найден, JSON возвращается как dict."""
+    html = (
+        "<html><head>"
+        '<script id="__NEXT_DATA__" type="application/json">'
+        '{"props":{"pageProps":{"buildingClass":"Комфорт"}}}'
+        "</script>"
+        "</head></html>"
+    )
+    result = extract_next_data(html)
+    assert isinstance(result, dict)
+    assert result["props"]["pageProps"]["buildingClass"] == "Комфорт"
+
+
+def test_extract_next_data_single_quotes() -> None:
+    """Тег с одинарными кавычками тоже должен парситься."""
+    html = "<script id='__NEXT_DATA__'>" '{"props":{"pageProps":{}}}' "</script>"
+    result = extract_next_data(html)
+    assert "props" in result
+
+
+def test_extract_next_data_not_found_raises() -> None:
+    """Если тег не найден — ValueError."""
+    with pytest.raises(ValueError, match="__NEXT_DATA__"):
+        extract_next_data("<html><body>no script here</body></html>")
+
+
+def test_extract_next_data_invalid_json_raises() -> None:
+    """Если JSON некорректный — ValueError."""
+    html = '<script id="__NEXT_DATA__">{broken json</script>'
+    with pytest.raises(ValueError):
+        extract_next_data(html)
+
+
+# ── parse_catalog_object — full sample ───────────────────────────────────────
+
+
+def _make_full_next_data() -> dict[str, Any]:
+    """Реалистичный full next_data для obj_id=65136 (подтверждён live)."""
+    return {
+        "props": {
+            "pageProps": {
+                "buildingClass": "Комфорт",
+                "wallMaterial": "Монолит-кирпич",
+                "objEnergyEfficiency": "B",
+                "parkingCount": 246,
+                "finishTypeCount": 1,
+                "freePlan": "Нет",
+                "publicationDate": "31.03.2025",
+                "additionalInfo": {
+                    "objectParkingPlaces": 43,
+                    "nearbyParkingPlaces": 0,
+                    "ceilingHeight": "2,7",
+                    "passengerElevatorsCount": 0,
+                    "cargoElevatorsCount": 0,
+                    "cargoPassengerElevatorCount": 4,
+                    "playgroundsCount": 6,
+                    "sportsgroundCount": 5,
+                    "bicycleLane": 0,
+                    "trashAreaCount": 3,
+                    "ramp": 0,
+                    "curbLowering": 1,
+                    "wheelchairElevatorsCount": 0,
+                    "parkingAvailabilityPerc": 60,
+                },
+                "quartography": {
+                    "objLivElemEntrCnt": 1,
+                    "objLivElemSqAvg": 46.2,
+                    "nonLivFirstFloor": 1,
+                },
+                "indexes": {
+                    "infrastructure": 10,
+                    "transport": 6,
+                },
+                "projectDeclaration": {
+                    "number": "66-001686",
+                },
+            }
+        }
+    }
+
+
+def test_parse_catalog_object_full() -> None:
+    """Полный sample: все 25+ полей должны быть замаплены корректно."""
+    data = parse_catalog_object(_make_full_next_data())
+
+    assert data["obj_class"] == "Комфорт"
+    assert data["wall_type"] == "Монолит-кирпич"
+    assert data["energy_eff"] == "B"
+    assert data["section_count"] == 1
+    assert data["parking_total_slots"] == 246
+    assert data["guest_parking_inside_count"] == 43
+    assert data["guest_parking_outside_count"] == 0
+    assert data["ceiling_height_m"] == pytest.approx(2.7)
+    assert data["finishing_variants_count"] == 1
+    assert data["has_free_planning"] is False
+    assert data["avg_flat_area_m2"] == pytest.approx(46.2)
+    assert data["elevators_passenger_count"] == 0
+    assert data["elevators_cargo_count"] == 4  # 0 + 4
+    assert data["playground_kids_count"] == 6
+    assert data["playground_sport_count"] == 5
+    assert data["has_bike_paths"] is False  # bicycleLane=0
+    assert data["trash_areas_count"] == 3
+    assert data["has_ramp"] is False  # ramp=0
+    assert data["has_low_platforms"] is True  # curbLowering=1
+    assert data["has_wheelchair_lift"] is False  # wheelchairElevatorsCount=0
+    assert data["first_floor_type"] == "нежилой"  # nonLivFirstFloor=1
+    assert data["parking_provision_pct"] == 60
+    assert data["project_published_at"] == date(2025, 3, 31)
+    assert data["project_declaration_num"] == "66-001686"
+    assert data["domrf_score_infrastructure"] == 10
+    assert data["domrf_score_transport"] == 6
+
+
+def test_parse_catalog_object_has_free_planning_da() -> None:
+    """freePlan='Да' → has_free_planning=True."""
+    nd: dict[str, Any] = {"props": {"pageProps": {"freePlan": "Да"}}}
+    data = parse_catalog_object(nd)
+    assert data["has_free_planning"] is True
+
+
+def test_parse_catalog_object_first_floor_zhiloj() -> None:
+    """nonLivFirstFloor=0 → first_floor_type='жилой'."""
+    nd: dict[str, Any] = {
+        "props": {
+            "pageProps": {
+                "quartography": {"nonLivFirstFloor": 0},
+            }
+        }
+    }
+    data = parse_catalog_object(nd)
+    assert data["first_floor_type"] == "жилой"
+
+
+def test_parse_catalog_object_elevators_cargo_sum() -> None:
+    """elevators_cargo_count = cargoElevatorsCount + cargoPassengerElevatorCount."""
+    nd: dict[str, Any] = {
+        "props": {
+            "pageProps": {
+                "additionalInfo": {
+                    "cargoElevatorsCount": 2,
+                    "cargoPassengerElevatorCount": 3,
+                }
+            }
+        }
+    }
+    data = parse_catalog_object(nd)
+    assert data["elevators_cargo_count"] == 5
+
+
+def test_parse_catalog_object_partial() -> None:
+    """Только buildingClass → остальные поля None, без исключений."""
+    nd: dict[str, Any] = {"props": {"pageProps": {"buildingClass": "Бизнес"}}}
+    data = parse_catalog_object(nd)
+    assert data["obj_class"] == "Бизнес"
+    assert data["wall_type"] is None
+    assert data["energy_eff"] is None
+    assert data["section_count"] is None
+    assert data["parking_total_slots"] is None
+    assert data["ceiling_height_m"] is None
+    assert data["has_free_planning"] is None
+    assert data["elevators_cargo_count"] is None
+    assert data["project_published_at"] is None
+    assert data["domrf_score_infrastructure"] is None
+
+
+def test_parse_catalog_object_empty() -> None:
+    """Полностью пустой next_data → все поля None, без исключений."""
+    data = parse_catalog_object({})
+    for v in data.values():
+        assert v is None
+
+
+def test_parking_provision_pct_preserves_float() -> None:
+    """parking_provision_pct should preserve fractional values (column is numeric(5,1))."""
+    next_data: dict[str, Any] = {
+        "props": {"pageProps": {"id": 65136, "additionalInfo": {"parkingAvailabilityPerc": 60.5}}}
+    }
+    result = parse_catalog_object(next_data)
+    assert result["parking_provision_pct"] == 60.5
+
+
+# ── _to_numeric_comma ─────────────────────────────────────────────────────────
+
+
+@pytest.mark.parametrize(
+    "inp,expected",
+    [
+        ("2,7", 2.7),
+        ("2.7", 2.7),
+        ("3,50", 3.5),
+        ("", None),
+        (None, None),
+        ("   ", None),
+        ("abc", None),
+    ],
+)
+def test_to_numeric_comma(inp: Any, expected: float | None) -> None:
+    result = _to_numeric_comma(inp)
+    if expected is None:
+        assert result is None
+    else:
+        assert result == pytest.approx(expected)
+
+
+# ── _to_bool_da_net ───────────────────────────────────────────────────────────
+
+
+@pytest.mark.parametrize(
+    "inp,expected",
+    [
+        ("Да", True),
+        ("да", True),
+        ("ДА", True),
+        ("Нет", False),
+        ("нет", False),
+        ("НЕТ", False),
+        ("", None),
+        (None, None),
+        ("maybe", None),
+        ("Yes", None),
+    ],
+)
+def test_to_bool_da_net(inp: Any, expected: bool | None) -> None:
+    assert _to_bool_da_net(inp) == expected
+
+
+# ── _to_bool_int ──────────────────────────────────────────────────────────────
+
+
+@pytest.mark.parametrize(
+    "inp,expected",
+    [
+        (0, False),
+        (1, True),
+        (5, True),
+        ("1", True),
+        ("0", False),
+        (None, None),
+    ],
+)
+def test_to_bool_int(inp: Any, expected: bool | None) -> None:
+    assert _to_bool_int(inp) == expected
+
+
+# ── _to_date_ddmmyyyy ─────────────────────────────────────────────────────────
+
+
+@pytest.mark.parametrize(
+    "inp,expected",
+    [
+        ("31.03.2025", date(2025, 3, 31)),
+        ("01.01.2024", date(2024, 1, 1)),
+        ("", None),
+        (None, None),
+        ("2025-03-31", None),  # неправильный формат → None
+        ("abc", None),
+        ("31.13.2025", None),  # невалидный месяц → None
+    ],
+)
+def test_to_date_ddmmyyyy(inp: Any, expected: date | None) -> None:
+    assert _to_date_ddmmyyyy(inp) == expected

data/sql/118_22d_catalog_object_scraped_at.sql

@@ -0,0 +1,15 @@
+-- Migration 118: добавить catalog_scraped_at в domrf_kn_objects
+-- Нужна для catalog-object scraper (issue #297, sub-task 22d):
+-- scraper обновляет поле после успешного UPDATE, beat task выбирает
+-- только объекты где это поле NULL или устарело (> 30 дней).
+-- Индекс NULLS FIRST ускоряет ORDER BY catalog_scraped_at NULLS FIRST LIMIT N.
+
+BEGIN;
+
+ALTER TABLE domrf_kn_objects
+    ADD COLUMN IF NOT EXISTS catalog_scraped_at TIMESTAMP;
+
+CREATE INDEX IF NOT EXISTS idx_domrf_kn_objects_catalog_scraped_at
+    ON domrf_kn_objects (catalog_scraped_at NULLS FIRST);
+
+COMMIT;