refactor(nspd): remove Playwright-based scraper, rosreestr2coord = default

УДАЛЕНО (~1700 строк): * backend/app/services/scrapers/nspd_kn.py (Playwright + WAF-bypass) * backend/app/workers/tasks/scrape_nspd.py * frontend/src/app/admin/scrape/nspd/page.tsx * 5 NSPD admin endpoints (POST /nspd, /nspd/release-lock, GET /nspd/runs, /logs, /coverage) * NSPD beat schedule + worker_ready resume hook + nav tab + link * _nspd_default_regions() helper СОХРАНЕНО (история): * nspd_scrape_runs / nspd_scrape_log таблицы (3 row + log) для аудита * UNION ALL в v_scrape_runs_unified / v_scrape_log_unified — старые runs видны под фильтром scraper_type='nspd' * settings scrape_nspd_* помечены DEPRECATED CHANGED: * nspd_geo_jobs.use_rosreestr2coord DEFAULT FALSE → TRUE * UI checkbox "rosreestr2coord lib" по умолчанию = checked * /admin/scrape/all + /admin/scrape/geo — единственный путь к NSPD-данным NSPD-доступ теперь только через rosreestr2coord (community lib) — upstream поддерживает обновления headers/WAF-tricks. Локальный nspd_lite.py (urllib) остаётся как fallback (use_rosreestr2coord=FALSE).
2026-05-11 09:21:12 +03:00 · 2026-05-11 09:21:12 +03:00 · 79e2e94e09
commit 79e2e94e09
parent 54dbc77348
11 changed files with 165 additions and 1756 deletions
--- a/backend/app/api/v1/admin_scrape.py
+++ b/backend/app/api/v1/admin_scrape.py
@ -358,212 +358,10 @@ def list_logs(
    ]
-# ── NSPD (cad_quarters / cad_buildings) ─────────────────────────────────────
+# NSPD-скрапер УДАЛЁН 2026-05-11: вместо Playwright-based nspd_kn.py теперь
-
+# используется bulk geo-fetcher через rosreestr2coord. См. /admin/scrape/geo
-
+# и backend/app/workers/tasks/nspd_geo.py + nspd_lite.py (urllib).
-class TriggerNspdRequest(BaseModel):
+# История прогонов осталась в nspd_scrape_runs/log + v_scrape_runs_unified.
    region_code: int = Field(default=66, ge=1, le=99)
    limit: int | None = Field(default=None, ge=1, le=2000)
    rate_ms: int = Field(default=600, ge=200, le=5000)
    force: bool = False
@router.post("/nspd")
 def trigger_nspd_sweep(
    payload: TriggerNspdRequest,
    x_admin_token: Annotated[str | None, Header(alias="X-Admin-Token")] = None,
 ) -> dict[str, Any]:
    _check_token(x_admin_token)
    from app.workers.tasks.scrape_nspd import (
        force_release_lock as nspd_force_release_lock,
    )
    from app.workers.tasks.scrape_nspd import (
        scrape_nspd_region,
    )
    lock_released = False
    if payload.force:
        lock_released = nspd_force_release_lock(payload.region_code)
    result = scrape_nspd_region.apply_async(
        kwargs={
            "region_code": payload.region_code,
            "triggered_by": "manual",
            "limit": payload.limit,
            "rate_ms": payload.rate_ms,
        },
    )
    return {
        "task_id": result.id,
        "region_code": payload.region_code,
        "limit": payload.limit,
        "rate_ms": payload.rate_ms,
        "force": payload.force,
        "lock_was_released": lock_released,
    }
 class NspdReleaseLockRequest(BaseModel):
    region_code: int = Field(..., ge=1, le=99)
@router.post("/nspd/release-lock")
 def nspd_release_lock(
    payload: NspdReleaseLockRequest,
    x_admin_token: Annotated[str | None, Header(alias="X-Admin-Token")] = None,
 ) -> dict[str, Any]:
    _check_token(x_admin_token)
    from app.workers.tasks.scrape_nspd import force_release_lock as nspd_force_release_lock
    released = nspd_force_release_lock(payload.region_code)
    return {"region_code": payload.region_code, "released": released}
@router.get("/nspd/runs")
 def list_nspd_runs(
    db: Annotated[Session, Depends(get_db)],
    x_admin_token: Annotated[str | None, Header(alias="X-Admin-Token")] = None,
    limit: int = 20,
 ) -> list[dict[str, Any]]:
    _check_token(x_admin_token)
    rows = (
        db.execute(
            text(
                """
                SELECT run_id, started_at, finished_at, heartbeat_at,
                       region_code, pending_count,
                       quarters_ok, quarters_failed, buildings_ok,
                       requests_count, waf_429_count,
                       status, error, triggered_by
                  FROM nspd_scrape_runs
                 ORDER BY started_at DESC
                 LIMIT :lim
                """
            ),
            {"lim": min(limit, 100)},
        )
        .mappings()
        .all()
    )
    return [
        {
            "run_id": r["run_id"],
            "started_at": r["started_at"].isoformat() if r["started_at"] else None,
            "finished_at": r["finished_at"].isoformat() if r["finished_at"] else None,
            "heartbeat_at": r["heartbeat_at"].isoformat() if r["heartbeat_at"] else None,
            "region_code": r["region_code"],
            "pending_count": r["pending_count"],
            "quarters_ok": r["quarters_ok"],
            "quarters_failed": r["quarters_failed"],
            "buildings_ok": r["buildings_ok"],
            "requests_count": r["requests_count"],
            "waf_429_count": r["waf_429_count"],
            "status": r["status"],
            "error": r["error"],
            "triggered_by": r["triggered_by"],
        }
        for r in rows
    ]
@router.get("/nspd/logs")
 def list_nspd_logs(
    db: Annotated[Session, Depends(get_db)],
    x_admin_token: Annotated[str | None, Header(alias="X-Admin-Token")] = None,
    run_id: int | None = None,
    since_id: int | None = None,
    limit: int = 200,
 ) -> list[dict[str, Any]]:
    _check_token(x_admin_token)
    where: list[str] = []
    params: dict[str, Any] = {"lim": min(limit, 1000)}
    if run_id is not None:
        where.append("run_id = :rid")
        params["rid"] = run_id
    if since_id is not None:
        where.append("log_id > :sid")
        params["sid"] = since_id
    where_sql = ("WHERE " + " AND ".join(where)) if where else ""
    rows = (
        db.execute(
            text(
                f"""
                SELECT log_id, run_id, ts, level, stage, cad_number, message
                  FROM nspd_scrape_log
                  {where_sql}
                 ORDER BY log_id DESC
                 LIMIT :lim
                """
            ),
            params,
        )
        .mappings()
        .all()
    )
    return [
        {
            "log_id": r["log_id"],
            "run_id": r["run_id"],
            "ts": r["ts"].isoformat() if r["ts"] else None,
            "level": r["level"],
            "stage": r["stage"],
            "cad_number": r["cad_number"],
            "message": r["message"],
        }
        for r in rows
    ]
@router.get("/nspd/coverage")
 def nspd_coverage(
    db: Annotated[Session, Depends(get_db)],
    x_admin_token: Annotated[str | None, Header(alias="X-Admin-Token")] = None,
    region_code: int = 66,
 ) -> dict[str, Any]:
    """Покрытие cad_quarters_geom относительно ДДУ-сделок rosreestr.
    Полезно перед запуском sweep — видишь сколько ещё осталось скрейпить."""
    _check_token(x_admin_token)
    # ВАЖНО: regex с двоеточиями (':00:', ':0000000') ломают SQLAlchemy text() —
    # двоеточие парсится как named-parameter prefix. Поэтому паттерны передаём
    # через bindparams. NOT LIKE безопасен и быстрее regex.
    row = (
        db.execute(
            text(
                """
                WITH pending AS (
                    SELECT DISTINCT quarter_cad_number AS cad
                      FROM rosreestr_deals
                     WHERE region_code = :rc
                       AND doc_type = 'ДДУ'
                       AND realestate_type_code = '002001003000'
                       AND quarter_cad_number IS NOT NULL
                       AND quarter_cad_number NOT LIKE :bad_prefix
                       AND quarter_cad_number NOT LIKE :bad_suffix
                ),
                done AS (SELECT cad_number AS cad FROM cad_quarters_geom)
                SELECT
                    (SELECT COUNT(*) FROM pending) AS total_with_ddu,
                    (SELECT COUNT(*) FROM pending p
                       WHERE EXISTS (SELECT 1 FROM done d WHERE d.cad = p.cad))
                        AS already_scraped,
                    (SELECT COUNT(*) FROM cad_buildings) AS buildings_total
                """
            ),
            {"rc": region_code, "bad_prefix": "00:00:%", "bad_suffix": "%:0000000"},
        )
        .mappings()
        .first()
    )
    total = int(row["total_with_ddu"] or 0)
    done = int(row["already_scraped"] or 0)
    return {
        "region_code": region_code,
        "total_quarters_with_ddu": total,
        "already_scraped": done,
        "pending": max(total - done, 0),
        "buildings_total": int(row["buildings_total"] or 0),
        "coverage_pct": round(done / max(total, 1) * 100, 1) if total else None,
    }
 # ── Objective ETL (Антоновский SQLite → PG) ─────────────────────────────────
--- a/backend/app/core/config.py
+++ b/backend/app/core/config.py
@ -34,14 +34,12 @@ class Settings(BaseSettings):
    # Empty string = endpoint disabled.
    scrape_admin_token: str = ""
-    # NSPD-scraper schedule (кадастровые кварталы / здания).
+    # NSPD-scraper (Playwright) УДАЛЁН 2026-05-11. Сменён на bulk geo-fetcher
-    # По умолчанию: 20-е число февраля/мая/августа/ноября в 03:30 МСК
+    # через rosreestr2coord — запускается вручную через /admin/scrape/geo.
-    # (после квартальных публикаций rosreestr_deals).
+    # Settings оставлены deprecated на случай отката (можно удалить позже).
-    scrape_nspd_cron: str = "30 3 20 2,5,8,11 *"
+    scrape_nspd_cron: str = "30 3 20 2,5,8,11 *"  # DEPRECATED
-    # Регионы для NSPD-sweep (comma-separated rosreestr region codes).
+    scrape_nspd_default_regions: str = "66"  # DEPRECATED
-    scrape_nspd_default_regions: str = "66"
+    scrape_nspd_rate_ms: int = 600  # DEPRECATED — see nspd_lite_rate_ms
    # Пауза между NSPD-запросами в мс. <600мс — высокий риск WAF 403.
    scrape_nspd_rate_ms: int = 600
    # Objective.ru API (api.objctv.ru) — платная аналитика первички/вторички.
    # Лежит наряду с DOM.РФ kn + rosreestr CSV как 3-й источник истины с
--- a/backend/app/services/scrapers/nspd_kn.py
+++ b/backend/app/services/scrapers/nspd_kn.py
@ -1,576 +0,0 @@
 """NSPD scraper — кадастровые кварталы и здания через nspd.gov.ru API.
 Endpoints:
    GET /api/geoportal/v2/search/geoportal?thematicSearchId=2&query={cad}
        → polygon кадастрового квартала (1 feature с label = cad-номер)
    GET /api/geoportal/v2/search/geoportal?thematicSearchId=1&query={cad}
        → объекты внутри квартала (здания, земля, линейные).
        Фильтруем categoryName='Здания' и cad_num LIKE '<region>:<district>:%'.
 WAF nspd.gov.ru банит burst-запросы и нероссийские IP. Стратегия:
 - rate-limit 600мс между запросами (≤100 req/min — наблюдаемый предел)
 - exponential backoff на 403 (60s + 30s × attempt)
 - запускать с RU IP (production worker должен быть в RU/CIS-zone)
 Этот модуль — рефакторинг data/sql/62_scrape_nspd_full.py из stand-alone скрипта
 в библиотечную функцию для Celery. Логика идентична, но:
 - работает через SQLAlchemy Session (FK к cad_quarters_geom/cad_buildings)
 - пишет heartbeat и progress в nspd_scrape_runs
 - логирует структурированно в nspd_scrape_log
 """
 from __future__ import annotations
 import json
 import logging
 import ssl
 import time
 import urllib.error
 import urllib.parse
 import urllib.request
 from datetime import datetime
 from typing import Any
 from sqlalchemy import text
 from sqlalchemy.orm import Session
 from app.core.db import SessionLocal
 logger = logging.getLogger(__name__)
 NSPD_BASE = "https://nspd.gov.ru/api/geoportal/v2/search/geoportal"
 HEADERS = {
    "User-Agent": (
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
        "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
    ),
    "Accept": "application/json",
    "Accept-Language": "ru-RU,ru;q=0.9",
    "Referer": "https://nspd.gov.ru/map",
 }
 SSL_CTX = ssl._create_unverified_context()
 DEFAULT_RATE_MS = 600
 # Heartbeat каждый cad — даёт UI видимый прогресс и быстрый zombie-detect
 # на прод-VM, где worker может тихо стопать без error в логе.
 DEFAULT_HEARTBEAT_EVERY = 1  # quarters
 DEFAULT_COMMIT_EVERY = 10
 # Retries × timeout × WAF backoff формирует worst-case на один cad.
 # Старые значения (5×30s + 60+90+120+150+180s = 600+150 = ~12.5 мин) дают
 # инвиз-зависание worker'а если NSPD WAF режет nonbrowser fingerprint.
 # Новые: до ~3 мин на cad, чтобы при 'тихом' WAF run падал быстрее.
 DEFAULT_RETRIES = 3
 DEFAULT_TIMEOUT_S = 12
 # region_code (rosreestr) → cad-prefix фильтр для cad_buildings.
 # 66 = Свердловская обл., 66:41 = ЕКБ. Для других регионов добавлять mapping.
 REGION_CAD_PREFIX: dict[int, str] = {
    66: "66:41:",  # Екатеринбург
 }
 # ── HTTP layer ────────────────────────────────────────────────────────────────
 class WafBlockedError(RuntimeError):
    """403 после всех retry — WAF банит, прогон не продолжаем."""
 def nspd_fetch(
    thematic_search_id: int,
    query: str,
    *,
    retries: int = DEFAULT_RETRIES,
    timeout: int = DEFAULT_TIMEOUT_S,
    on_403: Any = None,
 ) -> dict | None:
    """Fetch NSPD с retry/backoff. on_403 callable(attempt) — для трекинга WAF."""
    qs = urllib.parse.urlencode({"thematicSearchId": thematic_search_id, "query": query})
    url = f"{NSPD_BASE}?{qs}"
    req = urllib.request.Request(url, headers=HEADERS)
    last_err: Exception | None = None
    for attempt in range(retries):
        try:
            with urllib.request.urlopen(req, timeout=timeout, context=SSL_CTX) as r:
                return json.loads(r.read().decode("utf-8"))
        except urllib.error.HTTPError as e:
            last_err = e
            if e.code == 404:
                return None
            if e.code == 403:
                if on_403:
                    on_403(attempt)
                wait = 60 + 30 * attempt
                logger.warning(
                    "WAF 403 для %s (попытка %d/%d), пауза %ds", query, attempt + 1, retries, wait
                )
                time.sleep(wait)
                continue
            logger.warning("HTTP %s для %s (попытка %d)", e.code, query, attempt + 1)
        except (urllib.error.URLError, TimeoutError, OSError) as e:
            last_err = e
            logger.warning("Сетевая ошибка для %s: %s (попытка %d)", query, e, attempt + 1)
        time.sleep(min(2**attempt, 30))
    if isinstance(last_err, urllib.error.HTTPError) and last_err.code == 403:
        raise WafBlockedError(f"WAF banned after {retries} retries on {query}")
    logger.error("FAILED %s после %d попыток: %s", query, retries, last_err)
    return None
 # ── Geometry helpers ──────────────────────────────────────────────────────────
 def poly_to_wkt(geom: dict | None) -> str | None:
    if not geom:
        return None
    def ring(r: list) -> str:
        return "(" + ",".join(f"{p[0]} {p[1]}" for p in r) + ")"
    t = geom.get("type")
    if t == "Polygon":
        return "POLYGON(" + ",".join(ring(r) for r in geom["coordinates"]) + ")"
    if t == "MultiPolygon":
        return (
            "MULTIPOLYGON("
            + ",".join("(" + ",".join(ring(r) for r in p) + ")" for p in geom["coordinates"])
            + ")"
        )
    if t == "Point":
        return f"POINT({geom['coordinates'][0]} {geom['coordinates'][1]})"
    return None
 # ── SQL helpers ───────────────────────────────────────────────────────────────
 def _to_int(x: Any) -> int | None:
    if x is None or x == "":
        return None
    try:
        return int(x)
    except (TypeError, ValueError):
        return None
 def _to_num(x: Any) -> float | None:
    if x is None or x == "":
        return None
    try:
        return float(x)
    except (TypeError, ValueError):
        return None
 def _to_date(x: Any) -> str | None:
    if not x:
        return None
    s = str(x).split(" ")[0]
    return s or None
 def get_pending_cads(db: Session, region_code: int) -> list[str]:
    """Cad-кварталы с ДДУ-сделками в регионе минус уже scraped.
    NOTE: regex с двоеточиями ('!~ ":0000000$"') ломают SQLAlchemy text() —
    двоеточие парсится как named-parameter prefix. Используем NOT LIKE через
    bindparam (быстрее regex и безопасно).
    """
    rows = db.execute(
        text(
            """
            SELECT DISTINCT quarter_cad_number
              FROM rosreestr_deals
             WHERE region_code = :rc
               AND doc_type = 'ДДУ'
               AND realestate_type_code = '002001003000'
               AND quarter_cad_number IS NOT NULL
               AND quarter_cad_number <> ''
               AND quarter_cad_number NOT LIKE :bad_prefix
               AND quarter_cad_number NOT LIKE :bad_suffix
            """
        ),
        {"rc": region_code, "bad_prefix": "00:00:%", "bad_suffix": "%:0000000"},
    ).all()
    pending = {r[0] for r in rows}
    done = {r[0] for r in db.execute(text("SELECT cad_number FROM cad_quarters_geom")).all()}
    return sorted(pending - done)
 def insert_quarter(db: Session, cad: str, wkt: str, raw_props: dict | None) -> None:
    db.execute(
        text(
            """
            INSERT INTO cad_quarters_geom (cad_number, geom, raw_props, source)
            VALUES (
                :cad,
                ST_Multi(ST_Transform(ST_SetSRID(ST_GeomFromText(:wkt), 3857), 4326))
                    ::geometry(MultiPolygon, 4326),
                CAST(:props AS jsonb),
                'nspd'
            )
            ON CONFLICT (cad_number) DO UPDATE SET
                geom = EXCLUDED.geom,
                raw_props = COALESCE(EXCLUDED.raw_props, cad_quarters_geom.raw_props),
                fetched_at = NOW()
            """
        ),
        {
            "cad": cad,
            "wkt": wkt,
            "props": json.dumps(raw_props, ensure_ascii=False) if raw_props else None,
        },
    )
 def insert_buildings(db: Session, qcad: str, features: list[dict], cad_prefix: str) -> int:
    n = 0
    for f in features:
        props_outer = f.get("properties") or {}
        if props_outer.get("categoryName") != "Здания":
            continue
        opts = props_outer.get("options") or {}
        cn = opts.get("cad_num")
        if not cn or not cn.startswith(cad_prefix):
            continue
        wkt = poly_to_wkt(f.get("geometry"))
        if not wkt:
            continue
        db.execute(
            text(
                """
                INSERT INTO cad_buildings (
                    cad_num, quarter_cad_num, geom, purpose, building_name,
                    readable_address, area, floors, year_built, year_commisioning,
                    cost_value, registration_date,
                    status, ownership_type, cultural_heritage, underground_floors,
                    build_record_area, build_record_type, common_data_status, obj_type,
                    raw_props
                )
                VALUES (
                    :cad_num, :qcad,
                    ST_Transform(ST_SetSRID(ST_GeomFromText(:wkt), 3857), 4326),
                    :purpose, :name, :addr, :area, :floors, :yb, :yc,
                    :cost, :reg_date,
                    :status, :ownership, :cultural, :underground,
                    :build_rec_area, :build_rec_type, :common_status, :obj_type,
                    CAST(:raw_props AS jsonb)
                )
                ON CONFLICT (cad_num) DO UPDATE SET
                    quarter_cad_num = EXCLUDED.quarter_cad_num,
                    geom = EXCLUDED.geom,
                    purpose = EXCLUDED.purpose,
                    building_name = EXCLUDED.building_name,
                    readable_address = EXCLUDED.readable_address,
                    area = EXCLUDED.area,
                    floors = EXCLUDED.floors,
                    year_built = EXCLUDED.year_built,
                    year_commisioning = EXCLUDED.year_commisioning,
                    cost_value = EXCLUDED.cost_value,
                    registration_date = EXCLUDED.registration_date,
                    status = EXCLUDED.status,
                    ownership_type = EXCLUDED.ownership_type,
                    cultural_heritage = EXCLUDED.cultural_heritage,
                    underground_floors = EXCLUDED.underground_floors,
                    build_record_area = EXCLUDED.build_record_area,
                    build_record_type = EXCLUDED.build_record_type,
                    common_data_status = EXCLUDED.common_data_status,
                    obj_type = EXCLUDED.obj_type,
                    raw_props = COALESCE(EXCLUDED.raw_props, cad_buildings.raw_props),
                    fetched_at = NOW()
                """
            ),
            {
                "cad_num": cn,
                "qcad": qcad,
                "wkt": wkt,
                "purpose": opts.get("purpose"),
                "name": opts.get("building_name"),
                "addr": opts.get("readable_address"),
                "area": _to_num(opts.get("area")),
                # floors / underground_floors хранятся TEXT — NSPD отдаёт
                # диапазоны '1-2', '2-3'. _to_int съел бы их в NULL и потерял
                # данные. Аналитика делает CAST через regex '^[0-9]+$'.
                "floors": (
                    str(opts["floors"]).strip() if opts.get("floors") not in (None, "") else None
                ),
                "yb": _to_int(opts.get("year_built")),
                "yc": _to_int(opts.get("year_commisioning")),
                "cost": _to_num(opts.get("cost_value")),
                "reg_date": _to_date(opts.get("registration_date")),
                "status": opts.get("status"),
                "ownership": opts.get("ownership_type"),
                "cultural": opts.get("cultural_heritage"),
                "underground": (
                    str(opts["underground_floors"]).strip()
                    if opts.get("underground_floors") not in (None, "")
                    else None
                ),
                "build_rec_area": _to_num(opts.get("build_record_area")),
                "build_rec_type": opts.get("build_record_type"),
                "common_status": opts.get("common_data_status"),
                "obj_type": opts.get("obj_type"),
                "raw_props": json.dumps(opts, ensure_ascii=False) if opts else None,
            },
        )
        n += 1
    return n
 # ── Run lifecycle ─────────────────────────────────────────────────────────────
 def _start_run(db: Session, region_code: int, triggered_by: str, pending_count: int) -> int:
    row = db.execute(
        text(
            """
            INSERT INTO nspd_scrape_runs (region_code, triggered_by, pending_count, status)
            VALUES (:rc, :tb, :pc, 'running')
            RETURNING run_id
            """
        ),
        {"rc": region_code, "tb": triggered_by, "pc": pending_count},
    ).scalar_one()
    db.commit()
    return int(row)
 def _heartbeat(db: Session, run_id: int, **counts: int) -> None:
    sets = ["heartbeat_at = NOW()"]
    params: dict[str, Any] = {"rid": run_id}
    for k, v in counts.items():
        sets.append(f"{k} = :{k}")
        params[k] = v
    db.execute(
        text(f"UPDATE nspd_scrape_runs SET {', '.join(sets)} WHERE run_id = :rid"),
        params,
    )
    db.commit()
 def _finish_run(
    db: Session,
    run_id: int,
    *,
    status: str,
    error: str | None = None,
    **counts: int,
 ) -> None:
    sets = ["finished_at = NOW()", "status = :status"]
    params: dict[str, Any] = {"rid": run_id, "status": status, "error": error}
    sets.append("error = :error")
    for k, v in counts.items():
        sets.append(f"{k} = :{k}")
        params[k] = v
    db.execute(
        text(f"UPDATE nspd_scrape_runs SET {', '.join(sets)} WHERE run_id = :rid"),
        params,
    )
    db.commit()
 def _log(
    db: Session,
    run_id: int | None,
    *,
    level: str,
    stage: str,
    message: str,
    cad: str | None = None,
 ) -> None:
    try:
        db.execute(
            text(
                """
                INSERT INTO nspd_scrape_log (run_id, level, stage, cad_number, message)
                VALUES (:rid, :lvl, :st, :cad, :msg)
                """
            ),
            {"rid": run_id, "lvl": level, "st": stage, "cad": cad, "msg": message[:1000]},
        )
        db.commit()
    except Exception as e:
        logger.warning("nspd_scrape_log insert failed: %s", e)
 # ── Main entrypoint ───────────────────────────────────────────────────────────
 def run_region_scrape(
    *,
    region_code: int,
    triggered_by: str = "beat",
    limit: int | None = None,
    rate_ms: int = DEFAULT_RATE_MS,
    commit_every: int = DEFAULT_COMMIT_EVERY,
    heartbeat_every: int = DEFAULT_HEARTBEAT_EVERY,
 ) -> dict[str, Any]:
    """Полный sweep одного региона. Безопасен для повторных запусков —
    pending = (rosreestr cads) − (cad_quarters_geom cads)."""
    cad_prefix = REGION_CAD_PREFIX.get(region_code)
    if not cad_prefix:
        raise ValueError(f"region_code={region_code} unknown — добавь в REGION_CAD_PREFIX")
    db = SessionLocal()
    run_id: int | None = None
    try:
        pending = get_pending_cads(db, region_code)
        if limit:
            pending = pending[:limit]
        run_id = _start_run(db, region_code, triggered_by, len(pending))
        _log(
            db,
            run_id,
            level="info",
            stage="discover",
            message=f"region={region_code} pending={len(pending)} cads",
        )
        if not pending:
            _finish_run(db, run_id, status="done", quarters_ok=0, quarters_failed=0, buildings_ok=0)
            return {"run_id": run_id, "pending": 0, "ok": 0, "failed": 0, "buildings": 0}
        ok = 0
        failed = 0
        n_buildings = 0
        n_requests = 0
        n_waf = 0
        started = time.time()
        for i, cn in enumerate(pending, 1):
            try:
                # Pre-fetch лог в DB — чтобы видеть «застрял на cad X» если
                # urllib зависнет в read(). Heartbeat не пишется во время
                # nspd_fetch, и без этого лога run выглядит как «discover, тишина».
                logger.info("nspd[%d/%d] %s — fetching polygon", i, len(pending), cn)
                if i == 1 or i % 10 == 0:
                    _log(
                        db,
                        run_id,
                        level="info",
                        stage="quarter_fetch",
                        cad=cn,
                        message=f"fetching polygon ({i}/{len(pending)})",
                    )
                j2 = nspd_fetch(2, cn, on_403=lambda _a: None)
                n_requests += 1
                qf = None
                q_props: dict | None = None
                if j2:
                    for f in j2.get("data", {}).get("features") or []:
                        if (f.get("properties") or {}).get("label") == cn:
                            qf = f
                            q_props = (f.get("properties") or {}).get("options") or {}
                            break
                wkt = poly_to_wkt(qf.get("geometry") if qf else None) if qf else None
                if wkt:
                    insert_quarter(db, cn, wkt, q_props)
                    ok += 1
                else:
                    failed += 1
                    _log(
                        db,
                        run_id,
                        level="warn",
                        stage="quarter_fetch",
                        cad=cn,
                        message="no polygon returned",
                    )
                j1 = nspd_fetch(1, cn, on_403=lambda _a: None)
                n_requests += 1
                if j1:
                    features = j1.get("data", {}).get("features") or []
                    n_buildings += insert_buildings(db, cn, features, cad_prefix)
                if i % commit_every == 0:
                    db.commit()
                if i % heartbeat_every == 0:
                    _heartbeat(
                        db,
                        run_id,
                        quarters_ok=ok,
                        quarters_failed=failed,
                        buildings_ok=n_buildings,
                        requests_count=n_requests,
                        waf_429_count=n_waf,
                    )
            except WafBlockedError as e:
                _log(
                    db,
                    run_id,
                    level="error",
                    stage="quarter_fetch",
                    cad=cn,
                    message=f"WAF blocked: {e}",
                )
                _finish_run(
                    db,
                    run_id,
                    status="failed",
                    error=str(e),
                    quarters_ok=ok,
                    quarters_failed=failed,
                    buildings_ok=n_buildings,
                    requests_count=n_requests,
                    waf_429_count=n_waf + 1,
                )
                raise
            except Exception as e:
                failed += 1
                db.rollback()
                _log(
                    db,
                    run_id,
                    level="error",
                    stage="quarter_fetch",
                    cad=cn,
                    message=f"{type(e).__name__}: {e}",
                )
            time.sleep(rate_ms / 1000.0)
        db.commit()
        elapsed = time.time() - started
        _finish_run(
            db,
            run_id,
            status="done",
            quarters_ok=ok,
            quarters_failed=failed,
            buildings_ok=n_buildings,
            requests_count=n_requests,
            waf_429_count=n_waf,
        )
        _log(
            db,
            run_id,
            level="info",
            stage="done",
            message=(
                f"elapsed={elapsed:.0f}s ok={ok} failed={failed} "
                f"buildings={n_buildings} reqs={n_requests}"
            ),
        )
        return {
            "run_id": run_id,
            "pending": len(pending),
            "ok": ok,
            "failed": failed,
            "buildings": n_buildings,
            "requests": n_requests,
            "elapsed_s": int(elapsed),
            "started_at": datetime.utcfromtimestamp(started).isoformat() + "Z",
        }
    except WafBlockedError:
        raise
    except Exception as e:
        if run_id:
            try:
                _finish_run(db, run_id, status="failed", error=f"{type(e).__name__}: {e}")
            except Exception:
                pass
        raise
    finally:
        db.close()
--- a/backend/app/workers/celery_app.py
+++ b/backend/app/workers/celery_app.py
@ -30,17 +30,12 @@ def _default_regions() -> list[int]:
    return [int(x.strip()) for x in settings.scrape_kn_default_regions.split(",") if x.strip()]
 def _nspd_default_regions() -> list[int]:
    return [int(x.strip()) for x in settings.scrape_nspd_default_regions.split(",") if x.strip()]
 celery_app = Celery(
    "gendesign",
    broker=settings.redis_url,
    backend=settings.redis_url,
    include=[
        "app.workers.tasks.scrape_kn",
        "app.workers.tasks.scrape_nspd",
        "app.workers.tasks.refresh_analytics",
        "app.workers.tasks.scrape_objective",
        "app.workers.tasks.objective_etl",
@ -61,23 +56,10 @@ celery_app.conf.beat_schedule = {
    for rc in _default_regions()
 }
-# NSPD-скрейп — квартально, после публикации rosreestr_deals.
+# NSPD beat schedule УДАЛЁН 2026-05-11.
-# Pending = (новые quarter_cad_number из rosreestr_deals) − (cad_quarters_geom).
+# Старый Playwright-based scraper (tasks.scrape_nspd) сменён на bulk geo-fetcher
-# При limit=None прогон захватит все накопленные cad-кварталы за квартал.
+# через rosreestr2coord — запускается вручную через UI /admin/scrape/geo.
-celery_app.conf.beat_schedule.update(
+# Расписания по умолчанию нет (cron можно добавить через beat позже если надо).
    {
        f"nspd-region-{rc}": {
            "task": "tasks.scrape_nspd.scrape_nspd_region",
            "schedule": _parse_cron(settings.scrape_nspd_cron),
            "kwargs": {
                "region_code": rc,
                "triggered_by": "beat",
                "rate_ms": settings.scrape_nspd_rate_ms,
            },
        }
        for rc in _nspd_default_regions()
    }
 )
 # Refresh ekb_districts медианы — ежемесячно 5-го числа в 04:00 МСК
 # (после публикации новых rosreestr-кварталов и NSPD beat-cycle).
@ -194,71 +176,9 @@ def _resume_zombie_runs(sender=None, **_kwargs) -> None:
        except Exception as e:
            logger.warning("worker_ready: failed to enqueue resume for run=%s: %s", rid, e)
-    # NSPD-runs: помечаем зомби (>15 мин без heartbeat — NSPD сам по себе
+    # NSPD-runs (старый Playwright-scraper) resume УДАЛЁН 2026-05-11. Скрапер
-    # медленнее kn, дельта больше). Resume = просто заново ставим
+    # снят с эксплуатации в пользу bulk geo-fetcher (nspd_geo). История
-    # scrape_nspd_region с тем же region_code и triggered_by='resume':
+    # старых runs сохранена в nspd_scrape_runs — никаких side-effects.
    # сервис идемпотентен — pending = (rosreestr ДДУ cads) − (cad_quarters_geom done),
    # уже скрейпнутые квартала пропустятся автоматом.
    db = SessionLocal()
    nspd_resume_targets: list[int] = []
    try:
        zombie_rows = (
            db.execute(
                text(
                    """
                    UPDATE nspd_scrape_runs
                       SET status = 'zombie',
                           finished_at = NOW(),
                           error = COALESCE(error,
                               'auto-zombie at worker_ready, resume scheduled')
                     WHERE status = 'running'
                       AND COALESCE(heartbeat_at, started_at)
                           < NOW() - INTERVAL '15 minutes'
                    RETURNING run_id, region_code
                    """
                )
            )
            .mappings()
            .all()
        )
        db.commit()
        # Уникальные регионы для resume (разные ран-id для одного региона —
        # повторно не enqueueum, lock и idempotency пропустят дубль).
        seen: set[int] = set()
        for r in zombie_rows:
            rc = int(r["region_code"])
            if rc not in seen:
                seen.add(rc)
                nspd_resume_targets.append(rc)
                logger.info(
                    "worker_ready: NSPD zombie run=%s region=%s — resume scheduled",
                    r["run_id"],
                    rc,
                )
    except Exception as e:
        logger.warning("worker_ready nspd zombie scan failed: %s", e)
        try:
            db.rollback()
        except Exception:
            pass
    finally:
        db.close()
    if nspd_resume_targets:
        from app.workers.tasks.scrape_nspd import scrape_nspd_region
        for rc in nspd_resume_targets:
            try:
                scrape_nspd_region.apply_async(
                    kwargs={"region_code": rc, "triggered_by": "resume"},
                )
                logger.info("worker_ready: scrape_nspd_region enqueued region=%s", rc)
            except Exception as e:
                logger.warning(
                    "worker_ready: failed to enqueue NSPD resume for region=%s: %s",
                    rc,
                    e,
                )
    # NSPD geo-jobs: bulk-fetcher с собственной resume-логикой через
    # nspd_geo_jobs / nspd_geo_targets. Жадный resume: status='running' со
--- a/backend/app/workers/tasks/nspd_geo.py
+++ b/backend/app/workers/tasks/nspd_geo.py
@ -45,6 +45,29 @@ WAF_BACKOFF_BASE_S = 30
 # ── Helpers для job/target lifecycle ────────────────────────────────────────
 def _log(
    db: Session, job_id: int, level: str, stage: str, message: str, cad_num: str | None = None
 ) -> None:
    """Запись в nspd_geo_log (виден через v_scrape_log_unified в admin UI)."""
    try:
        db.execute(
            text(
                """
                INSERT INTO nspd_geo_log (job_id, level, stage, cad_num, message)
                VALUES (:jid, :lvl, :stg, :cad, :msg)
                """
            ),
            {"jid": job_id, "lvl": level, "stg": stage, "cad": cad_num, "msg": message[:1000]},
        )
        db.commit()
    except Exception as e:
        logger.warning("nspd_geo_log write failed for job=%s: %s", job_id, e)
        try:
            db.rollback()
        except Exception:
            pass
 def _start_job(db: Session, job_id: int) -> None:
    db.execute(
        text(
@ -60,6 +83,7 @@ def _start_job(db: Session, job_id: int) -> None:
        {"id": job_id},
    )
    db.commit()
    _log(db, job_id, "info", "start", "job started/resumed")
 def _heartbeat(db: Session, job_id: int, **counts: int) -> None:
@ -366,6 +390,14 @@ def process_nspd_geo_job(self: Any, job_id: int) -> dict[str, Any]:
                )
                # НЕ помечаем failed — остаётся pending для следующей итерации
                _heartbeat(db, job_id, requests_count=n_requests, waf_blocked_count=n_waf)
                _log(
                    db,
                    job_id,
                    "warn",
                    "waf",
                    f"WAF/429 (consecutive={consecutive_waf}), backoff {wait}s",
                    cad_num=cad,
                )
                time.sleep(wait)
                if consecutive_waf >= 8:
                    # 8 подряд WAF — пауза job'а до следующего worker_ready
@ -375,11 +407,19 @@ def process_nspd_geo_job(self: Any, job_id: int) -> dict[str, Any]:
                        "paused",
                        error=f"WAF blocked {consecutive_waf} consecutive",
                    )
                    _log(
                        db,
                        job_id,
                        "error",
                        "paused",
                        f"job paused after {consecutive_waf} consecutive WAF blocks",
                    )
                    return {"job_id": job_id, "paused": True, "reason": "waf"}
                continue
            except (NspdLiteError, Exception) as e:
                logger.warning("fetch failed for %s: %s", cad, e)
                _log(db, job_id, "warn", "fetch_fail", str(e)[:300], cad_num=cad)
                db.execute(
                    text(
                        """
@ -418,6 +458,13 @@ def process_nspd_geo_job(self: Any, job_id: int) -> dict[str, Any]:
        )
        # done даже при failed targets — UI покажет failed_count отдельно
        _finish_job(db, job_id, "done")
        _log(
            db,
            job_id,
            "info",
            "done",
            f"done: {done} ok / {failed} failed / {n_requests} req / {n_waf} WAF blocks",
        )
        logger.info(
            "process_nspd_geo_job done: job=%s done=%d failed=%d req=%d waf=%d",
            job_id,
--- a/backend/app/workers/tasks/scrape_nspd.py
+++ b/backend/app/workers/tasks/scrape_nspd.py
@ -1,104 +0,0 @@
 """Celery task wrapper для NSPD-скрейпа кадастровых кварталов."""
 from __future__ import annotations
 import logging
 from contextlib import contextmanager
 from typing import Any
 import redis
 from app.core.config import settings
 from app.services.scrapers.nspd_kn import WafBlockedError, run_region_scrape
 from app.workers.celery_app import celery_app
 logger = logging.getLogger(__name__)
 # NSPD-скрейпы 700+ cad-кварталов на регион занимают ~2 часа (600мс/req × 2 req/cad).
 # Lock TTL 3 часа — потолок с запасом, не оставляет зомби-lock на долго.
 _LOCK_TTL_SECONDS = 3 * 60 * 60
 def _lock_key(region_code: int) -> str:
    return f"scrape:nspd:lock:{region_code}"
 def force_release_lock(region_code: int) -> bool:
    """Принудительно удалить Redis-lock для emergency manual trigger."""
    key = _lock_key(region_code)
    r = redis.Redis.from_url(settings.redis_url)
    try:
        return bool(r.delete(key))
    except Exception as e:
        logger.warning("force_release_lock %s failed: %s", key, e)
        return False
@contextmanager
 def _region_lock(region_code: int):
    key = _lock_key(region_code)
    r = redis.Redis.from_url(settings.redis_url)
    acquired = r.set(key, "1", nx=True, ex=_LOCK_TTL_SECONDS)
    try:
        yield bool(acquired)
    finally:
        if acquired:
            try:
                r.delete(key)
            except Exception:
                pass
@celery_app.task(
    bind=True,
    name="tasks.scrape_nspd.scrape_nspd_region",
    max_retries=2,
    autoretry_for=(WafBlockedError,),
    retry_backoff=600,  # 10 мин начальный, удваивается
    retry_backoff_max=7200,  # потолок 2 часа
 )
 def scrape_nspd_region(
    self: Any,
    region_code: int = 66,
    triggered_by: str = "beat",
    limit: int | None = None,
    rate_ms: int = 600,
 ) -> dict[str, Any]:
    """Запуск NSPD-скрейпа одного региона. По расписанию: 20-е число
    февраля/мая/августа/ноября (после квартальных публикаций rosreestr).
    Args:
        region_code: код региона rosreestr (66 = Свердл).
        triggered_by: 'beat' | 'manual' | 'resume' — для журналирования.
        limit: ограничить N cad-кварталов (smoke-тест).
        rate_ms: пауза между запросами (по умолчанию 600мс — наблюдаемый предел WAF).
    Returns:
        dict с run_id и итогами (pending/ok/failed/buildings/requests/elapsed_s),
        либо {'skipped': True, 'reason': 'lock_held'} если уже идёт прогон.
    """
    with _region_lock(region_code) as got_lock:
        if not got_lock:
            logger.warning(
                "scrape_nspd_region SKIPPED region=%s — другой sweep идёт (Redis lock)",
                region_code,
            )
            return {
                "skipped": True,
                "reason": "lock_held",
                "region_code": region_code,
                "lock_key": _lock_key(region_code),
            }
        logger.info(
            "scrape_nspd_region start region=%s trigger=%s limit=%s",
            region_code,
            triggered_by,
            limit,
        )
        return run_region_scrape(
            region_code=region_code,
            triggered_by=triggered_by,
            limit=limit,
            rate_ms=rate_ms,
        )
--- a/data/sql/77_nspd_geo_jobs.sql
+++ b/data/sql/77_nspd_geo_jobs.sql
@ -28,7 +28,9 @@ CREATE TABLE IF NOT EXISTS nspd_geo_jobs (
    source_params   jsonb,                         -- что-то вроде {"region_codes":[66,74,72,59], "thematic_id":2}
    -- Параметры fetch
    rate_ms         INTEGER NOT NULL DEFAULT 600,
-    use_rosreestr2coord BOOLEAN NOT NULL DEFAULT FALSE,  -- если TRUE — community-lib, иначе наш nspd_lite
+    -- TRUE = community-lib rosreestr2coord (default с 2026-05-11),
    -- FALSE = наш nspd_lite urllib-based (fallback).
    use_rosreestr2coord BOOLEAN NOT NULL DEFAULT TRUE,
    -- Состояние
    status          TEXT NOT NULL DEFAULT 'queued', -- queued | running | done | failed | paused
    started_at      timestamptz,
@ -83,3 +85,67 @@ CREATE INDEX IF NOT EXISTS nspd_geo_targets_failed_idx
 COMMENT ON TABLE nspd_geo_targets IS
    'Список cad-номеров для обработки в рамках job. Resume-state: '
    'task просто SELECT WHERE status=pending и продолжает.';
 -- ── nspd_geo_log: логи (видны через v_scrape_log_unified) ───────────────────
 CREATE TABLE IF NOT EXISTS nspd_geo_log (
    log_id      BIGSERIAL PRIMARY KEY,
    job_id      BIGINT REFERENCES nspd_geo_jobs(job_id) ON DELETE CASCADE,
    ts          timestamptz NOT NULL DEFAULT NOW(),
    level       TEXT NOT NULL DEFAULT 'info',
    stage       TEXT,
    cad_num     TEXT,
    message     TEXT NOT NULL
 );
 CREATE INDEX IF NOT EXISTS nspd_geo_log_job_idx
    ON nspd_geo_log(job_id, log_id DESC);
 CREATE INDEX IF NOT EXISTS nspd_geo_log_ts_idx
    ON nspd_geo_log(ts DESC);
 -- ── Расширение unified-views на nspd_geo (UNION ALL) ────────────────────────
 CREATE OR REPLACE VIEW v_scrape_runs_unified AS
 SELECT 'kn' AS scraper_type, run_id, started_at, finished_at, heartbeat_at,
       status, error, NULL::text AS triggered_by,
       region_codes::text AS scope, requests_count,
       objects_count AS items_ok, NULL::integer AS items_failed,
       flats_count AS sub_items_ok,
       progress_obj_index AS progress_idx, total_obj_count AS progress_total,
       jsonb_build_object('developers', developer_ids, 'snapshot_date', snapshot_date,
                          'resumed_from', resumed_from_run_id, 'params', params) AS extra
  FROM kn_scrape_runs
 UNION ALL
 SELECT 'nspd', run_id, started_at, finished_at, heartbeat_at, status, error,
       triggered_by, region_code::text, requests_count,
       quarters_ok, quarters_failed, buildings_ok,
       NULL, pending_count,
       jsonb_build_object('waf_429_count', waf_429_count)
  FROM nspd_scrape_runs
 UNION ALL
 SELECT 'objective', run_id, started_at, finished_at, heartbeat_at, status, error,
       triggered_by, group_name, requests_count,
       reports_ok, reports_failed, rows_lots,
       NULL, NULL,
       jsonb_build_object('rows_corpus_room', rows_corpus_room, 'rows_history', rows_history)
  FROM objective_scrape_runs
 UNION ALL
 SELECT 'nspd_geo', job_id, started_at, finished_at, heartbeat_at, status, error,
       triggered_by, COALESCE(name, job_kind),
       requests_count, targets_done, targets_failed, targets_skipped,
       targets_done, targets_total,
       jsonb_build_object('job_kind', job_kind, 'source_kind', source_kind,
                          'rate_ms', rate_ms, 'use_rosreestr2coord', use_rosreestr2coord,
                          'waf_blocked_count', waf_blocked_count,
                          'source_params', source_params)
  FROM nspd_geo_jobs;
 CREATE OR REPLACE VIEW v_scrape_log_unified AS
 SELECT 'kn' AS scraper_type, log_id, run_id, ts, level, stage,
       obj_id::text AS entity_id, message
  FROM kn_scrape_log
 UNION ALL
 SELECT 'nspd', log_id, run_id, ts, level, stage, cad_number, message
  FROM nspd_scrape_log
 UNION ALL
 SELECT 'nspd_geo', log_id, job_id AS run_id, ts, level, stage, cad_num, message
  FROM nspd_geo_log;
--- a/frontend/src/app/admin/layout.tsx
+++ b/frontend/src/app/admin/layout.tsx
@ -6,8 +6,7 @@ import { usePathname } from "next/navigation";
 const TABS = [
  { href: "/admin/scrape/all", label: "📊 Все скраперы (сводно)" },
  { href: "/admin/scrape", label: "Скрапер DOM.РФ" },
-  { href: "/admin/scrape/nspd", label: "Скрапер NSPD" },
+  { href: "/admin/scrape/geo", label: "🗺 NSPD geo (rosreestr2coord)" },
  { href: "/admin/scrape/geo", label: "🗺 Geo bulk-jobs" },
  { href: "/admin/scrape/objective", label: "ETL Objective" },
  { href: "/admin/leads", label: "Лиды PRINZIP" },
 ];
--- a/frontend/src/app/admin/scrape/all/page.tsx
+++ b/frontend/src/app/admin/scrape/all/page.tsx
@ -6,7 +6,7 @@ import { useQuery } from "@tanstack/react-query";
 import { apiFetch } from "@/lib/api";
 interface UnifiedRunRow {
-  scraper_type: "kn" | "nspd" | "objective";
+  scraper_type: "kn" | "nspd" | "objective" | "nspd_geo";
  run_id: number;
  started_at: string | null;
  finished_at: string | null;
@ -25,7 +25,7 @@ interface UnifiedRunRow {
 }
 interface UnifiedLogRow {
-  scraper_type: "kn" | "nspd";
+  scraper_type: "kn" | "nspd" | "nspd_geo";
  log_id: number;
  run_id: number | null;
  ts: string | null;
@ -40,6 +40,7 @@ const SCRAPER_BADGE: Record<string, { bg: string; fg: string; label: string }> =
    kn: { bg: "#dbeafe", fg: "#1d4ed8", label: "DOM.РФ kn" },
    nspd: { bg: "#fef3c7", fg: "#a16207", label: "NSPD" },
    objective: { bg: "#d1fae5", fg: "#059669", label: "Objective" },
    nspd_geo: { bg: "#ede9fe", fg: "#6d28d9", label: "🗺 NSPD geo" },
  };
 const STATUS_COLOR: Record<string, string> = {
@ -68,9 +69,9 @@ export default function ScrapeAllAdminPage() {
      ? ""
      : (localStorage.getItem("admin_token") ?? ""),
  );
-  const [filter, setFilter] = useState<"all" | "kn" | "nspd" | "objective">(
+  const [filter, setFilter] = useState<
-    "all",
+    "all" | "kn" | "nspd" | "objective" | "nspd_geo"
-  );
+  >("all");
  const [logsRunId, setLogsRunId] = useState<number | "">("");
  const saveToken = (v: string) => {
@ -124,20 +125,22 @@ export default function ScrapeAllAdminPage() {
        Скраперы — сводный dashboard
      </h2>
      <p style={{ color: "#5b6066", fontSize: 13, marginTop: 4 }}>
-        Все 3 скрапера в одной таблице (через <code>v_scrape_runs_unified</code>
+        Все скраперы в одной таблице (через <code>v_scrape_runs_unified</code>).
-        ). Для запуска / триггеров — отдельные страницы:{" "}
+        Для запуска / триггеров — отдельные страницы:{" "}
        <a href="/admin/scrape" style={{ color: "#1d4ed8" }}>
-          kn
+          DOM.РФ kn
        </a>{" "}
        ·{" "}
-        <a href="/admin/scrape/nspd" style={{ color: "#1d4ed8" }}>
+        <a href="/admin/scrape/geo" style={{ color: "#1d4ed8" }}>
-          NSPD
+          🗺 NSPD geo
        </a>{" "}
        ·{" "}
        <a href="/admin/scrape/objective" style={{ color: "#1d4ed8" }}>
          Objective
        </a>
-        .
+        . Старый Playwright-based NSPD-скрапер снят с эксплуатации 2026-05-11 —
        история его прогонов осталась под фильтром <code>nspd</code> только для
        аудита.
      </p>
      <section style={cardStyle}>
@ -177,20 +180,22 @@ export default function ScrapeAllAdminPage() {
      <section style={{ ...cardStyle, marginTop: 16 }}>
        <div style={{ display: "flex", gap: 8, alignItems: "center" }}>
          <span style={{ ...labelStyle, marginBottom: 0 }}>Filter:</span>
-          {(["all", "kn", "nspd", "objective"] as const).map((t) => (
+          {(["all", "kn", "nspd", "nspd_geo", "objective"] as const).map(
-            <button
+            (t) => (
-              key={t}
+              <button
-              type="button"
+                key={t}
-              onClick={() => setFilter(t)}
+                type="button"
-              style={{
+                onClick={() => setFilter(t)}
-                ...filterTab,
+                style={{
-                background: filter === t ? "#1d4ed8" : "#f3f4f6",
+                  ...filterTab,
-                color: filter === t ? "#fff" : "#1f2937",
+                  background: filter === t ? "#1d4ed8" : "#f3f4f6",
-              }}
+                  color: filter === t ? "#fff" : "#1f2937",
-            >
+                }}
-              {t === "all" ? "Все" : (SCRAPER_BADGE[t]?.label ?? t)}
+              >
-            </button>
+                {t === "all" ? "Все" : (SCRAPER_BADGE[t]?.label ?? t)}
-          ))}
+              </button>
            ),
          )}
        </div>
      </section>
@ -308,7 +313,8 @@ export default function ScrapeAllAdminPage() {
                    </td>
                    <td style={td}>
                      {(r.scraper_type === "kn" ||
-                        r.scraper_type === "nspd") && (
+                        r.scraper_type === "nspd" ||
                        r.scraper_type === "nspd_geo") && (
                        <button
                          type="button"
                          onClick={() => setLogsRunId(r.run_id)}
--- a/frontend/src/app/admin/scrape/geo/page.tsx
+++ b/frontend/src/app/admin/scrape/geo/page.tsx
@ -73,7 +73,7 @@ export default function GeoScrapeAdminPage() {
  const [cadNumsText, setCadNumsText] = useState("");
  const [regionCodes, setRegionCodes] = useState("66,74,72,59");
  const [rateMs, setRateMs] = useState(600);
-  const [useLib, setUseLib] = useState(false);
+  const [useLib, setUseLib] = useState(true);
  const [lastResp, setLastResp] = useState<EnqueueResp | null>(null);
  const queryClient = useQueryClient();
--- a/frontend/src/app/admin/scrape/nspd/page.tsx
+++ b/frontend/src/app/admin/scrape/nspd/page.tsx
@ -1,745 +0,0 @@
 "use client";
 import { useState } from "react";
 import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
 import { apiFetch } from "@/lib/api";
 interface NspdRunRow {
  run_id: number;
  started_at: string | null;
  finished_at: string | null;
  heartbeat_at: string | null;
  region_code: number;
  pending_count: number | null;
  quarters_ok: number;
  quarters_failed: number;
  buildings_ok: number;
  requests_count: number;
  waf_429_count: number;
  status: string;
  error: string | null;
  triggered_by: string;
 }
 interface NspdLogRow {
  log_id: number;
  run_id: number | null;
  ts: string | null;
  level: "debug" | "info" | "warn" | "error";
  stage: string | null;
  cad_number: string | null;
  message: string;
 }
 interface NspdCoverage {
  region_code: number;
  total_quarters_with_ddu: number;
  already_scraped: number;
  pending: number;
  buildings_total: number;
  coverage_pct: number | null;
 }
 interface NspdTriggerResp {
  task_id: string;
  region_code: number;
  limit: number | null;
  rate_ms: number;
  force: boolean;
  lock_was_released: boolean;
 }
 export default function NspdScrapeAdminPage() {
  const [token, setToken] = useState<string>(() =>
    typeof window === "undefined"
      ? ""
      : (localStorage.getItem("admin_token") ?? ""),
  );
  const [region, setRegion] = useState(66);
  const [limit, setLimit] = useState<string>("");
  const [rateMs, setRateMs] = useState(600);
  const [forceRun, setForceRun] = useState(false);
  const [logsRunId, setLogsRunId] = useState<number | "">("");
  const queryClient = useQueryClient();
  const saveToken = (v: string) => {
    setToken(v);
    localStorage.setItem("admin_token", v);
  };
  const coverage = useQuery({
    queryKey: ["nspd-coverage", region, token],
    queryFn: () =>
      apiFetch<NspdCoverage>(
        `/api/v1/admin/scrape/nspd/coverage?region_code=${region}`,
        {
          headers: { "X-Admin-Token": token },
        },
      ),
    enabled: !!token,
    refetchInterval: 10000,
  });
  const runs = useQuery({
    queryKey: ["nspd-runs", token],
    queryFn: () =>
      apiFetch<NspdRunRow[]>("/api/v1/admin/scrape/nspd/runs?limit=20", {
        headers: { "X-Admin-Token": token },
      }),
    enabled: !!token,
    refetchInterval: 5000,
  });
  const logs = useQuery({
    queryKey: ["nspd-logs", logsRunId, token],
    queryFn: () => {
      const q =
        logsRunId !== "" ? `?run_id=${logsRunId}&limit=200` : "?limit=200";
      return apiFetch<NspdLogRow[]>(`/api/v1/admin/scrape/nspd/logs${q}`, {
        headers: { "X-Admin-Token": token },
      });
    },
    enabled: !!token,
    refetchInterval: 3000,
  });
  const trigger = useMutation({
    mutationFn: () =>
      apiFetch<NspdTriggerResp>("/api/v1/admin/scrape/nspd", {
        method: "POST",
        headers: { "X-Admin-Token": token },
        body: JSON.stringify({
          region_code: region,
          limit: limit === "" ? null : Number(limit),
          rate_ms: rateMs,
          force: forceRun,
        }),
      }),
    onSuccess: () => {
      queryClient.invalidateQueries({ queryKey: ["nspd-runs"] });
      queryClient.invalidateQueries({ queryKey: ["nspd-coverage"] });
    },
  });
  const releaseLock = useMutation({
    mutationFn: () =>
      apiFetch<{ region_code: number; released: boolean }>(
        "/api/v1/admin/scrape/nspd/release-lock",
        {
          method: "POST",
          headers: { "X-Admin-Token": token },
          body: JSON.stringify({ region_code: region }),
        },
      ),
    onSuccess: () => {
      queryClient.invalidateQueries({ queryKey: ["nspd-runs"] });
    },
  });
  return (
    <>
      <h2 style={{ marginTop: 0, fontSize: 18 }}>
        Скрапер NSPD — кадастровые кварталы и здания
      </h2>
      <p style={{ color: "#5b6066", fontSize: 13, marginTop: 4 }}>
        Берёт quarter_cad_number из rosreestr_deals (ДДУ-сделки на квартиры) →
        тянет полигон cad-квартала + здания внутри из NSPD geoportal. Расписание
        (beat): 20-е число фев/мая/авг/нояб (после квартальных публикаций
        rosreestr). См. <code>SCRAPE_NSPD_CRON</code>.
      </p>
      <section style={cardStyle}>
        <label style={{ display: "block", marginBottom: 12 }}>
          <span style={labelStyle}>Admin Token</span>
          <input
            type="password"
            value={token}
            onChange={(e) => saveToken(e.target.value)}
            placeholder="SCRAPE_ADMIN_TOKEN"
            style={inputStyle}
          />
        </label>
        <div style={{ display: "flex", gap: 16, flexWrap: "wrap" }}>
          <label>
            <span style={labelStyle}>Регион</span>
            <input
              type="number"
              value={region}
              min={1}
              max={99}
              onChange={(e) => setRegion(Number(e.target.value))}
              style={{ ...inputStyle, width: 100 }}
            />
          </label>
          <label>
            <span style={labelStyle}>Limit (smoke)</span>
            <input
              type="number"
              value={limit}
              onChange={(e) => setLimit(e.target.value)}
              placeholder="напр. 10"
              min={1}
              max={2000}
              style={{ ...inputStyle, width: 120 }}
            />
          </label>
          <label>
            <span style={labelStyle}>Rate (мс)</span>
            <input
              type="number"
              value={rateMs}
              onChange={(e) => setRateMs(Number(e.target.value))}
              min={200}
              max={5000}
              step={100}
              style={{ ...inputStyle, width: 110 }}
              title="Пауза между NSPD-запросами. <600мс = WAF 403 риск"
            />
          </label>
          <label
            style={{
              display: "flex",
              alignItems: "center",
              gap: 6,
              marginTop: 22,
            }}
            title="Перед запуском удалить existing Redis-lock"
          >
            <input
              type="checkbox"
              checked={forceRun}
              onChange={(e) => setForceRun(e.target.checked)}
            />
            <span style={{ color: forceRun ? "#b3261e" : "inherit" }}>
              ⚡ Force (снять lock)
            </span>
          </label>
        </div>
        {/* Pre-launch summary: сколько кварталов реально пойдёт в обработку
            и сколько примерно займёт. Считаем тут, а не на бэке — дешевле. */}
        {token && coverage.data
          ? (() => {
              const limitNum = limit === "" ? null : Number(limit);
              const willProcess = limitNum
                ? Math.min(limitNum, coverage.data.pending)
                : coverage.data.pending;
              // 2 запроса на квартал (полигон + здания) × rate_ms, +1с на запись
              const etaSec = willProcess * ((rateMs * 2) / 1000 + 1);
              const etaMin = Math.round(etaSec / 60);
              const etaH = Math.floor(etaMin / 60);
              const etaText =
                etaMin >= 60 ? `~${etaH}ч ${etaMin % 60}мин` : `~${etaMin} мин`;
              return (
                <div
                  style={{
                    marginTop: 14,
                    padding: 10,
                    background: willProcess === 0 ? "#f0fdf4" : "#eff6ff",
                    border: `1px solid ${
                      willProcess === 0 ? "#bbf7d0" : "#bfdbfe"
                    }`,
                    borderRadius: 6,
                    fontSize: 13,
                    lineHeight: 1.5,
                  }}
                >
                  {willProcess === 0 ? (
                    <>
                      ✅ Всё уже скрейпнуто для региона {region}. Запускать
                      бесполезно — pending=0.
                    </>
                  ) : (
                    <>
                      При запуске будет обработано{" "}
                      <strong>
                        {willProcess.toLocaleString("ru")} кварталов
                      </strong>{" "}
                      {limitNum && limitNum < coverage.data.pending ? (
                        <span style={{ color: "#5b6066" }}>
                          (limit {limitNum} из {coverage.data.pending} pending)
                        </span>
                      ) : (
                        <span style={{ color: "#5b6066" }}>
                          (вся очередь pending)
                        </span>
                      )}
                      . Расчётное время: <strong>{etaText}</strong> при rate{" "}
                      {rateMs}мс ({((rateMs * 2) / 1000).toFixed(1)}с / квартал
                      × 2 запроса).
                    </>
                  )}
                </div>
              );
            })()
          : null}
        <div
          style={{ marginTop: 16, display: "flex", gap: 12, flexWrap: "wrap" }}
        >
          <button
            onClick={() => trigger.mutate()}
            disabled={
              !token || trigger.isPending || (coverage.data?.pending ?? 0) === 0
            }
            style={primaryBtn}
          >
            {trigger.isPending ? "Запускаю…" : "Запустить sweep"}
          </button>
          <button
            onClick={() => {
              if (
                confirm(
                  "Снять Redis-lock для NSPD? Используй когда подозреваешь зомби-worker.",
                )
              ) {
                releaseLock.mutate();
              }
            }}
            disabled={!token || releaseLock.isPending}
            style={secondaryBtn}
          >
            {releaseLock.isPending ? "Снимаю…" : "🔓 Снять Redis-lock"}
          </button>
          {!token ? (
            <span
              style={{
                marginLeft: 12,
                color: "#9a6700",
                fontSize: 13,
                alignSelf: "center",
              }}
            >
              ← введите Admin Token выше
            </span>
          ) : null}
        </div>
        {trigger.isError ? (
          <div
            style={{
              marginTop: 12,
              padding: 12,
              background: "#fef2f2",
              border: "1px solid #fecaca",
              borderRadius: 6,
              color: "#991b1b",
              fontSize: 14,
              wordBreak: "break-word",
            }}
          >
            <strong>Ошибка:</strong> {String(trigger.error)}
          </div>
        ) : null}
        {trigger.isSuccess ? (
          <div
            style={{
              marginTop: 12,
              padding: 12,
              background: "#f0fdf4",
              border: "1px solid #bbf7d0",
              borderRadius: 6,
              color: "#065f46",
              fontSize: 14,
            }}
          >
            <strong>✓ Поставлено в очередь.</strong> task_id={" "}
            <code
              style={{
                background: "#dcfce7",
                padding: "2px 6px",
                borderRadius: 3,
              }}
            >
              {trigger.data.task_id}
            </code>{" "}
            · регион {trigger.data.region_code}
            {trigger.data.limit ? ` · limit=${trigger.data.limit}` : ""} · rate=
            {trigger.data.rate_ms}мс
          </div>
        ) : null}
      </section>
      <section style={{ ...cardStyle, marginTop: 16 }}>
        <h2 style={{ margin: 0, fontSize: 16, marginBottom: 12 }}>
          Покрытие cad_quarters_geom (регион {region})
        </h2>
        {!token ? (
          <p style={{ color: "#5b6066", fontSize: 13 }}>
            Введите Admin Token выше.
          </p>
        ) : coverage.isLoading ? (
          <p style={{ color: "#5b6066" }}>Загрузка…</p>
        ) : coverage.isError ? (
          <p style={{ color: "#b3261e" }}>Ошибка: {String(coverage.error)}</p>
        ) : coverage.data ? (
          <div
            style={{
              display: "grid",
              gridTemplateColumns: "repeat(auto-fit, minmax(160px, 1fr))",
              gap: 12,
            }}
          >
            <CoverageCard
              label="ДДУ-кварталов в rosreestr"
              value={coverage.data.total_quarters_with_ddu.toLocaleString("ru")}
            />
            <CoverageCard
              label="Уже скрейпнуто"
              value={coverage.data.already_scraped.toLocaleString("ru")}
              color="#0a7a3a"
            />
            <CoverageCard
              label="Осталось"
              value={coverage.data.pending.toLocaleString("ru")}
              color={coverage.data.pending > 0 ? "#9a6700" : "#0a7a3a"}
            />
            <CoverageCard
              label="Зданий в БД"
              value={coverage.data.buildings_total.toLocaleString("ru")}
            />
            <CoverageCard
              label="Покрытие, %"
              value={
                coverage.data.coverage_pct != null
                  ? `${coverage.data.coverage_pct}%`
                  : "—"
              }
              color={
                coverage.data.coverage_pct != null &&
                coverage.data.coverage_pct >= 90
                  ? "#0a7a3a"
                  : "#9a6700"
              }
            />
          </div>
        ) : null}
      </section>
      <section style={{ ...cardStyle, marginTop: 16 }}>
        <div
          style={{
            display: "flex",
            alignItems: "center",
            justifyContent: "space-between",
            marginBottom: 12,
          }}
        >
          <h2 style={{ margin: 0, fontSize: 16 }}>
            Лог скрейпа{" "}
            <span style={{ fontWeight: 400, color: "#5b6066", fontSize: 13 }}>
              · обновляется каждые 3 сек
            </span>
          </h2>
          <input
            type="number"
            value={logsRunId}
            onChange={(e) =>
              setLogsRunId(e.target.value === "" ? "" : Number(e.target.value))
            }
            placeholder="run_id (опц.)"
            style={{ ...inputStyle, width: 130 }}
          />
        </div>
        {!token ? (
          <p style={{ color: "#5b6066", fontSize: 13 }}>
            Введите Admin Token чтобы видеть лог.
          </p>
        ) : logs.isLoading ? (
          <p style={{ color: "#5b6066" }}>Загрузка…</p>
        ) : logs.isError ? (
          <p style={{ color: "#b3261e" }}>Ошибка: {String(logs.error)}</p>
        ) : (logs.data?.length ?? 0) === 0 ? (
          <p style={{ color: "#5b6066", fontSize: 13 }}>
            Нет событий{logsRunId !== "" ? ` для run_id=${logsRunId}` : ""}.
          </p>
        ) : (
          <div
            style={{
              maxHeight: 360,
              overflowY: "auto",
              fontFamily: "ui-monospace, SFMono-Regular, Menlo, monospace",
              fontSize: 12,
              border: "1px solid #eef0f3",
              borderRadius: 6,
              background: "#0f172a",
              color: "#e2e8f0",
              padding: 10,
            }}
          >
            {(logs.data ?? []).map((l) => {
              const color =
                l.level === "error"
                  ? "#fca5a5"
                  : l.level === "warn"
                    ? "#fcd34d"
                    : "#a3e635";
              return (
                <div key={l.log_id} style={{ marginBottom: 2 }}>
                  <span style={{ color: "#94a3b8" }}>
                    {l.ts?.slice(11, 19)}
                  </span>{" "}
                  <span style={{ color }}>{l.level.toUpperCase()}</span>{" "}
                  <span style={{ color: "#7dd3fc" }}>
                    [run-{l.run_id}
                    {l.stage ? ` · ${l.stage}` : ""}
                    {l.cad_number ? ` · ${l.cad_number}` : ""}]
                  </span>{" "}
                  <span>{l.message}</span>
                </div>
              );
            })}
          </div>
        )}
      </section>
      <section style={{ ...cardStyle, marginTop: 16 }}>
        <h2 style={{ marginTop: 0, fontSize: 16 }}>Последние прогоны</h2>
        {!token ? (
          <p style={{ color: "#5b6066" }}>
            Введите Admin Token чтобы увидеть журнал.
          </p>
        ) : runs.isLoading ? (
          <p>Загрузка…</p>
        ) : runs.isError ? (
          <p style={{ color: "#b3261e" }}>Ошибка: {String(runs.error)}</p>
        ) : (
          <table
            style={{ width: "100%", borderCollapse: "collapse", fontSize: 13 }}
          >
            <thead>
              <tr style={{ background: "#f6f7f9" }}>
                {[
                  "#",
                  "Старт",
                  "HB",
                  "Статус",
                  "Прогресс",
                  "Pending",
                  "OK",
                  "Fail",
                  "Зданий",
                  "Req",
                  "WAF",
                  "Trigger",
                  "Ошибка",
                ].map((h) => (
                  <th key={h} style={th}>
                    {h}
                  </th>
                ))}
              </tr>
            </thead>
            <tbody>
              {(runs.data ?? []).map((r) => (
                <tr
                  key={r.run_id}
                  style={{ borderBottom: "1px solid #eef0f3" }}
                >
                  <td style={td}>{r.run_id}</td>
                  <td style={td}>
                    {r.started_at?.slice(0, 19).replace("T", " ")}
                  </td>
                  <td style={td}>
                    {r.heartbeat_at ? r.heartbeat_at.slice(11, 19) : "—"}
                  </td>
                  <td
                    style={{
                      ...td,
                      color:
                        r.status === "done"
                          ? "#0a7a3a"
                          : r.status === "failed" || r.status === "zombie"
                            ? "#b3261e"
                            : "#9a6700",
                      fontWeight: 600,
                    }}
                  >
                    {r.status}
                  </td>
                  <td style={{ ...td, minWidth: 110 }}>
                    {r.pending_count && r.pending_count > 0
                      ? (() => {
                          const pct = Math.min(
                            100,
                            Math.round(
                              ((r.quarters_ok + r.quarters_failed) /
                                r.pending_count) *
                                100,
                            ),
                          );
                          return (
                            <div
                              style={{
                                fontFamily: "ui-monospace, monospace",
                                fontSize: 12,
                                display: "flex",
                                alignItems: "center",
                                gap: 6,
                              }}
                            >
                              <div
                                style={{
                                  width: 70,
                                  height: 6,
                                  background: "#eef0f3",
                                  borderRadius: 3,
                                  overflow: "hidden",
                                }}
                              >
                                <div
                                  style={{
                                    width: `${pct}%`,
                                    height: "100%",
                                    background:
                                      pct === 100
                                        ? "#0a7a3a"
                                        : pct > 0
                                          ? "#1d4ed8"
                                          : "#9a6700",
                                  }}
                                />
                              </div>
                              <span>{pct}%</span>
                            </div>
                          );
                        })()
                      : "—"}
                  </td>
                  <td style={td}>{r.pending_count ?? "—"}</td>
                  <td style={td}>{r.quarters_ok}</td>
                  <td
                    style={{
                      ...td,
                      color: r.quarters_failed > 0 ? "#b3261e" : undefined,
                    }}
                  >
                    {r.quarters_failed}
                  </td>
                  <td style={td}>{r.buildings_ok}</td>
                  <td style={td}>{r.requests_count}</td>
                  <td
                    style={{
                      ...td,
                      color: r.waf_429_count > 0 ? "#b3261e" : undefined,
                    }}
                  >
                    {r.waf_429_count}
                  </td>
                  <td style={td}>{r.triggered_by}</td>
                  <td
                    style={{
                      ...td,
                      color: "#b3261e",
                      maxWidth: 240,
                      overflow: "hidden",
                      textOverflow: "ellipsis",
                      whiteSpace: "nowrap",
                    }}
                    title={r.error ?? ""}
                  >
                    {r.error ?? ""}
                  </td>
                </tr>
              ))}
            </tbody>
          </table>
        )}
      </section>
    </>
  );
 }
 function CoverageCard({
  label,
  value,
  color,
 }: {
  label: string;
  value: string;
  color?: string;
 }) {
  return (
    <div
      style={{
        background: "#fff",
        border: "1px solid #e6e8ec",
        borderRadius: 8,
        padding: "10px 12px",
      }}
    >
      <div
        style={{
          fontSize: 11,
          color: "#5b6066",
          textTransform: "uppercase",
          letterSpacing: 0.4,
          marginBottom: 4,
        }}
      >
        {label}
      </div>
      <div
        style={{
          fontSize: 22,
          fontWeight: 700,
          color: color ?? "#0f172a",
          lineHeight: 1.1,
        }}
      >
        {value}
      </div>
    </div>
  );
 }
 const cardStyle = {
  background: "#fff",
  border: "1px solid #e6e8ec",
  borderRadius: 12,
  padding: 20,
 };
 const labelStyle = {
  display: "block",
  fontSize: 12,
  color: "#5b6066",
  marginBottom: 4,
  textTransform: "uppercase" as const,
  letterSpacing: 0.4,
 };
 const inputStyle = {
  padding: "8px 10px",
  border: "1px solid #d1d5db",
  borderRadius: 6,
  fontSize: 14,
  width: "100%",
  boxSizing: "border-box" as const,
 };
 const primaryBtn = {
  padding: "10px 18px",
  background: "#1d4ed8",
  color: "#fff",
  border: "none",
  borderRadius: 6,
  fontSize: 14,
  fontWeight: 600,
  cursor: "pointer",
 };
 const secondaryBtn = {
  padding: "8px 14px",
  background: "#f3f4f6",
  color: "#1f2937",
  border: "1px solid #d1d5db",
  borderRadius: 6,
  fontSize: 13,
  cursor: "pointer",
 };
 const th = {
  padding: "8px 10px",
  textAlign: "left" as const,
  fontWeight: 600,
  borderBottom: "1px solid #e6e8ec",
 };
 const td = { padding: "8px 10px" };