lekss361/gendesign
1
0
Fork 0
Code Issues 57 Pull requests Projects Releases Packages Wiki Activity Actions

refactor(nspd): remove Playwright-based scraper, rosreestr2coord = default

Browse source 
УДАЛЕНО (~1700 строк):
* backend/app/services/scrapers/nspd_kn.py (Playwright + WAF-bypass)
* backend/app/workers/tasks/scrape_nspd.py
* frontend/src/app/admin/scrape/nspd/page.tsx
* 5 NSPD admin endpoints (POST /nspd, /nspd/release-lock,
  GET /nspd/runs, /logs, /coverage)
* NSPD beat schedule + worker_ready resume hook + nav tab + link
* _nspd_default_regions() helper

СОХРАНЕНО (история):
* nspd_scrape_runs / nspd_scrape_log таблицы (3 row + log) для аудита
* UNION ALL в v_scrape_runs_unified / v_scrape_log_unified —
  старые runs видны под фильтром scraper_type='nspd'
* settings scrape_nspd_* помечены DEPRECATED

CHANGED:
* nspd_geo_jobs.use_rosreestr2coord DEFAULT FALSE → TRUE
* UI checkbox "rosreestr2coord lib" по умолчанию = checked
* /admin/scrape/all + /admin/scrape/geo — единственный путь к NSPD-данным

NSPD-доступ теперь только через rosreestr2coord (community lib) —
upstream поддерживает обновления headers/WAF-tricks. Локальный
nspd_lite.py (urllib) остаётся как fallback (use_rosreestr2coord=FALSE).
This commit is contained in:
lekss361 2026-05-11 09:21:12 +03:00
parent 54dbc77348
commit 79e2e94e09
11 changed files with 165 additions and 1756 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

210
backend/app/api/v1/admin_scrape.py
View file

@ -358,212 +358,10 @@ def list_logs(
]
# ── NSPD (cad_quarters / cad_buildings) ─────────────────────────────────────
class TriggerNspdRequest(BaseModel):
region_code: int = Field(default=66, ge=1, le=99)
limit: int | None = Field(default=None, ge=1, le=2000)
rate_ms: int = Field(default=600, ge=200, le=5000)
force: bool = False
@router.post("/nspd")
def trigger_nspd_sweep(
payload: TriggerNspdRequest,
x_admin_token: Annotated[str | None, Header(alias="X-Admin-Token")] = None,
) -> dict[str, Any]:
_check_token(x_admin_token)
from app.workers.tasks.scrape_nspd import (
force_release_lock as nspd_force_release_lock,
)
from app.workers.tasks.scrape_nspd import (
scrape_nspd_region,
)
lock_released = False
if payload.force:
lock_released = nspd_force_release_lock(payload.region_code)
result = scrape_nspd_region.apply_async(
kwargs={
"region_code": payload.region_code,
"triggered_by": "manual",
"limit": payload.limit,
"rate_ms": payload.rate_ms,
},
)
return {
"task_id": result.id,
"region_code": payload.region_code,
"limit": payload.limit,
"rate_ms": payload.rate_ms,
"force": payload.force,
"lock_was_released": lock_released,
}
class NspdReleaseLockRequest(BaseModel):
region_code: int = Field(..., ge=1, le=99)
@router.post("/nspd/release-lock")
def nspd_release_lock(
payload: NspdReleaseLockRequest,
x_admin_token: Annotated[str | None, Header(alias="X-Admin-Token")] = None,
) -> dict[str, Any]:
_check_token(x_admin_token)
from app.workers.tasks.scrape_nspd import force_release_lock as nspd_force_release_lock
released = nspd_force_release_lock(payload.region_code)
return {"region_code": payload.region_code, "released": released}
@router.get("/nspd/runs")
def list_nspd_runs(
db: Annotated[Session, Depends(get_db)],
x_admin_token: Annotated[str | None, Header(alias="X-Admin-Token")] = None,
limit: int = 20,
) -> list[dict[str, Any]]:
_check_token(x_admin_token)
rows = (
db.execute(
text(
"""
SELECT run_id, started_at, finished_at, heartbeat_at,
region_code, pending_count,
quarters_ok, quarters_failed, buildings_ok,
requests_count, waf_429_count,
status, error, triggered_by
FROM nspd_scrape_runs
ORDER BY started_at DESC
LIMIT :lim
"""
),
{"lim": min(limit, 100)},
)
.mappings()
.all()
)
return [
{
"run_id": r["run_id"],
"started_at": r["started_at"].isoformat() if r["started_at"] else None,
"finished_at": r["finished_at"].isoformat() if r["finished_at"] else None,
"heartbeat_at": r["heartbeat_at"].isoformat() if r["heartbeat_at"] else None,
"region_code": r["region_code"],
"pending_count": r["pending_count"],
"quarters_ok": r["quarters_ok"],
"quarters_failed": r["quarters_failed"],
"buildings_ok": r["buildings_ok"],
"requests_count": r["requests_count"],
"waf_429_count": r["waf_429_count"],
"status": r["status"],
"error": r["error"],
"triggered_by": r["triggered_by"],
}
for r in rows
]
@router.get("/nspd/logs")
def list_nspd_logs(
db: Annotated[Session, Depends(get_db)],
x_admin_token: Annotated[str | None, Header(alias="X-Admin-Token")] = None,
run_id: int | None = None,
since_id: int | None = None,
limit: int = 200,
) -> list[dict[str, Any]]:
_check_token(x_admin_token)
where: list[str] = []
params: dict[str, Any] = {"lim": min(limit, 1000)}
if run_id is not None:
where.append("run_id = :rid")
params["rid"] = run_id
if since_id is not None:
where.append("log_id > :sid")
params["sid"] = since_id
where_sql = ("WHERE " + " AND ".join(where)) if where else ""
rows = (
db.execute(
text(
f"""
SELECT log_id, run_id, ts, level, stage, cad_number, message
FROM nspd_scrape_log
{where_sql}
ORDER BY log_id DESC
LIMIT :lim
"""
),
params,
)
.mappings()
.all()
)
return [
{
"log_id": r["log_id"],
"run_id": r["run_id"],
"ts": r["ts"].isoformat() if r["ts"] else None,
"level": r["level"],
"stage": r["stage"],
"cad_number": r["cad_number"],
"message": r["message"],
}
for r in rows
]
@router.get("/nspd/coverage")
def nspd_coverage(
db: Annotated[Session, Depends(get_db)],
x_admin_token: Annotated[str | None, Header(alias="X-Admin-Token")] = None,
region_code: int = 66,
) -> dict[str, Any]:
"""Покрытие cad_quarters_geom относительно ДДУ-сделок rosreestr.
Полезно перед запуском sweep — видишь сколько ещё осталось скрейпить."""
_check_token(x_admin_token)
# ВАЖНО: regex с двоеточиями (':00:', ':0000000') ломают SQLAlchemy text() —
# двоеточие парсится как named-parameter prefix. Поэтому паттерны передаём
# через bindparams. NOT LIKE безопасен и быстрее regex.
row = (
db.execute(
text(
"""
WITH pending AS (
SELECT DISTINCT quarter_cad_number AS cad
FROM rosreestr_deals
WHERE region_code = :rc
AND doc_type = 'ДДУ'
AND realestate_type_code = '002001003000'
AND quarter_cad_number IS NOT NULL
AND quarter_cad_number NOT LIKE :bad_prefix
AND quarter_cad_number NOT LIKE :bad_suffix
),
done AS (SELECT cad_number AS cad FROM cad_quarters_geom)
SELECT
(SELECT COUNT(*) FROM pending) AS total_with_ddu,
(SELECT COUNT(*) FROM pending p
WHERE EXISTS (SELECT 1 FROM done d WHERE d.cad = p.cad))
AS already_scraped,
(SELECT COUNT(*) FROM cad_buildings) AS buildings_total
"""
),
{"rc": region_code, "bad_prefix": "00:00:%", "bad_suffix": "%:0000000"},
)
.mappings()
.first()
)
total = int(row["total_with_ddu"] or 0)
done = int(row["already_scraped"] or 0)
return {
"region_code": region_code,
"total_quarters_with_ddu": total,
"already_scraped": done,
"pending": max(total - done, 0),
"buildings_total": int(row["buildings_total"] or 0),
"coverage_pct": round(done / max(total, 1) * 100, 1) if total else None,
}
# NSPD-скрапер УДАЛЁН 2026-05-11: вместо Playwright-based nspd_kn.py теперь
# используется bulk geo-fetcher через rosreestr2coord. См. /admin/scrape/geo
# и backend/app/workers/tasks/nspd_geo.py + nspd_lite.py (urllib).
# История прогонов осталась в nspd_scrape_runs/log + v_scrape_runs_unified.
# ── Objective ETL (Антоновский SQLite → PG) ─────────────────────────────────

14
backend/app/core/config.py
View file

@ -34,14 +34,12 @@ class Settings(BaseSettings):
# Empty string = endpoint disabled.
scrape_admin_token: str = ""
# NSPD-scraper schedule (кадастровые кварталы / здания).
# По умолчанию: 20-е число февраля/мая/августа/ноября в 03:30 МСК
# (после квартальных публикаций rosreestr_deals).
scrape_nspd_cron: str = "30 3 20 2,5,8,11 *"
# Регионы для NSPD-sweep (comma-separated rosreestr region codes).
scrape_nspd_default_regions: str = "66"
# Пауза между NSPD-запросами в мс. <600мс — высокий риск WAF 403.
scrape_nspd_rate_ms: int = 600
# NSPD-scraper (Playwright) УДАЛЁН 2026-05-11. Сменён на bulk geo-fetcher
# через rosreestr2coord — запускается вручную через /admin/scrape/geo.
# Settings оставлены deprecated на случай отката (можно удалить позже).
scrape_nspd_cron: str = "30 3 20 2,5,8,11 *" # DEPRECATED
scrape_nspd_default_regions: str = "66" # DEPRECATED
scrape_nspd_rate_ms: int = 600 # DEPRECATED — see nspd_lite_rate_ms
# Objective.ru API (api.objctv.ru) — платная аналитика первички/вторички.
# Лежит наряду с DOM.РФ kn + rosreestr CSV как 3-й источник истины с

576
backend/app/services/scrapers/nspd_kn.py
View file

@ -1,576 +0,0 @@
"""NSPD scraper — кадастровые кварталы и здания через nspd.gov.ru API.
Endpoints:
GET /api/geoportal/v2/search/geoportal?thematicSearchId=2&query={cad}
→ polygon кадастрового квартала (1 feature с label = cad-номер)
GET /api/geoportal/v2/search/geoportal?thematicSearchId=1&query={cad}
→ объекты внутри квартала (здания, земля, линейные).
Фильтруем categoryName='Здания' и cad_num LIKE '<region>:<district>:%'.
WAF nspd.gov.ru банит burst-запросы и нероссийские IP. Стратегия:
- rate-limit 600мс между запросами (≤100 req/min — наблюдаемый предел)
- exponential backoff на 403 (60s + 30s × attempt)
- запускать с RU IP (production worker должен быть в RU/CIS-zone)
Этот модуль — рефакторинг data/sql/62_scrape_nspd_full.py из stand-alone скрипта
в библиотечную функцию для Celery. Логика идентична, но:
- работает через SQLAlchemy Session (FK к cad_quarters_geom/cad_buildings)
- пишет heartbeat и progress в nspd_scrape_runs
- логирует структурированно в nspd_scrape_log
"""
from __future__ import annotations
import json
import logging
import ssl
import time
import urllib.error
import urllib.parse
import urllib.request
from datetime import datetime
from typing import Any
from sqlalchemy import text
from sqlalchemy.orm import Session
from app.core.db import SessionLocal
logger = logging.getLogger(__name__)
NSPD_BASE = "https://nspd.gov.ru/api/geoportal/v2/search/geoportal"
HEADERS = {
"User-Agent": (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
),
"Accept": "application/json",
"Accept-Language": "ru-RU,ru;q=0.9",
"Referer": "https://nspd.gov.ru/map",
}
SSL_CTX = ssl._create_unverified_context()
DEFAULT_RATE_MS = 600
# Heartbeat каждый cad — даёт UI видимый прогресс и быстрый zombie-detect
# на прод-VM, где worker может тихо стопать без error в логе.
DEFAULT_HEARTBEAT_EVERY = 1 # quarters
DEFAULT_COMMIT_EVERY = 10
# Retries × timeout × WAF backoff формирует worst-case на один cad.
# Старые значения (5×30s + 60+90+120+150+180s = 600+150 = ~12.5 мин) дают
# инвиз-зависание worker'а если NSPD WAF режет nonbrowser fingerprint.
# Новые: до ~3 мин на cad, чтобы при 'тихом' WAF run падал быстрее.
DEFAULT_RETRIES = 3
DEFAULT_TIMEOUT_S = 12
# region_code (rosreestr) → cad-prefix фильтр для cad_buildings.
# 66 = Свердловская обл., 66:41 = ЕКБ. Для других регионов добавлять mapping.
REGION_CAD_PREFIX: dict[int, str] = {
66: "66:41:", # Екатеринбург
}
# ── HTTP layer ────────────────────────────────────────────────────────────────
class WafBlockedError(RuntimeError):
"""403 после всех retry — WAF банит, прогон не продолжаем."""
def nspd_fetch(
thematic_search_id: int,
query: str,
*,
retries: int = DEFAULT_RETRIES,
timeout: int = DEFAULT_TIMEOUT_S,
on_403: Any = None,
) -> dict | None:
"""Fetch NSPD с retry/backoff. on_403 callable(attempt) — для трекинга WAF."""
qs = urllib.parse.urlencode({"thematicSearchId": thematic_search_id, "query": query})
url = f"{NSPD_BASE}?{qs}"
req = urllib.request.Request(url, headers=HEADERS)
last_err: Exception | None = None
for attempt in range(retries):
try:
with urllib.request.urlopen(req, timeout=timeout, context=SSL_CTX) as r:
return json.loads(r.read().decode("utf-8"))
except urllib.error.HTTPError as e:
last_err = e
if e.code == 404:
return None
if e.code == 403:
if on_403:
on_403(attempt)
wait = 60 + 30 * attempt
logger.warning(
"WAF 403 для %s (попытка %d/%d), пауза %ds", query, attempt + 1, retries, wait
)
time.sleep(wait)
continue
logger.warning("HTTP %s для %s (попытка %d)", e.code, query, attempt + 1)
except (urllib.error.URLError, TimeoutError, OSError) as e:
last_err = e
logger.warning("Сетевая ошибка для %s: %s (попытка %d)", query, e, attempt + 1)
time.sleep(min(2**attempt, 30))
if isinstance(last_err, urllib.error.HTTPError) and last_err.code == 403:
raise WafBlockedError(f"WAF banned after {retries} retries on {query}")
logger.error("FAILED %s после %d попыток: %s", query, retries, last_err)
return None
# ── Geometry helpers ──────────────────────────────────────────────────────────
def poly_to_wkt(geom: dict | None) -> str | None:
if not geom:
return None
def ring(r: list) -> str:
return "(" + ",".join(f"{p[0]} {p[1]}" for p in r) + ")"
t = geom.get("type")
if t == "Polygon":
return "POLYGON(" + ",".join(ring(r) for r in geom["coordinates"]) + ")"
if t == "MultiPolygon":
return (
"MULTIPOLYGON("
+ ",".join("(" + ",".join(ring(r) for r in p) + ")" for p in geom["coordinates"])
+ ")"
)
if t == "Point":
return f"POINT({geom['coordinates'][0]} {geom['coordinates'][1]})"
return None
# ── SQL helpers ───────────────────────────────────────────────────────────────
def _to_int(x: Any) -> int | None:
if x is None or x == "":
return None
try:
return int(x)
except (TypeError, ValueError):
return None
def _to_num(x: Any) -> float | None:
if x is None or x == "":
return None
try:
return float(x)
except (TypeError, ValueError):
return None
def _to_date(x: Any) -> str | None:
if not x:
return None
s = str(x).split(" ")[0]
return s or None
def get_pending_cads(db: Session, region_code: int) -> list[str]:
"""Cad-кварталы с ДДУ-сделками в регионе минус уже scraped.
NOTE: regex с двоеточиями ('!~ ":0000000$"') ломают SQLAlchemy text() —
двоеточие парсится как named-parameter prefix. Используем NOT LIKE через
bindparam (быстрее regex и безопасно).
"""
rows = db.execute(
text(
"""
SELECT DISTINCT quarter_cad_number
FROM rosreestr_deals
WHERE region_code = :rc
AND doc_type = 'ДДУ'
AND realestate_type_code = '002001003000'
AND quarter_cad_number IS NOT NULL
AND quarter_cad_number <> ''
AND quarter_cad_number NOT LIKE :bad_prefix
AND quarter_cad_number NOT LIKE :bad_suffix
"""
),
{"rc": region_code, "bad_prefix": "00:00:%", "bad_suffix": "%:0000000"},
).all()
pending = {r[0] for r in rows}
done = {r[0] for r in db.execute(text("SELECT cad_number FROM cad_quarters_geom")).all()}
return sorted(pending - done)
def insert_quarter(db: Session, cad: str, wkt: str, raw_props: dict | None) -> None:
db.execute(
text(
"""
INSERT INTO cad_quarters_geom (cad_number, geom, raw_props, source)
VALUES (
:cad,
ST_Multi(ST_Transform(ST_SetSRID(ST_GeomFromText(:wkt), 3857), 4326))
::geometry(MultiPolygon, 4326),
CAST(:props AS jsonb),
'nspd'
)
ON CONFLICT (cad_number) DO UPDATE SET
geom = EXCLUDED.geom,
raw_props = COALESCE(EXCLUDED.raw_props, cad_quarters_geom.raw_props),
fetched_at = NOW()
"""
),
{
"cad": cad,
"wkt": wkt,
"props": json.dumps(raw_props, ensure_ascii=False) if raw_props else None,
},
)
def insert_buildings(db: Session, qcad: str, features: list[dict], cad_prefix: str) -> int:
n = 0
for f in features:
props_outer = f.get("properties") or {}
if props_outer.get("categoryName") != "Здания":
continue
opts = props_outer.get("options") or {}
cn = opts.get("cad_num")
if not cn or not cn.startswith(cad_prefix):
continue
wkt = poly_to_wkt(f.get("geometry"))
if not wkt:
continue
db.execute(
text(
"""
INSERT INTO cad_buildings (
cad_num, quarter_cad_num, geom, purpose, building_name,
readable_address, area, floors, year_built, year_commisioning,
cost_value, registration_date,
status, ownership_type, cultural_heritage, underground_floors,
build_record_area, build_record_type, common_data_status, obj_type,
raw_props
)
VALUES (
:cad_num, :qcad,
ST_Transform(ST_SetSRID(ST_GeomFromText(:wkt), 3857), 4326),
:purpose, :name, :addr, :area, :floors, :yb, :yc,
:cost, :reg_date,
:status, :ownership, :cultural, :underground,
:build_rec_area, :build_rec_type, :common_status, :obj_type,
CAST(:raw_props AS jsonb)
)
ON CONFLICT (cad_num) DO UPDATE SET
quarter_cad_num = EXCLUDED.quarter_cad_num,
geom = EXCLUDED.geom,
purpose = EXCLUDED.purpose,
building_name = EXCLUDED.building_name,
readable_address = EXCLUDED.readable_address,
area = EXCLUDED.area,
floors = EXCLUDED.floors,
year_built = EXCLUDED.year_built,
year_commisioning = EXCLUDED.year_commisioning,
cost_value = EXCLUDED.cost_value,
registration_date = EXCLUDED.registration_date,
status = EXCLUDED.status,
ownership_type = EXCLUDED.ownership_type,
cultural_heritage = EXCLUDED.cultural_heritage,
underground_floors = EXCLUDED.underground_floors,
build_record_area = EXCLUDED.build_record_area,
build_record_type = EXCLUDED.build_record_type,
common_data_status = EXCLUDED.common_data_status,
obj_type = EXCLUDED.obj_type,
raw_props = COALESCE(EXCLUDED.raw_props, cad_buildings.raw_props),
fetched_at = NOW()
"""
),
{
"cad_num": cn,
"qcad": qcad,
"wkt": wkt,
"purpose": opts.get("purpose"),
"name": opts.get("building_name"),
"addr": opts.get("readable_address"),
"area": _to_num(opts.get("area")),
# floors / underground_floors хранятся TEXT — NSPD отдаёт
# диапазоны '1-2', '2-3'. _to_int съел бы их в NULL и потерял
# данные. Аналитика делает CAST через regex '^[0-9]+$'.
"floors": (
str(opts["floors"]).strip() if opts.get("floors") not in (None, "") else None
),
"yb": _to_int(opts.get("year_built")),
"yc": _to_int(opts.get("year_commisioning")),
"cost": _to_num(opts.get("cost_value")),
"reg_date": _to_date(opts.get("registration_date")),
"status": opts.get("status"),
"ownership": opts.get("ownership_type"),
"cultural": opts.get("cultural_heritage"),
"underground": (
str(opts["underground_floors"]).strip()
if opts.get("underground_floors") not in (None, "")
else None
),
"build_rec_area": _to_num(opts.get("build_record_area")),
"build_rec_type": opts.get("build_record_type"),
"common_status": opts.get("common_data_status"),
"obj_type": opts.get("obj_type"),
"raw_props": json.dumps(opts, ensure_ascii=False) if opts else None,
},
)
n += 1
return n
# ── Run lifecycle ─────────────────────────────────────────────────────────────
def _start_run(db: Session, region_code: int, triggered_by: str, pending_count: int) -> int:
row = db.execute(
text(
"""
INSERT INTO nspd_scrape_runs (region_code, triggered_by, pending_count, status)
VALUES (:rc, :tb, :pc, 'running')
RETURNING run_id
"""
),
{"rc": region_code, "tb": triggered_by, "pc": pending_count},
).scalar_one()
db.commit()
return int(row)
def _heartbeat(db: Session, run_id: int, **counts: int) -> None:
sets = ["heartbeat_at = NOW()"]
params: dict[str, Any] = {"rid": run_id}
for k, v in counts.items():
sets.append(f"{k} = :{k}")
params[k] = v
db.execute(
text(f"UPDATE nspd_scrape_runs SET {', '.join(sets)} WHERE run_id = :rid"),
params,
)
db.commit()
def _finish_run(
db: Session,
run_id: int,
*,
status: str,
error: str | None = None,
**counts: int,
) -> None:
sets = ["finished_at = NOW()", "status = :status"]
params: dict[str, Any] = {"rid": run_id, "status": status, "error": error}
sets.append("error = :error")
for k, v in counts.items():
sets.append(f"{k} = :{k}")
params[k] = v
db.execute(
text(f"UPDATE nspd_scrape_runs SET {', '.join(sets)} WHERE run_id = :rid"),
params,
)
db.commit()
def _log(
db: Session,
run_id: int | None,
*,
level: str,
stage: str,
message: str,
cad: str | None = None,
) -> None:
try:
db.execute(
text(
"""
INSERT INTO nspd_scrape_log (run_id, level, stage, cad_number, message)
VALUES (:rid, :lvl, :st, :cad, :msg)
"""
),
{"rid": run_id, "lvl": level, "st": stage, "cad": cad, "msg": message[:1000]},
)
db.commit()
except Exception as e:
logger.warning("nspd_scrape_log insert failed: %s", e)
# ── Main entrypoint ───────────────────────────────────────────────────────────
def run_region_scrape(
*,
region_code: int,
triggered_by: str = "beat",
limit: int | None = None,
rate_ms: int = DEFAULT_RATE_MS,
commit_every: int = DEFAULT_COMMIT_EVERY,
heartbeat_every: int = DEFAULT_HEARTBEAT_EVERY,
) -> dict[str, Any]:
"""Полный sweep одного региона. Безопасен для повторных запусков —
pending = (rosreestr cads) − (cad_quarters_geom cads)."""
cad_prefix = REGION_CAD_PREFIX.get(region_code)
if not cad_prefix:
raise ValueError(f"region_code={region_code} unknown — добавь в REGION_CAD_PREFIX")
db = SessionLocal()
run_id: int | None = None
try:
pending = get_pending_cads(db, region_code)
if limit:
pending = pending[:limit]
run_id = _start_run(db, region_code, triggered_by, len(pending))
_log(
db,
run_id,
level="info",
stage="discover",
message=f"region={region_code} pending={len(pending)} cads",
)
if not pending:
_finish_run(db, run_id, status="done", quarters_ok=0, quarters_failed=0, buildings_ok=0)
return {"run_id": run_id, "pending": 0, "ok": 0, "failed": 0, "buildings": 0}
ok = 0
failed = 0
n_buildings = 0
n_requests = 0
n_waf = 0
started = time.time()
for i, cn in enumerate(pending, 1):
try:
# Pre-fetch лог в DB — чтобы видеть «застрял на cad X» если
# urllib зависнет в read(). Heartbeat не пишется во время
# nspd_fetch, и без этого лога run выглядит как «discover, тишина».
logger.info("nspd[%d/%d] %s — fetching polygon", i, len(pending), cn)
if i == 1 or i % 10 == 0:
_log(
db,
run_id,
level="info",
stage="quarter_fetch",
cad=cn,
message=f"fetching polygon ({i}/{len(pending)})",
)
j2 = nspd_fetch(2, cn, on_403=lambda _a: None)
n_requests += 1
qf = None
q_props: dict | None = None
if j2:
for f in j2.get("data", {}).get("features") or []:
if (f.get("properties") or {}).get("label") == cn:
qf = f
q_props = (f.get("properties") or {}).get("options") or {}
break
wkt = poly_to_wkt(qf.get("geometry") if qf else None) if qf else None
if wkt:
insert_quarter(db, cn, wkt, q_props)
ok += 1
else:
failed += 1
_log(
db,
run_id,
level="warn",
stage="quarter_fetch",
cad=cn,
message="no polygon returned",
)
j1 = nspd_fetch(1, cn, on_403=lambda _a: None)
n_requests += 1
if j1:
features = j1.get("data", {}).get("features") or []
n_buildings += insert_buildings(db, cn, features, cad_prefix)
if i % commit_every == 0:
db.commit()
if i % heartbeat_every == 0:
_heartbeat(
db,
run_id,
quarters_ok=ok,
quarters_failed=failed,
buildings_ok=n_buildings,
requests_count=n_requests,
waf_429_count=n_waf,
)
except WafBlockedError as e:
_log(
db,
run_id,
level="error",
stage="quarter_fetch",
cad=cn,
message=f"WAF blocked: {e}",
)
_finish_run(
db,
run_id,
status="failed",
error=str(e),
quarters_ok=ok,
quarters_failed=failed,
buildings_ok=n_buildings,
requests_count=n_requests,
waf_429_count=n_waf + 1,
)
raise
except Exception as e:
failed += 1
db.rollback()
_log(
db,
run_id,
level="error",
stage="quarter_fetch",
cad=cn,
message=f"{type(e).__name__}: {e}",
)
time.sleep(rate_ms / 1000.0)
db.commit()
elapsed = time.time() - started
_finish_run(
db,
run_id,
status="done",
quarters_ok=ok,
quarters_failed=failed,
buildings_ok=n_buildings,
requests_count=n_requests,
waf_429_count=n_waf,
)
_log(
db,
run_id,
level="info",
stage="done",
message=(
f"elapsed={elapsed:.0f}s ok={ok} failed={failed} "
f"buildings={n_buildings} reqs={n_requests}"
),
)
return {
"run_id": run_id,
"pending": len(pending),
"ok": ok,
"failed": failed,
"buildings": n_buildings,
"requests": n_requests,
"elapsed_s": int(elapsed),
"started_at": datetime.utcfromtimestamp(started).isoformat() + "Z",
}
except WafBlockedError:
raise
except Exception as e:
if run_id:
try:
_finish_run(db, run_id, status="failed", error=f"{type(e).__name__}: {e}")
except Exception:
pass
raise
finally:
db.close()

94
backend/app/workers/celery_app.py
View file

@ -30,17 +30,12 @@ def _default_regions() -> list[int]:
return [int(x.strip()) for x in settings.scrape_kn_default_regions.split(",") if x.strip()]
def _nspd_default_regions() -> list[int]:
return [int(x.strip()) for x in settings.scrape_nspd_default_regions.split(",") if x.strip()]
celery_app = Celery(
"gendesign",
broker=settings.redis_url,
backend=settings.redis_url,
include=[
"app.workers.tasks.scrape_kn",
"app.workers.tasks.scrape_nspd",
"app.workers.tasks.refresh_analytics",
"app.workers.tasks.scrape_objective",
"app.workers.tasks.objective_etl",
@ -61,23 +56,10 @@ celery_app.conf.beat_schedule = {
for rc in _default_regions()
}
# NSPD-скрейп — квартально, после публикации rosreestr_deals.
# Pending = (новые quarter_cad_number из rosreestr_deals) − (cad_quarters_geom).
# При limit=None прогон захватит все накопленные cad-кварталы за квартал.
celery_app.conf.beat_schedule.update(
{
f"nspd-region-{rc}": {
"task": "tasks.scrape_nspd.scrape_nspd_region",
"schedule": _parse_cron(settings.scrape_nspd_cron),
"kwargs": {
"region_code": rc,
"triggered_by": "beat",
"rate_ms": settings.scrape_nspd_rate_ms,
},
}
for rc in _nspd_default_regions()
}
)
# NSPD beat schedule УДАЛЁН 2026-05-11.
# Старый Playwright-based scraper (tasks.scrape_nspd) сменён на bulk geo-fetcher
# через rosreestr2coord — запускается вручную через UI /admin/scrape/geo.
# Расписания по умолчанию нет (cron можно добавить через beat позже если надо).
# Refresh ekb_districts медианы — ежемесячно 5-го числа в 04:00 МСК
# (после публикации новых rosreestr-кварталов и NSPD beat-cycle).
@ -194,71 +176,9 @@ def _resume_zombie_runs(sender=None, **_kwargs) -> None:
except Exception as e:
logger.warning("worker_ready: failed to enqueue resume for run=%s: %s", rid, e)
# NSPD-runs: помечаем зомби (>15 мин без heartbeat — NSPD сам по себе
# медленнее kn, дельта больше). Resume = просто заново ставим
# scrape_nspd_region с тем же region_code и triggered_by='resume':
# сервис идемпотентен — pending = (rosreestr ДДУ cads) − (cad_quarters_geom done),
# уже скрейпнутые квартала пропустятся автоматом.
db = SessionLocal()
nspd_resume_targets: list[int] = []
try:
zombie_rows = (
db.execute(
text(
"""
UPDATE nspd_scrape_runs
SET status = 'zombie',
finished_at = NOW(),
error = COALESCE(error,
'auto-zombie at worker_ready, resume scheduled')
WHERE status = 'running'
AND COALESCE(heartbeat_at, started_at)
< NOW() - INTERVAL '15 minutes'
RETURNING run_id, region_code
"""
)
)
.mappings()
.all()
)
db.commit()
# Уникальные регионы для resume (разные ран-id для одного региона —
# повторно не enqueueum, lock и idempotency пропустят дубль).
seen: set[int] = set()
for r in zombie_rows:
rc = int(r["region_code"])
if rc not in seen:
seen.add(rc)
nspd_resume_targets.append(rc)
logger.info(
"worker_ready: NSPD zombie run=%s region=%s — resume scheduled",
r["run_id"],
rc,
)
except Exception as e:
logger.warning("worker_ready nspd zombie scan failed: %s", e)
try:
db.rollback()
except Exception:
pass
finally:
db.close()
if nspd_resume_targets:
from app.workers.tasks.scrape_nspd import scrape_nspd_region
for rc in nspd_resume_targets:
try:
scrape_nspd_region.apply_async(
kwargs={"region_code": rc, "triggered_by": "resume"},
)
logger.info("worker_ready: scrape_nspd_region enqueued region=%s", rc)
except Exception as e:
logger.warning(
"worker_ready: failed to enqueue NSPD resume for region=%s: %s",
rc,
e,
)
# NSPD-runs (старый Playwright-scraper) resume УДАЛЁН 2026-05-11. Скрапер
# снят с эксплуатации в пользу bulk geo-fetcher (nspd_geo). История
# старых runs сохранена в nspd_scrape_runs — никаких side-effects.
# NSPD geo-jobs: bulk-fetcher с собственной resume-логикой через
# nspd_geo_jobs / nspd_geo_targets. Жадный resume: status='running' со

47
backend/app/workers/tasks/nspd_geo.py
View file

@ -45,6 +45,29 @@ WAF_BACKOFF_BASE_S = 30
# ── Helpers для job/target lifecycle ────────────────────────────────────────
def _log(
db: Session, job_id: int, level: str, stage: str, message: str, cad_num: str | None = None
) -> None:
"""Запись в nspd_geo_log (виден через v_scrape_log_unified в admin UI)."""
try:
db.execute(
text(
"""
INSERT INTO nspd_geo_log (job_id, level, stage, cad_num, message)
VALUES (:jid, :lvl, :stg, :cad, :msg)
"""
),
{"jid": job_id, "lvl": level, "stg": stage, "cad": cad_num, "msg": message[:1000]},
)
db.commit()
except Exception as e:
logger.warning("nspd_geo_log write failed for job=%s: %s", job_id, e)
try:
db.rollback()
except Exception:
pass
def _start_job(db: Session, job_id: int) -> None:
db.execute(
text(
@ -60,6 +83,7 @@ def _start_job(db: Session, job_id: int) -> None:
{"id": job_id},
)
db.commit()
_log(db, job_id, "info", "start", "job started/resumed")
def _heartbeat(db: Session, job_id: int, **counts: int) -> None:
@ -366,6 +390,14 @@ def process_nspd_geo_job(self: Any, job_id: int) -> dict[str, Any]:
)
# НЕ помечаем failed — остаётся pending для следующей итерации
_heartbeat(db, job_id, requests_count=n_requests, waf_blocked_count=n_waf)
_log(
db,
job_id,
"warn",
"waf",
f"WAF/429 (consecutive={consecutive_waf}), backoff {wait}s",
cad_num=cad,
)
time.sleep(wait)
if consecutive_waf >= 8:
# 8 подряд WAF — пауза job'а до следующего worker_ready
@ -375,11 +407,19 @@ def process_nspd_geo_job(self: Any, job_id: int) -> dict[str, Any]:
"paused",
error=f"WAF blocked {consecutive_waf} consecutive",
)
_log(
db,
job_id,
"error",
"paused",
f"job paused after {consecutive_waf} consecutive WAF blocks",
)
return {"job_id": job_id, "paused": True, "reason": "waf"}
continue
except (NspdLiteError, Exception) as e:
logger.warning("fetch failed for %s: %s", cad, e)
_log(db, job_id, "warn", "fetch_fail", str(e)[:300], cad_num=cad)
db.execute(
text(
"""
@ -418,6 +458,13 @@ def process_nspd_geo_job(self: Any, job_id: int) -> dict[str, Any]:
)
# done даже при failed targets — UI покажет failed_count отдельно
_finish_job(db, job_id, "done")
_log(
db,
job_id,
"info",
"done",
f"done: {done} ok / {failed} failed / {n_requests} req / {n_waf} WAF blocks",
)
logger.info(
"process_nspd_geo_job done: job=%s done=%d failed=%d req=%d waf=%d",
job_id,

104
backend/app/workers/tasks/scrape_nspd.py
View file

@ -1,104 +0,0 @@
"""Celery task wrapper для NSPD-скрейпа кадастровых кварталов."""
from __future__ import annotations
import logging
from contextlib import contextmanager
from typing import Any
import redis
from app.core.config import settings
from app.services.scrapers.nspd_kn import WafBlockedError, run_region_scrape
from app.workers.celery_app import celery_app
logger = logging.getLogger(__name__)
# NSPD-скрейпы 700+ cad-кварталов на регион занимают ~2 часа (600мс/req × 2 req/cad).
# Lock TTL 3 часа — потолок с запасом, не оставляет зомби-lock на долго.
_LOCK_TTL_SECONDS = 3 * 60 * 60
def _lock_key(region_code: int) -> str:
return f"scrape:nspd:lock:{region_code}"
def force_release_lock(region_code: int) -> bool:
"""Принудительно удалить Redis-lock для emergency manual trigger."""
key = _lock_key(region_code)
r = redis.Redis.from_url(settings.redis_url)
try:
return bool(r.delete(key))
except Exception as e:
logger.warning("force_release_lock %s failed: %s", key, e)
return False
@contextmanager
def _region_lock(region_code: int):
key = _lock_key(region_code)
r = redis.Redis.from_url(settings.redis_url)
acquired = r.set(key, "1", nx=True, ex=_LOCK_TTL_SECONDS)
try:
yield bool(acquired)
finally:
if acquired:
try:
r.delete(key)
except Exception:
pass
@celery_app.task(
bind=True,
name="tasks.scrape_nspd.scrape_nspd_region",
max_retries=2,
autoretry_for=(WafBlockedError,),
retry_backoff=600, # 10 мин начальный, удваивается
retry_backoff_max=7200, # потолок 2 часа
)
def scrape_nspd_region(
self: Any,
region_code: int = 66,
triggered_by: str = "beat",
limit: int | None = None,
rate_ms: int = 600,
) -> dict[str, Any]:
"""Запуск NSPD-скрейпа одного региона. По расписанию: 20-е число
февраля/мая/августа/ноября (после квартальных публикаций rosreestr).
Args:
region_code: код региона rosreestr (66 = Свердл).
triggered_by: 'beat' | 'manual' | 'resume' — для журналирования.
limit: ограничить N cad-кварталов (smoke-тест).
rate_ms: пауза между запросами (по умолчанию 600мс — наблюдаемый предел WAF).
Returns:
dict с run_id и итогами (pending/ok/failed/buildings/requests/elapsed_s),
либо {'skipped': True, 'reason': 'lock_held'} если уже идёт прогон.
"""
with _region_lock(region_code) as got_lock:
if not got_lock:
logger.warning(
"scrape_nspd_region SKIPPED region=%s — другой sweep идёт (Redis lock)",
region_code,
)
return {
"skipped": True,
"reason": "lock_held",
"region_code": region_code,
"lock_key": _lock_key(region_code),
}
logger.info(
"scrape_nspd_region start region=%s trigger=%s limit=%s",
region_code,
triggered_by,
limit,
)
return run_region_scrape(
region_code=region_code,
triggered_by=triggered_by,
limit=limit,
rate_ms=rate_ms,
)

68
data/sql/77_nspd_geo_jobs.sql
View file

@ -28,7 +28,9 @@ CREATE TABLE IF NOT EXISTS nspd_geo_jobs (
source_params jsonb, -- что-то вроде {"region_codes":[66,74,72,59], "thematic_id":2}
-- Параметры fetch
rate_ms INTEGER NOT NULL DEFAULT 600,
use_rosreestr2coord BOOLEAN NOT NULL DEFAULT FALSE, -- если TRUE — community-lib, иначе наш nspd_lite
-- TRUE = community-lib rosreestr2coord (default с 2026-05-11),
-- FALSE = наш nspd_lite urllib-based (fallback).
use_rosreestr2coord BOOLEAN NOT NULL DEFAULT TRUE,
-- Состояние
status TEXT NOT NULL DEFAULT 'queued', -- queued | running | done | failed | paused
started_at timestamptz,
@ -83,3 +85,67 @@ CREATE INDEX IF NOT EXISTS nspd_geo_targets_failed_idx
COMMENT ON TABLE nspd_geo_targets IS
'Список cad-номеров для обработки в рамках job. Resume-state: '
'task просто SELECT WHERE status=pending и продолжает.';
-- ── nspd_geo_log: логи (видны через v_scrape_log_unified) ───────────────────
CREATE TABLE IF NOT EXISTS nspd_geo_log (
log_id BIGSERIAL PRIMARY KEY,
job_id BIGINT REFERENCES nspd_geo_jobs(job_id) ON DELETE CASCADE,
ts timestamptz NOT NULL DEFAULT NOW(),
level TEXT NOT NULL DEFAULT 'info',
stage TEXT,
cad_num TEXT,
message TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS nspd_geo_log_job_idx
ON nspd_geo_log(job_id, log_id DESC);
CREATE INDEX IF NOT EXISTS nspd_geo_log_ts_idx
ON nspd_geo_log(ts DESC);
-- ── Расширение unified-views на nspd_geo (UNION ALL) ────────────────────────
CREATE OR REPLACE VIEW v_scrape_runs_unified AS
SELECT 'kn' AS scraper_type, run_id, started_at, finished_at, heartbeat_at,
status, error, NULL::text AS triggered_by,
region_codes::text AS scope, requests_count,
objects_count AS items_ok, NULL::integer AS items_failed,
flats_count AS sub_items_ok,
progress_obj_index AS progress_idx, total_obj_count AS progress_total,
jsonb_build_object('developers', developer_ids, 'snapshot_date', snapshot_date,
'resumed_from', resumed_from_run_id, 'params', params) AS extra
FROM kn_scrape_runs
UNION ALL
SELECT 'nspd', run_id, started_at, finished_at, heartbeat_at, status, error,
triggered_by, region_code::text, requests_count,
quarters_ok, quarters_failed, buildings_ok,
NULL, pending_count,
jsonb_build_object('waf_429_count', waf_429_count)
FROM nspd_scrape_runs
UNION ALL
SELECT 'objective', run_id, started_at, finished_at, heartbeat_at, status, error,
triggered_by, group_name, requests_count,
reports_ok, reports_failed, rows_lots,
NULL, NULL,
jsonb_build_object('rows_corpus_room', rows_corpus_room, 'rows_history', rows_history)
FROM objective_scrape_runs
UNION ALL
SELECT 'nspd_geo', job_id, started_at, finished_at, heartbeat_at, status, error,
triggered_by, COALESCE(name, job_kind),
requests_count, targets_done, targets_failed, targets_skipped,
targets_done, targets_total,
jsonb_build_object('job_kind', job_kind, 'source_kind', source_kind,
'rate_ms', rate_ms, 'use_rosreestr2coord', use_rosreestr2coord,
'waf_blocked_count', waf_blocked_count,
'source_params', source_params)
FROM nspd_geo_jobs;
CREATE OR REPLACE VIEW v_scrape_log_unified AS
SELECT 'kn' AS scraper_type, log_id, run_id, ts, level, stage,
obj_id::text AS entity_id, message
FROM kn_scrape_log
UNION ALL
SELECT 'nspd', log_id, run_id, ts, level, stage, cad_number, message
FROM nspd_scrape_log
UNION ALL
SELECT 'nspd_geo', log_id, job_id AS run_id, ts, level, stage, cad_num, message
FROM nspd_geo_log;

3
frontend/src/app/admin/layout.tsx
View file

@ -6,8 +6,7 @@ import { usePathname } from "next/navigation";
const TABS = [
{ href: "/admin/scrape/all", label: "📊 Все скраперы (сводно)" },
{ href: "/admin/scrape", label: "Скрапер DOM.РФ" },
{ href: "/admin/scrape/nspd", label: "Скрапер NSPD" },
{ href: "/admin/scrape/geo", label: "🗺 Geo bulk-jobs" },
{ href: "/admin/scrape/geo", label: "🗺 NSPD geo (rosreestr2coord)" },
{ href: "/admin/scrape/objective", label: "ETL Objective" },
{ href: "/admin/leads", label: "Лиды PRINZIP" },
];

58
frontend/src/app/admin/scrape/all/page.tsx
View file

@ -6,7 +6,7 @@ import { useQuery } from "@tanstack/react-query";
import { apiFetch } from "@/lib/api";
interface UnifiedRunRow {
scraper_type: "kn" | "nspd" | "objective";
scraper_type: "kn" | "nspd" | "objective" | "nspd_geo";
run_id: number;
started_at: string | null;
finished_at: string | null;
@ -25,7 +25,7 @@ interface UnifiedRunRow {
}
interface UnifiedLogRow {
scraper_type: "kn" | "nspd";
scraper_type: "kn" | "nspd" | "nspd_geo";
log_id: number;
run_id: number | null;
ts: string | null;
@ -40,6 +40,7 @@ const SCRAPER_BADGE: Record<string, { bg: string; fg: string; label: string }> =
kn: { bg: "#dbeafe", fg: "#1d4ed8", label: "DOM.РФ kn" },
nspd: { bg: "#fef3c7", fg: "#a16207", label: "NSPD" },
objective: { bg: "#d1fae5", fg: "#059669", label: "Objective" },
nspd_geo: { bg: "#ede9fe", fg: "#6d28d9", label: "🗺 NSPD geo" },
};
const STATUS_COLOR: Record<string, string> = {
@ -68,9 +69,9 @@ export default function ScrapeAllAdminPage() {
? ""
: (localStorage.getItem("admin_token") ?? ""),
);
const [filter, setFilter] = useState<"all" | "kn" | "nspd" | "objective">(
"all",
);
const [filter, setFilter] = useState<
"all" | "kn" | "nspd" | "objective" | "nspd_geo"
>("all");
const [logsRunId, setLogsRunId] = useState<number | "">("");
const saveToken = (v: string) => {
@ -124,20 +125,22 @@ export default function ScrapeAllAdminPage() {
Скраперы — сводный dashboard
</h2>
<p style={{ color: "#5b6066", fontSize: 13, marginTop: 4 }}>
Все 3 скрапера в одной таблице (через <code>v_scrape_runs_unified</code>
). Для запуска / триггеров — отдельные страницы:{" "}
Все скраперы в одной таблице (через <code>v_scrape_runs_unified</code>).
Для запуска / триггеров — отдельные страницы:{" "}
<a href="/admin/scrape" style={{ color: "#1d4ed8" }}>
kn
DOM.РФ kn
</a>{" "}
·{" "}
<a href="/admin/scrape/nspd" style={{ color: "#1d4ed8" }}>
NSPD
<a href="/admin/scrape/geo" style={{ color: "#1d4ed8" }}>
🗺 NSPD geo
</a>{" "}
·{" "}
<a href="/admin/scrape/objective" style={{ color: "#1d4ed8" }}>
Objective
</a>
.
. Старый Playwright-based NSPD-скрапер снят с эксплуатации 2026-05-11 —
история его прогонов осталась под фильтром <code>nspd</code> только для
аудита.
</p>
<section style={cardStyle}>
@ -177,20 +180,22 @@ export default function ScrapeAllAdminPage() {
<section style={{ ...cardStyle, marginTop: 16 }}>
<div style={{ display: "flex", gap: 8, alignItems: "center" }}>
<span style={{ ...labelStyle, marginBottom: 0 }}>Filter:</span>
{(["all", "kn", "nspd", "objective"] as const).map((t) => (
<button
key={t}
type="button"
onClick={() => setFilter(t)}
style={{
...filterTab,
background: filter === t ? "#1d4ed8" : "#f3f4f6",
color: filter === t ? "#fff" : "#1f2937",
}}
>
{t === "all" ? "Все" : (SCRAPER_BADGE[t]?.label ?? t)}
</button>
))}
{(["all", "kn", "nspd", "nspd_geo", "objective"] as const).map(
(t) => (
<button
key={t}
type="button"
onClick={() => setFilter(t)}
style={{
...filterTab,
background: filter === t ? "#1d4ed8" : "#f3f4f6",
color: filter === t ? "#fff" : "#1f2937",
}}
>
{t === "all" ? "Все" : (SCRAPER_BADGE[t]?.label ?? t)}
</button>
),
)}
</div>
</section>
@ -308,7 +313,8 @@ export default function ScrapeAllAdminPage() {
</td>
<td style={td}>
{(r.scraper_type === "kn" ||
r.scraper_type === "nspd") && (
r.scraper_type === "nspd" ||
r.scraper_type === "nspd_geo") && (
<button
type="button"
onClick={() => setLogsRunId(r.run_id)}

2
frontend/src/app/admin/scrape/geo/page.tsx
View file

@ -73,7 +73,7 @@ export default function GeoScrapeAdminPage() {
const [cadNumsText, setCadNumsText] = useState("");
const [regionCodes, setRegionCodes] = useState("66,74,72,59");
const [rateMs, setRateMs] = useState(600);
const [useLib, setUseLib] = useState(false);
const [useLib, setUseLib] = useState(true);
const [lastResp, setLastResp] = useState<EnqueueResp | null>(null);
const queryClient = useQueryClient();

745
frontend/src/app/admin/scrape/nspd/page.tsx
View file

@ -1,745 +0,0 @@
"use client";
import { useState } from "react";
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
import { apiFetch } from "@/lib/api";
interface NspdRunRow {
run_id: number;
started_at: string | null;
finished_at: string | null;
heartbeat_at: string | null;
region_code: number;
pending_count: number | null;
quarters_ok: number;
quarters_failed: number;
buildings_ok: number;
requests_count: number;
waf_429_count: number;
status: string;
error: string | null;
triggered_by: string;
}
interface NspdLogRow {
log_id: number;
run_id: number | null;
ts: string | null;
level: "debug" | "info" | "warn" | "error";
stage: string | null;
cad_number: string | null;
message: string;
}
interface NspdCoverage {
region_code: number;
total_quarters_with_ddu: number;
already_scraped: number;
pending: number;
buildings_total: number;
coverage_pct: number | null;
}
interface NspdTriggerResp {
task_id: string;
region_code: number;
limit: number | null;
rate_ms: number;
force: boolean;
lock_was_released: boolean;
}
export default function NspdScrapeAdminPage() {
const [token, setToken] = useState<string>(() =>
typeof window === "undefined"
? ""
: (localStorage.getItem("admin_token") ?? ""),
);
const [region, setRegion] = useState(66);
const [limit, setLimit] = useState<string>("");
const [rateMs, setRateMs] = useState(600);
const [forceRun, setForceRun] = useState(false);
const [logsRunId, setLogsRunId] = useState<number | "">("");
const queryClient = useQueryClient();
const saveToken = (v: string) => {
setToken(v);
localStorage.setItem("admin_token", v);
};
const coverage = useQuery({
queryKey: ["nspd-coverage", region, token],
queryFn: () =>
apiFetch<NspdCoverage>(
`/api/v1/admin/scrape/nspd/coverage?region_code=${region}`,
{
headers: { "X-Admin-Token": token },
},
),
enabled: !!token,
refetchInterval: 10000,
});
const runs = useQuery({
queryKey: ["nspd-runs", token],
queryFn: () =>
apiFetch<NspdRunRow[]>("/api/v1/admin/scrape/nspd/runs?limit=20", {
headers: { "X-Admin-Token": token },
}),
enabled: !!token,
refetchInterval: 5000,
});
const logs = useQuery({
queryKey: ["nspd-logs", logsRunId, token],
queryFn: () => {
const q =
logsRunId !== "" ? `?run_id=${logsRunId}&limit=200` : "?limit=200";
return apiFetch<NspdLogRow[]>(`/api/v1/admin/scrape/nspd/logs${q}`, {
headers: { "X-Admin-Token": token },
});
},
enabled: !!token,
refetchInterval: 3000,
});
const trigger = useMutation({
mutationFn: () =>
apiFetch<NspdTriggerResp>("/api/v1/admin/scrape/nspd", {
method: "POST",
headers: { "X-Admin-Token": token },
body: JSON.stringify({
region_code: region,
limit: limit === "" ? null : Number(limit),
rate_ms: rateMs,
force: forceRun,
}),
}),
onSuccess: () => {
queryClient.invalidateQueries({ queryKey: ["nspd-runs"] });
queryClient.invalidateQueries({ queryKey: ["nspd-coverage"] });
},
});
const releaseLock = useMutation({
mutationFn: () =>
apiFetch<{ region_code: number; released: boolean }>(
"/api/v1/admin/scrape/nspd/release-lock",
{
method: "POST",
headers: { "X-Admin-Token": token },
body: JSON.stringify({ region_code: region }),
},
),
onSuccess: () => {
queryClient.invalidateQueries({ queryKey: ["nspd-runs"] });
},
});
return (
<>
<h2 style={{ marginTop: 0, fontSize: 18 }}>
Скрапер NSPD — кадастровые кварталы и здания
</h2>
<p style={{ color: "#5b6066", fontSize: 13, marginTop: 4 }}>
Берёт quarter_cad_number из rosreestr_deals (ДДУ-сделки на квартиры) →
тянет полигон cad-квартала + здания внутри из NSPD geoportal. Расписание
(beat): 20-е число фев/мая/авг/нояб (после квартальных публикаций
rosreestr). См. <code>SCRAPE_NSPD_CRON</code>.
</p>
<section style={cardStyle}>
<label style={{ display: "block", marginBottom: 12 }}>
<span style={labelStyle}>Admin Token</span>
<input
type="password"
value={token}
onChange={(e) => saveToken(e.target.value)}
placeholder="SCRAPE_ADMIN_TOKEN"
style={inputStyle}
/>
</label>
<div style={{ display: "flex", gap: 16, flexWrap: "wrap" }}>
<label>
<span style={labelStyle}>Регион</span>
<input
type="number"
value={region}
min={1}
max={99}
onChange={(e) => setRegion(Number(e.target.value))}
style={{ ...inputStyle, width: 100 }}
/>
</label>
<label>
<span style={labelStyle}>Limit (smoke)</span>
<input
type="number"
value={limit}
onChange={(e) => setLimit(e.target.value)}
placeholder="напр. 10"
min={1}
max={2000}
style={{ ...inputStyle, width: 120 }}
/>
</label>
<label>
<span style={labelStyle}>Rate (мс)</span>
<input
type="number"
value={rateMs}
onChange={(e) => setRateMs(Number(e.target.value))}
min={200}
max={5000}
step={100}
style={{ ...inputStyle, width: 110 }}
title="Пауза между NSPD-запросами. <600мс = WAF 403 риск"
/>
</label>
<label
style={{
display: "flex",
alignItems: "center",
gap: 6,
marginTop: 22,
}}
title="Перед запуском удалить existing Redis-lock"
>
<input
type="checkbox"
checked={forceRun}
onChange={(e) => setForceRun(e.target.checked)}
/>
<span style={{ color: forceRun ? "#b3261e" : "inherit" }}>
⚡ Force (снять lock)
</span>
</label>
</div>
{/* Pre-launch summary: сколько кварталов реально пойдёт в обработку
и сколько примерно займёт. Считаем тут, а не на бэке — дешевле. */}
{token && coverage.data
? (() => {
const limitNum = limit === "" ? null : Number(limit);
const willProcess = limitNum
? Math.min(limitNum, coverage.data.pending)
: coverage.data.pending;
// 2 запроса на квартал (полигон + здания) × rate_ms, +1с на запись
const etaSec = willProcess * ((rateMs * 2) / 1000 + 1);
const etaMin = Math.round(etaSec / 60);
const etaH = Math.floor(etaMin / 60);
const etaText =
etaMin >= 60 ? `~${etaH}ч ${etaMin % 60}мин` : `~${etaMin} мин`;
return (
<div
style={{
marginTop: 14,
padding: 10,
background: willProcess === 0 ? "#f0fdf4" : "#eff6ff",
border: `1px solid ${
willProcess === 0 ? "#bbf7d0" : "#bfdbfe"
}`,
borderRadius: 6,
fontSize: 13,
lineHeight: 1.5,
}}
>
{willProcess === 0 ? (
<>
✅ Всё уже скрейпнуто для региона {region}. Запускать
бесполезно — pending=0.
</>
) : (
<>
При запуске будет обработано{" "}
<strong>
{willProcess.toLocaleString("ru")} кварталов
</strong>{" "}
{limitNum && limitNum < coverage.data.pending ? (
<span style={{ color: "#5b6066" }}>
(limit {limitNum} из {coverage.data.pending} pending)
</span>
) : (
<span style={{ color: "#5b6066" }}>
(вся очередь pending)
</span>
)}
. Расчётное время: <strong>{etaText}</strong> при rate{" "}
{rateMs}мс ({((rateMs * 2) / 1000).toFixed(1)}с / квартал
× 2 запроса).
</>
)}
</div>
);
})()
: null}
<div
style={{ marginTop: 16, display: "flex", gap: 12, flexWrap: "wrap" }}
>
<button
onClick={() => trigger.mutate()}
disabled={
!token || trigger.isPending || (coverage.data?.pending ?? 0) === 0
}
style={primaryBtn}
>
{trigger.isPending ? "Запускаю…" : "Запустить sweep"}
</button>
<button
onClick={() => {
if (
confirm(
"Снять Redis-lock для NSPD? Используй когда подозреваешь зомби-worker.",
)
) {
releaseLock.mutate();
}
}}
disabled={!token || releaseLock.isPending}
style={secondaryBtn}
>
{releaseLock.isPending ? "Снимаю…" : "🔓 Снять Redis-lock"}
</button>
{!token ? (
<span
style={{
marginLeft: 12,
color: "#9a6700",
fontSize: 13,
alignSelf: "center",
}}
>
← введите Admin Token выше
</span>
) : null}
</div>
{trigger.isError ? (
<div
style={{
marginTop: 12,
padding: 12,
background: "#fef2f2",
border: "1px solid #fecaca",
borderRadius: 6,
color: "#991b1b",
fontSize: 14,
wordBreak: "break-word",
}}
>
<strong>Ошибка:</strong> {String(trigger.error)}
</div>
) : null}
{trigger.isSuccess ? (
<div
style={{
marginTop: 12,
padding: 12,
background: "#f0fdf4",
border: "1px solid #bbf7d0",
borderRadius: 6,
color: "#065f46",
fontSize: 14,
}}
>
<strong>✓ Поставлено в очередь.</strong> task_id={" "}
<code
style={{
background: "#dcfce7",
padding: "2px 6px",
borderRadius: 3,
}}
>
{trigger.data.task_id}
</code>{" "}
· регион {trigger.data.region_code}
{trigger.data.limit ? ` · limit=${trigger.data.limit}` : ""} · rate=
{trigger.data.rate_ms}мс
</div>
) : null}
</section>
<section style={{ ...cardStyle, marginTop: 16 }}>
<h2 style={{ margin: 0, fontSize: 16, marginBottom: 12 }}>
Покрытие cad_quarters_geom (регион {region})
</h2>
{!token ? (
<p style={{ color: "#5b6066", fontSize: 13 }}>
Введите Admin Token выше.
</p>
) : coverage.isLoading ? (
<p style={{ color: "#5b6066" }}>Загрузка…</p>
) : coverage.isError ? (
<p style={{ color: "#b3261e" }}>Ошибка: {String(coverage.error)}</p>
) : coverage.data ? (
<div
style={{
display: "grid",
gridTemplateColumns: "repeat(auto-fit, minmax(160px, 1fr))",
gap: 12,
}}
>
<CoverageCard
label="ДДУ-кварталов в rosreestr"
value={coverage.data.total_quarters_with_ddu.toLocaleString("ru")}
/>
<CoverageCard
label="Уже скрейпнуто"
value={coverage.data.already_scraped.toLocaleString("ru")}
color="#0a7a3a"
/>
<CoverageCard
label="Осталось"
value={coverage.data.pending.toLocaleString("ru")}
color={coverage.data.pending > 0 ? "#9a6700" : "#0a7a3a"}
/>
<CoverageCard
label="Зданий в БД"
value={coverage.data.buildings_total.toLocaleString("ru")}
/>
<CoverageCard
label="Покрытие, %"
value={
coverage.data.coverage_pct != null
? `${coverage.data.coverage_pct}%`
: "—"
}
color={
coverage.data.coverage_pct != null &&
coverage.data.coverage_pct >= 90
? "#0a7a3a"
: "#9a6700"
}
/>
</div>
) : null}
</section>
<section style={{ ...cardStyle, marginTop: 16 }}>
<div
style={{
display: "flex",
alignItems: "center",
justifyContent: "space-between",
marginBottom: 12,
}}
>
<h2 style={{ margin: 0, fontSize: 16 }}>
Лог скрейпа{" "}
<span style={{ fontWeight: 400, color: "#5b6066", fontSize: 13 }}>
· обновляется каждые 3 сек
</span>
</h2>
<input
type="number"
value={logsRunId}
onChange={(e) =>
setLogsRunId(e.target.value === "" ? "" : Number(e.target.value))
}
placeholder="run_id (опц.)"
style={{ ...inputStyle, width: 130 }}
/>
</div>
{!token ? (
<p style={{ color: "#5b6066", fontSize: 13 }}>
Введите Admin Token чтобы видеть лог.
</p>
) : logs.isLoading ? (
<p style={{ color: "#5b6066" }}>Загрузка…</p>
) : logs.isError ? (
<p style={{ color: "#b3261e" }}>Ошибка: {String(logs.error)}</p>
) : (logs.data?.length ?? 0) === 0 ? (
<p style={{ color: "#5b6066", fontSize: 13 }}>
Нет событий{logsRunId !== "" ? ` для run_id=${logsRunId}` : ""}.
</p>
) : (
<div
style={{
maxHeight: 360,
overflowY: "auto",
fontFamily: "ui-monospace, SFMono-Regular, Menlo, monospace",
fontSize: 12,
border: "1px solid #eef0f3",
borderRadius: 6,
background: "#0f172a",
color: "#e2e8f0",
padding: 10,
}}
>
{(logs.data ?? []).map((l) => {
const color =
l.level === "error"
? "#fca5a5"
: l.level === "warn"
? "#fcd34d"
: "#a3e635";
return (
<div key={l.log_id} style={{ marginBottom: 2 }}>
<span style={{ color: "#94a3b8" }}>
{l.ts?.slice(11, 19)}
</span>{" "}
<span style={{ color }}>{l.level.toUpperCase()}</span>{" "}
<span style={{ color: "#7dd3fc" }}>
[run-{l.run_id}
{l.stage ? ` · ${l.stage}` : ""}
{l.cad_number ? ` · ${l.cad_number}` : ""}]
</span>{" "}
<span>{l.message}</span>
</div>
);
})}
</div>
)}
</section>
<section style={{ ...cardStyle, marginTop: 16 }}>
<h2 style={{ marginTop: 0, fontSize: 16 }}>Последние прогоны</h2>
{!token ? (
<p style={{ color: "#5b6066" }}>
Введите Admin Token чтобы увидеть журнал.
</p>
) : runs.isLoading ? (
<p>Загрузка…</p>
) : runs.isError ? (
<p style={{ color: "#b3261e" }}>Ошибка: {String(runs.error)}</p>
) : (
<table
style={{ width: "100%", borderCollapse: "collapse", fontSize: 13 }}
>
<thead>
<tr style={{ background: "#f6f7f9" }}>
{[
"#",
"Старт",
"HB",
"Статус",
"Прогресс",
"Pending",
"OK",
"Fail",
"Зданий",
"Req",
"WAF",
"Trigger",
"Ошибка",
].map((h) => (
<th key={h} style={th}>
{h}
</th>
))}
</tr>
</thead>
<tbody>
{(runs.data ?? []).map((r) => (
<tr
key={r.run_id}
style={{ borderBottom: "1px solid #eef0f3" }}
>
<td style={td}>{r.run_id}</td>
<td style={td}>
{r.started_at?.slice(0, 19).replace("T", " ")}
</td>
<td style={td}>
{r.heartbeat_at ? r.heartbeat_at.slice(11, 19) : "—"}
</td>
<td
style={{
...td,
color:
r.status === "done"
? "#0a7a3a"
: r.status === "failed" || r.status === "zombie"
? "#b3261e"
: "#9a6700",
fontWeight: 600,
}}
>
{r.status}
</td>
<td style={{ ...td, minWidth: 110 }}>
{r.pending_count && r.pending_count > 0
? (() => {
const pct = Math.min(
100,
Math.round(
((r.quarters_ok + r.quarters_failed) /
r.pending_count) *
100,
),
);
return (
<div
style={{
fontFamily: "ui-monospace, monospace",
fontSize: 12,
display: "flex",
alignItems: "center",
gap: 6,
}}
>
<div
style={{
width: 70,
height: 6,
background: "#eef0f3",
borderRadius: 3,
overflow: "hidden",
}}
>
<div
style={{
width: `${pct}%`,
height: "100%",
background:
pct === 100
? "#0a7a3a"
: pct > 0
? "#1d4ed8"
: "#9a6700",
}}
/>
</div>
<span>{pct}%</span>
</div>
);
})()
: "—"}
</td>
<td style={td}>{r.pending_count ?? "—"}</td>
<td style={td}>{r.quarters_ok}</td>
<td
style={{
...td,
color: r.quarters_failed > 0 ? "#b3261e" : undefined,
}}
>
{r.quarters_failed}
</td>
<td style={td}>{r.buildings_ok}</td>
<td style={td}>{r.requests_count}</td>
<td
style={{
...td,
color: r.waf_429_count > 0 ? "#b3261e" : undefined,
}}
>
{r.waf_429_count}
</td>
<td style={td}>{r.triggered_by}</td>
<td
style={{
...td,
color: "#b3261e",
maxWidth: 240,
overflow: "hidden",
textOverflow: "ellipsis",
whiteSpace: "nowrap",
}}
title={r.error ?? ""}
>
{r.error ?? ""}
</td>
</tr>
))}
</tbody>
</table>
)}
</section>
</>
);
}
function CoverageCard({
label,
value,
color,
}: {
label: string;
value: string;
color?: string;
}) {
return (
<div
style={{
background: "#fff",
border: "1px solid #e6e8ec",
borderRadius: 8,
padding: "10px 12px",
}}
>
<div
style={{
fontSize: 11,
color: "#5b6066",
textTransform: "uppercase",
letterSpacing: 0.4,
marginBottom: 4,
}}
>
{label}
</div>
<div
style={{
fontSize: 22,
fontWeight: 700,
color: color ?? "#0f172a",
lineHeight: 1.1,
}}
>
{value}
</div>
</div>
);
}
const cardStyle = {
background: "#fff",
border: "1px solid #e6e8ec",
borderRadius: 12,
padding: 20,
};
const labelStyle = {
display: "block",
fontSize: 12,
color: "#5b6066",
marginBottom: 4,
textTransform: "uppercase" as const,
letterSpacing: 0.4,
};
const inputStyle = {
padding: "8px 10px",
border: "1px solid #d1d5db",
borderRadius: 6,
fontSize: 14,
width: "100%",
boxSizing: "border-box" as const,
};
const primaryBtn = {
padding: "10px 18px",
background: "#1d4ed8",
color: "#fff",
border: "none",
borderRadius: 6,
fontSize: 14,
fontWeight: 600,
cursor: "pointer",
};
const secondaryBtn = {
padding: "8px 14px",
background: "#f3f4f6",
color: "#1f2937",
border: "1px solid #d1d5db",
borderRadius: 6,
fontSize: 13,
cursor: "pointer",
};
const th = {
padding: "8px 10px",
textAlign: "left" as const,
fontWeight: 600,
borderBottom: "1px solid #e6e8ec",
};
const td = { padding: "8px 10px" };