From 79e2e94e0967a705a64d41e7f111f574948d7dcc Mon Sep 17 00:00:00 2001 From: lekss361 Date: Mon, 11 May 2026 09:21:12 +0300 Subject: [PATCH] refactor(nspd): remove Playwright-based scraper, rosreestr2coord = default MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit УДАЛЕНО (~1700 строк): * backend/app/services/scrapers/nspd_kn.py (Playwright + WAF-bypass) * backend/app/workers/tasks/scrape_nspd.py * frontend/src/app/admin/scrape/nspd/page.tsx * 5 NSPD admin endpoints (POST /nspd, /nspd/release-lock, GET /nspd/runs, /logs, /coverage) * NSPD beat schedule + worker_ready resume hook + nav tab + link * _nspd_default_regions() helper СОХРАНЕНО (история): * nspd_scrape_runs / nspd_scrape_log таблицы (3 row + log) для аудита * UNION ALL в v_scrape_runs_unified / v_scrape_log_unified — старые runs видны под фильтром scraper_type='nspd' * settings scrape_nspd_* помечены DEPRECATED CHANGED: * nspd_geo_jobs.use_rosreestr2coord DEFAULT FALSE → TRUE * UI checkbox "rosreestr2coord lib" по умолчанию = checked * /admin/scrape/all + /admin/scrape/geo — единственный путь к NSPD-данным NSPD-доступ теперь только через rosreestr2coord (community lib) — upstream поддерживает обновления headers/WAF-tricks. Локальный nspd_lite.py (urllib) остаётся как fallback (use_rosreestr2coord=FALSE). --- backend/app/api/v1/admin_scrape.py | 210 +----- backend/app/core/config.py | 14 +- backend/app/services/scrapers/nspd_kn.py | 576 --------------- backend/app/workers/celery_app.py | 94 +-- backend/app/workers/tasks/nspd_geo.py | 47 ++ backend/app/workers/tasks/scrape_nspd.py | 104 --- data/sql/77_nspd_geo_jobs.sql | 68 +- frontend/src/app/admin/layout.tsx | 3 +- frontend/src/app/admin/scrape/all/page.tsx | 58 +- frontend/src/app/admin/scrape/geo/page.tsx | 2 +- frontend/src/app/admin/scrape/nspd/page.tsx | 745 -------------------- 11 files changed, 165 insertions(+), 1756 deletions(-) delete mode 100644 backend/app/services/scrapers/nspd_kn.py delete mode 100644 backend/app/workers/tasks/scrape_nspd.py delete mode 100644 frontend/src/app/admin/scrape/nspd/page.tsx diff --git a/backend/app/api/v1/admin_scrape.py b/backend/app/api/v1/admin_scrape.py index c1d7d0b4..173afb9f 100644 --- a/backend/app/api/v1/admin_scrape.py +++ b/backend/app/api/v1/admin_scrape.py @@ -358,212 +358,10 @@ def list_logs( ] -# ── NSPD (cad_quarters / cad_buildings) ───────────────────────────────────── - - -class TriggerNspdRequest(BaseModel): - region_code: int = Field(default=66, ge=1, le=99) - limit: int | None = Field(default=None, ge=1, le=2000) - rate_ms: int = Field(default=600, ge=200, le=5000) - force: bool = False - - -@router.post("/nspd") -def trigger_nspd_sweep( - payload: TriggerNspdRequest, - x_admin_token: Annotated[str | None, Header(alias="X-Admin-Token")] = None, -) -> dict[str, Any]: - _check_token(x_admin_token) - from app.workers.tasks.scrape_nspd import ( - force_release_lock as nspd_force_release_lock, - ) - from app.workers.tasks.scrape_nspd import ( - scrape_nspd_region, - ) - - lock_released = False - if payload.force: - lock_released = nspd_force_release_lock(payload.region_code) - - result = scrape_nspd_region.apply_async( - kwargs={ - "region_code": payload.region_code, - "triggered_by": "manual", - "limit": payload.limit, - "rate_ms": payload.rate_ms, - }, - ) - return { - "task_id": result.id, - "region_code": payload.region_code, - "limit": payload.limit, - "rate_ms": payload.rate_ms, - "force": payload.force, - "lock_was_released": lock_released, - } - - -class NspdReleaseLockRequest(BaseModel): - region_code: int = Field(..., ge=1, le=99) - - -@router.post("/nspd/release-lock") -def nspd_release_lock( - payload: NspdReleaseLockRequest, - x_admin_token: Annotated[str | None, Header(alias="X-Admin-Token")] = None, -) -> dict[str, Any]: - _check_token(x_admin_token) - from app.workers.tasks.scrape_nspd import force_release_lock as nspd_force_release_lock - - released = nspd_force_release_lock(payload.region_code) - return {"region_code": payload.region_code, "released": released} - - -@router.get("/nspd/runs") -def list_nspd_runs( - db: Annotated[Session, Depends(get_db)], - x_admin_token: Annotated[str | None, Header(alias="X-Admin-Token")] = None, - limit: int = 20, -) -> list[dict[str, Any]]: - _check_token(x_admin_token) - rows = ( - db.execute( - text( - """ - SELECT run_id, started_at, finished_at, heartbeat_at, - region_code, pending_count, - quarters_ok, quarters_failed, buildings_ok, - requests_count, waf_429_count, - status, error, triggered_by - FROM nspd_scrape_runs - ORDER BY started_at DESC - LIMIT :lim - """ - ), - {"lim": min(limit, 100)}, - ) - .mappings() - .all() - ) - return [ - { - "run_id": r["run_id"], - "started_at": r["started_at"].isoformat() if r["started_at"] else None, - "finished_at": r["finished_at"].isoformat() if r["finished_at"] else None, - "heartbeat_at": r["heartbeat_at"].isoformat() if r["heartbeat_at"] else None, - "region_code": r["region_code"], - "pending_count": r["pending_count"], - "quarters_ok": r["quarters_ok"], - "quarters_failed": r["quarters_failed"], - "buildings_ok": r["buildings_ok"], - "requests_count": r["requests_count"], - "waf_429_count": r["waf_429_count"], - "status": r["status"], - "error": r["error"], - "triggered_by": r["triggered_by"], - } - for r in rows - ] - - -@router.get("/nspd/logs") -def list_nspd_logs( - db: Annotated[Session, Depends(get_db)], - x_admin_token: Annotated[str | None, Header(alias="X-Admin-Token")] = None, - run_id: int | None = None, - since_id: int | None = None, - limit: int = 200, -) -> list[dict[str, Any]]: - _check_token(x_admin_token) - where: list[str] = [] - params: dict[str, Any] = {"lim": min(limit, 1000)} - if run_id is not None: - where.append("run_id = :rid") - params["rid"] = run_id - if since_id is not None: - where.append("log_id > :sid") - params["sid"] = since_id - where_sql = ("WHERE " + " AND ".join(where)) if where else "" - rows = ( - db.execute( - text( - f""" - SELECT log_id, run_id, ts, level, stage, cad_number, message - FROM nspd_scrape_log - {where_sql} - ORDER BY log_id DESC - LIMIT :lim - """ - ), - params, - ) - .mappings() - .all() - ) - return [ - { - "log_id": r["log_id"], - "run_id": r["run_id"], - "ts": r["ts"].isoformat() if r["ts"] else None, - "level": r["level"], - "stage": r["stage"], - "cad_number": r["cad_number"], - "message": r["message"], - } - for r in rows - ] - - -@router.get("/nspd/coverage") -def nspd_coverage( - db: Annotated[Session, Depends(get_db)], - x_admin_token: Annotated[str | None, Header(alias="X-Admin-Token")] = None, - region_code: int = 66, -) -> dict[str, Any]: - """Покрытие cad_quarters_geom относительно ДДУ-сделок rosreestr. - Полезно перед запуском sweep — видишь сколько ещё осталось скрейпить.""" - _check_token(x_admin_token) - # ВАЖНО: regex с двоеточиями (':00:', ':0000000') ломают SQLAlchemy text() — - # двоеточие парсится как named-parameter prefix. Поэтому паттерны передаём - # через bindparams. NOT LIKE безопасен и быстрее regex. - row = ( - db.execute( - text( - """ - WITH pending AS ( - SELECT DISTINCT quarter_cad_number AS cad - FROM rosreestr_deals - WHERE region_code = :rc - AND doc_type = 'ДДУ' - AND realestate_type_code = '002001003000' - AND quarter_cad_number IS NOT NULL - AND quarter_cad_number NOT LIKE :bad_prefix - AND quarter_cad_number NOT LIKE :bad_suffix - ), - done AS (SELECT cad_number AS cad FROM cad_quarters_geom) - SELECT - (SELECT COUNT(*) FROM pending) AS total_with_ddu, - (SELECT COUNT(*) FROM pending p - WHERE EXISTS (SELECT 1 FROM done d WHERE d.cad = p.cad)) - AS already_scraped, - (SELECT COUNT(*) FROM cad_buildings) AS buildings_total - """ - ), - {"rc": region_code, "bad_prefix": "00:00:%", "bad_suffix": "%:0000000"}, - ) - .mappings() - .first() - ) - total = int(row["total_with_ddu"] or 0) - done = int(row["already_scraped"] or 0) - return { - "region_code": region_code, - "total_quarters_with_ddu": total, - "already_scraped": done, - "pending": max(total - done, 0), - "buildings_total": int(row["buildings_total"] or 0), - "coverage_pct": round(done / max(total, 1) * 100, 1) if total else None, - } +# NSPD-скрапер УДАЛЁН 2026-05-11: вместо Playwright-based nspd_kn.py теперь +# используется bulk geo-fetcher через rosreestr2coord. См. /admin/scrape/geo +# и backend/app/workers/tasks/nspd_geo.py + nspd_lite.py (urllib). +# История прогонов осталась в nspd_scrape_runs/log + v_scrape_runs_unified. # ── Objective ETL (Антоновский SQLite → PG) ───────────────────────────────── diff --git a/backend/app/core/config.py b/backend/app/core/config.py index 82a9e755..c6c89974 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -34,14 +34,12 @@ class Settings(BaseSettings): # Empty string = endpoint disabled. scrape_admin_token: str = "" - # NSPD-scraper schedule (кадастровые кварталы / здания). - # По умолчанию: 20-е число февраля/мая/августа/ноября в 03:30 МСК - # (после квартальных публикаций rosreestr_deals). - scrape_nspd_cron: str = "30 3 20 2,5,8,11 *" - # Регионы для NSPD-sweep (comma-separated rosreestr region codes). - scrape_nspd_default_regions: str = "66" - # Пауза между NSPD-запросами в мс. <600мс — высокий риск WAF 403. - scrape_nspd_rate_ms: int = 600 + # NSPD-scraper (Playwright) УДАЛЁН 2026-05-11. Сменён на bulk geo-fetcher + # через rosreestr2coord — запускается вручную через /admin/scrape/geo. + # Settings оставлены deprecated на случай отката (можно удалить позже). + scrape_nspd_cron: str = "30 3 20 2,5,8,11 *" # DEPRECATED + scrape_nspd_default_regions: str = "66" # DEPRECATED + scrape_nspd_rate_ms: int = 600 # DEPRECATED — see nspd_lite_rate_ms # Objective.ru API (api.objctv.ru) — платная аналитика первички/вторички. # Лежит наряду с DOM.РФ kn + rosreestr CSV как 3-й источник истины с diff --git a/backend/app/services/scrapers/nspd_kn.py b/backend/app/services/scrapers/nspd_kn.py deleted file mode 100644 index fed335c0..00000000 --- a/backend/app/services/scrapers/nspd_kn.py +++ /dev/null @@ -1,576 +0,0 @@ -"""NSPD scraper — кадастровые кварталы и здания через nspd.gov.ru API. - -Endpoints: - GET /api/geoportal/v2/search/geoportal?thematicSearchId=2&query={cad} - → polygon кадастрового квартала (1 feature с label = cad-номер) - GET /api/geoportal/v2/search/geoportal?thematicSearchId=1&query={cad} - → объекты внутри квартала (здания, земля, линейные). - Фильтруем categoryName='Здания' и cad_num LIKE '::%'. - -WAF nspd.gov.ru банит burst-запросы и нероссийские IP. Стратегия: -- rate-limit 600мс между запросами (≤100 req/min — наблюдаемый предел) -- exponential backoff на 403 (60s + 30s × attempt) -- запускать с RU IP (production worker должен быть в RU/CIS-zone) - -Этот модуль — рефакторинг data/sql/62_scrape_nspd_full.py из stand-alone скрипта -в библиотечную функцию для Celery. Логика идентична, но: -- работает через SQLAlchemy Session (FK к cad_quarters_geom/cad_buildings) -- пишет heartbeat и progress в nspd_scrape_runs -- логирует структурированно в nspd_scrape_log -""" - -from __future__ import annotations - -import json -import logging -import ssl -import time -import urllib.error -import urllib.parse -import urllib.request -from datetime import datetime -from typing import Any - -from sqlalchemy import text -from sqlalchemy.orm import Session - -from app.core.db import SessionLocal - -logger = logging.getLogger(__name__) - -NSPD_BASE = "https://nspd.gov.ru/api/geoportal/v2/search/geoportal" -HEADERS = { - "User-Agent": ( - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " - "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" - ), - "Accept": "application/json", - "Accept-Language": "ru-RU,ru;q=0.9", - "Referer": "https://nspd.gov.ru/map", -} -SSL_CTX = ssl._create_unverified_context() - -DEFAULT_RATE_MS = 600 -# Heartbeat каждый cad — даёт UI видимый прогресс и быстрый zombie-detect -# на прод-VM, где worker может тихо стопать без error в логе. -DEFAULT_HEARTBEAT_EVERY = 1 # quarters -DEFAULT_COMMIT_EVERY = 10 -# Retries × timeout × WAF backoff формирует worst-case на один cad. -# Старые значения (5×30s + 60+90+120+150+180s = 600+150 = ~12.5 мин) дают -# инвиз-зависание worker'а если NSPD WAF режет nonbrowser fingerprint. -# Новые: до ~3 мин на cad, чтобы при 'тихом' WAF run падал быстрее. -DEFAULT_RETRIES = 3 -DEFAULT_TIMEOUT_S = 12 - -# region_code (rosreestr) → cad-prefix фильтр для cad_buildings. -# 66 = Свердловская обл., 66:41 = ЕКБ. Для других регионов добавлять mapping. -REGION_CAD_PREFIX: dict[int, str] = { - 66: "66:41:", # Екатеринбург -} - - -# ── HTTP layer ──────────────────────────────────────────────────────────────── - - -class WafBlockedError(RuntimeError): - """403 после всех retry — WAF банит, прогон не продолжаем.""" - - -def nspd_fetch( - thematic_search_id: int, - query: str, - *, - retries: int = DEFAULT_RETRIES, - timeout: int = DEFAULT_TIMEOUT_S, - on_403: Any = None, -) -> dict | None: - """Fetch NSPD с retry/backoff. on_403 callable(attempt) — для трекинга WAF.""" - qs = urllib.parse.urlencode({"thematicSearchId": thematic_search_id, "query": query}) - url = f"{NSPD_BASE}?{qs}" - req = urllib.request.Request(url, headers=HEADERS) - last_err: Exception | None = None - for attempt in range(retries): - try: - with urllib.request.urlopen(req, timeout=timeout, context=SSL_CTX) as r: - return json.loads(r.read().decode("utf-8")) - except urllib.error.HTTPError as e: - last_err = e - if e.code == 404: - return None - if e.code == 403: - if on_403: - on_403(attempt) - wait = 60 + 30 * attempt - logger.warning( - "WAF 403 для %s (попытка %d/%d), пауза %ds", query, attempt + 1, retries, wait - ) - time.sleep(wait) - continue - logger.warning("HTTP %s для %s (попытка %d)", e.code, query, attempt + 1) - except (urllib.error.URLError, TimeoutError, OSError) as e: - last_err = e - logger.warning("Сетевая ошибка для %s: %s (попытка %d)", query, e, attempt + 1) - time.sleep(min(2**attempt, 30)) - if isinstance(last_err, urllib.error.HTTPError) and last_err.code == 403: - raise WafBlockedError(f"WAF banned after {retries} retries on {query}") - logger.error("FAILED %s после %d попыток: %s", query, retries, last_err) - return None - - -# ── Geometry helpers ────────────────────────────────────────────────────────── - - -def poly_to_wkt(geom: dict | None) -> str | None: - if not geom: - return None - - def ring(r: list) -> str: - return "(" + ",".join(f"{p[0]} {p[1]}" for p in r) + ")" - - t = geom.get("type") - if t == "Polygon": - return "POLYGON(" + ",".join(ring(r) for r in geom["coordinates"]) + ")" - if t == "MultiPolygon": - return ( - "MULTIPOLYGON(" - + ",".join("(" + ",".join(ring(r) for r in p) + ")" for p in geom["coordinates"]) - + ")" - ) - if t == "Point": - return f"POINT({geom['coordinates'][0]} {geom['coordinates'][1]})" - return None - - -# ── SQL helpers ─────────────────────────────────────────────────────────────── - - -def _to_int(x: Any) -> int | None: - if x is None or x == "": - return None - try: - return int(x) - except (TypeError, ValueError): - return None - - -def _to_num(x: Any) -> float | None: - if x is None or x == "": - return None - try: - return float(x) - except (TypeError, ValueError): - return None - - -def _to_date(x: Any) -> str | None: - if not x: - return None - s = str(x).split(" ")[0] - return s or None - - -def get_pending_cads(db: Session, region_code: int) -> list[str]: - """Cad-кварталы с ДДУ-сделками в регионе минус уже scraped. - - NOTE: regex с двоеточиями ('!~ ":0000000$"') ломают SQLAlchemy text() — - двоеточие парсится как named-parameter prefix. Используем NOT LIKE через - bindparam (быстрее regex и безопасно). - """ - rows = db.execute( - text( - """ - SELECT DISTINCT quarter_cad_number - FROM rosreestr_deals - WHERE region_code = :rc - AND doc_type = 'ДДУ' - AND realestate_type_code = '002001003000' - AND quarter_cad_number IS NOT NULL - AND quarter_cad_number <> '' - AND quarter_cad_number NOT LIKE :bad_prefix - AND quarter_cad_number NOT LIKE :bad_suffix - """ - ), - {"rc": region_code, "bad_prefix": "00:00:%", "bad_suffix": "%:0000000"}, - ).all() - pending = {r[0] for r in rows} - done = {r[0] for r in db.execute(text("SELECT cad_number FROM cad_quarters_geom")).all()} - return sorted(pending - done) - - -def insert_quarter(db: Session, cad: str, wkt: str, raw_props: dict | None) -> None: - db.execute( - text( - """ - INSERT INTO cad_quarters_geom (cad_number, geom, raw_props, source) - VALUES ( - :cad, - ST_Multi(ST_Transform(ST_SetSRID(ST_GeomFromText(:wkt), 3857), 4326)) - ::geometry(MultiPolygon, 4326), - CAST(:props AS jsonb), - 'nspd' - ) - ON CONFLICT (cad_number) DO UPDATE SET - geom = EXCLUDED.geom, - raw_props = COALESCE(EXCLUDED.raw_props, cad_quarters_geom.raw_props), - fetched_at = NOW() - """ - ), - { - "cad": cad, - "wkt": wkt, - "props": json.dumps(raw_props, ensure_ascii=False) if raw_props else None, - }, - ) - - -def insert_buildings(db: Session, qcad: str, features: list[dict], cad_prefix: str) -> int: - n = 0 - for f in features: - props_outer = f.get("properties") or {} - if props_outer.get("categoryName") != "Здания": - continue - opts = props_outer.get("options") or {} - cn = opts.get("cad_num") - if not cn or not cn.startswith(cad_prefix): - continue - wkt = poly_to_wkt(f.get("geometry")) - if not wkt: - continue - db.execute( - text( - """ - INSERT INTO cad_buildings ( - cad_num, quarter_cad_num, geom, purpose, building_name, - readable_address, area, floors, year_built, year_commisioning, - cost_value, registration_date, - status, ownership_type, cultural_heritage, underground_floors, - build_record_area, build_record_type, common_data_status, obj_type, - raw_props - ) - VALUES ( - :cad_num, :qcad, - ST_Transform(ST_SetSRID(ST_GeomFromText(:wkt), 3857), 4326), - :purpose, :name, :addr, :area, :floors, :yb, :yc, - :cost, :reg_date, - :status, :ownership, :cultural, :underground, - :build_rec_area, :build_rec_type, :common_status, :obj_type, - CAST(:raw_props AS jsonb) - ) - ON CONFLICT (cad_num) DO UPDATE SET - quarter_cad_num = EXCLUDED.quarter_cad_num, - geom = EXCLUDED.geom, - purpose = EXCLUDED.purpose, - building_name = EXCLUDED.building_name, - readable_address = EXCLUDED.readable_address, - area = EXCLUDED.area, - floors = EXCLUDED.floors, - year_built = EXCLUDED.year_built, - year_commisioning = EXCLUDED.year_commisioning, - cost_value = EXCLUDED.cost_value, - registration_date = EXCLUDED.registration_date, - status = EXCLUDED.status, - ownership_type = EXCLUDED.ownership_type, - cultural_heritage = EXCLUDED.cultural_heritage, - underground_floors = EXCLUDED.underground_floors, - build_record_area = EXCLUDED.build_record_area, - build_record_type = EXCLUDED.build_record_type, - common_data_status = EXCLUDED.common_data_status, - obj_type = EXCLUDED.obj_type, - raw_props = COALESCE(EXCLUDED.raw_props, cad_buildings.raw_props), - fetched_at = NOW() - """ - ), - { - "cad_num": cn, - "qcad": qcad, - "wkt": wkt, - "purpose": opts.get("purpose"), - "name": opts.get("building_name"), - "addr": opts.get("readable_address"), - "area": _to_num(opts.get("area")), - # floors / underground_floors хранятся TEXT — NSPD отдаёт - # диапазоны '1-2', '2-3'. _to_int съел бы их в NULL и потерял - # данные. Аналитика делает CAST через regex '^[0-9]+$'. - "floors": ( - str(opts["floors"]).strip() if opts.get("floors") not in (None, "") else None - ), - "yb": _to_int(opts.get("year_built")), - "yc": _to_int(opts.get("year_commisioning")), - "cost": _to_num(opts.get("cost_value")), - "reg_date": _to_date(opts.get("registration_date")), - "status": opts.get("status"), - "ownership": opts.get("ownership_type"), - "cultural": opts.get("cultural_heritage"), - "underground": ( - str(opts["underground_floors"]).strip() - if opts.get("underground_floors") not in (None, "") - else None - ), - "build_rec_area": _to_num(opts.get("build_record_area")), - "build_rec_type": opts.get("build_record_type"), - "common_status": opts.get("common_data_status"), - "obj_type": opts.get("obj_type"), - "raw_props": json.dumps(opts, ensure_ascii=False) if opts else None, - }, - ) - n += 1 - return n - - -# ── Run lifecycle ───────────────────────────────────────────────────────────── - - -def _start_run(db: Session, region_code: int, triggered_by: str, pending_count: int) -> int: - row = db.execute( - text( - """ - INSERT INTO nspd_scrape_runs (region_code, triggered_by, pending_count, status) - VALUES (:rc, :tb, :pc, 'running') - RETURNING run_id - """ - ), - {"rc": region_code, "tb": triggered_by, "pc": pending_count}, - ).scalar_one() - db.commit() - return int(row) - - -def _heartbeat(db: Session, run_id: int, **counts: int) -> None: - sets = ["heartbeat_at = NOW()"] - params: dict[str, Any] = {"rid": run_id} - for k, v in counts.items(): - sets.append(f"{k} = :{k}") - params[k] = v - db.execute( - text(f"UPDATE nspd_scrape_runs SET {', '.join(sets)} WHERE run_id = :rid"), - params, - ) - db.commit() - - -def _finish_run( - db: Session, - run_id: int, - *, - status: str, - error: str | None = None, - **counts: int, -) -> None: - sets = ["finished_at = NOW()", "status = :status"] - params: dict[str, Any] = {"rid": run_id, "status": status, "error": error} - sets.append("error = :error") - for k, v in counts.items(): - sets.append(f"{k} = :{k}") - params[k] = v - db.execute( - text(f"UPDATE nspd_scrape_runs SET {', '.join(sets)} WHERE run_id = :rid"), - params, - ) - db.commit() - - -def _log( - db: Session, - run_id: int | None, - *, - level: str, - stage: str, - message: str, - cad: str | None = None, -) -> None: - try: - db.execute( - text( - """ - INSERT INTO nspd_scrape_log (run_id, level, stage, cad_number, message) - VALUES (:rid, :lvl, :st, :cad, :msg) - """ - ), - {"rid": run_id, "lvl": level, "st": stage, "cad": cad, "msg": message[:1000]}, - ) - db.commit() - except Exception as e: - logger.warning("nspd_scrape_log insert failed: %s", e) - - -# ── Main entrypoint ─────────────────────────────────────────────────────────── - - -def run_region_scrape( - *, - region_code: int, - triggered_by: str = "beat", - limit: int | None = None, - rate_ms: int = DEFAULT_RATE_MS, - commit_every: int = DEFAULT_COMMIT_EVERY, - heartbeat_every: int = DEFAULT_HEARTBEAT_EVERY, -) -> dict[str, Any]: - """Полный sweep одного региона. Безопасен для повторных запусков — - pending = (rosreestr cads) − (cad_quarters_geom cads).""" - cad_prefix = REGION_CAD_PREFIX.get(region_code) - if not cad_prefix: - raise ValueError(f"region_code={region_code} unknown — добавь в REGION_CAD_PREFIX") - - db = SessionLocal() - run_id: int | None = None - try: - pending = get_pending_cads(db, region_code) - if limit: - pending = pending[:limit] - run_id = _start_run(db, region_code, triggered_by, len(pending)) - _log( - db, - run_id, - level="info", - stage="discover", - message=f"region={region_code} pending={len(pending)} cads", - ) - - if not pending: - _finish_run(db, run_id, status="done", quarters_ok=0, quarters_failed=0, buildings_ok=0) - return {"run_id": run_id, "pending": 0, "ok": 0, "failed": 0, "buildings": 0} - - ok = 0 - failed = 0 - n_buildings = 0 - n_requests = 0 - n_waf = 0 - started = time.time() - - for i, cn in enumerate(pending, 1): - try: - # Pre-fetch лог в DB — чтобы видеть «застрял на cad X» если - # urllib зависнет в read(). Heartbeat не пишется во время - # nspd_fetch, и без этого лога run выглядит как «discover, тишина». - logger.info("nspd[%d/%d] %s — fetching polygon", i, len(pending), cn) - if i == 1 or i % 10 == 0: - _log( - db, - run_id, - level="info", - stage="quarter_fetch", - cad=cn, - message=f"fetching polygon ({i}/{len(pending)})", - ) - j2 = nspd_fetch(2, cn, on_403=lambda _a: None) - n_requests += 1 - qf = None - q_props: dict | None = None - if j2: - for f in j2.get("data", {}).get("features") or []: - if (f.get("properties") or {}).get("label") == cn: - qf = f - q_props = (f.get("properties") or {}).get("options") or {} - break - wkt = poly_to_wkt(qf.get("geometry") if qf else None) if qf else None - if wkt: - insert_quarter(db, cn, wkt, q_props) - ok += 1 - else: - failed += 1 - _log( - db, - run_id, - level="warn", - stage="quarter_fetch", - cad=cn, - message="no polygon returned", - ) - - j1 = nspd_fetch(1, cn, on_403=lambda _a: None) - n_requests += 1 - if j1: - features = j1.get("data", {}).get("features") or [] - n_buildings += insert_buildings(db, cn, features, cad_prefix) - - if i % commit_every == 0: - db.commit() - if i % heartbeat_every == 0: - _heartbeat( - db, - run_id, - quarters_ok=ok, - quarters_failed=failed, - buildings_ok=n_buildings, - requests_count=n_requests, - waf_429_count=n_waf, - ) - - except WafBlockedError as e: - _log( - db, - run_id, - level="error", - stage="quarter_fetch", - cad=cn, - message=f"WAF blocked: {e}", - ) - _finish_run( - db, - run_id, - status="failed", - error=str(e), - quarters_ok=ok, - quarters_failed=failed, - buildings_ok=n_buildings, - requests_count=n_requests, - waf_429_count=n_waf + 1, - ) - raise - except Exception as e: - failed += 1 - db.rollback() - _log( - db, - run_id, - level="error", - stage="quarter_fetch", - cad=cn, - message=f"{type(e).__name__}: {e}", - ) - - time.sleep(rate_ms / 1000.0) - - db.commit() - elapsed = time.time() - started - _finish_run( - db, - run_id, - status="done", - quarters_ok=ok, - quarters_failed=failed, - buildings_ok=n_buildings, - requests_count=n_requests, - waf_429_count=n_waf, - ) - _log( - db, - run_id, - level="info", - stage="done", - message=( - f"elapsed={elapsed:.0f}s ok={ok} failed={failed} " - f"buildings={n_buildings} reqs={n_requests}" - ), - ) - return { - "run_id": run_id, - "pending": len(pending), - "ok": ok, - "failed": failed, - "buildings": n_buildings, - "requests": n_requests, - "elapsed_s": int(elapsed), - "started_at": datetime.utcfromtimestamp(started).isoformat() + "Z", - } - except WafBlockedError: - raise - except Exception as e: - if run_id: - try: - _finish_run(db, run_id, status="failed", error=f"{type(e).__name__}: {e}") - except Exception: - pass - raise - finally: - db.close() diff --git a/backend/app/workers/celery_app.py b/backend/app/workers/celery_app.py index ae3818b0..ff52418d 100644 --- a/backend/app/workers/celery_app.py +++ b/backend/app/workers/celery_app.py @@ -30,17 +30,12 @@ def _default_regions() -> list[int]: return [int(x.strip()) for x in settings.scrape_kn_default_regions.split(",") if x.strip()] -def _nspd_default_regions() -> list[int]: - return [int(x.strip()) for x in settings.scrape_nspd_default_regions.split(",") if x.strip()] - - celery_app = Celery( "gendesign", broker=settings.redis_url, backend=settings.redis_url, include=[ "app.workers.tasks.scrape_kn", - "app.workers.tasks.scrape_nspd", "app.workers.tasks.refresh_analytics", "app.workers.tasks.scrape_objective", "app.workers.tasks.objective_etl", @@ -61,23 +56,10 @@ celery_app.conf.beat_schedule = { for rc in _default_regions() } -# NSPD-скрейп — квартально, после публикации rosreestr_deals. -# Pending = (новые quarter_cad_number из rosreestr_deals) − (cad_quarters_geom). -# При limit=None прогон захватит все накопленные cad-кварталы за квартал. -celery_app.conf.beat_schedule.update( - { - f"nspd-region-{rc}": { - "task": "tasks.scrape_nspd.scrape_nspd_region", - "schedule": _parse_cron(settings.scrape_nspd_cron), - "kwargs": { - "region_code": rc, - "triggered_by": "beat", - "rate_ms": settings.scrape_nspd_rate_ms, - }, - } - for rc in _nspd_default_regions() - } -) +# NSPD beat schedule УДАЛЁН 2026-05-11. +# Старый Playwright-based scraper (tasks.scrape_nspd) сменён на bulk geo-fetcher +# через rosreestr2coord — запускается вручную через UI /admin/scrape/geo. +# Расписания по умолчанию нет (cron можно добавить через beat позже если надо). # Refresh ekb_districts медианы — ежемесячно 5-го числа в 04:00 МСК # (после публикации новых rosreestr-кварталов и NSPD beat-cycle). @@ -194,71 +176,9 @@ def _resume_zombie_runs(sender=None, **_kwargs) -> None: except Exception as e: logger.warning("worker_ready: failed to enqueue resume for run=%s: %s", rid, e) - # NSPD-runs: помечаем зомби (>15 мин без heartbeat — NSPD сам по себе - # медленнее kn, дельта больше). Resume = просто заново ставим - # scrape_nspd_region с тем же region_code и triggered_by='resume': - # сервис идемпотентен — pending = (rosreestr ДДУ cads) − (cad_quarters_geom done), - # уже скрейпнутые квартала пропустятся автоматом. - db = SessionLocal() - nspd_resume_targets: list[int] = [] - try: - zombie_rows = ( - db.execute( - text( - """ - UPDATE nspd_scrape_runs - SET status = 'zombie', - finished_at = NOW(), - error = COALESCE(error, - 'auto-zombie at worker_ready, resume scheduled') - WHERE status = 'running' - AND COALESCE(heartbeat_at, started_at) - < NOW() - INTERVAL '15 minutes' - RETURNING run_id, region_code - """ - ) - ) - .mappings() - .all() - ) - db.commit() - # Уникальные регионы для resume (разные ран-id для одного региона — - # повторно не enqueueum, lock и idempotency пропустят дубль). - seen: set[int] = set() - for r in zombie_rows: - rc = int(r["region_code"]) - if rc not in seen: - seen.add(rc) - nspd_resume_targets.append(rc) - logger.info( - "worker_ready: NSPD zombie run=%s region=%s — resume scheduled", - r["run_id"], - rc, - ) - except Exception as e: - logger.warning("worker_ready nspd zombie scan failed: %s", e) - try: - db.rollback() - except Exception: - pass - finally: - db.close() - - if nspd_resume_targets: - from app.workers.tasks.scrape_nspd import scrape_nspd_region - - for rc in nspd_resume_targets: - try: - scrape_nspd_region.apply_async( - kwargs={"region_code": rc, "triggered_by": "resume"}, - ) - logger.info("worker_ready: scrape_nspd_region enqueued region=%s", rc) - except Exception as e: - logger.warning( - "worker_ready: failed to enqueue NSPD resume for region=%s: %s", - rc, - e, - ) + # NSPD-runs (старый Playwright-scraper) resume УДАЛЁН 2026-05-11. Скрапер + # снят с эксплуатации в пользу bulk geo-fetcher (nspd_geo). История + # старых runs сохранена в nspd_scrape_runs — никаких side-effects. # NSPD geo-jobs: bulk-fetcher с собственной resume-логикой через # nspd_geo_jobs / nspd_geo_targets. Жадный resume: status='running' со diff --git a/backend/app/workers/tasks/nspd_geo.py b/backend/app/workers/tasks/nspd_geo.py index ba0ceccc..d69cff42 100644 --- a/backend/app/workers/tasks/nspd_geo.py +++ b/backend/app/workers/tasks/nspd_geo.py @@ -45,6 +45,29 @@ WAF_BACKOFF_BASE_S = 30 # ── Helpers для job/target lifecycle ──────────────────────────────────────── +def _log( + db: Session, job_id: int, level: str, stage: str, message: str, cad_num: str | None = None +) -> None: + """Запись в nspd_geo_log (виден через v_scrape_log_unified в admin UI).""" + try: + db.execute( + text( + """ + INSERT INTO nspd_geo_log (job_id, level, stage, cad_num, message) + VALUES (:jid, :lvl, :stg, :cad, :msg) + """ + ), + {"jid": job_id, "lvl": level, "stg": stage, "cad": cad_num, "msg": message[:1000]}, + ) + db.commit() + except Exception as e: + logger.warning("nspd_geo_log write failed for job=%s: %s", job_id, e) + try: + db.rollback() + except Exception: + pass + + def _start_job(db: Session, job_id: int) -> None: db.execute( text( @@ -60,6 +83,7 @@ def _start_job(db: Session, job_id: int) -> None: {"id": job_id}, ) db.commit() + _log(db, job_id, "info", "start", "job started/resumed") def _heartbeat(db: Session, job_id: int, **counts: int) -> None: @@ -366,6 +390,14 @@ def process_nspd_geo_job(self: Any, job_id: int) -> dict[str, Any]: ) # НЕ помечаем failed — остаётся pending для следующей итерации _heartbeat(db, job_id, requests_count=n_requests, waf_blocked_count=n_waf) + _log( + db, + job_id, + "warn", + "waf", + f"WAF/429 (consecutive={consecutive_waf}), backoff {wait}s", + cad_num=cad, + ) time.sleep(wait) if consecutive_waf >= 8: # 8 подряд WAF — пауза job'а до следующего worker_ready @@ -375,11 +407,19 @@ def process_nspd_geo_job(self: Any, job_id: int) -> dict[str, Any]: "paused", error=f"WAF blocked {consecutive_waf} consecutive", ) + _log( + db, + job_id, + "error", + "paused", + f"job paused after {consecutive_waf} consecutive WAF blocks", + ) return {"job_id": job_id, "paused": True, "reason": "waf"} continue except (NspdLiteError, Exception) as e: logger.warning("fetch failed for %s: %s", cad, e) + _log(db, job_id, "warn", "fetch_fail", str(e)[:300], cad_num=cad) db.execute( text( """ @@ -418,6 +458,13 @@ def process_nspd_geo_job(self: Any, job_id: int) -> dict[str, Any]: ) # done даже при failed targets — UI покажет failed_count отдельно _finish_job(db, job_id, "done") + _log( + db, + job_id, + "info", + "done", + f"done: {done} ok / {failed} failed / {n_requests} req / {n_waf} WAF blocks", + ) logger.info( "process_nspd_geo_job done: job=%s done=%d failed=%d req=%d waf=%d", job_id, diff --git a/backend/app/workers/tasks/scrape_nspd.py b/backend/app/workers/tasks/scrape_nspd.py deleted file mode 100644 index aab2693f..00000000 --- a/backend/app/workers/tasks/scrape_nspd.py +++ /dev/null @@ -1,104 +0,0 @@ -"""Celery task wrapper для NSPD-скрейпа кадастровых кварталов.""" - -from __future__ import annotations - -import logging -from contextlib import contextmanager -from typing import Any - -import redis - -from app.core.config import settings -from app.services.scrapers.nspd_kn import WafBlockedError, run_region_scrape -from app.workers.celery_app import celery_app - -logger = logging.getLogger(__name__) - -# NSPD-скрейпы 700+ cad-кварталов на регион занимают ~2 часа (600мс/req × 2 req/cad). -# Lock TTL 3 часа — потолок с запасом, не оставляет зомби-lock на долго. -_LOCK_TTL_SECONDS = 3 * 60 * 60 - - -def _lock_key(region_code: int) -> str: - return f"scrape:nspd:lock:{region_code}" - - -def force_release_lock(region_code: int) -> bool: - """Принудительно удалить Redis-lock для emergency manual trigger.""" - key = _lock_key(region_code) - r = redis.Redis.from_url(settings.redis_url) - try: - return bool(r.delete(key)) - except Exception as e: - logger.warning("force_release_lock %s failed: %s", key, e) - return False - - -@contextmanager -def _region_lock(region_code: int): - key = _lock_key(region_code) - r = redis.Redis.from_url(settings.redis_url) - acquired = r.set(key, "1", nx=True, ex=_LOCK_TTL_SECONDS) - try: - yield bool(acquired) - finally: - if acquired: - try: - r.delete(key) - except Exception: - pass - - -@celery_app.task( - bind=True, - name="tasks.scrape_nspd.scrape_nspd_region", - max_retries=2, - autoretry_for=(WafBlockedError,), - retry_backoff=600, # 10 мин начальный, удваивается - retry_backoff_max=7200, # потолок 2 часа -) -def scrape_nspd_region( - self: Any, - region_code: int = 66, - triggered_by: str = "beat", - limit: int | None = None, - rate_ms: int = 600, -) -> dict[str, Any]: - """Запуск NSPD-скрейпа одного региона. По расписанию: 20-е число - февраля/мая/августа/ноября (после квартальных публикаций rosreestr). - - Args: - region_code: код региона rosreestr (66 = Свердл). - triggered_by: 'beat' | 'manual' | 'resume' — для журналирования. - limit: ограничить N cad-кварталов (smoke-тест). - rate_ms: пауза между запросами (по умолчанию 600мс — наблюдаемый предел WAF). - - Returns: - dict с run_id и итогами (pending/ok/failed/buildings/requests/elapsed_s), - либо {'skipped': True, 'reason': 'lock_held'} если уже идёт прогон. - """ - with _region_lock(region_code) as got_lock: - if not got_lock: - logger.warning( - "scrape_nspd_region SKIPPED region=%s — другой sweep идёт (Redis lock)", - region_code, - ) - return { - "skipped": True, - "reason": "lock_held", - "region_code": region_code, - "lock_key": _lock_key(region_code), - } - - logger.info( - "scrape_nspd_region start region=%s trigger=%s limit=%s", - region_code, - triggered_by, - limit, - ) - return run_region_scrape( - region_code=region_code, - triggered_by=triggered_by, - limit=limit, - rate_ms=rate_ms, - ) diff --git a/data/sql/77_nspd_geo_jobs.sql b/data/sql/77_nspd_geo_jobs.sql index 617cb5cd..2ed738da 100644 --- a/data/sql/77_nspd_geo_jobs.sql +++ b/data/sql/77_nspd_geo_jobs.sql @@ -28,7 +28,9 @@ CREATE TABLE IF NOT EXISTS nspd_geo_jobs ( source_params jsonb, -- что-то вроде {"region_codes":[66,74,72,59], "thematic_id":2} -- Параметры fetch rate_ms INTEGER NOT NULL DEFAULT 600, - use_rosreestr2coord BOOLEAN NOT NULL DEFAULT FALSE, -- если TRUE — community-lib, иначе наш nspd_lite + -- TRUE = community-lib rosreestr2coord (default с 2026-05-11), + -- FALSE = наш nspd_lite urllib-based (fallback). + use_rosreestr2coord BOOLEAN NOT NULL DEFAULT TRUE, -- Состояние status TEXT NOT NULL DEFAULT 'queued', -- queued | running | done | failed | paused started_at timestamptz, @@ -83,3 +85,67 @@ CREATE INDEX IF NOT EXISTS nspd_geo_targets_failed_idx COMMENT ON TABLE nspd_geo_targets IS 'Список cad-номеров для обработки в рамках job. Resume-state: ' 'task просто SELECT WHERE status=pending и продолжает.'; + +-- ── nspd_geo_log: логи (видны через v_scrape_log_unified) ─────────────────── + +CREATE TABLE IF NOT EXISTS nspd_geo_log ( + log_id BIGSERIAL PRIMARY KEY, + job_id BIGINT REFERENCES nspd_geo_jobs(job_id) ON DELETE CASCADE, + ts timestamptz NOT NULL DEFAULT NOW(), + level TEXT NOT NULL DEFAULT 'info', + stage TEXT, + cad_num TEXT, + message TEXT NOT NULL +); +CREATE INDEX IF NOT EXISTS nspd_geo_log_job_idx + ON nspd_geo_log(job_id, log_id DESC); +CREATE INDEX IF NOT EXISTS nspd_geo_log_ts_idx + ON nspd_geo_log(ts DESC); + +-- ── Расширение unified-views на nspd_geo (UNION ALL) ──────────────────────── + +CREATE OR REPLACE VIEW v_scrape_runs_unified AS +SELECT 'kn' AS scraper_type, run_id, started_at, finished_at, heartbeat_at, + status, error, NULL::text AS triggered_by, + region_codes::text AS scope, requests_count, + objects_count AS items_ok, NULL::integer AS items_failed, + flats_count AS sub_items_ok, + progress_obj_index AS progress_idx, total_obj_count AS progress_total, + jsonb_build_object('developers', developer_ids, 'snapshot_date', snapshot_date, + 'resumed_from', resumed_from_run_id, 'params', params) AS extra + FROM kn_scrape_runs +UNION ALL +SELECT 'nspd', run_id, started_at, finished_at, heartbeat_at, status, error, + triggered_by, region_code::text, requests_count, + quarters_ok, quarters_failed, buildings_ok, + NULL, pending_count, + jsonb_build_object('waf_429_count', waf_429_count) + FROM nspd_scrape_runs +UNION ALL +SELECT 'objective', run_id, started_at, finished_at, heartbeat_at, status, error, + triggered_by, group_name, requests_count, + reports_ok, reports_failed, rows_lots, + NULL, NULL, + jsonb_build_object('rows_corpus_room', rows_corpus_room, 'rows_history', rows_history) + FROM objective_scrape_runs +UNION ALL +SELECT 'nspd_geo', job_id, started_at, finished_at, heartbeat_at, status, error, + triggered_by, COALESCE(name, job_kind), + requests_count, targets_done, targets_failed, targets_skipped, + targets_done, targets_total, + jsonb_build_object('job_kind', job_kind, 'source_kind', source_kind, + 'rate_ms', rate_ms, 'use_rosreestr2coord', use_rosreestr2coord, + 'waf_blocked_count', waf_blocked_count, + 'source_params', source_params) + FROM nspd_geo_jobs; + +CREATE OR REPLACE VIEW v_scrape_log_unified AS +SELECT 'kn' AS scraper_type, log_id, run_id, ts, level, stage, + obj_id::text AS entity_id, message + FROM kn_scrape_log +UNION ALL +SELECT 'nspd', log_id, run_id, ts, level, stage, cad_number, message + FROM nspd_scrape_log +UNION ALL +SELECT 'nspd_geo', log_id, job_id AS run_id, ts, level, stage, cad_num, message + FROM nspd_geo_log; diff --git a/frontend/src/app/admin/layout.tsx b/frontend/src/app/admin/layout.tsx index 01e0720d..8f4bad2c 100644 --- a/frontend/src/app/admin/layout.tsx +++ b/frontend/src/app/admin/layout.tsx @@ -6,8 +6,7 @@ import { usePathname } from "next/navigation"; const TABS = [ { href: "/admin/scrape/all", label: "📊 Все скраперы (сводно)" }, { href: "/admin/scrape", label: "Скрапер DOM.РФ" }, - { href: "/admin/scrape/nspd", label: "Скрапер NSPD" }, - { href: "/admin/scrape/geo", label: "🗺 Geo bulk-jobs" }, + { href: "/admin/scrape/geo", label: "🗺 NSPD geo (rosreestr2coord)" }, { href: "/admin/scrape/objective", label: "ETL Objective" }, { href: "/admin/leads", label: "Лиды PRINZIP" }, ]; diff --git a/frontend/src/app/admin/scrape/all/page.tsx b/frontend/src/app/admin/scrape/all/page.tsx index 6f899022..30e8d5e7 100644 --- a/frontend/src/app/admin/scrape/all/page.tsx +++ b/frontend/src/app/admin/scrape/all/page.tsx @@ -6,7 +6,7 @@ import { useQuery } from "@tanstack/react-query"; import { apiFetch } from "@/lib/api"; interface UnifiedRunRow { - scraper_type: "kn" | "nspd" | "objective"; + scraper_type: "kn" | "nspd" | "objective" | "nspd_geo"; run_id: number; started_at: string | null; finished_at: string | null; @@ -25,7 +25,7 @@ interface UnifiedRunRow { } interface UnifiedLogRow { - scraper_type: "kn" | "nspd"; + scraper_type: "kn" | "nspd" | "nspd_geo"; log_id: number; run_id: number | null; ts: string | null; @@ -40,6 +40,7 @@ const SCRAPER_BADGE: Record = kn: { bg: "#dbeafe", fg: "#1d4ed8", label: "DOM.РФ kn" }, nspd: { bg: "#fef3c7", fg: "#a16207", label: "NSPD" }, objective: { bg: "#d1fae5", fg: "#059669", label: "Objective" }, + nspd_geo: { bg: "#ede9fe", fg: "#6d28d9", label: "🗺 NSPD geo" }, }; const STATUS_COLOR: Record = { @@ -68,9 +69,9 @@ export default function ScrapeAllAdminPage() { ? "" : (localStorage.getItem("admin_token") ?? ""), ); - const [filter, setFilter] = useState<"all" | "kn" | "nspd" | "objective">( - "all", - ); + const [filter, setFilter] = useState< + "all" | "kn" | "nspd" | "objective" | "nspd_geo" + >("all"); const [logsRunId, setLogsRunId] = useState(""); const saveToken = (v: string) => { @@ -124,20 +125,22 @@ export default function ScrapeAllAdminPage() { Скраперы — сводный dashboard

- Все 3 скрапера в одной таблице (через v_scrape_runs_unified - ). Для запуска / триггеров — отдельные страницы:{" "} + Все скраперы в одной таблице (через v_scrape_runs_unified). + Для запуска / триггеров — отдельные страницы:{" "} - kn + DOM.РФ kn {" "} ·{" "} - - NSPD + + 🗺 NSPD geo {" "} ·{" "} Objective - . + . Старый Playwright-based NSPD-скрапер снят с эксплуатации 2026-05-11 — + история его прогонов осталась под фильтром nspd только для + аудита.

@@ -177,20 +180,22 @@ export default function ScrapeAllAdminPage() {
Filter: - {(["all", "kn", "nspd", "objective"] as const).map((t) => ( - - ))} + {(["all", "kn", "nspd", "nspd_geo", "objective"] as const).map( + (t) => ( + + ), + )}
@@ -308,7 +313,8 @@ export default function ScrapeAllAdminPage() { {(r.scraper_type === "kn" || - r.scraper_type === "nspd") && ( + r.scraper_type === "nspd" || + r.scraper_type === "nspd_geo") && ( - - {!token ? ( - - ← введите Admin Token выше - - ) : null} - - - {trigger.isError ? ( -
- Ошибка: {String(trigger.error)} -
- ) : null} - - {trigger.isSuccess ? ( -
- ✓ Поставлено в очередь. task_id={" "} - - {trigger.data.task_id} - {" "} - · регион {trigger.data.region_code} - {trigger.data.limit ? ` · limit=${trigger.data.limit}` : ""} · rate= - {trigger.data.rate_ms}мс -
- ) : null} -
- -
-

- Покрытие cad_quarters_geom (регион {region}) -

- {!token ? ( -

- Введите Admin Token выше. -

- ) : coverage.isLoading ? ( -

Загрузка…

- ) : coverage.isError ? ( -

Ошибка: {String(coverage.error)}

- ) : coverage.data ? ( -
- - - 0 ? "#9a6700" : "#0a7a3a"} - /> - - = 90 - ? "#0a7a3a" - : "#9a6700" - } - /> -
- ) : null} -
- -
-
-

- Лог скрейпа{" "} - - · обновляется каждые 3 сек - -

- - setLogsRunId(e.target.value === "" ? "" : Number(e.target.value)) - } - placeholder="run_id (опц.)" - style={{ ...inputStyle, width: 130 }} - /> -
- {!token ? ( -

- Введите Admin Token чтобы видеть лог. -

- ) : logs.isLoading ? ( -

Загрузка…

- ) : logs.isError ? ( -

Ошибка: {String(logs.error)}

- ) : (logs.data?.length ?? 0) === 0 ? ( -

- Нет событий{logsRunId !== "" ? ` для run_id=${logsRunId}` : ""}. -

- ) : ( -
- {(logs.data ?? []).map((l) => { - const color = - l.level === "error" - ? "#fca5a5" - : l.level === "warn" - ? "#fcd34d" - : "#a3e635"; - return ( -
- - {l.ts?.slice(11, 19)} - {" "} - {l.level.toUpperCase()}{" "} - - [run-{l.run_id} - {l.stage ? ` · ${l.stage}` : ""} - {l.cad_number ? ` · ${l.cad_number}` : ""}] - {" "} - {l.message} -
- ); - })} -
- )} -
- -
-

Последние прогоны

- {!token ? ( -

- Введите Admin Token чтобы увидеть журнал. -

- ) : runs.isLoading ? ( -

Загрузка…

- ) : runs.isError ? ( -

Ошибка: {String(runs.error)}

- ) : ( - - - - {[ - "#", - "Старт", - "HB", - "Статус", - "Прогресс", - "Pending", - "OK", - "Fail", - "Зданий", - "Req", - "WAF", - "Trigger", - "Ошибка", - ].map((h) => ( - - ))} - - - - {(runs.data ?? []).map((r) => ( - - - - - - - - - - - - - - - - ))} - -
- {h} -
{r.run_id} - {r.started_at?.slice(0, 19).replace("T", " ")} - - {r.heartbeat_at ? r.heartbeat_at.slice(11, 19) : "—"} - - {r.status} - - {r.pending_count && r.pending_count > 0 - ? (() => { - const pct = Math.min( - 100, - Math.round( - ((r.quarters_ok + r.quarters_failed) / - r.pending_count) * - 100, - ), - ); - return ( -
-
-
0 - ? "#1d4ed8" - : "#9a6700", - }} - /> -
- {pct}% -
- ); - })() - : "—"} -
{r.pending_count ?? "—"}{r.quarters_ok} 0 ? "#b3261e" : undefined, - }} - > - {r.quarters_failed} - {r.buildings_ok}{r.requests_count} 0 ? "#b3261e" : undefined, - }} - > - {r.waf_429_count} - {r.triggered_by} - {r.error ?? ""} -
- )} -
- - ); -} - -function CoverageCard({ - label, - value, - color, -}: { - label: string; - value: string; - color?: string; -}) { - return ( -
-
- {label} -
-
- {value} -
-
- ); -} - -const cardStyle = { - background: "#fff", - border: "1px solid #e6e8ec", - borderRadius: 12, - padding: 20, -}; -const labelStyle = { - display: "block", - fontSize: 12, - color: "#5b6066", - marginBottom: 4, - textTransform: "uppercase" as const, - letterSpacing: 0.4, -}; -const inputStyle = { - padding: "8px 10px", - border: "1px solid #d1d5db", - borderRadius: 6, - fontSize: 14, - width: "100%", - boxSizing: "border-box" as const, -}; -const primaryBtn = { - padding: "10px 18px", - background: "#1d4ed8", - color: "#fff", - border: "none", - borderRadius: 6, - fontSize: 14, - fontWeight: 600, - cursor: "pointer", -}; -const secondaryBtn = { - padding: "8px 14px", - background: "#f3f4f6", - color: "#1f2937", - border: "1px solid #d1d5db", - borderRadius: 6, - fontSize: 13, - cursor: "pointer", -}; -const th = { - padding: "8px 10px", - textAlign: "left" as const, - fontWeight: 600, - borderBottom: "1px solid #e6e8ec", -}; -const td = { padding: "8px 10px" };