diff --git a/backend/app/api/v1/admin_scrape.py b/backend/app/api/v1/admin_scrape.py index 173afb9f..eb0faac4 100644 --- a/backend/app/api/v1/admin_scrape.py +++ b/backend/app/api/v1/admin_scrape.py @@ -618,7 +618,6 @@ class EnqueueGeoJobRequest(BaseModel): default=None, max_length=20, description="Для source=rosreestr_pending" ) rate_ms: int = Field(default=600, ge=100, le=10000) - use_rosreestr2coord: bool = False @router.post("/geo/jobs") @@ -681,7 +680,6 @@ def enqueue_geo_job( source_params={"region_codes": payload.region_codes, "thematic_id": payload.thematic_id}, cad_nums_with_thematic=cad_with_thematic, rate_ms=payload.rate_ms, - use_rosreestr2coord=payload.use_rosreestr2coord, triggered_by="manual", ) process_nspd_geo_job.apply_async(args=[job_id]) @@ -708,7 +706,7 @@ def list_geo_jobs( started_at, finished_at, heartbeat_at, targets_total, targets_done, targets_failed, targets_skipped, requests_count, waf_blocked_count, error, - rate_ms, use_rosreestr2coord, created_at + rate_ms, created_at FROM nspd_geo_jobs ORDER BY created_at DESC LIMIT :lim @@ -738,7 +736,6 @@ def list_geo_jobs( "waf_blocked_count": r["waf_blocked_count"], "error": r["error"], "rate_ms": r["rate_ms"], - "use_rosreestr2coord": r["use_rosreestr2coord"], "created_at": r["created_at"].isoformat() if r["created_at"] else None, "progress_pct": round( 100.0 * (r["targets_done"] or 0) / max(r["targets_total"] or 1, 1), 1 diff --git a/backend/app/services/scrapers/nspd_lite.py b/backend/app/services/scrapers/nspd_lite.py index 07ca9f3f..bd47e76b 100644 --- a/backend/app/services/scrapers/nspd_lite.py +++ b/backend/app/services/scrapers/nspd_lite.py @@ -145,16 +145,21 @@ def fetch_via_rosreestr2coord( cad_num: str, area_type: int = 1, timeout: int = 15, - delay: float = 1.0, + delay: float = 0.0, # legacy-параметр, оставлен для совместимости signature ) -> dict[str, Any] | None: - """Тот же запрос но через community-библиотеку rosreestr2coord. + """Тот же запрос но через community-библиотеку rosreestr2coord (v5+). Преимущество: они поддерживают upstream обновления headers/WAF-tricks. Использовать когда наш fetch_* поломается из-за изменений WAF. + Параметр `delay` сохранён в signature для обратной совместимости с воркером, + но в rosreestr2coord v5 он убран — rate limiting между запросами теперь + делает наш воркер через `rate_ms` (см. nspd_geo.py). + Returns: GeoJSON Feature или None если participant не найден. """ + _ = delay # silence unused — см. docstring выше try: from rosreestr2coord.parser import Area except ImportError as e: @@ -164,7 +169,6 @@ def fetch_via_rosreestr2coord( code=cad_num, area_type=area_type, timeout=timeout, - delay=delay, with_log=False, ) try: diff --git a/backend/app/workers/tasks/nspd_geo.py b/backend/app/workers/tasks/nspd_geo.py index d69cff42..6220c3eb 100644 --- a/backend/app/workers/tasks/nspd_geo.py +++ b/backend/app/workers/tasks/nspd_geo.py @@ -29,7 +29,6 @@ from app.core.db import SessionLocal from app.services.scrapers.nspd_lite import ( NspdLiteError, NspdLiteWafError, - fetch_geoportal, fetch_via_rosreestr2coord, ) from app.workers.celery_app import celery_app @@ -228,7 +227,6 @@ def enqueue_geo_job( source_params: dict[str, Any], cad_nums_with_thematic: list[tuple[str, int]], rate_ms: int = 600, - use_rosreestr2coord: bool = False, triggered_by: str = "manual", ) -> int: """Создаёт job + targets и возвращает job_id. Не enqueue'ит task сам. @@ -243,8 +241,8 @@ def enqueue_geo_job( """ INSERT INTO nspd_geo_jobs ( name, job_kind, source_kind, source_params, - rate_ms, use_rosreestr2coord, triggered_by, status, targets_total - ) VALUES (:n, :k, :sk, CAST(:sp AS jsonb), :r, :ur, :tb, 'queued', :tt) + rate_ms, triggered_by, status, targets_total + ) VALUES (:n, :k, :sk, CAST(:sp AS jsonb), :r, :tb, 'queued', :tt) RETURNING job_id """ ), @@ -254,7 +252,6 @@ def enqueue_geo_job( "sk": source_kind, "sp": _json.dumps(source_params, ensure_ascii=False), "r": rate_ms, - "ur": use_rosreestr2coord, "tb": triggered_by, "tt": len(cad_nums_with_thematic), }, @@ -311,7 +308,6 @@ def process_nspd_geo_job(self: Any, job_id: int) -> dict[str, Any]: _start_job(db, job_id) rate_ms = int(job["rate_ms"] or 600) - use_lib = bool(job["use_rosreestr2coord"]) # Текущие счётчики (для resume — продолжаем с тех значений) done = int(job["targets_done"] or 0) @@ -321,9 +317,7 @@ def process_nspd_geo_job(self: Any, job_id: int) -> dict[str, Any]: n_waf = int(job["waf_blocked_count"] or 0) consecutive_waf = 0 - logger.info( - "process_nspd_geo_job start: job=%s use_lib=%s rate=%dms", job_id, use_lib, rate_ms - ) + logger.info("process_nspd_geo_job start: job=%s rate=%dms", job_id, rate_ms) while True: target = ( @@ -349,15 +343,19 @@ def process_nspd_geo_job(self: Any, job_id: int) -> dict[str, Any]: cad = target["cad_num"] try: - if use_lib: - payload = { - "data": { - "features": [fetch_via_rosreestr2coord(cad, area_type=tid, timeout=15)] - } + # NSPD geo fetch теперь только через rosreestr2coord lib (v5+). + # Старая ветка с urllib-fetch (fetch_geoportal) удалена — community + # пакет нам надёжнее, авторы обновляют под WAF. + payload = { + "data": { + "features": [fetch_via_rosreestr2coord(cad, area_type=tid, timeout=15)] } - payload["data"]["features"] = [f for f in payload["data"]["features"] if f] - else: - payload = fetch_geoportal(cad, thematic_id=tid, rate_ms=rate_ms) + } + payload["data"]["features"] = [f for f in payload["data"]["features"] if f] + # rosreestr2coord v5 убрал internal delay-параметр → rate limit + # делаем здесь, чтобы не словить WAF за высокую частоту. + if rate_ms > 0: + time.sleep(rate_ms / 1000.0) n_requests += 1 consecutive_waf = 0 diff --git a/data/sql/77_nspd_geo_jobs.sql b/data/sql/77_nspd_geo_jobs.sql index 2ed738da..d6890b09 100644 --- a/data/sql/77_nspd_geo_jobs.sql +++ b/data/sql/77_nspd_geo_jobs.sql @@ -26,11 +26,9 @@ CREATE TABLE IF NOT EXISTS nspd_geo_jobs ( -- Источник целей source_kind TEXT NOT NULL, -- 'rosreestr_pending' | 'manual_list' | 'region_bbox' source_params jsonb, -- что-то вроде {"region_codes":[66,74,72,59], "thematic_id":2} - -- Параметры fetch + -- Параметры fetch (всегда через rosreestr2coord lib v5+; + -- legacy колонка `use_rosreestr2coord` удалена миграцией 78_). rate_ms INTEGER NOT NULL DEFAULT 600, - -- TRUE = community-lib rosreestr2coord (default с 2026-05-11), - -- FALSE = наш nspd_lite urllib-based (fallback). - use_rosreestr2coord BOOLEAN NOT NULL DEFAULT TRUE, -- Состояние status TEXT NOT NULL DEFAULT 'queued', -- queued | running | done | failed | paused started_at timestamptz, @@ -134,7 +132,7 @@ SELECT 'nspd_geo', job_id, started_at, finished_at, heartbeat_at, status, error, requests_count, targets_done, targets_failed, targets_skipped, targets_done, targets_total, jsonb_build_object('job_kind', job_kind, 'source_kind', source_kind, - 'rate_ms', rate_ms, 'use_rosreestr2coord', use_rosreestr2coord, + 'rate_ms', rate_ms, 'waf_blocked_count', waf_blocked_count, 'source_params', source_params) FROM nspd_geo_jobs; diff --git a/data/sql/78_drop_use_rosreestr2coord.sql b/data/sql/78_drop_use_rosreestr2coord.sql new file mode 100644 index 00000000..8e1385cb --- /dev/null +++ b/data/sql/78_drop_use_rosreestr2coord.sql @@ -0,0 +1,58 @@ +-- Drop legacy `use_rosreestr2coord` column from nspd_geo_jobs. +-- +-- Контекст: +-- В первой итерации NSPD geo bulk-fetcher был выбор между нашим urllib-fetcher +-- (fetch_geoportal) и community-библиотекой rosreestr2coord. На практике +-- библиотека намного устойчивее к WAF — её авторы регулярно обновляют +-- headers/tricks. Поэтому отказались от urllib-ветки и теперь ВСЕГДА используем +-- rosreestr2coord. Колонка больше не несёт смысла. +-- +-- Порядок: сначала пересоздаём v_scrape_runs_unified без зависимости от +-- колонки, потом DROP COLUMN. + +BEGIN; + +-- Чистим возможную мусорную view с опечаткой (создавалась предыдущей попыткой +-- миграции — без 's' в имени, ошибка). +DROP VIEW IF EXISTS v_scrape_run_unified; + +-- Главная unified-view (см. 77_nspd_geo_jobs.sql) пересоздаётся без +-- use_rosreestr2coord в JSONB-секции nspd_geo. +CREATE OR REPLACE VIEW v_scrape_runs_unified AS +SELECT 'kn' AS scraper_type, run_id, started_at, finished_at, heartbeat_at, + status, error, NULL::text AS triggered_by, + region_codes::text AS scope, requests_count, + objects_count AS items_ok, NULL::integer AS items_failed, + flats_count AS sub_items_ok, + progress_obj_index AS progress_idx, total_obj_count AS progress_total, + jsonb_build_object('developers', developer_ids, 'snapshot_date', snapshot_date, + 'resumed_from', resumed_from_run_id, 'params', params) AS extra + FROM kn_scrape_runs +UNION ALL +SELECT 'nspd', run_id, started_at, finished_at, heartbeat_at, status, error, + triggered_by, region_code::text, requests_count, + quarters_ok, quarters_failed, buildings_ok, + NULL, pending_count, + jsonb_build_object('waf_429_count', waf_429_count) + FROM nspd_scrape_runs +UNION ALL +SELECT 'objective', run_id, started_at, finished_at, heartbeat_at, status, error, + triggered_by, group_name, requests_count, + reports_ok, reports_failed, rows_lots, + NULL, NULL, + jsonb_build_object('rows_corpus_room', rows_corpus_room, 'rows_history', rows_history) + FROM objective_scrape_runs +UNION ALL +SELECT 'nspd_geo', job_id, started_at, finished_at, heartbeat_at, status, error, + triggered_by, COALESCE(name, job_kind), + requests_count, targets_done, targets_failed, targets_skipped, + targets_done, targets_total, + jsonb_build_object('job_kind', job_kind, 'source_kind', source_kind, + 'rate_ms', rate_ms, + 'waf_blocked_count', waf_blocked_count, + 'source_params', source_params) + FROM nspd_geo_jobs; + +ALTER TABLE nspd_geo_jobs DROP COLUMN IF EXISTS use_rosreestr2coord; + +COMMIT; diff --git a/debug.log b/debug.log new file mode 100644 index 00000000..e69de29b diff --git a/frontend/src/app/admin/scrape/geo/page.tsx b/frontend/src/app/admin/scrape/geo/page.tsx index 36da53a8..69f7636c 100644 --- a/frontend/src/app/admin/scrape/geo/page.tsx +++ b/frontend/src/app/admin/scrape/geo/page.tsx @@ -24,7 +24,6 @@ interface GeoJob { waf_blocked_count: number; error: string | null; rate_ms: number; - use_rosreestr2coord: boolean; created_at: string | null; progress_pct: number; } @@ -71,10 +70,18 @@ export default function GeoScrapeAdminPage() { "quarters", ); const [thematicId, setThematicId] = useState(2); + + // Авто-синк thematic_id при смене job_kind. Пользователь может override через + // селект ниже (например quarters + thematic_id=15 для комплексов). + const handleJobKindChange = (kind: "quarters" | "parcels" | "buildings") => { + setJobKind(kind); + if (kind === "quarters") setThematicId(2); + else if (kind === "parcels") setThematicId(1); + else if (kind === "buildings") setThematicId(5); + }; const [cadNumsText, setCadNumsText] = useState(""); const [regionCodes, setRegionCodes] = useState("66,74,72,59"); const [rateMs, setRateMs] = useState(600); - const [useLib, setUseLib] = useState(true); const [lastResp, setLastResp] = useState(null); const queryClient = useQueryClient(); @@ -120,7 +127,6 @@ export default function GeoScrapeAdminPage() { thematic_id: thematicId, region_codes, rate_ms: rateMs, - use_rosreestr2coord: useLib, }), }); }, @@ -220,27 +226,33 @@ export default function GeoScrapeAdminPage() { @@ -292,16 +304,6 @@ export default function GeoScrapeAdminPage() { style={inputStyle} /> -
@@ -520,9 +522,9 @@ export default function GeoScrapeAdminPage() { diff --git a/frontend/src/components/admin/ScrapeLogsPanel.tsx b/frontend/src/components/admin/ScrapeLogsPanel.tsx index 3e610e38..73d3ad38 100644 --- a/frontend/src/components/admin/ScrapeLogsPanel.tsx +++ b/frontend/src/components/admin/ScrapeLogsPanel.tsx @@ -19,8 +19,8 @@ interface LogRow { interface Props { /** Admin token (для X-Admin-Token header). */ token: string; - /** scraper_type filter — `kn`, `nspd`, `objective`. Передаётся в `/admin/scrape/all/logs`. */ - scraperType: "kn" | "nspd" | "objective"; + /** scraper_type filter — kn / nspd / nspd_geo / objective. */ + scraperType: "kn" | "nspd" | "nspd_geo" | "objective"; /** Имя поля по которому фильтруем run в запросе (по умолчанию `run_id`). */ runIdParam?: string; /** Лимит записей (default 200). */