From b233bf91cc1f05336725f29435c357fcabd4c19b Mon Sep 17 00:00:00 2001 From: lekss361 Date: Sat, 23 May 2026 10:41:22 +0000 Subject: [PATCH] =?UTF-8?q?refactor(security):=20=D1=83=D0=B1=D1=80=D0=B0?= =?UTF-8?q?=D1=82=D1=8C=20X-Admin-Token=20(Caddy=20basic=5Fauth=20=D0=B4?= =?UTF-8?q?=D0=BE=D1=81=D1=82=D0=B0=D1=82=D0=BE=D1=87=D0=B5=D0=BD)=20(#437?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/.env.example | 5 ++- backend/app/api/v1/admin_cadastre.py | 9 ++--- backend/app/api/v1/admin_etl.py | 6 +-- backend/app/api/v1/admin_jobs.py | 7 +--- backend/app/api/v1/admin_leads.py | 10 +---- backend/app/api/v1/admin_scrape.py | 41 +++------------------ backend/app/api/v1/admin_weight_profiles.py | 11 ++---- 7 files changed, 22 insertions(+), 67 deletions(-) diff --git a/backend/.env.example b/backend/.env.example index 32bd1d6e..d17d9dd7 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -27,6 +27,7 @@ SCRAPE_KN_JITTER_SECONDS=1800 SCRAPE_KN_DEFAULT_REGIONS=66 # Путь к Playwright storage_state.json (commited в git, обновляется --save-state). SCRAPE_KN_STATE_PATH=data/playwright_state.json -# Токен для POST /api/v1/admin/scrape/kn (заголовок X-Admin-Token). -# Пустая строка = endpoint disabled (503). Сгенерировать: openssl rand -hex 32 +# DEPRECATED 2026-05-23: app-level admin auth removed (PR #436, Caddy basic_auth достаточен). +# Reinstate: revert changes in admin_*.py чтобы вернуть AdminTokenAuth dep. +# Переменная сохранена в core/deps.py для быстрого rollback. SCRAPE_ADMIN_TOKEN= diff --git a/backend/app/api/v1/admin_cadastre.py b/backend/app/api/v1/admin_cadastre.py index ff85f0d0..b24e794a 100644 --- a/backend/app/api/v1/admin_cadastre.py +++ b/backend/app/api/v1/admin_cadastre.py @@ -11,6 +11,9 @@ Scope variants: pilot — первые 50 кварталов ЕКБ (66:41:%) ekb_full — все ~2408 кварталов ЕКБ manual_list — явный список quarters в body.quarters + +Auth: gendsgn.ru-wide Caddy basic_auth gate (PR #426). App-level X-Admin-Token +header removed 2026-05-23 — двойная auth избыточна для pilot. """ from __future__ import annotations @@ -25,7 +28,6 @@ from sqlalchemy import text from sqlalchemy.orm import Session from app.core.db import get_db -from app.core.deps import AdminTokenAuth logger = logging.getLogger(__name__) @@ -185,7 +187,6 @@ def _serialize_job(row: Any) -> dict[str, Any]: def create_cadastre_job( body: CreateCadastreJobRequest, db: Annotated[Session, Depends(get_db)], - _: AdminTokenAuth, ) -> dict[str, Any]: """Создать bulk cadastre harvest job и поставить в очередь. @@ -234,7 +235,6 @@ def create_cadastre_job( @router.get("/jobs") def list_cadastre_jobs( db: Annotated[Session, Depends(get_db)], - _: AdminTokenAuth, limit: int = 30, ) -> list[dict[str, Any]]: """Список последних cadastre harvest jobs.""" @@ -262,7 +262,6 @@ def list_cadastre_jobs( def get_cadastre_job( job_id: int, db: Annotated[Session, Depends(get_db)], - _: AdminTokenAuth, ) -> dict[str, Any]: """Детали одного cadastre harvest job.""" row = ( @@ -297,7 +296,6 @@ def get_cadastre_job( def cancel_cadastre_job( job_id: int, db: Annotated[Session, Depends(get_db)], - _: AdminTokenAuth, ) -> dict[str, Any]: """Отменить job. Воркеры увидят status='cancelled' при следующей итерации и skip.""" result = db.execute( @@ -327,7 +325,6 @@ def cancel_cadastre_job( def resume_cadastre_job( job_id: int, db: Annotated[Session, Depends(get_db)], - _: AdminTokenAuth, ) -> dict[str, Any]: """Возобновить paused/failed job. Re-enqueue enqueue_cadastre_harvest.""" from app.workers.tasks.scrape_cadastre import enqueue_cadastre_harvest diff --git a/backend/app/api/v1/admin_etl.py b/backend/app/api/v1/admin_etl.py index 4490c989..97d38942 100644 --- a/backend/app/api/v1/admin_etl.py +++ b/backend/app/api/v1/admin_etl.py @@ -9,7 +9,8 @@ POST /api/v1/admin/etl/objective-backfill POST /api/v1/admin/etl/nspd-denorm-backfill Запустить backfill nspd_parcels/nspd_buildings из всех nspd_quarter_dumps. -Header: X-Admin-Token: +Auth: gendsgn.ru-wide Caddy basic_auth gate (PR #426). App-level X-Admin-Token +header removed 2026-05-23 — двойная auth избыточна для pilot. """ from __future__ import annotations @@ -21,7 +22,6 @@ from fastapi import APIRouter, Depends, Query from sqlalchemy.orm import Session from app.core.db import get_db -from app.core.deps import AdminTokenAuth from app.services.etl.objective_backfill import ( AUTO_ACCEPT_THRESHOLD, AUTO_ACCEPT_THRESHOLD_V2, @@ -38,7 +38,6 @@ router = APIRouter() @router.post("/objective-backfill") def run_objective_backfill( db: Annotated[Session, Depends(get_db)], - _: AdminTokenAuth, dry_run: Annotated[bool, Query(description="Preview без insertions")] = False, refresh_mv: Annotated[ bool, Query(description="REFRESH mv_layout_velocity после backfill") @@ -107,7 +106,6 @@ def run_objective_backfill( @router.post("/nspd-denorm-backfill") def run_nspd_denorm_backfill( - _: AdminTokenAuth, limit: Annotated[ int | None, Query(description="Максимум кварталов для обработки (None = все)"), diff --git a/backend/app/api/v1/admin_jobs.py b/backend/app/api/v1/admin_jobs.py index 05ed03cc..4fcd7eac 100644 --- a/backend/app/api/v1/admin_jobs.py +++ b/backend/app/api/v1/admin_jobs.py @@ -4,7 +4,8 @@ GET /api/v1/admin/jobs/settings — список всех job_settings GET /api/v1/admin/jobs/settings/{type} — одна строка по job_type PUT /api/v1/admin/jobs/settings/{type} — PATCH-style update -Auth: X-Admin-Token header (тот же токен что у admin_scrape endpoints). +Auth: gendsgn.ru-wide Caddy basic_auth gate (PR #426). App-level X-Admin-Token +header removed 2026-05-23 — двойная auth избыточна для pilot. """ from __future__ import annotations @@ -15,7 +16,6 @@ from fastapi import APIRouter, Depends from sqlalchemy.orm import Session from app.core.db import get_db -from app.core.deps import AdminTokenAuth from app.schemas.job_settings import JobSettingRead, JobSettingUpdate router = APIRouter() @@ -24,7 +24,6 @@ router = APIRouter() @router.get("/settings", response_model=list[JobSettingRead]) def list_job_settings( db: Annotated[Session, Depends(get_db)], - _: AdminTokenAuth, ) -> Any: """Список всех job_settings (все job_type).""" from app.services.job_settings import get_all @@ -36,7 +35,6 @@ def list_job_settings( def get_job_setting( job_type: str, db: Annotated[Session, Depends(get_db)], - _: AdminTokenAuth, ) -> Any: """Настройки одного job_type. Возвращает fallback defaults если строки нет в БД.""" from app.services.job_settings import get_one @@ -49,7 +47,6 @@ def update_job_setting( job_type: str, payload: JobSettingUpdate, db: Annotated[Session, Depends(get_db)], - _: AdminTokenAuth, ) -> Any: """PATCH-style update: передавать только изменяемые поля. diff --git a/backend/app/api/v1/admin_leads.py b/backend/app/api/v1/admin_leads.py index 45d899e4..71e9b54c 100644 --- a/backend/app/api/v1/admin_leads.py +++ b/backend/app/api/v1/admin_leads.py @@ -3,7 +3,8 @@ GET /api/v1/admin/leads — list with filters + pagination GET /api/v1/admin/leads/stats — KPI summary (total/conversion/revenue) -Auth: same X-Admin-Token as /admin/scrape (settings.scrape_admin_token). +Auth: gendsgn.ru-wide Caddy basic_auth gate (PR #426). App-level X-Admin-Token +header removed 2026-05-23 — двойная auth избыточна для pilot. PII compliance: only phone_last4 and email_hash exposed; raw phone/email were never persisted (см. data/sql/52_import_prinzip_crm.py). """ @@ -17,7 +18,6 @@ from sqlalchemy import text from sqlalchemy.orm import Session from app.core.db import get_db -from app.core.deps import AdminTokenAuth from app.services import analytics_queries as q router = APIRouter() @@ -26,7 +26,6 @@ router = APIRouter() @router.get("/") def list_leads( db: Annotated[Session, Depends(get_db)], - _: AdminTokenAuth, status: Annotated[str | None, Query()] = None, source: Annotated[str | None, Query()] = None, converted: Annotated[bool | None, Query()] = None, @@ -129,7 +128,6 @@ def list_leads( @router.get("/stats") def leads_stats( db: Annotated[Session, Depends(get_db)], - _: AdminTokenAuth, months: Annotated[int, Query(ge=1, le=120)] = 12, ) -> dict[str, Any]: """KPI summary за последние N месяцев.""" @@ -190,7 +188,6 @@ def leads_stats( @router.get("/funnel/monthly") def funnel_monthly( db: Annotated[Session, Depends(get_db)], - _: AdminTokenAuth, months: Annotated[int, Query(ge=1, le=120)] = 24, ) -> list[dict[str, Any]]: """Воронка по месяцам: leads → engaged → converted (по source).""" @@ -200,7 +197,6 @@ def funnel_monthly( @router.get("/funnel/by-source") def funnel_by_source( db: Annotated[Session, Depends(get_db)], - _: AdminTokenAuth, months: Annotated[int, Query(ge=1, le=120)] = 12, ) -> list[dict[str, Any]]: """Splitting по source: кто конвертит лучше за последние N месяцев.""" @@ -210,7 +206,6 @@ def funnel_by_source( @router.get("/funnel/by-object") def funnel_by_object( db: Annotated[Session, Depends(get_db)], - _: AdminTokenAuth, ) -> list[dict[str, Any]]: """Воронка для каждого ЖК PRINZIP: leads / deals / revenue.""" return q.prinzip_funnel_by_object(db) @@ -219,7 +214,6 @@ def funnel_by_object( @router.get("/sources") def list_sources( db: Annotated[Session, Depends(get_db)], - _: AdminTokenAuth, ) -> list[str]: """Distinct list of source values for filter dropdown.""" rows = db.execute( diff --git a/backend/app/api/v1/admin_scrape.py b/backend/app/api/v1/admin_scrape.py index c0f4685e..d5bc4a95 100644 --- a/backend/app/api/v1/admin_scrape.py +++ b/backend/app/api/v1/admin_scrape.py @@ -2,9 +2,10 @@ POST /api/v1/admin/scrape/kn Body: {"region_code": 66, "developers": ["6208_0"]?, "fetch_flats": true?} -Header: X-Admin-Token: -Disabled if SCRAPE_ADMIN_TOKEN is empty (default). +Auth: gendsgn.ru-wide Caddy basic_auth gate (PR #426). App-level X-Admin-Token +header removed 2026-05-23 — двойная auth избыточна для pilot. + Returns the Celery task id; poll /api/v1/admin/scrape/runs/{run_id} to track DB-side progress. """ @@ -19,7 +20,6 @@ from sqlalchemy.orm import Session from app.core.config import settings from app.core.db import get_db -from app.core.deps import AdminTokenAuth router = APIRouter() @@ -38,7 +38,6 @@ class TriggerKnRequest(BaseModel): @router.post("/kn") def trigger_kn_sweep( payload: TriggerKnRequest, - _: AdminTokenAuth, ) -> dict[str, Any]: # Defer Celery import so the API works without a broker in dev. from app.workers.tasks.scrape_kn import ( @@ -72,7 +71,6 @@ def trigger_kn_sweep( @router.get("/queue") def queue_status( db: Annotated[Session, Depends(get_db)], - _: AdminTokenAuth, ) -> dict[str, Any]: """Snapshot of Celery state, optimised for UI poll latency. @@ -217,7 +215,6 @@ class ReleaseLockRequest(BaseModel): @router.post("/release-lock") def release_lock( payload: ReleaseLockRequest, - _: AdminTokenAuth, ) -> dict[str, Any]: """Принудительно удалить Redis-lock для (region, devs). Использовать когда зомби-worker оставил lock и новые задачи скипаются с reason='lock_held'.""" @@ -239,7 +236,6 @@ class RevokeRequest(BaseModel): @router.post("/revoke") def revoke_task( payload: RevokeRequest, - _: AdminTokenAuth, ) -> dict[str, Any]: """Cancel a queued or running task by id.""" from app.workers.celery_app import celery_app @@ -251,7 +247,6 @@ def revoke_task( @router.get("/failures") def list_failures( db: Annotated[Session, Depends(get_db)], - _: AdminTokenAuth, run_id: int | None = None, limit: int = 50, ) -> list[dict[str, Any]]: @@ -297,7 +292,6 @@ def list_failures( @router.get("/logs") def list_logs( db: Annotated[Session, Depends(get_db)], - _: AdminTokenAuth, run_id: int | None = None, since_id: int | None = None, limit: int = 200, @@ -353,9 +347,7 @@ def list_logs( @router.post("/noise-sync") -def trigger_noise_sync( - _: AdminTokenAuth, -) -> dict[str, Any]: +def trigger_noise_sync() -> dict[str, Any]: """Manual trigger для синхронизации OSM шумовых источников ЕКБ из Overpass API. Обычно запускается еженедельно через beat (понед 3:30 МСК). @@ -385,7 +377,6 @@ class HarvestQuarterRequest(BaseModel): @router.post("/nspd/harvest-quarter") def trigger_harvest_quarter( payload: HarvestQuarterRequest, - _: AdminTokenAuth, ) -> dict[str, Any]: """Manual trigger одного quarter harvest (для testing / конкретного квартала). @@ -412,9 +403,7 @@ def trigger_harvest_quarter( @router.post("/pzz-sync") -def trigger_pzz_sync( - _: AdminTokenAuth, -) -> dict[str, Any]: +def trigger_pzz_sync() -> dict[str, Any]: """Manual trigger для импорта ПЗЗ территориальных зон ЕКБ из Росреестр PKK6. Запускать после деплоя миграции 85_pzz_zones_ekb.sql и при необходимости @@ -427,9 +416,7 @@ def trigger_pzz_sync( @router.post("/poi-sync") -def trigger_poi_sync( - _: AdminTokenAuth, -) -> dict[str, Any]: +def trigger_poi_sync() -> dict[str, Any]: """Manual trigger для синхронизации OSM POI ЕКБ из Overpass API. Обычно запускается еженедельно через beat (понед 3:00 МСК). @@ -452,7 +439,6 @@ class TriggerObjectiveEtlRequest(BaseModel): @router.post("/objective") def trigger_objective_etl( payload: TriggerObjectiveEtlRequest, - _: AdminTokenAuth, ) -> dict[str, Any]: """Запустить ETL Антоновского SQLite → наша PG. @@ -477,7 +463,6 @@ def trigger_objective_etl( @router.get("/objective/runs") def list_objective_runs( db: Annotated[Session, Depends(get_db)], - _: AdminTokenAuth, limit: int = 20, ) -> list[dict[str, Any]]: rows = ( @@ -535,7 +520,6 @@ class TriggerObjectiveSyncRequest(BaseModel): @router.post("/objective/sync-our") def trigger_objective_sync_our( payload: TriggerObjectiveSyncRequest, - _: AdminTokenAuth, ) -> dict[str, Any]: """Запустить НАШ sync_all_groups (тратит limits Объектива). @@ -584,7 +568,6 @@ class ObjectiveConfigUpdateRequest(BaseModel): @router.get("/objective/config") def get_objective_config( db: Annotated[Session, Depends(get_db)], - _: AdminTokenAuth, ) -> dict[str, Any]: """Текущая динамическая конфигурация Objective sync (single row).""" from app.services.objective_sync_config import get_config @@ -596,7 +579,6 @@ def get_objective_config( def update_objective_config( payload: ObjectiveConfigUpdateRequest, db: Annotated[Session, Depends(get_db)], - _: AdminTokenAuth, ) -> dict[str, Any]: """PATCH-style update. После изменения cron_schedule нужен restart beat, остальные поля применяются на следующем sync-вызове автоматически.""" @@ -626,7 +608,6 @@ def update_objective_config( @router.get("/objective/coverage") def objective_coverage( db: Annotated[Session, Depends(get_db)], - _: AdminTokenAuth, ) -> dict[str, Any]: """Что у нас в БД сейчас + что лежит в SQLite Антона (size, last modified). @@ -693,7 +674,6 @@ class EnqueueGeoJobRequest(BaseModel): def enqueue_geo_job( payload: EnqueueGeoJobRequest, db: Annotated[Session, Depends(get_db)], - _: AdminTokenAuth, ) -> dict[str, Any]: """Создать bulk geo-job и поставить в очередь. @@ -874,7 +854,6 @@ def _collect_all_in_region( def bulk_enqueue_geo( payload: BulkGeoEnqueueRequest, db: Annotated[Session, Depends(get_db)], - _: AdminTokenAuth, ) -> dict[str, Any]: """Разбить pending cad-номера на N чанков и запустить N×len(thematic_ids) geo-jobs. @@ -966,7 +945,6 @@ def bulk_enqueue_geo( @router.get("/geo/jobs") def list_geo_jobs( db: Annotated[Session, Depends(get_db)], - _: AdminTokenAuth, limit: int = 30, ) -> list[dict[str, Any]]: """Список последних geo-jobs (для UI dashboard).""" @@ -1021,7 +999,6 @@ def list_geo_jobs( def cancel_geo_job( job_id: int, db: Annotated[Session, Depends(get_db)], - _: AdminTokenAuth, ) -> dict[str, Any]: """Пометить job как cancelled. Worker увидит при следующей итерации.""" db.execute( @@ -1042,7 +1019,6 @@ def cancel_geo_job( def resume_geo_job( job_id: int, db: Annotated[Session, Depends(get_db)], - _: AdminTokenAuth, ) -> dict[str, Any]: """Re-enqueue paused/failed job. Resume idempotent через pending targets.""" from app.services.job_settings import get_setting_value @@ -1064,7 +1040,6 @@ def resume_geo_job( @router.get("/all/runs") def list_all_runs( db: Annotated[Session, Depends(get_db)], - _: AdminTokenAuth, scraper_type: str | None = None, limit: int = 30, ) -> list[dict[str, Any]]: @@ -1122,7 +1097,6 @@ def list_all_runs( @router.get("/all/logs") def list_all_logs( db: Annotated[Session, Depends(get_db)], - _: AdminTokenAuth, scraper_type: str | None = None, run_id: int | None = None, limit: int = 200, @@ -1174,7 +1148,6 @@ def list_all_logs( @router.get("/runs") def list_runs( db: Annotated[Session, Depends(get_db)], - _: AdminTokenAuth, limit: int = 20, ) -> list[dict[str, Any]]: rows = ( @@ -1229,7 +1202,6 @@ class TriggerEkburgPermitsRequest(BaseModel): @router.post("/ekburg-permits") def trigger_ekburg_permits( payload: TriggerEkburgPermitsRequest, - _: AdminTokenAuth, ) -> dict[str, Any]: """Manual trigger для синхронизации РНС/РВЭ ЕКБ из екатеринбург.рф (xlsx). @@ -1267,7 +1239,6 @@ class TriggerKnCatalogObjectsRequest(BaseModel): @router.post("/kn-catalog-objects") def trigger_kn_catalog_objects( payload: TriggerKnCatalogObjectsRequest, - _: AdminTokenAuth, ) -> dict[str, Any]: """Manual trigger для catalog-OBJECT scraper (заполняет wall_type, energy_eff, ceiling_height_m, parking_*, playground_*, scores из SSR __NEXT_DATA__). diff --git a/backend/app/api/v1/admin_weight_profiles.py b/backend/app/api/v1/admin_weight_profiles.py index fc187596..243a8823 100644 --- a/backend/app/api/v1/admin_weight_profiles.py +++ b/backend/app/api/v1/admin_weight_profiles.py @@ -1,6 +1,9 @@ """Admin endpoints для управления weight profiles. -Per #114 (Макс feedback: садики ≠ Мегамарт). Endpoints под X-Admin-Token. +Per #114 (Макс feedback: садики ≠ Мегамарт). + +Auth: gendsgn.ru-wide Caddy basic_auth gate (PR #426). App-level X-Admin-Token +header removed 2026-05-23 — двойная auth избыточна для pilot. GET /api/v1/admin/site-finder/weight-profiles?user_id=... → list POST /api/v1/admin/site-finder/weight-profiles → create @@ -17,7 +20,6 @@ from fastapi import APIRouter, Depends, HTTPException, Query, status from sqlalchemy.orm import Session from app.core.db import get_db -from app.core.deps import AdminTokenAuth from app.services.site_finder.weight_profiles import ( WeightProfile, WeightProfileCreate, @@ -37,7 +39,6 @@ router = APIRouter() def list_user_profiles( user_id: Annotated[str, Query(min_length=1, description="user_id для фильтра профилей")], db: Annotated[Session, Depends(get_db)], - _: AdminTokenAuth, include_system: Annotated[ bool, Query(description="Включить системные preset-профили (Эконом/Комфорт/Бизнес)"), @@ -57,7 +58,6 @@ def list_user_profiles( def create_user_profile( payload: WeightProfileCreate, db: Annotated[Session, Depends(get_db)], - _: AdminTokenAuth, ) -> Any: """Создать новый weight profile. @@ -72,7 +72,6 @@ def get_user_profile( profile_id: int, user_id: Annotated[str, Query(min_length=1, description="Владелец профиля")], db: Annotated[Session, Depends(get_db)], - _: AdminTokenAuth, ) -> Any: """Получить один weight profile по id (scoped к user_id).""" profile = get_profile(db, user_id, profile_id) @@ -87,7 +86,6 @@ def update_user_profile( user_id: Annotated[str, Query(min_length=1, description="Владелец профиля")], payload: WeightProfileUpdate, db: Annotated[Session, Depends(get_db)], - _: AdminTokenAuth, ) -> Any: """PATCH-style обновление weight profile. @@ -105,7 +103,6 @@ def delete_user_profile( profile_id: int, user_id: Annotated[str, Query(min_length=1, description="Владелец профиля")], db: Annotated[Session, Depends(get_db)], - _: AdminTokenAuth, ) -> None: """Удалить weight profile. 404 если не найден.""" deleted = delete_profile(db, user_id, profile_id)