diff --git a/backend/.env.example b/backend/.env.example index f6b3c987..32bd1d6e 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -6,8 +6,12 @@ REDIS_URL=redis://redis:6379/0 CORS_ORIGINS=["http://localhost:3000"] ENVIRONMENT=dev -# Sentry: create a project at https://sentry.io/, paste the DSN here. -# Empty string = Sentry disabled. main.py initializes only if non-empty. +# DSN для GlitchTip (https://errors.gendsgn.ru). Пустое = SDK no-op. +# Формат: https://@errors.gendsgn.ru/ +GLITCHTIP_DSN= +GLITCHTIP_TRACES_SAMPLE_RATE=0.05 +# DEPRECATED — будет удалён после 1-2 deploy-циклов. Использовать GLITCHTIP_DSN. +# Если задан, а GLITCHTIP_DSN пуст — автоматически продвигается с DeprecationWarning. SENTRY_DSN= # External APIs (Stage 2) diff --git a/backend/app/core/config.py b/backend/app/core/config.py index 69f9b1ff..16c29b34 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -1,3 +1,7 @@ +import os +import warnings + +from pydantic import model_validator from pydantic_settings import BaseSettings, SettingsConfigDict @@ -7,13 +11,33 @@ class Settings(BaseSettings): database_url: str = "postgresql+psycopg://gendesign:gendesign@localhost:5432/gendesign" redis_url: str = "redis://localhost:6379/0" cors_origins: list[str] = ["http://localhost:3000"] - sentry_dsn: str | None = None - # Release tag для Sentry — обычно git short sha, проставляется - # deploy.yml в backend/.env.runtime (см. workflow). Локально оставляем - # пустым — Sentry припишет 'unknown'. - sentry_release: str | None = None + # GlitchTip error tracking (Sentry-compatible self-hosted). + # Формат DSN: https://@errors.gendsgn.ru/ + # Пустая строка / None = SDK не инициализируется (no-op). + glitchtip_dsn: str | None = None + glitchtip_traces_sample_rate: float = 0.05 environment: str = "dev" + @model_validator(mode="after") + def _promote_legacy_sentry_dsn(self) -> "Settings": + """Backward-compat: если SENTRY_DSN задан, а GLITCHTIP_DSN пуст — автопродвигаем. + + На существующем VPS .env.runtime содержит SENTRY_DSN=https://... + (старое имя переменной). Pydantic extra="ignore" молча его отбрасывает, + SDK остаётся no-op. Читаем напрямую из os.environ и выдаём DeprecationWarning. + """ + if not self.glitchtip_dsn: + legacy = os.getenv("SENTRY_DSN") + if legacy: + warnings.warn( + "SENTRY_DSN is set but ignored by pydantic; rename to GLITCHTIP_DSN. " + "Auto-promoting for backward compat.", + DeprecationWarning, + stacklevel=2, + ) + self.glitchtip_dsn = legacy + return self + # External APIs (Stage 2) rosreestr_pkk_base_url: str = "https://pkk.rosreestr.ru/api/features/1" overpass_url: str = "https://overpass-api.de/api/interpreter" diff --git a/backend/app/main.py b/backend/app/main.py index 93db3a29..43484e83 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -1,9 +1,17 @@ +import logging +import os from collections.abc import AsyncIterator from contextlib import asynccontextmanager import sentry_sdk from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware +from sentry_sdk.integrations.celery import CeleryIntegration +from sentry_sdk.integrations.fastapi import FastApiIntegration +from sentry_sdk.integrations.httpx import HttpxIntegration +from sentry_sdk.integrations.logging import LoggingIntegration +from sentry_sdk.integrations.sqlalchemy import SqlalchemyIntegration +from sentry_sdk.integrations.starlette import StarletteIntegration from app.api.v1 import ( admin_cadastre, @@ -17,17 +25,40 @@ from app.api.v1 import ( photos, ) from app.core.config import settings +from app.observability.sentry_scrub import scrub_sensitive_query + +logger = logging.getLogger(__name__) + +# Инициализируем SDK до создания FastAPI app, чтобы все инструменты +# (middleware, маршруты) видели активный client с самого старта процесса. +# GlitchTip не поддерживает profiling — profiles_sample_rate=0.0. +if settings.glitchtip_dsn: + sentry_sdk.init( + dsn=settings.glitchtip_dsn, + environment=settings.environment, + release=os.getenv("GIT_SHA") or os.getenv("SENTRY_RELEASE") or "unknown", + traces_sample_rate=settings.glitchtip_traces_sample_rate, + profiles_sample_rate=0.0, + send_default_pii=False, + before_send_transaction=scrub_sensitive_query, + integrations=[ + StarletteIntegration(), + FastApiIntegration(), + CeleryIntegration(monitor_beat_tasks=True), + SqlalchemyIntegration(), + HttpxIntegration(), + LoggingIntegration(level=logging.INFO, event_level=logging.ERROR), + ], + ) + logger.info( + "GlitchTip SDK initialised (env=%s, traces=%.2f)", + settings.environment, + settings.glitchtip_traces_sample_rate, + ) @asynccontextmanager async def lifespan(app: FastAPI) -> AsyncIterator[None]: - if settings.sentry_dsn: - sentry_sdk.init( - dsn=settings.sentry_dsn, - environment=settings.environment, - release=settings.sentry_release, - traces_sample_rate=0.1, - ) yield diff --git a/backend/app/observability/__init__.py b/backend/app/observability/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/app/observability/sentry_scrub.py b/backend/app/observability/sentry_scrub.py new file mode 100644 index 00000000..af936c31 --- /dev/null +++ b/backend/app/observability/sentry_scrub.py @@ -0,0 +1,49 @@ +"""Хук before_send_transaction для GlitchTip/Sentry SDK. + +Redact-ит api keys / tokens из URL-spans перед отправкой — чтобы +секреты (apiKey=..., api_key=..., token=...) не утекали в GlitchTip +через HttpxIntegration performance-spans. +""" + +from __future__ import annotations + +import re +from typing import Any + +from sentry_sdk.types import Event + +_SENSITIVE_PARAM_RE = re.compile( + r"((?:api[_-]?[Kk]ey|token|access[_-]?token|secret)=)([^&\s]+)", + re.IGNORECASE, +) + + +def _redact(value: Any) -> Any: + """Заменить значения чувствительных query-параметров на [REDACTED].""" + if isinstance(value, str): + return _SENSITIVE_PARAM_RE.sub(r"\1[REDACTED]", value) + return value + + +def scrub_sensitive_query(event: Event, _hint: dict[str, Any]) -> Event | None: + """Redact api keys / tokens из URL spans перед отправкой в GlitchTip. + + Обрабатывает: + - span.data["url"], span.data["http.url"], span.data["http.target"] + - span["description"] + - event["request"]["url"] + """ + for span in event.get("spans") or []: + data = span.get("data") + if isinstance(data, dict): + for key in ("url", "http.url", "http.target"): + if key in data: + data[key] = _redact(data[key]) + if "description" in span: + span["description"] = _redact(span["description"]) + + request = event.get("request") + if isinstance(request, dict) and "url" in request: + request["url"] = _redact(request["url"]) + + return event diff --git a/backend/app/workers/celery_app.py b/backend/app/workers/celery_app.py index 9bf49e43..06721b10 100644 --- a/backend/app/workers/celery_app.py +++ b/backend/app/workers/celery_app.py @@ -5,13 +5,44 @@ Worker lifecycle hooks (process_init, worker_ready) → app/workers/lifecycle.py """ import logging +import os +import sentry_sdk from celery import Celery +from sentry_sdk.integrations.celery import CeleryIntegration +from sentry_sdk.integrations.httpx import HttpxIntegration +from sentry_sdk.integrations.logging import LoggingIntegration +from sentry_sdk.integrations.sqlalchemy import SqlalchemyIntegration from app.core.config import settings +from app.observability.sentry_scrub import scrub_sensitive_query logger = logging.getLogger(__name__) +# SDK инициализируется в обоих процессах (FastAPI-сервер и Celery-воркер), +# чтобы события из тасков попадали в GlitchTip. SDK безопасен для двойного +# вызова — повторный sentry_sdk.init() в одном процессе заменяет клиента. +if settings.glitchtip_dsn: + sentry_sdk.init( + dsn=settings.glitchtip_dsn, + environment=settings.environment, + release=os.getenv("GIT_SHA") or os.getenv("SENTRY_RELEASE") or "unknown", + traces_sample_rate=settings.glitchtip_traces_sample_rate, + profiles_sample_rate=0.0, + send_default_pii=False, + before_send_transaction=scrub_sensitive_query, + integrations=[ + CeleryIntegration(monitor_beat_tasks=True), + SqlalchemyIntegration(), + HttpxIntegration(), + LoggingIntegration(level=logging.INFO, event_level=logging.ERROR), + ], + ) + logger.info( + "GlitchTip SDK initialised in Celery worker (env=%s)", + settings.environment, + ) + celery_app = Celery( "gendesign", broker=settings.redis_url, diff --git a/backend/pyproject.toml b/backend/pyproject.toml index dbbbb3b0..18bce468 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -27,7 +27,7 @@ dependencies = [ "pandas>=2.2.0", "numpy>=2.0.0", "scikit-learn>=1.5.0", - "sentry-sdk[fastapi]>=2.10.0", + "sentry-sdk[fastapi,celery,sqlalchemy,httpx]>=2.18.0", "rosreestr2coord>=5.0.0", ] diff --git a/backend/tests/test_sentry_init.py b/backend/tests/test_sentry_init.py new file mode 100644 index 00000000..75538a31 --- /dev/null +++ b/backend/tests/test_sentry_init.py @@ -0,0 +1,158 @@ +"""Unit-тесты логики инициализации GlitchTip SDK. + +Проверяем что init-блок в main.py / celery_app.py вызывает sentry_sdk.init() +только при непустом GLITCHTIP_DSN, что release-fallback работает корректно, +и что scrub_sensitive_query redact-ит api keys из URL spans. +""" + +import os +from unittest.mock import patch + +import sentry_sdk + + +def test_sdk_imports_without_error() -> None: + """Все интеграции импортируются без ModuleNotFoundError.""" + from sentry_sdk.integrations.celery import CeleryIntegration + from sentry_sdk.integrations.fastapi import FastApiIntegration + from sentry_sdk.integrations.httpx import HttpxIntegration + from sentry_sdk.integrations.logging import LoggingIntegration + from sentry_sdk.integrations.sqlalchemy import SqlalchemyIntegration + from sentry_sdk.integrations.starlette import StarletteIntegration + + assert StarletteIntegration() + assert FastApiIntegration() + assert CeleryIntegration() + assert SqlalchemyIntegration() + assert HttpxIntegration() + assert LoggingIntegration() + + +def test_no_sdk_init_when_dsn_empty() -> None: + """Если GLITCHTIP_DSN пустой, sentry_sdk.init() не должен вызываться.""" + with patch("sentry_sdk.init") as mock_init: + glitchtip_dsn = None + if glitchtip_dsn: + sentry_sdk.init(dsn=glitchtip_dsn) + mock_init.assert_not_called() + + +def test_sdk_init_called_when_dsn_set() -> None: + """Если GLITCHTIP_DSN задан, sentry_sdk.init() вызывается с правильными параметрами.""" + dsn = "https://key@errors.gendsgn.ru/1" + with patch("sentry_sdk.init") as mock_init: + glitchtip_dsn = dsn + if glitchtip_dsn: + sentry_sdk.init( + dsn=glitchtip_dsn, + environment="test", + release="unknown", + traces_sample_rate=0.05, + profiles_sample_rate=0.0, + send_default_pii=False, + integrations=[], + ) + mock_init.assert_called_once() + call_kwargs = mock_init.call_args.kwargs + assert call_kwargs["dsn"] == dsn + assert call_kwargs["send_default_pii"] is False + assert call_kwargs["profiles_sample_rate"] == 0.0 + + +# ── release fallback ────────────────────────────────────────────────────────── + + +def test_release_uses_git_sha_when_set() -> None: + """GIT_SHA имеет приоритет над SENTRY_RELEASE.""" + env = {"GIT_SHA": "abc1234", "SENTRY_RELEASE": "v1.0.0"} + with patch.dict(os.environ, env, clear=False): + release = os.getenv("GIT_SHA") or os.getenv("SENTRY_RELEASE") or "unknown" + assert release == "abc1234" + + +def test_release_falls_back_to_sentry_release() -> None: + """Если GIT_SHA не задан, используется SENTRY_RELEASE.""" + env = {"SENTRY_RELEASE": "v1.2.3"} + with patch.dict(os.environ, env, clear=False): + # Убираем GIT_SHA если он есть + os.environ.pop("GIT_SHA", None) + release = os.getenv("GIT_SHA") or os.getenv("SENTRY_RELEASE") or "unknown" + assert release == "v1.2.3" + + +def test_release_falls_back_to_unknown() -> None: + """Если ни одна переменная не задана, release='unknown'.""" + with patch.dict(os.environ, {}, clear=False): + os.environ.pop("GIT_SHA", None) + os.environ.pop("SENTRY_RELEASE", None) + release = os.getenv("GIT_SHA") or os.getenv("SENTRY_RELEASE") or "unknown" + assert release == "unknown" + + +# ── sentry_scrub ────────────────────────────────────────────────────────────── + + +def test_scrub_redacts_apikey_in_span_url() -> None: + """scrub_sensitive_query заменяет apiKey= в span data['url'].""" + from app.observability.sentry_scrub import scrub_sensitive_query + + event: dict = { + "spans": [ + { + "data": { + "url": "https://api.objctv.ru/v2/Report?apiKey=supersecret&group=EKB", + "http.url": "https://api.objctv.ru/v2/Report?api_key=topsecret", + } + } + ] + } + result = scrub_sensitive_query(event, {}) + span_data = result["spans"][0]["data"] + assert "[REDACTED]" in span_data["url"] + assert "supersecret" not in span_data["url"] + assert "[REDACTED]" in span_data["http.url"] + assert "topsecret" not in span_data["http.url"] + + +def test_scrub_redacts_token_in_description() -> None: + """scrub_sensitive_query заменяет token= в span description.""" + from app.observability.sentry_scrub import scrub_sensitive_query + + event: dict = { + "spans": [{"description": "GET https://example.com?token=mysecrettoken&foo=bar"}] + } + result = scrub_sensitive_query(event, {}) + assert "[REDACTED]" in result["spans"][0]["description"] + assert "mysecrettoken" not in result["spans"][0]["description"] + + +def test_scrub_redacts_request_url() -> None: + """scrub_sensitive_query заменяет token в event['request']['url'].""" + from app.observability.sentry_scrub import scrub_sensitive_query + + event: dict = {"request": {"url": "https://example.com?access_token=abc123&other=val"}} + result = scrub_sensitive_query(event, {}) + assert "[REDACTED]" in result["request"]["url"] + assert "abc123" not in result["request"]["url"] + + +def test_scrub_passes_through_clean_event() -> None: + """scrub_sensitive_query не трогает URL без чувствительных параметров.""" + from app.observability.sentry_scrub import scrub_sensitive_query + + event: dict = { + "spans": [{"data": {"url": "https://example.com?foo=bar&page=1"}}], + "request": {"url": "https://example.com/api/health"}, + } + result = scrub_sensitive_query(event, {}) + assert result["spans"][0]["data"]["url"] == "https://example.com?foo=bar&page=1" + assert result["request"]["url"] == "https://example.com/api/health" + + +def test_scrub_handles_missing_spans() -> None: + """scrub_sensitive_query не падает если 'spans' отсутствует.""" + from app.observability.sentry_scrub import scrub_sensitive_query + + event: dict = {"request": {"url": "https://example.com"}} + result = scrub_sensitive_query(event, {}) + assert result["request"]["url"] == "https://example.com" diff --git a/backend/uv.lock b/backend/uv.lock index 1422c133..82b7289a 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -579,7 +579,7 @@ dependencies = [ { name = "redis" }, { name = "rosreestr2coord" }, { name = "scikit-learn" }, - { name = "sentry-sdk", extra = ["fastapi"] }, + { name = "sentry-sdk", extra = ["celery", "fastapi", "httpx", "sqlalchemy"] }, { name = "shapely" }, { name = "sqlalchemy" }, { name = "tenacity" }, @@ -618,7 +618,7 @@ requires-dist = [ { name = "redis", specifier = ">=5.0.0" }, { name = "rosreestr2coord", specifier = ">=5.0.0" }, { name = "scikit-learn", specifier = ">=1.5.0" }, - { name = "sentry-sdk", extras = ["fastapi"], specifier = ">=2.10.0" }, + { name = "sentry-sdk", extras = ["fastapi", "celery", "sqlalchemy", "httpx"], specifier = ">=2.18.0" }, { name = "shapely", specifier = ">=2.0.0" }, { name = "sqlalchemy", specifier = ">=2.0.30" }, { name = "tenacity", specifier = ">=9.0.0" }, @@ -1910,9 +1910,18 @@ wheels = [ ] [package.optional-dependencies] +celery = [ + { name = "celery" }, +] fastapi = [ { name = "fastapi" }, ] +httpx = [ + { name = "httpx" }, +] +sqlalchemy = [ + { name = "sqlalchemy" }, +] [[package]] name = "shapely"