Light1YT
dd773d6bde
All checks were successful
CI / changes (push) Successful in 8s
CI / frontend-tests (push) Has been skipped
CI / changes (pull_request) Successful in 9s
CI / frontend-tests (pull_request) Has been skipped
CI / backend-tests (push) Successful in 6m42s
CI / backend-tests (pull_request) Successful in 6m36s
В complete-loop'е min(...,30) применялся ТОЛЬКО к exponential backoff'у
(else-ветка). Серверный Retry-After уходил в time.sleep как есть.
_parse_retry_after принимает любое число секунд ("86400".isdigit() → True),
а provider'ы шлют Retry-After до 86400с при quota-exhaustion (OpenAI,
CDN-503). Эффект: time.sleep блокирует поток anyio-threadpool на часы
(до 48ч при llm_max_retries=2). Async-консьюмер (chat.py) мостится через
run_in_threadpool — поток держит токен пула (~40 потоков, общий с sync
Depends(get_db)) и DB-сессию → пул исчерпывается → стопор приложения.
Patch: вынес кап на module-level _MAX_BACKOFF_S=30s, применяю к ОБЕИМ
веткам (Retry-After И exp.backoff). raw_wait логируется отдельно для
наблюдаемости (видно когда провайдер просил больше).
Новый юнит-тест test_rate_limited_retry_after_capped: provider шлёт
retry_after=86400 → time.sleep вызывается с 30 (не 86400). 14/14
client тестов + 55/55 LLM-suite зелёные. ruff clean.
Closes #1209
319 lines
13 KiB
Python
319 lines
13 KiB
Python
"""LLM client orchestration tests (#960).
|
||
|
||
Покрываем: disabled→fallback (БЕЗ сетевой попытки), no-key→fallback, call-cap,
|
||
redaction hard-block→fallback, happy-path (тело запроса проскраблено),
|
||
guardrails (timeout→fallback, 429-retry-then-fallback), complete_or_raise.
|
||
|
||
Сеть НЕ дёргается: либо внедряем fake-провайдера, либо ставим tripwire на httpx.Client.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
from typing import Any
|
||
|
||
import pytest
|
||
|
||
from app.core.config import settings
|
||
from app.services.llm import client as llm_client
|
||
from app.services.llm.client import LLMResult, LLMUnavailableError, complete, complete_or_raise
|
||
from app.services.llm.provider import (
|
||
LLMProvider,
|
||
LLMRateLimitedError,
|
||
LLMTimeoutError,
|
||
ProviderResponse,
|
||
ToolCall,
|
||
)
|
||
from app.services.llm.redaction import SafePayload
|
||
|
||
# ── Fake провайдеры (внедряются через provider=...) ────────────────────────────
|
||
|
||
|
||
class _FakeOpenAILike(LLMProvider):
|
||
"""Внешний (is_external=True) fake: записывает messages, отдаёт фикс. ответ."""
|
||
|
||
def __init__(self, response: ProviderResponse | None = None) -> None:
|
||
self.captured_messages: list[dict[str, Any]] | None = None
|
||
self.captured_tools: list[dict[str, Any]] | None = None
|
||
self.calls = 0
|
||
self._response = response or ProviderResponse(
|
||
content="ответ модели", prompt_tokens=10, completion_tokens=5, model="gpt-4o-mini"
|
||
)
|
||
|
||
@property
|
||
def is_external(self) -> bool:
|
||
return True
|
||
|
||
@property
|
||
def model(self) -> str:
|
||
return "gpt-4o-mini"
|
||
|
||
def complete(
|
||
self,
|
||
messages: list[dict[str, Any]],
|
||
*,
|
||
tools: list[dict[str, Any]] | None = None,
|
||
max_output_tokens: int,
|
||
) -> ProviderResponse:
|
||
self.calls += 1
|
||
self.captured_messages = messages
|
||
self.captured_tools = tools
|
||
return self._response
|
||
|
||
|
||
class _RaisingProvider(LLMProvider):
|
||
"""Внешний fake, всегда бросающий заданное исключение (для guardrail-тестов)."""
|
||
|
||
def __init__(self, exc: Exception) -> None:
|
||
self._exc = exc
|
||
self.calls = 0
|
||
|
||
@property
|
||
def is_external(self) -> bool:
|
||
return True
|
||
|
||
@property
|
||
def model(self) -> str:
|
||
return "gpt-4o-mini"
|
||
|
||
def complete(self, messages: Any, *, tools: Any = None, max_output_tokens: int) -> Any:
|
||
self.calls += 1
|
||
raise self._exc
|
||
|
||
|
||
@pytest.fixture
|
||
def _enabled(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
"""Включить LLM с fake-ключом (без реальных секретов)."""
|
||
monkeypatch.setattr(settings, "llm_enabled", True)
|
||
monkeypatch.setattr(settings, "openai_api_key", "test-fake-key-not-real")
|
||
|
||
|
||
# ── Guard #1/#2: disabled / no key ─────────────────────────────────────────────
|
||
|
||
|
||
def test_disabled_returns_fallback_without_network(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
"""llm_enabled=False → fallback, и НИ ОДНОЙ попытки создать httpx.Client."""
|
||
monkeypatch.setattr(settings, "llm_enabled", False)
|
||
|
||
def _tripwire(*a: Any, **k: Any) -> None:
|
||
raise AssertionError("network must NOT be touched when llm_enabled=False")
|
||
|
||
# Любая попытка сетевого вызова через httpx завалит тест.
|
||
monkeypatch.setattr("app.services.llm.provider.httpx.Client", _tripwire)
|
||
|
||
res = complete(system_prompt="sys", payload=SafePayload(text="hi"))
|
||
assert res.ok is False
|
||
assert res.fallback_used is True
|
||
assert res.reason == "disabled"
|
||
|
||
|
||
def test_enabled_but_no_key_returns_fallback(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
"""llm_enabled=True но ключ None → fallback (no_api_key), без сети."""
|
||
monkeypatch.setattr(settings, "llm_enabled", True)
|
||
monkeypatch.setattr(settings, "openai_api_key", None)
|
||
|
||
def _tripwire(*a: Any, **k: Any) -> None:
|
||
raise AssertionError("no network without a key")
|
||
|
||
monkeypatch.setattr("app.services.llm.provider.httpx.Client", _tripwire)
|
||
res = complete(system_prompt="sys", payload=SafePayload(text="hi"))
|
||
assert res.reason == "no_api_key"
|
||
assert res.fallback_used is True
|
||
|
||
|
||
# ── Call cap ───────────────────────────────────────────────────────────────────
|
||
|
||
|
||
def test_call_cap_returns_fallback(_enabled: None, monkeypatch: pytest.MonkeyPatch) -> None:
|
||
"""call_index >= llm_max_calls_per_request → fallback, провайдер не вызывается."""
|
||
monkeypatch.setattr(settings, "llm_max_calls_per_request", 2)
|
||
prov = _FakeOpenAILike()
|
||
res = complete(
|
||
system_prompt="sys", payload=SafePayload(text="hi"), provider=prov, call_index=2
|
||
)
|
||
assert res.reason == "call_cap"
|
||
assert prov.calls == 0
|
||
|
||
|
||
# ── Redaction integration ──────────────────────────────────────────────────────
|
||
|
||
|
||
def test_confidential_payload_returns_fallback(_enabled: None) -> None:
|
||
"""is_confidential=True (external) → fallback redaction_refused, провайдер не зван."""
|
||
prov = _FakeOpenAILike()
|
||
res = complete(
|
||
system_prompt="sys",
|
||
payload=SafePayload(text="секрет", is_confidential=True),
|
||
provider=prov,
|
||
)
|
||
assert res.reason == "redaction_refused"
|
||
assert prov.calls == 0
|
||
|
||
|
||
def test_request_body_is_redacted(_enabled: None) -> None:
|
||
"""ГЛАВНОЕ: тело запроса к провайдеру НЕ содержит сырого PII — оно проскраблено."""
|
||
prov = _FakeOpenAILike()
|
||
payload = SafePayload(
|
||
text="Свяжитесь +7 912 345 67 89 или mail@example.ru",
|
||
fields={"owner": "Иванов Иван Иванович", "zone": "Ц-1"},
|
||
)
|
||
res = complete(system_prompt="Ты ассистент", payload=payload, provider=prov)
|
||
|
||
assert res.ok is True
|
||
assert prov.captured_messages is not None
|
||
blob = str(prov.captured_messages)
|
||
# Сырого PII в отправленных messages быть не должно.
|
||
assert "+7 912 345 67 89" not in blob
|
||
assert "mail@example.ru" not in blob
|
||
assert "Иванов Иван Иванович" not in blob
|
||
# Зато плейсхолдеры и безопасные поля — на месте.
|
||
assert "[REDACTED:phone]" in blob
|
||
assert "[REDACTED:email]" in blob
|
||
assert "[REDACTED:name]" in blob
|
||
assert "Ц-1" in blob
|
||
|
||
|
||
# ── Happy path ─────────────────────────────────────────────────────────────────
|
||
|
||
|
||
def test_happy_path_returns_ok_result(_enabled: None) -> None:
|
||
"""Успешный ответ → ok=True, content и токены проброшены."""
|
||
prov = _FakeOpenAILike(
|
||
ProviderResponse(
|
||
content="готово", prompt_tokens=12, completion_tokens=4, model="gpt-4o-mini"
|
||
)
|
||
)
|
||
res = complete(system_prompt="sys", payload=SafePayload(text="вопрос"), provider=prov)
|
||
assert res.ok is True
|
||
assert res.content == "готово"
|
||
assert res.prompt_tokens == 12
|
||
assert res.completion_tokens == 4
|
||
assert res.model == "gpt-4o-mini"
|
||
|
||
|
||
def test_tool_calls_passed_through(_enabled: None) -> None:
|
||
"""tool_calls из ответа провайдера прокидываются в LLMResult."""
|
||
prov = _FakeOpenAILike(
|
||
ProviderResponse(
|
||
content=None,
|
||
tool_calls=[ToolCall(id="c1", name="extract", arguments='{"x":1}')],
|
||
model="gpt-4o-mini",
|
||
)
|
||
)
|
||
res = complete(system_prompt="sys", payload=SafePayload(text="q"), provider=prov)
|
||
assert res.ok is True
|
||
assert len(res.tool_calls) == 1
|
||
assert res.tool_calls[0].name == "extract"
|
||
|
||
|
||
def test_tools_forwarded_to_provider(_enabled: None) -> None:
|
||
"""tools прокидываются в provider.complete."""
|
||
prov = _FakeOpenAILike()
|
||
tools = [{"type": "function", "function": {"name": "f"}}]
|
||
complete(system_prompt="sys", payload=SafePayload(text="q"), provider=prov, tools=tools)
|
||
assert prov.captured_tools == tools
|
||
|
||
|
||
# ── Guardrails: timeout / rate-limit / retries ─────────────────────────────────
|
||
|
||
|
||
def test_timeout_degrades_to_fallback(_enabled: None) -> None:
|
||
"""LLMTimeoutError → fallback(timeout), без ретраев (1 вызов)."""
|
||
prov = _RaisingProvider(LLMTimeoutError("slow"))
|
||
res = complete(system_prompt="sys", payload=SafePayload(text="q"), provider=prov)
|
||
assert res.reason == "timeout"
|
||
assert res.fallback_used is True
|
||
assert prov.calls == 1
|
||
|
||
|
||
def test_rate_limited_retries_then_fallback(
|
||
_enabled: None, monkeypatch: pytest.MonkeyPatch
|
||
) -> None:
|
||
"""429 на всех попытках → ретраи (cap) затем fallback(rate_limited). sleep замокан."""
|
||
monkeypatch.setattr(settings, "llm_max_retries", 2)
|
||
# Не спим в тесте.
|
||
monkeypatch.setattr(llm_client.time, "sleep", lambda s: None)
|
||
prov = _RaisingProvider(LLMRateLimitedError("429", status_code=429, retry_after=None))
|
||
|
||
res = complete(system_prompt="sys", payload=SafePayload(text="q"), provider=prov)
|
||
assert res.reason == "rate_limited"
|
||
assert res.fallback_used is True
|
||
# 1 первичный + 2 ретрая = 3 вызова.
|
||
assert prov.calls == 3
|
||
|
||
|
||
def test_rate_limited_retry_after_capped(
|
||
_enabled: None, monkeypatch: pytest.MonkeyPatch
|
||
) -> None:
|
||
"""#1209: серверный Retry-After (86400с при quota-exhaustion) должен капаться
|
||
_MAX_BACKOFF_S — иначе time.sleep блокирует anyio-threadpool на часы.
|
||
"""
|
||
monkeypatch.setattr(settings, "llm_max_retries", 1)
|
||
sleeps: list[float] = []
|
||
monkeypatch.setattr(llm_client.time, "sleep", lambda s: sleeps.append(s))
|
||
|
||
class _BigRetryAfterProvider(LLMProvider):
|
||
@property
|
||
def is_external(self) -> bool:
|
||
return True
|
||
|
||
@property
|
||
def model(self) -> str:
|
||
return "gpt-4o-mini"
|
||
|
||
def complete(self, messages: Any, *, tools: Any = None, max_output_tokens: int) -> Any:
|
||
raise LLMRateLimitedError("429", status_code=429, retry_after=86400.0)
|
||
|
||
prov = _BigRetryAfterProvider()
|
||
res = complete(system_prompt="sys", payload=SafePayload(text="q"), provider=prov)
|
||
assert res.ok is False
|
||
assert res.reason == "rate_limited"
|
||
# ровно один backoff (1 ретрай); значение должно быть капано _MAX_BACKOFF_S (30с).
|
||
assert len(sleeps) == 1
|
||
assert sleeps[0] == llm_client._MAX_BACKOFF_S
|
||
|
||
|
||
def test_rate_limited_then_success(_enabled: None, monkeypatch: pytest.MonkeyPatch) -> None:
|
||
"""429 один раз, затем успех → ok (ретрай сработал)."""
|
||
monkeypatch.setattr(settings, "llm_max_retries", 2)
|
||
monkeypatch.setattr(llm_client.time, "sleep", lambda s: None)
|
||
|
||
class _FlakyProvider(LLMProvider):
|
||
def __init__(self) -> None:
|
||
self.calls = 0
|
||
|
||
@property
|
||
def is_external(self) -> bool:
|
||
return True
|
||
|
||
@property
|
||
def model(self) -> str:
|
||
return "gpt-4o-mini"
|
||
|
||
def complete(self, messages: Any, *, tools: Any = None, max_output_tokens: int) -> Any:
|
||
self.calls += 1
|
||
if self.calls == 1:
|
||
raise LLMRateLimitedError("429", status_code=429, retry_after=None)
|
||
return ProviderResponse(content="ок после ретрая", model="gpt-4o-mini")
|
||
|
||
prov = _FlakyProvider()
|
||
res = complete(system_prompt="sys", payload=SafePayload(text="q"), provider=prov)
|
||
assert res.ok is True
|
||
assert res.content == "ок после ретрая"
|
||
assert prov.calls == 2
|
||
|
||
|
||
# ── complete_or_raise ──────────────────────────────────────────────────────────
|
||
|
||
|
||
def test_complete_or_raise_raises_on_fallback(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
"""complete_or_raise бросает LLMUnavailableError когда LLM недоступен (disabled)."""
|
||
monkeypatch.setattr(settings, "llm_enabled", False)
|
||
with pytest.raises(LLMUnavailableError):
|
||
complete_or_raise(system_prompt="sys", payload=SafePayload(text="hi"))
|
||
|
||
|
||
def test_complete_or_raise_returns_on_success(_enabled: None) -> None:
|
||
prov = _FakeOpenAILike(ProviderResponse(content="ok", model="gpt-4o-mini"))
|
||
res = complete_or_raise(system_prompt="sys", payload=SafePayload(text="q"), provider=prov)
|
||
assert isinstance(res, LLMResult)
|
||
assert res.ok is True
|