"""LLM client orchestration tests (#960). Покрываем: disabled→fallback (БЕЗ сетевой попытки), no-key→fallback, call-cap, redaction hard-block→fallback, happy-path (тело запроса проскраблено), guardrails (timeout→fallback, 429-retry-then-fallback), complete_or_raise. Сеть НЕ дёргается: либо внедряем fake-провайдера, либо ставим tripwire на httpx.Client. """ from __future__ import annotations from typing import Any import pytest from app.core.config import settings from app.services.llm import client as llm_client from app.services.llm.client import LLMResult, LLMUnavailableError, complete, complete_or_raise from app.services.llm.provider import ( LLMProvider, LLMRateLimitedError, LLMTimeoutError, ProviderResponse, ToolCall, ) from app.services.llm.redaction import SafePayload # ── Fake провайдеры (внедряются через provider=...) ──────────────────────────── class _FakeOpenAILike(LLMProvider): """Внешний (is_external=True) fake: записывает messages, отдаёт фикс. ответ.""" def __init__(self, response: ProviderResponse | None = None) -> None: self.captured_messages: list[dict[str, Any]] | None = None self.captured_tools: list[dict[str, Any]] | None = None self.calls = 0 self._response = response or ProviderResponse( content="ответ модели", prompt_tokens=10, completion_tokens=5, model="gpt-4o-mini" ) @property def is_external(self) -> bool: return True @property def model(self) -> str: return "gpt-4o-mini" def complete( self, messages: list[dict[str, Any]], *, tools: list[dict[str, Any]] | None = None, max_output_tokens: int, ) -> ProviderResponse: self.calls += 1 self.captured_messages = messages self.captured_tools = tools return self._response class _RaisingProvider(LLMProvider): """Внешний fake, всегда бросающий заданное исключение (для guardrail-тестов).""" def __init__(self, exc: Exception) -> None: self._exc = exc self.calls = 0 @property def is_external(self) -> bool: return True @property def model(self) -> str: return "gpt-4o-mini" def complete(self, messages: Any, *, tools: Any = None, max_output_tokens: int) -> Any: self.calls += 1 raise self._exc @pytest.fixture def _enabled(monkeypatch: pytest.MonkeyPatch) -> None: """Включить LLM с fake-ключом (без реальных секретов).""" monkeypatch.setattr(settings, "llm_enabled", True) monkeypatch.setattr(settings, "openai_api_key", "test-fake-key-not-real") # ── Guard #1/#2: disabled / no key ───────────────────────────────────────────── def test_disabled_returns_fallback_without_network(monkeypatch: pytest.MonkeyPatch) -> None: """llm_enabled=False → fallback, и НИ ОДНОЙ попытки создать httpx.Client.""" monkeypatch.setattr(settings, "llm_enabled", False) def _tripwire(*a: Any, **k: Any) -> None: raise AssertionError("network must NOT be touched when llm_enabled=False") # Любая попытка сетевого вызова через httpx завалит тест. monkeypatch.setattr("app.services.llm.provider.httpx.Client", _tripwire) res = complete(system_prompt="sys", payload=SafePayload(text="hi")) assert res.ok is False assert res.fallback_used is True assert res.reason == "disabled" def test_enabled_but_no_key_returns_fallback(monkeypatch: pytest.MonkeyPatch) -> None: """llm_enabled=True но ключ None → fallback (no_api_key), без сети.""" monkeypatch.setattr(settings, "llm_enabled", True) monkeypatch.setattr(settings, "openai_api_key", None) def _tripwire(*a: Any, **k: Any) -> None: raise AssertionError("no network without a key") monkeypatch.setattr("app.services.llm.provider.httpx.Client", _tripwire) res = complete(system_prompt="sys", payload=SafePayload(text="hi")) assert res.reason == "no_api_key" assert res.fallback_used is True # ── Call cap ─────────────────────────────────────────────────────────────────── def test_call_cap_returns_fallback(_enabled: None, monkeypatch: pytest.MonkeyPatch) -> None: """call_index >= llm_max_calls_per_request → fallback, провайдер не вызывается.""" monkeypatch.setattr(settings, "llm_max_calls_per_request", 2) prov = _FakeOpenAILike() res = complete( system_prompt="sys", payload=SafePayload(text="hi"), provider=prov, call_index=2 ) assert res.reason == "call_cap" assert prov.calls == 0 # ── Redaction integration ────────────────────────────────────────────────────── def test_confidential_payload_returns_fallback(_enabled: None) -> None: """is_confidential=True (external) → fallback redaction_refused, провайдер не зван.""" prov = _FakeOpenAILike() res = complete( system_prompt="sys", payload=SafePayload(text="секрет", is_confidential=True), provider=prov, ) assert res.reason == "redaction_refused" assert prov.calls == 0 def test_request_body_is_redacted(_enabled: None) -> None: """ГЛАВНОЕ: тело запроса к провайдеру НЕ содержит сырого PII — оно проскраблено.""" prov = _FakeOpenAILike() payload = SafePayload( text="Свяжитесь +7 912 345 67 89 или mail@example.ru", fields={"owner": "Иванов Иван Иванович", "zone": "Ц-1"}, ) res = complete(system_prompt="Ты ассистент", payload=payload, provider=prov) assert res.ok is True assert prov.captured_messages is not None blob = str(prov.captured_messages) # Сырого PII в отправленных messages быть не должно. assert "+7 912 345 67 89" not in blob assert "mail@example.ru" not in blob assert "Иванов Иван Иванович" not in blob # Зато плейсхолдеры и безопасные поля — на месте. assert "[REDACTED:phone]" in blob assert "[REDACTED:email]" in blob assert "[REDACTED:name]" in blob assert "Ц-1" in blob # ── Happy path ───────────────────────────────────────────────────────────────── def test_happy_path_returns_ok_result(_enabled: None) -> None: """Успешный ответ → ok=True, content и токены проброшены.""" prov = _FakeOpenAILike( ProviderResponse( content="готово", prompt_tokens=12, completion_tokens=4, model="gpt-4o-mini" ) ) res = complete(system_prompt="sys", payload=SafePayload(text="вопрос"), provider=prov) assert res.ok is True assert res.content == "готово" assert res.prompt_tokens == 12 assert res.completion_tokens == 4 assert res.model == "gpt-4o-mini" def test_tool_calls_passed_through(_enabled: None) -> None: """tool_calls из ответа провайдера прокидываются в LLMResult.""" prov = _FakeOpenAILike( ProviderResponse( content=None, tool_calls=[ToolCall(id="c1", name="extract", arguments='{"x":1}')], model="gpt-4o-mini", ) ) res = complete(system_prompt="sys", payload=SafePayload(text="q"), provider=prov) assert res.ok is True assert len(res.tool_calls) == 1 assert res.tool_calls[0].name == "extract" def test_tools_forwarded_to_provider(_enabled: None) -> None: """tools прокидываются в provider.complete.""" prov = _FakeOpenAILike() tools = [{"type": "function", "function": {"name": "f"}}] complete(system_prompt="sys", payload=SafePayload(text="q"), provider=prov, tools=tools) assert prov.captured_tools == tools # ── Guardrails: timeout / rate-limit / retries ───────────────────────────────── def test_timeout_degrades_to_fallback(_enabled: None) -> None: """LLMTimeoutError → fallback(timeout), без ретраев (1 вызов).""" prov = _RaisingProvider(LLMTimeoutError("slow")) res = complete(system_prompt="sys", payload=SafePayload(text="q"), provider=prov) assert res.reason == "timeout" assert res.fallback_used is True assert prov.calls == 1 def test_rate_limited_retries_then_fallback( _enabled: None, monkeypatch: pytest.MonkeyPatch ) -> None: """429 на всех попытках → ретраи (cap) затем fallback(rate_limited). sleep замокан.""" monkeypatch.setattr(settings, "llm_max_retries", 2) # Не спим в тесте. monkeypatch.setattr(llm_client.time, "sleep", lambda s: None) prov = _RaisingProvider(LLMRateLimitedError("429", status_code=429, retry_after=None)) res = complete(system_prompt="sys", payload=SafePayload(text="q"), provider=prov) assert res.reason == "rate_limited" assert res.fallback_used is True # 1 первичный + 2 ретрая = 3 вызова. assert prov.calls == 3 def test_rate_limited_retry_after_capped( _enabled: None, monkeypatch: pytest.MonkeyPatch ) -> None: """#1209: серверный Retry-After (86400с при quota-exhaustion) должен капаться _MAX_BACKOFF_S — иначе time.sleep блокирует anyio-threadpool на часы. """ monkeypatch.setattr(settings, "llm_max_retries", 1) sleeps: list[float] = [] monkeypatch.setattr(llm_client.time, "sleep", lambda s: sleeps.append(s)) class _BigRetryAfterProvider(LLMProvider): @property def is_external(self) -> bool: return True @property def model(self) -> str: return "gpt-4o-mini" def complete(self, messages: Any, *, tools: Any = None, max_output_tokens: int) -> Any: raise LLMRateLimitedError("429", status_code=429, retry_after=86400.0) prov = _BigRetryAfterProvider() res = complete(system_prompt="sys", payload=SafePayload(text="q"), provider=prov) assert res.ok is False assert res.reason == "rate_limited" # ровно один backoff (1 ретрай); значение должно быть капано _MAX_BACKOFF_S (30с). assert len(sleeps) == 1 assert sleeps[0] == llm_client._MAX_BACKOFF_S def test_rate_limited_then_success(_enabled: None, monkeypatch: pytest.MonkeyPatch) -> None: """429 один раз, затем успех → ok (ретрай сработал).""" monkeypatch.setattr(settings, "llm_max_retries", 2) monkeypatch.setattr(llm_client.time, "sleep", lambda s: None) class _FlakyProvider(LLMProvider): def __init__(self) -> None: self.calls = 0 @property def is_external(self) -> bool: return True @property def model(self) -> str: return "gpt-4o-mini" def complete(self, messages: Any, *, tools: Any = None, max_output_tokens: int) -> Any: self.calls += 1 if self.calls == 1: raise LLMRateLimitedError("429", status_code=429, retry_after=None) return ProviderResponse(content="ок после ретрая", model="gpt-4o-mini") prov = _FlakyProvider() res = complete(system_prompt="sys", payload=SafePayload(text="q"), provider=prov) assert res.ok is True assert res.content == "ок после ретрая" assert prov.calls == 2 # ── complete_or_raise ────────────────────────────────────────────────────────── def test_complete_or_raise_raises_on_fallback(monkeypatch: pytest.MonkeyPatch) -> None: """complete_or_raise бросает LLMUnavailableError когда LLM недоступен (disabled).""" monkeypatch.setattr(settings, "llm_enabled", False) with pytest.raises(LLMUnavailableError): complete_or_raise(system_prompt="sys", payload=SafePayload(text="hi")) def test_complete_or_raise_returns_on_success(_enabled: None) -> None: prov = _FakeOpenAILike(ProviderResponse(content="ok", model="gpt-4o-mini")) res = complete_or_raise(system_prompt="sys", payload=SafePayload(text="q"), provider=prov) assert isinstance(res, LLMResult) assert res.ok is True