"""Offline-тесты пула прокси (#2162). Покрытие БЕЗ live-сети/БД: stateful FakeSession эмулирует таблицу scrape_proxies и интерпретирует SQL по ключевым фрагментам, так что acquire/release/mark_health/ reap_stale_leases проверяются по фактическому изменению состояния строк. - acquire: возвращает свободный+здоровый прокси нужного affinity; ставит lease. - два acquire подряд → РАЗНЫЕ прокси (первый лизнут → выпал из выборки второго). - release освобождает (leased_by → NULL), прокси снова acquire-абелен. - mark_health fail → инкремент consecutive_fails, авто-disable при DISABLE_THRESHOLD. - mark_health ok → сброс fails + exit_ip/latency. - reap_stale_leases освобождает старый lease, свежий не трогает. - affinity-фильтр: acquire('avito') не берёт cian-only прокси. - acquire пропускает disabled и «нездоровые» (fails >= MAX_CONSECUTIVE_FAILS). - run_proxy_healthcheck: reap + проба каждого enabled + mark_health (проба замокана). """ from __future__ import annotations import os os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test") from datetime import UTC, datetime, timedelta from typing import Any import pytest from app.services import proxy_pool from app.services.proxy_pool import ( DISABLE_THRESHOLD, MAX_CONSECUTIVE_FAILS, acquire, mark_health, reap_stale_leases, release, ) # ── stateful fake session ──────────────────────────────────────────────────── class _FakeResult: def __init__(self, rows: list[dict[str, Any]]): self._rows = rows def mappings(self) -> _FakeResult: return self def fetchone(self) -> dict[str, Any] | None: return self._rows[0] if self._rows else None def all(self) -> list[dict[str, Any]]: return list(self._rows) def fetchall(self) -> list[Any]: # для RETURNING id: код делает r.id → нужен attribute-access return [type("Row", (), r)() for r in self._rows] class FakeSession: """Эмуляция Session поверх in-memory списка scrape_proxies-строк.""" def __init__(self, rows: list[dict[str, Any]]): self.rows = rows # helpers def _by_id(self, pid: int) -> dict[str, Any] | None: return next((r for r in self.rows if r["id"] == pid), None) def execute(self, stmt: Any, params: dict[str, Any] | None = None) -> _FakeResult: sql = str(stmt) p = params or {} if "FOR UPDATE SKIP LOCKED" in sql: # acquire SELECT provider = p["provider"] max_fails = p["max_fails"] cands = [ r for r in self.rows if r["enabled"] and r["consecutive_fails"] < max_fails and r["provider_affinity"] in (provider, "any") and r["leased_by"] is None ] # ORDER BY last_ok_at NULLS LAST, id cands.sort( key=lambda r: ( r["last_ok_at"] is None, r["last_ok_at"] or datetime.min.replace(tzinfo=UTC), r["id"], ) ) return _FakeResult(cands[:1]) if "SET leased_by = CAST(:run_id" in sql: # acquire lease UPDATE row = self._by_id(p["id"]) if row is not None: row["leased_by"] = p["run_id"] row["leased_at"] = datetime.now(UTC) return _FakeResult([]) if "make_interval(mins =>" in sql and "leased_by IS NOT NULL" in sql: # reap cutoff = datetime.now(UTC) - timedelta(minutes=p["mins"]) freed: list[dict[str, Any]] = [] for r in self.rows: if ( r["leased_by"] is not None and r["leased_at"] is not None and r["leased_at"] < cutoff ): r["leased_by"] = None r["leased_at"] = None freed.append({"id": r["id"]}) return _FakeResult(freed) if "SET leased_by = NULL" in sql: # release (WHERE id) row = self._by_id(p["id"]) if row is not None: row["leased_by"] = None row["leased_at"] = None return _FakeResult([]) if "SET consecutive_fails = 0" in sql: # mark_health ok row = self._by_id(p["id"]) if row is not None: row["consecutive_fails"] = 0 row["exit_ip"] = p["exit_ip"] row["latency_ms"] = p["latency_ms"] row["last_ok_at"] = datetime.now(UTC) return _FakeResult([]) if "consecutive_fails = consecutive_fails + 1" in sql: # mark_health fail row = self._by_id(p["id"]) if row is not None: row["consecutive_fails"] += 1 if row["consecutive_fails"] >= p["disable_threshold"]: row["enabled"] = False return _FakeResult([]) if "WHERE enabled" in sql and "ORDER BY id" in sql: # healthcheck SELECT rows = sorted((r for r in self.rows if r["enabled"]), key=lambda r: r["id"]) return _FakeResult([dict(r) for r in rows]) raise AssertionError(f"unhandled SQL: {sql}") def commit(self) -> None: pass def rollback(self) -> None: pass def _proxy( pid: int, *, affinity: str = "any", enabled: bool = True, fails: int = 0, leased_by: int | None = None, leased_at: datetime | None = None, last_ok_at: datetime | None = None, kind: str = "http", rotate_url: str | None = None, ) -> dict[str, Any]: return { "id": pid, "url": f"http://u:p@h{pid}:8080", "kind": kind, "rotate_url": rotate_url, "provider_affinity": affinity, "enabled": enabled, "consecutive_fails": fails, "leased_by": leased_by, "leased_at": leased_at, "last_ok_at": last_ok_at, "exit_ip": None, "latency_ms": None, } # ── acquire ────────────────────────────────────────────────────────────────── def test_acquire_returns_free_healthy_of_affinity() -> None: db = FakeSession([_proxy(1, affinity="avito"), _proxy(2, affinity="cian")]) lease = acquire(db, "avito", run_id=100) # type: ignore[arg-type] assert lease is not None assert lease.id == 1 assert lease.url == "http://u:p@h1:8080" assert db._by_id(1)["leased_by"] == 100 # lease проставлен def test_acquire_includes_any_affinity() -> None: db = FakeSession([_proxy(1, affinity="any")]) lease = acquire(db, "avito", run_id=5) # type: ignore[arg-type] assert lease is not None and lease.id == 1 def test_two_acquire_return_distinct_proxies() -> None: db = FakeSession([_proxy(1, affinity="avito"), _proxy(2, affinity="avito")]) first = acquire(db, "avito", run_id=1) # type: ignore[arg-type] second = acquire(db, "avito", run_id=2) # type: ignore[arg-type] assert first is not None and second is not None assert first.id != second.id # первый лизнут → второй берёт другой def test_acquire_empty_pool_returns_none() -> None: db = FakeSession([_proxy(1, affinity="avito", leased_by=99)]) # единственный занят assert acquire(db, "avito", run_id=1) is None # type: ignore[arg-type] def test_acquire_affinity_filter_excludes_other_provider() -> None: db = FakeSession([_proxy(1, affinity="cian")]) assert acquire(db, "avito", run_id=1) is None # type: ignore[arg-type] def test_acquire_skips_disabled() -> None: db = FakeSession([_proxy(1, affinity="avito", enabled=False)]) assert acquire(db, "avito", run_id=1) is None # type: ignore[arg-type] def test_acquire_skips_unhealthy() -> None: db = FakeSession([_proxy(1, affinity="avito", fails=MAX_CONSECUTIVE_FAILS)]) assert acquire(db, "avito", run_id=1) is None # type: ignore[arg-type] def test_acquire_without_run_id_uses_marker() -> None: db = FakeSession([_proxy(1, affinity="avito")]) lease = acquire(db, "avito") # run_id=None # type: ignore[arg-type] assert lease is not None assert db._by_id(1)["leased_by"] == proxy_pool.NON_RUN_LEASE_MARKER # ── release ────────────────────────────────────────────────────────────────── def test_release_frees_proxy() -> None: db = FakeSession([_proxy(1, affinity="avito")]) lease = acquire(db, "avito", run_id=7) # type: ignore[arg-type] assert lease is not None assert acquire(db, "avito", run_id=8) is None # type: ignore[arg-type] # занят release(db, 1) # type: ignore[arg-type] assert db._by_id(1)["leased_by"] is None assert acquire(db, "avito", run_id=9) is not None # type: ignore[arg-type] # снова свободен # ── mark_health ────────────────────────────────────────────────────────────── def test_mark_health_fail_increments() -> None: db = FakeSession([_proxy(1, fails=0)]) mark_health(db, 1, ok=False) # type: ignore[arg-type] assert db._by_id(1)["consecutive_fails"] == 1 assert db._by_id(1)["enabled"] is True # ещё не порог def test_mark_health_fail_disables_at_threshold() -> None: db = FakeSession([_proxy(1, fails=DISABLE_THRESHOLD - 1)]) mark_health(db, 1, ok=False) # type: ignore[arg-type] assert db._by_id(1)["consecutive_fails"] == DISABLE_THRESHOLD assert db._by_id(1)["enabled"] is False # авто-disable def test_mark_health_ok_resets_and_records() -> None: db = FakeSession([_proxy(1, fails=4)]) mark_health(db, 1, ok=True, exit_ip="1.2.3.4", latency_ms=88) # type: ignore[arg-type] row = db._by_id(1) assert row["consecutive_fails"] == 0 assert row["exit_ip"] == "1.2.3.4" assert row["latency_ms"] == 88 assert row["last_ok_at"] is not None # ── reap_stale_leases ──────────────────────────────────────────────────────── def test_reap_frees_stale_lease_keeps_fresh() -> None: old = datetime.now(UTC) - timedelta(minutes=120) fresh = datetime.now(UTC) - timedelta(minutes=1) db = FakeSession( [ _proxy(1, leased_by=50, leased_at=old), _proxy(2, leased_by=51, leased_at=fresh), ] ) freed = reap_stale_leases(db, older_than_minutes=30) # type: ignore[arg-type] assert freed == 1 assert db._by_id(1)["leased_by"] is None # протухший освобождён assert db._by_id(2)["leased_by"] == 51 # свежий не тронут # ── run_proxy_healthcheck ──────────────────────────────────────────────────── async def test_healthcheck_probes_enabled_and_marks_health( monkeypatch: pytest.MonkeyPatch, ) -> None: db = FakeSession( [ _proxy(1, fails=2), _proxy(2, enabled=False), # disabled — не проверяется _proxy(3, fails=0), ] ) async def _fake_probe(url: str) -> tuple[bool, str | None, int | None]: # прокси 1 «жив», прокси 3 «мёртв» if "h1:" in url: return True, "9.9.9.9", 42 return False, None, None monkeypatch.setattr(proxy_pool, "_probe_proxy", _fake_probe) counters = await proxy_pool.run_proxy_healthcheck(db) # type: ignore[arg-type] assert counters["checked"] == 2 # только enabled (1 и 3) assert counters["ok"] == 1 assert counters["failed"] == 1 assert db._by_id(1)["consecutive_fails"] == 0 # ok → сброс assert db._by_id(1)["exit_ip"] == "9.9.9.9" assert db._by_id(3)["consecutive_fails"] == 1 # fail → инкремент