lekss361
d7f4873a53
Some checks failed
Deploy Trade-In / changes (push) Successful in 16s
Deploy Trade-In / build-browser (push) Has been skipped
Deploy Trade-In / test (push) Failing after 1m36s
Deploy Trade-In / build-backend (push) Has been skipped
Deploy Trade-In / deploy (push) Has been skipped
Deploy Trade-In / build-frontend (push) Successful in 33s
321 lines
12 KiB
Python
321 lines
12 KiB
Python
"""Offline-тесты пула прокси (#2162).
|
||
|
||
Покрытие БЕЗ live-сети/БД: stateful FakeSession эмулирует таблицу scrape_proxies и
|
||
интерпретирует SQL по ключевым фрагментам, так что acquire/release/mark_health/
|
||
reap_stale_leases проверяются по фактическому изменению состояния строк.
|
||
|
||
- acquire: возвращает свободный+здоровый прокси нужного affinity; ставит lease.
|
||
- два acquire подряд → РАЗНЫЕ прокси (первый лизнут → выпал из выборки второго).
|
||
- release освобождает (leased_by → NULL), прокси снова acquire-абелен.
|
||
- mark_health fail → инкремент consecutive_fails, авто-disable при DISABLE_THRESHOLD.
|
||
- mark_health ok → сброс fails + exit_ip/latency.
|
||
- reap_stale_leases освобождает старый lease, свежий не трогает.
|
||
- affinity-фильтр: acquire('avito') не берёт cian-only прокси.
|
||
- acquire пропускает disabled и «нездоровые» (fails >= MAX_CONSECUTIVE_FAILS).
|
||
- run_proxy_healthcheck: reap + проба каждого enabled + mark_health (проба замокана).
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import os
|
||
|
||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
|
||
|
||
from datetime import UTC, datetime, timedelta
|
||
from typing import Any
|
||
|
||
import pytest
|
||
|
||
from app.services import proxy_pool
|
||
from app.services.proxy_pool import (
|
||
DISABLE_THRESHOLD,
|
||
MAX_CONSECUTIVE_FAILS,
|
||
acquire,
|
||
mark_health,
|
||
reap_stale_leases,
|
||
release,
|
||
)
|
||
|
||
# ── stateful fake session ────────────────────────────────────────────────────
|
||
|
||
|
||
class _FakeResult:
|
||
def __init__(self, rows: list[dict[str, Any]]):
|
||
self._rows = rows
|
||
|
||
def mappings(self) -> _FakeResult:
|
||
return self
|
||
|
||
def fetchone(self) -> dict[str, Any] | None:
|
||
return self._rows[0] if self._rows else None
|
||
|
||
def all(self) -> list[dict[str, Any]]:
|
||
return list(self._rows)
|
||
|
||
def fetchall(self) -> list[Any]:
|
||
# для RETURNING id: код делает r.id → нужен attribute-access
|
||
return [type("Row", (), r)() for r in self._rows]
|
||
|
||
|
||
class FakeSession:
|
||
"""Эмуляция Session поверх in-memory списка scrape_proxies-строк."""
|
||
|
||
def __init__(self, rows: list[dict[str, Any]]):
|
||
self.rows = rows
|
||
|
||
# helpers
|
||
def _by_id(self, pid: int) -> dict[str, Any] | None:
|
||
return next((r for r in self.rows if r["id"] == pid), None)
|
||
|
||
def execute(self, stmt: Any, params: dict[str, Any] | None = None) -> _FakeResult:
|
||
sql = str(stmt)
|
||
p = params or {}
|
||
|
||
if "FOR UPDATE SKIP LOCKED" in sql: # acquire SELECT
|
||
provider = p["provider"]
|
||
max_fails = p["max_fails"]
|
||
cands = [
|
||
r
|
||
for r in self.rows
|
||
if r["enabled"]
|
||
and r["consecutive_fails"] < max_fails
|
||
and r["provider_affinity"] in (provider, "any")
|
||
and r["leased_by"] is None
|
||
]
|
||
# ORDER BY last_ok_at NULLS LAST, id
|
||
cands.sort(
|
||
key=lambda r: (
|
||
r["last_ok_at"] is None,
|
||
r["last_ok_at"] or datetime.min.replace(tzinfo=UTC),
|
||
r["id"],
|
||
)
|
||
)
|
||
return _FakeResult(cands[:1])
|
||
|
||
if "SET leased_by = CAST(:run_id" in sql: # acquire lease UPDATE
|
||
row = self._by_id(p["id"])
|
||
if row is not None:
|
||
row["leased_by"] = p["run_id"]
|
||
row["leased_at"] = datetime.now(UTC)
|
||
return _FakeResult([])
|
||
|
||
if "make_interval(mins =>" in sql and "leased_by IS NOT NULL" in sql: # reap
|
||
cutoff = datetime.now(UTC) - timedelta(minutes=p["mins"])
|
||
freed: list[dict[str, Any]] = []
|
||
for r in self.rows:
|
||
if (
|
||
r["leased_by"] is not None
|
||
and r["leased_at"] is not None
|
||
and r["leased_at"] < cutoff
|
||
):
|
||
r["leased_by"] = None
|
||
r["leased_at"] = None
|
||
freed.append({"id": r["id"]})
|
||
return _FakeResult(freed)
|
||
|
||
if "SET leased_by = NULL" in sql: # release (WHERE id)
|
||
row = self._by_id(p["id"])
|
||
if row is not None:
|
||
row["leased_by"] = None
|
||
row["leased_at"] = None
|
||
return _FakeResult([])
|
||
|
||
if "SET consecutive_fails = 0" in sql: # mark_health ok
|
||
row = self._by_id(p["id"])
|
||
if row is not None:
|
||
row["consecutive_fails"] = 0
|
||
row["exit_ip"] = p["exit_ip"]
|
||
row["latency_ms"] = p["latency_ms"]
|
||
row["last_ok_at"] = datetime.now(UTC)
|
||
return _FakeResult([])
|
||
|
||
if "consecutive_fails = consecutive_fails + 1" in sql: # mark_health fail
|
||
row = self._by_id(p["id"])
|
||
if row is not None:
|
||
row["consecutive_fails"] += 1
|
||
if row["consecutive_fails"] >= p["disable_threshold"]:
|
||
row["enabled"] = False
|
||
return _FakeResult([])
|
||
|
||
if "WHERE enabled" in sql and "ORDER BY id" in sql: # healthcheck SELECT
|
||
rows = sorted((r for r in self.rows if r["enabled"]), key=lambda r: r["id"])
|
||
return _FakeResult([dict(r) for r in rows])
|
||
|
||
raise AssertionError(f"unhandled SQL: {sql}")
|
||
|
||
def commit(self) -> None:
|
||
pass
|
||
|
||
def rollback(self) -> None:
|
||
pass
|
||
|
||
|
||
def _proxy(
|
||
pid: int,
|
||
*,
|
||
affinity: str = "any",
|
||
enabled: bool = True,
|
||
fails: int = 0,
|
||
leased_by: int | None = None,
|
||
leased_at: datetime | None = None,
|
||
last_ok_at: datetime | None = None,
|
||
kind: str = "http",
|
||
rotate_url: str | None = None,
|
||
) -> dict[str, Any]:
|
||
return {
|
||
"id": pid,
|
||
"url": f"http://u:p@h{pid}:8080",
|
||
"kind": kind,
|
||
"rotate_url": rotate_url,
|
||
"provider_affinity": affinity,
|
||
"enabled": enabled,
|
||
"consecutive_fails": fails,
|
||
"leased_by": leased_by,
|
||
"leased_at": leased_at,
|
||
"last_ok_at": last_ok_at,
|
||
"exit_ip": None,
|
||
"latency_ms": None,
|
||
}
|
||
|
||
|
||
# ── acquire ──────────────────────────────────────────────────────────────────
|
||
|
||
|
||
def test_acquire_returns_free_healthy_of_affinity() -> None:
|
||
db = FakeSession([_proxy(1, affinity="avito"), _proxy(2, affinity="cian")])
|
||
lease = acquire(db, "avito", run_id=100) # type: ignore[arg-type]
|
||
assert lease is not None
|
||
assert lease.id == 1
|
||
assert lease.url == "http://u:p@h1:8080"
|
||
assert db._by_id(1)["leased_by"] == 100 # lease проставлен
|
||
|
||
|
||
def test_acquire_includes_any_affinity() -> None:
|
||
db = FakeSession([_proxy(1, affinity="any")])
|
||
lease = acquire(db, "avito", run_id=5) # type: ignore[arg-type]
|
||
assert lease is not None and lease.id == 1
|
||
|
||
|
||
def test_two_acquire_return_distinct_proxies() -> None:
|
||
db = FakeSession([_proxy(1, affinity="avito"), _proxy(2, affinity="avito")])
|
||
first = acquire(db, "avito", run_id=1) # type: ignore[arg-type]
|
||
second = acquire(db, "avito", run_id=2) # type: ignore[arg-type]
|
||
assert first is not None and second is not None
|
||
assert first.id != second.id # первый лизнут → второй берёт другой
|
||
|
||
|
||
def test_acquire_empty_pool_returns_none() -> None:
|
||
db = FakeSession([_proxy(1, affinity="avito", leased_by=99)]) # единственный занят
|
||
assert acquire(db, "avito", run_id=1) is None # type: ignore[arg-type]
|
||
|
||
|
||
def test_acquire_affinity_filter_excludes_other_provider() -> None:
|
||
db = FakeSession([_proxy(1, affinity="cian")])
|
||
assert acquire(db, "avito", run_id=1) is None # type: ignore[arg-type]
|
||
|
||
|
||
def test_acquire_skips_disabled() -> None:
|
||
db = FakeSession([_proxy(1, affinity="avito", enabled=False)])
|
||
assert acquire(db, "avito", run_id=1) is None # type: ignore[arg-type]
|
||
|
||
|
||
def test_acquire_skips_unhealthy() -> None:
|
||
db = FakeSession([_proxy(1, affinity="avito", fails=MAX_CONSECUTIVE_FAILS)])
|
||
assert acquire(db, "avito", run_id=1) is None # type: ignore[arg-type]
|
||
|
||
|
||
def test_acquire_without_run_id_uses_marker() -> None:
|
||
db = FakeSession([_proxy(1, affinity="avito")])
|
||
lease = acquire(db, "avito") # run_id=None # type: ignore[arg-type]
|
||
assert lease is not None
|
||
assert db._by_id(1)["leased_by"] == proxy_pool.NON_RUN_LEASE_MARKER
|
||
|
||
|
||
# ── release ──────────────────────────────────────────────────────────────────
|
||
|
||
|
||
def test_release_frees_proxy() -> None:
|
||
db = FakeSession([_proxy(1, affinity="avito")])
|
||
lease = acquire(db, "avito", run_id=7) # type: ignore[arg-type]
|
||
assert lease is not None
|
||
assert acquire(db, "avito", run_id=8) is None # type: ignore[arg-type] # занят
|
||
release(db, 1) # type: ignore[arg-type]
|
||
assert db._by_id(1)["leased_by"] is None
|
||
assert acquire(db, "avito", run_id=9) is not None # type: ignore[arg-type] # снова свободен
|
||
|
||
|
||
# ── mark_health ──────────────────────────────────────────────────────────────
|
||
|
||
|
||
def test_mark_health_fail_increments() -> None:
|
||
db = FakeSession([_proxy(1, fails=0)])
|
||
mark_health(db, 1, ok=False) # type: ignore[arg-type]
|
||
assert db._by_id(1)["consecutive_fails"] == 1
|
||
assert db._by_id(1)["enabled"] is True # ещё не порог
|
||
|
||
|
||
def test_mark_health_fail_disables_at_threshold() -> None:
|
||
db = FakeSession([_proxy(1, fails=DISABLE_THRESHOLD - 1)])
|
||
mark_health(db, 1, ok=False) # type: ignore[arg-type]
|
||
assert db._by_id(1)["consecutive_fails"] == DISABLE_THRESHOLD
|
||
assert db._by_id(1)["enabled"] is False # авто-disable
|
||
|
||
|
||
def test_mark_health_ok_resets_and_records() -> None:
|
||
db = FakeSession([_proxy(1, fails=4)])
|
||
mark_health(db, 1, ok=True, exit_ip="1.2.3.4", latency_ms=88) # type: ignore[arg-type]
|
||
row = db._by_id(1)
|
||
assert row["consecutive_fails"] == 0
|
||
assert row["exit_ip"] == "1.2.3.4"
|
||
assert row["latency_ms"] == 88
|
||
assert row["last_ok_at"] is not None
|
||
|
||
|
||
# ── reap_stale_leases ────────────────────────────────────────────────────────
|
||
|
||
|
||
def test_reap_frees_stale_lease_keeps_fresh() -> None:
|
||
old = datetime.now(UTC) - timedelta(minutes=120)
|
||
fresh = datetime.now(UTC) - timedelta(minutes=1)
|
||
db = FakeSession(
|
||
[
|
||
_proxy(1, leased_by=50, leased_at=old),
|
||
_proxy(2, leased_by=51, leased_at=fresh),
|
||
]
|
||
)
|
||
freed = reap_stale_leases(db, older_than_minutes=30) # type: ignore[arg-type]
|
||
assert freed == 1
|
||
assert db._by_id(1)["leased_by"] is None # протухший освобождён
|
||
assert db._by_id(2)["leased_by"] == 51 # свежий не тронут
|
||
|
||
|
||
# ── run_proxy_healthcheck ────────────────────────────────────────────────────
|
||
|
||
|
||
async def test_healthcheck_probes_enabled_and_marks_health(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
db = FakeSession(
|
||
[
|
||
_proxy(1, fails=2),
|
||
_proxy(2, enabled=False), # disabled — не проверяется
|
||
_proxy(3, fails=0),
|
||
]
|
||
)
|
||
|
||
async def _fake_probe(url: str) -> tuple[bool, str | None, int | None]:
|
||
# прокси 1 «жив», прокси 3 «мёртв»
|
||
if "h1:" in url:
|
||
return True, "9.9.9.9", 42
|
||
return False, None, None
|
||
|
||
monkeypatch.setattr(proxy_pool, "_probe_proxy", _fake_probe)
|
||
|
||
counters = await proxy_pool.run_proxy_healthcheck(db) # type: ignore[arg-type]
|
||
|
||
assert counters["checked"] == 2 # только enabled (1 и 3)
|
||
assert counters["ok"] == 1
|
||
assert counters["failed"] == 1
|
||
assert db._by_id(1)["consecutive_fails"] == 0 # ok → сброс
|
||
assert db._by_id(1)["exit_ip"] == "9.9.9.9"
|
||
assert db._by_id(3)["consecutive_fails"] == 1 # fail → инкремент
|