bot-backend
d1fa450f48
Some checks failed
Deploy / build-backend (push) Blocked by required conditions
Deploy / build-worker (push) Blocked by required conditions
Deploy / build-frontend (push) Blocked by required conditions
Deploy / deploy (push) Blocked by required conditions
Deploy / changes (push) Has been cancelled
269 lines
11 KiB
Python
269 lines
11 KiB
Python
"""Тесты pravo_gov66_client — offline, без живой сети.
|
||
|
||
Покрывают: парсинг search-HTML (doc_id/title/url), парсинг страницы документа
|
||
(PDF-ссылка), fallback на ecp-вариант, отсутствие PDF, pacing/backoff (#2445 D4)
|
||
между последовательными запросами к pravo.gov66.ru.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import httpx
|
||
import pytest
|
||
|
||
from app.services.scrapers.pravo_gov66_client import (
|
||
_LAW_TYPE_POSTANOVLENIE,
|
||
_extract_pdf_url,
|
||
_get_with_backoff,
|
||
_parse_search_html,
|
||
)
|
||
|
||
# ── Фикстура HTML страницы поиска ─────────────────────────────────────────────
|
||
# Реальный фрагмент страницы https://www.pravo.gov66.ru/search/?q=изъятие+земельных+участков
|
||
# Содержит ссылки вида /NNNNN/ с заголовком постановления.
|
||
_SEARCH_HTML_FIXTURE = """
|
||
<!DOCTYPE html>
|
||
<html>
|
||
<head><title>Поиск</title></head>
|
||
<body>
|
||
<div class="search-results">
|
||
<ul>
|
||
<li class="result-item">
|
||
<a href="/35496/">
|
||
Об изъятии земельных участков для государственных нужд
|
||
Свердловской области в целях реконструкции ул. Татищева
|
||
в г. Екатеринбурге
|
||
</a>
|
||
</li>
|
||
<li class="result-item">
|
||
<a href="/34812/">
|
||
О резервировании земельных участков для нужд Екатеринбург
|
||
</a>
|
||
</li>
|
||
<li class="result-item">
|
||
<!-- Ссылка на категорию, не документ — должна игнорироваться -->
|
||
<a href="/category/law/">Категория</a>
|
||
</li>
|
||
<li class="result-item">
|
||
<!-- Дубль doc_id 35496 — должен дедуплицироваться -->
|
||
<a href="/35496/">Дубль</a>
|
||
</li>
|
||
<li class="result-item">
|
||
<!-- Нечисловой id — не должен попасть -->
|
||
<a href="/some-slug/">Текстовый slug</a>
|
||
</li>
|
||
</ul>
|
||
</div>
|
||
</body>
|
||
</html>
|
||
"""
|
||
|
||
# ── Фикстура HTML страницы документа с обычным PDF ────────────────────────────
|
||
_DOC_HTML_WITH_PDF = """
|
||
<!DOCTYPE html>
|
||
<html>
|
||
<body>
|
||
<h1>Постановление № 509-ПП</h1>
|
||
<p>
|
||
Скачать документ:
|
||
<a href="/media/pravo/509-ПП_ab1234cd.pdf">Скачать PDF</a>
|
||
</p>
|
||
<p>
|
||
Подписанная версия:
|
||
<a href="/media/pravo/ecp/509-ПП.pdf">Скачать подписанный PDF</a>
|
||
</p>
|
||
</body>
|
||
</html>
|
||
"""
|
||
|
||
# ── Фикстура HTML страницы документа только с ecp PDF ─────────────────────────
|
||
_DOC_HTML_ECP_ONLY = """
|
||
<!DOCTYPE html>
|
||
<html>
|
||
<body>
|
||
<h1>Постановление № 312-ПП</h1>
|
||
<p>
|
||
<a href="/media/pravo/ecp/312-ПП.pdf">Подписанный PDF</a>
|
||
</p>
|
||
</body>
|
||
</html>
|
||
"""
|
||
|
||
# ── Фикстура HTML страницы документа без PDF ──────────────────────────────────
|
||
_DOC_HTML_NO_PDF = """
|
||
<!DOCTYPE html>
|
||
<html>
|
||
<body>
|
||
<h1>Постановление без PDF</h1>
|
||
<p>Текст постановления размещён ниже.</p>
|
||
</body>
|
||
</html>
|
||
"""
|
||
|
||
|
||
class TestLawTypeConstant:
|
||
def test_postanovlenie_is_numeric_string(self) -> None:
|
||
"""Числовой код — сервер игнорирует текстовые значения типа «Постановление»."""
|
||
assert _LAW_TYPE_POSTANOVLENIE == "1"
|
||
assert _LAW_TYPE_POSTANOVLENIE.isdigit()
|
||
|
||
|
||
class TestParseSearchHtml:
|
||
def test_extracts_doc_ids_and_titles(self) -> None:
|
||
results = _parse_search_html(_SEARCH_HTML_FIXTURE)
|
||
ids = [r["doc_id"] for r in results]
|
||
assert "35496" in ids
|
||
assert "34812" in ids
|
||
|
||
def test_ignores_non_numeric_paths(self) -> None:
|
||
results = _parse_search_html(_SEARCH_HTML_FIXTURE)
|
||
ids = [r["doc_id"] for r in results]
|
||
assert "some-slug" not in ids
|
||
|
||
def test_deduplicates_doc_ids(self) -> None:
|
||
results = _parse_search_html(_SEARCH_HTML_FIXTURE)
|
||
ids = [r["doc_id"] for r in results]
|
||
assert ids.count("35496") == 1
|
||
|
||
def test_ignores_category_links(self) -> None:
|
||
results = _parse_search_html(_SEARCH_HTML_FIXTURE)
|
||
# /category/law/ не числовой id — не попадает.
|
||
ids = [r["doc_id"] for r in results]
|
||
assert len(ids) == 2
|
||
|
||
def test_url_is_absolute(self) -> None:
|
||
results = _parse_search_html(_SEARCH_HTML_FIXTURE)
|
||
for r in results:
|
||
assert r["url"].startswith("https://www.pravo.gov66.ru/")
|
||
|
||
def test_title_extracted(self) -> None:
|
||
results = _parse_search_html(_SEARCH_HTML_FIXTURE)
|
||
doc = next(r for r in results if r["doc_id"] == "35496")
|
||
assert "изъятии" in doc["title"].lower()
|
||
|
||
def test_empty_html_returns_empty(self) -> None:
|
||
assert _parse_search_html("") == []
|
||
|
||
def test_html_no_doc_links_returns_empty(self) -> None:
|
||
html = "<html><body><a href='/category/'>Cat</a></body></html>"
|
||
assert _parse_search_html(html) == []
|
||
|
||
|
||
class TestExtractPdfUrl:
|
||
def test_prefers_non_ecp_pdf(self) -> None:
|
||
url = _extract_pdf_url(_DOC_HTML_WITH_PDF)
|
||
assert url is not None
|
||
assert "ecp" not in url
|
||
assert url.endswith(".pdf")
|
||
|
||
def test_non_ecp_pdf_is_absolute(self) -> None:
|
||
url = _extract_pdf_url(_DOC_HTML_WITH_PDF)
|
||
assert url is not None
|
||
assert url.startswith("https://www.pravo.gov66.ru/media/pravo/")
|
||
|
||
def test_falls_back_to_ecp_when_no_regular_pdf(self) -> None:
|
||
url = _extract_pdf_url(_DOC_HTML_ECP_ONLY)
|
||
assert url is not None
|
||
assert "ecp" in url
|
||
assert url.endswith(".pdf")
|
||
|
||
def test_returns_none_when_no_pdf(self) -> None:
|
||
url = _extract_pdf_url(_DOC_HTML_NO_PDF)
|
||
assert url is None
|
||
|
||
def test_ecp_fallback_is_absolute(self) -> None:
|
||
url = _extract_pdf_url(_DOC_HTML_ECP_ONLY)
|
||
assert url is not None
|
||
assert url.startswith("https://www.pravo.gov66.ru/")
|
||
|
||
|
||
# ── Тесты pacing/backoff (#2445 D4) — offline через httpx.MockTransport ───────
|
||
|
||
|
||
def _client_with_responses(responses: list[httpx.Response]) -> httpx.Client:
|
||
"""httpx.Client на MockTransport, отдающий responses по очереди (без реальной сети)."""
|
||
calls = iter(responses)
|
||
|
||
def _handler(request: httpx.Request) -> httpx.Response:
|
||
return next(calls)
|
||
|
||
return httpx.Client(transport=httpx.MockTransport(_handler))
|
||
|
||
|
||
class TestGetWithBackoff:
|
||
def test_sleeps_before_request(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||
"""Пейсинг: time.sleep вызывается ПЕРЕД запросом, даже на первой/единственной попытке."""
|
||
sleeps: list[float] = []
|
||
monkeypatch.setattr(
|
||
"app.services.scrapers.pravo_gov66_client.time.sleep",
|
||
lambda s: sleeps.append(s),
|
||
)
|
||
client = _client_with_responses([httpx.Response(200, text="ok")])
|
||
_get_with_backoff(client, "https://www.pravo.gov66.ru/1/")
|
||
assert len(sleeps) == 1
|
||
|
||
def test_paces_between_successive_fetches(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||
"""Несколько последовательных fetch → time.sleep между КАЖДЫМ (не только ретраи)."""
|
||
sleeps: list[float] = []
|
||
monkeypatch.setattr(
|
||
"app.services.scrapers.pravo_gov66_client.time.sleep",
|
||
lambda s: sleeps.append(s),
|
||
)
|
||
client = _client_with_responses(
|
||
[
|
||
httpx.Response(200, text="a"),
|
||
httpx.Response(200, text="b"),
|
||
httpx.Response(200, text="c"),
|
||
]
|
||
)
|
||
for i in range(3):
|
||
_get_with_backoff(client, f"https://www.pravo.gov66.ru/{i}/")
|
||
assert len(sleeps) == 3
|
||
|
||
def test_retries_on_429_then_succeeds(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||
"""429 → backoff sleep + ретрай → 200 успешно, без исключения."""
|
||
sleeps: list[float] = []
|
||
monkeypatch.setattr(
|
||
"app.services.scrapers.pravo_gov66_client.time.sleep",
|
||
lambda s: sleeps.append(s),
|
||
)
|
||
client = _client_with_responses(
|
||
[httpx.Response(429, text="rate limited"), httpx.Response(200, text="ok")]
|
||
)
|
||
resp = _get_with_backoff(client, "https://www.pravo.gov66.ru/1/")
|
||
assert resp.status_code == 200
|
||
# 1 pacing-sleep перед 1й попыткой + 1 backoff-sleep перед ретраем + 1 pacing перед 2й.
|
||
assert len(sleeps) >= 2
|
||
|
||
def test_retries_on_5xx_then_succeeds(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||
sleeps: list[float] = []
|
||
monkeypatch.setattr(
|
||
"app.services.scrapers.pravo_gov66_client.time.sleep",
|
||
lambda s: sleeps.append(s),
|
||
)
|
||
client = _client_with_responses(
|
||
[httpx.Response(503, text="unavailable"), httpx.Response(200, text="ok")]
|
||
)
|
||
resp = _get_with_backoff(client, "https://www.pravo.gov66.ru/1/")
|
||
assert resp.status_code == 200
|
||
|
||
def test_raises_after_exhausting_retries_on_429(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||
monkeypatch.setattr("app.services.scrapers.pravo_gov66_client.time.sleep", lambda s: None)
|
||
# _MAX_RETRIES=2 → до 3 попыток, все 429.
|
||
client = _client_with_responses([httpx.Response(429, text="x") for _ in range(3)])
|
||
with pytest.raises(httpx.HTTPStatusError):
|
||
_get_with_backoff(client, "https://www.pravo.gov66.ru/1/")
|
||
|
||
def test_4xx_other_than_429_raises_immediately_without_retry(
|
||
self, monkeypatch: pytest.MonkeyPatch
|
||
) -> None:
|
||
"""404 не ретраится — сразу raise_for_status(), задача не тратит время на 3 попытки."""
|
||
sleeps: list[float] = []
|
||
monkeypatch.setattr(
|
||
"app.services.scrapers.pravo_gov66_client.time.sleep",
|
||
lambda s: sleeps.append(s),
|
||
)
|
||
client = _client_with_responses([httpx.Response(404, text="not found")])
|
||
with pytest.raises(httpx.HTTPStatusError):
|
||
_get_with_backoff(client, "https://www.pravo.gov66.ru/1/")
|
||
# Только 1 pacing-sleep перед единственной попыткой, никаких backoff-ретраев.
|
||
assert len(sleeps) == 1
|