bot-backend
23f04af80f
All checks were successful
CI / changes (push) Successful in 7s
CI / frontend-tests (push) Has been skipped
CI / changes (pull_request) Successful in 9s
CI / frontend-tests (pull_request) Has been skipped
CI / backend-tests (push) Successful in 6m17s
CI / backend-tests (pull_request) Successful in 6m18s
- Новый клиент pravo_gov66_client: search_documents (GET + BeautifulSoup, doc_id/title/url), get_document_pdf_url (предпочитает /media/pravo/, fallback ecp), fetch_pdf (verify=False). - ingest_reservations переписан: enumerate по _RESERVATION_QUERIES 2-3 слова, фильтр заголовков (изъят|резерв + Екатеринбург|Свердлов), дедуп doc_id, SAVEPOINT per-row UPSERT в land_reservation; docs= override сохранён для back-compat. - 28 тестов offline (mock search HTML, mock doc page, mock client в task).
289 lines
12 KiB
Python
289 lines
12 KiB
Python
"""Тесты ingest_reservations (#1062) — offline, без живой сети и реальной БД.
|
||
|
||
Мокаем: pravo_gov66_client (search/get_pdf_url/fetch_pdf) + extract_from_pdf + SessionLocal.
|
||
Проверяем: фильтрацию заголовков, дедупликацию doc_id, upsert-параметры, graceful на ошибках.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
from contextlib import contextmanager
|
||
from datetime import date
|
||
from typing import Any
|
||
|
||
from app.services.scrapers.page_reservation_parser import ReservationRecord
|
||
from app.workers.tasks import reservation_ingest
|
||
|
||
# ── Двойники ──────────────────────────────────────────────────────────────────
|
||
|
||
|
||
class _FakeDB:
|
||
"""Session-двойник: пишет execute-вызовы, begin_nested как no-op CM."""
|
||
|
||
def __init__(self) -> None:
|
||
self.executed: list[tuple[Any, dict[str, Any] | None]] = []
|
||
self.commit_count = 0
|
||
self.closed = False
|
||
|
||
@contextmanager
|
||
def begin_nested(self) -> Any:
|
||
yield
|
||
|
||
def execute(self, sql: Any, params: dict[str, Any] | None = None) -> Any:
|
||
self.executed.append((sql, params))
|
||
return None
|
||
|
||
def commit(self) -> None:
|
||
self.commit_count += 1
|
||
|
||
def rollback(self) -> None:
|
||
pass
|
||
|
||
def close(self) -> None:
|
||
self.closed = True
|
||
|
||
|
||
# ── Фикстуры ──────────────────────────────────────────────────────────────────
|
||
|
||
_SAMPLE_RECORDS = [
|
||
ReservationRecord(
|
||
cad_num="66:41:0101001:1",
|
||
reservation_kind="изъятие",
|
||
act_number="509-ПП",
|
||
act_date=date(2024, 3, 12),
|
||
purpose="реконструкция ул. Татищева",
|
||
raw_excerpt="...участок 66:41:0101001:1...",
|
||
),
|
||
ReservationRecord(
|
||
cad_num="66:41:0101001:2",
|
||
reservation_kind="изъятие",
|
||
act_number="509-ПП",
|
||
act_date=date(2024, 3, 12),
|
||
purpose="реконструкция ул. Татищева",
|
||
raw_excerpt=None,
|
||
),
|
||
]
|
||
|
||
_RELEVANT_HITS = [
|
||
{
|
||
"doc_id": "35496",
|
||
"title": "Об изъятии земельных участков Екатеринбург",
|
||
"url": "https://www.pravo.gov66.ru/35496/",
|
||
},
|
||
{
|
||
"doc_id": "34812",
|
||
"title": "О резервировании земельных участков Свердловской области",
|
||
"url": "https://www.pravo.gov66.ru/34812/",
|
||
},
|
||
]
|
||
|
||
_IRRELEVANT_HIT = {
|
||
"doc_id": "99999",
|
||
"title": "О бюджете на 2024 год",
|
||
"url": "https://www.pravo.gov66.ru/99999/",
|
||
}
|
||
|
||
_PDF_BYTES = b"%PDF-1.4 fake content"
|
||
|
||
|
||
# ── Helpers ───────────────────────────────────────────────────────────────────
|
||
|
||
|
||
def _patch_enumerate(
|
||
monkeypatch: Any,
|
||
db: _FakeDB,
|
||
hits: list[dict[str, Any]],
|
||
pdf_url: str | None = "https://www.pravo.gov66.ru/media/pravo/509-ПП_abc.pdf",
|
||
pdf_bytes: bytes = _PDF_BYTES,
|
||
records: list[ReservationRecord] = _SAMPLE_RECORDS,
|
||
) -> None:
|
||
"""Патчит все зависимости ingest_reservations для enumerate-режима."""
|
||
monkeypatch.setattr(reservation_ingest, "SessionLocal", lambda: db)
|
||
monkeypatch.setattr(reservation_ingest, "search_documents", lambda q, **kw: hits)
|
||
monkeypatch.setattr(reservation_ingest, "get_document_pdf_url", lambda doc_id: pdf_url)
|
||
monkeypatch.setattr(reservation_ingest, "fetch_pdf", lambda url: pdf_bytes)
|
||
monkeypatch.setattr(reservation_ingest, "extract_from_pdf", lambda b: records)
|
||
|
||
|
||
# ── Тесты фильтрации заголовков ───────────────────────────────────────────────
|
||
|
||
|
||
class TestIsRelevantTitle:
|
||
def test_relevant_izyatie_ekb(self) -> None:
|
||
assert reservation_ingest._is_relevant_title(
|
||
"Об изъятии земельных участков в г. Екатеринбурге"
|
||
)
|
||
|
||
def test_relevant_rezervirovanie_sverdlov(self) -> None:
|
||
assert reservation_ingest._is_relevant_title(
|
||
"О резервировании земельных участков Свердловской области"
|
||
)
|
||
|
||
def test_irrelevant_no_geo(self) -> None:
|
||
assert not reservation_ingest._is_relevant_title(
|
||
"Об изъятии земельных участков в г. Нижний Тагил"
|
||
)
|
||
|
||
def test_irrelevant_no_operation(self) -> None:
|
||
assert not reservation_ingest._is_relevant_title(
|
||
"О бюджете Свердловской области на 2024 год"
|
||
)
|
||
|
||
def test_irrelevant_empty(self) -> None:
|
||
assert not reservation_ingest._is_relevant_title("")
|
||
|
||
|
||
# ── Тесты enumerate-режима ────────────────────────────────────────────────────
|
||
|
||
|
||
class TestIngestEnumerate:
|
||
def test_returns_docs_and_parcels(self, monkeypatch: Any) -> None:
|
||
db = _FakeDB()
|
||
_patch_enumerate(monkeypatch, db, hits=_RELEVANT_HITS)
|
||
result = reservation_ingest.ingest_reservations()
|
||
assert result["docs"] >= 1
|
||
assert result["parcels"] >= 1
|
||
|
||
def test_filters_irrelevant_titles(self, monkeypatch: Any) -> None:
|
||
"""Документ с нерелевантным заголовком не должен попасть в ingest."""
|
||
db = _FakeDB()
|
||
irrelevant_only = [_IRRELEVANT_HIT]
|
||
_patch_enumerate(monkeypatch, db, hits=irrelevant_only)
|
||
result = reservation_ingest.ingest_reservations()
|
||
assert result["docs"] == 0
|
||
assert result["parcels"] == 0
|
||
assert db.executed == []
|
||
|
||
def test_deduplicates_doc_id_across_queries(self, monkeypatch: Any) -> None:
|
||
"""Один и тот же doc_id из двух запросов инжестируется один раз."""
|
||
db = _FakeDB()
|
||
same_hit = _RELEVANT_HITS[:1] # один хит
|
||
|
||
call_count = [0]
|
||
|
||
def _search_twice(q: str, **kw: Any) -> list[dict[str, Any]]:
|
||
call_count[0] += 1
|
||
return same_hit # оба запроса возвращают один и тот же doc_id
|
||
|
||
monkeypatch.setattr(reservation_ingest, "SessionLocal", lambda: db)
|
||
monkeypatch.setattr(reservation_ingest, "search_documents", _search_twice)
|
||
monkeypatch.setattr(
|
||
reservation_ingest,
|
||
"get_document_pdf_url",
|
||
lambda doc_id: "https://www.pravo.gov66.ru/media/pravo/test.pdf",
|
||
)
|
||
monkeypatch.setattr(reservation_ingest, "fetch_pdf", lambda url: _PDF_BYTES)
|
||
monkeypatch.setattr(reservation_ingest, "extract_from_pdf", lambda b: _SAMPLE_RECORDS[:1])
|
||
|
||
result = reservation_ingest.ingest_reservations()
|
||
# 2 запроса, но 1 уникальный doc_id → fetched=1
|
||
assert result["docs"] == 1
|
||
|
||
def test_skips_doc_when_no_pdf_url(self, monkeypatch: Any) -> None:
|
||
db = _FakeDB()
|
||
_patch_enumerate(monkeypatch, db, hits=_RELEVANT_HITS[:1], pdf_url=None)
|
||
result = reservation_ingest.ingest_reservations()
|
||
assert result["docs"] == 0
|
||
assert result["parcels"] == 0
|
||
|
||
def test_skips_doc_when_pdf_fetch_fails(self, monkeypatch: Any) -> None:
|
||
db = _FakeDB()
|
||
monkeypatch.setattr(reservation_ingest, "SessionLocal", lambda: db)
|
||
monkeypatch.setattr(
|
||
reservation_ingest, "search_documents", lambda q, **kw: _RELEVANT_HITS[:1]
|
||
)
|
||
monkeypatch.setattr(
|
||
reservation_ingest,
|
||
"get_document_pdf_url",
|
||
lambda doc_id: "https://www.pravo.gov66.ru/media/pravo/test.pdf",
|
||
)
|
||
monkeypatch.setattr(
|
||
reservation_ingest, "fetch_pdf", lambda url: (_ for _ in ()).throw(OSError("net err"))
|
||
)
|
||
monkeypatch.setattr(reservation_ingest, "extract_from_pdf", lambda b: _SAMPLE_RECORDS[:1])
|
||
result = reservation_ingest.ingest_reservations()
|
||
assert result["docs"] == 0
|
||
|
||
def test_upsert_params_contain_cad_num(self, monkeypatch: Any) -> None:
|
||
db = _FakeDB()
|
||
_patch_enumerate(monkeypatch, db, hits=_RELEVANT_HITS[:1], records=_SAMPLE_RECORDS[:1])
|
||
reservation_ingest.ingest_reservations()
|
||
assert db.executed, "должен быть хотя бы один execute"
|
||
_, params = db.executed[0]
|
||
assert params is not None
|
||
assert params["cad_num"] == "66:41:0101001:1"
|
||
assert params["source"] == "pravo_gov66"
|
||
|
||
def test_db_closed_after_run(self, monkeypatch: Any) -> None:
|
||
db = _FakeDB()
|
||
_patch_enumerate(monkeypatch, db, hits=[])
|
||
reservation_ingest.ingest_reservations()
|
||
assert db.closed
|
||
|
||
def test_skips_empty_records(self, monkeypatch: Any) -> None:
|
||
"""PDF без кад-номеров → ни одного execute."""
|
||
db = _FakeDB()
|
||
_patch_enumerate(monkeypatch, db, hits=_RELEVANT_HITS[:1], records=[])
|
||
result = reservation_ingest.ingest_reservations()
|
||
assert result["parcels"] == 0
|
||
assert db.executed == []
|
||
|
||
|
||
# ── Тесты explicit-docs (back-compat) ─────────────────────────────────────────
|
||
|
||
|
||
class TestIngestExplicitDocs:
|
||
def test_explicit_docs_bypass_enumerate(self, monkeypatch: Any) -> None:
|
||
"""docs=... → enumerate не вызывается, search_documents не дёргается."""
|
||
db = _FakeDB()
|
||
monkeypatch.setattr(reservation_ingest, "SessionLocal", lambda: db)
|
||
|
||
def _should_not_call(*a: Any, **kw: Any) -> list[dict[str, Any]]:
|
||
raise AssertionError("enumerate не должен вызываться")
|
||
|
||
monkeypatch.setattr(reservation_ingest, "search_documents", _should_not_call)
|
||
monkeypatch.setattr(reservation_ingest, "fetch_pdf", lambda url: _PDF_BYTES)
|
||
monkeypatch.setattr(reservation_ingest, "extract_from_pdf", lambda b: _SAMPLE_RECORDS[:1])
|
||
|
||
explicit = [
|
||
{
|
||
"url": "https://www.pravo.gov66.ru/media/pravo/509-ПП_abc.pdf",
|
||
"act_number": "509-ПП",
|
||
"basis_act": "Постановление № 509-ПП",
|
||
"purpose": "реконструкция",
|
||
"kind": "изъятие",
|
||
}
|
||
]
|
||
result = reservation_ingest.ingest_reservations(docs=explicit)
|
||
assert result["docs"] == 1
|
||
assert result["parcels"] == 1
|
||
|
||
def test_explicit_docs_upsert_uses_doc_kind(self, monkeypatch: Any) -> None:
|
||
"""Если rec.reservation_kind пуст, берётся kind из doc."""
|
||
db = _FakeDB()
|
||
monkeypatch.setattr(reservation_ingest, "SessionLocal", lambda: db)
|
||
monkeypatch.setattr(reservation_ingest, "fetch_pdf", lambda url: _PDF_BYTES)
|
||
|
||
no_kind_record = ReservationRecord(
|
||
cad_num="66:41:0000001:1",
|
||
reservation_kind="", # пустой — должен взяться из doc
|
||
act_number=None,
|
||
act_date=None,
|
||
purpose=None,
|
||
raw_excerpt=None,
|
||
)
|
||
monkeypatch.setattr(reservation_ingest, "extract_from_pdf", lambda b: [no_kind_record])
|
||
|
||
explicit = [
|
||
{
|
||
"url": "https://x.ru/test.pdf",
|
||
"act_number": "1-ПП",
|
||
"basis_act": "Постановление",
|
||
"purpose": None,
|
||
"kind": "резервирование",
|
||
}
|
||
]
|
||
reservation_ingest.ingest_reservations(docs=explicit)
|
||
assert db.executed
|
||
_, params = db.executed[0]
|
||
assert params is not None
|
||
assert params["reservation_kind"] == "резервирование"
|