bot-backend
d1fa450f48
Some checks failed
Deploy / build-backend (push) Blocked by required conditions
Deploy / build-worker (push) Blocked by required conditions
Deploy / build-frontend (push) Blocked by required conditions
Deploy / deploy (push) Blocked by required conditions
Deploy / changes (push) Has been cancelled
339 lines
15 KiB
Python
339 lines
15 KiB
Python
"""Тесты ingest_reservations (#1062) — offline, без живой сети и реальной БД.
|
||
|
||
Мокаем: pravo_gov66_client (search/get_pdf_url/fetch_pdf) + extract_from_pdf + SessionLocal.
|
||
Проверяем: фильтрацию заголовков, дедупликацию doc_id, upsert-параметры, graceful на ошибках,
|
||
session lifecycle (#2445 D4: сессия НЕ держится через весь цикл — открывается заново на
|
||
каждый документ) и pacing к pravo.gov66.ru (time.sleep между запросами клиента).
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
from contextlib import contextmanager
|
||
from datetime import date
|
||
from typing import Any
|
||
|
||
from app.services.scrapers.page_reservation_parser import ReservationRecord
|
||
from app.workers.tasks import reservation_ingest
|
||
|
||
# ── Двойники ──────────────────────────────────────────────────────────────────
|
||
|
||
|
||
class _FakeDB:
|
||
"""Session-двойник: пишет execute-вызовы, begin_nested как no-op CM."""
|
||
|
||
def __init__(self) -> None:
|
||
self.executed: list[tuple[Any, dict[str, Any] | None]] = []
|
||
self.commit_count = 0
|
||
self.closed = False
|
||
|
||
@contextmanager
|
||
def begin_nested(self) -> Any:
|
||
yield
|
||
|
||
def execute(self, sql: Any, params: dict[str, Any] | None = None) -> Any:
|
||
self.executed.append((sql, params))
|
||
return None
|
||
|
||
def commit(self) -> None:
|
||
self.commit_count += 1
|
||
|
||
def rollback(self) -> None:
|
||
pass
|
||
|
||
def close(self) -> None:
|
||
self.closed = True
|
||
|
||
|
||
class _FakeSessionFactory:
|
||
"""SessionLocal-двойник: считает, сколько раз вызывалась фабрика (open) и сколько
|
||
раз каждая выданная сессия была закрыта СРАЗУ (до следующего open) — доказательство,
|
||
что сессия не держится открытой через весь цикл, а открывается/закрывается per-item.
|
||
"""
|
||
|
||
def __init__(self) -> None:
|
||
self.sessions: list[_FakeDB] = []
|
||
# True если на момент следующего SessionLocal() предыдущая сессия уже была closed.
|
||
self.closed_before_next_open: list[bool] = []
|
||
|
||
def __call__(self) -> _FakeDB:
|
||
if self.sessions:
|
||
self.closed_before_next_open.append(self.sessions[-1].closed)
|
||
db = _FakeDB()
|
||
self.sessions.append(db)
|
||
return db
|
||
|
||
|
||
# ── Фикстуры ──────────────────────────────────────────────────────────────────
|
||
|
||
_SAMPLE_RECORDS = [
|
||
ReservationRecord(
|
||
cad_num="66:41:0101001:1",
|
||
reservation_kind="изъятие",
|
||
act_number="509-ПП",
|
||
act_date=date(2024, 3, 12),
|
||
purpose="реконструкция ул. Татищева",
|
||
raw_excerpt="...участок 66:41:0101001:1...",
|
||
),
|
||
ReservationRecord(
|
||
cad_num="66:41:0101001:2",
|
||
reservation_kind="изъятие",
|
||
act_number="509-ПП",
|
||
act_date=date(2024, 3, 12),
|
||
purpose="реконструкция ул. Татищева",
|
||
raw_excerpt=None,
|
||
),
|
||
]
|
||
|
||
_RELEVANT_HITS = [
|
||
{
|
||
"doc_id": "35496",
|
||
"title": "Об изъятии земельных участков Екатеринбург",
|
||
"url": "https://www.pravo.gov66.ru/35496/",
|
||
},
|
||
{
|
||
"doc_id": "34812",
|
||
"title": "О резервировании земельных участков Свердловской области",
|
||
"url": "https://www.pravo.gov66.ru/34812/",
|
||
},
|
||
]
|
||
|
||
_IRRELEVANT_HIT = {
|
||
"doc_id": "99999",
|
||
"title": "О бюджете на 2024 год",
|
||
"url": "https://www.pravo.gov66.ru/99999/",
|
||
}
|
||
|
||
_PDF_BYTES = b"%PDF-1.4 fake content"
|
||
|
||
|
||
# ── Helpers ───────────────────────────────────────────────────────────────────
|
||
|
||
|
||
def _patch_enumerate(
|
||
monkeypatch: Any,
|
||
db: _FakeDB,
|
||
hits: list[dict[str, Any]],
|
||
pdf_url: str | None = "https://www.pravo.gov66.ru/media/pravo/509-ПП_abc.pdf",
|
||
pdf_bytes: bytes = _PDF_BYTES,
|
||
records: list[ReservationRecord] = _SAMPLE_RECORDS,
|
||
) -> None:
|
||
"""Патчит все зависимости ingest_reservations для enumerate-режима."""
|
||
monkeypatch.setattr(reservation_ingest, "SessionLocal", lambda: db)
|
||
monkeypatch.setattr(reservation_ingest, "search_documents", lambda q, **kw: hits)
|
||
monkeypatch.setattr(reservation_ingest, "get_document_pdf_url", lambda doc_id: pdf_url)
|
||
monkeypatch.setattr(reservation_ingest, "fetch_pdf", lambda url: pdf_bytes)
|
||
monkeypatch.setattr(reservation_ingest, "extract_from_pdf", lambda b: records)
|
||
|
||
|
||
# ── Тесты фильтрации заголовков ───────────────────────────────────────────────
|
||
|
||
|
||
class TestIsRelevantTitle:
|
||
def test_relevant_izyatie_ekb(self) -> None:
|
||
assert reservation_ingest._is_relevant_title(
|
||
"Об изъятии земельных участков в г. Екатеринбурге"
|
||
)
|
||
|
||
def test_relevant_rezervirovanie_sverdlov(self) -> None:
|
||
assert reservation_ingest._is_relevant_title(
|
||
"О резервировании земельных участков Свердловской области"
|
||
)
|
||
|
||
def test_irrelevant_no_geo(self) -> None:
|
||
assert not reservation_ingest._is_relevant_title(
|
||
"Об изъятии земельных участков в г. Нижний Тагил"
|
||
)
|
||
|
||
def test_irrelevant_no_operation(self) -> None:
|
||
assert not reservation_ingest._is_relevant_title(
|
||
"О бюджете Свердловской области на 2024 год"
|
||
)
|
||
|
||
def test_irrelevant_empty(self) -> None:
|
||
assert not reservation_ingest._is_relevant_title("")
|
||
|
||
|
||
# ── Тесты enumerate-режима ────────────────────────────────────────────────────
|
||
|
||
|
||
class TestIngestEnumerate:
|
||
def test_returns_docs_and_parcels(self, monkeypatch: Any) -> None:
|
||
db = _FakeDB()
|
||
_patch_enumerate(monkeypatch, db, hits=_RELEVANT_HITS)
|
||
result = reservation_ingest.ingest_reservations()
|
||
assert result["docs"] >= 1
|
||
assert result["parcels"] >= 1
|
||
|
||
def test_filters_irrelevant_titles(self, monkeypatch: Any) -> None:
|
||
"""Документ с нерелевантным заголовком не должен попасть в ingest."""
|
||
db = _FakeDB()
|
||
irrelevant_only = [_IRRELEVANT_HIT]
|
||
_patch_enumerate(monkeypatch, db, hits=irrelevant_only)
|
||
result = reservation_ingest.ingest_reservations()
|
||
assert result["docs"] == 0
|
||
assert result["parcels"] == 0
|
||
assert db.executed == []
|
||
|
||
def test_deduplicates_doc_id_across_queries(self, monkeypatch: Any) -> None:
|
||
"""Один и тот же doc_id из двух запросов инжестируется один раз."""
|
||
db = _FakeDB()
|
||
same_hit = _RELEVANT_HITS[:1] # один хит
|
||
|
||
call_count = [0]
|
||
|
||
def _search_twice(q: str, **kw: Any) -> list[dict[str, Any]]:
|
||
call_count[0] += 1
|
||
return same_hit # оба запроса возвращают один и тот же doc_id
|
||
|
||
monkeypatch.setattr(reservation_ingest, "SessionLocal", lambda: db)
|
||
monkeypatch.setattr(reservation_ingest, "search_documents", _search_twice)
|
||
monkeypatch.setattr(
|
||
reservation_ingest,
|
||
"get_document_pdf_url",
|
||
lambda doc_id: "https://www.pravo.gov66.ru/media/pravo/test.pdf",
|
||
)
|
||
monkeypatch.setattr(reservation_ingest, "fetch_pdf", lambda url: _PDF_BYTES)
|
||
monkeypatch.setattr(reservation_ingest, "extract_from_pdf", lambda b: _SAMPLE_RECORDS[:1])
|
||
|
||
result = reservation_ingest.ingest_reservations()
|
||
# 2 запроса, но 1 уникальный doc_id → fetched=1
|
||
assert result["docs"] == 1
|
||
|
||
def test_skips_doc_when_no_pdf_url(self, monkeypatch: Any) -> None:
|
||
db = _FakeDB()
|
||
_patch_enumerate(monkeypatch, db, hits=_RELEVANT_HITS[:1], pdf_url=None)
|
||
result = reservation_ingest.ingest_reservations()
|
||
assert result["docs"] == 0
|
||
assert result["parcels"] == 0
|
||
|
||
def test_skips_doc_when_pdf_fetch_fails(self, monkeypatch: Any) -> None:
|
||
db = _FakeDB()
|
||
monkeypatch.setattr(reservation_ingest, "SessionLocal", lambda: db)
|
||
monkeypatch.setattr(
|
||
reservation_ingest, "search_documents", lambda q, **kw: _RELEVANT_HITS[:1]
|
||
)
|
||
monkeypatch.setattr(
|
||
reservation_ingest,
|
||
"get_document_pdf_url",
|
||
lambda doc_id: "https://www.pravo.gov66.ru/media/pravo/test.pdf",
|
||
)
|
||
monkeypatch.setattr(
|
||
reservation_ingest, "fetch_pdf", lambda url: (_ for _ in ()).throw(OSError("net err"))
|
||
)
|
||
monkeypatch.setattr(reservation_ingest, "extract_from_pdf", lambda b: _SAMPLE_RECORDS[:1])
|
||
result = reservation_ingest.ingest_reservations()
|
||
assert result["docs"] == 0
|
||
|
||
def test_upsert_params_contain_cad_num(self, monkeypatch: Any) -> None:
|
||
db = _FakeDB()
|
||
_patch_enumerate(monkeypatch, db, hits=_RELEVANT_HITS[:1], records=_SAMPLE_RECORDS[:1])
|
||
reservation_ingest.ingest_reservations()
|
||
assert db.executed, "должен быть хотя бы один execute"
|
||
_, params = db.executed[0]
|
||
assert params is not None
|
||
assert params["cad_num"] == "66:41:0101001:1"
|
||
assert params["source"] == "pravo_gov66"
|
||
|
||
def test_db_closed_after_each_doc(self, monkeypatch: Any) -> None:
|
||
"""Session lifecycle (#2445 D4): сессия открывается/закрывается НА КАЖДЫЙ документ,
|
||
не одна на весь прогон — доказывает, что она не держится через PDF-fetch цикла.
|
||
"""
|
||
factory = _FakeSessionFactory()
|
||
monkeypatch.setattr(reservation_ingest, "SessionLocal", factory)
|
||
monkeypatch.setattr(reservation_ingest, "search_documents", lambda q, **kw: _RELEVANT_HITS)
|
||
monkeypatch.setattr(
|
||
reservation_ingest,
|
||
"get_document_pdf_url",
|
||
lambda doc_id: "https://www.pravo.gov66.ru/media/pravo/test.pdf",
|
||
)
|
||
monkeypatch.setattr(reservation_ingest, "fetch_pdf", lambda url: _PDF_BYTES)
|
||
monkeypatch.setattr(reservation_ingest, "extract_from_pdf", lambda b: _SAMPLE_RECORDS[:1])
|
||
|
||
reservation_ingest.ingest_reservations()
|
||
|
||
# 2 релевантных хита → 2 документа с записями → SessionLocal() вызван 2 раза.
|
||
assert len(factory.sessions) == 2
|
||
# Каждая сессия должна быть closed к моменту следующего open (per-item lifecycle,
|
||
# не одна сессия держится открытой на весь цикл enumerate).
|
||
assert factory.closed_before_next_open == [True]
|
||
# Последняя сессия тоже закрыта после прогона.
|
||
assert factory.sessions[-1].closed
|
||
# И каждая закоммичена (per-item commit, идемпотентность сохранена).
|
||
assert all(s.commit_count == 1 for s in factory.sessions)
|
||
|
||
def test_no_session_opened_when_no_hits(self, monkeypatch: Any) -> None:
|
||
"""Нет хитов → нет документов → SessionLocal вообще не вызывается (нечего писать)."""
|
||
factory = _FakeSessionFactory()
|
||
monkeypatch.setattr(reservation_ingest, "SessionLocal", factory)
|
||
monkeypatch.setattr(reservation_ingest, "search_documents", lambda q, **kw: [])
|
||
reservation_ingest.ingest_reservations()
|
||
assert factory.sessions == []
|
||
|
||
def test_skips_empty_records(self, monkeypatch: Any) -> None:
|
||
"""PDF без кад-номеров → ни одного execute."""
|
||
db = _FakeDB()
|
||
_patch_enumerate(monkeypatch, db, hits=_RELEVANT_HITS[:1], records=[])
|
||
result = reservation_ingest.ingest_reservations()
|
||
assert result["parcels"] == 0
|
||
assert db.executed == []
|
||
|
||
|
||
# ── Тесты explicit-docs (back-compat) ─────────────────────────────────────────
|
||
|
||
|
||
class TestIngestExplicitDocs:
|
||
def test_explicit_docs_bypass_enumerate(self, monkeypatch: Any) -> None:
|
||
"""docs=... → enumerate не вызывается, search_documents не дёргается."""
|
||
db = _FakeDB()
|
||
monkeypatch.setattr(reservation_ingest, "SessionLocal", lambda: db)
|
||
|
||
def _should_not_call(*a: Any, **kw: Any) -> list[dict[str, Any]]:
|
||
raise AssertionError("enumerate не должен вызываться")
|
||
|
||
monkeypatch.setattr(reservation_ingest, "search_documents", _should_not_call)
|
||
monkeypatch.setattr(reservation_ingest, "fetch_pdf", lambda url: _PDF_BYTES)
|
||
monkeypatch.setattr(reservation_ingest, "extract_from_pdf", lambda b: _SAMPLE_RECORDS[:1])
|
||
|
||
explicit = [
|
||
{
|
||
"url": "https://www.pravo.gov66.ru/media/pravo/509-ПП_abc.pdf",
|
||
"act_number": "509-ПП",
|
||
"basis_act": "Постановление № 509-ПП",
|
||
"purpose": "реконструкция",
|
||
"kind": "изъятие",
|
||
}
|
||
]
|
||
result = reservation_ingest.ingest_reservations(docs=explicit)
|
||
assert result["docs"] == 1
|
||
assert result["parcels"] == 1
|
||
|
||
def test_explicit_docs_upsert_uses_doc_kind(self, monkeypatch: Any) -> None:
|
||
"""Если rec.reservation_kind пуст, берётся kind из doc."""
|
||
db = _FakeDB()
|
||
monkeypatch.setattr(reservation_ingest, "SessionLocal", lambda: db)
|
||
monkeypatch.setattr(reservation_ingest, "fetch_pdf", lambda url: _PDF_BYTES)
|
||
|
||
no_kind_record = ReservationRecord(
|
||
cad_num="66:41:0000001:1",
|
||
reservation_kind="", # пустой — должен взяться из doc
|
||
act_number=None,
|
||
act_date=None,
|
||
purpose=None,
|
||
raw_excerpt=None,
|
||
)
|
||
monkeypatch.setattr(reservation_ingest, "extract_from_pdf", lambda b: [no_kind_record])
|
||
|
||
explicit = [
|
||
{
|
||
"url": "https://x.ru/test.pdf",
|
||
"act_number": "1-ПП",
|
||
"basis_act": "Постановление",
|
||
"purpose": None,
|
||
"kind": "резервирование",
|
||
}
|
||
]
|
||
reservation_ingest.ingest_reservations(docs=explicit)
|
||
assert db.executed
|
||
_, params = db.executed[0]
|
||
assert params is not None
|
||
assert params["reservation_kind"] == "резервирование"
|