"""Тесты ingest_reservations (#1062) — offline, без живой сети и реальной БД. Мокаем: pravo_gov66_client (search/get_pdf_url/fetch_pdf) + extract_from_pdf + SessionLocal. Проверяем: фильтрацию заголовков, дедупликацию doc_id, upsert-параметры, graceful на ошибках. """ from __future__ import annotations from contextlib import contextmanager from datetime import date from typing import Any from app.services.scrapers.page_reservation_parser import ReservationRecord from app.workers.tasks import reservation_ingest # ── Двойники ────────────────────────────────────────────────────────────────── class _FakeDB: """Session-двойник: пишет execute-вызовы, begin_nested как no-op CM.""" def __init__(self) -> None: self.executed: list[tuple[Any, dict[str, Any] | None]] = [] self.commit_count = 0 self.closed = False @contextmanager def begin_nested(self) -> Any: yield def execute(self, sql: Any, params: dict[str, Any] | None = None) -> Any: self.executed.append((sql, params)) return None def commit(self) -> None: self.commit_count += 1 def rollback(self) -> None: pass def close(self) -> None: self.closed = True # ── Фикстуры ────────────────────────────────────────────────────────────────── _SAMPLE_RECORDS = [ ReservationRecord( cad_num="66:41:0101001:1", reservation_kind="изъятие", act_number="509-ПП", act_date=date(2024, 3, 12), purpose="реконструкция ул. Татищева", raw_excerpt="...участок 66:41:0101001:1...", ), ReservationRecord( cad_num="66:41:0101001:2", reservation_kind="изъятие", act_number="509-ПП", act_date=date(2024, 3, 12), purpose="реконструкция ул. Татищева", raw_excerpt=None, ), ] _RELEVANT_HITS = [ { "doc_id": "35496", "title": "Об изъятии земельных участков Екатеринбург", "url": "https://www.pravo.gov66.ru/35496/", }, { "doc_id": "34812", "title": "О резервировании земельных участков Свердловской области", "url": "https://www.pravo.gov66.ru/34812/", }, ] _IRRELEVANT_HIT = { "doc_id": "99999", "title": "О бюджете на 2024 год", "url": "https://www.pravo.gov66.ru/99999/", } _PDF_BYTES = b"%PDF-1.4 fake content" # ── Helpers ─────────────────────────────────────────────────────────────────── def _patch_enumerate( monkeypatch: Any, db: _FakeDB, hits: list[dict[str, Any]], pdf_url: str | None = "https://www.pravo.gov66.ru/media/pravo/509-ПП_abc.pdf", pdf_bytes: bytes = _PDF_BYTES, records: list[ReservationRecord] = _SAMPLE_RECORDS, ) -> None: """Патчит все зависимости ingest_reservations для enumerate-режима.""" monkeypatch.setattr(reservation_ingest, "SessionLocal", lambda: db) monkeypatch.setattr(reservation_ingest, "search_documents", lambda q, **kw: hits) monkeypatch.setattr(reservation_ingest, "get_document_pdf_url", lambda doc_id: pdf_url) monkeypatch.setattr(reservation_ingest, "fetch_pdf", lambda url: pdf_bytes) monkeypatch.setattr(reservation_ingest, "extract_from_pdf", lambda b: records) # ── Тесты фильтрации заголовков ─────────────────────────────────────────────── class TestIsRelevantTitle: def test_relevant_izyatie_ekb(self) -> None: assert reservation_ingest._is_relevant_title( "Об изъятии земельных участков в г. Екатеринбурге" ) def test_relevant_rezervirovanie_sverdlov(self) -> None: assert reservation_ingest._is_relevant_title( "О резервировании земельных участков Свердловской области" ) def test_irrelevant_no_geo(self) -> None: assert not reservation_ingest._is_relevant_title( "Об изъятии земельных участков в г. Нижний Тагил" ) def test_irrelevant_no_operation(self) -> None: assert not reservation_ingest._is_relevant_title( "О бюджете Свердловской области на 2024 год" ) def test_irrelevant_empty(self) -> None: assert not reservation_ingest._is_relevant_title("") # ── Тесты enumerate-режима ──────────────────────────────────────────────────── class TestIngestEnumerate: def test_returns_docs_and_parcels(self, monkeypatch: Any) -> None: db = _FakeDB() _patch_enumerate(monkeypatch, db, hits=_RELEVANT_HITS) result = reservation_ingest.ingest_reservations() assert result["docs"] >= 1 assert result["parcels"] >= 1 def test_filters_irrelevant_titles(self, monkeypatch: Any) -> None: """Документ с нерелевантным заголовком не должен попасть в ingest.""" db = _FakeDB() irrelevant_only = [_IRRELEVANT_HIT] _patch_enumerate(monkeypatch, db, hits=irrelevant_only) result = reservation_ingest.ingest_reservations() assert result["docs"] == 0 assert result["parcels"] == 0 assert db.executed == [] def test_deduplicates_doc_id_across_queries(self, monkeypatch: Any) -> None: """Один и тот же doc_id из двух запросов инжестируется один раз.""" db = _FakeDB() same_hit = _RELEVANT_HITS[:1] # один хит call_count = [0] def _search_twice(q: str, **kw: Any) -> list[dict[str, Any]]: call_count[0] += 1 return same_hit # оба запроса возвращают один и тот же doc_id monkeypatch.setattr(reservation_ingest, "SessionLocal", lambda: db) monkeypatch.setattr(reservation_ingest, "search_documents", _search_twice) monkeypatch.setattr( reservation_ingest, "get_document_pdf_url", lambda doc_id: "https://www.pravo.gov66.ru/media/pravo/test.pdf", ) monkeypatch.setattr(reservation_ingest, "fetch_pdf", lambda url: _PDF_BYTES) monkeypatch.setattr(reservation_ingest, "extract_from_pdf", lambda b: _SAMPLE_RECORDS[:1]) result = reservation_ingest.ingest_reservations() # 2 запроса, но 1 уникальный doc_id → fetched=1 assert result["docs"] == 1 def test_skips_doc_when_no_pdf_url(self, monkeypatch: Any) -> None: db = _FakeDB() _patch_enumerate(monkeypatch, db, hits=_RELEVANT_HITS[:1], pdf_url=None) result = reservation_ingest.ingest_reservations() assert result["docs"] == 0 assert result["parcels"] == 0 def test_skips_doc_when_pdf_fetch_fails(self, monkeypatch: Any) -> None: db = _FakeDB() monkeypatch.setattr(reservation_ingest, "SessionLocal", lambda: db) monkeypatch.setattr( reservation_ingest, "search_documents", lambda q, **kw: _RELEVANT_HITS[:1] ) monkeypatch.setattr( reservation_ingest, "get_document_pdf_url", lambda doc_id: "https://www.pravo.gov66.ru/media/pravo/test.pdf", ) monkeypatch.setattr( reservation_ingest, "fetch_pdf", lambda url: (_ for _ in ()).throw(OSError("net err")) ) monkeypatch.setattr(reservation_ingest, "extract_from_pdf", lambda b: _SAMPLE_RECORDS[:1]) result = reservation_ingest.ingest_reservations() assert result["docs"] == 0 def test_upsert_params_contain_cad_num(self, monkeypatch: Any) -> None: db = _FakeDB() _patch_enumerate(monkeypatch, db, hits=_RELEVANT_HITS[:1], records=_SAMPLE_RECORDS[:1]) reservation_ingest.ingest_reservations() assert db.executed, "должен быть хотя бы один execute" _, params = db.executed[0] assert params is not None assert params["cad_num"] == "66:41:0101001:1" assert params["source"] == "pravo_gov66" def test_db_closed_after_run(self, monkeypatch: Any) -> None: db = _FakeDB() _patch_enumerate(monkeypatch, db, hits=[]) reservation_ingest.ingest_reservations() assert db.closed def test_skips_empty_records(self, monkeypatch: Any) -> None: """PDF без кад-номеров → ни одного execute.""" db = _FakeDB() _patch_enumerate(monkeypatch, db, hits=_RELEVANT_HITS[:1], records=[]) result = reservation_ingest.ingest_reservations() assert result["parcels"] == 0 assert db.executed == [] # ── Тесты explicit-docs (back-compat) ───────────────────────────────────────── class TestIngestExplicitDocs: def test_explicit_docs_bypass_enumerate(self, monkeypatch: Any) -> None: """docs=... → enumerate не вызывается, search_documents не дёргается.""" db = _FakeDB() monkeypatch.setattr(reservation_ingest, "SessionLocal", lambda: db) def _should_not_call(*a: Any, **kw: Any) -> list[dict[str, Any]]: raise AssertionError("enumerate не должен вызываться") monkeypatch.setattr(reservation_ingest, "search_documents", _should_not_call) monkeypatch.setattr(reservation_ingest, "fetch_pdf", lambda url: _PDF_BYTES) monkeypatch.setattr(reservation_ingest, "extract_from_pdf", lambda b: _SAMPLE_RECORDS[:1]) explicit = [ { "url": "https://www.pravo.gov66.ru/media/pravo/509-ПП_abc.pdf", "act_number": "509-ПП", "basis_act": "Постановление № 509-ПП", "purpose": "реконструкция", "kind": "изъятие", } ] result = reservation_ingest.ingest_reservations(docs=explicit) assert result["docs"] == 1 assert result["parcels"] == 1 def test_explicit_docs_upsert_uses_doc_kind(self, monkeypatch: Any) -> None: """Если rec.reservation_kind пуст, берётся kind из doc.""" db = _FakeDB() monkeypatch.setattr(reservation_ingest, "SessionLocal", lambda: db) monkeypatch.setattr(reservation_ingest, "fetch_pdf", lambda url: _PDF_BYTES) no_kind_record = ReservationRecord( cad_num="66:41:0000001:1", reservation_kind="", # пустой — должен взяться из doc act_number=None, act_date=None, purpose=None, raw_excerpt=None, ) monkeypatch.setattr(reservation_ingest, "extract_from_pdf", lambda b: [no_kind_record]) explicit = [ { "url": "https://x.ru/test.pdf", "act_number": "1-ПП", "basis_act": "Постановление", "purpose": None, "kind": "резервирование", } ] reservation_ingest.ingest_reservations(docs=explicit) assert db.executed _, params = db.executed[0] assert params is not None assert params["reservation_kind"] == "резервирование"