"""Unit tests for rosreestr_poll service (#888). Coverage: (a) _next_quarter: boundary cases (Q4 → Q1 of next year, mid-year). (b) latest_loaded_quarter: derives correct (year, quarter) from a known deal_date. (c) rosreestr_dataset_url: cp1251 percent-encoding matches the real portal URL. (d) check_new_quarter_available — new /data-sets/ autoindex flow: - real application/zip + large Content-Length → True - soft-404 regression: file found but Content-Type text/html → False (this is the exact bug that broke the old /opendata/ flat-URL poller) - tiny Content-Length (placeholder/error file) → False - quarter folder missing from autoindex → False (not yet published) - dataset file missing from quarter folder → False - network error / timeout → False, never raises (e) poll_rosreestr_new_quarter orchestration: available=True/False end-to-end, no DB data fallback, network error propagation. """ from __future__ import annotations import os import sys from datetime import date from unittest.mock import AsyncMock, MagicMock, patch from urllib.parse import quote os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test") # WeasyPrint stub — not installed in CI without GTK _wp_mock = MagicMock() sys.modules.setdefault("weasyprint", _wp_mock) import pytest # noqa: E402 from app.services.rosreestr_poll import ( # noqa: E402 _next_quarter, _period_start_to_quarter, check_new_quarter_available, latest_loaded_quarter, poll_rosreestr_new_quarter, rosreestr_dataset_url, ) # --------------------------------------------------------------------------- # cp1251 href fixtures — built the same way the real portal encodes folder/file # names (percent-encoded cp1251 bytes), so tests exercise the actual decode path. # --------------------------------------------------------------------------- def _cp1251_href(name: str) -> str: return quote(name.encode("cp1251"), safe="") def _autoindex_html(*folder_names: str) -> str: """Build a fake Apache-autoindex-style HTML listing for /data-sets/.""" links = ['../'] for name in folder_names: href = _cp1251_href(name) + "/" links.append(f'{name}/') return "

" + "\n".join(links) + "

" def _folder_html(*file_names: str) -> str: """Build a fake Apache-autoindex-style HTML listing for a quarter folder.""" links = ['../'] for name in file_names: href = _cp1251_href(name) links.append(f'{name}') return "

" + "\n".join(links) + "

" _SDELKI_FILE_2026_Q1 = "dataset_СДЕЛКИ_r-r_01-92_y_2026_q_1.csv.zip" _KADASTR_FILE_2026_Q1 = "dataset_КАДАСТРСТОИМОСТЬ_r-r_01-92_y_2026_q_1.csv.zip" def _mock_response(*, status_code: int = 200, text: str = "", headers: dict | None = None): resp = MagicMock() resp.status_code = status_code resp.text = text resp.headers = headers or {} return resp # --------------------------------------------------------------------------- # (a) _next_quarter # --------------------------------------------------------------------------- def test_next_quarter_mid_year() -> None: assert _next_quarter(2026, 1) == (2026, 2) assert _next_quarter(2026, 2) == (2026, 3) assert _next_quarter(2026, 3) == (2026, 4) def test_next_quarter_year_rollover() -> None: assert _next_quarter(2026, 4) == (2027, 1) assert _next_quarter(2024, 4) == (2025, 1) def test_period_start_to_quarter_from_date() -> None: assert _period_start_to_quarter(date(2026, 1, 1)) == (2026, 1) assert _period_start_to_quarter(date(2026, 4, 1)) == (2026, 2) assert _period_start_to_quarter(date(2026, 7, 15)) == (2026, 3) assert _period_start_to_quarter(date(2026, 10, 31)) == (2026, 4) assert _period_start_to_quarter(date(2025, 12, 31)) == (2025, 4) def test_period_start_to_quarter_from_string() -> None: assert _period_start_to_quarter("2026-01-15") == (2026, 1) assert _period_start_to_quarter("2026-07-01") == (2026, 3) # --------------------------------------------------------------------------- # (c) rosreestr_dataset_url — cp1251 percent-encoding must match the real portal. # --------------------------------------------------------------------------- def test_rosreestr_dataset_url_matches_verified_real_encoding() -> None: """Encoding must byte-match the live-verified rosreestr.gov.ru/data-sets/ URL.""" url = rosreestr_dataset_url(2026, 1) assert url == ( "https://rosreestr.gov.ru/data-sets/" "1%20%EA%E2%E0%F0%F2%E0%EB%202026%E3./" "dataset_%D1%C4%C5%CB%CA%C8_r-r_01-92_y_2026_q_1.csv.zip" ) def test_rosreestr_dataset_url_contains_year_quarter() -> None: url = rosreestr_dataset_url(2026, 2) assert "2026" in url assert "q_2" in url # --------------------------------------------------------------------------- # Fake DB helpers # --------------------------------------------------------------------------- class _FakeRow: """Mimics a DB row with __getitem__.""" def __init__(self, val: object) -> None: self._val = val def __getitem__(self, idx: int) -> object: return self._val class _FakeDB: """Minimal fake Session returning a configurable deal_date via execute().fetchone().""" def __init__(self, max_deal_date: date | None) -> None: self._row = _FakeRow(max_deal_date) if max_deal_date is not None else None def execute(self, stmt: object, params: object = None) -> MagicMock: mock = MagicMock() mock.fetchone.return_value = self._row return mock def commit(self) -> None: pass # --------------------------------------------------------------------------- # (b) latest_loaded_quarter # --------------------------------------------------------------------------- def test_latest_loaded_quarter_returns_correct_year_quarter() -> None: """Q1 2026 deal_date → (2026, 1).""" db = _FakeDB(date(2026, 1, 31)) result = latest_loaded_quarter(db) # type: ignore[arg-type] assert result == (2026, 1) def test_latest_loaded_quarter_q4() -> None: """Dec deal_date → Q4.""" db = _FakeDB(date(2025, 12, 15)) result = latest_loaded_quarter(db) # type: ignore[arg-type] assert result == (2025, 4) def test_latest_loaded_quarter_returns_none_when_empty() -> None: """No rows → None.""" db = _FakeDB(None) result = latest_loaded_quarter(db) # type: ignore[arg-type] assert result is None # --------------------------------------------------------------------------- # (d) check_new_quarter_available — /data-sets/ autoindex flow # --------------------------------------------------------------------------- @pytest.mark.asyncio async def test_check_available_true_for_real_zip() -> None: """Folder found, file found, HEAD returns application/zip + large Content-Length → True.""" index_html = _autoindex_html("1 квартал 2026г.", "4 квартал 2025г.") folder_html = _folder_html(_SDELKI_FILE_2026_Q1, _KADASTR_FILE_2026_Q1) mock_client = AsyncMock() mock_client.get.side_effect = [ _mock_response(text=index_html), _mock_response(text=folder_html), ] mock_client.head.return_value = _mock_response( headers={"content-type": "application/zip", "content-length": "9500000"} ) result = await check_new_quarter_available(mock_client, 2026, 1) # type: ignore[arg-type] assert result is True assert mock_client.get.call_count == 2 assert mock_client.head.call_count == 1 @pytest.mark.asyncio async def test_check_available_false_soft_404_text_html() -> None: """Regression guard: HTTP 200 with Content-Type text/html (Bitrix soft-404) → False. This is the exact failure mode that broke the old /opendata/ flat-URL poller — it treated bare HTTP 200 as availability. Even if a matching file href is found, a non-zip Content-Type must never be treated as available. """ index_html = _autoindex_html("1 квартал 2026г.") folder_html = _folder_html(_SDELKI_FILE_2026_Q1) mock_client = AsyncMock() mock_client.get.side_effect = [ _mock_response(text=index_html), _mock_response(text=folder_html), ] mock_client.head.return_value = _mock_response( status_code=200, headers={"content-type": "text/html; charset=utf-8", "content-length": "1234"}, ) result = await check_new_quarter_available(mock_client, 2026, 1) # type: ignore[arg-type] assert result is False @pytest.mark.asyncio async def test_check_available_false_tiny_content_length() -> None: """application/zip but tiny Content-Length (placeholder/error file) → False.""" index_html = _autoindex_html("1 квартал 2026г.") folder_html = _folder_html(_SDELKI_FILE_2026_Q1) mock_client = AsyncMock() mock_client.get.side_effect = [ _mock_response(text=index_html), _mock_response(text=folder_html), ] mock_client.head.return_value = _mock_response( headers={"content-type": "application/zip", "content-length": "512"} ) result = await check_new_quarter_available(mock_client, 2026, 1) # type: ignore[arg-type] assert result is False @pytest.mark.asyncio async def test_check_available_false_quarter_folder_missing() -> None: """Autoindex doesn't have the target quarter folder yet → False, no further GET/HEAD.""" index_html = _autoindex_html("4 квартал 2025г.") # 2026 Q1 not published yet mock_client = AsyncMock() mock_client.get.return_value = _mock_response(text=index_html) result = await check_new_quarter_available(mock_client, 2026, 1) # type: ignore[arg-type] assert result is False assert mock_client.get.call_count == 1 # never fetched a (non-existent) folder mock_client.head.assert_not_called() @pytest.mark.asyncio async def test_check_available_false_dataset_file_missing_in_folder() -> None: """Quarter folder exists but has no dataset_СДЕЛКИ file (only КАДАСТРСТОИМОСТЬ) → False.""" index_html = _autoindex_html("1 квартал 2026г.") folder_html = _folder_html(_KADASTR_FILE_2026_Q1) # no СДЕЛКИ file mock_client = AsyncMock() mock_client.get.side_effect = [ _mock_response(text=index_html), _mock_response(text=folder_html), ] result = await check_new_quarter_available(mock_client, 2026, 1) # type: ignore[arg-type] assert result is False mock_client.head.assert_not_called() @pytest.mark.asyncio async def test_check_available_false_on_index_404() -> None: """Unexpected non-200 on the base autoindex → False, no raise.""" mock_client = AsyncMock() mock_client.get.return_value = _mock_response(status_code=404, text="") result = await check_new_quarter_available(mock_client, 2026, 1) # type: ignore[arg-type] assert result is False @pytest.mark.asyncio async def test_check_new_quarter_available_timeout_returns_false() -> None: """Timeout → returns False, no exception.""" import httpx mock_client = AsyncMock() mock_client.get.side_effect = httpx.TimeoutException("timeout") result = await check_new_quarter_available(mock_client, 2026, 2) # type: ignore[arg-type] assert result is False @pytest.mark.asyncio async def test_check_new_quarter_available_network_error_returns_false() -> None: """Network error mid-flow (folder GET) → returns False, no exception.""" import httpx index_html = _autoindex_html("1 квартал 2026г.") mock_client = AsyncMock() mock_client.get.side_effect = [ _mock_response(text=index_html), httpx.ConnectError("simulated network error"), ] result = await check_new_quarter_available(mock_client, 2026, 1) # type: ignore[arg-type] assert result is False # --------------------------------------------------------------------------- # (e) poll_rosreestr_new_quarter — orchestration # --------------------------------------------------------------------------- def _patched_client_for(index_html: str, folder_html: str, head_response) -> AsyncMock: mock_client = AsyncMock() mock_client.get.side_effect = [ _mock_response(text=index_html), _mock_response(text=folder_html), ] mock_client.head.return_value = head_response mock_client.__aenter__ = AsyncMock(return_value=mock_client) mock_client.__aexit__ = AsyncMock(return_value=False) return mock_client @pytest.mark.asyncio async def test_poll_available_true_returns_correct_next_quarter() -> None: """If latest loaded is Q1 2026 and the Q2 2026 dataset is a real zip, available=True.""" db = _FakeDB(date(2026, 3, 31)) # latest = Q1 2026 → next = Q2 2026 index_html = _autoindex_html("2 квартал 2026г.") file_name = "dataset_СДЕЛКИ_r-r_01-92_y_2026_q_2.csv.zip" folder_html = _folder_html(file_name) head_response = _mock_response( headers={"content-type": "application/zip", "content-length": "9500000"} ) mock_client = _patched_client_for(index_html, folder_html, head_response) with patch("app.services.rosreestr_poll.httpx.AsyncClient", return_value=mock_client): result = await poll_rosreestr_new_quarter(db) # type: ignore[arg-type] assert result["available"] is True assert result["year"] == 2026 assert result["quarter"] == 2 assert result["latest_loaded_year"] == 2026 assert result["latest_loaded_quarter"] == 1 @pytest.mark.asyncio async def test_poll_available_false_when_folder_not_published() -> None: """Quarter folder not yet published (autoindex has no matching entry) → available=False.""" db = _FakeDB(date(2026, 3, 15)) # Q1 2026 → next = Q2 2026 index_html = _autoindex_html("1 квартал 2026г.") # Q2 2026 missing mock_client = AsyncMock() mock_client.get.return_value = _mock_response(text=index_html) mock_client.__aenter__ = AsyncMock(return_value=mock_client) mock_client.__aexit__ = AsyncMock(return_value=False) with patch("app.services.rosreestr_poll.httpx.AsyncClient", return_value=mock_client): result = await poll_rosreestr_new_quarter(db) # type: ignore[arg-type] assert result["available"] is False assert result["year"] == 2026 assert result["quarter"] == 2 @pytest.mark.asyncio async def test_poll_network_error_returns_false_no_raise() -> None: """Network error → available=False, no exception propagated.""" import httpx db = _FakeDB(date(2026, 1, 20)) # Q1 2026 mock_client = AsyncMock() mock_client.get.side_effect = httpx.ConnectError("simulated network error") mock_client.__aenter__ = AsyncMock(return_value=mock_client) mock_client.__aexit__ = AsyncMock(return_value=False) with patch("app.services.rosreestr_poll.httpx.AsyncClient", return_value=mock_client): # Must not raise result = await poll_rosreestr_new_quarter(db) # type: ignore[arg-type] assert result["available"] is False # --------------------------------------------------------------------------- # (f) No DB data → fallback baseline checks 2026Q1 # --------------------------------------------------------------------------- @pytest.mark.asyncio async def test_poll_fallback_when_no_db_data() -> None: """No rosreestr deals in DB → fallback 2025Q4 baseline, checks 2026Q1.""" db = _FakeDB(None) index_html = _autoindex_html("4 квартал 2025г.") # 2026 Q1 not yet published mock_client = AsyncMock() mock_client.get.return_value = _mock_response(text=index_html) mock_client.__aenter__ = AsyncMock(return_value=mock_client) mock_client.__aexit__ = AsyncMock(return_value=False) with patch("app.services.rosreestr_poll.httpx.AsyncClient", return_value=mock_client): result = await poll_rosreestr_new_quarter(db) # type: ignore[arg-type] # Fallback: 2025Q4 → check 2026Q1 assert result["year"] == 2026 assert result["quarter"] == 1 assert result["available"] is False # latest_loaded_year/quarter should be None (no DB data) assert result["latest_loaded_year"] is None assert result["latest_loaded_quarter"] is None