bot-backend
ed504cbbd9
All checks were successful
Deploy Trade-In / changes (push) Successful in 10s
Deploy Trade-In / build-frontend (push) Has been skipped
Deploy Trade-In / build-browser (push) Has been skipped
Deploy Trade-In / test (push) Successful in 45s
Deploy Trade-In / build-backend (push) Successful in 1m4s
Deploy Trade-In / deploy (push) Successful in 49s
442 lines
16 KiB
Python
442 lines
16 KiB
Python
"""Unit tests for rosreestr_poll service (#888).
|
||
|
||
Coverage:
|
||
(a) _next_quarter: boundary cases (Q4 → Q1 of next year, mid-year).
|
||
(b) latest_loaded_quarter: derives correct (year, quarter) from a known deal_date.
|
||
(c) rosreestr_dataset_url: cp1251 percent-encoding matches the real portal URL.
|
||
(d) check_new_quarter_available — new /data-sets/ autoindex flow:
|
||
- real application/zip + large Content-Length → True
|
||
- soft-404 regression: file found but Content-Type text/html → False
|
||
(this is the exact bug that broke the old /opendata/ flat-URL poller)
|
||
- tiny Content-Length (placeholder/error file) → False
|
||
- quarter folder missing from autoindex → False (not yet published)
|
||
- dataset file missing from quarter folder → False
|
||
- network error / timeout → False, never raises
|
||
(e) poll_rosreestr_new_quarter orchestration: available=True/False end-to-end,
|
||
no DB data fallback, network error propagation.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import os
|
||
import sys
|
||
from datetime import date
|
||
from unittest.mock import AsyncMock, MagicMock, patch
|
||
from urllib.parse import quote
|
||
|
||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
|
||
|
||
# WeasyPrint stub — not installed in CI without GTK
|
||
_wp_mock = MagicMock()
|
||
sys.modules.setdefault("weasyprint", _wp_mock)
|
||
|
||
import pytest # noqa: E402
|
||
|
||
from app.services.rosreestr_poll import ( # noqa: E402
|
||
_next_quarter,
|
||
_period_start_to_quarter,
|
||
check_new_quarter_available,
|
||
latest_loaded_quarter,
|
||
poll_rosreestr_new_quarter,
|
||
rosreestr_dataset_url,
|
||
)
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# cp1251 href fixtures — built the same way the real portal encodes folder/file
|
||
# names (percent-encoded cp1251 bytes), so tests exercise the actual decode path.
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def _cp1251_href(name: str) -> str:
|
||
return quote(name.encode("cp1251"), safe="")
|
||
|
||
|
||
def _autoindex_html(*folder_names: str) -> str:
|
||
"""Build a fake Apache-autoindex-style HTML listing for /data-sets/."""
|
||
links = ['<a href="../">../</a>']
|
||
for name in folder_names:
|
||
href = _cp1251_href(name) + "/"
|
||
links.append(f'<a href="{href}">{name}/</a>')
|
||
return "<html><body><pre>" + "\n".join(links) + "</pre></body></html>"
|
||
|
||
|
||
def _folder_html(*file_names: str) -> str:
|
||
"""Build a fake Apache-autoindex-style HTML listing for a quarter folder."""
|
||
links = ['<a href="../">../</a>']
|
||
for name in file_names:
|
||
href = _cp1251_href(name)
|
||
links.append(f'<a href="{href}">{name}</a>')
|
||
return "<html><body><pre>" + "\n".join(links) + "</pre></body></html>"
|
||
|
||
|
||
_SDELKI_FILE_2026_Q1 = "dataset_СДЕЛКИ_r-r_01-92_y_2026_q_1.csv.zip"
|
||
_KADASTR_FILE_2026_Q1 = "dataset_КАДАСТРСТОИМОСТЬ_r-r_01-92_y_2026_q_1.csv.zip"
|
||
|
||
|
||
def _mock_response(*, status_code: int = 200, text: str = "", headers: dict | None = None):
|
||
resp = MagicMock()
|
||
resp.status_code = status_code
|
||
resp.text = text
|
||
resp.headers = headers or {}
|
||
return resp
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# (a) _next_quarter
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_next_quarter_mid_year() -> None:
|
||
assert _next_quarter(2026, 1) == (2026, 2)
|
||
assert _next_quarter(2026, 2) == (2026, 3)
|
||
assert _next_quarter(2026, 3) == (2026, 4)
|
||
|
||
|
||
def test_next_quarter_year_rollover() -> None:
|
||
assert _next_quarter(2026, 4) == (2027, 1)
|
||
assert _next_quarter(2024, 4) == (2025, 1)
|
||
|
||
|
||
def test_period_start_to_quarter_from_date() -> None:
|
||
assert _period_start_to_quarter(date(2026, 1, 1)) == (2026, 1)
|
||
assert _period_start_to_quarter(date(2026, 4, 1)) == (2026, 2)
|
||
assert _period_start_to_quarter(date(2026, 7, 15)) == (2026, 3)
|
||
assert _period_start_to_quarter(date(2026, 10, 31)) == (2026, 4)
|
||
assert _period_start_to_quarter(date(2025, 12, 31)) == (2025, 4)
|
||
|
||
|
||
def test_period_start_to_quarter_from_string() -> None:
|
||
assert _period_start_to_quarter("2026-01-15") == (2026, 1)
|
||
assert _period_start_to_quarter("2026-07-01") == (2026, 3)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# (c) rosreestr_dataset_url — cp1251 percent-encoding must match the real portal.
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_rosreestr_dataset_url_matches_verified_real_encoding() -> None:
|
||
"""Encoding must byte-match the live-verified rosreestr.gov.ru/data-sets/ URL."""
|
||
url = rosreestr_dataset_url(2026, 1)
|
||
assert url == (
|
||
"https://rosreestr.gov.ru/data-sets/"
|
||
"1%20%EA%E2%E0%F0%F2%E0%EB%202026%E3./"
|
||
"dataset_%D1%C4%C5%CB%CA%C8_r-r_01-92_y_2026_q_1.csv.zip"
|
||
)
|
||
|
||
|
||
def test_rosreestr_dataset_url_contains_year_quarter() -> None:
|
||
url = rosreestr_dataset_url(2026, 2)
|
||
assert "2026" in url
|
||
assert "q_2" in url
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Fake DB helpers
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
class _FakeRow:
|
||
"""Mimics a DB row with __getitem__."""
|
||
|
||
def __init__(self, val: object) -> None:
|
||
self._val = val
|
||
|
||
def __getitem__(self, idx: int) -> object:
|
||
return self._val
|
||
|
||
|
||
class _FakeDB:
|
||
"""Minimal fake Session returning a configurable deal_date via execute().fetchone()."""
|
||
|
||
def __init__(self, max_deal_date: date | None) -> None:
|
||
self._row = _FakeRow(max_deal_date) if max_deal_date is not None else None
|
||
|
||
def execute(self, stmt: object, params: object = None) -> MagicMock:
|
||
mock = MagicMock()
|
||
mock.fetchone.return_value = self._row
|
||
return mock
|
||
|
||
def commit(self) -> None:
|
||
pass
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# (b) latest_loaded_quarter
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_latest_loaded_quarter_returns_correct_year_quarter() -> None:
|
||
"""Q1 2026 deal_date → (2026, 1)."""
|
||
db = _FakeDB(date(2026, 1, 31))
|
||
result = latest_loaded_quarter(db) # type: ignore[arg-type]
|
||
assert result == (2026, 1)
|
||
|
||
|
||
def test_latest_loaded_quarter_q4() -> None:
|
||
"""Dec deal_date → Q4."""
|
||
db = _FakeDB(date(2025, 12, 15))
|
||
result = latest_loaded_quarter(db) # type: ignore[arg-type]
|
||
assert result == (2025, 4)
|
||
|
||
|
||
def test_latest_loaded_quarter_returns_none_when_empty() -> None:
|
||
"""No rows → None."""
|
||
db = _FakeDB(None)
|
||
result = latest_loaded_quarter(db) # type: ignore[arg-type]
|
||
assert result is None
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# (d) check_new_quarter_available — /data-sets/ autoindex flow
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_check_available_true_for_real_zip() -> None:
|
||
"""Folder found, file found, HEAD returns application/zip + large Content-Length → True."""
|
||
index_html = _autoindex_html("1 квартал 2026г.", "4 квартал 2025г.")
|
||
folder_html = _folder_html(_SDELKI_FILE_2026_Q1, _KADASTR_FILE_2026_Q1)
|
||
|
||
mock_client = AsyncMock()
|
||
mock_client.get.side_effect = [
|
||
_mock_response(text=index_html),
|
||
_mock_response(text=folder_html),
|
||
]
|
||
mock_client.head.return_value = _mock_response(
|
||
headers={"content-type": "application/zip", "content-length": "9500000"}
|
||
)
|
||
|
||
result = await check_new_quarter_available(mock_client, 2026, 1) # type: ignore[arg-type]
|
||
|
||
assert result is True
|
||
assert mock_client.get.call_count == 2
|
||
assert mock_client.head.call_count == 1
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_check_available_false_soft_404_text_html() -> None:
|
||
"""Regression guard: HTTP 200 with Content-Type text/html (Bitrix soft-404) → False.
|
||
|
||
This is the exact failure mode that broke the old /opendata/ flat-URL poller —
|
||
it treated bare HTTP 200 as availability. Even if a matching file href is found,
|
||
a non-zip Content-Type must never be treated as available.
|
||
"""
|
||
index_html = _autoindex_html("1 квартал 2026г.")
|
||
folder_html = _folder_html(_SDELKI_FILE_2026_Q1)
|
||
|
||
mock_client = AsyncMock()
|
||
mock_client.get.side_effect = [
|
||
_mock_response(text=index_html),
|
||
_mock_response(text=folder_html),
|
||
]
|
||
mock_client.head.return_value = _mock_response(
|
||
status_code=200,
|
||
headers={"content-type": "text/html; charset=utf-8", "content-length": "1234"},
|
||
)
|
||
|
||
result = await check_new_quarter_available(mock_client, 2026, 1) # type: ignore[arg-type]
|
||
|
||
assert result is False
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_check_available_false_tiny_content_length() -> None:
|
||
"""application/zip but tiny Content-Length (placeholder/error file) → False."""
|
||
index_html = _autoindex_html("1 квартал 2026г.")
|
||
folder_html = _folder_html(_SDELKI_FILE_2026_Q1)
|
||
|
||
mock_client = AsyncMock()
|
||
mock_client.get.side_effect = [
|
||
_mock_response(text=index_html),
|
||
_mock_response(text=folder_html),
|
||
]
|
||
mock_client.head.return_value = _mock_response(
|
||
headers={"content-type": "application/zip", "content-length": "512"}
|
||
)
|
||
|
||
result = await check_new_quarter_available(mock_client, 2026, 1) # type: ignore[arg-type]
|
||
|
||
assert result is False
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_check_available_false_quarter_folder_missing() -> None:
|
||
"""Autoindex doesn't have the target quarter folder yet → False, no further GET/HEAD."""
|
||
index_html = _autoindex_html("4 квартал 2025г.") # 2026 Q1 not published yet
|
||
|
||
mock_client = AsyncMock()
|
||
mock_client.get.return_value = _mock_response(text=index_html)
|
||
|
||
result = await check_new_quarter_available(mock_client, 2026, 1) # type: ignore[arg-type]
|
||
|
||
assert result is False
|
||
assert mock_client.get.call_count == 1 # never fetched a (non-existent) folder
|
||
mock_client.head.assert_not_called()
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_check_available_false_dataset_file_missing_in_folder() -> None:
|
||
"""Quarter folder exists but has no dataset_СДЕЛКИ file (only КАДАСТРСТОИМОСТЬ) → False."""
|
||
index_html = _autoindex_html("1 квартал 2026г.")
|
||
folder_html = _folder_html(_KADASTR_FILE_2026_Q1) # no СДЕЛКИ file
|
||
|
||
mock_client = AsyncMock()
|
||
mock_client.get.side_effect = [
|
||
_mock_response(text=index_html),
|
||
_mock_response(text=folder_html),
|
||
]
|
||
|
||
result = await check_new_quarter_available(mock_client, 2026, 1) # type: ignore[arg-type]
|
||
|
||
assert result is False
|
||
mock_client.head.assert_not_called()
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_check_available_false_on_index_404() -> None:
|
||
"""Unexpected non-200 on the base autoindex → False, no raise."""
|
||
mock_client = AsyncMock()
|
||
mock_client.get.return_value = _mock_response(status_code=404, text="")
|
||
|
||
result = await check_new_quarter_available(mock_client, 2026, 1) # type: ignore[arg-type]
|
||
|
||
assert result is False
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_check_new_quarter_available_timeout_returns_false() -> None:
|
||
"""Timeout → returns False, no exception."""
|
||
import httpx
|
||
|
||
mock_client = AsyncMock()
|
||
mock_client.get.side_effect = httpx.TimeoutException("timeout")
|
||
|
||
result = await check_new_quarter_available(mock_client, 2026, 2) # type: ignore[arg-type]
|
||
assert result is False
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_check_new_quarter_available_network_error_returns_false() -> None:
|
||
"""Network error mid-flow (folder GET) → returns False, no exception."""
|
||
import httpx
|
||
|
||
index_html = _autoindex_html("1 квартал 2026г.")
|
||
|
||
mock_client = AsyncMock()
|
||
mock_client.get.side_effect = [
|
||
_mock_response(text=index_html),
|
||
httpx.ConnectError("simulated network error"),
|
||
]
|
||
|
||
result = await check_new_quarter_available(mock_client, 2026, 1) # type: ignore[arg-type]
|
||
assert result is False
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# (e) poll_rosreestr_new_quarter — orchestration
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def _patched_client_for(index_html: str, folder_html: str, head_response) -> AsyncMock:
|
||
mock_client = AsyncMock()
|
||
mock_client.get.side_effect = [
|
||
_mock_response(text=index_html),
|
||
_mock_response(text=folder_html),
|
||
]
|
||
mock_client.head.return_value = head_response
|
||
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
|
||
mock_client.__aexit__ = AsyncMock(return_value=False)
|
||
return mock_client
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_poll_available_true_returns_correct_next_quarter() -> None:
|
||
"""If latest loaded is Q1 2026 and the Q2 2026 dataset is a real zip, available=True."""
|
||
db = _FakeDB(date(2026, 3, 31)) # latest = Q1 2026 → next = Q2 2026
|
||
|
||
index_html = _autoindex_html("2 квартал 2026г.")
|
||
file_name = "dataset_СДЕЛКИ_r-r_01-92_y_2026_q_2.csv.zip"
|
||
folder_html = _folder_html(file_name)
|
||
head_response = _mock_response(
|
||
headers={"content-type": "application/zip", "content-length": "9500000"}
|
||
)
|
||
|
||
mock_client = _patched_client_for(index_html, folder_html, head_response)
|
||
|
||
with patch("app.services.rosreestr_poll.httpx.AsyncClient", return_value=mock_client):
|
||
result = await poll_rosreestr_new_quarter(db) # type: ignore[arg-type]
|
||
|
||
assert result["available"] is True
|
||
assert result["year"] == 2026
|
||
assert result["quarter"] == 2
|
||
assert result["latest_loaded_year"] == 2026
|
||
assert result["latest_loaded_quarter"] == 1
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_poll_available_false_when_folder_not_published() -> None:
|
||
"""Quarter folder not yet published (autoindex has no matching entry) → available=False."""
|
||
db = _FakeDB(date(2026, 3, 15)) # Q1 2026 → next = Q2 2026
|
||
|
||
index_html = _autoindex_html("1 квартал 2026г.") # Q2 2026 missing
|
||
|
||
mock_client = AsyncMock()
|
||
mock_client.get.return_value = _mock_response(text=index_html)
|
||
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
|
||
mock_client.__aexit__ = AsyncMock(return_value=False)
|
||
|
||
with patch("app.services.rosreestr_poll.httpx.AsyncClient", return_value=mock_client):
|
||
result = await poll_rosreestr_new_quarter(db) # type: ignore[arg-type]
|
||
|
||
assert result["available"] is False
|
||
assert result["year"] == 2026
|
||
assert result["quarter"] == 2
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_poll_network_error_returns_false_no_raise() -> None:
|
||
"""Network error → available=False, no exception propagated."""
|
||
import httpx
|
||
|
||
db = _FakeDB(date(2026, 1, 20)) # Q1 2026
|
||
|
||
mock_client = AsyncMock()
|
||
mock_client.get.side_effect = httpx.ConnectError("simulated network error")
|
||
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
|
||
mock_client.__aexit__ = AsyncMock(return_value=False)
|
||
|
||
with patch("app.services.rosreestr_poll.httpx.AsyncClient", return_value=mock_client):
|
||
# Must not raise
|
||
result = await poll_rosreestr_new_quarter(db) # type: ignore[arg-type]
|
||
|
||
assert result["available"] is False
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# (f) No DB data → fallback baseline checks 2026Q1
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_poll_fallback_when_no_db_data() -> None:
|
||
"""No rosreestr deals in DB → fallback 2025Q4 baseline, checks 2026Q1."""
|
||
db = _FakeDB(None)
|
||
|
||
index_html = _autoindex_html("4 квартал 2025г.") # 2026 Q1 not yet published
|
||
|
||
mock_client = AsyncMock()
|
||
mock_client.get.return_value = _mock_response(text=index_html)
|
||
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
|
||
mock_client.__aexit__ = AsyncMock(return_value=False)
|
||
|
||
with patch("app.services.rosreestr_poll.httpx.AsyncClient", return_value=mock_client):
|
||
result = await poll_rosreestr_new_quarter(db) # type: ignore[arg-type]
|
||
|
||
# Fallback: 2025Q4 → check 2026Q1
|
||
assert result["year"] == 2026
|
||
assert result["quarter"] == 1
|
||
assert result["available"] is False
|
||
# latest_loaded_year/quarter should be None (no DB data)
|
||
assert result["latest_loaded_year"] is None
|
||
assert result["latest_loaded_quarter"] is None
|