bot-backend
5a9fb806bb
The backfill service (app/services/yandex_address_backfill.py) + its manual admin endpoint already exist, but it was never wired into the in-app scheduler — so residual street-only yandex listings (no house number) were only fixed on manual trigger. Add a recurring scheduler task: trigger_yandex_address_backfill_run mirrors trigger_cian_backfill_run (claim run + mark done/failed), new task wrapper app/tasks/yandex_address_backfill.py delegates to the existing backfill_yandex_addresses(), dispatch branch + seed migration 091 (window 02:00-03:00 UTC, ON CONFLICT(source) idempotent). EKB-only (region_code=66). Refs #855, #726
195 lines
7.4 KiB
Python
195 lines
7.4 KiB
Python
"""Tests for yandex_address_backfill — pure regex extraction, no network/DB."""
|
||
|
||
from __future__ import annotations
|
||
|
||
import os
|
||
import re
|
||
import sys
|
||
from unittest.mock import AsyncMock, MagicMock, patch
|
||
|
||
# DATABASE_URL required by config before any app import.
|
||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
|
||
|
||
# WeasyPrint stub — not installed in CI without GTK.
|
||
_wp_mock = MagicMock()
|
||
sys.modules.setdefault("weasyprint", _wp_mock)
|
||
|
||
import pytest # noqa: E402
|
||
|
||
from app.services.yandex_address_backfill import ( # noqa: E402
|
||
YandexAddressBackfillResult,
|
||
_extract_address_from_title,
|
||
)
|
||
|
||
# Regex used to assert that an extracted address contains a house number.
|
||
_RE_HAS_HOUSE_NUMBER = re.compile(r",\s*\d+")
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Pure regex / extraction tests (no network, no DB)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_extract_address_standard_title() -> None:
|
||
"""Standard title: «Продажа квартиры — Екатеринбург, улица Горького, 36 — id 7654321»."""
|
||
html = (
|
||
"<title>Продажа квартиры — Екатеринбург, улица Горького, 36 "
|
||
"— id 7654321 на Яндекс.Недвижимости</title>"
|
||
)
|
||
addr = _extract_address_from_title(html)
|
||
assert addr is not None, "Should extract address from standard title"
|
||
assert _RE_HAS_HOUSE_NUMBER.search(
|
||
addr
|
||
), f"Extracted address should contain house number: {addr!r}"
|
||
assert "Екатеринбург" in addr
|
||
assert "Горького" in addr
|
||
assert "36" in addr
|
||
|
||
|
||
def test_extract_address_with_zhk_prefix() -> None:
|
||
"""Title with optional ЖК prefix before city: «— ЖК Каменный Ручей, Екатеринбург, …»."""
|
||
html = (
|
||
"<title>Продажа квартиры — ЖК Каменный Ручей, "
|
||
"Екатеринбург, проспект Космонавтов, 110А — id 99001 </title>"
|
||
)
|
||
addr = _extract_address_from_title(html)
|
||
assert addr is not None, "Should extract address when ЖК prefix is present"
|
||
assert _RE_HAS_HOUSE_NUMBER.search(
|
||
addr
|
||
), f"Extracted address should contain house number: {addr!r}"
|
||
assert "Екатеринбург" in addr
|
||
|
||
|
||
def test_extract_address_nbsp_replaced() -> None:
|
||
"""\\xa0 (non-breaking space) in title should be normalised before regex match."""
|
||
html = (
|
||
"<title>Продажа\xa0квартиры\xa0—\xa0Екатеринбург,\xa0улица\xa0Ленина,\xa012"
|
||
"\xa0—\xa0id\xa0123456</title>"
|
||
)
|
||
addr = _extract_address_from_title(html)
|
||
assert addr is not None, "\\xa0 should be replaced before regex"
|
||
assert _RE_HAS_HOUSE_NUMBER.search(
|
||
addr
|
||
), f"Extracted address should contain house number: {addr!r}"
|
||
|
||
|
||
def test_extract_address_no_title_returns_none() -> None:
|
||
"""HTML without a matching <title> returns None."""
|
||
html = "<html><body>Объявление снято</body></html>"
|
||
addr = _extract_address_from_title(html)
|
||
assert addr is None
|
||
|
||
|
||
def test_extract_address_title_without_id_returns_none() -> None:
|
||
"""Title that matches city but has no '— id NNN' sentinel returns None."""
|
||
html = "<title>Екатеринбург, улица Горького, 36</title>"
|
||
addr = _extract_address_from_title(html)
|
||
assert addr is None
|
||
|
||
|
||
def test_extract_address_street_only_still_extracted() -> None:
|
||
"""Title with street-only (no house number) is extracted but fails the house-number check.
|
||
|
||
The extractor itself does not gate on house number presence — the caller
|
||
(backfill_yandex_addresses) checks _RE_HAS_HOUSE_NUMBER separately and skips.
|
||
"""
|
||
html = "<title>Продажа квартиры — Екатеринбург, улица Горького — id 7654321</title>"
|
||
addr = _extract_address_from_title(html)
|
||
# May or may not match depending on whether «улица Горького» satisfies the regex —
|
||
# the key assertion is that IF it does, it does NOT contain a house number.
|
||
if addr is not None:
|
||
assert not _RE_HAS_HOUSE_NUMBER.search(
|
||
addr
|
||
), "Street-only extracted address should NOT contain a house number"
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Task-level smoke tests (no real DB, no real HTTP)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_run_yandex_address_backfill_no_eligible_listings() -> None:
|
||
"""No eligible listings → run marked done with zeroed counters, no HTTP calls."""
|
||
from app.tasks.yandex_address_backfill import run_yandex_address_backfill
|
||
|
||
db = MagicMock()
|
||
|
||
empty_result = YandexAddressBackfillResult(
|
||
checked=0, saved=0, skipped=0, errors=0, duration_sec=0.01
|
||
)
|
||
|
||
with (
|
||
patch(
|
||
"app.tasks.yandex_address_backfill.backfill_yandex_addresses",
|
||
new_callable=AsyncMock,
|
||
return_value=empty_result,
|
||
) as mock_backfill,
|
||
patch("app.tasks.yandex_address_backfill.runs_mod") as mock_runs,
|
||
):
|
||
result = await run_yandex_address_backfill(db, run_id=42, params={})
|
||
|
||
mock_backfill.assert_awaited_once()
|
||
mock_runs.update_heartbeat.assert_called_once_with(
|
||
db, 42, {"checked": 0, "saved": 0, "skipped": 0, "errors": 0}
|
||
)
|
||
mock_runs.mark_done.assert_called_once()
|
||
mock_runs.mark_failed.assert_not_called()
|
||
|
||
assert result.checked == 0
|
||
assert result.saved == 0
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_run_yandex_address_backfill_saves_enriched_count() -> None:
|
||
"""5 checked, 3 saved → result reflects correct counters."""
|
||
from app.tasks.yandex_address_backfill import run_yandex_address_backfill
|
||
|
||
db = MagicMock()
|
||
|
||
backfill_result = YandexAddressBackfillResult(
|
||
checked=5, saved=3, skipped=1, errors=1, duration_sec=15.0
|
||
)
|
||
|
||
with (
|
||
patch(
|
||
"app.tasks.yandex_address_backfill.backfill_yandex_addresses",
|
||
new_callable=AsyncMock,
|
||
return_value=backfill_result,
|
||
),
|
||
patch("app.tasks.yandex_address_backfill.runs_mod") as mock_runs,
|
||
):
|
||
result = await run_yandex_address_backfill(
|
||
db, run_id=99, params={"batch_size": 100, "request_delay_sec": 2.0}
|
||
)
|
||
|
||
mock_runs.mark_done.assert_called_once()
|
||
done_counters = mock_runs.mark_done.call_args[0][2] # positional: (db, run_id, counters)
|
||
assert done_counters["checked"] == 5
|
||
assert done_counters["saved"] == 3
|
||
assert done_counters["errors"] == 1
|
||
|
||
assert result.saved == 3
|
||
assert result.errors == 1
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_run_yandex_address_backfill_marks_failed_on_exception() -> None:
|
||
"""backfill_yandex_addresses raises → mark_failed called, exception re-raised."""
|
||
from app.tasks.yandex_address_backfill import run_yandex_address_backfill
|
||
|
||
db = MagicMock()
|
||
|
||
with (
|
||
patch(
|
||
"app.tasks.yandex_address_backfill.backfill_yandex_addresses",
|
||
new_callable=AsyncMock,
|
||
side_effect=RuntimeError("network timeout"),
|
||
),
|
||
patch("app.tasks.yandex_address_backfill.runs_mod") as mock_runs,
|
||
):
|
||
with pytest.raises(RuntimeError, match="network timeout"):
|
||
await run_yandex_address_backfill(db, run_id=7, params={})
|
||
|
||
mock_runs.mark_failed.assert_called_once()
|
||
mock_runs.mark_done.assert_not_called()
|