bot-backend
a56420bdfb
Root cause: 7674 avito/yandex/n1 listings з null-geom мають ненульовий address, але geocode_tried_at IS NULL — OS cron геокодує тільки deals, ніколи listings. Parts: - geocode_missing.py: WHERE додає geocode_tried_at IS NULL OR tried_at < 7d (loop-safe); при failure (geo=None / exception) позначаємо tried_at=NOW(). Новий run_geocode_missing_listings: batch loop з wall-clock budget (30m), update_heartbeat / mark_done / mark_failed lifecycle через scrape_runs. - scheduler.py: trigger_geocode_missing_listings_run + dispatch elif. - migration 110: seed scrape_schedules row (enabled=true, window 06-09 UTC). - tests: оновлено 2 тести (commit на tried_at UPDATE); +7 нових тестів. 1794 passed.
496 lines
18 KiB
Python
496 lines
18 KiB
Python
"""Tests for geocode_missing_listings batch task."""
|
||
|
||
from __future__ import annotations
|
||
|
||
import os
|
||
import sys
|
||
from unittest.mock import AsyncMock, MagicMock, patch
|
||
|
||
# DATABASE_URL required by config before any app import.
|
||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
|
||
|
||
# WeasyPrint stub — not installed in CI without GTK.
|
||
_wp_mock = MagicMock()
|
||
sys.modules.setdefault("weasyprint", _wp_mock)
|
||
|
||
import pytest # noqa: E402
|
||
|
||
from app.services.geocoder import GeocodeResult # noqa: E402
|
||
from app.tasks.geocode_missing import ( # noqa: E402
|
||
GeocodeBackfillResult,
|
||
geocode_missing_listings,
|
||
run_geocode_missing_listings,
|
||
)
|
||
|
||
|
||
def _make_geocode_result(provider: str = "nominatim") -> GeocodeResult:
|
||
return GeocodeResult(
|
||
lat=56.838,
|
||
lon=60.605,
|
||
full_address="Екатеринбург, ул. Малышева, 30",
|
||
provider=provider, # type: ignore[arg-type]
|
||
confidence="exact",
|
||
)
|
||
|
||
|
||
def _mock_db_rows(rows: list[dict]) -> MagicMock:
|
||
"""Возвращает mock Session где .execute().mappings().all() → rows."""
|
||
db = MagicMock()
|
||
mappings_mock = MagicMock()
|
||
mappings_mock.all.return_value = rows
|
||
db.execute.return_value.mappings.return_value = mappings_mock
|
||
# execute().rowcount для UPDATE
|
||
db.execute.return_value.rowcount = len(rows)
|
||
return db
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_geocode_missing_empty_returns_zero_result() -> None:
|
||
"""Нет адресов с NULL coords → возвращает нулевой результат без ошибок."""
|
||
db = _mock_db_rows([])
|
||
|
||
with patch("app.tasks.geocode_missing.geocode", new_callable=AsyncMock) as mock_geo:
|
||
result = await geocode_missing_listings(db, batch_size=50)
|
||
|
||
assert isinstance(result, GeocodeBackfillResult)
|
||
assert result.addresses_total == 0
|
||
assert result.addresses_processed == 0
|
||
assert result.listings_updated == 0
|
||
mock_geo.assert_not_called()
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_geocode_missing_processes_pending_addresses() -> None:
|
||
"""2 unique addresses → geocode вызывается 2 раза, listings_updated суммируется."""
|
||
rows = [
|
||
{"address": "ул. Малышева, 30", "listings_count": 3},
|
||
{"address": "ул. Ленина, 10", "listings_count": 1},
|
||
]
|
||
|
||
db = MagicMock()
|
||
# Первый call .execute().mappings().all() → rows (SELECT)
|
||
# Последующие .execute() → UPDATE (rowcount)
|
||
select_result = MagicMock()
|
||
select_result.mappings.return_value.all.return_value = rows
|
||
|
||
update_result_1 = MagicMock()
|
||
update_result_1.rowcount = 3
|
||
|
||
update_result_2 = MagicMock()
|
||
update_result_2.rowcount = 1
|
||
|
||
db.execute.side_effect = [select_result, update_result_1, update_result_2]
|
||
|
||
with patch(
|
||
"app.tasks.geocode_missing.geocode",
|
||
new_callable=AsyncMock,
|
||
return_value=_make_geocode_result("nominatim"),
|
||
) as mock_geo:
|
||
result = await geocode_missing_listings(db, batch_size=200)
|
||
|
||
assert result.addresses_total == 2
|
||
assert result.addresses_processed == 2
|
||
assert result.addresses_geocoded == 2
|
||
assert result.addresses_failed == 0
|
||
assert result.listings_updated == 4 # 3 + 1
|
||
assert result.cache_misses == 2
|
||
assert result.cache_hits == 0
|
||
assert mock_geo.call_count == 2
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_geocode_missing_dedup_addresses_single_geocode_call() -> None:
|
||
"""5 listings с тем же адресом → SELECT группирует → 1 geocode call."""
|
||
rows = [{"address": "ул. Тургенева, 5", "listings_count": 5}]
|
||
|
||
db = MagicMock()
|
||
select_result = MagicMock()
|
||
select_result.mappings.return_value.all.return_value = rows
|
||
update_result = MagicMock()
|
||
update_result.rowcount = 5
|
||
db.execute.side_effect = [select_result, update_result]
|
||
|
||
with patch(
|
||
"app.tasks.geocode_missing.geocode",
|
||
new_callable=AsyncMock,
|
||
return_value=_make_geocode_result("nominatim"),
|
||
) as mock_geo:
|
||
result = await geocode_missing_listings(db, batch_size=200)
|
||
|
||
assert mock_geo.call_count == 1
|
||
assert result.listings_updated == 5
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_geocode_missing_cache_hit_counted() -> None:
|
||
"""Адрес из geocode_cache → cache_hits++, cache_misses не растёт."""
|
||
rows = [{"address": "ул. Горького, 7", "listings_count": 2}]
|
||
|
||
db = MagicMock()
|
||
select_result = MagicMock()
|
||
select_result.mappings.return_value.all.return_value = rows
|
||
update_result = MagicMock()
|
||
update_result.rowcount = 2
|
||
db.execute.side_effect = [select_result, update_result]
|
||
|
||
with patch(
|
||
"app.tasks.geocode_missing.geocode",
|
||
new_callable=AsyncMock,
|
||
return_value=_make_geocode_result("cache"),
|
||
):
|
||
result = await geocode_missing_listings(db, batch_size=200)
|
||
|
||
assert result.cache_hits == 1
|
||
assert result.cache_misses == 0
|
||
assert result.addresses_geocoded == 1
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_geocode_missing_skips_failed_addresses() -> None:
|
||
"""Geocoder возвращает None → failed++, lat UPDATE не вызывается, tried_at помечается."""
|
||
rows = [{"address": "несуществующий адрес", "listings_count": 1}]
|
||
|
||
db = MagicMock()
|
||
select_result = MagicMock()
|
||
select_result.mappings.return_value.all.return_value = rows
|
||
tried_at_update_result = MagicMock()
|
||
tried_at_update_result.rowcount = 1
|
||
# execute side_effect: [SELECT, UPDATE tried_at]
|
||
db.execute.side_effect = [select_result, tried_at_update_result]
|
||
|
||
with patch(
|
||
"app.tasks.geocode_missing.geocode",
|
||
new_callable=AsyncMock,
|
||
return_value=None,
|
||
):
|
||
result = await geocode_missing_listings(db, batch_size=200)
|
||
|
||
assert result.addresses_failed == 1
|
||
assert result.addresses_geocoded == 0
|
||
assert result.listings_updated == 0
|
||
# commit вызывается для tried_at UPDATE (loop-safe backoff)
|
||
db.commit.assert_called_once()
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_geocode_missing_dry_run_does_not_update() -> None:
|
||
"""dry_run=True → addresses_processed > 0 но listings_updated == 0, нет commit."""
|
||
rows = [
|
||
{"address": "ул. Пушкина, 1", "listings_count": 3},
|
||
{"address": "пр. Ленина, 50", "listings_count": 2},
|
||
]
|
||
|
||
db = MagicMock()
|
||
select_result = MagicMock()
|
||
select_result.mappings.return_value.all.return_value = rows
|
||
db.execute.return_value = select_result
|
||
|
||
with patch(
|
||
"app.tasks.geocode_missing.geocode",
|
||
new_callable=AsyncMock,
|
||
return_value=_make_geocode_result("nominatim"),
|
||
):
|
||
result = await geocode_missing_listings(db, batch_size=200, dry_run=True)
|
||
|
||
assert result.addresses_processed == 2
|
||
assert result.addresses_geocoded == 2
|
||
assert result.listings_updated == 0
|
||
db.commit.assert_not_called()
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_geocode_missing_batch_size_respected() -> None:
|
||
"""batch_size=3 → SQL LIMIT=3 (проверяем параметр в вызове execute)."""
|
||
db = MagicMock()
|
||
select_result = MagicMock()
|
||
select_result.mappings.return_value.all.return_value = []
|
||
db.execute.return_value = select_result
|
||
|
||
with patch("app.tasks.geocode_missing.geocode", new_callable=AsyncMock):
|
||
await geocode_missing_listings(db, batch_size=3)
|
||
|
||
# Первый вызов execute — SELECT с LIMIT
|
||
first_call = db.execute.call_args_list[0]
|
||
params = first_call[0][1] # positional arg[1] = params dict
|
||
assert params["limit"] == 3
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_geocode_missing_geocode_exception_counted_as_failed() -> None:
|
||
"""Geocoder raises Exception → failed++, tried_at помечается, обработка продолжается."""
|
||
rows = [
|
||
{"address": "ул. Сломанная, 1", "listings_count": 1},
|
||
{"address": "ул. Рабочая, 5", "listings_count": 2},
|
||
]
|
||
|
||
db = MagicMock()
|
||
select_result = MagicMock()
|
||
select_result.mappings.return_value.all.return_value = rows
|
||
tried_at_result = MagicMock()
|
||
tried_at_result.rowcount = 1
|
||
update_result = MagicMock()
|
||
update_result.rowcount = 2
|
||
# SELECT → tried_at UPDATE (exception path) → lat/lon UPDATE (success path)
|
||
db.execute.side_effect = [select_result, tried_at_result, update_result]
|
||
|
||
geo_side_effects = [RuntimeError("network error"), _make_geocode_result("nominatim")]
|
||
|
||
with patch(
|
||
"app.tasks.geocode_missing.geocode",
|
||
new_callable=AsyncMock,
|
||
side_effect=geo_side_effects,
|
||
):
|
||
result = await geocode_missing_listings(db, batch_size=200)
|
||
|
||
assert result.addresses_failed == 1
|
||
assert result.addresses_geocoded == 1
|
||
assert result.listings_updated == 2
|
||
|
||
|
||
# ── New tests: geocode_tried_at / loop-safe / run_geocode_missing_listings ───
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_geocode_missing_failure_marks_tried_at() -> None:
|
||
"""geo=None → UPDATE geocode_tried_at выполняется (loop-safe backoff)."""
|
||
rows = [{"address": "несуществующий адрес", "listings_count": 2}]
|
||
|
||
db = MagicMock()
|
||
select_result = MagicMock()
|
||
select_result.mappings.return_value.all.return_value = rows
|
||
tried_at_result = MagicMock()
|
||
db.execute.side_effect = [select_result, tried_at_result]
|
||
|
||
with patch(
|
||
"app.tasks.geocode_missing.geocode",
|
||
new_callable=AsyncMock,
|
||
return_value=None,
|
||
):
|
||
result = await geocode_missing_listings(db, batch_size=200)
|
||
|
||
assert result.addresses_failed == 1
|
||
assert result.listings_updated == 0
|
||
# Второй execute — UPDATE tried_at
|
||
assert db.execute.call_count == 2
|
||
second_call_sql = str(db.execute.call_args_list[1][0][0])
|
||
assert "geocode_tried_at" in second_call_sql
|
||
db.commit.assert_called_once()
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_geocode_missing_exception_marks_tried_at() -> None:
|
||
"""geocode() raises → UPDATE geocode_tried_at выполняется, commit вызывается."""
|
||
rows = [{"address": "ул. Битая, 99", "listings_count": 1}]
|
||
|
||
db = MagicMock()
|
||
select_result = MagicMock()
|
||
select_result.mappings.return_value.all.return_value = rows
|
||
tried_at_result = MagicMock()
|
||
db.execute.side_effect = [select_result, tried_at_result]
|
||
|
||
with patch(
|
||
"app.tasks.geocode_missing.geocode",
|
||
new_callable=AsyncMock,
|
||
side_effect=RuntimeError("timeout"),
|
||
):
|
||
result = await geocode_missing_listings(db, batch_size=200)
|
||
|
||
assert result.addresses_failed == 1
|
||
assert db.execute.call_count == 2
|
||
second_call_sql = str(db.execute.call_args_list[1][0][0])
|
||
assert "geocode_tried_at" in second_call_sql
|
||
db.commit.assert_called_once()
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_geocode_missing_dry_run_no_tried_at_update() -> None:
|
||
"""dry_run=True + geo=None → tried_at НЕ обновляется, commit не вызывается."""
|
||
rows = [{"address": "несуществующий адрес", "listings_count": 1}]
|
||
|
||
db = MagicMock()
|
||
select_result = MagicMock()
|
||
select_result.mappings.return_value.all.return_value = rows
|
||
db.execute.return_value = select_result
|
||
|
||
with patch(
|
||
"app.tasks.geocode_missing.geocode",
|
||
new_callable=AsyncMock,
|
||
return_value=None,
|
||
):
|
||
result = await geocode_missing_listings(db, batch_size=200, dry_run=True)
|
||
|
||
assert result.addresses_failed == 1
|
||
# Только SELECT, никаких UPDATE
|
||
assert db.execute.call_count == 1
|
||
db.commit.assert_not_called()
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_geocode_missing_recent_tried_at_excluded_via_where() -> None:
|
||
"""SELECT содержит geocode_tried_at IS NULL OR tried_at < NOW() - 7d (loop-safe WHERE)."""
|
||
db = MagicMock()
|
||
select_result = MagicMock()
|
||
select_result.mappings.return_value.all.return_value = []
|
||
db.execute.return_value = select_result
|
||
|
||
with patch("app.tasks.geocode_missing.geocode", new_callable=AsyncMock):
|
||
await geocode_missing_listings(db, batch_size=10)
|
||
|
||
first_call = db.execute.call_args_list[0]
|
||
sql_text = str(first_call[0][0])
|
||
assert "geocode_tried_at IS NULL" in sql_text
|
||
assert "7 days" in sql_text
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_run_geocode_missing_listings_terminates_on_drained() -> None:
|
||
"""run_geocode_missing_listings завершается когда addresses_total == 0 (ничего pending)."""
|
||
db = MagicMock()
|
||
drained = GeocodeBackfillResult(addresses_total=0)
|
||
|
||
with (
|
||
patch(
|
||
"app.tasks.geocode_missing.geocode_missing_listings",
|
||
new_callable=AsyncMock,
|
||
return_value=drained,
|
||
),
|
||
patch("app.tasks.geocode_missing.runs_mod") as mock_runs,
|
||
):
|
||
result = await run_geocode_missing_listings(db, run_id=42, params={})
|
||
|
||
mock_runs.update_heartbeat.assert_called()
|
||
mock_runs.mark_done.assert_called_once()
|
||
mock_runs.mark_failed.assert_not_called()
|
||
assert result.addresses_total == 0
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_run_geocode_missing_listings_terminates_on_partial_batch() -> None:
|
||
"""Последний batch меньше batch_size → завершаем (дренаж)."""
|
||
db = MagicMock()
|
||
partial = GeocodeBackfillResult(
|
||
addresses_total=50, # < batch_size=200
|
||
addresses_processed=50,
|
||
listings_updated=30,
|
||
)
|
||
|
||
with (
|
||
patch(
|
||
"app.tasks.geocode_missing.geocode_missing_listings",
|
||
new_callable=AsyncMock,
|
||
return_value=partial,
|
||
),
|
||
patch("app.tasks.geocode_missing.runs_mod") as mock_runs,
|
||
):
|
||
result = await run_geocode_missing_listings(db, run_id=7, params={"batch_size": 200})
|
||
|
||
mock_runs.mark_done.assert_called_once()
|
||
assert result.addresses_processed == 50
|
||
assert result.listings_updated == 30
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_run_geocode_missing_listings_mark_failed_on_exception() -> None:
|
||
"""При исключении внутри loop: mark_failed вызывается, re-raise."""
|
||
db = MagicMock()
|
||
|
||
with (
|
||
patch(
|
||
"app.tasks.geocode_missing.geocode_missing_listings",
|
||
new_callable=AsyncMock,
|
||
side_effect=RuntimeError("DB down"),
|
||
),
|
||
patch("app.tasks.geocode_missing.runs_mod") as mock_runs,
|
||
):
|
||
with pytest.raises(RuntimeError, match="DB down"):
|
||
await run_geocode_missing_listings(db, run_id=99, params={})
|
||
|
||
mock_runs.mark_failed.assert_called_once()
|
||
mock_runs.mark_done.assert_not_called()
|
||
|
||
|
||
# ── Integration-style: estimator Avito exclusion removed ─────────────────────
|
||
|
||
|
||
def test_estimator_fetch_analogs_includes_avito() -> None:
|
||
"""estimator._fetch_analogs не содержит AND source <> 'avito'."""
|
||
import inspect
|
||
|
||
from app.services import estimator
|
||
|
||
source = inspect.getsource(estimator._fetch_analogs)
|
||
assert (
|
||
"source <> 'avito'" not in source
|
||
), "Avito exclusion должен быть удалён из estimator._fetch_analogs"
|
||
|
||
|
||
# ── Admin endpoint smoke (schema-only, no real DB) ────────────────────────────
|
||
|
||
|
||
def test_admin_geocode_missing_endpoint_exists() -> None:
|
||
"""GET /api/v1/admin/scrape/geocode-missing-listings/status → endpoint registered."""
|
||
from fastapi import FastAPI
|
||
from fastapi.testclient import TestClient
|
||
|
||
from app.api.v1 import admin as admin_module
|
||
from app.core.db import get_db
|
||
|
||
app = FastAPI()
|
||
app.include_router(admin_module.router, prefix="/api/v1/admin")
|
||
|
||
mock_db = MagicMock()
|
||
# status endpoint calls execute twice — stats + unique + cache_size
|
||
stats_result = MagicMock()
|
||
stats_result.mappings.return_value.all.return_value = []
|
||
scalar_result = MagicMock()
|
||
scalar_result.scalar.return_value = 0
|
||
mock_db.execute.side_effect = [stats_result, scalar_result, scalar_result]
|
||
|
||
app.dependency_overrides[get_db] = lambda: mock_db
|
||
|
||
client = TestClient(app)
|
||
resp = client.get("/api/v1/admin/scrape/geocode-missing-listings/status")
|
||
# Если endpoint зарегистрирован корректно — не 404
|
||
assert resp.status_code != 404
|
||
|
||
|
||
def test_admin_geocode_missing_post_dry_run_endpoint_exists() -> None:
|
||
"""POST /api/v1/admin/scrape/geocode-missing-listings?dry_run=true → endpoint registered."""
|
||
from fastapi import FastAPI
|
||
from fastapi.testclient import TestClient
|
||
|
||
from app.api.v1 import admin as admin_module
|
||
from app.core.db import get_db
|
||
|
||
app = FastAPI()
|
||
app.include_router(admin_module.router, prefix="/api/v1/admin")
|
||
|
||
# Mock db: не нужен реальный — тест только на routing
|
||
mock_db = MagicMock()
|
||
app.dependency_overrides[get_db] = lambda: mock_db
|
||
|
||
client = TestClient(app)
|
||
# Без реального geocode — mocked через patch
|
||
with patch(
|
||
"app.tasks.geocode_missing.geocode_missing_listings",
|
||
new_callable=AsyncMock,
|
||
return_value=GeocodeBackfillResult(
|
||
addresses_total=0,
|
||
addresses_processed=0,
|
||
addresses_geocoded=0,
|
||
addresses_failed=0,
|
||
listings_updated=0,
|
||
cache_hits=0,
|
||
cache_misses=0,
|
||
duration_sec=0.01,
|
||
),
|
||
):
|
||
resp = client.post(
|
||
"/api/v1/admin/scrape/geocode-missing-listings?dry_run=true&batch_size=10"
|
||
)
|
||
|
||
assert resp.status_code != 404
|
||
if resp.status_code == 200:
|
||
data = resp.json()
|
||
assert "status" in data
|
||
assert "addresses_total" in data
|