fdfb74ca88
После C-5 fix (PR #487) Avito coords либо NULL либо реальные. Estimator исключал source='avito' из radius search потому что раньше там был jitter (±0.005°) от 5 anchor cron'ов. Сейчас: - 3580 Avito + 641 Yandex + 106 N1 + 38 Cian listings без coords - estimator._fetch_analogs ловил только cian/yandex/n1 = ~30% эффективной базы Bundled fix: 1. app/tasks/geocode_missing.py — batch geocoder (Nominatim 1/s, dedup по address) 2. POST /admin/scrape/geocode-missing-listings + GET status — manual trigger 3. estimator.py: убран AND source <> 'avito' — после backfill Avito включён в radius Result: после backfill +4166 Avito listings в radius search = +40% эффективной data. Confidence "high" будет в ~70% случаев вместо ~30%. Tests: 11 новых tests/tasks/test_geocode_missing.py — all pass.
318 lines
11 KiB
Python
318 lines
11 KiB
Python
"""Tests for geocode_missing_listings batch task."""
|
||
from __future__ import annotations
|
||
|
||
import os
|
||
import sys
|
||
from unittest.mock import AsyncMock, MagicMock, call, patch
|
||
|
||
# DATABASE_URL required by config before any app import.
|
||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
|
||
|
||
# WeasyPrint stub — not installed in CI without GTK.
|
||
_wp_mock = MagicMock()
|
||
sys.modules.setdefault("weasyprint", _wp_mock)
|
||
|
||
import pytest
|
||
|
||
from app.services.geocoder import GeocodeResult
|
||
from app.tasks.geocode_missing import GeocodeBackfillResult, geocode_missing_listings
|
||
|
||
|
||
def _make_geocode_result(provider: str = "nominatim") -> GeocodeResult:
|
||
return GeocodeResult(
|
||
lat=56.838,
|
||
lon=60.605,
|
||
full_address="Екатеринбург, ул. Малышева, 30",
|
||
provider=provider, # type: ignore[arg-type]
|
||
confidence="exact",
|
||
)
|
||
|
||
|
||
def _mock_db_rows(rows: list[dict]) -> MagicMock:
|
||
"""Возвращает mock Session где .execute().mappings().all() → rows."""
|
||
db = MagicMock()
|
||
mappings_mock = MagicMock()
|
||
mappings_mock.all.return_value = rows
|
||
db.execute.return_value.mappings.return_value = mappings_mock
|
||
# execute().rowcount для UPDATE
|
||
db.execute.return_value.rowcount = len(rows)
|
||
return db
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_geocode_missing_empty_returns_zero_result() -> None:
|
||
"""Нет адресов с NULL coords → возвращает нулевой результат без ошибок."""
|
||
db = _mock_db_rows([])
|
||
|
||
with patch("app.tasks.geocode_missing.geocode", new_callable=AsyncMock) as mock_geo:
|
||
result = await geocode_missing_listings(db, batch_size=50)
|
||
|
||
assert isinstance(result, GeocodeBackfillResult)
|
||
assert result.addresses_total == 0
|
||
assert result.addresses_processed == 0
|
||
assert result.listings_updated == 0
|
||
mock_geo.assert_not_called()
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_geocode_missing_processes_pending_addresses() -> None:
|
||
"""2 unique addresses → geocode вызывается 2 раза, listings_updated суммируется."""
|
||
rows = [
|
||
{"address": "ул. Малышева, 30", "listings_count": 3},
|
||
{"address": "ул. Ленина, 10", "listings_count": 1},
|
||
]
|
||
|
||
db = MagicMock()
|
||
# Первый call .execute().mappings().all() → rows (SELECT)
|
||
# Последующие .execute() → UPDATE (rowcount)
|
||
select_result = MagicMock()
|
||
select_result.mappings.return_value.all.return_value = rows
|
||
|
||
update_result_1 = MagicMock()
|
||
update_result_1.rowcount = 3
|
||
|
||
update_result_2 = MagicMock()
|
||
update_result_2.rowcount = 1
|
||
|
||
db.execute.side_effect = [select_result, update_result_1, update_result_2]
|
||
|
||
with patch(
|
||
"app.tasks.geocode_missing.geocode",
|
||
new_callable=AsyncMock,
|
||
return_value=_make_geocode_result("nominatim"),
|
||
) as mock_geo:
|
||
result = await geocode_missing_listings(db, batch_size=200)
|
||
|
||
assert result.addresses_total == 2
|
||
assert result.addresses_processed == 2
|
||
assert result.addresses_geocoded == 2
|
||
assert result.addresses_failed == 0
|
||
assert result.listings_updated == 4 # 3 + 1
|
||
assert result.cache_misses == 2
|
||
assert result.cache_hits == 0
|
||
assert mock_geo.call_count == 2
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_geocode_missing_dedup_addresses_single_geocode_call() -> None:
|
||
"""5 listings с тем же адресом → SELECT группирует → 1 geocode call."""
|
||
rows = [{"address": "ул. Тургенева, 5", "listings_count": 5}]
|
||
|
||
db = MagicMock()
|
||
select_result = MagicMock()
|
||
select_result.mappings.return_value.all.return_value = rows
|
||
update_result = MagicMock()
|
||
update_result.rowcount = 5
|
||
db.execute.side_effect = [select_result, update_result]
|
||
|
||
with patch(
|
||
"app.tasks.geocode_missing.geocode",
|
||
new_callable=AsyncMock,
|
||
return_value=_make_geocode_result("nominatim"),
|
||
) as mock_geo:
|
||
result = await geocode_missing_listings(db, batch_size=200)
|
||
|
||
assert mock_geo.call_count == 1
|
||
assert result.listings_updated == 5
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_geocode_missing_cache_hit_counted() -> None:
|
||
"""Адрес из geocode_cache → cache_hits++, cache_misses не растёт."""
|
||
rows = [{"address": "ул. Горького, 7", "listings_count": 2}]
|
||
|
||
db = MagicMock()
|
||
select_result = MagicMock()
|
||
select_result.mappings.return_value.all.return_value = rows
|
||
update_result = MagicMock()
|
||
update_result.rowcount = 2
|
||
db.execute.side_effect = [select_result, update_result]
|
||
|
||
with patch(
|
||
"app.tasks.geocode_missing.geocode",
|
||
new_callable=AsyncMock,
|
||
return_value=_make_geocode_result("cache"),
|
||
):
|
||
result = await geocode_missing_listings(db, batch_size=200)
|
||
|
||
assert result.cache_hits == 1
|
||
assert result.cache_misses == 0
|
||
assert result.addresses_geocoded == 1
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_geocode_missing_skips_failed_addresses() -> None:
|
||
"""Geocoder возвращает None → failed++, UPDATE не вызывается."""
|
||
rows = [{"address": "несуществующий адрес", "listings_count": 1}]
|
||
|
||
db = MagicMock()
|
||
select_result = MagicMock()
|
||
select_result.mappings.return_value.all.return_value = rows
|
||
db.execute.return_value = select_result
|
||
|
||
with patch(
|
||
"app.tasks.geocode_missing.geocode",
|
||
new_callable=AsyncMock,
|
||
return_value=None,
|
||
):
|
||
result = await geocode_missing_listings(db, batch_size=200)
|
||
|
||
assert result.addresses_failed == 1
|
||
assert result.addresses_geocoded == 0
|
||
assert result.listings_updated == 0
|
||
# commit не должен был вызываться после failed geocode
|
||
db.commit.assert_not_called()
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_geocode_missing_dry_run_does_not_update() -> None:
|
||
"""dry_run=True → addresses_processed > 0 но listings_updated == 0, нет commit."""
|
||
rows = [
|
||
{"address": "ул. Пушкина, 1", "listings_count": 3},
|
||
{"address": "пр. Ленина, 50", "listings_count": 2},
|
||
]
|
||
|
||
db = MagicMock()
|
||
select_result = MagicMock()
|
||
select_result.mappings.return_value.all.return_value = rows
|
||
db.execute.return_value = select_result
|
||
|
||
with patch(
|
||
"app.tasks.geocode_missing.geocode",
|
||
new_callable=AsyncMock,
|
||
return_value=_make_geocode_result("nominatim"),
|
||
):
|
||
result = await geocode_missing_listings(db, batch_size=200, dry_run=True)
|
||
|
||
assert result.addresses_processed == 2
|
||
assert result.addresses_geocoded == 2
|
||
assert result.listings_updated == 0
|
||
db.commit.assert_not_called()
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_geocode_missing_batch_size_respected() -> None:
|
||
"""batch_size=3 → SQL LIMIT=3 (проверяем параметр в вызове execute)."""
|
||
db = MagicMock()
|
||
select_result = MagicMock()
|
||
select_result.mappings.return_value.all.return_value = []
|
||
db.execute.return_value = select_result
|
||
|
||
with patch("app.tasks.geocode_missing.geocode", new_callable=AsyncMock):
|
||
await geocode_missing_listings(db, batch_size=3)
|
||
|
||
# Первый вызов execute — SELECT с LIMIT
|
||
first_call = db.execute.call_args_list[0]
|
||
params = first_call[0][1] # positional arg[1] = params dict
|
||
assert params["limit"] == 3
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_geocode_missing_geocode_exception_counted_as_failed() -> None:
|
||
"""Geocoder raises Exception → failed++, обработка продолжается."""
|
||
rows = [
|
||
{"address": "ул. Сломанная, 1", "listings_count": 1},
|
||
{"address": "ул. Рабочая, 5", "listings_count": 2},
|
||
]
|
||
|
||
db = MagicMock()
|
||
select_result = MagicMock()
|
||
select_result.mappings.return_value.all.return_value = rows
|
||
update_result = MagicMock()
|
||
update_result.rowcount = 2
|
||
db.execute.side_effect = [select_result, update_result]
|
||
|
||
geo_side_effects = [RuntimeError("network error"), _make_geocode_result("nominatim")]
|
||
|
||
with patch(
|
||
"app.tasks.geocode_missing.geocode",
|
||
new_callable=AsyncMock,
|
||
side_effect=geo_side_effects,
|
||
):
|
||
result = await geocode_missing_listings(db, batch_size=200)
|
||
|
||
assert result.addresses_failed == 1
|
||
assert result.addresses_geocoded == 1
|
||
assert result.listings_updated == 2
|
||
|
||
|
||
# ── Integration-style: estimator Avito exclusion removed ─────────────────────
|
||
|
||
|
||
def test_estimator_fetch_analogs_includes_avito() -> None:
|
||
"""estimator._fetch_analogs не содержит AND source <> 'avito'."""
|
||
import inspect
|
||
|
||
from app.services import estimator
|
||
|
||
source = inspect.getsource(estimator._fetch_analogs)
|
||
assert "source <> 'avito'" not in source, (
|
||
"Avito exclusion должен быть удалён из estimator._fetch_analogs"
|
||
)
|
||
|
||
|
||
# ── Admin endpoint smoke (schema-only, no real DB) ────────────────────────────
|
||
|
||
|
||
def test_admin_geocode_missing_endpoint_exists() -> None:
|
||
"""GET /api/v1/admin/scrape/geocode-missing-listings/status → endpoint registered."""
|
||
from fastapi import FastAPI
|
||
from fastapi.testclient import TestClient
|
||
|
||
from app.api.v1 import admin as admin_module
|
||
from app.core.db import get_db
|
||
|
||
app = FastAPI()
|
||
app.include_router(admin_module.router, prefix="/api/v1/admin")
|
||
|
||
mock_db = MagicMock()
|
||
# status endpoint calls execute twice — stats + unique + cache_size
|
||
stats_result = MagicMock()
|
||
stats_result.mappings.return_value.all.return_value = []
|
||
scalar_result = MagicMock()
|
||
scalar_result.scalar.return_value = 0
|
||
mock_db.execute.side_effect = [stats_result, scalar_result, scalar_result]
|
||
|
||
app.dependency_overrides[get_db] = lambda: mock_db
|
||
|
||
client = TestClient(app)
|
||
resp = client.get("/api/v1/admin/scrape/geocode-missing-listings/status")
|
||
# Если endpoint зарегистрирован корректно — не 404
|
||
assert resp.status_code != 404
|
||
|
||
|
||
def test_admin_geocode_missing_post_dry_run_endpoint_exists() -> None:
|
||
"""POST /api/v1/admin/scrape/geocode-missing-listings?dry_run=true → endpoint registered."""
|
||
from fastapi import FastAPI
|
||
from fastapi.testclient import TestClient
|
||
|
||
from app.api.v1 import admin as admin_module
|
||
from app.core.db import get_db
|
||
|
||
app = FastAPI()
|
||
app.include_router(admin_module.router, prefix="/api/v1/admin")
|
||
|
||
# Mock db: не нужен реальный — тест только на routing
|
||
mock_db = MagicMock()
|
||
app.dependency_overrides[get_db] = lambda: mock_db
|
||
|
||
client = TestClient(app)
|
||
# Без реального geocode — mocked через patch
|
||
with patch(
|
||
"app.tasks.geocode_missing.geocode_missing_listings",
|
||
new_callable=AsyncMock,
|
||
return_value=GeocodeBackfillResult(
|
||
addresses_total=0, addresses_processed=0, addresses_geocoded=0,
|
||
addresses_failed=0, listings_updated=0, cache_hits=0, cache_misses=0,
|
||
duration_sec=0.01,
|
||
),
|
||
):
|
||
resp = client.post(
|
||
"/api/v1/admin/scrape/geocode-missing-listings?dry_run=true&batch_size=10"
|
||
)
|
||
|
||
assert resp.status_code != 404
|
||
if resp.status_code == 200:
|
||
data = resp.json()
|
||
assert "status" in data
|
||
assert "addresses_total" in data
|