bd472b9b57
All checks were successful
CI / changes (pull_request) Successful in 8s
CI Trade-In / changes (pull_request) Successful in 10s
CI Trade-In / frontend-checks (pull_request) Has been skipped
CI / backend-tests (pull_request) Has been skipped
CI / frontend-tests (pull_request) Has been skipped
CI / openapi-codegen-check (pull_request) Has been skipped
CI Trade-In / backend-tests (pull_request) Successful in 2m29s
Ночная очередь geocode_missing_listings была на 98.5% забита is_active=false объявлениями чужих регионов (Новосибирск/Казань/Челябинск/Тюмень/Ижевск...) без улицы и дома. ORDER BY listings_count DESC ставил такой мусор в начало очереди (у 'Новосибирская обл.,Новосибирск' — 214 listings, у реального адреса — 1-2), поэтому Nominatim-бюджет (1 req/sec) съедался мусором и до активных адресов дело не доходило: 8 ночных прогонов подряд saved=0. Добавлен AND is_active в SELECT. UPDATE (lat/lon и оба tried_at) намеренно оставлены без этого фильтра — координаты и backoff-метка принадлежат паре (address, city) как тексту, не конкретному listing; is_active=false дубликат той же пары и так навсегда исключён из будущих SELECT, а unfiltered UPDATE проставляет ему ответ бесплатно (Nominatim-вызов уже оплачен активным листингом) на случай реактивации. Refs #2604
819 lines
32 KiB
Python
819 lines
32 KiB
Python
"""Tests for geocode_missing_listings batch task."""
|
||
|
||
from __future__ import annotations
|
||
|
||
import os
|
||
import sys
|
||
from unittest.mock import AsyncMock, MagicMock, patch
|
||
|
||
# DATABASE_URL required by config before any app import.
|
||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
|
||
|
||
# WeasyPrint stub — not installed in CI without GTK.
|
||
_wp_mock = MagicMock()
|
||
sys.modules.setdefault("weasyprint", _wp_mock)
|
||
|
||
import pytest # noqa: E402
|
||
|
||
from app.services.geocoder import GeocodeResult # noqa: E402
|
||
from app.tasks.geocode_missing import ( # noqa: E402
|
||
GeocodeBackfillResult,
|
||
geocode_missing_listings,
|
||
run_geocode_missing_listings,
|
||
)
|
||
|
||
|
||
def _make_geocode_result(provider: str = "nominatim") -> GeocodeResult:
|
||
return GeocodeResult(
|
||
lat=56.838,
|
||
lon=60.605,
|
||
full_address="Екатеринбург, ул. Малышева, 30",
|
||
provider=provider, # type: ignore[arg-type]
|
||
confidence="exact",
|
||
)
|
||
|
||
|
||
def _mock_db_rows(rows: list[dict]) -> MagicMock:
|
||
"""Возвращает mock Session где .execute().mappings().all() → rows."""
|
||
db = MagicMock()
|
||
mappings_mock = MagicMock()
|
||
mappings_mock.all.return_value = rows
|
||
db.execute.return_value.mappings.return_value = mappings_mock
|
||
# execute().rowcount для UPDATE
|
||
db.execute.return_value.rowcount = len(rows)
|
||
return db
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_geocode_missing_empty_returns_zero_result() -> None:
|
||
"""Нет адресов с NULL coords → возвращает нулевой результат без ошибок."""
|
||
db = _mock_db_rows([])
|
||
|
||
with patch("app.tasks.geocode_missing.geocode", new_callable=AsyncMock) as mock_geo:
|
||
result = await geocode_missing_listings(db, batch_size=50)
|
||
|
||
assert isinstance(result, GeocodeBackfillResult)
|
||
assert result.addresses_total == 0
|
||
assert result.addresses_processed == 0
|
||
assert result.listings_updated == 0
|
||
mock_geo.assert_not_called()
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_geocode_missing_processes_pending_addresses() -> None:
|
||
"""2 unique addresses → geocode вызывается 2 раза, listings_updated суммируется."""
|
||
rows = [
|
||
{"address": "ул. Малышева, 30", "listings_count": 3},
|
||
{"address": "ул. Ленина, 10", "listings_count": 1},
|
||
]
|
||
|
||
db = MagicMock()
|
||
# Первый call .execute().mappings().all() → rows (SELECT)
|
||
# Последующие .execute() → UPDATE (rowcount)
|
||
select_result = MagicMock()
|
||
select_result.mappings.return_value.all.return_value = rows
|
||
|
||
update_result_1 = MagicMock()
|
||
update_result_1.rowcount = 3
|
||
|
||
update_result_2 = MagicMock()
|
||
update_result_2.rowcount = 1
|
||
|
||
db.execute.side_effect = [select_result, update_result_1, update_result_2]
|
||
|
||
with patch(
|
||
"app.tasks.geocode_missing.geocode",
|
||
new_callable=AsyncMock,
|
||
return_value=_make_geocode_result("nominatim"),
|
||
) as mock_geo:
|
||
result = await geocode_missing_listings(db, batch_size=200)
|
||
|
||
assert result.addresses_total == 2
|
||
assert result.addresses_processed == 2
|
||
assert result.addresses_geocoded == 2
|
||
assert result.addresses_failed == 0
|
||
assert result.listings_updated == 4 # 3 + 1
|
||
assert result.cache_misses == 2
|
||
assert result.cache_hits == 0
|
||
assert mock_geo.call_count == 2
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_geocode_missing_dedup_addresses_single_geocode_call() -> None:
|
||
"""5 listings с тем же адресом → SELECT группирует → 1 geocode call."""
|
||
rows = [{"address": "ул. Тургенева, 5", "listings_count": 5}]
|
||
|
||
db = MagicMock()
|
||
select_result = MagicMock()
|
||
select_result.mappings.return_value.all.return_value = rows
|
||
update_result = MagicMock()
|
||
update_result.rowcount = 5
|
||
db.execute.side_effect = [select_result, update_result]
|
||
|
||
with patch(
|
||
"app.tasks.geocode_missing.geocode",
|
||
new_callable=AsyncMock,
|
||
return_value=_make_geocode_result("nominatim"),
|
||
) as mock_geo:
|
||
result = await geocode_missing_listings(db, batch_size=200)
|
||
|
||
assert mock_geo.call_count == 1
|
||
assert result.listings_updated == 5
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_geocode_missing_cache_hit_counted() -> None:
|
||
"""Адрес из geocode_cache → cache_hits++, cache_misses не растёт."""
|
||
rows = [{"address": "ул. Горького, 7", "listings_count": 2}]
|
||
|
||
db = MagicMock()
|
||
select_result = MagicMock()
|
||
select_result.mappings.return_value.all.return_value = rows
|
||
update_result = MagicMock()
|
||
update_result.rowcount = 2
|
||
db.execute.side_effect = [select_result, update_result]
|
||
|
||
with patch(
|
||
"app.tasks.geocode_missing.geocode",
|
||
new_callable=AsyncMock,
|
||
return_value=_make_geocode_result("cache"),
|
||
):
|
||
result = await geocode_missing_listings(db, batch_size=200)
|
||
|
||
assert result.cache_hits == 1
|
||
assert result.cache_misses == 0
|
||
assert result.addresses_geocoded == 1
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_geocode_missing_skips_failed_addresses() -> None:
|
||
"""Geocoder возвращает None → failed++, lat UPDATE не вызывается, tried_at помечается."""
|
||
rows = [{"address": "несуществующий адрес", "listings_count": 1}]
|
||
|
||
db = MagicMock()
|
||
select_result = MagicMock()
|
||
select_result.mappings.return_value.all.return_value = rows
|
||
tried_at_update_result = MagicMock()
|
||
tried_at_update_result.rowcount = 1
|
||
# execute side_effect: [SELECT, UPDATE tried_at]
|
||
db.execute.side_effect = [select_result, tried_at_update_result]
|
||
|
||
with patch(
|
||
"app.tasks.geocode_missing.geocode",
|
||
new_callable=AsyncMock,
|
||
return_value=None,
|
||
):
|
||
result = await geocode_missing_listings(db, batch_size=200)
|
||
|
||
assert result.addresses_failed == 1
|
||
assert result.addresses_geocoded == 0
|
||
assert result.listings_updated == 0
|
||
# commit вызывается для tried_at UPDATE (loop-safe backoff)
|
||
db.commit.assert_called_once()
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_geocode_missing_dry_run_does_not_update() -> None:
|
||
"""dry_run=True → addresses_processed > 0 но listings_updated == 0, нет commit."""
|
||
rows = [
|
||
{"address": "ул. Пушкина, 1", "listings_count": 3},
|
||
{"address": "пр. Ленина, 50", "listings_count": 2},
|
||
]
|
||
|
||
db = MagicMock()
|
||
select_result = MagicMock()
|
||
select_result.mappings.return_value.all.return_value = rows
|
||
db.execute.return_value = select_result
|
||
|
||
with patch(
|
||
"app.tasks.geocode_missing.geocode",
|
||
new_callable=AsyncMock,
|
||
return_value=_make_geocode_result("nominatim"),
|
||
):
|
||
result = await geocode_missing_listings(db, batch_size=200, dry_run=True)
|
||
|
||
assert result.addresses_processed == 2
|
||
assert result.addresses_geocoded == 2
|
||
assert result.listings_updated == 0
|
||
db.commit.assert_not_called()
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_geocode_missing_batch_size_respected() -> None:
|
||
"""batch_size=3 → SQL LIMIT=3 (проверяем параметр в вызове execute)."""
|
||
db = MagicMock()
|
||
select_result = MagicMock()
|
||
select_result.mappings.return_value.all.return_value = []
|
||
db.execute.return_value = select_result
|
||
|
||
with patch("app.tasks.geocode_missing.geocode", new_callable=AsyncMock):
|
||
await geocode_missing_listings(db, batch_size=3)
|
||
|
||
# Первый вызов execute — SELECT с LIMIT
|
||
first_call = db.execute.call_args_list[0]
|
||
params = first_call[0][1] # positional arg[1] = params dict
|
||
assert params["limit"] == 3
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_geocode_missing_geocode_exception_counted_as_failed() -> None:
|
||
"""Geocoder raises Exception → failed++, tried_at помечается, обработка продолжается."""
|
||
rows = [
|
||
{"address": "ул. Сломанная, 1", "listings_count": 1},
|
||
{"address": "ул. Рабочая, 5", "listings_count": 2},
|
||
]
|
||
|
||
db = MagicMock()
|
||
select_result = MagicMock()
|
||
select_result.mappings.return_value.all.return_value = rows
|
||
tried_at_result = MagicMock()
|
||
tried_at_result.rowcount = 1
|
||
update_result = MagicMock()
|
||
update_result.rowcount = 2
|
||
# SELECT → tried_at UPDATE (exception path) → lat/lon UPDATE (success path)
|
||
db.execute.side_effect = [select_result, tried_at_result, update_result]
|
||
|
||
geo_side_effects = [RuntimeError("network error"), _make_geocode_result("nominatim")]
|
||
|
||
with patch(
|
||
"app.tasks.geocode_missing.geocode",
|
||
new_callable=AsyncMock,
|
||
side_effect=geo_side_effects,
|
||
):
|
||
result = await geocode_missing_listings(db, batch_size=200)
|
||
|
||
assert result.addresses_failed == 1
|
||
assert result.addresses_geocoded == 1
|
||
assert result.listings_updated == 2
|
||
|
||
|
||
# ── New tests: geocode_tried_at / loop-safe / run_geocode_missing_listings ───
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_geocode_missing_failure_marks_tried_at() -> None:
|
||
"""geo=None → UPDATE geocode_tried_at выполняется (loop-safe backoff)."""
|
||
rows = [{"address": "несуществующий адрес", "listings_count": 2}]
|
||
|
||
db = MagicMock()
|
||
select_result = MagicMock()
|
||
select_result.mappings.return_value.all.return_value = rows
|
||
tried_at_result = MagicMock()
|
||
db.execute.side_effect = [select_result, tried_at_result]
|
||
|
||
with patch(
|
||
"app.tasks.geocode_missing.geocode",
|
||
new_callable=AsyncMock,
|
||
return_value=None,
|
||
):
|
||
result = await geocode_missing_listings(db, batch_size=200)
|
||
|
||
assert result.addresses_failed == 1
|
||
assert result.listings_updated == 0
|
||
# Второй execute — UPDATE tried_at
|
||
assert db.execute.call_count == 2
|
||
second_call_sql = str(db.execute.call_args_list[1][0][0])
|
||
assert "geocode_tried_at" in second_call_sql
|
||
db.commit.assert_called_once()
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_geocode_missing_exception_marks_tried_at() -> None:
|
||
"""geocode() raises → UPDATE geocode_tried_at выполняется, commit вызывается."""
|
||
rows = [{"address": "ул. Битая, 99", "listings_count": 1}]
|
||
|
||
db = MagicMock()
|
||
select_result = MagicMock()
|
||
select_result.mappings.return_value.all.return_value = rows
|
||
tried_at_result = MagicMock()
|
||
db.execute.side_effect = [select_result, tried_at_result]
|
||
|
||
with patch(
|
||
"app.tasks.geocode_missing.geocode",
|
||
new_callable=AsyncMock,
|
||
side_effect=RuntimeError("timeout"),
|
||
):
|
||
result = await geocode_missing_listings(db, batch_size=200)
|
||
|
||
assert result.addresses_failed == 1
|
||
assert db.execute.call_count == 2
|
||
second_call_sql = str(db.execute.call_args_list[1][0][0])
|
||
assert "geocode_tried_at" in second_call_sql
|
||
db.commit.assert_called_once()
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_geocode_missing_dry_run_no_tried_at_update() -> None:
|
||
"""dry_run=True + geo=None → tried_at НЕ обновляется, commit не вызывается."""
|
||
rows = [{"address": "несуществующий адрес", "listings_count": 1}]
|
||
|
||
db = MagicMock()
|
||
select_result = MagicMock()
|
||
select_result.mappings.return_value.all.return_value = rows
|
||
db.execute.return_value = select_result
|
||
|
||
with patch(
|
||
"app.tasks.geocode_missing.geocode",
|
||
new_callable=AsyncMock,
|
||
return_value=None,
|
||
):
|
||
result = await geocode_missing_listings(db, batch_size=200, dry_run=True)
|
||
|
||
assert result.addresses_failed == 1
|
||
# Только SELECT, никаких UPDATE
|
||
assert db.execute.call_count == 1
|
||
db.commit.assert_not_called()
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_geocode_missing_recent_tried_at_excluded_via_where() -> None:
|
||
"""SELECT содержит geocode_tried_at IS NULL OR tried_at < NOW() - 7d (loop-safe WHERE)."""
|
||
db = MagicMock()
|
||
select_result = MagicMock()
|
||
select_result.mappings.return_value.all.return_value = []
|
||
db.execute.return_value = select_result
|
||
|
||
with patch("app.tasks.geocode_missing.geocode", new_callable=AsyncMock):
|
||
await geocode_missing_listings(db, batch_size=10)
|
||
|
||
first_call = db.execute.call_args_list[0]
|
||
sql_text = str(first_call[0][0])
|
||
assert "geocode_tried_at IS NULL" in sql_text
|
||
assert "7 days" in sql_text
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_geocode_missing_select_filters_is_active() -> None:
|
||
"""SELECT содержит `AND is_active` (#2604 п.1).
|
||
|
||
На проде очередь без этого фильтра была на 98.5% забита is_active=false
|
||
объявлениями чужих регионов (Новосибирск/Казань/Челябинск/…) без улицы и дома;
|
||
`ORDER BY listings_count DESC` ставил самый мусорный адрес («Новосибирская
|
||
обл.,Новосибирск», 214 listings) В НАЧАЛО очереди — весь Nominatim-бюджет
|
||
(1 req/sec) съедался мусором, до реальных активных адресов дело не доходило
|
||
(8 ночных прогонов подряд: saved=0). Falsification-проба: на коде ДО фикса
|
||
`"AND is_active" in sql_text` ложно, тест падает; после фикса проходит.
|
||
"""
|
||
db = MagicMock()
|
||
select_result = MagicMock()
|
||
select_result.mappings.return_value.all.return_value = []
|
||
db.execute.return_value = select_result
|
||
|
||
with patch("app.tasks.geocode_missing.geocode", new_callable=AsyncMock):
|
||
await geocode_missing_listings(db, batch_size=10)
|
||
|
||
first_call = db.execute.call_args_list[0]
|
||
sql_text = str(first_call[0][0])
|
||
assert "AND is_active" in sql_text
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_run_geocode_missing_listings_terminates_on_drained() -> None:
|
||
"""run_geocode_missing_listings завершается когда addresses_total == 0 (ничего pending)."""
|
||
db = MagicMock()
|
||
drained = GeocodeBackfillResult(addresses_total=0)
|
||
|
||
with (
|
||
patch(
|
||
"app.tasks.geocode_missing.geocode_missing_listings",
|
||
new_callable=AsyncMock,
|
||
return_value=drained,
|
||
),
|
||
patch("app.tasks.geocode_missing.runs_mod") as mock_runs,
|
||
):
|
||
result = await run_geocode_missing_listings(db, run_id=42, params={})
|
||
|
||
mock_runs.update_heartbeat.assert_called()
|
||
mock_runs.mark_done.assert_called_once()
|
||
mock_runs.mark_failed.assert_not_called()
|
||
assert result.addresses_total == 0
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_run_geocode_missing_listings_terminates_on_partial_batch() -> None:
|
||
"""Последний batch меньше batch_size → завершаем (дренаж)."""
|
||
db = MagicMock()
|
||
partial = GeocodeBackfillResult(
|
||
addresses_total=50, # < batch_size=200
|
||
addresses_processed=50,
|
||
listings_updated=30,
|
||
)
|
||
|
||
with (
|
||
patch(
|
||
"app.tasks.geocode_missing.geocode_missing_listings",
|
||
new_callable=AsyncMock,
|
||
return_value=partial,
|
||
),
|
||
patch("app.tasks.geocode_missing.runs_mod") as mock_runs,
|
||
):
|
||
result = await run_geocode_missing_listings(db, run_id=7, params={"batch_size": 200})
|
||
|
||
mock_runs.mark_done.assert_called_once()
|
||
assert result.addresses_processed == 50
|
||
assert result.listings_updated == 30
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_run_geocode_missing_listings_mark_failed_on_exception() -> None:
|
||
"""При исключении внутри loop: mark_failed вызывается, re-raise."""
|
||
db = MagicMock()
|
||
|
||
with (
|
||
patch(
|
||
"app.tasks.geocode_missing.geocode_missing_listings",
|
||
new_callable=AsyncMock,
|
||
side_effect=RuntimeError("DB down"),
|
||
),
|
||
patch("app.tasks.geocode_missing.runs_mod") as mock_runs,
|
||
):
|
||
with pytest.raises(RuntimeError, match="DB down"):
|
||
await run_geocode_missing_listings(db, run_id=99, params={})
|
||
|
||
mock_runs.mark_failed.assert_called_once()
|
||
mock_runs.mark_done.assert_not_called()
|
||
|
||
|
||
# ── (address, city) pair grouping / city_hint (#2594 шаг 2/3) ────────────────
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_geocode_missing_same_address_different_city_independent_calls_and_updates() -> None:
|
||
"""Ключевой сценарий #2594: два ряда с ОДИНАКОВЫМ текстом address, но РАЗНЫМИ
|
||
city — каждый должен получить свой geocode-вызов (city_hint) и свой UPDATE, не
|
||
затрагивающий другую пару. Раньше группировка была только по address: SELECT
|
||
группировал по тексту, а UPDATE бил по WHERE address = :addr без city — второй
|
||
UPDATE (для Нижнего Тагила) перезаписал бы координаты, уже проставленные первым
|
||
(для Екатеринбурга), и наоборот.
|
||
"""
|
||
rows = [
|
||
{"address": "ул. Победы, 30", "city": "Екатеринбург", "listings_count": 1},
|
||
{"address": "ул. Победы, 30", "city": "Нижний Тагил", "listings_count": 1},
|
||
]
|
||
|
||
ekb_geo = GeocodeResult(
|
||
lat=56.838,
|
||
lon=60.605,
|
||
full_address="Екатеринбург, ул. Победы, 30",
|
||
provider="nominatim", # type: ignore[arg-type]
|
||
confidence="exact",
|
||
)
|
||
tagil_geo = GeocodeResult(
|
||
lat=57.910,
|
||
lon=59.985,
|
||
full_address="Нижний Тагил, ул. Победы, 30",
|
||
provider="nominatim", # type: ignore[arg-type]
|
||
confidence="exact",
|
||
)
|
||
|
||
db = MagicMock()
|
||
select_result = MagicMock()
|
||
select_result.mappings.return_value.all.return_value = rows
|
||
update_ekb = MagicMock()
|
||
update_ekb.rowcount = 1
|
||
update_tagil = MagicMock()
|
||
update_tagil.rowcount = 1
|
||
db.execute.side_effect = [select_result, update_ekb, update_tagil]
|
||
|
||
with patch(
|
||
"app.tasks.geocode_missing.geocode",
|
||
new_callable=AsyncMock,
|
||
side_effect=[ekb_geo, tagil_geo],
|
||
) as mock_geo:
|
||
result = await geocode_missing_listings(db, batch_size=200)
|
||
|
||
# 2 отдельных geocode-вызова — по одному на пару (address, city), не 1 на address.
|
||
assert mock_geo.call_count == 2
|
||
ekb_call = mock_geo.call_args_list[0]
|
||
tagil_call = mock_geo.call_args_list[1]
|
||
assert ekb_call.args[0] == "ул. Победы, 30"
|
||
assert ekb_call.kwargs["city_hint"] == "Екатеринбург"
|
||
assert tagil_call.args[0] == "ул. Победы, 30"
|
||
assert tagil_call.kwargs["city_hint"] == "Нижний Тагил"
|
||
|
||
# 2 отдельных UPDATE, каждый со своим city в WHERE — не задевает другую пару.
|
||
update_calls = db.execute.call_args_list[1:]
|
||
assert len(update_calls) == 2
|
||
expected = [("Екатеринбург", 56.838), ("Нижний Тагил", 57.910)]
|
||
for call, (expected_city, expected_lat) in zip(update_calls, expected, strict=True):
|
||
sql = str(call.args[0])
|
||
params = call.args[1]
|
||
assert "IS NOT DISTINCT FROM" in sql
|
||
assert params["addr"] == "ул. Победы, 30"
|
||
assert params["city"] == expected_city
|
||
assert params["lat"] == pytest.approx(expected_lat)
|
||
|
||
assert result.addresses_processed == 2
|
||
assert result.addresses_geocoded == 2
|
||
assert result.listings_updated == 2
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_geocode_missing_null_city_group_uses_is_not_distinct_from() -> None:
|
||
"""city IS NULL — своя группа. city_hint=None передаётся геокодеру, UPDATE
|
||
использует IS NOT DISTINCT FROM (обычный `=` никогда не совпал бы с NULL —
|
||
группа NULL-city вообще не обновилась бы обычным equality-сравнением)."""
|
||
rows = [{"address": "ул. Дружинина, 33", "city": None, "listings_count": 2}]
|
||
|
||
db = MagicMock()
|
||
select_result = MagicMock()
|
||
select_result.mappings.return_value.all.return_value = rows
|
||
update_result = MagicMock()
|
||
update_result.rowcount = 2
|
||
db.execute.side_effect = [select_result, update_result]
|
||
|
||
with patch(
|
||
"app.tasks.geocode_missing.geocode",
|
||
new_callable=AsyncMock,
|
||
return_value=_make_geocode_result("nominatim"),
|
||
) as mock_geo:
|
||
result = await geocode_missing_listings(db, batch_size=200)
|
||
|
||
mock_geo.assert_called_once_with("ул. Дружинина, 33", db, city_hint=None)
|
||
|
||
update_call = db.execute.call_args_list[1]
|
||
sql = str(update_call.args[0])
|
||
params = update_call.args[1]
|
||
assert "IS NOT DISTINCT FROM" in sql
|
||
assert params["city"] is None
|
||
assert result.listings_updated == 2
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_geocode_missing_select_groups_by_address_and_city() -> None:
|
||
"""SELECT содержит GROUP BY address, city — НЕ только по address (#2594)."""
|
||
db = MagicMock()
|
||
select_result = MagicMock()
|
||
select_result.mappings.return_value.all.return_value = []
|
||
db.execute.return_value = select_result
|
||
|
||
with patch("app.tasks.geocode_missing.geocode", new_callable=AsyncMock):
|
||
await geocode_missing_listings(db, batch_size=10)
|
||
|
||
first_call = db.execute.call_args_list[0]
|
||
sql_text = str(first_call[0][0])
|
||
assert "GROUP BY address, city" in sql_text
|
||
assert "SELECT address, city, COUNT(*)" in sql_text
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_geocode_missing_failed_pair_tried_at_update_scoped_to_city() -> None:
|
||
"""Failed geocode (geo=None) для (address, city) → UPDATE tried_at ограничен
|
||
ЭТОЙ парой (IS NOT DISTINCT FROM city), не всеми строками с тем же address."""
|
||
rows = [{"address": "несуществующий адрес", "city": "Нижний Тагил", "listings_count": 1}]
|
||
|
||
db = MagicMock()
|
||
select_result = MagicMock()
|
||
select_result.mappings.return_value.all.return_value = rows
|
||
tried_at_result = MagicMock()
|
||
db.execute.side_effect = [select_result, tried_at_result]
|
||
|
||
with patch(
|
||
"app.tasks.geocode_missing.geocode",
|
||
new_callable=AsyncMock,
|
||
return_value=None,
|
||
) as mock_geo:
|
||
result = await geocode_missing_listings(db, batch_size=200)
|
||
|
||
mock_geo.assert_called_once_with("несуществующий адрес", db, city_hint="Нижний Тагил")
|
||
assert result.addresses_failed == 1
|
||
|
||
update_call = db.execute.call_args_list[1]
|
||
sql = str(update_call.args[0])
|
||
params = update_call.args[1]
|
||
assert "IS NOT DISTINCT FROM" in sql
|
||
assert params["city"] == "Нижний Тагил"
|
||
|
||
|
||
# ── #2604 п.1/п.2: UPDATE decisions — locked in by test, not just comment ────
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_geocode_missing_success_update_not_filtered_by_is_active() -> None:
|
||
"""Decision #2604 п.1 (UPDATE lat/lon): намеренно БЕЗ `is_active` в WHERE.
|
||
|
||
Координаты — свойство физического адреса (address, city), не свойство
|
||
конкретного listing. is_active=false дубликат ЭТОЙ ЖЕ пары никогда не будет
|
||
независимо отобран SELECT'ом (он навсегда исключён оттуда) — без unfiltered
|
||
UPDATE такой дубликат остался бы с NULL lat/lon навсегда, хотя ответ уже
|
||
получен и оплачен Nominatim-вызовом активного листинга.
|
||
"""
|
||
rows = [{"address": "ул. Тестовая, 1", "city": "Екатеринбург", "listings_count": 2}]
|
||
db = MagicMock()
|
||
select_result = MagicMock()
|
||
select_result.mappings.return_value.all.return_value = rows
|
||
update_result = MagicMock()
|
||
update_result.rowcount = 2
|
||
db.execute.side_effect = [select_result, update_result]
|
||
|
||
with patch(
|
||
"app.tasks.geocode_missing.geocode",
|
||
new_callable=AsyncMock,
|
||
return_value=_make_geocode_result("nominatim"),
|
||
):
|
||
await geocode_missing_listings(db, batch_size=200)
|
||
|
||
update_call = db.execute.call_args_list[1]
|
||
sql = str(update_call.args[0])
|
||
assert "is_active" not in sql
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_geocode_missing_notfound_tried_at_update_not_filtered_by_is_active() -> None:
|
||
"""Decision #2604 п.2 (geo is None → tried_at UPDATE): намеренно БЕЗ `is_active`.
|
||
|
||
tried_at — backoff-метка для (address, city) КАК ТЕКСТА, не для конкретного
|
||
listing; is_active=false дубликат и так никогда не переотбирается SELECT'ом.
|
||
Единственный сценарий где это важно — реактивация (is_active → true) той же
|
||
строки: backoff уже стоит и корректно защищает от немедленного повтора
|
||
заведомо неудачного адреса.
|
||
"""
|
||
rows = [{"address": "несуществующий адрес", "city": None, "listings_count": 1}]
|
||
db = MagicMock()
|
||
select_result = MagicMock()
|
||
select_result.mappings.return_value.all.return_value = rows
|
||
tried_at_result = MagicMock()
|
||
db.execute.side_effect = [select_result, tried_at_result]
|
||
|
||
with patch(
|
||
"app.tasks.geocode_missing.geocode",
|
||
new_callable=AsyncMock,
|
||
return_value=None,
|
||
):
|
||
await geocode_missing_listings(db, batch_size=200)
|
||
|
||
update_call = db.execute.call_args_list[1]
|
||
sql = str(update_call.args[0])
|
||
assert "is_active" not in sql
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_geocode_missing_exception_tried_at_update_not_filtered_by_is_active() -> None:
|
||
"""Decision #2604 п.2 (geocode() raises → tried_at UPDATE): та же логика, что и
|
||
в NOT-FOUND ветке выше — намеренно БЕЗ `is_active`, зафиксировано тестом."""
|
||
rows = [{"address": "ул. Битая, 99", "city": None, "listings_count": 1}]
|
||
db = MagicMock()
|
||
select_result = MagicMock()
|
||
select_result.mappings.return_value.all.return_value = rows
|
||
tried_at_result = MagicMock()
|
||
db.execute.side_effect = [select_result, tried_at_result]
|
||
|
||
with patch(
|
||
"app.tasks.geocode_missing.geocode",
|
||
new_callable=AsyncMock,
|
||
side_effect=RuntimeError("timeout"),
|
||
):
|
||
await geocode_missing_listings(db, batch_size=200)
|
||
|
||
update_call = db.execute.call_args_list[1]
|
||
sql = str(update_call.args[0])
|
||
assert "is_active" not in sql
|
||
|
||
|
||
# ── Integration-style: estimator Avito exclusion removed ─────────────────────
|
||
|
||
|
||
def test_estimator_fetch_analogs_includes_avito() -> None:
|
||
"""estimator._fetch_analogs не содержит AND source <> 'avito'."""
|
||
import inspect
|
||
|
||
from app.services import estimator
|
||
|
||
source = inspect.getsource(estimator._fetch_analogs)
|
||
assert (
|
||
"source <> 'avito'" not in source
|
||
), "Avito exclusion должен быть удалён из estimator._fetch_analogs"
|
||
|
||
|
||
# ── Admin endpoint smoke (schema-only, no real DB) ────────────────────────────
|
||
|
||
|
||
def test_admin_geocode_missing_endpoint_exists() -> None:
|
||
"""GET /api/v1/admin/scrape/geocode-missing-listings/status → endpoint registered."""
|
||
from fastapi import FastAPI
|
||
from fastapi.testclient import TestClient
|
||
|
||
from app.api.v1 import admin as admin_module
|
||
from app.core.db import get_db
|
||
|
||
app = FastAPI()
|
||
app.include_router(admin_module.router, prefix="/api/v1/admin")
|
||
|
||
mock_db = MagicMock()
|
||
# status endpoint calls execute twice — stats + unique + cache_size
|
||
stats_result = MagicMock()
|
||
stats_result.mappings.return_value.all.return_value = []
|
||
scalar_result = MagicMock()
|
||
scalar_result.scalar.return_value = 0
|
||
mock_db.execute.side_effect = [stats_result, scalar_result, scalar_result]
|
||
|
||
app.dependency_overrides[get_db] = lambda: mock_db
|
||
|
||
client = TestClient(app)
|
||
resp = client.get("/api/v1/admin/scrape/geocode-missing-listings/status")
|
||
# Если endpoint зарегистрирован корректно — не 404
|
||
assert resp.status_code != 404
|
||
|
||
|
||
def test_admin_geocode_missing_post_dry_run_endpoint_exists() -> None:
|
||
"""POST /api/v1/admin/scrape/geocode-missing-listings?dry_run=true → endpoint registered."""
|
||
from fastapi import FastAPI
|
||
from fastapi.testclient import TestClient
|
||
|
||
from app.api.v1 import admin as admin_module
|
||
from app.core.db import get_db
|
||
|
||
app = FastAPI()
|
||
app.include_router(admin_module.router, prefix="/api/v1/admin")
|
||
|
||
# Mock db: не нужен реальный — тест только на routing
|
||
mock_db = MagicMock()
|
||
app.dependency_overrides[get_db] = lambda: mock_db
|
||
|
||
client = TestClient(app)
|
||
# Без реального geocode — mocked через patch
|
||
with patch(
|
||
"app.tasks.geocode_missing.geocode_missing_listings",
|
||
new_callable=AsyncMock,
|
||
return_value=GeocodeBackfillResult(
|
||
addresses_total=0,
|
||
addresses_processed=0,
|
||
addresses_geocoded=0,
|
||
addresses_failed=0,
|
||
listings_updated=0,
|
||
cache_hits=0,
|
||
cache_misses=0,
|
||
duration_sec=0.01,
|
||
),
|
||
):
|
||
resp = client.post(
|
||
"/api/v1/admin/scrape/geocode-missing-listings?dry_run=true&batch_size=10"
|
||
)
|
||
|
||
assert resp.status_code != 404
|
||
if resp.status_code == 200:
|
||
data = resp.json()
|
||
assert "status" in data
|
||
assert "addresses_total" in data
|
||
|
||
|
||
# ── admin.geocode_missing (per-ID endpoint) city_hint (#2594 шаг 2/3) ────────
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_admin_geocode_missing_passes_city_hint() -> None:
|
||
"""POST /admin/geocode-missing читает city из SELECT и передаёт как city_hint.
|
||
|
||
Раньше endpoint читал только row["address"] и звал geocode(clean, db) без
|
||
города — голый тагильский адрес без города в тексте уходил в Екатеринбург.
|
||
"""
|
||
from app.api.v1 import admin as admin_module
|
||
|
||
rows = [{"id": 55, "address": "ул. Победы, 30", "city": "Нижний Тагил"}]
|
||
|
||
db = MagicMock()
|
||
select_result = MagicMock()
|
||
select_result.mappings.return_value.all.return_value = rows
|
||
update_result = MagicMock()
|
||
remaining_result = MagicMock()
|
||
remaining_result.scalar.return_value = 0
|
||
db.execute.side_effect = [select_result, update_result, remaining_result]
|
||
|
||
geo = GeocodeResult(
|
||
lat=57.910,
|
||
lon=59.985,
|
||
full_address="Нижний Тагил, ул. Победы, 30",
|
||
provider="nominatim", # type: ignore[arg-type]
|
||
confidence="exact",
|
||
)
|
||
|
||
with patch(
|
||
"app.api.v1.admin.geocode",
|
||
new_callable=AsyncMock,
|
||
return_value=geo,
|
||
) as mock_geo:
|
||
result = await admin_module.geocode_missing(db, limit=100, target="listings")
|
||
|
||
mock_geo.assert_called_once_with("ул. Победы, 30", db, city_hint="Нижний Тагил")
|
||
assert result["geocoded"] == 1
|
||
assert result["skipped"] == 0
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_admin_geocode_missing_select_includes_city_column() -> None:
|
||
"""SELECT в admin.geocode_missing содержит колонку city (#2594)."""
|
||
from app.api.v1 import admin as admin_module
|
||
|
||
db = MagicMock()
|
||
select_result = MagicMock()
|
||
select_result.mappings.return_value.all.return_value = []
|
||
remaining_result = MagicMock()
|
||
remaining_result.scalar.return_value = 0
|
||
db.execute.side_effect = [select_result, remaining_result]
|
||
|
||
with patch("app.api.v1.admin.geocode", new_callable=AsyncMock):
|
||
await admin_module.geocode_missing(db, limit=100, target="listings")
|
||
|
||
first_call = db.execute.call_args_list[0]
|
||
sql_text = str(first_call[0][0])
|
||
assert "SELECT id, address, city" in sql_text
|