chore(tradein): удалить остатки источника N1 (#2593)
All checks were successful
CI / changes (pull_request) Successful in 9s
CI Trade-In / changes (pull_request) Successful in 9s
CI / backend-tests (pull_request) Has been skipped
CI / frontend-tests (pull_request) Has been skipped
CI / openapi-codegen-check (pull_request) Has been skipped
CI Trade-In / frontend-checks (pull_request) Successful in 1m13s
CI Trade-In / backend-tests (pull_request) Successful in 2m29s
All checks were successful
CI / changes (pull_request) Successful in 9s
CI Trade-In / changes (pull_request) Successful in 9s
CI / backend-tests (pull_request) Has been skipped
CI / frontend-tests (pull_request) Has been skipped
CI / openapi-codegen-check (pull_request) Has been skipped
CI Trade-In / frontend-checks (pull_request) Successful in 1m13s
CI Trade-In / backend-tests (pull_request) Successful in 2m29s
N1 не собирается с 16 июня, в scrape_schedules его нет. Миграция 165 удалила источник на 90% (allowlist/scheduler/settings) — оставались точечные литералы: - SourcesMap.tsx: цвет для мёртвого source в легенде карты (fallback серый). - admin.py geocode-missing: N1-ветка address-плейсхолдер фильтра + стале докстринги, упоминавшие N1 как активный источник listings. - test_estimator_source_quota.py: докстринг регрессии с упоминанием N1 среди вытесняемых источников. Данные (382 listings source='n1', is_active=false) не трогаются — все поверхности уже провайдер-агностичны с safe fallback для неизвестных id (source-registry.ts, trade_in_pdf.py _SOURCE_LOGO_COLORS.get, SourcesMap.tsx colorForSource). Денормализованные счётчики (TOTAL_SOURCES/mappers.ts, _TOTAL_SOURCES/trade_in_pdf.py, LIVE_SOURCE_COUNT/source-registry.ts) уже производные от актуальных ростеров без n1 — индексация не затронута.
This commit is contained in:
parent
57fe26d3f3
commit
146139fbf9
3 changed files with 44 additions and 29 deletions
|
|
@ -246,7 +246,8 @@ def _clean_address_for_geocode(addr: str) -> str:
|
||||||
"""Чистим address для геокодера.
|
"""Чистим address для геокодера.
|
||||||
|
|
||||||
Cian отдаёт «улица Латвийская, 56/3 · р-н Чкаловский» — суффикс ' · ...'
|
Cian отдаёт «улица Латвийская, 56/3 · р-н Чкаловский» — суффикс ' · ...'
|
||||||
мешает Nominatim. Берём часть до ' · '. N1 отдаёт «Репина, 75/2 стр.» — ок.
|
мешает Nominatim. Берём часть до ' · '. Остальные источники такого суффикса
|
||||||
|
не используют — адрес остаётся без изменений.
|
||||||
"""
|
"""
|
||||||
main = addr.split(" · ")[0].strip()
|
main = addr.split(" · ")[0].strip()
|
||||||
return main or addr
|
return main or addr
|
||||||
|
|
@ -260,7 +261,7 @@ async def geocode_missing(
|
||||||
) -> dict:
|
) -> dict:
|
||||||
"""Геокодинг listings ИЛИ deals у которых нет lat/lon (используя address).
|
"""Геокодинг listings ИЛИ deals у которых нет lat/lon (используя address).
|
||||||
|
|
||||||
target=listings (по умолч.) — объявления Cian/N1; target=deals — сделки Росреестра.
|
target=listings (по умолч.) — объявления; target=deals — сделки Росреестра.
|
||||||
Чанк-обработка с бюджетом по времени (~240с, заведомо меньше cron
|
Чанк-обработка с бюджетом по времени (~240с, заведомо меньше cron
|
||||||
`curl -m 320`): за вызов геокодим сколько успеваем, остаток уходит в
|
`curl -m 320`): за вызов геокодим сколько успеваем, остаток уходит в
|
||||||
`remaining`, cron вызывает в цикле пока `remaining` > 0.
|
`remaining`, cron вызывает в цикле пока `remaining` > 0.
|
||||||
|
|
@ -269,12 +270,8 @@ async def geocode_missing(
|
||||||
адреса не выбираются повторно 7 дней → cron-loop завершается, не зацикливается.
|
адреса не выбираются повторно 7 дней → cron-loop завершается, не зацикливается.
|
||||||
geom обновляется автоматически триггером.
|
geom обновляется автоматически триггером.
|
||||||
"""
|
"""
|
||||||
# Доп. фильтр для listings — у Avito/N1 встречаются плейсхолдер-адреса.
|
# Доп. фильтр для listings — у Avito встречаются плейсхолдер-адреса.
|
||||||
extra_filter = (
|
extra_filter = "AND address NOT LIKE '%(Avito)%'" if target == "listings" else ""
|
||||||
"AND address NOT LIKE '%(Avito)%' AND address NOT LIKE '%(N1)%'"
|
|
||||||
if target == "listings"
|
|
||||||
else ""
|
|
||||||
)
|
|
||||||
rows = (
|
rows = (
|
||||||
db.execute(
|
db.execute(
|
||||||
text(
|
text(
|
||||||
|
|
|
||||||
|
|
@ -1,9 +1,10 @@
|
||||||
"""Tests for _fetch_analogs per-address cap and per-source quota (source starvation fix).
|
"""Tests for _fetch_analogs per-address cap and per-source quota (source starvation fix).
|
||||||
|
|
||||||
Regression: Монтёрская 8/2 — 91 Avito listings с distance=0 выдавливали
|
Regression: Монтёрская 8/2 — 91 Avito listings с distance=0 выдавливали
|
||||||
Cian/Yandex/N1 из топ-50, т.к. pure-distance sort + LIMIT 50.
|
Cian/Yandex из топ-50, т.к. pure-distance sort + LIMIT 50.
|
||||||
Fix: MAX_ANALOGS_PER_ADDRESS cap в SQL + MIN_ANALOGS_PER_SOURCE quota в Python.
|
Fix: MAX_ANALOGS_PER_ADDRESS cap в SQL + MIN_ANALOGS_PER_SOURCE quota в Python.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import os
|
import os
|
||||||
|
|
||||||
# Settings requires DATABASE_URL at init time. Set dummy DSN before any app import.
|
# Settings requires DATABASE_URL at init time. Set dummy DSN before any app import.
|
||||||
|
|
@ -15,6 +16,7 @@ from unittest.mock import MagicMock
|
||||||
|
|
||||||
# ── Helpers ───────────────────────────────────────────────────────────────────
|
# ── Helpers ───────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
def _make_listing(
|
def _make_listing(
|
||||||
*,
|
*,
|
||||||
source: str,
|
source: str,
|
||||||
|
|
@ -58,6 +60,7 @@ def _make_db_mock(rows: list[dict[str, Any]]) -> MagicMock:
|
||||||
|
|
||||||
# ── Test 1: per-address cap ───────────────────────────────────────────────────
|
# ── Test 1: per-address cap ───────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
def test_address_cap_limits_per_address_listings() -> None:
|
def test_address_cap_limits_per_address_listings() -> None:
|
||||||
"""_fetch_analogs caps at MAX_ANALOGS_PER_ADDRESS listings from a single address.
|
"""_fetch_analogs caps at MAX_ANALOGS_PER_ADDRESS listings from a single address.
|
||||||
|
|
||||||
|
|
@ -91,6 +94,7 @@ def test_address_cap_limits_per_address_listings() -> None:
|
||||||
|
|
||||||
# ── Test 2: source quota (regression for Cian starvation) ────────────────────
|
# ── Test 2: source quota (regression for Cian starvation) ────────────────────
|
||||||
|
|
||||||
|
|
||||||
def test_source_quota_prevents_cian_starvation() -> None:
|
def test_source_quota_prevents_cian_starvation() -> None:
|
||||||
"""MIN_ANALOGS_PER_SOURCE guarantees Cian is represented despite Avito dominance.
|
"""MIN_ANALOGS_PER_SOURCE guarantees Cian is represented despite Avito dominance.
|
||||||
|
|
||||||
|
|
@ -103,22 +107,28 @@ def test_source_quota_prevents_cian_starvation() -> None:
|
||||||
# SQL already applied address cap. Simulate SQL result after cap:
|
# SQL already applied address cap. Simulate SQL result after cap:
|
||||||
# 5 avito (cap applied to large block), 8 cian (different address, 200m away).
|
# 5 avito (cap applied to large block), 8 cian (different address, 200m away).
|
||||||
avito_rows = [
|
avito_rows = [
|
||||||
_make_listing(source="avito", address="ул. Монтёрская, 8/2", distance_m=0.0,
|
_make_listing(
|
||||||
relevance_score=float(i) * 0.01)
|
source="avito",
|
||||||
|
address="ул. Монтёрская, 8/2",
|
||||||
|
distance_m=0.0,
|
||||||
|
relevance_score=float(i) * 0.01,
|
||||||
|
)
|
||||||
for i in range(5)
|
for i in range(5)
|
||||||
]
|
]
|
||||||
cian_rows = [
|
cian_rows = [
|
||||||
_make_listing(source="cian", address="ул. Монтёрская, 1", distance_m=200.0,
|
_make_listing(
|
||||||
relevance_score=0.2 + float(i) * 0.01)
|
source="cian",
|
||||||
|
address="ул. Монтёрская, 1",
|
||||||
|
distance_m=200.0,
|
||||||
|
relevance_score=0.2 + float(i) * 0.01,
|
||||||
|
)
|
||||||
for i in range(8)
|
for i in range(8)
|
||||||
]
|
]
|
||||||
# SQL returns avito first (better relevance), then cian
|
# SQL returns avito first (better relevance), then cian
|
||||||
sql_rows = avito_rows + cian_rows
|
sql_rows = avito_rows + cian_rows
|
||||||
db = _make_db_mock(sql_rows)
|
db = _make_db_mock(sql_rows)
|
||||||
|
|
||||||
result, _, _ = _fetch_analogs(
|
result, _, _ = _fetch_analogs(db, lat=56.838, lon=60.595, rooms=1, area=38.0, radius_m=1000)
|
||||||
db, lat=56.838, lon=60.595, rooms=1, area=38.0, radius_m=1000
|
|
||||||
)
|
|
||||||
|
|
||||||
cian_count = sum(1 for r in result if r["source"] == "cian")
|
cian_count = sum(1 for r in result if r["source"] == "cian")
|
||||||
expected_min = min(8, MIN_ANALOGS_PER_SOURCE)
|
expected_min = min(8, MIN_ANALOGS_PER_SOURCE)
|
||||||
|
|
@ -130,6 +140,7 @@ def test_source_quota_prevents_cian_starvation() -> None:
|
||||||
|
|
||||||
# ── Test 3: no source starvation when quota > supply ─────────────────────────
|
# ── Test 3: no source starvation when quota > supply ─────────────────────────
|
||||||
|
|
||||||
|
|
||||||
def test_source_quota_includes_all_when_supply_below_min() -> None:
|
def test_source_quota_includes_all_when_supply_below_min() -> None:
|
||||||
"""When a source has fewer listings than MIN_ANALOGS_PER_SOURCE, all are included.
|
"""When a source has fewer listings than MIN_ANALOGS_PER_SOURCE, all are included.
|
||||||
|
|
||||||
|
|
@ -138,31 +149,38 @@ def test_source_quota_includes_all_when_supply_below_min() -> None:
|
||||||
from app.services.estimator import _fetch_analogs
|
from app.services.estimator import _fetch_analogs
|
||||||
|
|
||||||
avito_rows = [
|
avito_rows = [
|
||||||
_make_listing(source="avito", address="ул. Монтёрская, 8/2", distance_m=0.0,
|
_make_listing(
|
||||||
relevance_score=float(i) * 0.01)
|
source="avito",
|
||||||
|
address="ул. Монтёрская, 8/2",
|
||||||
|
distance_m=0.0,
|
||||||
|
relevance_score=float(i) * 0.01,
|
||||||
|
)
|
||||||
for i in range(5)
|
for i in range(5)
|
||||||
]
|
]
|
||||||
cian_rows = [
|
cian_rows = [
|
||||||
_make_listing(source="cian", address="ул. Монтёрская, 3", distance_m=300.0,
|
_make_listing(
|
||||||
relevance_score=0.3 + float(i) * 0.01)
|
source="cian",
|
||||||
|
address="ул. Монтёрская, 3",
|
||||||
|
distance_m=300.0,
|
||||||
|
relevance_score=0.3 + float(i) * 0.01,
|
||||||
|
)
|
||||||
for i in range(3)
|
for i in range(3)
|
||||||
]
|
]
|
||||||
sql_rows = avito_rows + cian_rows
|
sql_rows = avito_rows + cian_rows
|
||||||
db = _make_db_mock(sql_rows)
|
db = _make_db_mock(sql_rows)
|
||||||
|
|
||||||
result, _, _ = _fetch_analogs(
|
result, _, _ = _fetch_analogs(db, lat=56.838, lon=60.595, rooms=1, area=38.0, radius_m=1000)
|
||||||
db, lat=56.838, lon=60.595, rooms=1, area=38.0, radius_m=1000
|
|
||||||
)
|
|
||||||
|
|
||||||
cian_count = sum(1 for r in result if r["source"] == "cian")
|
cian_count = sum(1 for r in result if r["source"] == "cian")
|
||||||
assert cian_count == 3, (
|
assert (
|
||||||
f"All 3 Cian listings (below MIN quota) must be included, got {cian_count}"
|
cian_count == 3
|
||||||
)
|
), f"All 3 Cian listings (below MIN quota) must be included, got {cian_count}"
|
||||||
assert len(result) == 8 # 5 avito + 3 cian
|
assert len(result) == 8 # 5 avito + 3 cian
|
||||||
|
|
||||||
|
|
||||||
# ── Test 4: fallback signal preserved ────────────────────────────────────────
|
# ── Test 4: fallback signal preserved ────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
def test_fallback_signal_reflects_radius() -> None:
|
def test_fallback_signal_reflects_radius() -> None:
|
||||||
"""_fetch_analogs returns correct fallback_used boolean based on radius_m.
|
"""_fetch_analogs returns correct fallback_used boolean based on radius_m.
|
||||||
|
|
||||||
|
|
@ -172,8 +190,9 @@ def test_fallback_signal_reflects_radius() -> None:
|
||||||
from app.services.estimator import DEFAULT_RADIUS_M, FALLBACK_RADIUS_M, _fetch_analogs
|
from app.services.estimator import DEFAULT_RADIUS_M, FALLBACK_RADIUS_M, _fetch_analogs
|
||||||
|
|
||||||
rows = [
|
rows = [
|
||||||
_make_listing(source="avito", address="ул. Ленина, 1", distance_m=100.0,
|
_make_listing(
|
||||||
relevance_score=0.1),
|
source="avito", address="ул. Ленина, 1", distance_m=100.0, relevance_score=0.1
|
||||||
|
),
|
||||||
]
|
]
|
||||||
|
|
||||||
db_default = _make_db_mock(rows)
|
db_default = _make_db_mock(rows)
|
||||||
|
|
|
||||||
|
|
@ -89,7 +89,6 @@ const SOURCE_COLORS: Record<string, string> = {
|
||||||
cian: "#1baf7a",
|
cian: "#1baf7a",
|
||||||
yandex: "#eda100",
|
yandex: "#eda100",
|
||||||
domklik: "#008300",
|
domklik: "#008300",
|
||||||
n1: "#4a3aa7",
|
|
||||||
};
|
};
|
||||||
const FALLBACK_SOURCE_COLOR = "#6b7280"; // неизвестный/прочий листинговый источник
|
const FALLBACK_SOURCE_COLOR = "#6b7280"; // неизвестный/прочий листинговый источник
|
||||||
const COLOR_TARGET = "#f59e0b"; // та же семантика, что и MapCard.COLOR_TARGET
|
const COLOR_TARGET = "#f59e0b"; // та же семантика, что и MapCard.COLOR_TARGET
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue