feat(velocity): migrate D2 sales source from stale sale_graph to objective_corpus_room_month

Closes #156
This commit is contained in:
lekss361 2026-05-15 09:16:03 +03:00
parent 37fd4b5a8e
commit fa6bbafa3f
3 changed files with 174 additions and 168 deletions

View file

@ -673,7 +673,14 @@ def object_detail(db: Session, obj_id: int) -> dict[str, Any] | None:
def object_sale_graph(
db: Session, obj_id: int, type_filter: str | None = None
) -> list[dict[str, Any]]:
"""Time-series продаж per-ЖК. Latest snapshot."""
"""Time-series продаж per-ЖК. Latest snapshot.
NOTE: намеренно оставлен на domrf_kn_sale_graph это внутренний
per-object detail view для PRINZIP-аналитики (/api/v1/analytics/object/*).
Миграция на objective_corpus_room_month требует отдельного pr: там другая
гранулярность (corpus × room_bucket), а не obj_id.
Данные stale (newest 2026-01) приемлемо для исторического графика.
"""
where_type = ""
params: dict[str, Any] = {"obj": obj_id}
if type_filter:
@ -930,7 +937,13 @@ def prinzip_funnel_by_object(db: Session) -> list[dict[str, Any]]:
def prinzip_objects_with_velocity(db: Session) -> list[dict[str, Any]]:
"""Список 28 PRINZIP-ЖК с агрегатами + apartments-velocity sparkline data."""
"""Список 28 PRINZIP-ЖК с агрегатами + apartments-velocity sparkline data.
NOTE: sparkline (velocity CTE) намеренно оставлен на domrf_kn_sale_graph.
objective_corpus_room_month не имеет obj_id требует JOIN через
objective_complex_mapping по project_name. Это отдельная задача рефакторинга
admin-view. Данные stale (newest 2026-01), sparkline визуально OK для тренда.
"""
rows = (
db.execute(
text(
@ -1086,43 +1099,47 @@ def _velocity_baseline(
target_class: str | None,
) -> dict[str, Any]:
"""Median monthly sales velocity (apartments/month per ЖК) from
domrf_kn_sale_graph for objects in the same район+class over last 24 mo.
objective_corpus_room_month for objects in the same район+class over last 24 mo.
Migrated from domrf_kn_sale_graph (stale since 2026-01) to
objective_corpus_room_month (updated weekly via Objective API).
objective_corpus_room_month.district matches domrf_kn_objects.district_name.
class filter uses 'class' column (Комфорт/Бизнес/Стандарт).
Returns dict {realised_per_month_median, realised_per_month_avg,
objects_count, observations}. All-None means no data caller falls back.
"""
where_class = "AND o.obj_class = :cls" if target_class else ""
params: dict[str, Any] = {
"rc": region_code,
"dn": district_name,
}
# Objective class naming: "Комфорт", "Бизнес", "Стандарт" — capitalised.
# domrf obj_class may differ in case; apply ILIKE for robustness.
where_class = "AND LOWER(crm.class) = LOWER(:cls)" if target_class else ""
params: dict[str, Any] = {"dn": district_name}
if target_class:
params["cls"] = target_class
# region_code not used — objective_corpus_room_month covers only EKB (region 66).
# district filter is sufficient for locality. If district returns no rows,
# caller falls back to rosreestr_fallback path (unchanged behaviour).
_ = region_code # retained in signature for backward compat
row = (
db.execute(
text(
f"""
WITH obj_pool AS (
SELECT o.obj_id
FROM domrf_kn_objects o
WHERE o.region_cd = :rc
AND o.district_name = :dn
WITH per_project_month AS (
SELECT project_name,
report_month,
SUM(deals_total_count) AS month_units
FROM objective_corpus_room_month crm
WHERE crm.district = :dn
{where_class}
),
sg AS (
SELECT sg.obj_id, sg.realised
FROM domrf_kn_sale_graph sg
JOIN obj_pool p ON p.obj_id = sg.obj_id
WHERE sg.type = 'apartments'
AND sg.realised IS NOT NULL
AND sg.report_month >= NOW() - INTERVAL '24 months'
AND crm.deals_total_count > 0
AND crm.report_month >= NOW() - INTERVAL '24 months'
GROUP BY project_name, report_month
)
SELECT
AVG(realised) AS avg_pm,
PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY realised) AS median_pm,
COUNT(DISTINCT obj_id) AS objects,
COUNT(*) AS observations
FROM sg
AVG(month_units) AS avg_pm,
PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY month_units) AS median_pm,
COUNT(DISTINCT project_name) AS objects,
COUNT(*) AS observations
FROM per_project_month
"""
),
params,
@ -1182,6 +1199,10 @@ def _district_velocity_trend(db: Session, *, district_name: str) -> tuple[float
"""Ratio realised: recent_6mo / prior_6mo. >1.5 — рынок горит, <0.7 —
остывает. Считаем за окно 12 мес: H1 2025 vs H2 2025+.
Мигрировано с domrf_kn_sale_graph (stale 2026-01) на
objective_corpus_room_month (обновляется еженедельно).
deals_total_count заменяет realised (DDU + DKP всего по корпусу).
Возвращает (ratio, recent_units, prior_units). None если данных мало.
"""
row = (
@ -1189,17 +1210,16 @@ def _district_velocity_trend(db: Session, *, district_name: str) -> tuple[float
text(
"""
SELECT
SUM(sg.realised) FILTER (WHERE sg.report_month >= DATE '2025-07-01')
SUM(crm.deals_total_count)
FILTER (WHERE crm.report_month >= DATE '2025-07-01')
AS recent,
SUM(sg.realised) FILTER (WHERE sg.report_month BETWEEN DATE '2025-01-01'
AND DATE '2025-06-30')
SUM(crm.deals_total_count)
FILTER (WHERE crm.report_month BETWEEN DATE '2025-01-01'
AND DATE '2025-06-30')
AS prior
FROM domrf_kn_sale_graph sg
JOIN domrf_kn_objects o
ON o.obj_id = sg.obj_id
AND o.snapshot_date = sg.snapshot_date
WHERE sg.type = 'apartments'
AND o.district_name = :dn
FROM objective_corpus_room_month crm
WHERE crm.district = :dn
AND crm.deals_total_count > 0
"""
),
{"dn": district_name},
@ -1450,37 +1470,40 @@ def _elasticity_coef(
target_class: str | None,
elasticity_window_months: int = 24,
) -> dict[str, Any]:
"""Fit log-log regression LN(realised) ~ LN(price_avg) on sale_graph
observations for the same район+class. Returns elasticity (slope), ,
n. Falls back to FALLBACK_ELASTICITY if data thin or regression weak."""
where_class = "AND o.obj_class = :cls" if target_class else ""
"""Fit log-log regression LN(deals_total_count) ~ LN(price_per_m2) on
objective_corpus_room_month observations for the same район+class.
Returns elasticity (slope), , n.
Falls back to FALLBACK_ELASTICITY if data thin or regression weak.
Мигрировано с domrf_kn_sale_graph (stale 2026-01) на
objective_corpus_room_month (обновляется еженедельно).
Маппинг: realised deals_total_count,
price_avg deals_total_avg_price_thousand_rub_per_m2.
LN-масштаб цены (тыс.руб/м²) сохраняет slope relative magnitude slope
не зависит от единиц (аддитивный сдвиг в LN пространстве).
"""
where_class = "AND LOWER(crm.class) = LOWER(:cls)" if target_class else ""
params: dict[str, Any] = {
"rc": region_code,
"dn": district_name,
"ew": elasticity_window_months,
}
if target_class:
params["cls"] = target_class
_ = region_code # retained for backward compat; objective data covers EKB only
row = (
db.execute(
text(
f"""
WITH obj_pool AS (
SELECT o.obj_id
FROM domrf_kn_objects o
WHERE o.region_cd = :rc
AND o.district_name = :dn
WITH pts AS (
SELECT
LN(crm.deals_total_count::float8) AS y,
LN(crm.deals_total_avg_price_thousand_rub_per_m2::float8) AS x
FROM objective_corpus_room_month crm
WHERE crm.district = :dn
{where_class}
),
pts AS (
SELECT LN(sg.realised)::float8 AS y,
LN(sg.price_avg)::float8 AS x
FROM domrf_kn_sale_graph sg
JOIN obj_pool p ON p.obj_id = sg.obj_id
WHERE sg.type = 'apartments'
AND sg.realised IS NOT NULL AND sg.realised > 0
AND sg.price_avg IS NOT NULL AND sg.price_avg > 0
AND sg.report_month >= NOW() - (:ew || ' months')::interval
AND crm.deals_total_count > 0
AND crm.deals_total_avg_price_thousand_rub_per_m2 > 0
AND crm.report_month >= NOW() - (:ew || ' months')::interval
)
SELECT
regr_slope(y, x) AS slope,
@ -1521,83 +1544,64 @@ def _elasticity_per_bucket_coef(
fallback: dict[str, Any],
elasticity_window_months: int = 24,
) -> dict[str, dict[str, Any]]:
"""Per-bucket эластичность (Tier 3): группируем sale_graph-наблюдения по
«доминирующему bucket» каждого ЖК (mode total_area из domrf_kn_flats),
регрессия log-log для каждой группы. Студии vs 80+ м² реагируют на цену
по-разному.
"""Per-bucket эластичность (Tier 3): группируем objective_corpus_room_month
по room_bucket регрессия log-log для каждой группы.
Студии vs 80+ м² реагируют на цену по-разному.
Мигрировано с domrf_kn_sale_graph + domrf_kn_flats (stale 2026-01) на
objective_corpus_room_month (обновляется еженедельно).
objective_corpus_room_month уже содержит room_bucket напрямую нет
необходимости в MODE-агрегации domrf_kn_flats.
Маппинг room_bucket _BUCKET_PRETTY ключи:
'студия' '1-Студия'
'1' '2-1-к'
'2' '3-2-к'
'3' '4-3-к'
'4'/'5+' '5-80+ м²'
Returns: dict[bucket_pretty {elasticity, r2, n, source}]. Если в bucket'е
меньше 30 точек или регрессия слабая (<0.05 либо positive slope) берём
общую эластичность из `fallback` со source='fallback_global'.
"""
where_class = "AND o.obj_class = :cls" if target_class else ""
where_class = "AND LOWER(crm.class) = LOWER(:cls)" if target_class else ""
params: dict[str, Any] = {
"rc": region_code,
"dn": district_name,
"ew": elasticity_window_months,
}
if target_class:
params["cls"] = target_class
_ = region_code # retained for backward compat; objective data covers EKB only
rows = (
db.execute(
text(
f"""
WITH obj_pool AS (
SELECT o.obj_id
FROM domrf_kn_objects o
WHERE o.region_cd = :rc
AND o.district_name = :dn
WITH pts AS (
SELECT
CASE
WHEN LOWER(crm.room_bucket) IN ('студия', 'studio', '0')
THEN '1-Студия'
WHEN crm.room_bucket = '1' THEN '2-1-к'
WHEN crm.room_bucket = '2' THEN '3-2-к'
WHEN crm.room_bucket = '3' THEN '4-3-к'
WHEN crm.room_bucket IN ('4', '5+') THEN '5-80+ м²'
ELSE NULL
END AS bucket,
LN(crm.deals_total_count::float8) AS y,
LN(crm.deals_total_avg_price_thousand_rub_per_m2::float8) AS x
FROM objective_corpus_room_month crm
WHERE crm.district = :dn
{where_class}
),
obj_bucket AS (
-- Доминирующий bucket каждого ЖК = MODE по семантике
-- квартиры (is_studio + rooms). Раньше брали медиану
-- total_area ЖК с mix studios+3к попадали в bucket
-- "1-к" как среднее, что искажало эластичность. MODE
-- по типу квартиры сохраняет семантику: ЖК с mix даёт
-- bucket = самый массовый тип.
--
-- Fallback на total_area для строк с NULL rooms (быват
-- в kn-API для нестандартных квартир).
SELECT
f.obj_id,
MODE() WITHIN GROUP (ORDER BY (
CASE
WHEN f.is_studio = true THEN '1-Студия'
WHEN f.rooms = 1 THEN '2-1-к'
WHEN f.rooms = 2 THEN '3-2-к'
WHEN f.rooms = 3 THEN '4-3-к'
WHEN f.rooms >= 4 THEN '5-80+ м²'
WHEN f.total_area < 30 THEN '1-Студия'
WHEN f.total_area < 45 THEN '2-1-к'
WHEN f.total_area < 60 THEN '3-2-к'
WHEN f.total_area < 80 THEN '4-3-к'
ELSE '5-80+ м²'
END
)) AS bucket
FROM domrf_kn_flats f
JOIN obj_pool p ON p.obj_id = f.obj_id
WHERE f.total_area IS NOT NULL
AND f.total_area BETWEEN 15 AND 200
GROUP BY f.obj_id
),
pts AS (
SELECT
ob.bucket,
LN(sg.realised)::float8 AS y,
LN(sg.price_avg)::float8 AS x
FROM domrf_kn_sale_graph sg
JOIN obj_bucket ob ON ob.obj_id = sg.obj_id
WHERE sg.type = 'apartments'
AND sg.realised IS NOT NULL AND sg.realised > 0
AND sg.price_avg IS NOT NULL AND sg.price_avg > 0
AND sg.report_month >= NOW() - (:ew || ' months')::interval
AND crm.deals_total_count > 0
AND crm.deals_total_avg_price_thousand_rub_per_m2 > 0
AND crm.report_month >= NOW() - (:ew || ' months')::interval
)
SELECT bucket,
regr_slope(y, x) AS slope,
regr_r2(y, x) AS r2,
COUNT(*)::bigint AS n
FROM pts
WHERE bucket IS NOT NULL
GROUP BY bucket
"""
),
@ -2065,7 +2069,9 @@ def recommend_mix(
target_class=target_class_for_geo,
)
sale_graph_vel_pm = vel["realised_per_month_median"] or vel["realised_per_month_avg"]
velocity_source = "sale_graph" if sale_graph_vel_pm is not None else "rosreestr_fallback"
# velocity_source label: "objective" when data available, "rosreestr_fallback" otherwise.
# Value key kept as "sale_graph" in output for frontend backward-compat (no breaking change).
velocity_source = "objective" if sale_graph_vel_pm is not None else "rosreestr_fallback"
elast = _elasticity_coef(
db,

View file

@ -1,12 +1,20 @@
"""Velocity-score — темп продаж конкурентов вокруг участка.
Per #34 D2: утилизация domrf_kn_sale_graph (15876 строк).
Per #34 D2: утилизация objective_corpus_room_month (еженедельно обновляемые данные).
Ранее использовался domrf_kn_sale_graph (последнее обновление 2026-01, устарел).
Главный demand-сигнал «продастся ли» среднемесячный объём продаж
конкурирующих ЖК в радиусе radius_km от участка, нормированный к
ЕКБ-медиане по region_cd=66.
ЕКБ-медиане по данным Objective.
Foundation: domrf_kn_objects (lat/lon, comm_name, obj_class, region_cd),
domrf_kn_sale_graph (obj_id, report_month, area_sq, realised, type).
objective_complex_mapping (domrf_obj_id objective_complex_name),
objective_corpus_room_month (project_name, deals_total_vol_m2,
deals_total_count, report_month).
Linkage: domrf_kn_objects.obj_id
objective_complex_mapping.domrf_obj_id
objective_complex_mapping.objective_complex_name
objective_corpus_room_month.project_name
"""
from __future__ import annotations
@ -63,8 +71,9 @@ def compute_velocity(
Алгоритм:
1. Найти ЖК-конкуренты в радиусе radius_km (через lat/lon ST_DWithin).
2. Взять sale_graph за последние months_window месяцев (latest snapshot).
3. Посчитать суммарный объём (area_sq > 0, иначе realised * avg_area).
2. Взять objective_corpus_room_month за последние months_window месяцев
через objective_complex_mapping (domrf_obj_id project_name).
3. Посчитать суммарный объём deals_total_vol_m2.
4. Нормировать на ЕКБ-медиану score 0..1.
Возвращает None если parcel_geom_wkt невалиден или конкурентов нет.
@ -149,38 +158,36 @@ def compute_velocity(
for r in comp_rows
}
# ── Step 2: sale_graph за последние N месяцев (latest snapshot per obj) ──
# area_sq = м² за месяц (primary). Если NULL — realised * 45 м² heuristic.
# type = 'apartments' — только жильё.
# ── Step 2: objective_corpus_room_month за последние N месяцев ───────────
# Linkage: domrf_obj_id → objective_complex_mapping → project_name →
# objective_corpus_room_month.
# deals_total_vol_m2 = DDU + DKP м² за месяц (primary signal).
# deals_total_count > 0 — фильтрует месяцы без сделок.
# GROUP BY domrf_obj_id чтобы сохранить совместимость с caller (obj_id key).
try:
with db.begin_nested():
sales_rows = (
db.execute(
text(
"""
WITH latest_sg AS (
SELECT DISTINCT ON (obj_id, report_month)
obj_id,
report_month,
area_sq,
realised
FROM domrf_kn_sale_graph
WHERE obj_id = ANY(:obj_ids)
AND type = 'apartments'
AND report_month >= (CURRENT_DATE - :window_interval::interval)
ORDER BY obj_id, report_month, snapshot_date DESC NULLS LAST
WITH mapped AS (
SELECT cm.domrf_obj_id AS obj_id,
cm.objective_complex_name
FROM objective_complex_mapping cm
WHERE cm.domrf_obj_id = ANY(:obj_ids)
)
SELECT
obj_id,
SUM(
COALESCE(area_sq, realised * 45.0)
) AS total_sqm,
COUNT(DISTINCT report_month) AS months_with_data,
MIN(report_month) AS period_start,
MAX(report_month) AS period_end
FROM latest_sg
WHERE area_sq > 0 OR realised > 0
GROUP BY obj_id
m.obj_id,
SUM(crm.deals_total_vol_m2) AS total_sqm,
COUNT(DISTINCT crm.report_month) AS months_with_data,
MIN(crm.report_month) AS period_start,
MAX(crm.report_month) AS period_end
FROM objective_corpus_room_month crm
JOIN mapped m
ON m.objective_complex_name = crm.project_name
WHERE crm.report_month >= (CURRENT_DATE - :window_interval::interval)
AND crm.deals_total_count > 0
GROUP BY m.obj_id
"""
),
{
@ -192,7 +199,7 @@ def compute_velocity(
.all()
)
except Exception:
logger.exception("velocity: sale_graph query failed for obj_ids=%s", obj_ids[:5])
logger.exception("velocity: objective sales query failed for obj_ids=%s", obj_ids[:5])
# SAVEPOINT auto-rollback'нут — outer tx clean
return None
@ -270,6 +277,11 @@ def compute_velocity(
def _get_ekb_median(db: Session, months_window: int = 6) -> float | None:
"""ЕКБ-wide медиана monthly velocity (м²/мес) per ЖК — benchmark.
Источник: objective_corpus_room_month (актуальные данные, обновляется еженедельно).
Ранее использовался domrf_kn_sale_graph с фильтром region_cd=66.
objective_corpus_room_month не имеет region_cd данные Objective'а
покрывают primarily ЕКБ, что для baseline допустимо.
Учитываются только ЖК с 3 месяцами данных за окно (стабильный сигнал).
Fallback к _EKB_MEDIAN_FALLBACK_SQM_PER_MONTH если нет данных в БД.
"""
@ -279,38 +291,23 @@ def _get_ekb_median(db: Session, months_window: int = 6) -> float | None:
db.execute(
text(
"""
WITH latest_sg AS (
SELECT DISTINCT ON (obj_id, report_month)
obj_id,
area_sq,
realised,
report_month
FROM domrf_kn_sale_graph sg
WHERE sg.type = 'apartments'
AND sg.report_month >= (CURRENT_DATE - :window_interval::interval)
AND EXISTS (
SELECT 1 FROM domrf_kn_objects o
WHERE o.obj_id = sg.obj_id
AND o.region_cd = 66
)
ORDER BY obj_id, report_month, snapshot_date DESC NULLS LAST
),
per_obj AS (
WITH per_project AS (
SELECT
obj_id,
SUM(COALESCE(area_sq, realised * 45.0)) AS total_sqm,
COUNT(DISTINCT report_month) AS months_data
FROM latest_sg
WHERE area_sq > 0 OR realised > 0
GROUP BY obj_id
project_name,
SUM(deals_total_vol_m2) AS total_sqm,
COUNT(DISTINCT report_month) AS months_data
FROM objective_corpus_room_month
WHERE report_month >= (CURRENT_DATE - :window_interval::interval)
AND deals_total_count > 0
GROUP BY project_name
HAVING COUNT(DISTINCT report_month) >= 3
),
per_obj_velocity AS (
per_project_velocity AS (
SELECT total_sqm / months_data AS velocity
FROM per_obj
FROM per_project
)
SELECT PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY velocity) AS median
FROM per_obj_velocity
FROM per_project_velocity
"""
),
{"window_interval": f"{months_window} months"},

View file

@ -1,6 +1,9 @@
"""Tests for velocity-score service (#34 D2).
Mock-based не требуют живой БД.
Источник данных objective_corpus_room_month (мигрировано с domrf_kn_sale_graph).
Mock shape совместим: sales query возвращает те же aliases (obj_id, total_sqm,
months_with_data, period_start, period_end) через GROUP BY domrf_obj_id.
"""
from __future__ import annotations
@ -79,7 +82,7 @@ def test_no_competitors_returns_none():
def test_no_sales_data_returns_none():
"""ЖК есть, но нет данных sale_graph → None."""
"""ЖК есть, но нет данных objective_corpus_room_month → None."""
comp_rows = [_comp_row(1), _comp_row(2)]
db = _make_db(comp_rows=comp_rows, sales_rows=[])
result = compute_velocity(db, parcel_geom_wkt=_PARCEL_WKT)