gendesign/backend/app/services/site_finder/best_layouts.py
lekss361 c701636515 Merge forgejo/main into fix/sf-01-velocity-time-window
Resolve conflict in best_layouts.py: keep SF-01 inline velocity
(deals_window numerator) + add SF-05 sold_pct clamp at 100% and
is_oversold flag from main.
2026-05-17 12:15:40 +03:00

631 lines
26 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""Анализ лучших планировок конкурентов по velocity (Issue #113 Phase 2.1).
Источники:
cad_parcels_geom / cad_quarters_geom — центроид участка
domrf_kn_objects — ЖК в радиусе (latitude/longitude → geography)
objective_corpus_room_month — ежемесячные сделки по (project_name, room_bucket)
objective_complex_mapping — domrf_obj_id ↔ objective_complex_name
domrf_kn_flats — supply count по (room_bucket, area_bin)
Алгоритм:
Step 1: центроид участка (cad_parcels_geom → cad_quarters_geom fallback).
Step 2: obj_id конкурентов в радиусе (domrf_kn_objects + фильтры).
Step 3: inline SQL из objective_corpus_room_month с честным WHERE report_month фильтром.
Step 4: velocity_per_month = deals_window / months_in_window (честный time_window).
Step 5: supply side из domrf_kn_flats — один батч-запрос.
Step 6: per-row signature + sold_pct.
Step 7: фильтр min_velocity + sort + rank.
Step 8: build recommendation_for_tz (unit-mix, price, rationale).
Step 9: data_quality (coverage + confidence).
Fix SF-01: раньше mv_layout_velocity (24 мес) делился на divisor (4/12) — данные
не менялись при смене time_window. Теперь inline SQL с реальным фильтром report_month.
"""
from __future__ import annotations
import datetime as dt
import logging
from typing import Any
from sqlalchemy import text
from sqlalchemy.orm import Session
from app.schemas.parcel import (
BestLayoutsRequest,
BestLayoutsResponse,
LayoutDataQuality,
LayoutTzMixRow,
LayoutTzRecommendation,
TopLayoutRow,
)
from app.services.site_finder.layout_signature import area_bin, layout_signature
logger = logging.getLogger(__name__)
# Confidence thresholds (per coverage % of objects with MV velocity data)
# Tune via PR if business feedback требует.
LAYOUT_CONFIDENCE_HIGH_PCT = 50.0
LAYOUT_CONFIDENCE_MEDIUM_PCT = 20.0
# Параметры time_window: (PostgreSQL interval string, months divisor для velocity_per_month).
# Используются в _INLINE_VELOCITY_SQL — реальный фильтр по report_month.
# Fix SF-01: убраны _VELOCITY_DIVISORS, которые делили MV (24 мес) без изменения данных.
_TIME_WINDOW_PARAMS: dict[str, tuple[str, float]] = {
"last_month": ("1 month", 1.0),
"last_quarter": ("3 months", 3.0),
"last_year": ("12 months", 12.0),
}
# ── SQL: центроид участка ─────────────────────────────────────────────────────
_PARCEL_CENTROID_SQL = text("""
SELECT ST_X(pt) AS center_lon,
ST_Y(pt) AS center_lat
FROM (
SELECT ST_Centroid(geom) AS pt
FROM cad_parcels_geom
WHERE cad_num = :cad_num AND geom IS NOT NULL
UNION ALL
SELECT ST_Centroid(geom) AS pt
FROM cad_quarters_geom
WHERE cad_number = :quarter AND geom IS NOT NULL
) sub
LIMIT 1
""")
# ── SQL: obj_id конкурентов в радиусе ─────────────────────────────────────────
# Геометрия domrf_kn_objects вычисляется on-the-fly из (latitude, longitude)
# как ST_SetSRID(ST_MakePoint(longitude, latitude), 4326)::geography
# (consistency с competitors.py).
# obj_class_filter: NULL = все классы.
# filter_competitor_obj_ids: NULL = не фильтровать по списку.
_COMPETITORS_IN_RADIUS_SQL = text("""
SELECT DISTINCT ON (obj_id) obj_id
FROM domrf_kn_objects
WHERE latitude IS NOT NULL AND longitude IS NOT NULL
AND ST_DWithin(
ST_SetSRID(ST_MakePoint(longitude, latitude), 4326)::geography,
ST_SetSRID(
ST_MakePoint(CAST(:center_lon AS float), CAST(:center_lat AS float)),
4326
)::geography,
CAST(:radius_m AS float)
)
AND (
CAST(:obj_class_filter AS text) IS NULL
OR obj_class = CAST(:obj_class_filter AS text)
)
ORDER BY obj_id, snapshot_date DESC NULLS LAST
""")
# ── SQL: inline velocity из objective_corpus_room_month + mapping ─────────────
# Fix SF-01: честный фильтр по report_month вместо деления mv_layout_velocity (24 мес).
# Параметры:
# :window_interval — PostgreSQL interval string ('1 month', '3 months', '12 months')
# :competitor_obj_ids — list[int] obj_id конкурентов в радиусе
# CAST(:window_interval AS interval) — psycopg v3 / SQLAlchemy 2.0 safe (не ::interval).
_INLINE_VELOCITY_SQL = text("""
SELECT
CASE
WHEN crm.room_bucket = 'студия' THEN 'studio'
ELSE crm.room_bucket
END AS room_bucket,
SUM(crm.deals_total_count) AS deals_window,
AVG(crm.deals_total_avg_area_m2) AS avg_area_m2,
AVG(crm.deals_total_avg_price_thousand_rub_per_m2)
* 1000.0 AS avg_price_per_m2_rub,
array_agg(DISTINCT cm.domrf_obj_id) AS competitor_obj_ids,
COUNT(DISTINCT cm.domrf_obj_id) AS competitor_count,
MIN(crm.report_month) AS window_start,
MAX(crm.report_month) AS window_end
FROM objective_corpus_room_month crm
JOIN objective_complex_mapping cm
ON cm.objective_complex_name = crm.project_name
WHERE crm.report_month >= (NOW() - CAST(:window_interval AS interval))::date
AND cm.domrf_obj_id = ANY(:competitor_obj_ids)
AND crm.room_bucket IS NOT NULL
GROUP BY
CASE
WHEN crm.room_bucket = 'студия' THEN 'studio'
ELSE crm.room_bucket
END
""")
# ── SQL: supply по (room_bucket, area_bin) за последний снимок ───────────────
# Один батч-запрос вместо N — возвращает map (rb, ab) → count.
# room_bucket и area_bin вычисляются в SQL аналогично layout_signature.py.
_SUPPLY_BATCH_SQL = text("""
SELECT
CASE
WHEN f.is_studio = TRUE OR f.flat_type = 'Квартира-студия' THEN 'studio'
WHEN f.rooms = 0 THEN 'studio'
WHEN f.rooms IN (1, 2, 3) THEN f.rooms::text
WHEN f.rooms >= 4 THEN '4+'
ELSE '1'
END AS rb,
CASE
WHEN f.total_area < 25 THEN '<25'
WHEN f.total_area < 40 THEN '25-40'
WHEN f.total_area < 60 THEN '40-60'
WHEN f.total_area < 80 THEN '60-80'
WHEN f.total_area < 100 THEN '80-100'
ELSE '100+'
END AS ab,
COUNT(*) AS units
FROM domrf_kn_flats f
JOIN domrf_kn_objects o ON f.obj_id = o.obj_id
WHERE o.latitude IS NOT NULL AND o.longitude IS NOT NULL
AND ST_DWithin(
ST_SetSRID(ST_MakePoint(o.longitude, o.latitude), 4326)::geography,
ST_SetSRID(
ST_MakePoint(CAST(:center_lon AS float), CAST(:center_lat AS float)),
4326
)::geography,
CAST(:radius_m AS float)
)
AND f.snapshot_date = CAST(:latest_snap AS date)
GROUP BY rb, ab
""")
# ── Вспомогательные функции ───────────────────────────────────────────────────
def _quarter_from_cad(cad_num: str) -> str:
"""Извлечь кадастровый квартал: '66:41:0303161:123''66:41:0303161'."""
parts = cad_num.split(":")
if len(parts) >= 3:
return ":".join(parts[:3])
return cad_num
def _normalize_pct(buckets: dict[str, float]) -> dict[str, int]:
"""Нормировать доли до целых процентов с суммой ровно 100.
Алгоритм largest-remainder (Hamilton method):
1. Floor каждого значения.
2. Остаток 100 sum_floors распределить в top-bucket по дробной части.
"""
if not buckets:
return {}
total = sum(buckets.values())
if total <= 0:
n = len(buckets)
base = 100 // n
result = {k: base for k in buckets}
# распределить остаток
remainder = 100 - base * n
for k in list(buckets.keys())[:remainder]:
result[k] += 1
return result
raw = {k: v / total * 100.0 for k, v in buckets.items()}
floors = {k: int(v) for k, v in raw.items()}
remainder = 100 - sum(floors.values())
# sort by fractional part desc
fracs = sorted(buckets.keys(), key=lambda k: -(raw[k] - floors[k]))
for k in fracs[:remainder]:
floors[k] += 1
return floors
# ── Главная функция ───────────────────────────────────────────────────────────
def get_best_layouts(
db: Session,
cad_num: str,
request: BestLayoutsRequest,
) -> BestLayoutsResponse:
"""Top layouts (rooms × area_bin) конкурентов с рейтингом по velocity.
Raises:
ValueError: если центроид участка не найден (caller → HTTP 404).
"""
quarter = _quarter_from_cad(cad_num)
radius_m = request.radius_km * 1000.0
# time_window → (interval_str, months divisor)
window_interval, months_in_window = _TIME_WINDOW_PARAMS.get(
request.time_window, ("3 months", 3.0)
)
# ── Step 1: центроид участка ─────────────────────────────────────────────
try:
coord_row = (
db.execute(
_PARCEL_CENTROID_SQL,
{"cad_num": cad_num, "quarter": quarter},
)
.mappings()
.first()
)
except Exception:
logger.exception("best_layouts: centroid query failed for cad_num=%s", cad_num)
raise
if not coord_row:
raise ValueError(f"Геометрия для {cad_num} не найдена")
center_lon = float(coord_row["center_lon"])
center_lat = float(coord_row["center_lat"])
# ── Step 2: obj_id конкурентов в радиусе ────────────────────────────────
try:
id_rows = (
db.execute(
_COMPETITORS_IN_RADIUS_SQL,
{
"center_lon": center_lon,
"center_lat": center_lat,
"radius_m": radius_m,
"obj_class_filter": request.obj_class_filter,
},
)
.mappings()
.all()
)
except Exception:
logger.exception("best_layouts: competitors-in-radius query failed for cad_num=%s", cad_num)
raise
all_obj_ids: list[int] = [int(r["obj_id"]) for r in id_rows]
objects_total_in_radius = len(all_obj_ids)
# Применить exclude / filter из request
exclude_set = set(request.exclude_competitor_obj_ids)
if exclude_set:
all_obj_ids = [oid for oid in all_obj_ids if oid not in exclude_set]
if request.filter_competitor_obj_ids is not None:
filter_set = set(request.filter_competitor_obj_ids)
all_obj_ids = [oid for oid in all_obj_ids if oid in filter_set]
if not all_obj_ids:
return _empty_response(
radius_km=request.radius_km,
time_window=request.time_window,
objects_total_in_radius=objects_total_in_radius,
)
# ── Step 3: inline velocity из objective_corpus_room_month ──────────────
# Fix SF-01: честный фильтр report_month >= NOW() - window_interval.
# Разные time_window → разные deals_window, разный mix.
try:
vel_rows = (
db.execute(
_INLINE_VELOCITY_SQL,
{
"window_interval": window_interval,
"competitor_obj_ids": all_obj_ids,
},
)
.mappings()
.all()
)
except Exception:
logger.exception(
"best_layouts: inline velocity query failed for cad_num=%s obj_count=%d",
cad_num,
len(all_obj_ids),
)
raise
if not vel_rows:
return _empty_response(
radius_km=request.radius_km,
time_window=request.time_window,
objects_total_in_radius=objects_total_in_radius,
)
# ── Step 5: supply side (батч-запрос) ────────────────────────────────────
# Pre-compute последний snapshot_date один раз — избегаем subquery на каждый scan.
latest_snap: dt.date | None = db.scalar(text("SELECT MAX(snapshot_date) FROM domrf_kn_flats"))
if latest_snap is None:
logger.warning("best_layouts: domrf_kn_flats пустой (нет snapshot_date), supply=0 fallback")
supply_rows = []
else:
try:
supply_rows = (
db.execute(
_SUPPLY_BATCH_SQL,
{
"center_lon": center_lon,
"center_lat": center_lat,
"radius_m": radius_m,
"latest_snap": latest_snap,
},
)
.mappings()
.all()
)
except Exception:
logger.warning("best_layouts: supply query failed, supply=0 fallback")
supply_rows = []
supply_map: dict[tuple[str, str], int] = {
(str(r["rb"]), str(r["ab"])): int(r["units"]) for r in supply_rows
}
# ── Step 4 + 6: velocity из реального окна и enrichment per row ─────────
# Fix SF-01: velocity_per_month = deals_window / months_in_window.
# deals_window уже отфильтрован по report_month — разные time_window дают разные данные.
enriched: list[dict[str, Any]] = []
window_start: dt.date | None = None
window_end: dt.date | None = None
# Собираем obj_ids с данными в objective_corpus_room_month (для data_quality)
obj_ids_with_data: set[int] = set()
for r in vel_rows:
room_bucket = str(r["room_bucket"])
deals_window = float(r["deals_window"]) if r["deals_window"] is not None else 0.0
avg_area = float(r["avg_area_m2"]) if r["avg_area_m2"] is not None else 0.0
price_rub = (
float(r["avg_price_per_m2_rub"]) if r["avg_price_per_m2_rub"] is not None else None
)
competitor_obj_ids: list[int] = (
[int(oid) for oid in r["competitor_obj_ids"]] if r["competitor_obj_ids"] else []
)
competitor_count = int(r["competitor_count"])
obj_ids_with_data.update(competitor_obj_ids)
# Step 4: честный velocity = сделки за окно / длина окна в месяцах
velocity_per_month = round(deals_window / months_in_window, 2)
# Step 6: area_bin по avg_area (layout_signature.area_bin)
ab = area_bin(avg_area) if avg_area > 0 else "<25"
sig = layout_signature(room_bucket, ab) # type: ignore[arg-type]
supply_count = supply_map.get((room_bucket, ab), 0)
sold_pct: float | None = None
is_oversold = False
if supply_count > 0:
sold_pct_raw = deals_window / supply_count * 100.0
is_oversold = sold_pct_raw > 100.0
# Clamp at 100%: сделки за 24 мес / текущий snapshot supply несопоставимы.
# Значения >100% артефакт окна, не реальная «распроданность».
sold_pct = round(min(sold_pct_raw, 100.0), 1)
# data window
if r["window_start"] is not None:
ws = r["window_start"]
if isinstance(ws, str):
ws = dt.date.fromisoformat(ws)
elif isinstance(ws, dt.datetime):
ws = ws.date()
window_start = ws if window_start is None else min(window_start, ws)
if r["window_end"] is not None:
we = r["window_end"]
if isinstance(we, str):
we = dt.date.fromisoformat(we)
elif isinstance(we, dt.datetime):
we = we.date()
window_end = we if window_end is None else max(window_end, we)
enriched.append(
{
"room_bucket": room_bucket,
"area_bin": ab,
"signature": sig,
"competitor_obj_ids": competitor_obj_ids,
"competitor_count": competitor_count,
"sum_deals": deals_window,
"velocity_per_month": velocity_per_month,
"avg_price_per_m2_rub": price_rub,
"avg_area_m2": avg_area,
"supply_units_in_radius": supply_count,
"sold_pct_of_supply": sold_pct,
"is_oversold": is_oversold,
}
)
# ── Step 7: фильтр min_velocity + sort + rank ────────────────────────────
filtered = [
row for row in enriched if row["velocity_per_month"] >= request.min_velocity_per_month
]
filtered.sort(key=lambda r: r["velocity_per_month"], reverse=True)
top_layouts: list[TopLayoutRow] = []
for rank_idx, row in enumerate(filtered, start=1):
top_layouts.append(
TopLayoutRow(
rank=rank_idx,
room_bucket=row["room_bucket"],
area_bin=row["area_bin"],
signature=row["signature"],
competitor_obj_ids=row["competitor_obj_ids"],
competitor_count=row["competitor_count"],
total_sold_in_window=int(row["sum_deals"]),
velocity_per_month=row["velocity_per_month"],
avg_price_per_m2_rub=row["avg_price_per_m2_rub"],
avg_area_m2=round(row["avg_area_m2"], 1),
supply_units_in_radius=row["supply_units_in_radius"],
sold_pct_of_supply=row["sold_pct_of_supply"],
is_oversold=row["is_oversold"],
)
)
# ── Step 8: build recommendation_for_tz ─────────────────────────────────
# Используем filtered (только > min_velocity) для recommendation.
# Если после фильтрации всё пустое — используем enriched (все данные без фильтра).
rec_source = filtered if filtered else enriched
today = dt.date.today()
ws_date = window_start if window_start is not None else today
we_date = window_end if window_end is not None else today
recommendation = _build_recommendation(
rows=rec_source,
radius_km=request.radius_km,
time_window=request.time_window,
target_total_flats=request.target_total_flats,
window_start=ws_date,
window_end=we_date,
all_enriched=enriched,
)
# ── Step 9: data_quality ─────────────────────────────────────────────────
# Denominator = post-filter set (effective consideration set после exclude/filter).
objects_total_after_filter = len(all_obj_ids)
objects_with_data = len(obj_ids_with_data & set(all_obj_ids))
coverage_pct = (
round(objects_with_data / objects_total_after_filter * 100.0, 1)
if objects_total_after_filter > 0
else 0.0
)
if coverage_pct >= LAYOUT_CONFIDENCE_HIGH_PCT:
confidence: str = "high"
elif coverage_pct >= LAYOUT_CONFIDENCE_MEDIUM_PCT:
confidence = "medium"
else:
confidence = "low"
data_quality = LayoutDataQuality(
objects_with_velocity_data=objects_with_data,
objects_total_in_radius=objects_total_after_filter,
velocity_coverage_pct=coverage_pct,
confidence=confidence, # type: ignore[arg-type]
)
return BestLayoutsResponse(
top_layouts=top_layouts,
recommendation_for_tz=recommendation,
data_quality=data_quality,
)
def _build_recommendation(
rows: list[dict[str, Any]],
radius_km: float,
time_window: str,
target_total_flats: int | None,
window_start: dt.date,
window_end: dt.date,
all_enriched: list[dict[str, Any]],
) -> LayoutTzRecommendation:
"""Собрать LayoutTzRecommendation из enriched rows."""
if not rows:
return LayoutTzRecommendation(
rationale_text=(
f"В радиусе {radius_km}км: нет layout-паттернов с достаточной velocity."
),
mix=[],
weighted_avg_price_per_m2_rub=None,
based_on_obj_count=0,
based_on_total_deals=0,
data_window_start=window_start,
data_window_end=window_end,
)
# Группировка по room_bucket (строки уже могут быть per-bucket из MV GROUP BY)
rb_deals: dict[str, float] = {}
rb_area_weighted: dict[str, float] = {}
rb_price_weighted: dict[str, float] = {}
rb_price_total_deals: dict[str, float] = {}
all_competitor_ids: set[int] = set()
for row in rows:
rb = row["room_bucket"]
sd = float(row["sum_deals"])
rb_deals[rb] = rb_deals.get(rb, 0.0) + sd
rb_area_weighted[rb] = rb_area_weighted.get(rb, 0.0) + row["avg_area_m2"] * sd
all_competitor_ids.update(row["competitor_obj_ids"])
if row["avg_price_per_m2_rub"] is not None:
rb_price_weighted[rb] = rb_price_weighted.get(rb, 0.0) + (
row["avg_price_per_m2_rub"] * sd
)
rb_price_total_deals[rb] = rb_price_total_deals.get(rb, 0.0) + sd
total_deals = sum(rb_deals.values())
pct_map = _normalize_pct(rb_deals)
mix: list[LayoutTzMixRow] = []
for rb, pct in sorted(pct_map.items(), key=lambda x: -x[1]):
avg_area = (
round(rb_area_weighted[rb] / rb_deals[rb], 1) if rb_deals.get(rb, 0) > 0 else None
)
abs_units: int | None = None
if target_total_flats is not None:
abs_units = round(pct / 100.0 * target_total_flats)
mix.append(
LayoutTzMixRow(
room_bucket=rb,
pct=pct,
abs_units=abs_units,
avg_target_area_m2=avg_area,
)
)
# Weighted avg price across all room_buckets
total_price_deals = sum(rb_price_total_deals.values())
weighted_price: float | None = None
if total_price_deals > 0:
weighted_price = round(sum(rb_price_weighted.values()) / total_price_deals, 0)
# Rationale
competitor_count = len(all_competitor_ids)
tw_label = {"last_month": "1 мес", "last_quarter": "квартал", "last_year": "год"}.get(
time_window, time_window
)
rationale_text = (
f"В радиусе {radius_km}км за {tw_label}: "
f"{len(rows)} активных layout-паттернов, "
f"total {int(total_deals)} продаж в {competitor_count} ЖК"
)
# based_on_obj_count из all_enriched (уникальные obj_id с данными MV)
all_mv_obj_ids: set[int] = set()
for row in all_enriched:
all_mv_obj_ids.update(row["competitor_obj_ids"])
return LayoutTzRecommendation(
rationale_text=rationale_text,
mix=mix,
weighted_avg_price_per_m2_rub=weighted_price,
based_on_obj_count=len(all_mv_obj_ids),
based_on_total_deals=int(total_deals),
data_window_start=window_start,
data_window_end=window_end,
)
def _empty_response(
radius_km: float,
time_window: str,
objects_total_in_radius: int,
) -> BestLayoutsResponse:
"""Ответ когда нет конкурентов или нет MV данных."""
today = dt.date.today()
tw_label = {"last_month": "1 мес", "last_quarter": "квартал", "last_year": "год"}.get(
time_window, time_window
)
return BestLayoutsResponse(
top_layouts=[],
recommendation_for_tz=LayoutTzRecommendation(
rationale_text=(
f"В радиусе {radius_km}км за {tw_label}: "
f"конкуренты не найдены или нет данных velocity."
),
mix=[],
weighted_avg_price_per_m2_rub=None,
based_on_obj_count=0,
based_on_total_deals=0,
data_window_start=today,
data_window_end=today,
),
data_quality=LayoutDataQuality(
objects_with_velocity_data=0,
objects_total_in_radius=objects_total_in_radius,
velocity_coverage_pct=0.0,
confidence="low",
),
)