merge main + renumber 163→167 + guard riasurt harvest (#108 review)
All checks were successful
CI / changes (pull_request) Successful in 6s
CI / frontend-tests (pull_request) Has been skipped
CI / openapi-codegen-check (pull_request) Successful in 2m33s
CI / backend-tests (pull_request) Successful in 10m10s

This commit is contained in:
bot-backend 2026-06-17 22:52:15 +03:00
commit 0a72ef9491
36 changed files with 1251 additions and 205 deletions

View file

@ -15,23 +15,22 @@ name: CI
# FUTURE: добавить `postgis/postgis:16-3.4` service + гонять mv_layout — см.
# .github/workflows/ci.yml как образец service-блока.
on:
# ТОЛЬКО pull_request — НЕТ push-триггера на feature-ветки (CI-шторм #1709).
# WHY: раньше был и push: [feat/**,fix/**,...]. Каждый коммит в ветку с открытым
# PR триггерил ДВА прогона на ОДИН SHA: push-событие (github.ref=refs/heads/<branch>)
# и pull_request-событие (github.ref=refs/pull/<N>/merge). Разный github.ref →
# разные concurrency-группы (см. ниже) → прогоны НЕ отменяют друг друга → 2× job
# при и так дефицитных раннерах. В bot-пайплайне каждый коммит идёт через PR, так
# что pull_request гейтит его полностью; push-прогон был чистым дублем.
# Trade-off: push в feature-ветку БЕЗ открытого PR не получит CI до открытия PR
# (бот открывает PR сразу после первого push) — приемлемо.
pull_request:
branches: [main]
push:
branches:
# Mirror the bot-PR / feature-branch flow в .claude/rules/git-pr.md:
# PR получает gate, прямые пуши в feature-ветки — тоже.
- "feat/**"
- "fix/**"
- "refactor/**"
- "chore/**"
- "docs/**"
- "perf/**"
- "test/**"
- "ci/**"
- "hotfix/**"
concurrency:
# Теперь, когда остался только pull_request, github.ref стабилен на весь PR
# (refs/pull/<N>/merge) → новый push в ветку PR отменяет предыдущий незавершённый
# прогон ЭТОГО PR (cancel-in-progress) вместо накопления параллельных.
group: ci-${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
@ -89,6 +88,19 @@ jobs:
curl -LsSf https://astral.sh/uv/install.sh | sh
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
- name: Cache uv packages
# Кросс-прогонный кэш скачанных/собранных wheel'ов (~/.cache/uv по умолч.).
# `uv sync --frozen` без него каждый прогон тянет весь geo-стек заново —
# доминирующая часть времени job (#1709). Ключ по uv.lock; continue-on-error
# чтобы сбой cache-бэкенда раннера НИКОГДА не ронял gate.
uses: actions/cache@v4
continue-on-error: true
with:
path: ~/.cache/uv
key: uv-${{ runner.os }}-${{ hashFiles('backend/uv.lock') }}
restore-keys: |
uv-${{ runner.os }}-
- name: Install system deps for geo + WeasyPrint
# libpq/gdal/proj/geos — geo-стек (geopandas/shapely/pyproj).
# libcairo2/libpango* — нативные либы WeasyPrint: с ними PDF-тесты
@ -230,6 +242,16 @@ jobs:
curl -LsSf https://astral.sh/uv/install.sh | sh
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
- name: Cache uv packages
# См. backend-tests: кросс-прогонный кэш ~/.cache/uv, тот же ключ по uv.lock.
uses: actions/cache@v4
continue-on-error: true
with:
path: ~/.cache/uv
key: uv-${{ runner.os }}-${{ hashFiles('backend/uv.lock') }}
restore-keys: |
uv-${{ runner.os }}-
- name: Install system deps for geo + WeasyPrint
# app.main транзитивно тянет geo/PDF-модули. На macOS-dev импорт схемы
# проходит и без этих либ, но на ubuntu ставим как backend-tests
@ -239,9 +261,12 @@ jobs:
sudo apt-get install -y libpq-dev libgdal-dev libproj-dev libgeos-dev \
libcairo2 libpango-1.0-0 libpangoft2-1.0-0
- name: Install backend deps (uv sync --frozen)
- name: Install backend deps (uv sync --frozen --no-dev)
working-directory: backend
run: uv sync --frozen
# --no-dev: этот job только дампит app.openapi() (нужен runtime app.main).
# pytest/ruff/coverage не используются → не ставим dev-группу (быстрее).
# Dockerfile тоже собирает с --no-dev → импорт app.main гарантированно ок.
run: uv sync --frozen --no-dev
- name: Install frontend deps (npm ci)
working-directory: frontend

View file

@ -138,7 +138,7 @@ def leads_stats(
WITH window_leads AS (
SELECT *
FROM prinzip_leads
WHERE created_at >= NOW() - (:m || ' months')::interval
WHERE created_at >= NOW() - make_interval(months => :m)
)
SELECT
(SELECT COUNT(*) FROM prinzip_leads) AS leads_total,

View file

@ -184,8 +184,7 @@ def quartirography(db: Session, source: str, region_id: int = 66) -> list[dict[s
-- ('2025-07-01' расширял «recent»-окно каждую неделю по мере
-- доливки ETL новых report_months перекос в сторону всё
-- более длинной истории). Тот же фикс, что #1203 и _BUCKET_SQL.
AND period_start_date >= NOW()
- (:months_window || ' months')::INTERVAL
AND period_start_date >= NOW() - make_interval(months => :months_window)
),
bucketed AS (
SELECT CASE
@ -1169,7 +1168,7 @@ def prinzip_funnel_monthly(db: Session, months: int = 24) -> list[dict[str, Any]
"""
SELECT month, source, leads, engaged, converted, conv_pct
FROM prinzip_funnel_monthly
WHERE month >= (CURRENT_DATE - (:months || ' months')::interval)::date
WHERE month >= (CURRENT_DATE - make_interval(months => :months))::date
ORDER BY month DESC, leads DESC
"""
),
@ -1203,7 +1202,7 @@ def prinzip_funnel_by_source(db: Session, months: int = 12) -> list[dict[str, An
SUM(converted) AS converted,
ROUND(100.0 * SUM(converted) / NULLIF(SUM(leads), 0), 2) AS conv_pct
FROM prinzip_funnel_monthly
WHERE month >= (CURRENT_DATE - (:months || ' months')::interval)::date
WHERE month >= (CURRENT_DATE - make_interval(months => :months))::date
GROUP BY source
ORDER BY leads DESC
"""
@ -1364,8 +1363,7 @@ _BUCKET_SQL = text(
AND deal_count > 0
AND (area / deal_count) BETWEEN 15 AND 200
AND price_per_sqm BETWEEN 30000 AND 1000000
AND period_start_date >= NOW()
- (:months_window || ' months')::INTERVAL
AND period_start_date >= NOW() - make_interval(months => :months_window)
),
bucketed AS (
SELECT CASE
@ -1995,7 +1993,7 @@ def _elasticity_coef(
{where_district}
{where_class}
AND crm.deals_total_avg_price_thousand_rub_per_m2 > 0
AND crm.report_month >= NOW() - (:ew || ' months')::interval
AND crm.report_month >= NOW() - make_interval(months => :ew)
)
SELECT
regr_slope(y, x) AS slope,
@ -2086,7 +2084,7 @@ def _elasticity_per_bucket_coef(
{where_class}
AND crm.deals_total_count > 0
AND crm.deals_total_avg_price_thousand_rub_per_m2 > 0
AND crm.report_month >= NOW() - (:ew || ' months')::interval
AND crm.report_month >= NOW() - make_interval(months => :ew)
)
SELECT bucket,
regr_slope(y, x) AS slope,

View file

@ -667,7 +667,14 @@ class EmissRow:
"""Одна готовая к upsert строка macro_indicator из ЕМИСС (source='emiss').
Отдельно от ``MacroRow`` (open-data, source='rosstat', yearly): ЕМИСС-ряды несут
свою frequency (quarterly/monthly) и source контракт upsert'а у них иной.
свою frequency (quarterly/monthly), source и period_type контракт upsert'а у
них иной.
period_type: гранулярность под-периода ('year' | 'quarter' | 'month' | 'unknown').
Берётся из _emiss_period_granularity(PERIOD). Необходима как часть PK
macro_indicator (migration 163), чтобы годовой агрегат ('год' 'year') и
Q1 ('I квартал' 'quarter') за один год не перезаписывали друг друга при
ON CONFLICT DO UPDATE (оба дают obs_date=YYYY-01-01) (#1606).
"""
indicator_type: str
@ -677,6 +684,7 @@ class EmissRow:
unit: str
frequency: str
comment: str
period_type: str = "unknown"
def _emiss_period_to_month(period: str) -> int | None:
@ -817,6 +825,7 @@ def parse_emiss_sdmx(raw: bytes | str, spec: EmissIndicatorSpec) -> list[EmissRo
unit=spec.unit,
frequency=spec.frequency,
comment=spec.comment,
period_type=granularity,
)
return [by_key[k] for k in sorted(by_key, key=lambda k: (k[0], k[1]))]

View file

@ -0,0 +1,95 @@
"""Refresh helper for the sales-tracker MVs (Issue #61).
Two independent materialized views built from the Объектив sales-tracker
("шахматки") snapshots (objective_lots / objective_lots_history), created by
data/sql/164_mv_sales_tracker_velocity_absorption.sql:
1. mv_sales_tracker_velocity_by_district per (district, month) sold/total/
avg-sold-price. Feeds the Site Finder Velocity Score (4th scoring criterion).
2. mv_sales_tracker_absorption_curves cumulative sold% as f(months from
sales_start_date) per (rooms_int, area_bucket). Foundation for recommend_mix
+ sellout forecast.
The two MVs do not depend on each other, so refresh order is irrelevant; both
are refreshed in the same call.
Scheduled via Celery beat hardcoded entry in workers/beat_schedule.py
('mv-sales-tracker-refresh-weekly', Mon 04:30 MSK).
Usage example (manual, via psql-connected shell or admin endpoint):
from app.services.site_finder.sales_tracker_mv_refresh import refresh_sales_tracker_mvs
counts = refresh_sales_tracker_mvs(db)
# logs: "mv_sales_tracker_velocity_by_district refreshed: 70 rows", etc.
"""
from __future__ import annotations
import logging
from sqlalchemy import text
from sqlalchemy.exc import DatabaseError
from sqlalchemy.orm import Session
logger = logging.getLogger(__name__)
_MV_NAMES: tuple[str, ...] = (
"mv_sales_tracker_velocity_by_district",
"mv_sales_tracker_absorption_curves",
)
def _refresh_mv(db: Session, mv_name: str, *, concurrently: bool) -> int:
"""Run REFRESH MATERIALIZED VIEW [CONCURRENTLY] <mv_name>, return row count.
Falls back to non-concurrent on the known "cannot refresh concurrently"
error (MV empty or no UNIQUE index should not happen in prod since the
migration creates the UNIQUE index and populates the MV, but provides a
safe recovery path for first-run / post-recreation edge cases).
"""
try:
if concurrently:
db.execute(text(f"REFRESH MATERIALIZED VIEW CONCURRENTLY {mv_name}"))
else:
db.execute(text(f"REFRESH MATERIALIZED VIEW {mv_name}"))
db.commit()
except DatabaseError as e:
# PostgreSQL emits "CONCURRENTLY cannot be used when the materialized
# view ... is not populated" (matview.c, SQLSTATE 55000), surfaced by
# psycopg3 as an InternalError (a DatabaseError sibling).
if concurrently and "concurrently cannot be used" in str(e).lower():
logger.warning(
"%s: CONCURRENTLY failed (MV likely not populated), "
"falling back to non-concurrent refresh",
mv_name,
)
db.rollback()
db.execute(text(f"REFRESH MATERIALIZED VIEW {mv_name}"))
db.commit()
else:
raise
row = db.execute(text(f"SELECT COUNT(*) FROM {mv_name}")).first()
count = int(row[0]) if row else 0
logger.info("%s refreshed: %d rows", mv_name, count)
return count
def refresh_sales_tracker_mvs(db: Session, *, concurrently: bool = True) -> dict[str, int]:
"""Refresh both sales-tracker MVs.
Args:
db: SQLAlchemy Session (sync).
concurrently: When True, uses REFRESH CONCURRENTLY (non-blocking
readers continue). Requires the per-MV UNIQUE indexes
(mv_sales_tracker_velocity_by_district_pk,
mv_sales_tracker_absorption_curves_pk) and the MVs to be already
populated. Pass False only for first populate or after recreation.
Returns:
Mapping mv_name -> row count after refresh (for observability).
"""
counts: dict[str, int] = {}
for mv_name in _MV_NAMES:
counts[mv_name] = _refresh_mv(db, mv_name, concurrently=concurrently)
return counts

View file

@ -16,9 +16,13 @@ Foundation: domrf_kn_objects (lat/lon, comm_name, obj_class, region_cd),
Fallback: rosreestr_deals (quarter_cad_number, deal_count, period_start_date).
Linkage: domrf_kn_objects.obj_id
objective_complex_mapping.domrf_obj_id
objective_complex_mapping.domrf_obj_id (gated: is_reviewed/manual/score0.85)
objective_complex_mapping.objective_complex_name
objective_corpus_room_month.project_name
OBJ-2 (#307): маппинги фильтруются по confidence (_MAPPING_CONFIDENCE_GATE) —
unreviewed low-score auto-matches (#1331/#1333 backfill) исключаются как
false-positive risk.
"""
from __future__ import annotations
@ -40,6 +44,24 @@ _EKB_MEDIAN_FALLBACK_SQM_PER_MONTH: float = 4500.0
# пытаемся rosreestr_fallback.
_OBJECTIVE_COVERAGE_MIN_RATIO: float = 0.50
# OBJ-2 (#307): gate objective_complex_mapping by confidence перед использованием
# в velocity. Fuzzy-trgm backfill (#1331/#1333) добавил ~115 auto-matched строк с
# is_reviewed=false и низким match_score (вплоть до 0.50-0.625) — false-positive
# risk, который раздувал/искажал velocity конкурентов.
#
# Принимаем mapping только если:
# - is_reviewed = TRUE (человек подтвердил), ИЛИ
# - match_method = 'manual' (ручной маппинг, score обычно NULL), ИЛИ
# - match_score >= 0.85 (AUTO_ACCEPT_THRESHOLD — high-confidence auto,
# vault: fuzzy_trgm 0.85+ надёжен для auto-use).
#
# Строгий gate только на is_reviewed=true дал бы 2 строки из 303 → обнулил бы
# velocity-покрытие; 0.85-порог сохраняет 264/303 EKB-маппингов, отбрасывая 39
# низкоуверенных. Совпадает с AUTO_ACCEPT_THRESHOLD из objective_backfill.py.
_MAPPING_CONFIDENCE_GATE: str = (
"(cm.is_reviewed = TRUE OR cm.match_method = 'manual' OR cm.match_score >= 0.85)"
)
@dataclass(frozen=True)
class VelocityResult:
@ -195,7 +217,7 @@ def compute_velocity(
sales_rows = (
db.execute(
text(
"""
f"""
WITH all_competitors AS (
SELECT unnest(CAST(:obj_ids AS int[])) AS obj_id
),
@ -204,6 +226,7 @@ def compute_velocity(
cm.objective_complex_name
FROM objective_complex_mapping cm
WHERE cm.domrf_obj_id = ANY(:obj_ids)
AND {_MAPPING_CONFIDENCE_GATE}
)
SELECT
ac.obj_id,
@ -311,12 +334,13 @@ def compute_velocity(
bucket_rows = (
db.execute(
text(
"""
f"""
WITH mapped AS (
SELECT cm.domrf_obj_id AS obj_id,
cm.objective_complex_name
FROM objective_complex_mapping cm
WHERE cm.domrf_obj_id = ANY(:obj_ids)
AND {_MAPPING_CONFIDENCE_GATE}
)
SELECT
m.obj_id,

View file

@ -540,4 +540,21 @@ def build_beat_schedule() -> dict:
"options": {"queue": "celery"},
}
# Sales-tracker MVs (#61) — питают Site Finder Velocity Score (4-й критерий) +
# recommend_mix / sellout-forecast. Оба MV (mv_sales_tracker_velocity_by_district,
# mv_sales_tracker_absorption_curves) рефрешатся CONCURRENTLY (non-blocking, требуют
# unique-индексы из миграции 161). Источник — objective_lots / objective_lots_history
# (Объектив-шахматки), наполняются objective_sync (Mon 04:15 МСК по умолчанию).
#
# Понедельник 04:30 МСК (Celery conf.timezone=Europe/Moscow → crontab в МСК, #1233) —
# ПОСЛЕ objective_sync (04:15), чтобы агрегаты считались по свежему снапшоту; в
# окне до тяжёлого monday-кластера site_finder-рефрешей (ird 05:00, gknspecial 05:30,
# supply-layers 06:00). Refresh лёгкий (~6с на 1.1M lots). Техническая infra-задача,
# не в job_settings (как refresh-quarter-price-index / refresh-layout-velocity).
schedule["mv-sales-tracker-refresh-weekly"] = {
"task": "tasks.mv_sales_tracker_refresh.refresh_sales_tracker_mvs",
"schedule": _parse_cron("30 4 * * mon"), # 04:30 MSK, понедельник
"options": {"queue": "celery"},
}
return schedule

View file

@ -83,6 +83,7 @@ celery_app = Celery(
"app.workers.tasks.developer_registry_refresh",
"app.workers.tasks.refresh_layout_velocity",
"app.workers.tasks.riasurt_sverdl_harvest",
"app.workers.tasks.mv_sales_tracker_refresh",
],
)
celery_app.conf.timezone = "Europe/Moscow"

View file

@ -41,19 +41,21 @@ from app.workers.celery_app import celery_app
logger = logging.getLogger(__name__)
# psycopg v3: CAST(:x AS type) — НИКОГДА :x::type (SQLAlchemy+psycopg3 роняет
# синтаксис на ::). Контракт колонок совпадает с macro_indicator (migration 123):
# (indicator_type, region, obs_date, value, source, frequency, unit, comment,
# updated_at) PK (indicator_type, region, obs_date).
# синтаксис на ::). Контракт колонок совпадает с macro_indicator (migration 163):
# (indicator_type, region, obs_date, period_type, value, source, frequency, unit,
# comment, updated_at). PK (indicator_type, region, obs_date, period_type).
# CBR-ряды используют period_type='unknown' (литерал) — они не несут sub-period
# granularity и различаются по obs_date.
UPSERT_KEY_RATE_SQL = text(
"""
INSERT INTO macro_indicator (
indicator_type, region, obs_date, value,
indicator_type, region, obs_date, period_type, value,
source, frequency, unit, comment
) VALUES (
'key_rate', 'rf', CAST(:d AS date), CAST(:v AS numeric),
'key_rate', 'rf', CAST(:d AS date), 'unknown', CAST(:v AS numeric),
'cbr', 'daily', '%', 'CBR key rate'
)
ON CONFLICT (indicator_type, region, obs_date) DO UPDATE SET
ON CONFLICT (indicator_type, region, obs_date, period_type) DO UPDATE SET
value = EXCLUDED.value,
updated_at = now()
"""
@ -61,16 +63,17 @@ UPSERT_KEY_RATE_SQL = text(
# Инфляция «% г/г» (ИПЦ YoY): indicator_type='inflation_yoy', monthly, region='rf'.
# obs_date уже нормализован к 1-му числу месяца парсером (parse_inflation_xlsx).
# period_type='unknown' — месячный ряд без sub-period granularity.
UPSERT_INFLATION_SQL = text(
"""
INSERT INTO macro_indicator (
indicator_type, region, obs_date, value,
indicator_type, region, obs_date, period_type, value,
source, frequency, unit, comment
) VALUES (
'inflation_yoy', 'rf', CAST(:d AS date), CAST(:v AS numeric),
'inflation_yoy', 'rf', CAST(:d AS date), 'unknown', CAST(:v AS numeric),
'cbr', 'monthly', '%', 'CBR inflation YoY (ИПЦ, % г/г)'
)
ON CONFLICT (indicator_type, region, obs_date) DO UPDATE SET
ON CONFLICT (indicator_type, region, obs_date, period_type) DO UPDATE SET
value = EXCLUDED.value,
updated_at = now()
"""

View file

@ -0,0 +1,52 @@
"""Celery task: refresh the sales-tracker MVs (Issue #61).
Scheduled via hardcoded beat entry in workers/beat_schedule.py:
'mv-sales-tracker-refresh-weekly' weekly on Monday at 04:30 MSK.
Refreshes (both CONCURRENTLY, non-blocking):
- mv_sales_tracker_velocity_by_district (Site Finder Velocity Score, 4th criterion)
- mv_sales_tracker_absorption_curves (recommend_mix + sellout forecast foundation)
Both MVs are built from the Объектив sales-tracker ("шахматки") snapshots
(objective_lots / objective_lots_history). Source data refreshes via the
objective_sync beat job, so a weekly MV refresh keeps the aggregates current.
MV-source migration: data/sql/164_mv_sales_tracker_velocity_absorption.sql.
"""
from __future__ import annotations
import logging
from typing import Any
from app.core.db import SessionLocal
from app.services.site_finder.sales_tracker_mv_refresh import refresh_sales_tracker_mvs
from app.workers.celery_app import celery_app
logger = logging.getLogger(__name__)
@celery_app.task(
bind=True,
name="tasks.mv_sales_tracker_refresh.refresh_sales_tracker_mvs",
max_retries=2,
)
def refresh_sales_tracker_mvs_task(self: Any) -> dict[str, Any]:
"""REFRESH both sales-tracker MVs (#61).
Both MVs are refreshed CONCURRENTLY (non-blocking, require their UNIQUE
indexes created by migration 161); the service falls back to non-concurrent
if an MV is found unpopulated (first-run edge case).
Returns result dict for the Celery task result store / logging.
"""
db = SessionLocal()
try:
counts = refresh_sales_tracker_mvs(db, concurrently=True)
logger.info("refresh_sales_tracker_mvs: completed, rows=%s", counts)
return {"status": "ok", "rows": counts}
except Exception as e:
logger.exception("refresh_sales_tracker_mvs failed: %s", e)
raise
finally:
db.close()

View file

@ -47,21 +47,23 @@ from app.workers.celery_app import celery_app
logger = logging.getLogger(__name__)
# psycopg v3: CAST(:x AS type) — НИКОГДА :x::type (SQLAlchemy+psycopg3 роняет
# синтаксис на ::). Контракт колонок совпадает с macro_indicator (migration 123):
# (indicator_type, region, obs_date, value, source, frequency, unit, comment,
# updated_at) PK (indicator_type, region, obs_date). source='rosstat',
# frequency='yearly' (текущие ряды Росстата — годовые: obs_date = 1 января года).
# синтаксис на ::). Контракт колонок совпадает с macro_indicator (migration 163):
# (indicator_type, region, obs_date, period_type, value, source, frequency, unit,
# comment, updated_at). PK (indicator_type, region, obs_date, period_type).
# Не-ЕМИСС источники (open-data, СМР) используют period_type='unknown' (литерал) —
# они различаются по obs_date без sub-period granularity.
# source='rosstat', frequency='yearly' (текущие ряды Росстата — годовые).
UPSERT_ROSSTAT_SQL = text(
"""
INSERT INTO macro_indicator (
indicator_type, region, obs_date, value,
indicator_type, region, obs_date, period_type, value,
source, frequency, unit, comment
) VALUES (
CAST(:itype AS text), CAST(:region AS text), CAST(:d AS date),
CAST(:v AS numeric),
'unknown', CAST(:v AS numeric),
'rosstat', 'yearly', CAST(:unit AS text), CAST(:comment AS text)
)
ON CONFLICT (indicator_type, region, obs_date) DO UPDATE SET
ON CONFLICT (indicator_type, region, obs_date, period_type) DO UPDATE SET
value = EXCLUDED.value,
source = EXCLUDED.source,
frequency = EXCLUDED.frequency,
@ -71,20 +73,23 @@ UPSERT_ROSSTAT_SQL = text(
"""
)
# ЕМИСС-ряды (source='emiss'): frequency параметризована (quarterly для доходов,
# monthly для ИПЦ когда добавится) — в отличие от open-data, где она фикс 'yearly'.
# Тот же PK и ON CONFLICT-контракт. CAST(:x AS type) — НИКОГДА :x::type (psycopg v3).
# ЕМИСС-ряды (source='emiss'): period_type параметризован — 'year' | 'quarter' |
# 'month' (из _emiss_period_granularity). Это ключевое отличие от не-ЕМИСС источников:
# период ЕМИСС-ряда несёт granularity, необходимую для разделения годового агрегата
# ('год' → 'year') и Q1 ('I квартал' → 'quarter'), оба с obs_date=YYYY-01-01 (#1606).
# frequency параметризована (quarterly для доходов, monthly для ИПЦ когда добавится).
# CAST(:x AS type) — НИКОГДА :x::type (psycopg v3).
UPSERT_EMISS_SQL = text(
"""
INSERT INTO macro_indicator (
indicator_type, region, obs_date, value,
indicator_type, region, obs_date, period_type, value,
source, frequency, unit, comment
) VALUES (
CAST(:itype AS text), CAST(:region AS text), CAST(:d AS date),
CAST(:v AS numeric),
CAST(:period_type AS text), CAST(:v AS numeric),
'emiss', CAST(:freq AS text), CAST(:unit AS text), CAST(:comment AS text)
)
ON CONFLICT (indicator_type, region, obs_date) DO UPDATE SET
ON CONFLICT (indicator_type, region, obs_date, period_type) DO UPDATE SET
value = EXCLUDED.value,
source = EXCLUDED.source,
frequency = EXCLUDED.frequency,
@ -96,19 +101,19 @@ UPSERT_EMISS_SQL = text(
# Индекс цен на СМР — открытый xlsx Росстата: source='rosstat' (rosstat.gov.ru-файл),
# НО frequency='monthly' (в отличие от open-data демографии — там фикс 'yearly'),
# поэтому frequency параметризована. MacroRow не несёт frequency-поля, подставляем
# литералом в bind. Тот же PK и ON CONFLICT-контракт. CAST(:x AS type) — psycopg v3.
# поэтому frequency параметризована. MacroRow не несёт frequency/period_type-полей;
# подставляем литералами. CAST(:x AS type) — psycopg v3.
UPSERT_ROSSTAT_MONTHLY_SQL = text(
"""
INSERT INTO macro_indicator (
indicator_type, region, obs_date, value,
indicator_type, region, obs_date, period_type, value,
source, frequency, unit, comment
) VALUES (
CAST(:itype AS text), CAST(:region AS text), CAST(:d AS date),
CAST(:v AS numeric),
'unknown', CAST(:v AS numeric),
'rosstat', 'monthly', CAST(:unit AS text), CAST(:comment AS text)
)
ON CONFLICT (indicator_type, region, obs_date) DO UPDATE SET
ON CONFLICT (indicator_type, region, obs_date, period_type) DO UPDATE SET
value = EXCLUDED.value,
source = EXCLUDED.source,
frequency = EXCLUDED.frequency,
@ -185,9 +190,15 @@ def _upsert_monthly_rows(db: Session, rows: list[MacroRow]) -> int:
def _upsert_emiss_rows(db: Session, rows: list[EmissRow]) -> int:
"""Апсертит EmissRow в macro_indicator (source='emiss', frequency per-row).
"""Апсертит EmissRow в macro_indicator (source='emiss', frequency+period_type per-row).
SAVEPOINT per-row, чтобы один битый ряд не откатывал всю транзакцию. Возвращает
число успешных upsert'ов."""
число успешных upsert'ов.
period_type (из r.period_type) часть нового PK (migration 163): позволяет
годовому агрегату ('year') и Q1 ('quarter') за один год коexist в таблице без
взаимной перезаписи (#1606).
"""
upserted = 0
for r in rows:
try:
@ -198,6 +209,7 @@ def _upsert_emiss_rows(db: Session, rows: list[EmissRow]) -> int:
"itype": r.indicator_type,
"region": r.region,
"d": r.obs_date,
"period_type": r.period_type,
"v": r.value,
"freq": r.frequency,
"unit": r.unit,
@ -207,10 +219,11 @@ def _upsert_emiss_rows(db: Session, rows: list[EmissRow]) -> int:
upserted += 1
except Exception as e:
logger.warning(
"upsert emiss %s/%s@%s=%s failed: %s",
"upsert emiss %s/%s@%s[%s]=%s failed: %s",
r.indicator_type,
r.region,
r.obs_date,
r.period_type,
r.value,
e,
)

View file

@ -108,19 +108,30 @@ def test_yearly_and_q1_both_survive_dedup() -> None:
assert Decimal("42000") in values, "Q1-наблюдение потеряно"
# obs_date у обоих одинаковый (это нормально — коллизия теперь на стороне DB-upsert)
assert all(r.obs_date == date(2023, 1, 1) for r in rows)
# period_type: годовой → 'year', Q1 → 'quarter' (часть PK в macro_indicator, migration 163)
period_types = {r.period_type for r in rows}
assert period_types == {
"year",
"quarter",
}, f"ожидались period_type year+quarter, получили {period_types}"
# ── real-fixture extraction (income id=57039) ─────────────────────────────────────
def test_income_extracts_sverdlovsk_only() -> None:
"""Из реальной SDMX-выгрузки извлекается ТОЛЬКО Свердл (ОКАТО 65) — РФ/ЦФО/Адыгея нет."""
"""Из реальной SDMX-выгрузки извлекается ТОЛЬКО Свердл (ОКАТО 65) — РФ/ЦФО/Адыгея нет.
Все строки квартальные period_type='quarter' (часть PK macro_indicator, migration 163).
"""
rows = parse_emiss_sdmx(_load("emiss_income_57039.xml"), INCOME_PER_CAPITA_SPEC)
assert rows, "ожидались строки по Свердл"
assert {r.region for r in rows} == {"sverdl"}
assert all(r.indicator_type == "income_per_capita" for r in rows)
assert all(r.unit == "руб" for r in rows)
assert all(r.frequency == "quarterly" for r in rows)
assert all(
r.period_type == "quarter" for r in rows
), "квартальные строки должны иметь period_type='quarter'"
def test_income_concrete_values_and_dates() -> None:

View file

@ -77,7 +77,12 @@ def test_find_match_candidates_returns_candidates() -> None:
def test_auto_apply_matches_dry_run_no_inserts() -> None:
"""dry_run=True возвращает счётчики без обращения к БД (execute не вызывается)."""
"""dry_run=True возвращает projected-счётчики без обращения к БД.
auto_accepted = сколько кандидатов БЫЛО БЫ принято (preview), а не 0
смысл dry-run в admin-endpoint'е именно показать оператору объём перед
реальным insert. execute/commit при этом не вызываются.
"""
mock_db = MagicMock()
candidates = [
@ -88,8 +93,9 @@ def test_auto_apply_matches_dry_run_no_inserts() -> None:
result = auto_apply_matches(mock_db, candidates, dry_run=True)
assert result["auto_accepted"] == 0
assert result["auto_accepted"] == 1 # projected: 1 кандидат >= AUTO_ACCEPT_THRESHOLD
assert result["review_queue"] == 2
assert result["skipped"] == 0
mock_db.execute.assert_not_called()
mock_db.commit.assert_not_called()

View file

@ -327,6 +327,39 @@ def test_by_room_bucket_empty_when_no_bucket_data():
assert result.by_room_bucket == {}
def test_mapping_confidence_gate_in_sales_query():
"""OBJ-2 (#307): sales/bucket queries фильтруют objective_complex_mapping
по confidence unreviewed low-score auto-matches исключены.
Проверяем, что SQL, переданный в db.execute для маппинг-CTE, содержит
предикат gate (is_reviewed / manual / match_score >= 0.85). Это гарантирует,
что ~115 fuzzy-trgm строк с is_reviewed=false и score<0.85 не попадают в velocity.
"""
comp_rows = [_comp_row(1), _comp_row(2)]
sales_rows = [
_sales_row(1, total_sqm=4000.0, months=4),
_sales_row(2, total_sqm=3000.0, months=4),
]
bucket_rows = [_bucket_row(1, "1", units_sold=20, sqm_sold=900.0)]
db = _make_db(comp_rows=comp_rows, sales_rows=sales_rows, bucket_rows=bucket_rows)
with patch(
"app.services.site_finder.velocity._get_ekb_median",
return_value=_EKB_MEDIAN_FALLBACK_SQM_PER_MONTH,
):
compute_velocity(db, parcel_geom_wkt=_PARCEL_WKT)
# db.execute вызывается 3 раза: comp / sales / bucket. SQL берём из call_args.
executed_sql = [str(call.args[0]) for call in db.execute.call_args_list]
# comp-query НЕ трогает mapping; sales (idx 1) и bucket (idx 2) — должны иметь gate.
mapping_queries = [s for s in executed_sql if "objective_complex_mapping" in s]
assert len(mapping_queries) >= 2, "ожидаются sales + bucket запросы с mapping"
for sql in mapping_queries:
assert "cm.is_reviewed = TRUE" in sql
assert "cm.match_method = 'manual'" in sql
assert "cm.match_score >= 0.85" in sql
def test_sample_competitors_include_by_room_bucket():
"""sample_competitors каждого элемента содержит by_room_bucket."""
comp_rows = [_comp_row(1), _comp_row(2)]

View file

@ -48,14 +48,16 @@ def test_upsert_sql_contract() -> None:
assert "'rf'" in sql
assert "'cbr'" in sql
assert "'daily'" in sql
# ON CONFLICT по полному PK + обновление value/updated_at
assert "ON CONFLICT (INDICATOR_TYPE, REGION, OBS_DATE) DO UPDATE" in upper
# ON CONFLICT по новому PK включает period_type (migration 163)
assert "ON CONFLICT (INDICATOR_TYPE, REGION, OBS_DATE, PERIOD_TYPE) DO UPDATE" in upper
assert "VALUE = EXCLUDED.VALUE" in upper
assert "UPDATED_AT = NOW()" in upper
# psycopg v3: CAST(:x AS type), НИКОГДА :x::type
assert "CAST(:D AS DATE)" in upper
assert "CAST(:V AS NUMERIC)" in upper
assert "::" not in sql
# CBR-строки используют period_type='unknown' (литерал в SQL)
assert "'unknown'" in sql
def test_task_upserts_each_row_with_correct_params() -> None:
@ -192,13 +194,14 @@ def test_upsert_inflation_sql_contract() -> None:
assert "'rf'" in sql
assert "'cbr'" in sql
assert "'monthly'" in sql
assert "ON CONFLICT (INDICATOR_TYPE, REGION, OBS_DATE) DO UPDATE" in upper
assert "ON CONFLICT (INDICATOR_TYPE, REGION, OBS_DATE, PERIOD_TYPE) DO UPDATE" in upper
assert "VALUE = EXCLUDED.VALUE" in upper
assert "UPDATED_AT = NOW()" in upper
# psycopg v3: CAST(:x AS type), НИКОГДА :x::type
assert "CAST(:D AS DATE)" in upper
assert "CAST(:V AS NUMERIC)" in upper
assert "::" not in sql
assert "'unknown'" in sql
def test_task_registered_in_beat_weekly() -> None:

View file

@ -128,9 +128,12 @@ def test_harvest_for_mo_skips_geomless_feature(monkeypatch: Any) -> None:
def test_harvest_all_iterates_5_mo(monkeypatch: Any) -> None:
"""harvest_all прогоняет все 5 МО агломерации."""
"""harvest_all прогоняет все 5 МО агломерации (при включённом флаге)."""
db = _FakeDB()
_patch(monkeypatch, db)
monkeypatch.setattr(
riasurt_sverdl_harvest.settings, "enable_riasurt_harvest", True, raising=False
)
res = riasurt_sverdl_harvest.harvest_all_riasurt_sverdl([845274])
assert res["mo"] == 5
@ -138,6 +141,21 @@ def test_harvest_all_iterates_5_mo(monkeypatch: Any) -> None:
assert res["features"] == 5
def test_harvest_all_disabled_by_default(monkeypatch: Any) -> None:
"""Гейт #108: при выключенном флаге harvest_all возвращает early без WMS-вызовов."""
db = _FakeDB()
client = _patch(monkeypatch, db)
monkeypatch.setattr(
riasurt_sverdl_harvest.settings, "enable_riasurt_harvest", False, raising=False
)
res = riasurt_sverdl_harvest.harvest_all_riasurt_sverdl([845274])
assert res == {"mo": 0, "features": 0}
# ни одного WMS-запроса, ни одной записи в БД
assert client.bbox_calls == []
assert db.executed == []
def test_mo_bboxes_has_5_agglomeration_municipalities() -> None:
"""MO_BBOXES содержит ровно 5 МО окраин агломерации."""
assert set(riasurt_sverdl_harvest.MO_BBOXES) == {

View file

@ -35,7 +35,7 @@ def _make_mock_db() -> tuple[MagicMock, list[dict[str, Any]]]:
def test_upsert_sql_contract() -> None:
"""UPSERT_ROSSTAT_SQL содержит обязательные литералы контракта macro_indicator
и НЕ использует :x::type (psycopg v3)."""
и НЕ использует :x::type (psycopg v3). PK включает period_type (migration 163)."""
from app.workers.tasks.rosstat_macro_sync import UPSERT_ROSSTAT_SQL
sql = str(UPSERT_ROSSTAT_SQL)
@ -44,7 +44,7 @@ def test_upsert_sql_contract() -> None:
assert "INTO MACRO_INDICATOR" in upper
assert "'rosstat'" in sql
assert "'yearly'" in sql
assert "ON CONFLICT (INDICATOR_TYPE, REGION, OBS_DATE) DO UPDATE" in upper
assert "ON CONFLICT (INDICATOR_TYPE, REGION, OBS_DATE, PERIOD_TYPE) DO UPDATE" in upper
assert "VALUE = EXCLUDED.VALUE" in upper
assert "UPDATED_AT = NOW()" in upper
# psycopg v3: CAST(:x AS type), НИКОГДА :x::type
@ -52,6 +52,8 @@ def test_upsert_sql_contract() -> None:
assert "CAST(:D AS DATE)" in upper
assert "CAST(:V AS NUMERIC)" in upper
assert "::" not in sql
# не-ЕМИСС строки используют period_type='unknown' (литерал в SQL)
assert "'unknown'" in sql
def test_task_upserts_each_row_with_correct_params() -> None:
@ -155,15 +157,24 @@ def test_task_emiss_failure_does_not_block_opendata() -> None:
def test_task_upserts_emiss_rows_with_correct_params() -> None:
"""Каждая EmissRow → execute UPSERT_EMISS_SQL с itype/region/d/v/freq/unit/comment."""
"""Каждая EmissRow → execute UPSERT_EMISS_SQL с itype/region/d/period_type/v/freq/unit/comment.
period_type передаётся из r.period_type (часть нового PK, migration 163).
"""
from app.services.scrapers.rosstat_emiss import EmissRow
from app.workers.tasks import rosstat_macro_sync as task_mod
db, captured = _make_mock_db()
emiss_rows = [
EmissRow(
"income_per_capita", "sverdl", date(2024, 1, 1),
Decimal("54006"), "руб", "quarterly", "c",
"income_per_capita",
"sverdl",
date(2024, 1, 1),
Decimal("54006"),
"руб",
"quarterly",
"c",
period_type="quarter",
),
]
@ -181,6 +192,7 @@ def test_task_upserts_emiss_rows_with_correct_params() -> None:
"itype": "income_per_capita",
"region": "sverdl",
"d": date(2024, 1, 1),
"period_type": "quarter",
"v": Decimal("54006"),
"freq": "quarterly",
"unit": "руб",
@ -190,7 +202,8 @@ def test_task_upserts_emiss_rows_with_correct_params() -> None:
def test_upsert_emiss_sql_contract() -> None:
"""UPSERT_EMISS_SQL: source='emiss', frequency параметризован, ON CONFLICT, CAST not ::."""
"""UPSERT_EMISS_SQL: source='emiss', frequency+period_type параметризованы,
ON CONFLICT включает period_type (migration 163), CAST not ::."""
from app.workers.tasks.rosstat_macro_sync import UPSERT_EMISS_SQL
sql = str(UPSERT_EMISS_SQL)
@ -198,14 +211,16 @@ def test_upsert_emiss_sql_contract() -> None:
assert "INTO MACRO_INDICATOR" in upper
assert "'emiss'" in sql
assert "CAST(:FREQ AS TEXT)" in upper
assert "ON CONFLICT (INDICATOR_TYPE, REGION, OBS_DATE) DO UPDATE" in upper
assert "CAST(:PERIOD_TYPE AS TEXT)" in upper
assert "ON CONFLICT (INDICATOR_TYPE, REGION, OBS_DATE, PERIOD_TYPE) DO UPDATE" in upper
assert "CAST(:ITYPE AS TEXT)" in upper
assert "CAST(:V AS NUMERIC)" in upper
assert "::" not in sql
def test_upsert_rosstat_monthly_sql_contract() -> None:
"""UPSERT_ROSSTAT_MONTHLY_SQL: source='rosstat', frequency='monthly', CAST not ::."""
"""UPSERT_ROSSTAT_MONTHLY_SQL: source='rosstat', frequency='monthly', period_type='unknown',
ON CONFLICT включает period_type (migration 163), CAST not ::."""
from app.workers.tasks.rosstat_macro_sync import UPSERT_ROSSTAT_MONTHLY_SQL
sql = str(UPSERT_ROSSTAT_MONTHLY_SQL)
@ -213,11 +228,12 @@ def test_upsert_rosstat_monthly_sql_contract() -> None:
assert "INTO MACRO_INDICATOR" in upper
assert "'rosstat'" in sql
assert "'monthly'" in sql
assert "ON CONFLICT (INDICATOR_TYPE, REGION, OBS_DATE) DO UPDATE" in upper
assert "ON CONFLICT (INDICATOR_TYPE, REGION, OBS_DATE, PERIOD_TYPE) DO UPDATE" in upper
assert "CAST(:ITYPE AS TEXT)" in upper
assert "CAST(:D AS DATE)" in upper
assert "CAST(:V AS NUMERIC)" in upper
assert "::" not in sql
assert "'unknown'" in sql
def test_task_upserts_construction_rows_with_correct_params() -> None:
@ -228,8 +244,12 @@ def test_task_upserts_construction_rows_with_correct_params() -> None:
db, captured = _make_mock_db()
constr_rows = [
MacroRow(
"construction_price_index", "rf", date(2025, 1, 1),
Decimal("100.8"), "%", "smr",
"construction_price_index",
"rf",
date(2025, 1, 1),
Decimal("100.8"),
"%",
"smr",
),
]

View file

@ -0,0 +1,92 @@
-- 163_emiss_pk_period_type.sql
-- #1606 follow-up: расширить PK macro_indicator колонкой period_type, чтобы годовой
-- агрегат ('год' → 'year') и Q1 ('I квартал' → 'quarter') за один год не перезаписывали
-- друг друга при ON CONFLICT DO UPDATE (оба дают obs_date=YYYY-01-01).
--
-- Контекст:
-- PR #1687 (#1606) исправил in-memory дедупликацию в parse_emiss_sdmx: ключ дедупа
-- стал трёхкомпонентным (region, obs_date, granularity). Но PK таблицы остался
-- (indicator_type, region, obs_date) — при апсерте в БД коллизия сохранялась.
-- Эта миграция фиксирует PK на стороне БД.
--
-- Что делает:
-- 1. Добавляет колонку period_type TEXT NOT NULL DEFAULT 'unknown'.
-- Все существующие строки (CBR, rosstat open-data, domrf) получают 'unknown' —
-- это корректно: у них нет sub-period granularity (они уже различаются по obs_date).
-- 2. Бэкфилл: строки source='emiss' обновляем до 'quarter' (единственный
-- активный ЕМИСС-индикатор — income_per_capita, он квартальный; годовых EMISS-
-- строк ещё нет в таблице до этой миграции). Значение ДОЛЖНО совпадать с тем,
-- что пишет UPSERT_EMISS_SQL (_emiss_period_granularity → 'quarter'/'year'/'month',
-- короткая форма), иначе ON CONFLICT не сматчит → дубли при следующем скрейпе.
-- 3. Пересоздаёт PRIMARY KEY: старый (indicator_type, region, obs_date) → новый
-- (indicator_type, region, obs_date, period_type).
-- 4. Пересоздаёт сопутствующий индекс idx_macro_type_region_date с тем же составом
-- (расширяется автоматически через новый PK-индекс; отдельный индекс DROP+CREATE).
--
-- Зависимости: macro_indicator (migration 123_macro_indicator.sql).
-- Нет VIEW/FK зависимостей от macro_indicator PK (проверено через pg_depend).
-- Idempotent: ADD COLUMN IF NOT EXISTS + IF NOT EXISTS в CREATE INDEX. Повторный
-- прогон безопасен: DROP CONSTRAINT / ADD CONSTRAINT могут упасть если PK уже
-- переименован — обёрнуты в DO $$ BEGIN ... EXCEPTION WHEN ... END $$; блоки.
--
-- Deploy order: SQL migration применяется ПЕРВОЙ (auto-deploy.yml), затем деплоится
-- backend со обновлёнными UPSERT_EMISS_SQL / UPSERT_ROSSTAT_SQL / UPSERT_KEY_RATE_SQL
-- / UPSERT_INFLATION_SQL / UPSERT_ROSSTAT_MONTHLY_SQL (они несут новый конфликт-таргет
-- + period_type в INSERT).
BEGIN;
-- ── 1. Добавить колонку period_type (DEFAULT 'unknown' для всех существующих строк) ──
ALTER TABLE macro_indicator
ADD COLUMN IF NOT EXISTS period_type TEXT NOT NULL DEFAULT 'unknown';
-- ── 2. Бэкфилл ЕМИСС-строк → 'quarter' ────────────────────────────────────────────
-- Все существующие ЕМИСС-строки — квартальные (income_per_capita). Значение 'quarter'
-- (короткая форма) обязано совпадать с UPSERT_EMISS_SQL, который параметризует
-- period_type из _emiss_period_granularity ('quarter'/'year'/'month'). Иначе после
-- деплоя следующий скрейп вставит 'quarter'-строки, не матчащие 'quarterly' на
-- ON CONFLICT → дубли для каждого квартального наблюдения. Если появятся 'year'-строки
-- (будущие индикаторы) — придут с корректным period_type из скрейпера.
UPDATE macro_indicator
SET period_type = 'quarter'
WHERE source = 'emiss'
AND period_type = 'unknown';
-- ── 3. Пересоздать PRIMARY KEY ────────────────────────────────────────────────────────
-- PostgreSQL не поддерживает ALTER PRIMARY KEY напрямую — нужно DROP + ADD.
-- Ловим случай когда старый PK уже дропнут (повторный прогон).
DO $$
BEGIN
ALTER TABLE macro_indicator DROP CONSTRAINT macro_indicator_pkey;
EXCEPTION
WHEN undefined_object THEN
NULL; -- PK уже удалён (повторный прогон)
WHEN feature_not_supported THEN
NULL;
END;
$$;
-- Новый PK включает period_type. Если constraint с этим именем уже существует —
-- ADD CONSTRAINT бросит duplicate_object (42710, дубль имени constraint), ловим.
DO $$
BEGIN
ALTER TABLE macro_indicator
ADD CONSTRAINT macro_indicator_pkey
PRIMARY KEY (indicator_type, region, obs_date, period_type);
EXCEPTION
WHEN duplicate_object THEN
NULL; -- constraint с таким именем уже создан (повторный прогон)
WHEN invalid_table_definition THEN
NULL; -- уже существует тот же constraint
END;
$$;
-- ── 4. Пересоздать сопутствующий индекс ──────────────────────────────────────────────
-- Старый индекс (без period_type в составе) уже покрыт новым PK-индексом по факту,
-- но явно дропаем и пересоздаём с period_type для консистентности плана запросов.
DROP INDEX IF EXISTS idx_macro_type_region_date;
CREATE INDEX IF NOT EXISTS idx_macro_type_region_date
ON macro_indicator (indicator_type, region, obs_date DESC, period_type);
COMMIT;

View file

@ -0,0 +1,184 @@
-- 164_mv_sales_tracker_velocity_absorption.sql
-- Issue #61 — Velocity materialized views for Site Finder Velocity Score (4th scoring
-- criterion) + recommend_mix smart unit-mix. Foundation for sellout forecast.
--
-- B2-1 data source ("шахматки" / sales-tracker): the Объектив scraper
-- (backend/app/workers/tasks/scrape_objective.py) → tables:
-- objective_lots — 1.12M rows, one row per tracked lot (current state),
-- carries district / rooms_int / area_pd / sales_start_date /
-- is_sold / registration_date / contract_date / price_per_m2_rub.
-- objective_lots_history — 974k rows, daily-ish per-lot snapshots
-- (snapshot_date, is_sold, status, prices).
-- Snapshot history depth (as of 2026-06-17): 3 captures 2026-05-17 / 05-19 / 06-03 (spans
-- >2 weeks, sold count moved 193188->194893 => measurable absorption). Cohort/absorption
-- resolution improves automatically as the weekly scraper accumulates more snapshots.
--
-- -- MV 1: mv_sales_tracker_velocity_by_district --------------------------------------
-- Grain: (district, sale_month). One row per district per month.
-- Dedup: a lot appears in multiple snapshots within a month -> we keep that lot's LATEST
-- snapshot within the month (DISTINCT ON lot, snapshot_date DESC) before
-- aggregating, so total_count is lots-tracked-that-month (not snapshot rows).
-- Metrics: total_count, sold_count, avg_sold_price_per_m2, avg_sold_price_total,
-- sold_share (velocity proxy for SF Velocity Score).
--
-- -- MV 2: mv_sales_tracker_absorption_curves ----------------------------------------
-- Grain: (rooms_int, area_bucket, months_since_start). Cumulative sold% as f(months
-- from first_seen). "first_seen" = objective_lots.sales_start_date (true sales
-- launch — richer/longer than the 3-snapshot window). Sold-month anchor =
-- COALESCE(registration_date, contract_date). months_since_start clamped >= 0
-- (712 noise rows have anchor < start). 99.98% of sold lots carry both dates.
-- cohort_size = all lots in (rooms, area_bucket) cohort; cum_sold = sold lots
-- whose months_since_start <= the row's bucket; cum_sold_pct = cum_sold/cohort.
-- This is snapshot-sparsity-independent (driven by registration dates, not snapshots),
-- so the curve is usable today and the foundation for sellout forecast.
--
-- REFRESH CONCURRENTLY: both MVs get a UNIQUE index on their full grain immediately after
-- creation (on empty MV -> instant), enabling non-blocking weekly REFRESH CONCURRENTLY.
-- Scheduled via Celery beat `mv-sales-tracker-refresh-weekly` (Mon 04:30 MSK) ->
-- task app.workers.tasks.mv_sales_tracker_refresh.refresh_sales_tracker_mvs.
--
-- Deploy: auto-applied by deploy.yml via _schema_migrations tracking (one-shot, NN order).
-- Dependencies on existing objects: objective_lots, objective_lots_history (read-only).
-- No views depend on these MVs at creation time.
--
-- WARN: re-apply (DR / lost _schema_migrations / dev local) DROP ... CASCADE снесёт MV +
-- зависимости. После re-apply ПЕРВЫЙ refresh = non-concurrent (CONCURRENTLY падает
-- на пустой/не-populated MV). _schema_migrations нормально предотвращает re-apply.
BEGIN;
-- ====================================================================================
-- MV 1: velocity by district x month
-- ====================================================================================
DROP MATERIALIZED VIEW IF EXISTS mv_sales_tracker_velocity_by_district CASCADE;
CREATE MATERIALIZED VIEW mv_sales_tracker_velocity_by_district AS
WITH lot_month AS (
-- One row per (lot, month): the lot's latest snapshot within that month.
SELECT DISTINCT ON (h.objective_lot_id, date_trunc('month', h.snapshot_date))
l.district AS district,
date_trunc('month', h.snapshot_date)::date AS sale_month,
h.objective_lot_id,
h.is_sold,
h.price_per_m2_rub,
h.price_calculated_total_rub
FROM objective_lots_history h
JOIN objective_lots l ON l.objective_lot_id = h.objective_lot_id
WHERE l.district IS NOT NULL
ORDER BY h.objective_lot_id,
date_trunc('month', h.snapshot_date),
h.snapshot_date DESC
)
SELECT
district,
sale_month,
count(*)::int AS total_count,
count(*) FILTER (WHERE is_sold)::int AS sold_count,
round(
count(*) FILTER (WHERE is_sold)::numeric
/ NULLIF(count(*), 0), 4
) AS sold_share,
round(avg(price_per_m2_rub) FILTER (WHERE is_sold), 2) AS avg_sold_price_per_m2,
round(avg(price_calculated_total_rub) FILTER (WHERE is_sold), 2) AS avg_sold_price_total
FROM lot_month
GROUP BY district, sale_month
WITH NO DATA;
-- UNIQUE index on full grain -> enables REFRESH CONCURRENTLY (created on empty MV = instant)
CREATE UNIQUE INDEX mv_sales_tracker_velocity_by_district_pk
ON mv_sales_tracker_velocity_by_district (district, sale_month);
CREATE INDEX mv_sales_tracker_velocity_district_idx
ON mv_sales_tracker_velocity_by_district (district);
REFRESH MATERIALIZED VIEW mv_sales_tracker_velocity_by_district;
COMMENT ON MATERIALIZED VIEW mv_sales_tracker_velocity_by_district IS
'Issue #61. Per (district, month) sold/total/avg-sold-price from objective_lots_history '
'snapshots (Obektiv shahmatka), deduped to latest snapshot per lot per month. '
'Feeds Site Finder Velocity Score. Refresh weekly CONCURRENTLY.';
-- ====================================================================================
-- MV 2: absorption curves by room_count x area_bucket x months-from-first-seen
-- ====================================================================================
DROP MATERIALIZED VIEW IF EXISTS mv_sales_tracker_absorption_curves CASCADE;
CREATE MATERIALIZED VIEW mv_sales_tracker_absorption_curves AS
WITH base AS (
-- One row per lot. area_bucket from area_pd; months_since_start = whole months between
-- sales_start_date and the sold anchor (reg/contract). Unsold lots have NULL anchor.
SELECT
l.rooms_int,
CASE
WHEN l.area_pd < 30 THEN '<30'
WHEN l.area_pd < 45 THEN '30-45'
WHEN l.area_pd < 60 THEN '45-60'
WHEN l.area_pd < 80 THEN '60-80'
ELSE '80+'
END AS area_bucket,
l.is_sold,
CASE
WHEN l.is_sold
AND l.sales_start_date IS NOT NULL
AND COALESCE(l.registration_date, l.contract_date) IS NOT NULL
THEN GREATEST(
0,
(date_part('year', age(COALESCE(l.registration_date, l.contract_date),
l.sales_start_date)) * 12
+ date_part('month', age(COALESCE(l.registration_date, l.contract_date),
l.sales_start_date)))::int
)
END AS months_since_start
FROM objective_lots l
WHERE l.rooms_int IS NOT NULL
AND l.area_pd IS NOT NULL
AND l.sales_start_date IS NOT NULL
),
cohort AS (
SELECT rooms_int, area_bucket, count(*)::int AS cohort_size
FROM base
GROUP BY rooms_int, area_bucket
),
sold_at_month AS (
SELECT rooms_int, area_bucket, months_since_start, count(*)::int AS sold_in_month
FROM base
WHERE is_sold AND months_since_start IS NOT NULL
GROUP BY rooms_int, area_bucket, months_since_start
)
SELECT
s.rooms_int,
s.area_bucket,
s.months_since_start,
c.cohort_size,
-- cumulative sold up to and including this month-offset (per cohort)
SUM(s.sold_in_month) OVER (
PARTITION BY s.rooms_int, s.area_bucket
ORDER BY s.months_since_start
ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
)::int AS cum_sold,
round(
SUM(s.sold_in_month) OVER (
PARTITION BY s.rooms_int, s.area_bucket
ORDER BY s.months_since_start
ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
)::numeric / NULLIF(c.cohort_size, 0), 4
) AS cum_sold_pct
FROM sold_at_month s
JOIN cohort c ON c.rooms_int = s.rooms_int AND c.area_bucket = s.area_bucket
WITH NO DATA;
-- UNIQUE index on full grain -> enables REFRESH CONCURRENTLY
CREATE UNIQUE INDEX mv_sales_tracker_absorption_curves_pk
ON mv_sales_tracker_absorption_curves (rooms_int, area_bucket, months_since_start);
CREATE INDEX mv_sales_tracker_absorption_cohort_idx
ON mv_sales_tracker_absorption_curves (rooms_int, area_bucket);
REFRESH MATERIALIZED VIEW mv_sales_tracker_absorption_curves;
COMMENT ON MATERIALIZED VIEW mv_sales_tracker_absorption_curves IS
'Issue #61. Cumulative sold-pct as f(months from sales_start_date) per (rooms_int, '
'area_bucket). Anchor = COALESCE(registration_date, contract_date) from objective_lots. '
'Foundation for recommend_mix + sellout forecast. Refresh weekly CONCURRENTLY.';
COMMIT;

View file

@ -0,0 +1,93 @@
-- 165_velocity_mapping_reviewed_gate.sql
-- #307 OBJ-2 — gate objective_complex_mapping by confidence в mv_layout_velocity.
--
-- Контекст: fuzzy-trgm backfill (#1331/#1333, миграции 155/156/116/117/150) добавил
-- ~115+ auto-matched строк в objective_complex_mapping с is_reviewed=false и низким
-- match_score (вплоть до 0.50-0.625). Они с высокой вероятностью false-positive
-- (Objective project name ≠ тот же ЖК что domrf_obj_id) и искажали velocity-агрегаты
-- Site Finder, попадая в MV наравне с проверенными маппингами.
--
-- Fix: redefine MV с confidence-gate в JOIN — принимаем маппинг только если
-- is_reviewed = TRUE (человек подтвердил), ИЛИ
-- match_method = 'manual' (ручной маппинг, score обычно NULL), ИЛИ
-- match_score >= 0.85 (AUTO_ACCEPT_THRESHOLD из objective_backfill.py —
-- high-confidence auto, fuzzy_trgm 0.85+ надёжен).
--
-- Строгий gate только на is_reviewed=true дал бы 2 строки из 303 EKB-маппингов →
-- обнулил бы velocity. Порог 0.85 сохраняет ~264/303, отбрасывая ~39 низкоуверенных.
-- Тот же предикат применён в runtime-пути backend/app/services/site_finder/velocity.py.
--
-- Сохраняет weighted-average формулу из #21 (100_fix_mv_layout_velocity_weighted_avg.sql) —
-- меняется ТОЛЬКО WHERE/JOIN-условие на objective_complex_mapping.
--
-- Dependencies: mv_layout_velocity не имеет зависимых VIEW/MV (проверено 2026-05-17/100_fix).
-- Idempotency: DROP IF EXISTS → безопасен при повторном запуске.
-- Deploy order: SQL migration → app deploy. Auto-applied через _schema_migrations.
-- ВАЖНО: DROP + CREATE + REFRESH (non-concurrent, блокировка ~30 с на ~19 738 source rows).
BEGIN;
DROP MATERIALIZED VIEW IF EXISTS mv_layout_velocity CASCADE;
CREATE MATERIALIZED VIEW mv_layout_velocity AS
WITH last24mo AS (
SELECT
ocm.project_name,
CASE
WHEN ocm.room_bucket = 'студия' THEN 'studio'
ELSE ocm.room_bucket
END AS room_bucket,
ocm.deals_total_count,
ocm.deals_total_avg_area_m2,
ocm.deals_total_avg_price_thousand_rub_per_m2,
ocm.deals_total_vol_m2,
ocm.report_month
FROM objective_corpus_room_month ocm
WHERE ocm.report_month >= (NOW() - INTERVAL '24 months')::date
)
SELECT
cm.domrf_obj_id AS obj_id,
l.room_bucket,
SUM(l.deals_total_count)::int AS total_deals_24mo,
-- weighted average (#21) — нули из месяцев без сделок не тянут вниз.
(SUM(l.deals_total_avg_area_m2 * l.deals_total_count)
/ NULLIF(SUM(l.deals_total_count), 0))::numeric(10, 2) AS avg_area_m2,
(SUM(l.deals_total_avg_price_thousand_rub_per_m2 * l.deals_total_count)
/ NULLIF(SUM(l.deals_total_count), 0))::numeric(12, 2) AS avg_price_thousand_rub_per_m2,
SUM(l.deals_total_vol_m2)::numeric(12, 2) AS total_vol_m2,
MIN(l.report_month) AS window_start,
MAX(l.report_month) AS window_end,
COUNT(DISTINCT l.report_month)::int AS months_with_data
FROM last24mo l
JOIN objective_complex_mapping cm
ON cm.objective_complex_name = l.project_name
WHERE l.room_bucket IS NOT NULL
AND cm.domrf_obj_id IS NOT NULL
AND cm.objective_group = 'Екатеринбург'
-- #307 OBJ-2: confidence-gate — исключаем unreviewed low-score auto-matches.
AND (cm.is_reviewed = TRUE OR cm.match_method = 'manual' OR cm.match_score >= 0.85)
GROUP BY cm.domrf_obj_id, l.room_bucket
WITH NO DATA;
-- UNIQUE index: required for REFRESH CONCURRENTLY (periodic via layout_velocity_refresh.py).
CREATE UNIQUE INDEX mv_layout_velocity_pk
ON mv_layout_velocity (obj_id, room_bucket);
-- Lookup index for /best-layouts endpoint queries by obj_id.
CREATE INDEX mv_layout_velocity_obj_idx
ON mv_layout_velocity (obj_id);
-- Initial populate (non-concurrent — MV just created, CONCURRENTLY requires populated MV).
REFRESH MATERIALIZED VIEW mv_layout_velocity;
COMMENT ON MATERIALIZED VIEW mv_layout_velocity IS
'Per-(obj_id, room_bucket) deals aggregation за last 24 months. '
'WEIGHTED average площади и цены (SF Bug #21). '
'Confidence-gated mapping: is_reviewed/manual/score>=0.85 (#307 OBJ-2). '
'Source: objective_corpus_room_month × objective_complex_mapping (EKB only). '
'Refresh via layout_velocity_refresh.py (CONCURRENTLY после initial populate).';
COMMIT;

View file

@ -4168,6 +4168,8 @@ export interface components {
distance_m: number;
/** Weight */
weight: number;
/** Score Contribution */
score_contribution: number;
/** Address */
address: string | null;
};
@ -4177,6 +4179,8 @@ export interface components {
cad_num: string;
/** Radius M */
radius_m: number;
/** Poi Weighted Score */
poi_weighted_score: number;
/** Top Poi */
top_poi: components["schemas"]["PoiScoreItem"][];
};

View file

@ -6,9 +6,11 @@ from typing import Literal
from pydantic import BaseModel, Field, model_validator
SortKey = Literal[
"price_asc", "price_desc", "area_desc", "area_asc", "date_desc", "dist_asc"
]
SortKey = Literal["price_asc", "price_desc", "area_desc", "area_asc", "date_desc", "dist_asc"]
# Сегмент рынка (#1188, поверх canon-предиката #1186).
# NULL listing_segment (legacy вторичка до миграции 011) трактуется как 'vtorichka'.
SegmentKey = Literal["vtorichka", "novostroyki", "all"]
class SearchParams(BaseModel):
@ -49,6 +51,19 @@ class SearchParams(BaseModel):
address_query: str | None = Field(default=None, max_length=200)
description_query: str | None = Field(default=None, max_length=200)
# --- Market segment (#1188) ---
segment: SegmentKey = Field(
default="vtorichka",
description=(
"Сегмент рынка для фильтрации listings. "
"`vtorichka` (по умолчанию, back-compat) — вторичка; "
"строки с listing_segment=NULL (legacy до миграции 011) "
"считаются вторичкой согласно canon-предикату #1186. "
"`novostroyki` — только первичка. "
"`all` — без фильтра по сегменту."
),
)
# --- Sort + pagination ---
sort: SortKey = "date_desc"
page: int = Field(default=1, ge=1, le=1000)

View file

@ -106,9 +106,9 @@ class DkpCorridor(BaseModel):
"""
count: int # число ДКП-сделок в выборке
low_ppm2: int # min ₽/м² по сделкам (P10-ish — берём минимум)
low_ppm2: int # P10 ₽/м² по сделкам (робастный коридор)
median_ppm2: int # медиана ₽/м²
high_ppm2: int # max ₽/м²
high_ppm2: int # P90 ₽/м² по сделкам (робастный коридор)
period_months: int # окно поиска сделок

View file

@ -321,6 +321,7 @@ async def trigger_yandex_city_sweep_run(db: Session, schedule_row: dict[str, Any
request_delay_sec=float(params.get("request_delay_sec", 9.0)),
radius_m=int(params.get("radius_m", 1500)),
enrich_address=bool(params.get("enrich_address", True)),
segments=params.get("segments"),
)
except Exception:
logger.exception("scheduler: run_yandex_city_sweep crashed run_id=%d", run_id)

View file

@ -1033,6 +1033,7 @@ async def run_yandex_city_sweep(
enrich_address: bool = True,
rooms_list: list[str] | None = None,
price_ranges: list[tuple[int | None, int | None]] | None = None,
segments: list[str] | None = None,
) -> YandexCitySweepCounters:
"""Yandex.Недвижимость city sweep: rooms × price combos от центра ЕКБ → save → address-enrich.
@ -1078,6 +1079,11 @@ async def run_yandex_city_sweep(
_rooms_list = rooms_list or list(ROOM_PATH.keys())
_price_ranges = price_ranges or DEFAULT_PRICE_RANGES
# newFlat-сегменты: оба прохода (vtorichka + novostroyki) по одним combos.
# Дефолт ["NO", "YES"] — оба; dedup по source_id span'ит оба прохода (один seen
# внутри fetch_around_multi_room). Counters (lots_fetched/inserted/updated)
# агрегируют оба прохода через len(anchor_lots).
_segments = segments or ["NO", "YES"]
counters = YandexCitySweepCounters(anchors_total=len(_anchors))
inter_anchor_delay = request_delay_sec if request_delay_sec is not None else 7.0
@ -1086,16 +1092,21 @@ async def run_yandex_city_sweep(
consecutive_failures = 0
# Вычисляем watchdog-таймаут для combos-режима (центр, anchors=None).
# В combos-режиме один "anchor" выполняет num_combos × max_pages fetch'ей —
# каждый занимает примерно _resolved_delay + _YANDEX_COMBOS_PER_FETCH_S секунд.
# Для 30 combos × 3 pages × (9+12) + 300 ≈ 2190s (~37 мин).
# В combos-режиме один "anchor" выполняет num_segments × num_combos × max_pages
# fetch'ей — каждый занимает примерно _resolved_delay + _YANDEX_COMBOS_PER_FETCH_S сек.
# Для 2 segments × 30 combos × 3 pages × (9+12) + 300 ≈ 4080s (~68 мин).
# Множитель len(_segments) масштабирует watchdog пропорционально числу проходов
# (vtorichka + novostroyki ≈ ×2 wall-clock) — иначе второй проход гильотинится.
# В explicit-anchor (тест/override) режиме оставляем ANCHOR_TIMEOUT_SEC (240s) —
# там каждый anchor небольшой и watchdog работает как старый защитный барьер.
_num_combos = len(_rooms_list) * len(_price_ranges)
if anchors is None and _num_combos > 0:
_sweep_timeout = max(
ANCHOR_TIMEOUT_SEC,
_num_combos * pages_per_anchor * (_resolved_delay + _YANDEX_COMBOS_PER_FETCH_S)
len(_segments)
* _num_combos
* pages_per_anchor
* (_resolved_delay + _YANDEX_COMBOS_PER_FETCH_S)
+ _YANDEX_ADDRESS_ENRICH_BUDGET_S,
)
else:
@ -1162,6 +1173,7 @@ async def run_yandex_city_sweep(
max_pages=pages_per_anchor,
rooms_list=_rooms_list,
price_ranges=_price_ranges,
segments=_segments,
)
counters.lots_fetched += len(anchor_lots)
if anchor_lots:

View file

@ -74,15 +74,7 @@ _UNIT_DAYS: dict[str, int] = {
# поэтому отдельная ветка с неявным n=1. Юниты в винительном падеже («минуту»,
# «неделю») плюс именительный («час», «день»).
_REL_DATE_SINGULAR_RE = re.compile(
r"(?P<unit>"
r"секунду|"
r"минуту|"
r"час|"
r"день|"
r"неделю|"
r"месяц|"
r"год"
r")\s+назад",
r"(?P<unit>" r"секунду|" r"минуту|" r"час|" r"день|" r"неделю|" r"месяц|" r"год" r")\s+назад",
flags=re.I,
)
@ -751,12 +743,12 @@ class AvitoScraper(BaseScraper):
house_ext_id = parts[-1]
house_url = urljoin("https://www.avito.ru", h_href)
# listing_segment по item URL pattern
listing_segment: str | None = None
if "/kvartiry/" in href:
listing_segment = "vtorichka"
elif "/novostroyki/" in href:
listing_segment = "novostroyki"
# listing_segment по DOM-маркеру карточки: новостройки несут
# data-marker="item-development-name" (название ЖК/застройщика),
# вторичка — нет. URL-паттерн ненадёжен: каждый avito-URL квартиры
# содержит /kvartiry/ (вкл. новостройки) → ветка всегда vtorichka.
dev_name = card.css_first('[data-marker="item-development-name"]')
listing_segment = "novostroyki" if dev_name is not None else "vtorichka"
return ScrapedLot(
source="avito",

View file

@ -152,6 +152,7 @@ _SALE_TYPE_MAP: dict[str, str] = {
"свободная": "free",
"альтернатива": "alternative",
"аукцион": "auction",
"переуступка": "assignment",
}
_HOUSE_TYPE_MAP: dict[str, str] = {

View file

@ -48,6 +48,7 @@ class DetailEnrichment:
repair_type: str | None = None # 'cosmetic' / 'design' / 'no' — raw Cian value
repair_state: str | None = None # enum: needs_repair/standard/good/excellent
kitchen_area_m2: float | None = None # offer.kitchenArea (м²)
description: str | None = None # offer.description (текст объявления)
views_total: int | None = None # Cian stats.totalViewsFormattedString → int
views_today: int | None = None
@ -144,6 +145,7 @@ async def fetch_detail(
repair_type=_extract_repair_type(offer),
repair_state=_extract_repair_state(offer),
kitchen_area_m2=_parse_float(offer.get("kitchenArea")),
description=offer.get("description") or None,
raw_offer=offer,
)
@ -302,8 +304,10 @@ def save_detail_enrichment(db: Session, listing_id: int, enrichment: DetailEnric
repair_type = COALESCE(:rt, repair_type),
repair_state = COALESCE(:rs, repair_state),
kitchen_area_m2 = COALESCE(CAST(:ka AS double precision), kitchen_area_m2),
description = COALESCE(:descr, description),
views_total = COALESCE(:vt, views_total),
views_today = COALESCE(:vd, views_today)
views_today = COALESCE(:vd, views_today),
detail_enriched_at = NOW()
WHERE id = CAST(:lid AS bigint)
"""),
{
@ -315,6 +319,7 @@ def save_detail_enrichment(db: Session, listing_id: int, enrichment: DetailEnric
"rt": enrichment.repair_type,
"rs": enrichment.repair_state,
"ka": enrichment.kitchen_area_m2,
"descr": enrichment.description,
"vt": enrichment.views_total,
"vd": enrichment.views_today,
},

View file

@ -129,7 +129,14 @@ def _extract_json_from_content(content: str) -> str | None:
return None
def _parse_gate_json(payload: dict[str, Any], page_param: int = 1) -> list[ScrapedLot]:
def _segment_for(new_flat: str) -> str:
"""Map a newFlat gate-API value to the listings.listing_segment value."""
return "novostroyki" if new_flat == "YES" else "vtorichka"
def _parse_gate_json(
payload: dict[str, Any], page_param: int = 1, new_flat: str = "NO"
) -> list[ScrapedLot]:
"""Parse gate-API JSON payload into a list of ScrapedLot.
Verified field mapping (from .issdump/yandex_gate_api_sample.json + live prod 2026-06-17):
@ -150,7 +157,7 @@ def _parse_gate_json(payload: dict[str, Any], page_param: int = 1) -> list[Scrap
location.geocoderAddress -> address (full, with house number)
mainImages[] -> photo_urls (prepend "https:", up to 5)
building.siteId -> house_ext_id (JK linkage)
listing_segment = "vtorichka" (all requests use newFlat=NO)
listing_segment -> "novostroyki" (newFlat=YES) | "vtorichka" (newFlat=NO)
"""
result = _extract_gate_data(payload)
if result is None:
@ -160,13 +167,15 @@ def _parse_gate_json(payload: dict[str, Any], page_param: int = 1) -> list[Scrap
entities, _pager = result
lots: list[ScrapedLot] = []
for entity in entities:
lot = _entity_to_lot(entity, page_param=page_param)
lot = _entity_to_lot(entity, page_param=page_param, new_flat=new_flat)
if lot is not None:
lots.append(lot)
return lots
def _entity_to_lot(entity: dict[str, Any], page_param: int = 1) -> ScrapedLot | None:
def _entity_to_lot(
entity: dict[str, Any], page_param: int = 1, new_flat: str = "NO"
) -> ScrapedLot | None:
"""Convert one gate-API entity dict to ScrapedLot. None on missing required fields."""
try:
offer_id = str(entity.get("offerId") or "")
@ -263,7 +272,7 @@ def _entity_to_lot(entity: dict[str, Any], page_param: int = 1) -> ScrapedLot |
house_source=house_source,
house_ext_id=house_ext_id,
house_url=None,
listing_segment="vtorichka",
listing_segment=_segment_for(new_flat),
raw_payload={
"page_param": page_param,
"ceiling_height": ceiling_height,
@ -366,13 +375,18 @@ class YandexRealtyScraper(BaseScraper):
rooms: str | None = None,
price_min: int | None = None,
price_max: int | None = None,
new_flat: str = "NO",
) -> str:
"""Build gate-API URL. page: 1-based (page=0 -> API error)."""
"""Build gate-API URL. page: 1-based (page=0 -> API error).
new_flat: "NO" -> vtorichka (default, preserves legacy behavior),
"YES" -> novostroyki (primary-sale / reassignment segment).
"""
params: dict[str, str | int] = {
"rgid": _EKB_RGID,
"type": "SELL",
"category": "APARTMENT",
"newFlat": "NO",
"newFlat": new_flat,
"_pageType": "search",
"_providers": "react-search-results-data",
"page": page,
@ -391,9 +405,12 @@ class YandexRealtyScraper(BaseScraper):
page: int,
price_min: int | None,
price_max: int | None,
new_flat: str = "NO",
) -> dict[str, Any] | None:
"""Fetch one gate-API page. Returns parsed JSON dict or None on error/invalid."""
url = self._build_url(page=page, rooms=rooms, price_min=price_min, price_max=price_max)
url = self._build_url(
page=page, rooms=rooms, price_min=price_min, price_max=price_max, new_flat=new_flat
)
try:
resp = await self._http_get(url, timeout=60)
except Exception:
@ -421,14 +438,18 @@ class YandexRealtyScraper(BaseScraper):
rooms: str | None = None,
price_min: int | None = None,
price_max: int | None = None,
new_flat: str = "NO",
) -> list[ScrapedLot]:
"""Fetch ONE page of gate-API results. lat/lon/radius_m ignored (uses rgid).
page: 1-based. Tarpit resilience: status_code==0 or JSON error
-> rotate IP + retry up to _YANDEX_TARPIT_MAX_RETRIES times.
Non-tarpit failures (4xx/5xx) are not retried.
new_flat: "NO" -> vtorichka, "YES" -> novostroyki.
"""
url = self._build_url(page=page, rooms=rooms, price_min=price_min, price_max=price_max)
url = self._build_url(
page=page, rooms=rooms, price_min=price_min, price_max=price_max, new_flat=new_flat
)
payload: dict[str, Any] | None = None
for attempt in range(1 + _YANDEX_TARPIT_MAX_RETRIES):
@ -494,13 +515,14 @@ class YandexRealtyScraper(BaseScraper):
)
return []
lots = _parse_gate_json(payload, page_param=page)
lots = _parse_gate_json(payload, page_param=page, new_flat=new_flat)
logger.info(
"yandex gate page=%d rooms=%s price=[%s-%s]: %d cards",
"yandex gate page=%d rooms=%s price=[%s-%s] newFlat=%s: %d cards",
page,
rooms or "all",
price_min,
price_max,
new_flat,
len(lots),
)
await self.sleep_between_requests()
@ -514,16 +536,20 @@ class YandexRealtyScraper(BaseScraper):
max_pages: int = _GATE_MAX_PAGES_CAP,
rooms_list: list[str] | None = None,
price_ranges: list[tuple[int | None, int | None]] | None = None,
segments: list[str] | None = None,
**_legacy_kwargs: Any,
) -> list[ScrapedLot]:
"""Fetch via rooms x price-range combos; paginate each combo to totalPages.
"""Fetch via segment x rooms x price-range combos; paginate each combo to totalPages.
Pagination driven by pager.totalPages from first page response.
Each combo iterates page=1 .. min(totalPages, max_pages, _GATE_MAX_PAGES_CAP).
Deduplicates by source_id/source_url across all combos.
Deduplicates by source_id/source_url across ALL combos AND segments (one `seen`).
Legacy mode (rooms_list=None, price_ranges=None): single citywide sweep.
segments: list of newFlat values, default ["NO"] (vtorichka only). Pass
["NO", "YES"] to sweep both vtorichka and novostroyki in one call.
"""
seen: dict[str, ScrapedLot] = {}
_segments = segments or ["NO"]
if rooms_list is None and price_ranges is None:
combos: list[tuple[str | None, int | None, int | None]] = [(None, None, None)]
@ -532,112 +558,123 @@ class YandexRealtyScraper(BaseScraper):
p_ranges = price_ranges or DEFAULT_PRICE_RANGES
combos = [(r, lo, hi) for r in r_list for lo, hi in p_ranges]
for rooms, price_min, price_max in combos:
combo_label = _combo_label(rooms, price_min, price_max)
total_pages: int | None = None
for new_flat in _segments:
_seg = _segment_for(new_flat)
for rooms, price_min, price_max in combos:
combo_label = f"{_seg}/{_combo_label(rooms, price_min, price_max)}"
total_pages: int | None = None
for page in range(1, max_pages + 1):
if page == 1:
payload_p1: dict[str, Any] | None = None
p1_url = self._build_url(
page=1, rooms=rooms, price_min=price_min, price_max=price_max
)
for _attempt in range(1 + _YANDEX_TARPIT_MAX_RETRIES):
try:
resp = await self._http_get(p1_url, timeout=60)
except Exception:
logger.exception("yandex gate combo fetch failed combo=%s", combo_label)
for page in range(1, max_pages + 1):
if page == 1:
payload_p1: dict[str, Any] | None = None
p1_url = self._build_url(
page=1,
rooms=rooms,
price_min=price_min,
price_max=price_max,
new_flat=new_flat,
)
for _attempt in range(1 + _YANDEX_TARPIT_MAX_RETRIES):
try:
resp = await self._http_get(p1_url, timeout=60)
except Exception:
logger.exception(
"yandex gate combo fetch failed combo=%s", combo_label
)
break
if resp.status_code == 0: # type: ignore[union-attr]
logger.warning(
"yandex gate: tarpit combo=%s page=1 -- rotating", combo_label
)
await self._rotate_ip()
await asyncio.sleep(2)
continue
if resp.status_code != 200: # type: ignore[union-attr]
logger.warning(
"yandex gate: HTTP %d combo=%s page=1",
resp.status_code,
combo_label, # type: ignore[union-attr]
)
break
try:
payload_p1 = json.loads(resp.text) # type: ignore[union-attr]
except (json.JSONDecodeError, ValueError):
logger.warning(
"yandex gate: JSON parse error combo=%s page=1", combo_label
)
payload_p1 = None
await self._rotate_ip()
await asyncio.sleep(2)
continue
if _is_gate_error(payload_p1):
logger.warning(
"yandex gate: error payload combo=%s page=1", combo_label
)
payload_p1 = None
break
break
if resp.status_code == 0: # type: ignore[union-attr]
logger.warning(
"yandex gate: tarpit combo=%s page=1 -- rotating", combo_label
)
await self._rotate_ip()
await asyncio.sleep(2)
continue
if resp.status_code != 200: # type: ignore[union-attr]
logger.warning(
"yandex gate: HTTP %d combo=%s page=1",
resp.status_code,
combo_label, # type: ignore[union-attr]
if payload_p1 is None:
logger.debug(
"yandex gate combo [%s] page=1: failed -- skipping", combo_label
)
break
try:
payload_p1 = json.loads(resp.text) # type: ignore[union-attr]
except (json.JSONDecodeError, ValueError):
logger.warning(
"yandex gate: JSON parse error combo=%s page=1", combo_label
)
payload_p1 = None
await self._rotate_ip()
await asyncio.sleep(2)
continue
if _is_gate_error(payload_p1):
logger.warning(
"yandex gate: error payload combo=%s page=1", combo_label
)
payload_p1 = None
result = _extract_gate_data(payload_p1)
if result is None:
break
_entities_p1, pager_p1 = result
total_pages = min(
pager_p1.get("totalPages", 1),
max_pages,
_GATE_MAX_PAGES_CAP,
)
lots_p1 = _parse_gate_json(payload_p1, page_param=1, new_flat=new_flat)
if not lots_p1:
logger.debug(
"yandex gate combo [%s] page=1: empty -- stopping", combo_label
)
break
for lot in lots_p1:
key = lot.source_id or lot.source_url
if key and key not in seen:
seen[key] = lot
await self.sleep_between_requests()
if total_pages <= 1:
break
continue
if total_pages is not None and page > total_pages:
break
if payload_p1 is None:
lots = await self.fetch_around(
lat,
lon,
radius_m,
page=page,
rooms=rooms,
price_min=price_min,
price_max=price_max,
new_flat=new_flat,
)
if not lots:
logger.debug(
"yandex gate combo [%s] page=1: failed -- skipping", combo_label
"yandex gate combo [%s] page=%d: empty -- stopping",
combo_label,
page,
)
break
result = _extract_gate_data(payload_p1)
if result is None:
break
_entities_p1, pager_p1 = result
total_pages = min(
pager_p1.get("totalPages", 1),
max_pages,
_GATE_MAX_PAGES_CAP,
)
lots_p1 = _parse_gate_json(payload_p1, page_param=1)
if not lots_p1:
logger.debug(
"yandex gate combo [%s] page=1: empty -- stopping", combo_label
)
break
for lot in lots_p1:
for lot in lots:
key = lot.source_id or lot.source_url
if key and key not in seen:
seen[key] = lot
await self.sleep_between_requests()
if total_pages <= 1:
break
continue
if total_pages is not None and page > total_pages:
break
lots = await self.fetch_around(
lat,
lon,
radius_m,
page=page,
rooms=rooms,
price_min=price_min,
price_max=price_max,
)
if not lots:
logger.debug(
"yandex gate combo [%s] page=%d: empty -- stopping",
combo_label,
page,
)
break
for lot in lots:
key = lot.source_id or lot.source_url
if key and key not in seen:
seen[key] = lot
logger.info(
"yandex gate aggregate: %d unique lots (%d combos)",
"yandex gate aggregate: %d unique lots (%d combos x %d segments=%s)",
len(seen),
len(combos),
len(_segments),
_segments,
)
return list(seen.values())

View file

@ -17,6 +17,18 @@ _SORT_SQL: dict[str, str] = {
),
}
# Сегмент рынка (#1188). listings_search_mv не несёт колонку listing_segment,
# поэтому фильтруем через подзапрос к базовой таблице listings, переиспользуя
# canon-предикат #1186: NULL = legacy вторичка до миграции 011.
_VTORICHKA_GUARD = "(listing_segment IS NULL OR listing_segment = 'vtorichka')"
_SEGMENT_SQL: dict[str, str | None] = {
"vtorichka": (f"listing_id IN (SELECT id FROM listings WHERE {_VTORICHKA_GUARD})"),
"novostroyki": (
"listing_id IN (SELECT id FROM listings WHERE listing_segment = 'novostroyki')"
),
"all": None,
}
def build_search_query(params: SearchParams) -> tuple[str, dict[str, object]]:
"""Возвращает (sql, args) для SELECT из listings_search_mv."""
@ -86,6 +98,10 @@ def build_search_query(params: SearchParams) -> tuple[str, dict[str, object]]:
if params.has_kadastr:
where.append("cadastral_number IS NOT NULL")
segment_clause = _SEGMENT_SQL[params.segment]
if segment_clause is not None:
where.append(segment_clause)
if params.sources:
where.append("sources && CAST(:sources AS text[])")
args["sources"] = params.sources

View file

@ -78,6 +78,25 @@ def test_minimal_parse() -> None:
assert result.views_today == 12
def test_sale_type_assignment_pereustupka() -> None:
"""«Способ продажи: переуступка» → sale_type='assignment' (ДДУ переуступка,
сигнал новостройки). До фикса map не содержал ключ None."""
html = """
<html><body>
<div data-marker="item-view/item-id"> 8043003287</div>
<span itemprop="price" content="9500000">9 500 000 </span>
<div data-marker="item-view/item-params">
<ul>
<li>Количество комнат: 2</li>
<li>Способ продажи: переуступка</li>
</ul>
</div>
</body></html>
"""
result = parse_detail_html(html, "https://www.avito.ru/test_8043003287")
assert result.sale_type == "assignment"
def test_metro_extraction() -> None:
result = parse_detail_html(MINIMAL_HTML, SOURCE_URL)
# "Чкаловская 11-15 мин пешком" → metro_stations

View file

@ -60,3 +60,37 @@ def test_lazy_card_date_comes_from_json_not_dom() -> None:
lots = AvitoScraper()._parse_html(html, "https://www.avito.ru/x")
lazy = next(lot for lot in lots if "8043936560" in (lot.source_url or ""))
assert lazy.listing_date == _expected(_TS_LAZY)
# ── listing_segment по DOM-маркеру item-development-name ───────────────────────
# Каждый avito-URL квартиры содержит /kvartiry/ (вкл. новостройки), поэтому
# сегмент определяется не по URL, а по наличию data-marker="item-development-name"
# (название ЖК/застройщика рендерится только у карточек новостроек).
def _serp_card(item_id: str, *, with_dev_name: bool) -> str:
dev = '<div data-marker="item-development-name">ЖК «Федерация»</div>' if with_dev_name else ""
return f"""
<div data-marker="item" data-item-id="{item_id}">
<a data-marker="item-title" href="/ekaterinburg/kvartiry/2-k_kvartira_{item_id}">
2-к. квартира, 50 м², 5/20 эт.</a>
<meta itemprop="price" content="9500000"/>
{dev}
</div>
"""
def test_segment_novostroyki_when_development_name_present() -> None:
"""Карточка с data-marker='item-development-name' → listing_segment='novostroyki'."""
html = f"<html><body>{_serp_card('8043003287', with_dev_name=True)}</body></html>"
lots = AvitoScraper()._parse_html(html, "https://www.avito.ru/ekaterinburg/kvartiry")
assert len(lots) == 1
assert lots[0].listing_segment == "novostroyki"
def test_segment_vtorichka_when_no_development_name() -> None:
"""Карточка без маркера ЖК → listing_segment='vtorichka' (даже с /kvartiry/ в URL)."""
html = f"<html><body>{_serp_card('8043936560', with_dev_name=False)}</body></html>"
lots = AvitoScraper()._parse_html(html, "https://www.avito.ru/ekaterinburg/kvartiry")
assert len(lots) == 1
assert lots[0].listing_segment == "vtorichka"

View file

@ -462,6 +462,40 @@ async def test_fetch_detail_kitchen_area_none_when_missing(monkeypatch):
assert result.kitchen_area_m2 is None
@pytest.mark.asyncio
async def test_fetch_detail_extracts_description(monkeypatch):
"""offer.description → DetailEnrichment.description (для persist в listings)."""
fake_state = {
"offerData": {
"offer": {
"cianId": 555555,
"description": "Продаётся уютная квартира с видом на парк.",
}
}
}
monkeypatch.setattr(
"app.services.scrapers.cian_detail.extract_state",
lambda html, mfe, key: fake_state if key == "defaultState" else None,
)
monkeypatch.setattr(
"app.services.scrapers.cian_detail.extract_all_states",
lambda html: {},
)
response = MagicMock()
response.status_code = 200
response.text = "<html></html>"
session = MagicMock()
session.get = AsyncMock(return_value=response)
session.close = AsyncMock()
result = await fetch_detail("https://ekb.cian.ru/sale/flat/555555/", session=session)
assert result is not None
assert result.description == "Продаётся уютная квартира с видом на парк."
# ── save_detail_enrichment — kitchen_area_m2 persisted ───────────────────────
@ -504,6 +538,36 @@ def test_save_detail_enrichment_kitchen_area_none_passthrough():
assert update_call_params["ka"] is None
# ── save_detail_enrichment — description + detail_enriched_at stamp ───────────
def test_save_detail_enrichment_writes_description():
"""save_detail_enrichment передаёт description в UPDATE listings."""
db = MagicMock()
db.execute.return_value.fetchone.return_value = None
enrichment = DetailEnrichment(description="Светлая квартира рядом с парком")
save_detail_enrichment(db, listing_id=1002, enrichment=enrichment)
update_call_params = db.execute.call_args_list[0][0][1]
assert update_call_params["descr"] == "Светлая квартира рядом с парком"
def test_save_detail_enrichment_stamps_detail_enriched_at():
"""UPDATE listings содержит detail_enriched_at = NOW() — иначе строка
переобогащается каждый run (detail_enriched_at IS NULL filter)."""
db = MagicMock()
db.execute.return_value.fetchone.return_value = None
enrichment = DetailEnrichment(kitchen_area_m2=8.3)
save_detail_enrichment(db, listing_id=1003, enrichment=enrichment)
update_sql = str(db.execute.call_args_list[0][0][0])
assert "detail_enriched_at = NOW()" in update_sql
# ── fetch_detail browser_fetcher param ───────────────────────────────────────

View file

@ -62,6 +62,42 @@ def test_build_query_sources_array():
assert args["sources"] == ["avito", "cian"]
def test_build_query_segment_default_vtorichka():
"""Без параметра segment — back-compat: canon-предикат вторички (#1186/#1188)."""
sql, _ = build_search_query(SearchParams())
assert SearchParams().segment == "vtorichka"
assert "listing_id IN (SELECT id FROM listings WHERE" in sql
assert "(listing_segment IS NULL OR listing_segment = 'vtorichka')" in sql
def test_build_query_segment_novostroyki():
"""segment=novostroyki — только первичка, без NULL-вторички."""
sql, _ = build_search_query(SearchParams(segment="novostroyki"))
assert "listing_segment = 'novostroyki'" in sql
# Не должен попасть canon-guard вторички.
assert "(listing_segment IS NULL OR listing_segment = 'vtorichka')" not in sql
def test_build_query_segment_all_no_filter():
"""segment=all — без фильтрации по сегменту."""
sql, _ = build_search_query(SearchParams(segment="all"))
assert "listing_segment" not in sql
def test_build_count_query_inherits_segment():
"""COUNT-запрос наследует segment-фильтр из build_search_query."""
sql, _ = build_count_query(SearchParams(segment="novostroyki"))
assert "listing_segment = 'novostroyki'" in sql
sql_v, _ = build_count_query(SearchParams())
assert "(listing_segment IS NULL OR listing_segment = 'vtorichka')" in sql_v
def test_segment_invalid_rejected():
"""Невалидное значение segment отклоняется валидацией."""
with pytest.raises(ValueError):
SearchParams(segment="kommercia")
def test_build_count_query_strips_limit():
sql, args = build_count_query(SearchParams(rooms=2, page=3))
assert "count(*)" in sql

View file

@ -182,6 +182,29 @@ def test_entity_to_lot_full_entity():
assert lot.raw_payload["kitchen_area_m2"] == pytest.approx(12.7)
def test_entity_to_lot_segment_vtorichka_default():
"""Default new_flat='NO' -> listing_segment='vtorichka'."""
lot = _entity_to_lot(_ENTITY_FULL)
assert lot is not None
assert lot.listing_segment == "vtorichka"
def test_entity_to_lot_segment_novostroyki_on_new_flat_yes():
"""new_flat='YES' -> listing_segment='novostroyki'."""
lot = _entity_to_lot(_ENTITY_FULL, new_flat="YES")
assert lot is not None
assert lot.listing_segment == "novostroyki"
def test_parse_gate_json_propagates_segment():
"""_parse_gate_json threads new_flat down to each lot's listing_segment."""
payload = _make_gate_payload([_ENTITY_FULL, _ENTITY_STUDIO])
lots_nb = _parse_gate_json(payload, new_flat="YES")
assert lots_nb and all(lot.listing_segment == "novostroyki" for lot in lots_nb)
lots_vt = _parse_gate_json(payload, new_flat="NO")
assert lots_vt and all(lot.listing_segment == "vtorichka" for lot in lots_vt)
def test_entity_to_lot_studio_rooms_zero():
lot = _entity_to_lot(_ENTITY_STUDIO)
assert lot is not None
@ -350,6 +373,21 @@ def test_build_url_combos_are_unique():
assert len(urls) == expected, f"URL collisions: {expected} combos but {len(urls)} unique"
def test_build_url_new_flat_default_no():
"""Default new_flat must keep newFlat=NO (vtorichka) — backward compat."""
s = YandexRealtyScraper()
url = s._build_url(page=1)
assert "newFlat=NO" in url
assert "newFlat=YES" not in url
def test_build_url_new_flat_yes():
"""new_flat='YES' must produce newFlat=YES (novostroyki segment)."""
s = YandexRealtyScraper()
url = s._build_url(page=1, new_flat="YES")
assert "newFlat=YES" in url
def test_default_city():
assert DEFAULT_CITY == "ekaterinburg"
@ -665,6 +703,81 @@ async def test_fetch_around_multi_room_dedup():
assert len(ids) == 3
@pytest.mark.asyncio
async def test_fetch_around_multi_room_runs_both_segments_with_shared_dedup():
"""segments=['NO','YES'] sweeps both vtorichka+novostroyki; dedup spans both passes.
Mocks _http_get and inspects requested URLs: both newFlat=NO and newFlat=YES
must be requested. The same offerId returned in both passes must be deduped to
one lot (single shared `seen` across passes), tagged by the FIRST pass's segment.
"""
s = YandexRealtyScraper()
requested_new_flat: list[str] = []
entity_shared = dict(
_ENTITY_FULL, offerId="shared_xseg", url="//realty.yandex.ru/offer/shared_xseg"
)
entity_vt = dict(_ENTITY_FULL, offerId="vt_only", url="//realty.yandex.ru/offer/vt_only")
entity_nb = dict(_ENTITY_FULL, offerId="nb_only", url="//realty.yandex.ru/offer/nb_only")
pager = {"page": 0, "pageSize": 20, "totalItems": 2, "totalPages": 1}
async def fake_http_get(url: str, **kwargs: object) -> _CurlResponse:
if "newFlat=YES" in url:
requested_new_flat.append("YES")
payload = _make_gate_payload([entity_shared, entity_nb], pager)
else:
requested_new_flat.append("NO")
payload = _make_gate_payload([entity_shared, entity_vt], pager)
return _CurlResponse(status_code=200, text=json.dumps(payload))
with patch.object(s, "_http_get", side_effect=fake_http_get):
with patch.object(s, "sleep_between_requests", new_callable=AsyncMock):
lots = await s.fetch_around_multi_room(
lat=0.0,
lon=0.0,
rooms_list=["1"],
price_ranges=[(None, 5_000_000)],
segments=["NO", "YES"],
max_pages=1,
)
assert "NO" in requested_new_flat
assert "YES" in requested_new_flat
by_id = {lot.source_id: lot for lot in lots}
assert set(by_id) == {"shared_xseg", "vt_only", "nb_only"}
# Shared offer deduped to ONE lot, tagged by the first pass (NO -> vtorichka).
assert by_id["shared_xseg"].listing_segment == "vtorichka"
assert by_id["vt_only"].listing_segment == "vtorichka"
assert by_id["nb_only"].listing_segment == "novostroyki"
@pytest.mark.asyncio
async def test_fetch_around_multi_room_default_segment_no_only():
"""Without segments arg, only newFlat=NO (vtorichka) is requested (legacy default)."""
s = YandexRealtyScraper()
requested_new_flat: list[str] = []
pager = {"page": 0, "pageSize": 20, "totalItems": 1, "totalPages": 1}
async def fake_http_get(url: str, **kwargs: object) -> _CurlResponse:
requested_new_flat.append("YES" if "newFlat=YES" in url else "NO")
body = json.dumps(_make_gate_payload([_ENTITY_FULL], pager))
return _CurlResponse(status_code=200, text=body)
with patch.object(s, "_http_get", side_effect=fake_http_get):
with patch.object(s, "sleep_between_requests", new_callable=AsyncMock):
await s.fetch_around_multi_room(
lat=0.0,
lon=0.0,
rooms_list=["1"],
price_ranges=[(None, 5_000_000)],
max_pages=1,
)
assert set(requested_new_flat) == {"NO"}
@pytest.mark.asyncio
async def test_fetch_around_multi_room_legacy_single_sweep():
"""Legacy mode (no rooms_list/price_ranges) uses single citywide sweep."""

View file

@ -121,9 +121,9 @@ export interface AvitoImvSummary {
// нет сделок. count..period_months — required (бэкенд не отдаёт частичный объект).
export interface DkpCorridor {
count: number; // число ДКП-сделок в выборке
low_ppm2: number; // min ₽/м² по сделкам
low_ppm2: number; // P10 ₽/м² по сделкам (робастный коридор)
median_ppm2: number; // медиана ₽/м²
high_ppm2: number; // max ₽/м²
high_ppm2: number; // P90 ₽/м² по сделкам (робастный коридор)
period_months: number; // окно поиска сделок
}