merge main + renumber 163→167 + guard riasurt harvest (#108 review)
This commit is contained in:
commit
0a72ef9491
36 changed files with 1251 additions and 205 deletions
|
|
@ -15,23 +15,22 @@ name: CI
|
|||
# FUTURE: добавить `postgis/postgis:16-3.4` service + гонять mv_layout — см.
|
||||
# .github/workflows/ci.yml как образец service-блока.
|
||||
on:
|
||||
# ТОЛЬКО pull_request — НЕТ push-триггера на feature-ветки (CI-шторм #1709).
|
||||
# WHY: раньше был и push: [feat/**,fix/**,...]. Каждый коммит в ветку с открытым
|
||||
# PR триггерил ДВА прогона на ОДИН SHA: push-событие (github.ref=refs/heads/<branch>)
|
||||
# и pull_request-событие (github.ref=refs/pull/<N>/merge). Разный github.ref →
|
||||
# разные concurrency-группы (см. ниже) → прогоны НЕ отменяют друг друга → 2× job
|
||||
# при и так дефицитных раннерах. В bot-пайплайне каждый коммит идёт через PR, так
|
||||
# что pull_request гейтит его полностью; push-прогон был чистым дублем.
|
||||
# Trade-off: push в feature-ветку БЕЗ открытого PR не получит CI до открытия PR
|
||||
# (бот открывает PR сразу после первого push) — приемлемо.
|
||||
pull_request:
|
||||
branches: [main]
|
||||
push:
|
||||
branches:
|
||||
# Mirror the bot-PR / feature-branch flow в .claude/rules/git-pr.md:
|
||||
# PR получает gate, прямые пуши в feature-ветки — тоже.
|
||||
- "feat/**"
|
||||
- "fix/**"
|
||||
- "refactor/**"
|
||||
- "chore/**"
|
||||
- "docs/**"
|
||||
- "perf/**"
|
||||
- "test/**"
|
||||
- "ci/**"
|
||||
- "hotfix/**"
|
||||
|
||||
concurrency:
|
||||
# Теперь, когда остался только pull_request, github.ref стабилен на весь PR
|
||||
# (refs/pull/<N>/merge) → новый push в ветку PR отменяет предыдущий незавершённый
|
||||
# прогон ЭТОГО PR (cancel-in-progress) вместо накопления параллельных.
|
||||
group: ci-${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
|
|
@ -89,6 +88,19 @@ jobs:
|
|||
curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
|
||||
|
||||
- name: Cache uv packages
|
||||
# Кросс-прогонный кэш скачанных/собранных wheel'ов (~/.cache/uv по умолч.).
|
||||
# `uv sync --frozen` без него каждый прогон тянет весь geo-стек заново —
|
||||
# доминирующая часть времени job (#1709). Ключ по uv.lock; continue-on-error
|
||||
# чтобы сбой cache-бэкенда раннера НИКОГДА не ронял gate.
|
||||
uses: actions/cache@v4
|
||||
continue-on-error: true
|
||||
with:
|
||||
path: ~/.cache/uv
|
||||
key: uv-${{ runner.os }}-${{ hashFiles('backend/uv.lock') }}
|
||||
restore-keys: |
|
||||
uv-${{ runner.os }}-
|
||||
|
||||
- name: Install system deps for geo + WeasyPrint
|
||||
# libpq/gdal/proj/geos — geo-стек (geopandas/shapely/pyproj).
|
||||
# libcairo2/libpango* — нативные либы WeasyPrint: с ними PDF-тесты
|
||||
|
|
@ -230,6 +242,16 @@ jobs:
|
|||
curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
|
||||
|
||||
- name: Cache uv packages
|
||||
# См. backend-tests: кросс-прогонный кэш ~/.cache/uv, тот же ключ по uv.lock.
|
||||
uses: actions/cache@v4
|
||||
continue-on-error: true
|
||||
with:
|
||||
path: ~/.cache/uv
|
||||
key: uv-${{ runner.os }}-${{ hashFiles('backend/uv.lock') }}
|
||||
restore-keys: |
|
||||
uv-${{ runner.os }}-
|
||||
|
||||
- name: Install system deps for geo + WeasyPrint
|
||||
# app.main транзитивно тянет geo/PDF-модули. На macOS-dev импорт схемы
|
||||
# проходит и без этих либ, но на ubuntu ставим как backend-tests
|
||||
|
|
@ -239,9 +261,12 @@ jobs:
|
|||
sudo apt-get install -y libpq-dev libgdal-dev libproj-dev libgeos-dev \
|
||||
libcairo2 libpango-1.0-0 libpangoft2-1.0-0
|
||||
|
||||
- name: Install backend deps (uv sync --frozen)
|
||||
- name: Install backend deps (uv sync --frozen --no-dev)
|
||||
working-directory: backend
|
||||
run: uv sync --frozen
|
||||
# --no-dev: этот job только дампит app.openapi() (нужен runtime app.main).
|
||||
# pytest/ruff/coverage не используются → не ставим dev-группу (быстрее).
|
||||
# Dockerfile тоже собирает с --no-dev → импорт app.main гарантированно ок.
|
||||
run: uv sync --frozen --no-dev
|
||||
|
||||
- name: Install frontend deps (npm ci)
|
||||
working-directory: frontend
|
||||
|
|
|
|||
|
|
@ -138,7 +138,7 @@ def leads_stats(
|
|||
WITH window_leads AS (
|
||||
SELECT *
|
||||
FROM prinzip_leads
|
||||
WHERE created_at >= NOW() - (:m || ' months')::interval
|
||||
WHERE created_at >= NOW() - make_interval(months => :m)
|
||||
)
|
||||
SELECT
|
||||
(SELECT COUNT(*) FROM prinzip_leads) AS leads_total,
|
||||
|
|
|
|||
|
|
@ -184,8 +184,7 @@ def quartirography(db: Session, source: str, region_id: int = 66) -> list[dict[s
|
|||
-- ('2025-07-01' расширял «recent»-окно каждую неделю по мере
|
||||
-- доливки ETL новых report_months → перекос в сторону всё
|
||||
-- более длинной истории). Тот же фикс, что #1203 и _BUCKET_SQL.
|
||||
AND period_start_date >= NOW()
|
||||
- (:months_window || ' months')::INTERVAL
|
||||
AND period_start_date >= NOW() - make_interval(months => :months_window)
|
||||
),
|
||||
bucketed AS (
|
||||
SELECT CASE
|
||||
|
|
@ -1169,7 +1168,7 @@ def prinzip_funnel_monthly(db: Session, months: int = 24) -> list[dict[str, Any]
|
|||
"""
|
||||
SELECT month, source, leads, engaged, converted, conv_pct
|
||||
FROM prinzip_funnel_monthly
|
||||
WHERE month >= (CURRENT_DATE - (:months || ' months')::interval)::date
|
||||
WHERE month >= (CURRENT_DATE - make_interval(months => :months))::date
|
||||
ORDER BY month DESC, leads DESC
|
||||
"""
|
||||
),
|
||||
|
|
@ -1203,7 +1202,7 @@ def prinzip_funnel_by_source(db: Session, months: int = 12) -> list[dict[str, An
|
|||
SUM(converted) AS converted,
|
||||
ROUND(100.0 * SUM(converted) / NULLIF(SUM(leads), 0), 2) AS conv_pct
|
||||
FROM prinzip_funnel_monthly
|
||||
WHERE month >= (CURRENT_DATE - (:months || ' months')::interval)::date
|
||||
WHERE month >= (CURRENT_DATE - make_interval(months => :months))::date
|
||||
GROUP BY source
|
||||
ORDER BY leads DESC
|
||||
"""
|
||||
|
|
@ -1364,8 +1363,7 @@ _BUCKET_SQL = text(
|
|||
AND deal_count > 0
|
||||
AND (area / deal_count) BETWEEN 15 AND 200
|
||||
AND price_per_sqm BETWEEN 30000 AND 1000000
|
||||
AND period_start_date >= NOW()
|
||||
- (:months_window || ' months')::INTERVAL
|
||||
AND period_start_date >= NOW() - make_interval(months => :months_window)
|
||||
),
|
||||
bucketed AS (
|
||||
SELECT CASE
|
||||
|
|
@ -1995,7 +1993,7 @@ def _elasticity_coef(
|
|||
{where_district}
|
||||
{where_class}
|
||||
AND crm.deals_total_avg_price_thousand_rub_per_m2 > 0
|
||||
AND crm.report_month >= NOW() - (:ew || ' months')::interval
|
||||
AND crm.report_month >= NOW() - make_interval(months => :ew)
|
||||
)
|
||||
SELECT
|
||||
regr_slope(y, x) AS slope,
|
||||
|
|
@ -2086,7 +2084,7 @@ def _elasticity_per_bucket_coef(
|
|||
{where_class}
|
||||
AND crm.deals_total_count > 0
|
||||
AND crm.deals_total_avg_price_thousand_rub_per_m2 > 0
|
||||
AND crm.report_month >= NOW() - (:ew || ' months')::interval
|
||||
AND crm.report_month >= NOW() - make_interval(months => :ew)
|
||||
)
|
||||
SELECT bucket,
|
||||
regr_slope(y, x) AS slope,
|
||||
|
|
|
|||
|
|
@ -667,7 +667,14 @@ class EmissRow:
|
|||
"""Одна готовая к upsert строка macro_indicator из ЕМИСС (source='emiss').
|
||||
|
||||
Отдельно от ``MacroRow`` (open-data, source='rosstat', yearly): ЕМИСС-ряды несут
|
||||
свою frequency (quarterly/monthly) и source — контракт upsert'а у них иной.
|
||||
свою frequency (quarterly/monthly), source и period_type — контракт upsert'а у
|
||||
них иной.
|
||||
|
||||
period_type: гранулярность под-периода ('year' | 'quarter' | 'month' | 'unknown').
|
||||
Берётся из _emiss_period_granularity(PERIOD). Необходима как часть PK
|
||||
macro_indicator (migration 163), чтобы годовой агрегат ('год' → 'year') и
|
||||
Q1 ('I квартал' → 'quarter') за один год не перезаписывали друг друга при
|
||||
ON CONFLICT DO UPDATE (оба дают obs_date=YYYY-01-01) (#1606).
|
||||
"""
|
||||
|
||||
indicator_type: str
|
||||
|
|
@ -677,6 +684,7 @@ class EmissRow:
|
|||
unit: str
|
||||
frequency: str
|
||||
comment: str
|
||||
period_type: str = "unknown"
|
||||
|
||||
|
||||
def _emiss_period_to_month(period: str) -> int | None:
|
||||
|
|
@ -817,6 +825,7 @@ def parse_emiss_sdmx(raw: bytes | str, spec: EmissIndicatorSpec) -> list[EmissRo
|
|||
unit=spec.unit,
|
||||
frequency=spec.frequency,
|
||||
comment=spec.comment,
|
||||
period_type=granularity,
|
||||
)
|
||||
|
||||
return [by_key[k] for k in sorted(by_key, key=lambda k: (k[0], k[1]))]
|
||||
|
|
|
|||
95
backend/app/services/site_finder/sales_tracker_mv_refresh.py
Normal file
95
backend/app/services/site_finder/sales_tracker_mv_refresh.py
Normal file
|
|
@ -0,0 +1,95 @@
|
|||
"""Refresh helper for the sales-tracker MVs (Issue #61).
|
||||
|
||||
Two independent materialized views built from the Объектив sales-tracker
|
||||
("шахматки") snapshots (objective_lots / objective_lots_history), created by
|
||||
data/sql/164_mv_sales_tracker_velocity_absorption.sql:
|
||||
|
||||
1. mv_sales_tracker_velocity_by_district — per (district, month) sold/total/
|
||||
avg-sold-price. Feeds the Site Finder Velocity Score (4th scoring criterion).
|
||||
2. mv_sales_tracker_absorption_curves — cumulative sold% as f(months from
|
||||
sales_start_date) per (rooms_int, area_bucket). Foundation for recommend_mix
|
||||
+ sellout forecast.
|
||||
|
||||
The two MVs do not depend on each other, so refresh order is irrelevant; both
|
||||
are refreshed in the same call.
|
||||
|
||||
Scheduled via Celery beat hardcoded entry in workers/beat_schedule.py
|
||||
('mv-sales-tracker-refresh-weekly', Mon 04:30 MSK).
|
||||
|
||||
Usage example (manual, via psql-connected shell or admin endpoint):
|
||||
from app.services.site_finder.sales_tracker_mv_refresh import refresh_sales_tracker_mvs
|
||||
|
||||
counts = refresh_sales_tracker_mvs(db)
|
||||
# logs: "mv_sales_tracker_velocity_by_district refreshed: 70 rows", etc.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.exc import DatabaseError
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_MV_NAMES: tuple[str, ...] = (
|
||||
"mv_sales_tracker_velocity_by_district",
|
||||
"mv_sales_tracker_absorption_curves",
|
||||
)
|
||||
|
||||
|
||||
def _refresh_mv(db: Session, mv_name: str, *, concurrently: bool) -> int:
|
||||
"""Run REFRESH MATERIALIZED VIEW [CONCURRENTLY] <mv_name>, return row count.
|
||||
|
||||
Falls back to non-concurrent on the known "cannot refresh concurrently"
|
||||
error (MV empty or no UNIQUE index — should not happen in prod since the
|
||||
migration creates the UNIQUE index and populates the MV, but provides a
|
||||
safe recovery path for first-run / post-recreation edge cases).
|
||||
"""
|
||||
try:
|
||||
if concurrently:
|
||||
db.execute(text(f"REFRESH MATERIALIZED VIEW CONCURRENTLY {mv_name}"))
|
||||
else:
|
||||
db.execute(text(f"REFRESH MATERIALIZED VIEW {mv_name}"))
|
||||
db.commit()
|
||||
except DatabaseError as e:
|
||||
# PostgreSQL emits "CONCURRENTLY cannot be used when the materialized
|
||||
# view ... is not populated" (matview.c, SQLSTATE 55000), surfaced by
|
||||
# psycopg3 as an InternalError (a DatabaseError sibling).
|
||||
if concurrently and "concurrently cannot be used" in str(e).lower():
|
||||
logger.warning(
|
||||
"%s: CONCURRENTLY failed (MV likely not populated), "
|
||||
"falling back to non-concurrent refresh",
|
||||
mv_name,
|
||||
)
|
||||
db.rollback()
|
||||
db.execute(text(f"REFRESH MATERIALIZED VIEW {mv_name}"))
|
||||
db.commit()
|
||||
else:
|
||||
raise
|
||||
|
||||
row = db.execute(text(f"SELECT COUNT(*) FROM {mv_name}")).first()
|
||||
count = int(row[0]) if row else 0
|
||||
logger.info("%s refreshed: %d rows", mv_name, count)
|
||||
return count
|
||||
|
||||
|
||||
def refresh_sales_tracker_mvs(db: Session, *, concurrently: bool = True) -> dict[str, int]:
|
||||
"""Refresh both sales-tracker MVs.
|
||||
|
||||
Args:
|
||||
db: SQLAlchemy Session (sync).
|
||||
concurrently: When True, uses REFRESH CONCURRENTLY (non-blocking —
|
||||
readers continue). Requires the per-MV UNIQUE indexes
|
||||
(mv_sales_tracker_velocity_by_district_pk,
|
||||
mv_sales_tracker_absorption_curves_pk) and the MVs to be already
|
||||
populated. Pass False only for first populate or after recreation.
|
||||
|
||||
Returns:
|
||||
Mapping mv_name -> row count after refresh (for observability).
|
||||
"""
|
||||
counts: dict[str, int] = {}
|
||||
for mv_name in _MV_NAMES:
|
||||
counts[mv_name] = _refresh_mv(db, mv_name, concurrently=concurrently)
|
||||
return counts
|
||||
|
|
@ -16,9 +16,13 @@ Foundation: domrf_kn_objects (lat/lon, comm_name, obj_class, region_cd),
|
|||
Fallback: rosreestr_deals (quarter_cad_number, deal_count, period_start_date).
|
||||
|
||||
Linkage: domrf_kn_objects.obj_id
|
||||
→ objective_complex_mapping.domrf_obj_id
|
||||
→ objective_complex_mapping.domrf_obj_id (gated: is_reviewed/manual/score≥0.85)
|
||||
→ objective_complex_mapping.objective_complex_name
|
||||
→ objective_corpus_room_month.project_name
|
||||
|
||||
OBJ-2 (#307): маппинги фильтруются по confidence (_MAPPING_CONFIDENCE_GATE) —
|
||||
unreviewed low-score auto-matches (#1331/#1333 backfill) исключаются как
|
||||
false-positive risk.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -40,6 +44,24 @@ _EKB_MEDIAN_FALLBACK_SQM_PER_MONTH: float = 4500.0
|
|||
# пытаемся rosreestr_fallback.
|
||||
_OBJECTIVE_COVERAGE_MIN_RATIO: float = 0.50
|
||||
|
||||
# OBJ-2 (#307): gate objective_complex_mapping by confidence перед использованием
|
||||
# в velocity. Fuzzy-trgm backfill (#1331/#1333) добавил ~115 auto-matched строк с
|
||||
# is_reviewed=false и низким match_score (вплоть до 0.50-0.625) — false-positive
|
||||
# risk, который раздувал/искажал velocity конкурентов.
|
||||
#
|
||||
# Принимаем mapping только если:
|
||||
# - is_reviewed = TRUE (человек подтвердил), ИЛИ
|
||||
# - match_method = 'manual' (ручной маппинг, score обычно NULL), ИЛИ
|
||||
# - match_score >= 0.85 (AUTO_ACCEPT_THRESHOLD — high-confidence auto,
|
||||
# vault: fuzzy_trgm 0.85+ надёжен для auto-use).
|
||||
#
|
||||
# Строгий gate только на is_reviewed=true дал бы 2 строки из 303 → обнулил бы
|
||||
# velocity-покрытие; 0.85-порог сохраняет 264/303 EKB-маппингов, отбрасывая 39
|
||||
# низкоуверенных. Совпадает с AUTO_ACCEPT_THRESHOLD из objective_backfill.py.
|
||||
_MAPPING_CONFIDENCE_GATE: str = (
|
||||
"(cm.is_reviewed = TRUE OR cm.match_method = 'manual' OR cm.match_score >= 0.85)"
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class VelocityResult:
|
||||
|
|
@ -195,7 +217,7 @@ def compute_velocity(
|
|||
sales_rows = (
|
||||
db.execute(
|
||||
text(
|
||||
"""
|
||||
f"""
|
||||
WITH all_competitors AS (
|
||||
SELECT unnest(CAST(:obj_ids AS int[])) AS obj_id
|
||||
),
|
||||
|
|
@ -204,6 +226,7 @@ def compute_velocity(
|
|||
cm.objective_complex_name
|
||||
FROM objective_complex_mapping cm
|
||||
WHERE cm.domrf_obj_id = ANY(:obj_ids)
|
||||
AND {_MAPPING_CONFIDENCE_GATE}
|
||||
)
|
||||
SELECT
|
||||
ac.obj_id,
|
||||
|
|
@ -311,12 +334,13 @@ def compute_velocity(
|
|||
bucket_rows = (
|
||||
db.execute(
|
||||
text(
|
||||
"""
|
||||
f"""
|
||||
WITH mapped AS (
|
||||
SELECT cm.domrf_obj_id AS obj_id,
|
||||
cm.objective_complex_name
|
||||
FROM objective_complex_mapping cm
|
||||
WHERE cm.domrf_obj_id = ANY(:obj_ids)
|
||||
AND {_MAPPING_CONFIDENCE_GATE}
|
||||
)
|
||||
SELECT
|
||||
m.obj_id,
|
||||
|
|
|
|||
|
|
@ -540,4 +540,21 @@ def build_beat_schedule() -> dict:
|
|||
"options": {"queue": "celery"},
|
||||
}
|
||||
|
||||
# Sales-tracker MVs (#61) — питают Site Finder Velocity Score (4-й критерий) +
|
||||
# recommend_mix / sellout-forecast. Оба MV (mv_sales_tracker_velocity_by_district,
|
||||
# mv_sales_tracker_absorption_curves) рефрешатся CONCURRENTLY (non-blocking, требуют
|
||||
# unique-индексы из миграции 161). Источник — objective_lots / objective_lots_history
|
||||
# (Объектив-шахматки), наполняются objective_sync (Mon 04:15 МСК по умолчанию).
|
||||
#
|
||||
# Понедельник 04:30 МСК (Celery conf.timezone=Europe/Moscow → crontab в МСК, #1233) —
|
||||
# ПОСЛЕ objective_sync (04:15), чтобы агрегаты считались по свежему снапшоту; в
|
||||
# окне до тяжёлого monday-кластера site_finder-рефрешей (ird 05:00, gknspecial 05:30,
|
||||
# supply-layers 06:00). Refresh лёгкий (~6с на 1.1M lots). Техническая infra-задача,
|
||||
# не в job_settings (как refresh-quarter-price-index / refresh-layout-velocity).
|
||||
schedule["mv-sales-tracker-refresh-weekly"] = {
|
||||
"task": "tasks.mv_sales_tracker_refresh.refresh_sales_tracker_mvs",
|
||||
"schedule": _parse_cron("30 4 * * mon"), # 04:30 MSK, понедельник
|
||||
"options": {"queue": "celery"},
|
||||
}
|
||||
|
||||
return schedule
|
||||
|
|
|
|||
|
|
@ -83,6 +83,7 @@ celery_app = Celery(
|
|||
"app.workers.tasks.developer_registry_refresh",
|
||||
"app.workers.tasks.refresh_layout_velocity",
|
||||
"app.workers.tasks.riasurt_sverdl_harvest",
|
||||
"app.workers.tasks.mv_sales_tracker_refresh",
|
||||
],
|
||||
)
|
||||
celery_app.conf.timezone = "Europe/Moscow"
|
||||
|
|
|
|||
|
|
@ -41,19 +41,21 @@ from app.workers.celery_app import celery_app
|
|||
logger = logging.getLogger(__name__)
|
||||
|
||||
# psycopg v3: CAST(:x AS type) — НИКОГДА :x::type (SQLAlchemy+psycopg3 роняет
|
||||
# синтаксис на ::). Контракт колонок совпадает с macro_indicator (migration 123):
|
||||
# (indicator_type, region, obs_date, value, source, frequency, unit, comment,
|
||||
# updated_at) PK (indicator_type, region, obs_date).
|
||||
# синтаксис на ::). Контракт колонок совпадает с macro_indicator (migration 163):
|
||||
# (indicator_type, region, obs_date, period_type, value, source, frequency, unit,
|
||||
# comment, updated_at). PK (indicator_type, region, obs_date, period_type).
|
||||
# CBR-ряды используют period_type='unknown' (литерал) — они не несут sub-period
|
||||
# granularity и различаются по obs_date.
|
||||
UPSERT_KEY_RATE_SQL = text(
|
||||
"""
|
||||
INSERT INTO macro_indicator (
|
||||
indicator_type, region, obs_date, value,
|
||||
indicator_type, region, obs_date, period_type, value,
|
||||
source, frequency, unit, comment
|
||||
) VALUES (
|
||||
'key_rate', 'rf', CAST(:d AS date), CAST(:v AS numeric),
|
||||
'key_rate', 'rf', CAST(:d AS date), 'unknown', CAST(:v AS numeric),
|
||||
'cbr', 'daily', '%', 'CBR key rate'
|
||||
)
|
||||
ON CONFLICT (indicator_type, region, obs_date) DO UPDATE SET
|
||||
ON CONFLICT (indicator_type, region, obs_date, period_type) DO UPDATE SET
|
||||
value = EXCLUDED.value,
|
||||
updated_at = now()
|
||||
"""
|
||||
|
|
@ -61,16 +63,17 @@ UPSERT_KEY_RATE_SQL = text(
|
|||
|
||||
# Инфляция «% г/г» (ИПЦ YoY): indicator_type='inflation_yoy', monthly, region='rf'.
|
||||
# obs_date уже нормализован к 1-му числу месяца парсером (parse_inflation_xlsx).
|
||||
# period_type='unknown' — месячный ряд без sub-period granularity.
|
||||
UPSERT_INFLATION_SQL = text(
|
||||
"""
|
||||
INSERT INTO macro_indicator (
|
||||
indicator_type, region, obs_date, value,
|
||||
indicator_type, region, obs_date, period_type, value,
|
||||
source, frequency, unit, comment
|
||||
) VALUES (
|
||||
'inflation_yoy', 'rf', CAST(:d AS date), CAST(:v AS numeric),
|
||||
'inflation_yoy', 'rf', CAST(:d AS date), 'unknown', CAST(:v AS numeric),
|
||||
'cbr', 'monthly', '%', 'CBR inflation YoY (ИПЦ, % г/г)'
|
||||
)
|
||||
ON CONFLICT (indicator_type, region, obs_date) DO UPDATE SET
|
||||
ON CONFLICT (indicator_type, region, obs_date, period_type) DO UPDATE SET
|
||||
value = EXCLUDED.value,
|
||||
updated_at = now()
|
||||
"""
|
||||
|
|
|
|||
52
backend/app/workers/tasks/mv_sales_tracker_refresh.py
Normal file
52
backend/app/workers/tasks/mv_sales_tracker_refresh.py
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
"""Celery task: refresh the sales-tracker MVs (Issue #61).
|
||||
|
||||
Scheduled via hardcoded beat entry in workers/beat_schedule.py:
|
||||
'mv-sales-tracker-refresh-weekly' — weekly on Monday at 04:30 MSK.
|
||||
|
||||
Refreshes (both CONCURRENTLY, non-blocking):
|
||||
- mv_sales_tracker_velocity_by_district (Site Finder Velocity Score, 4th criterion)
|
||||
- mv_sales_tracker_absorption_curves (recommend_mix + sellout forecast foundation)
|
||||
|
||||
Both MVs are built from the Объектив sales-tracker ("шахматки") snapshots
|
||||
(objective_lots / objective_lots_history). Source data refreshes via the
|
||||
objective_sync beat job, so a weekly MV refresh keeps the aggregates current.
|
||||
|
||||
MV-source migration: data/sql/164_mv_sales_tracker_velocity_absorption.sql.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from app.core.db import SessionLocal
|
||||
from app.services.site_finder.sales_tracker_mv_refresh import refresh_sales_tracker_mvs
|
||||
from app.workers.celery_app import celery_app
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@celery_app.task(
|
||||
bind=True,
|
||||
name="tasks.mv_sales_tracker_refresh.refresh_sales_tracker_mvs",
|
||||
max_retries=2,
|
||||
)
|
||||
def refresh_sales_tracker_mvs_task(self: Any) -> dict[str, Any]:
|
||||
"""REFRESH both sales-tracker MVs (#61).
|
||||
|
||||
Both MVs are refreshed CONCURRENTLY (non-blocking, require their UNIQUE
|
||||
indexes created by migration 161); the service falls back to non-concurrent
|
||||
if an MV is found unpopulated (first-run edge case).
|
||||
|
||||
Returns result dict for the Celery task result store / logging.
|
||||
"""
|
||||
db = SessionLocal()
|
||||
try:
|
||||
counts = refresh_sales_tracker_mvs(db, concurrently=True)
|
||||
logger.info("refresh_sales_tracker_mvs: completed, rows=%s", counts)
|
||||
return {"status": "ok", "rows": counts}
|
||||
except Exception as e:
|
||||
logger.exception("refresh_sales_tracker_mvs failed: %s", e)
|
||||
raise
|
||||
finally:
|
||||
db.close()
|
||||
|
|
@ -47,21 +47,23 @@ from app.workers.celery_app import celery_app
|
|||
logger = logging.getLogger(__name__)
|
||||
|
||||
# psycopg v3: CAST(:x AS type) — НИКОГДА :x::type (SQLAlchemy+psycopg3 роняет
|
||||
# синтаксис на ::). Контракт колонок совпадает с macro_indicator (migration 123):
|
||||
# (indicator_type, region, obs_date, value, source, frequency, unit, comment,
|
||||
# updated_at) PK (indicator_type, region, obs_date). source='rosstat',
|
||||
# frequency='yearly' (текущие ряды Росстата — годовые: obs_date = 1 января года).
|
||||
# синтаксис на ::). Контракт колонок совпадает с macro_indicator (migration 163):
|
||||
# (indicator_type, region, obs_date, period_type, value, source, frequency, unit,
|
||||
# comment, updated_at). PK (indicator_type, region, obs_date, period_type).
|
||||
# Не-ЕМИСС источники (open-data, СМР) используют period_type='unknown' (литерал) —
|
||||
# они различаются по obs_date без sub-period granularity.
|
||||
# source='rosstat', frequency='yearly' (текущие ряды Росстата — годовые).
|
||||
UPSERT_ROSSTAT_SQL = text(
|
||||
"""
|
||||
INSERT INTO macro_indicator (
|
||||
indicator_type, region, obs_date, value,
|
||||
indicator_type, region, obs_date, period_type, value,
|
||||
source, frequency, unit, comment
|
||||
) VALUES (
|
||||
CAST(:itype AS text), CAST(:region AS text), CAST(:d AS date),
|
||||
CAST(:v AS numeric),
|
||||
'unknown', CAST(:v AS numeric),
|
||||
'rosstat', 'yearly', CAST(:unit AS text), CAST(:comment AS text)
|
||||
)
|
||||
ON CONFLICT (indicator_type, region, obs_date) DO UPDATE SET
|
||||
ON CONFLICT (indicator_type, region, obs_date, period_type) DO UPDATE SET
|
||||
value = EXCLUDED.value,
|
||||
source = EXCLUDED.source,
|
||||
frequency = EXCLUDED.frequency,
|
||||
|
|
@ -71,20 +73,23 @@ UPSERT_ROSSTAT_SQL = text(
|
|||
"""
|
||||
)
|
||||
|
||||
# ЕМИСС-ряды (source='emiss'): frequency параметризована (quarterly для доходов,
|
||||
# monthly для ИПЦ когда добавится) — в отличие от open-data, где она фикс 'yearly'.
|
||||
# Тот же PK и ON CONFLICT-контракт. CAST(:x AS type) — НИКОГДА :x::type (psycopg v3).
|
||||
# ЕМИСС-ряды (source='emiss'): period_type параметризован — 'year' | 'quarter' |
|
||||
# 'month' (из _emiss_period_granularity). Это ключевое отличие от не-ЕМИСС источников:
|
||||
# период ЕМИСС-ряда несёт granularity, необходимую для разделения годового агрегата
|
||||
# ('год' → 'year') и Q1 ('I квартал' → 'quarter'), оба с obs_date=YYYY-01-01 (#1606).
|
||||
# frequency параметризована (quarterly для доходов, monthly для ИПЦ когда добавится).
|
||||
# CAST(:x AS type) — НИКОГДА :x::type (psycopg v3).
|
||||
UPSERT_EMISS_SQL = text(
|
||||
"""
|
||||
INSERT INTO macro_indicator (
|
||||
indicator_type, region, obs_date, value,
|
||||
indicator_type, region, obs_date, period_type, value,
|
||||
source, frequency, unit, comment
|
||||
) VALUES (
|
||||
CAST(:itype AS text), CAST(:region AS text), CAST(:d AS date),
|
||||
CAST(:v AS numeric),
|
||||
CAST(:period_type AS text), CAST(:v AS numeric),
|
||||
'emiss', CAST(:freq AS text), CAST(:unit AS text), CAST(:comment AS text)
|
||||
)
|
||||
ON CONFLICT (indicator_type, region, obs_date) DO UPDATE SET
|
||||
ON CONFLICT (indicator_type, region, obs_date, period_type) DO UPDATE SET
|
||||
value = EXCLUDED.value,
|
||||
source = EXCLUDED.source,
|
||||
frequency = EXCLUDED.frequency,
|
||||
|
|
@ -96,19 +101,19 @@ UPSERT_EMISS_SQL = text(
|
|||
|
||||
# Индекс цен на СМР — открытый xlsx Росстата: source='rosstat' (rosstat.gov.ru-файл),
|
||||
# НО frequency='monthly' (в отличие от open-data демографии — там фикс 'yearly'),
|
||||
# поэтому frequency параметризована. MacroRow не несёт frequency-поля, подставляем
|
||||
# литералом в bind. Тот же PK и ON CONFLICT-контракт. CAST(:x AS type) — psycopg v3.
|
||||
# поэтому frequency параметризована. MacroRow не несёт frequency/period_type-полей;
|
||||
# подставляем литералами. CAST(:x AS type) — psycopg v3.
|
||||
UPSERT_ROSSTAT_MONTHLY_SQL = text(
|
||||
"""
|
||||
INSERT INTO macro_indicator (
|
||||
indicator_type, region, obs_date, value,
|
||||
indicator_type, region, obs_date, period_type, value,
|
||||
source, frequency, unit, comment
|
||||
) VALUES (
|
||||
CAST(:itype AS text), CAST(:region AS text), CAST(:d AS date),
|
||||
CAST(:v AS numeric),
|
||||
'unknown', CAST(:v AS numeric),
|
||||
'rosstat', 'monthly', CAST(:unit AS text), CAST(:comment AS text)
|
||||
)
|
||||
ON CONFLICT (indicator_type, region, obs_date) DO UPDATE SET
|
||||
ON CONFLICT (indicator_type, region, obs_date, period_type) DO UPDATE SET
|
||||
value = EXCLUDED.value,
|
||||
source = EXCLUDED.source,
|
||||
frequency = EXCLUDED.frequency,
|
||||
|
|
@ -185,9 +190,15 @@ def _upsert_monthly_rows(db: Session, rows: list[MacroRow]) -> int:
|
|||
|
||||
|
||||
def _upsert_emiss_rows(db: Session, rows: list[EmissRow]) -> int:
|
||||
"""Апсертит EmissRow в macro_indicator (source='emiss', frequency per-row).
|
||||
"""Апсертит EmissRow в macro_indicator (source='emiss', frequency+period_type per-row).
|
||||
|
||||
SAVEPOINT per-row, чтобы один битый ряд не откатывал всю транзакцию. Возвращает
|
||||
число успешных upsert'ов."""
|
||||
число успешных upsert'ов.
|
||||
|
||||
period_type (из r.period_type) — часть нового PK (migration 163): позволяет
|
||||
годовому агрегату ('year') и Q1 ('quarter') за один год коexist в таблице без
|
||||
взаимной перезаписи (#1606).
|
||||
"""
|
||||
upserted = 0
|
||||
for r in rows:
|
||||
try:
|
||||
|
|
@ -198,6 +209,7 @@ def _upsert_emiss_rows(db: Session, rows: list[EmissRow]) -> int:
|
|||
"itype": r.indicator_type,
|
||||
"region": r.region,
|
||||
"d": r.obs_date,
|
||||
"period_type": r.period_type,
|
||||
"v": r.value,
|
||||
"freq": r.frequency,
|
||||
"unit": r.unit,
|
||||
|
|
@ -207,10 +219,11 @@ def _upsert_emiss_rows(db: Session, rows: list[EmissRow]) -> int:
|
|||
upserted += 1
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"upsert emiss %s/%s@%s=%s failed: %s",
|
||||
"upsert emiss %s/%s@%s[%s]=%s failed: %s",
|
||||
r.indicator_type,
|
||||
r.region,
|
||||
r.obs_date,
|
||||
r.period_type,
|
||||
r.value,
|
||||
e,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -108,19 +108,30 @@ def test_yearly_and_q1_both_survive_dedup() -> None:
|
|||
assert Decimal("42000") in values, "Q1-наблюдение потеряно"
|
||||
# obs_date у обоих одинаковый (это нормально — коллизия теперь на стороне DB-upsert)
|
||||
assert all(r.obs_date == date(2023, 1, 1) for r in rows)
|
||||
# period_type: годовой → 'year', Q1 → 'quarter' (часть PK в macro_indicator, migration 163)
|
||||
period_types = {r.period_type for r in rows}
|
||||
assert period_types == {
|
||||
"year",
|
||||
"quarter",
|
||||
}, f"ожидались period_type year+quarter, получили {period_types}"
|
||||
|
||||
|
||||
# ── real-fixture extraction (income id=57039) ─────────────────────────────────────
|
||||
|
||||
|
||||
def test_income_extracts_sverdlovsk_only() -> None:
|
||||
"""Из реальной SDMX-выгрузки извлекается ТОЛЬКО Свердл (ОКАТО 65) — РФ/ЦФО/Адыгея нет."""
|
||||
"""Из реальной SDMX-выгрузки извлекается ТОЛЬКО Свердл (ОКАТО 65) — РФ/ЦФО/Адыгея нет.
|
||||
Все строки квартальные → period_type='quarter' (часть PK macro_indicator, migration 163).
|
||||
"""
|
||||
rows = parse_emiss_sdmx(_load("emiss_income_57039.xml"), INCOME_PER_CAPITA_SPEC)
|
||||
assert rows, "ожидались строки по Свердл"
|
||||
assert {r.region for r in rows} == {"sverdl"}
|
||||
assert all(r.indicator_type == "income_per_capita" for r in rows)
|
||||
assert all(r.unit == "руб" for r in rows)
|
||||
assert all(r.frequency == "quarterly" for r in rows)
|
||||
assert all(
|
||||
r.period_type == "quarter" for r in rows
|
||||
), "квартальные строки должны иметь period_type='quarter'"
|
||||
|
||||
|
||||
def test_income_concrete_values_and_dates() -> None:
|
||||
|
|
|
|||
|
|
@ -77,7 +77,12 @@ def test_find_match_candidates_returns_candidates() -> None:
|
|||
|
||||
|
||||
def test_auto_apply_matches_dry_run_no_inserts() -> None:
|
||||
"""dry_run=True возвращает счётчики без обращения к БД (execute не вызывается)."""
|
||||
"""dry_run=True возвращает projected-счётчики без обращения к БД.
|
||||
|
||||
auto_accepted = сколько кандидатов БЫЛО БЫ принято (preview), а не 0 —
|
||||
смысл dry-run в admin-endpoint'е именно показать оператору объём перед
|
||||
реальным insert. execute/commit при этом не вызываются.
|
||||
"""
|
||||
mock_db = MagicMock()
|
||||
|
||||
candidates = [
|
||||
|
|
@ -88,8 +93,9 @@ def test_auto_apply_matches_dry_run_no_inserts() -> None:
|
|||
|
||||
result = auto_apply_matches(mock_db, candidates, dry_run=True)
|
||||
|
||||
assert result["auto_accepted"] == 0
|
||||
assert result["auto_accepted"] == 1 # projected: 1 кандидат >= AUTO_ACCEPT_THRESHOLD
|
||||
assert result["review_queue"] == 2
|
||||
assert result["skipped"] == 0
|
||||
mock_db.execute.assert_not_called()
|
||||
mock_db.commit.assert_not_called()
|
||||
|
||||
|
|
|
|||
|
|
@ -327,6 +327,39 @@ def test_by_room_bucket_empty_when_no_bucket_data():
|
|||
assert result.by_room_bucket == {}
|
||||
|
||||
|
||||
def test_mapping_confidence_gate_in_sales_query():
|
||||
"""OBJ-2 (#307): sales/bucket queries фильтруют objective_complex_mapping
|
||||
по confidence — unreviewed low-score auto-matches исключены.
|
||||
|
||||
Проверяем, что SQL, переданный в db.execute для маппинг-CTE, содержит
|
||||
предикат gate (is_reviewed / manual / match_score >= 0.85). Это гарантирует,
|
||||
что ~115 fuzzy-trgm строк с is_reviewed=false и score<0.85 не попадают в velocity.
|
||||
"""
|
||||
comp_rows = [_comp_row(1), _comp_row(2)]
|
||||
sales_rows = [
|
||||
_sales_row(1, total_sqm=4000.0, months=4),
|
||||
_sales_row(2, total_sqm=3000.0, months=4),
|
||||
]
|
||||
bucket_rows = [_bucket_row(1, "1", units_sold=20, sqm_sold=900.0)]
|
||||
db = _make_db(comp_rows=comp_rows, sales_rows=sales_rows, bucket_rows=bucket_rows)
|
||||
|
||||
with patch(
|
||||
"app.services.site_finder.velocity._get_ekb_median",
|
||||
return_value=_EKB_MEDIAN_FALLBACK_SQM_PER_MONTH,
|
||||
):
|
||||
compute_velocity(db, parcel_geom_wkt=_PARCEL_WKT)
|
||||
|
||||
# db.execute вызывается 3 раза: comp / sales / bucket. SQL берём из call_args.
|
||||
executed_sql = [str(call.args[0]) for call in db.execute.call_args_list]
|
||||
# comp-query НЕ трогает mapping; sales (idx 1) и bucket (idx 2) — должны иметь gate.
|
||||
mapping_queries = [s for s in executed_sql if "objective_complex_mapping" in s]
|
||||
assert len(mapping_queries) >= 2, "ожидаются sales + bucket запросы с mapping"
|
||||
for sql in mapping_queries:
|
||||
assert "cm.is_reviewed = TRUE" in sql
|
||||
assert "cm.match_method = 'manual'" in sql
|
||||
assert "cm.match_score >= 0.85" in sql
|
||||
|
||||
|
||||
def test_sample_competitors_include_by_room_bucket():
|
||||
"""sample_competitors каждого элемента содержит by_room_bucket."""
|
||||
comp_rows = [_comp_row(1), _comp_row(2)]
|
||||
|
|
|
|||
|
|
@ -48,14 +48,16 @@ def test_upsert_sql_contract() -> None:
|
|||
assert "'rf'" in sql
|
||||
assert "'cbr'" in sql
|
||||
assert "'daily'" in sql
|
||||
# ON CONFLICT по полному PK + обновление value/updated_at
|
||||
assert "ON CONFLICT (INDICATOR_TYPE, REGION, OBS_DATE) DO UPDATE" in upper
|
||||
# ON CONFLICT по новому PK включает period_type (migration 163)
|
||||
assert "ON CONFLICT (INDICATOR_TYPE, REGION, OBS_DATE, PERIOD_TYPE) DO UPDATE" in upper
|
||||
assert "VALUE = EXCLUDED.VALUE" in upper
|
||||
assert "UPDATED_AT = NOW()" in upper
|
||||
# psycopg v3: CAST(:x AS type), НИКОГДА :x::type
|
||||
assert "CAST(:D AS DATE)" in upper
|
||||
assert "CAST(:V AS NUMERIC)" in upper
|
||||
assert "::" not in sql
|
||||
# CBR-строки используют period_type='unknown' (литерал в SQL)
|
||||
assert "'unknown'" in sql
|
||||
|
||||
|
||||
def test_task_upserts_each_row_with_correct_params() -> None:
|
||||
|
|
@ -192,13 +194,14 @@ def test_upsert_inflation_sql_contract() -> None:
|
|||
assert "'rf'" in sql
|
||||
assert "'cbr'" in sql
|
||||
assert "'monthly'" in sql
|
||||
assert "ON CONFLICT (INDICATOR_TYPE, REGION, OBS_DATE) DO UPDATE" in upper
|
||||
assert "ON CONFLICT (INDICATOR_TYPE, REGION, OBS_DATE, PERIOD_TYPE) DO UPDATE" in upper
|
||||
assert "VALUE = EXCLUDED.VALUE" in upper
|
||||
assert "UPDATED_AT = NOW()" in upper
|
||||
# psycopg v3: CAST(:x AS type), НИКОГДА :x::type
|
||||
assert "CAST(:D AS DATE)" in upper
|
||||
assert "CAST(:V AS NUMERIC)" in upper
|
||||
assert "::" not in sql
|
||||
assert "'unknown'" in sql
|
||||
|
||||
|
||||
def test_task_registered_in_beat_weekly() -> None:
|
||||
|
|
|
|||
|
|
@ -128,9 +128,12 @@ def test_harvest_for_mo_skips_geomless_feature(monkeypatch: Any) -> None:
|
|||
|
||||
|
||||
def test_harvest_all_iterates_5_mo(monkeypatch: Any) -> None:
|
||||
"""harvest_all прогоняет все 5 МО агломерации."""
|
||||
"""harvest_all прогоняет все 5 МО агломерации (при включённом флаге)."""
|
||||
db = _FakeDB()
|
||||
_patch(monkeypatch, db)
|
||||
monkeypatch.setattr(
|
||||
riasurt_sverdl_harvest.settings, "enable_riasurt_harvest", True, raising=False
|
||||
)
|
||||
|
||||
res = riasurt_sverdl_harvest.harvest_all_riasurt_sverdl([845274])
|
||||
assert res["mo"] == 5
|
||||
|
|
@ -138,6 +141,21 @@ def test_harvest_all_iterates_5_mo(monkeypatch: Any) -> None:
|
|||
assert res["features"] == 5
|
||||
|
||||
|
||||
def test_harvest_all_disabled_by_default(monkeypatch: Any) -> None:
|
||||
"""Гейт #108: при выключенном флаге harvest_all возвращает early без WMS-вызовов."""
|
||||
db = _FakeDB()
|
||||
client = _patch(monkeypatch, db)
|
||||
monkeypatch.setattr(
|
||||
riasurt_sverdl_harvest.settings, "enable_riasurt_harvest", False, raising=False
|
||||
)
|
||||
|
||||
res = riasurt_sverdl_harvest.harvest_all_riasurt_sverdl([845274])
|
||||
assert res == {"mo": 0, "features": 0}
|
||||
# ни одного WMS-запроса, ни одной записи в БД
|
||||
assert client.bbox_calls == []
|
||||
assert db.executed == []
|
||||
|
||||
|
||||
def test_mo_bboxes_has_5_agglomeration_municipalities() -> None:
|
||||
"""MO_BBOXES содержит ровно 5 МО окраин агломерации."""
|
||||
assert set(riasurt_sverdl_harvest.MO_BBOXES) == {
|
||||
|
|
|
|||
|
|
@ -35,7 +35,7 @@ def _make_mock_db() -> tuple[MagicMock, list[dict[str, Any]]]:
|
|||
|
||||
def test_upsert_sql_contract() -> None:
|
||||
"""UPSERT_ROSSTAT_SQL содержит обязательные литералы контракта macro_indicator
|
||||
и НЕ использует :x::type (psycopg v3)."""
|
||||
и НЕ использует :x::type (psycopg v3). PK включает period_type (migration 163)."""
|
||||
from app.workers.tasks.rosstat_macro_sync import UPSERT_ROSSTAT_SQL
|
||||
|
||||
sql = str(UPSERT_ROSSTAT_SQL)
|
||||
|
|
@ -44,7 +44,7 @@ def test_upsert_sql_contract() -> None:
|
|||
assert "INTO MACRO_INDICATOR" in upper
|
||||
assert "'rosstat'" in sql
|
||||
assert "'yearly'" in sql
|
||||
assert "ON CONFLICT (INDICATOR_TYPE, REGION, OBS_DATE) DO UPDATE" in upper
|
||||
assert "ON CONFLICT (INDICATOR_TYPE, REGION, OBS_DATE, PERIOD_TYPE) DO UPDATE" in upper
|
||||
assert "VALUE = EXCLUDED.VALUE" in upper
|
||||
assert "UPDATED_AT = NOW()" in upper
|
||||
# psycopg v3: CAST(:x AS type), НИКОГДА :x::type
|
||||
|
|
@ -52,6 +52,8 @@ def test_upsert_sql_contract() -> None:
|
|||
assert "CAST(:D AS DATE)" in upper
|
||||
assert "CAST(:V AS NUMERIC)" in upper
|
||||
assert "::" not in sql
|
||||
# не-ЕМИСС строки используют period_type='unknown' (литерал в SQL)
|
||||
assert "'unknown'" in sql
|
||||
|
||||
|
||||
def test_task_upserts_each_row_with_correct_params() -> None:
|
||||
|
|
@ -155,15 +157,24 @@ def test_task_emiss_failure_does_not_block_opendata() -> None:
|
|||
|
||||
|
||||
def test_task_upserts_emiss_rows_with_correct_params() -> None:
|
||||
"""Каждая EmissRow → execute UPSERT_EMISS_SQL с itype/region/d/v/freq/unit/comment."""
|
||||
"""Каждая EmissRow → execute UPSERT_EMISS_SQL с itype/region/d/period_type/v/freq/unit/comment.
|
||||
|
||||
period_type передаётся из r.period_type (часть нового PK, migration 163).
|
||||
"""
|
||||
from app.services.scrapers.rosstat_emiss import EmissRow
|
||||
from app.workers.tasks import rosstat_macro_sync as task_mod
|
||||
|
||||
db, captured = _make_mock_db()
|
||||
emiss_rows = [
|
||||
EmissRow(
|
||||
"income_per_capita", "sverdl", date(2024, 1, 1),
|
||||
Decimal("54006"), "руб", "quarterly", "c",
|
||||
"income_per_capita",
|
||||
"sverdl",
|
||||
date(2024, 1, 1),
|
||||
Decimal("54006"),
|
||||
"руб",
|
||||
"quarterly",
|
||||
"c",
|
||||
period_type="quarter",
|
||||
),
|
||||
]
|
||||
|
||||
|
|
@ -181,6 +192,7 @@ def test_task_upserts_emiss_rows_with_correct_params() -> None:
|
|||
"itype": "income_per_capita",
|
||||
"region": "sverdl",
|
||||
"d": date(2024, 1, 1),
|
||||
"period_type": "quarter",
|
||||
"v": Decimal("54006"),
|
||||
"freq": "quarterly",
|
||||
"unit": "руб",
|
||||
|
|
@ -190,7 +202,8 @@ def test_task_upserts_emiss_rows_with_correct_params() -> None:
|
|||
|
||||
|
||||
def test_upsert_emiss_sql_contract() -> None:
|
||||
"""UPSERT_EMISS_SQL: source='emiss', frequency параметризован, ON CONFLICT, CAST not ::."""
|
||||
"""UPSERT_EMISS_SQL: source='emiss', frequency+period_type параметризованы,
|
||||
ON CONFLICT включает period_type (migration 163), CAST not ::."""
|
||||
from app.workers.tasks.rosstat_macro_sync import UPSERT_EMISS_SQL
|
||||
|
||||
sql = str(UPSERT_EMISS_SQL)
|
||||
|
|
@ -198,14 +211,16 @@ def test_upsert_emiss_sql_contract() -> None:
|
|||
assert "INTO MACRO_INDICATOR" in upper
|
||||
assert "'emiss'" in sql
|
||||
assert "CAST(:FREQ AS TEXT)" in upper
|
||||
assert "ON CONFLICT (INDICATOR_TYPE, REGION, OBS_DATE) DO UPDATE" in upper
|
||||
assert "CAST(:PERIOD_TYPE AS TEXT)" in upper
|
||||
assert "ON CONFLICT (INDICATOR_TYPE, REGION, OBS_DATE, PERIOD_TYPE) DO UPDATE" in upper
|
||||
assert "CAST(:ITYPE AS TEXT)" in upper
|
||||
assert "CAST(:V AS NUMERIC)" in upper
|
||||
assert "::" not in sql
|
||||
|
||||
|
||||
def test_upsert_rosstat_monthly_sql_contract() -> None:
|
||||
"""UPSERT_ROSSTAT_MONTHLY_SQL: source='rosstat', frequency='monthly', CAST not ::."""
|
||||
"""UPSERT_ROSSTAT_MONTHLY_SQL: source='rosstat', frequency='monthly', period_type='unknown',
|
||||
ON CONFLICT включает period_type (migration 163), CAST not ::."""
|
||||
from app.workers.tasks.rosstat_macro_sync import UPSERT_ROSSTAT_MONTHLY_SQL
|
||||
|
||||
sql = str(UPSERT_ROSSTAT_MONTHLY_SQL)
|
||||
|
|
@ -213,11 +228,12 @@ def test_upsert_rosstat_monthly_sql_contract() -> None:
|
|||
assert "INTO MACRO_INDICATOR" in upper
|
||||
assert "'rosstat'" in sql
|
||||
assert "'monthly'" in sql
|
||||
assert "ON CONFLICT (INDICATOR_TYPE, REGION, OBS_DATE) DO UPDATE" in upper
|
||||
assert "ON CONFLICT (INDICATOR_TYPE, REGION, OBS_DATE, PERIOD_TYPE) DO UPDATE" in upper
|
||||
assert "CAST(:ITYPE AS TEXT)" in upper
|
||||
assert "CAST(:D AS DATE)" in upper
|
||||
assert "CAST(:V AS NUMERIC)" in upper
|
||||
assert "::" not in sql
|
||||
assert "'unknown'" in sql
|
||||
|
||||
|
||||
def test_task_upserts_construction_rows_with_correct_params() -> None:
|
||||
|
|
@ -228,8 +244,12 @@ def test_task_upserts_construction_rows_with_correct_params() -> None:
|
|||
db, captured = _make_mock_db()
|
||||
constr_rows = [
|
||||
MacroRow(
|
||||
"construction_price_index", "rf", date(2025, 1, 1),
|
||||
Decimal("100.8"), "%", "smr",
|
||||
"construction_price_index",
|
||||
"rf",
|
||||
date(2025, 1, 1),
|
||||
Decimal("100.8"),
|
||||
"%",
|
||||
"smr",
|
||||
),
|
||||
]
|
||||
|
||||
|
|
|
|||
92
data/sql/163_emiss_pk_period_type.sql
Normal file
92
data/sql/163_emiss_pk_period_type.sql
Normal file
|
|
@ -0,0 +1,92 @@
|
|||
-- 163_emiss_pk_period_type.sql
|
||||
-- #1606 follow-up: расширить PK macro_indicator колонкой period_type, чтобы годовой
|
||||
-- агрегат ('год' → 'year') и Q1 ('I квартал' → 'quarter') за один год не перезаписывали
|
||||
-- друг друга при ON CONFLICT DO UPDATE (оба дают obs_date=YYYY-01-01).
|
||||
--
|
||||
-- Контекст:
|
||||
-- PR #1687 (#1606) исправил in-memory дедупликацию в parse_emiss_sdmx: ключ дедупа
|
||||
-- стал трёхкомпонентным (region, obs_date, granularity). Но PK таблицы остался
|
||||
-- (indicator_type, region, obs_date) — при апсерте в БД коллизия сохранялась.
|
||||
-- Эта миграция фиксирует PK на стороне БД.
|
||||
--
|
||||
-- Что делает:
|
||||
-- 1. Добавляет колонку period_type TEXT NOT NULL DEFAULT 'unknown'.
|
||||
-- Все существующие строки (CBR, rosstat open-data, domrf) получают 'unknown' —
|
||||
-- это корректно: у них нет sub-period granularity (они уже различаются по obs_date).
|
||||
-- 2. Бэкфилл: строки source='emiss' обновляем до 'quarter' (единственный
|
||||
-- активный ЕМИСС-индикатор — income_per_capita, он квартальный; годовых EMISS-
|
||||
-- строк ещё нет в таблице до этой миграции). Значение ДОЛЖНО совпадать с тем,
|
||||
-- что пишет UPSERT_EMISS_SQL (_emiss_period_granularity → 'quarter'/'year'/'month',
|
||||
-- короткая форма), иначе ON CONFLICT не сматчит → дубли при следующем скрейпе.
|
||||
-- 3. Пересоздаёт PRIMARY KEY: старый (indicator_type, region, obs_date) → новый
|
||||
-- (indicator_type, region, obs_date, period_type).
|
||||
-- 4. Пересоздаёт сопутствующий индекс idx_macro_type_region_date с тем же составом
|
||||
-- (расширяется автоматически через новый PK-индекс; отдельный индекс DROP+CREATE).
|
||||
--
|
||||
-- Зависимости: macro_indicator (migration 123_macro_indicator.sql).
|
||||
-- Нет VIEW/FK зависимостей от macro_indicator PK (проверено через pg_depend).
|
||||
-- Idempotent: ADD COLUMN IF NOT EXISTS + IF NOT EXISTS в CREATE INDEX. Повторный
|
||||
-- прогон безопасен: DROP CONSTRAINT / ADD CONSTRAINT могут упасть если PK уже
|
||||
-- переименован — обёрнуты в DO $$ BEGIN ... EXCEPTION WHEN ... END $$; блоки.
|
||||
--
|
||||
-- Deploy order: SQL migration применяется ПЕРВОЙ (auto-deploy.yml), затем деплоится
|
||||
-- backend со обновлёнными UPSERT_EMISS_SQL / UPSERT_ROSSTAT_SQL / UPSERT_KEY_RATE_SQL
|
||||
-- / UPSERT_INFLATION_SQL / UPSERT_ROSSTAT_MONTHLY_SQL (они несут новый конфликт-таргет
|
||||
-- + period_type в INSERT).
|
||||
|
||||
BEGIN;
|
||||
|
||||
-- ── 1. Добавить колонку period_type (DEFAULT 'unknown' для всех существующих строк) ──
|
||||
ALTER TABLE macro_indicator
|
||||
ADD COLUMN IF NOT EXISTS period_type TEXT NOT NULL DEFAULT 'unknown';
|
||||
|
||||
-- ── 2. Бэкфилл ЕМИСС-строк → 'quarter' ────────────────────────────────────────────
|
||||
-- Все существующие ЕМИСС-строки — квартальные (income_per_capita). Значение 'quarter'
|
||||
-- (короткая форма) обязано совпадать с UPSERT_EMISS_SQL, который параметризует
|
||||
-- period_type из _emiss_period_granularity ('quarter'/'year'/'month'). Иначе после
|
||||
-- деплоя следующий скрейп вставит 'quarter'-строки, не матчащие 'quarterly' на
|
||||
-- ON CONFLICT → дубли для каждого квартального наблюдения. Если появятся 'year'-строки
|
||||
-- (будущие индикаторы) — придут с корректным period_type из скрейпера.
|
||||
UPDATE macro_indicator
|
||||
SET period_type = 'quarter'
|
||||
WHERE source = 'emiss'
|
||||
AND period_type = 'unknown';
|
||||
|
||||
-- ── 3. Пересоздать PRIMARY KEY ────────────────────────────────────────────────────────
|
||||
-- PostgreSQL не поддерживает ALTER PRIMARY KEY напрямую — нужно DROP + ADD.
|
||||
-- Ловим случай когда старый PK уже дропнут (повторный прогон).
|
||||
DO $$
|
||||
BEGIN
|
||||
ALTER TABLE macro_indicator DROP CONSTRAINT macro_indicator_pkey;
|
||||
EXCEPTION
|
||||
WHEN undefined_object THEN
|
||||
NULL; -- PK уже удалён (повторный прогон)
|
||||
WHEN feature_not_supported THEN
|
||||
NULL;
|
||||
END;
|
||||
$$;
|
||||
|
||||
-- Новый PK включает period_type. Если constraint с этим именем уже существует —
|
||||
-- ADD CONSTRAINT бросит duplicate_object (42710, дубль имени constraint), ловим.
|
||||
DO $$
|
||||
BEGIN
|
||||
ALTER TABLE macro_indicator
|
||||
ADD CONSTRAINT macro_indicator_pkey
|
||||
PRIMARY KEY (indicator_type, region, obs_date, period_type);
|
||||
EXCEPTION
|
||||
WHEN duplicate_object THEN
|
||||
NULL; -- constraint с таким именем уже создан (повторный прогон)
|
||||
WHEN invalid_table_definition THEN
|
||||
NULL; -- уже существует тот же constraint
|
||||
END;
|
||||
$$;
|
||||
|
||||
-- ── 4. Пересоздать сопутствующий индекс ──────────────────────────────────────────────
|
||||
-- Старый индекс (без period_type в составе) уже покрыт новым PK-индексом по факту,
|
||||
-- но явно дропаем и пересоздаём с period_type для консистентности плана запросов.
|
||||
DROP INDEX IF EXISTS idx_macro_type_region_date;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_macro_type_region_date
|
||||
ON macro_indicator (indicator_type, region, obs_date DESC, period_type);
|
||||
|
||||
COMMIT;
|
||||
184
data/sql/164_mv_sales_tracker_velocity_absorption.sql
Normal file
184
data/sql/164_mv_sales_tracker_velocity_absorption.sql
Normal file
|
|
@ -0,0 +1,184 @@
|
|||
-- 164_mv_sales_tracker_velocity_absorption.sql
|
||||
-- Issue #61 — Velocity materialized views for Site Finder Velocity Score (4th scoring
|
||||
-- criterion) + recommend_mix smart unit-mix. Foundation for sellout forecast.
|
||||
--
|
||||
-- B2-1 data source ("шахматки" / sales-tracker): the Объектив scraper
|
||||
-- (backend/app/workers/tasks/scrape_objective.py) → tables:
|
||||
-- objective_lots — 1.12M rows, one row per tracked lot (current state),
|
||||
-- carries district / rooms_int / area_pd / sales_start_date /
|
||||
-- is_sold / registration_date / contract_date / price_per_m2_rub.
|
||||
-- objective_lots_history — 974k rows, daily-ish per-lot snapshots
|
||||
-- (snapshot_date, is_sold, status, prices).
|
||||
-- Snapshot history depth (as of 2026-06-17): 3 captures 2026-05-17 / 05-19 / 06-03 (spans
|
||||
-- >2 weeks, sold count moved 193188->194893 => measurable absorption). Cohort/absorption
|
||||
-- resolution improves automatically as the weekly scraper accumulates more snapshots.
|
||||
--
|
||||
-- -- MV 1: mv_sales_tracker_velocity_by_district --------------------------------------
|
||||
-- Grain: (district, sale_month). One row per district per month.
|
||||
-- Dedup: a lot appears in multiple snapshots within a month -> we keep that lot's LATEST
|
||||
-- snapshot within the month (DISTINCT ON lot, snapshot_date DESC) before
|
||||
-- aggregating, so total_count is lots-tracked-that-month (not snapshot rows).
|
||||
-- Metrics: total_count, sold_count, avg_sold_price_per_m2, avg_sold_price_total,
|
||||
-- sold_share (velocity proxy for SF Velocity Score).
|
||||
--
|
||||
-- -- MV 2: mv_sales_tracker_absorption_curves ----------------------------------------
|
||||
-- Grain: (rooms_int, area_bucket, months_since_start). Cumulative sold% as f(months
|
||||
-- from first_seen). "first_seen" = objective_lots.sales_start_date (true sales
|
||||
-- launch — richer/longer than the 3-snapshot window). Sold-month anchor =
|
||||
-- COALESCE(registration_date, contract_date). months_since_start clamped >= 0
|
||||
-- (712 noise rows have anchor < start). 99.98% of sold lots carry both dates.
|
||||
-- cohort_size = all lots in (rooms, area_bucket) cohort; cum_sold = sold lots
|
||||
-- whose months_since_start <= the row's bucket; cum_sold_pct = cum_sold/cohort.
|
||||
-- This is snapshot-sparsity-independent (driven by registration dates, not snapshots),
|
||||
-- so the curve is usable today and the foundation for sellout forecast.
|
||||
--
|
||||
-- REFRESH CONCURRENTLY: both MVs get a UNIQUE index on their full grain immediately after
|
||||
-- creation (on empty MV -> instant), enabling non-blocking weekly REFRESH CONCURRENTLY.
|
||||
-- Scheduled via Celery beat `mv-sales-tracker-refresh-weekly` (Mon 04:30 MSK) ->
|
||||
-- task app.workers.tasks.mv_sales_tracker_refresh.refresh_sales_tracker_mvs.
|
||||
--
|
||||
-- Deploy: auto-applied by deploy.yml via _schema_migrations tracking (one-shot, NN order).
|
||||
-- Dependencies on existing objects: objective_lots, objective_lots_history (read-only).
|
||||
-- No views depend on these MVs at creation time.
|
||||
--
|
||||
-- WARN: re-apply (DR / lost _schema_migrations / dev local) DROP ... CASCADE снесёт MV +
|
||||
-- зависимости. После re-apply ПЕРВЫЙ refresh = non-concurrent (CONCURRENTLY падает
|
||||
-- на пустой/не-populated MV). _schema_migrations нормально предотвращает re-apply.
|
||||
|
||||
BEGIN;
|
||||
|
||||
-- ====================================================================================
|
||||
-- MV 1: velocity by district x month
|
||||
-- ====================================================================================
|
||||
DROP MATERIALIZED VIEW IF EXISTS mv_sales_tracker_velocity_by_district CASCADE;
|
||||
|
||||
CREATE MATERIALIZED VIEW mv_sales_tracker_velocity_by_district AS
|
||||
WITH lot_month AS (
|
||||
-- One row per (lot, month): the lot's latest snapshot within that month.
|
||||
SELECT DISTINCT ON (h.objective_lot_id, date_trunc('month', h.snapshot_date))
|
||||
l.district AS district,
|
||||
date_trunc('month', h.snapshot_date)::date AS sale_month,
|
||||
h.objective_lot_id,
|
||||
h.is_sold,
|
||||
h.price_per_m2_rub,
|
||||
h.price_calculated_total_rub
|
||||
FROM objective_lots_history h
|
||||
JOIN objective_lots l ON l.objective_lot_id = h.objective_lot_id
|
||||
WHERE l.district IS NOT NULL
|
||||
ORDER BY h.objective_lot_id,
|
||||
date_trunc('month', h.snapshot_date),
|
||||
h.snapshot_date DESC
|
||||
)
|
||||
SELECT
|
||||
district,
|
||||
sale_month,
|
||||
count(*)::int AS total_count,
|
||||
count(*) FILTER (WHERE is_sold)::int AS sold_count,
|
||||
round(
|
||||
count(*) FILTER (WHERE is_sold)::numeric
|
||||
/ NULLIF(count(*), 0), 4
|
||||
) AS sold_share,
|
||||
round(avg(price_per_m2_rub) FILTER (WHERE is_sold), 2) AS avg_sold_price_per_m2,
|
||||
round(avg(price_calculated_total_rub) FILTER (WHERE is_sold), 2) AS avg_sold_price_total
|
||||
FROM lot_month
|
||||
GROUP BY district, sale_month
|
||||
WITH NO DATA;
|
||||
|
||||
-- UNIQUE index on full grain -> enables REFRESH CONCURRENTLY (created on empty MV = instant)
|
||||
CREATE UNIQUE INDEX mv_sales_tracker_velocity_by_district_pk
|
||||
ON mv_sales_tracker_velocity_by_district (district, sale_month);
|
||||
|
||||
CREATE INDEX mv_sales_tracker_velocity_district_idx
|
||||
ON mv_sales_tracker_velocity_by_district (district);
|
||||
|
||||
REFRESH MATERIALIZED VIEW mv_sales_tracker_velocity_by_district;
|
||||
|
||||
COMMENT ON MATERIALIZED VIEW mv_sales_tracker_velocity_by_district IS
|
||||
'Issue #61. Per (district, month) sold/total/avg-sold-price from objective_lots_history '
|
||||
'snapshots (Obektiv shahmatka), deduped to latest snapshot per lot per month. '
|
||||
'Feeds Site Finder Velocity Score. Refresh weekly CONCURRENTLY.';
|
||||
|
||||
-- ====================================================================================
|
||||
-- MV 2: absorption curves by room_count x area_bucket x months-from-first-seen
|
||||
-- ====================================================================================
|
||||
DROP MATERIALIZED VIEW IF EXISTS mv_sales_tracker_absorption_curves CASCADE;
|
||||
|
||||
CREATE MATERIALIZED VIEW mv_sales_tracker_absorption_curves AS
|
||||
WITH base AS (
|
||||
-- One row per lot. area_bucket from area_pd; months_since_start = whole months between
|
||||
-- sales_start_date and the sold anchor (reg/contract). Unsold lots have NULL anchor.
|
||||
SELECT
|
||||
l.rooms_int,
|
||||
CASE
|
||||
WHEN l.area_pd < 30 THEN '<30'
|
||||
WHEN l.area_pd < 45 THEN '30-45'
|
||||
WHEN l.area_pd < 60 THEN '45-60'
|
||||
WHEN l.area_pd < 80 THEN '60-80'
|
||||
ELSE '80+'
|
||||
END AS area_bucket,
|
||||
l.is_sold,
|
||||
CASE
|
||||
WHEN l.is_sold
|
||||
AND l.sales_start_date IS NOT NULL
|
||||
AND COALESCE(l.registration_date, l.contract_date) IS NOT NULL
|
||||
THEN GREATEST(
|
||||
0,
|
||||
(date_part('year', age(COALESCE(l.registration_date, l.contract_date),
|
||||
l.sales_start_date)) * 12
|
||||
+ date_part('month', age(COALESCE(l.registration_date, l.contract_date),
|
||||
l.sales_start_date)))::int
|
||||
)
|
||||
END AS months_since_start
|
||||
FROM objective_lots l
|
||||
WHERE l.rooms_int IS NOT NULL
|
||||
AND l.area_pd IS NOT NULL
|
||||
AND l.sales_start_date IS NOT NULL
|
||||
),
|
||||
cohort AS (
|
||||
SELECT rooms_int, area_bucket, count(*)::int AS cohort_size
|
||||
FROM base
|
||||
GROUP BY rooms_int, area_bucket
|
||||
),
|
||||
sold_at_month AS (
|
||||
SELECT rooms_int, area_bucket, months_since_start, count(*)::int AS sold_in_month
|
||||
FROM base
|
||||
WHERE is_sold AND months_since_start IS NOT NULL
|
||||
GROUP BY rooms_int, area_bucket, months_since_start
|
||||
)
|
||||
SELECT
|
||||
s.rooms_int,
|
||||
s.area_bucket,
|
||||
s.months_since_start,
|
||||
c.cohort_size,
|
||||
-- cumulative sold up to and including this month-offset (per cohort)
|
||||
SUM(s.sold_in_month) OVER (
|
||||
PARTITION BY s.rooms_int, s.area_bucket
|
||||
ORDER BY s.months_since_start
|
||||
ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
|
||||
)::int AS cum_sold,
|
||||
round(
|
||||
SUM(s.sold_in_month) OVER (
|
||||
PARTITION BY s.rooms_int, s.area_bucket
|
||||
ORDER BY s.months_since_start
|
||||
ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
|
||||
)::numeric / NULLIF(c.cohort_size, 0), 4
|
||||
) AS cum_sold_pct
|
||||
FROM sold_at_month s
|
||||
JOIN cohort c ON c.rooms_int = s.rooms_int AND c.area_bucket = s.area_bucket
|
||||
WITH NO DATA;
|
||||
|
||||
-- UNIQUE index on full grain -> enables REFRESH CONCURRENTLY
|
||||
CREATE UNIQUE INDEX mv_sales_tracker_absorption_curves_pk
|
||||
ON mv_sales_tracker_absorption_curves (rooms_int, area_bucket, months_since_start);
|
||||
|
||||
CREATE INDEX mv_sales_tracker_absorption_cohort_idx
|
||||
ON mv_sales_tracker_absorption_curves (rooms_int, area_bucket);
|
||||
|
||||
REFRESH MATERIALIZED VIEW mv_sales_tracker_absorption_curves;
|
||||
|
||||
COMMENT ON MATERIALIZED VIEW mv_sales_tracker_absorption_curves IS
|
||||
'Issue #61. Cumulative sold-pct as f(months from sales_start_date) per (rooms_int, '
|
||||
'area_bucket). Anchor = COALESCE(registration_date, contract_date) from objective_lots. '
|
||||
'Foundation for recommend_mix + sellout forecast. Refresh weekly CONCURRENTLY.';
|
||||
|
||||
COMMIT;
|
||||
93
data/sql/165_velocity_mapping_reviewed_gate.sql
Normal file
93
data/sql/165_velocity_mapping_reviewed_gate.sql
Normal file
|
|
@ -0,0 +1,93 @@
|
|||
-- 165_velocity_mapping_reviewed_gate.sql
|
||||
-- #307 OBJ-2 — gate objective_complex_mapping by confidence в mv_layout_velocity.
|
||||
--
|
||||
-- Контекст: fuzzy-trgm backfill (#1331/#1333, миграции 155/156/116/117/150) добавил
|
||||
-- ~115+ auto-matched строк в objective_complex_mapping с is_reviewed=false и низким
|
||||
-- match_score (вплоть до 0.50-0.625). Они с высокой вероятностью false-positive
|
||||
-- (Objective project name ≠ тот же ЖК что domrf_obj_id) и искажали velocity-агрегаты
|
||||
-- Site Finder, попадая в MV наравне с проверенными маппингами.
|
||||
--
|
||||
-- Fix: redefine MV с confidence-gate в JOIN — принимаем маппинг только если
|
||||
-- is_reviewed = TRUE (человек подтвердил), ИЛИ
|
||||
-- match_method = 'manual' (ручной маппинг, score обычно NULL), ИЛИ
|
||||
-- match_score >= 0.85 (AUTO_ACCEPT_THRESHOLD из objective_backfill.py —
|
||||
-- high-confidence auto, fuzzy_trgm 0.85+ надёжен).
|
||||
--
|
||||
-- Строгий gate только на is_reviewed=true дал бы 2 строки из 303 EKB-маппингов →
|
||||
-- обнулил бы velocity. Порог 0.85 сохраняет ~264/303, отбрасывая ~39 низкоуверенных.
|
||||
-- Тот же предикат применён в runtime-пути backend/app/services/site_finder/velocity.py.
|
||||
--
|
||||
-- Сохраняет weighted-average формулу из #21 (100_fix_mv_layout_velocity_weighted_avg.sql) —
|
||||
-- меняется ТОЛЬКО WHERE/JOIN-условие на objective_complex_mapping.
|
||||
--
|
||||
-- Dependencies: mv_layout_velocity не имеет зависимых VIEW/MV (проверено 2026-05-17/100_fix).
|
||||
-- Idempotency: DROP IF EXISTS → безопасен при повторном запуске.
|
||||
-- Deploy order: SQL migration → app deploy. Auto-applied через _schema_migrations.
|
||||
-- ВАЖНО: DROP + CREATE + REFRESH (non-concurrent, блокировка ~30 с на ~19 738 source rows).
|
||||
|
||||
BEGIN;
|
||||
|
||||
DROP MATERIALIZED VIEW IF EXISTS mv_layout_velocity CASCADE;
|
||||
|
||||
CREATE MATERIALIZED VIEW mv_layout_velocity AS
|
||||
WITH last24mo AS (
|
||||
SELECT
|
||||
ocm.project_name,
|
||||
CASE
|
||||
WHEN ocm.room_bucket = 'студия' THEN 'studio'
|
||||
ELSE ocm.room_bucket
|
||||
END AS room_bucket,
|
||||
ocm.deals_total_count,
|
||||
ocm.deals_total_avg_area_m2,
|
||||
ocm.deals_total_avg_price_thousand_rub_per_m2,
|
||||
ocm.deals_total_vol_m2,
|
||||
ocm.report_month
|
||||
FROM objective_corpus_room_month ocm
|
||||
WHERE ocm.report_month >= (NOW() - INTERVAL '24 months')::date
|
||||
)
|
||||
SELECT
|
||||
cm.domrf_obj_id AS obj_id,
|
||||
l.room_bucket,
|
||||
SUM(l.deals_total_count)::int AS total_deals_24mo,
|
||||
|
||||
-- weighted average (#21) — нули из месяцев без сделок не тянут вниз.
|
||||
(SUM(l.deals_total_avg_area_m2 * l.deals_total_count)
|
||||
/ NULLIF(SUM(l.deals_total_count), 0))::numeric(10, 2) AS avg_area_m2,
|
||||
|
||||
(SUM(l.deals_total_avg_price_thousand_rub_per_m2 * l.deals_total_count)
|
||||
/ NULLIF(SUM(l.deals_total_count), 0))::numeric(12, 2) AS avg_price_thousand_rub_per_m2,
|
||||
|
||||
SUM(l.deals_total_vol_m2)::numeric(12, 2) AS total_vol_m2,
|
||||
MIN(l.report_month) AS window_start,
|
||||
MAX(l.report_month) AS window_end,
|
||||
COUNT(DISTINCT l.report_month)::int AS months_with_data
|
||||
FROM last24mo l
|
||||
JOIN objective_complex_mapping cm
|
||||
ON cm.objective_complex_name = l.project_name
|
||||
WHERE l.room_bucket IS NOT NULL
|
||||
AND cm.domrf_obj_id IS NOT NULL
|
||||
AND cm.objective_group = 'Екатеринбург'
|
||||
-- #307 OBJ-2: confidence-gate — исключаем unreviewed low-score auto-matches.
|
||||
AND (cm.is_reviewed = TRUE OR cm.match_method = 'manual' OR cm.match_score >= 0.85)
|
||||
GROUP BY cm.domrf_obj_id, l.room_bucket
|
||||
WITH NO DATA;
|
||||
|
||||
-- UNIQUE index: required for REFRESH CONCURRENTLY (periodic via layout_velocity_refresh.py).
|
||||
CREATE UNIQUE INDEX mv_layout_velocity_pk
|
||||
ON mv_layout_velocity (obj_id, room_bucket);
|
||||
|
||||
-- Lookup index for /best-layouts endpoint queries by obj_id.
|
||||
CREATE INDEX mv_layout_velocity_obj_idx
|
||||
ON mv_layout_velocity (obj_id);
|
||||
|
||||
-- Initial populate (non-concurrent — MV just created, CONCURRENTLY requires populated MV).
|
||||
REFRESH MATERIALIZED VIEW mv_layout_velocity;
|
||||
|
||||
COMMENT ON MATERIALIZED VIEW mv_layout_velocity IS
|
||||
'Per-(obj_id, room_bucket) deals aggregation за last 24 months. '
|
||||
'WEIGHTED average площади и цены (SF Bug #21). '
|
||||
'Confidence-gated mapping: is_reviewed/manual/score>=0.85 (#307 OBJ-2). '
|
||||
'Source: objective_corpus_room_month × objective_complex_mapping (EKB only). '
|
||||
'Refresh via layout_velocity_refresh.py (CONCURRENTLY после initial populate).';
|
||||
|
||||
COMMIT;
|
||||
|
|
@ -4168,6 +4168,8 @@ export interface components {
|
|||
distance_m: number;
|
||||
/** Weight */
|
||||
weight: number;
|
||||
/** Score Contribution */
|
||||
score_contribution: number;
|
||||
/** Address */
|
||||
address: string | null;
|
||||
};
|
||||
|
|
@ -4177,6 +4179,8 @@ export interface components {
|
|||
cad_num: string;
|
||||
/** Radius M */
|
||||
radius_m: number;
|
||||
/** Poi Weighted Score */
|
||||
poi_weighted_score: number;
|
||||
/** Top Poi */
|
||||
top_poi: components["schemas"]["PoiScoreItem"][];
|
||||
};
|
||||
|
|
|
|||
|
|
@ -6,9 +6,11 @@ from typing import Literal
|
|||
|
||||
from pydantic import BaseModel, Field, model_validator
|
||||
|
||||
SortKey = Literal[
|
||||
"price_asc", "price_desc", "area_desc", "area_asc", "date_desc", "dist_asc"
|
||||
]
|
||||
SortKey = Literal["price_asc", "price_desc", "area_desc", "area_asc", "date_desc", "dist_asc"]
|
||||
|
||||
# Сегмент рынка (#1188, поверх canon-предиката #1186).
|
||||
# NULL listing_segment (legacy вторичка до миграции 011) трактуется как 'vtorichka'.
|
||||
SegmentKey = Literal["vtorichka", "novostroyki", "all"]
|
||||
|
||||
|
||||
class SearchParams(BaseModel):
|
||||
|
|
@ -49,6 +51,19 @@ class SearchParams(BaseModel):
|
|||
address_query: str | None = Field(default=None, max_length=200)
|
||||
description_query: str | None = Field(default=None, max_length=200)
|
||||
|
||||
# --- Market segment (#1188) ---
|
||||
segment: SegmentKey = Field(
|
||||
default="vtorichka",
|
||||
description=(
|
||||
"Сегмент рынка для фильтрации listings. "
|
||||
"`vtorichka` (по умолчанию, back-compat) — вторичка; "
|
||||
"строки с listing_segment=NULL (legacy до миграции 011) "
|
||||
"считаются вторичкой согласно canon-предикату #1186. "
|
||||
"`novostroyki` — только первичка. "
|
||||
"`all` — без фильтра по сегменту."
|
||||
),
|
||||
)
|
||||
|
||||
# --- Sort + pagination ---
|
||||
sort: SortKey = "date_desc"
|
||||
page: int = Field(default=1, ge=1, le=1000)
|
||||
|
|
|
|||
|
|
@ -106,9 +106,9 @@ class DkpCorridor(BaseModel):
|
|||
"""
|
||||
|
||||
count: int # число ДКП-сделок в выборке
|
||||
low_ppm2: int # min ₽/м² по сделкам (P10-ish — берём минимум)
|
||||
low_ppm2: int # P10 ₽/м² по сделкам (робастный коридор)
|
||||
median_ppm2: int # медиана ₽/м²
|
||||
high_ppm2: int # max ₽/м²
|
||||
high_ppm2: int # P90 ₽/м² по сделкам (робастный коридор)
|
||||
period_months: int # окно поиска сделок
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -321,6 +321,7 @@ async def trigger_yandex_city_sweep_run(db: Session, schedule_row: dict[str, Any
|
|||
request_delay_sec=float(params.get("request_delay_sec", 9.0)),
|
||||
radius_m=int(params.get("radius_m", 1500)),
|
||||
enrich_address=bool(params.get("enrich_address", True)),
|
||||
segments=params.get("segments"),
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("scheduler: run_yandex_city_sweep crashed run_id=%d", run_id)
|
||||
|
|
|
|||
|
|
@ -1033,6 +1033,7 @@ async def run_yandex_city_sweep(
|
|||
enrich_address: bool = True,
|
||||
rooms_list: list[str] | None = None,
|
||||
price_ranges: list[tuple[int | None, int | None]] | None = None,
|
||||
segments: list[str] | None = None,
|
||||
) -> YandexCitySweepCounters:
|
||||
"""Yandex.Недвижимость city sweep: rooms × price combos от центра ЕКБ → save → address-enrich.
|
||||
|
||||
|
|
@ -1078,6 +1079,11 @@ async def run_yandex_city_sweep(
|
|||
|
||||
_rooms_list = rooms_list or list(ROOM_PATH.keys())
|
||||
_price_ranges = price_ranges or DEFAULT_PRICE_RANGES
|
||||
# newFlat-сегменты: оба прохода (vtorichka + novostroyki) по одним combos.
|
||||
# Дефолт ["NO", "YES"] — оба; dedup по source_id span'ит оба прохода (один seen
|
||||
# внутри fetch_around_multi_room). Counters (lots_fetched/inserted/updated)
|
||||
# агрегируют оба прохода через len(anchor_lots).
|
||||
_segments = segments or ["NO", "YES"]
|
||||
|
||||
counters = YandexCitySweepCounters(anchors_total=len(_anchors))
|
||||
inter_anchor_delay = request_delay_sec if request_delay_sec is not None else 7.0
|
||||
|
|
@ -1086,16 +1092,21 @@ async def run_yandex_city_sweep(
|
|||
consecutive_failures = 0
|
||||
|
||||
# Вычисляем watchdog-таймаут для combos-режима (центр, anchors=None).
|
||||
# В combos-режиме один "anchor" выполняет num_combos × max_pages fetch'ей —
|
||||
# каждый занимает примерно _resolved_delay + _YANDEX_COMBOS_PER_FETCH_S секунд.
|
||||
# Для 30 combos × 3 pages × (9+12) + 300 ≈ 2190s (~37 мин).
|
||||
# В combos-режиме один "anchor" выполняет num_segments × num_combos × max_pages
|
||||
# fetch'ей — каждый занимает примерно _resolved_delay + _YANDEX_COMBOS_PER_FETCH_S сек.
|
||||
# Для 2 segments × 30 combos × 3 pages × (9+12) + 300 ≈ 4080s (~68 мин).
|
||||
# Множитель len(_segments) масштабирует watchdog пропорционально числу проходов
|
||||
# (vtorichka + novostroyki ≈ ×2 wall-clock) — иначе второй проход гильотинится.
|
||||
# В explicit-anchor (тест/override) режиме оставляем ANCHOR_TIMEOUT_SEC (240s) —
|
||||
# там каждый anchor небольшой и watchdog работает как старый защитный барьер.
|
||||
_num_combos = len(_rooms_list) * len(_price_ranges)
|
||||
if anchors is None and _num_combos > 0:
|
||||
_sweep_timeout = max(
|
||||
ANCHOR_TIMEOUT_SEC,
|
||||
_num_combos * pages_per_anchor * (_resolved_delay + _YANDEX_COMBOS_PER_FETCH_S)
|
||||
len(_segments)
|
||||
* _num_combos
|
||||
* pages_per_anchor
|
||||
* (_resolved_delay + _YANDEX_COMBOS_PER_FETCH_S)
|
||||
+ _YANDEX_ADDRESS_ENRICH_BUDGET_S,
|
||||
)
|
||||
else:
|
||||
|
|
@ -1162,6 +1173,7 @@ async def run_yandex_city_sweep(
|
|||
max_pages=pages_per_anchor,
|
||||
rooms_list=_rooms_list,
|
||||
price_ranges=_price_ranges,
|
||||
segments=_segments,
|
||||
)
|
||||
counters.lots_fetched += len(anchor_lots)
|
||||
if anchor_lots:
|
||||
|
|
|
|||
|
|
@ -74,15 +74,7 @@ _UNIT_DAYS: dict[str, int] = {
|
|||
# поэтому отдельная ветка с неявным n=1. Юниты в винительном падеже («минуту»,
|
||||
# «неделю») плюс именительный («час», «день»).
|
||||
_REL_DATE_SINGULAR_RE = re.compile(
|
||||
r"(?P<unit>"
|
||||
r"секунду|"
|
||||
r"минуту|"
|
||||
r"час|"
|
||||
r"день|"
|
||||
r"неделю|"
|
||||
r"месяц|"
|
||||
r"год"
|
||||
r")\s+назад",
|
||||
r"(?P<unit>" r"секунду|" r"минуту|" r"час|" r"день|" r"неделю|" r"месяц|" r"год" r")\s+назад",
|
||||
flags=re.I,
|
||||
)
|
||||
|
||||
|
|
@ -751,12 +743,12 @@ class AvitoScraper(BaseScraper):
|
|||
house_ext_id = parts[-1]
|
||||
house_url = urljoin("https://www.avito.ru", h_href)
|
||||
|
||||
# listing_segment по item URL pattern
|
||||
listing_segment: str | None = None
|
||||
if "/kvartiry/" in href:
|
||||
listing_segment = "vtorichka"
|
||||
elif "/novostroyki/" in href:
|
||||
listing_segment = "novostroyki"
|
||||
# listing_segment по DOM-маркеру карточки: новостройки несут
|
||||
# data-marker="item-development-name" (название ЖК/застройщика),
|
||||
# вторичка — нет. URL-паттерн ненадёжен: каждый avito-URL квартиры
|
||||
# содержит /kvartiry/ (вкл. новостройки) → ветка всегда vtorichka.
|
||||
dev_name = card.css_first('[data-marker="item-development-name"]')
|
||||
listing_segment = "novostroyki" if dev_name is not None else "vtorichka"
|
||||
|
||||
return ScrapedLot(
|
||||
source="avito",
|
||||
|
|
|
|||
|
|
@ -152,6 +152,7 @@ _SALE_TYPE_MAP: dict[str, str] = {
|
|||
"свободная": "free",
|
||||
"альтернатива": "alternative",
|
||||
"аукцион": "auction",
|
||||
"переуступка": "assignment",
|
||||
}
|
||||
|
||||
_HOUSE_TYPE_MAP: dict[str, str] = {
|
||||
|
|
|
|||
|
|
@ -48,6 +48,7 @@ class DetailEnrichment:
|
|||
repair_type: str | None = None # 'cosmetic' / 'design' / 'no' — raw Cian value
|
||||
repair_state: str | None = None # enum: needs_repair/standard/good/excellent
|
||||
kitchen_area_m2: float | None = None # offer.kitchenArea (м²)
|
||||
description: str | None = None # offer.description (текст объявления)
|
||||
views_total: int | None = None # Cian stats.totalViewsFormattedString → int
|
||||
views_today: int | None = None
|
||||
|
||||
|
|
@ -144,6 +145,7 @@ async def fetch_detail(
|
|||
repair_type=_extract_repair_type(offer),
|
||||
repair_state=_extract_repair_state(offer),
|
||||
kitchen_area_m2=_parse_float(offer.get("kitchenArea")),
|
||||
description=offer.get("description") or None,
|
||||
raw_offer=offer,
|
||||
)
|
||||
|
||||
|
|
@ -302,8 +304,10 @@ def save_detail_enrichment(db: Session, listing_id: int, enrichment: DetailEnric
|
|||
repair_type = COALESCE(:rt, repair_type),
|
||||
repair_state = COALESCE(:rs, repair_state),
|
||||
kitchen_area_m2 = COALESCE(CAST(:ka AS double precision), kitchen_area_m2),
|
||||
description = COALESCE(:descr, description),
|
||||
views_total = COALESCE(:vt, views_total),
|
||||
views_today = COALESCE(:vd, views_today)
|
||||
views_today = COALESCE(:vd, views_today),
|
||||
detail_enriched_at = NOW()
|
||||
WHERE id = CAST(:lid AS bigint)
|
||||
"""),
|
||||
{
|
||||
|
|
@ -315,6 +319,7 @@ def save_detail_enrichment(db: Session, listing_id: int, enrichment: DetailEnric
|
|||
"rt": enrichment.repair_type,
|
||||
"rs": enrichment.repair_state,
|
||||
"ka": enrichment.kitchen_area_m2,
|
||||
"descr": enrichment.description,
|
||||
"vt": enrichment.views_total,
|
||||
"vd": enrichment.views_today,
|
||||
},
|
||||
|
|
|
|||
|
|
@ -129,7 +129,14 @@ def _extract_json_from_content(content: str) -> str | None:
|
|||
return None
|
||||
|
||||
|
||||
def _parse_gate_json(payload: dict[str, Any], page_param: int = 1) -> list[ScrapedLot]:
|
||||
def _segment_for(new_flat: str) -> str:
|
||||
"""Map a newFlat gate-API value to the listings.listing_segment value."""
|
||||
return "novostroyki" if new_flat == "YES" else "vtorichka"
|
||||
|
||||
|
||||
def _parse_gate_json(
|
||||
payload: dict[str, Any], page_param: int = 1, new_flat: str = "NO"
|
||||
) -> list[ScrapedLot]:
|
||||
"""Parse gate-API JSON payload into a list of ScrapedLot.
|
||||
|
||||
Verified field mapping (from .issdump/yandex_gate_api_sample.json + live prod 2026-06-17):
|
||||
|
|
@ -150,7 +157,7 @@ def _parse_gate_json(payload: dict[str, Any], page_param: int = 1) -> list[Scrap
|
|||
location.geocoderAddress -> address (full, with house number)
|
||||
mainImages[] -> photo_urls (prepend "https:", up to 5)
|
||||
building.siteId -> house_ext_id (JK linkage)
|
||||
listing_segment = "vtorichka" (all requests use newFlat=NO)
|
||||
listing_segment -> "novostroyki" (newFlat=YES) | "vtorichka" (newFlat=NO)
|
||||
"""
|
||||
result = _extract_gate_data(payload)
|
||||
if result is None:
|
||||
|
|
@ -160,13 +167,15 @@ def _parse_gate_json(payload: dict[str, Any], page_param: int = 1) -> list[Scrap
|
|||
entities, _pager = result
|
||||
lots: list[ScrapedLot] = []
|
||||
for entity in entities:
|
||||
lot = _entity_to_lot(entity, page_param=page_param)
|
||||
lot = _entity_to_lot(entity, page_param=page_param, new_flat=new_flat)
|
||||
if lot is not None:
|
||||
lots.append(lot)
|
||||
return lots
|
||||
|
||||
|
||||
def _entity_to_lot(entity: dict[str, Any], page_param: int = 1) -> ScrapedLot | None:
|
||||
def _entity_to_lot(
|
||||
entity: dict[str, Any], page_param: int = 1, new_flat: str = "NO"
|
||||
) -> ScrapedLot | None:
|
||||
"""Convert one gate-API entity dict to ScrapedLot. None on missing required fields."""
|
||||
try:
|
||||
offer_id = str(entity.get("offerId") or "")
|
||||
|
|
@ -263,7 +272,7 @@ def _entity_to_lot(entity: dict[str, Any], page_param: int = 1) -> ScrapedLot |
|
|||
house_source=house_source,
|
||||
house_ext_id=house_ext_id,
|
||||
house_url=None,
|
||||
listing_segment="vtorichka",
|
||||
listing_segment=_segment_for(new_flat),
|
||||
raw_payload={
|
||||
"page_param": page_param,
|
||||
"ceiling_height": ceiling_height,
|
||||
|
|
@ -366,13 +375,18 @@ class YandexRealtyScraper(BaseScraper):
|
|||
rooms: str | None = None,
|
||||
price_min: int | None = None,
|
||||
price_max: int | None = None,
|
||||
new_flat: str = "NO",
|
||||
) -> str:
|
||||
"""Build gate-API URL. page: 1-based (page=0 -> API error)."""
|
||||
"""Build gate-API URL. page: 1-based (page=0 -> API error).
|
||||
|
||||
new_flat: "NO" -> vtorichka (default, preserves legacy behavior),
|
||||
"YES" -> novostroyki (primary-sale / reassignment segment).
|
||||
"""
|
||||
params: dict[str, str | int] = {
|
||||
"rgid": _EKB_RGID,
|
||||
"type": "SELL",
|
||||
"category": "APARTMENT",
|
||||
"newFlat": "NO",
|
||||
"newFlat": new_flat,
|
||||
"_pageType": "search",
|
||||
"_providers": "react-search-results-data",
|
||||
"page": page,
|
||||
|
|
@ -391,9 +405,12 @@ class YandexRealtyScraper(BaseScraper):
|
|||
page: int,
|
||||
price_min: int | None,
|
||||
price_max: int | None,
|
||||
new_flat: str = "NO",
|
||||
) -> dict[str, Any] | None:
|
||||
"""Fetch one gate-API page. Returns parsed JSON dict or None on error/invalid."""
|
||||
url = self._build_url(page=page, rooms=rooms, price_min=price_min, price_max=price_max)
|
||||
url = self._build_url(
|
||||
page=page, rooms=rooms, price_min=price_min, price_max=price_max, new_flat=new_flat
|
||||
)
|
||||
try:
|
||||
resp = await self._http_get(url, timeout=60)
|
||||
except Exception:
|
||||
|
|
@ -421,14 +438,18 @@ class YandexRealtyScraper(BaseScraper):
|
|||
rooms: str | None = None,
|
||||
price_min: int | None = None,
|
||||
price_max: int | None = None,
|
||||
new_flat: str = "NO",
|
||||
) -> list[ScrapedLot]:
|
||||
"""Fetch ONE page of gate-API results. lat/lon/radius_m ignored (uses rgid).
|
||||
|
||||
page: 1-based. Tarpit resilience: status_code==0 or JSON error
|
||||
-> rotate IP + retry up to _YANDEX_TARPIT_MAX_RETRIES times.
|
||||
Non-tarpit failures (4xx/5xx) are not retried.
|
||||
new_flat: "NO" -> vtorichka, "YES" -> novostroyki.
|
||||
"""
|
||||
url = self._build_url(page=page, rooms=rooms, price_min=price_min, price_max=price_max)
|
||||
url = self._build_url(
|
||||
page=page, rooms=rooms, price_min=price_min, price_max=price_max, new_flat=new_flat
|
||||
)
|
||||
|
||||
payload: dict[str, Any] | None = None
|
||||
for attempt in range(1 + _YANDEX_TARPIT_MAX_RETRIES):
|
||||
|
|
@ -494,13 +515,14 @@ class YandexRealtyScraper(BaseScraper):
|
|||
)
|
||||
return []
|
||||
|
||||
lots = _parse_gate_json(payload, page_param=page)
|
||||
lots = _parse_gate_json(payload, page_param=page, new_flat=new_flat)
|
||||
logger.info(
|
||||
"yandex gate page=%d rooms=%s price=[%s-%s]: %d cards",
|
||||
"yandex gate page=%d rooms=%s price=[%s-%s] newFlat=%s: %d cards",
|
||||
page,
|
||||
rooms or "all",
|
||||
price_min,
|
||||
price_max,
|
||||
new_flat,
|
||||
len(lots),
|
||||
)
|
||||
await self.sleep_between_requests()
|
||||
|
|
@ -514,16 +536,20 @@ class YandexRealtyScraper(BaseScraper):
|
|||
max_pages: int = _GATE_MAX_PAGES_CAP,
|
||||
rooms_list: list[str] | None = None,
|
||||
price_ranges: list[tuple[int | None, int | None]] | None = None,
|
||||
segments: list[str] | None = None,
|
||||
**_legacy_kwargs: Any,
|
||||
) -> list[ScrapedLot]:
|
||||
"""Fetch via rooms x price-range combos; paginate each combo to totalPages.
|
||||
"""Fetch via segment x rooms x price-range combos; paginate each combo to totalPages.
|
||||
|
||||
Pagination driven by pager.totalPages from first page response.
|
||||
Each combo iterates page=1 .. min(totalPages, max_pages, _GATE_MAX_PAGES_CAP).
|
||||
Deduplicates by source_id/source_url across all combos.
|
||||
Deduplicates by source_id/source_url across ALL combos AND segments (one `seen`).
|
||||
Legacy mode (rooms_list=None, price_ranges=None): single citywide sweep.
|
||||
segments: list of newFlat values, default ["NO"] (vtorichka only). Pass
|
||||
["NO", "YES"] to sweep both vtorichka and novostroyki in one call.
|
||||
"""
|
||||
seen: dict[str, ScrapedLot] = {}
|
||||
_segments = segments or ["NO"]
|
||||
|
||||
if rooms_list is None and price_ranges is None:
|
||||
combos: list[tuple[str | None, int | None, int | None]] = [(None, None, None)]
|
||||
|
|
@ -532,112 +558,123 @@ class YandexRealtyScraper(BaseScraper):
|
|||
p_ranges = price_ranges or DEFAULT_PRICE_RANGES
|
||||
combos = [(r, lo, hi) for r in r_list for lo, hi in p_ranges]
|
||||
|
||||
for rooms, price_min, price_max in combos:
|
||||
combo_label = _combo_label(rooms, price_min, price_max)
|
||||
total_pages: int | None = None
|
||||
for new_flat in _segments:
|
||||
_seg = _segment_for(new_flat)
|
||||
for rooms, price_min, price_max in combos:
|
||||
combo_label = f"{_seg}/{_combo_label(rooms, price_min, price_max)}"
|
||||
total_pages: int | None = None
|
||||
|
||||
for page in range(1, max_pages + 1):
|
||||
if page == 1:
|
||||
payload_p1: dict[str, Any] | None = None
|
||||
p1_url = self._build_url(
|
||||
page=1, rooms=rooms, price_min=price_min, price_max=price_max
|
||||
)
|
||||
for _attempt in range(1 + _YANDEX_TARPIT_MAX_RETRIES):
|
||||
try:
|
||||
resp = await self._http_get(p1_url, timeout=60)
|
||||
except Exception:
|
||||
logger.exception("yandex gate combo fetch failed combo=%s", combo_label)
|
||||
for page in range(1, max_pages + 1):
|
||||
if page == 1:
|
||||
payload_p1: dict[str, Any] | None = None
|
||||
p1_url = self._build_url(
|
||||
page=1,
|
||||
rooms=rooms,
|
||||
price_min=price_min,
|
||||
price_max=price_max,
|
||||
new_flat=new_flat,
|
||||
)
|
||||
for _attempt in range(1 + _YANDEX_TARPIT_MAX_RETRIES):
|
||||
try:
|
||||
resp = await self._http_get(p1_url, timeout=60)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"yandex gate combo fetch failed combo=%s", combo_label
|
||||
)
|
||||
break
|
||||
if resp.status_code == 0: # type: ignore[union-attr]
|
||||
logger.warning(
|
||||
"yandex gate: tarpit combo=%s page=1 -- rotating", combo_label
|
||||
)
|
||||
await self._rotate_ip()
|
||||
await asyncio.sleep(2)
|
||||
continue
|
||||
if resp.status_code != 200: # type: ignore[union-attr]
|
||||
logger.warning(
|
||||
"yandex gate: HTTP %d combo=%s page=1",
|
||||
resp.status_code,
|
||||
combo_label, # type: ignore[union-attr]
|
||||
)
|
||||
break
|
||||
try:
|
||||
payload_p1 = json.loads(resp.text) # type: ignore[union-attr]
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
logger.warning(
|
||||
"yandex gate: JSON parse error combo=%s page=1", combo_label
|
||||
)
|
||||
payload_p1 = None
|
||||
await self._rotate_ip()
|
||||
await asyncio.sleep(2)
|
||||
continue
|
||||
if _is_gate_error(payload_p1):
|
||||
logger.warning(
|
||||
"yandex gate: error payload combo=%s page=1", combo_label
|
||||
)
|
||||
payload_p1 = None
|
||||
break
|
||||
break
|
||||
if resp.status_code == 0: # type: ignore[union-attr]
|
||||
logger.warning(
|
||||
"yandex gate: tarpit combo=%s page=1 -- rotating", combo_label
|
||||
)
|
||||
await self._rotate_ip()
|
||||
await asyncio.sleep(2)
|
||||
continue
|
||||
if resp.status_code != 200: # type: ignore[union-attr]
|
||||
logger.warning(
|
||||
"yandex gate: HTTP %d combo=%s page=1",
|
||||
resp.status_code,
|
||||
combo_label, # type: ignore[union-attr]
|
||||
|
||||
if payload_p1 is None:
|
||||
logger.debug(
|
||||
"yandex gate combo [%s] page=1: failed -- skipping", combo_label
|
||||
)
|
||||
break
|
||||
try:
|
||||
payload_p1 = json.loads(resp.text) # type: ignore[union-attr]
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
logger.warning(
|
||||
"yandex gate: JSON parse error combo=%s page=1", combo_label
|
||||
)
|
||||
payload_p1 = None
|
||||
await self._rotate_ip()
|
||||
await asyncio.sleep(2)
|
||||
continue
|
||||
if _is_gate_error(payload_p1):
|
||||
logger.warning(
|
||||
"yandex gate: error payload combo=%s page=1", combo_label
|
||||
)
|
||||
payload_p1 = None
|
||||
|
||||
result = _extract_gate_data(payload_p1)
|
||||
if result is None:
|
||||
break
|
||||
_entities_p1, pager_p1 = result
|
||||
total_pages = min(
|
||||
pager_p1.get("totalPages", 1),
|
||||
max_pages,
|
||||
_GATE_MAX_PAGES_CAP,
|
||||
)
|
||||
lots_p1 = _parse_gate_json(payload_p1, page_param=1, new_flat=new_flat)
|
||||
if not lots_p1:
|
||||
logger.debug(
|
||||
"yandex gate combo [%s] page=1: empty -- stopping", combo_label
|
||||
)
|
||||
break
|
||||
for lot in lots_p1:
|
||||
key = lot.source_id or lot.source_url
|
||||
if key and key not in seen:
|
||||
seen[key] = lot
|
||||
await self.sleep_between_requests()
|
||||
if total_pages <= 1:
|
||||
break
|
||||
continue
|
||||
|
||||
if total_pages is not None and page > total_pages:
|
||||
break
|
||||
|
||||
if payload_p1 is None:
|
||||
lots = await self.fetch_around(
|
||||
lat,
|
||||
lon,
|
||||
radius_m,
|
||||
page=page,
|
||||
rooms=rooms,
|
||||
price_min=price_min,
|
||||
price_max=price_max,
|
||||
new_flat=new_flat,
|
||||
)
|
||||
if not lots:
|
||||
logger.debug(
|
||||
"yandex gate combo [%s] page=1: failed -- skipping", combo_label
|
||||
"yandex gate combo [%s] page=%d: empty -- stopping",
|
||||
combo_label,
|
||||
page,
|
||||
)
|
||||
break
|
||||
|
||||
result = _extract_gate_data(payload_p1)
|
||||
if result is None:
|
||||
break
|
||||
_entities_p1, pager_p1 = result
|
||||
total_pages = min(
|
||||
pager_p1.get("totalPages", 1),
|
||||
max_pages,
|
||||
_GATE_MAX_PAGES_CAP,
|
||||
)
|
||||
lots_p1 = _parse_gate_json(payload_p1, page_param=1)
|
||||
if not lots_p1:
|
||||
logger.debug(
|
||||
"yandex gate combo [%s] page=1: empty -- stopping", combo_label
|
||||
)
|
||||
break
|
||||
for lot in lots_p1:
|
||||
for lot in lots:
|
||||
key = lot.source_id or lot.source_url
|
||||
if key and key not in seen:
|
||||
seen[key] = lot
|
||||
await self.sleep_between_requests()
|
||||
if total_pages <= 1:
|
||||
break
|
||||
continue
|
||||
|
||||
if total_pages is not None and page > total_pages:
|
||||
break
|
||||
|
||||
lots = await self.fetch_around(
|
||||
lat,
|
||||
lon,
|
||||
radius_m,
|
||||
page=page,
|
||||
rooms=rooms,
|
||||
price_min=price_min,
|
||||
price_max=price_max,
|
||||
)
|
||||
if not lots:
|
||||
logger.debug(
|
||||
"yandex gate combo [%s] page=%d: empty -- stopping",
|
||||
combo_label,
|
||||
page,
|
||||
)
|
||||
break
|
||||
for lot in lots:
|
||||
key = lot.source_id or lot.source_url
|
||||
if key and key not in seen:
|
||||
seen[key] = lot
|
||||
|
||||
logger.info(
|
||||
"yandex gate aggregate: %d unique lots (%d combos)",
|
||||
"yandex gate aggregate: %d unique lots (%d combos x %d segments=%s)",
|
||||
len(seen),
|
||||
len(combos),
|
||||
len(_segments),
|
||||
_segments,
|
||||
)
|
||||
return list(seen.values())
|
||||
|
||||
|
|
|
|||
|
|
@ -17,6 +17,18 @@ _SORT_SQL: dict[str, str] = {
|
|||
),
|
||||
}
|
||||
|
||||
# Сегмент рынка (#1188). listings_search_mv не несёт колонку listing_segment,
|
||||
# поэтому фильтруем через подзапрос к базовой таблице listings, переиспользуя
|
||||
# canon-предикат #1186: NULL = legacy вторичка до миграции 011.
|
||||
_VTORICHKA_GUARD = "(listing_segment IS NULL OR listing_segment = 'vtorichka')"
|
||||
_SEGMENT_SQL: dict[str, str | None] = {
|
||||
"vtorichka": (f"listing_id IN (SELECT id FROM listings WHERE {_VTORICHKA_GUARD})"),
|
||||
"novostroyki": (
|
||||
"listing_id IN (SELECT id FROM listings WHERE listing_segment = 'novostroyki')"
|
||||
),
|
||||
"all": None,
|
||||
}
|
||||
|
||||
|
||||
def build_search_query(params: SearchParams) -> tuple[str, dict[str, object]]:
|
||||
"""Возвращает (sql, args) для SELECT из listings_search_mv."""
|
||||
|
|
@ -86,6 +98,10 @@ def build_search_query(params: SearchParams) -> tuple[str, dict[str, object]]:
|
|||
if params.has_kadastr:
|
||||
where.append("cadastral_number IS NOT NULL")
|
||||
|
||||
segment_clause = _SEGMENT_SQL[params.segment]
|
||||
if segment_clause is not None:
|
||||
where.append(segment_clause)
|
||||
|
||||
if params.sources:
|
||||
where.append("sources && CAST(:sources AS text[])")
|
||||
args["sources"] = params.sources
|
||||
|
|
|
|||
|
|
@ -78,6 +78,25 @@ def test_minimal_parse() -> None:
|
|||
assert result.views_today == 12
|
||||
|
||||
|
||||
def test_sale_type_assignment_pereustupka() -> None:
|
||||
"""«Способ продажи: переуступка» → sale_type='assignment' (ДДУ переуступка,
|
||||
сигнал новостройки). До фикса map не содержал ключ → None."""
|
||||
html = """
|
||||
<html><body>
|
||||
<div data-marker="item-view/item-id">№ 8043003287</div>
|
||||
<span itemprop="price" content="9500000">9 500 000 ₽</span>
|
||||
<div data-marker="item-view/item-params">
|
||||
<ul>
|
||||
<li>Количество комнат: 2</li>
|
||||
<li>Способ продажи: переуступка</li>
|
||||
</ul>
|
||||
</div>
|
||||
</body></html>
|
||||
"""
|
||||
result = parse_detail_html(html, "https://www.avito.ru/test_8043003287")
|
||||
assert result.sale_type == "assignment"
|
||||
|
||||
|
||||
def test_metro_extraction() -> None:
|
||||
result = parse_detail_html(MINIMAL_HTML, SOURCE_URL)
|
||||
# "Чкаловская 11-15 мин пешком" → metro_stations
|
||||
|
|
|
|||
|
|
@ -60,3 +60,37 @@ def test_lazy_card_date_comes_from_json_not_dom() -> None:
|
|||
lots = AvitoScraper()._parse_html(html, "https://www.avito.ru/x")
|
||||
lazy = next(lot for lot in lots if "8043936560" in (lot.source_url or ""))
|
||||
assert lazy.listing_date == _expected(_TS_LAZY)
|
||||
|
||||
|
||||
# ── listing_segment по DOM-маркеру item-development-name ───────────────────────
|
||||
# Каждый avito-URL квартиры содержит /kvartiry/ (вкл. новостройки), поэтому
|
||||
# сегмент определяется не по URL, а по наличию data-marker="item-development-name"
|
||||
# (название ЖК/застройщика рендерится только у карточек новостроек).
|
||||
|
||||
|
||||
def _serp_card(item_id: str, *, with_dev_name: bool) -> str:
|
||||
dev = '<div data-marker="item-development-name">ЖК «Федерация»</div>' if with_dev_name else ""
|
||||
return f"""
|
||||
<div data-marker="item" data-item-id="{item_id}">
|
||||
<a data-marker="item-title" href="/ekaterinburg/kvartiry/2-k_kvartira_{item_id}">
|
||||
2-к. квартира, 50 м², 5/20 эт.</a>
|
||||
<meta itemprop="price" content="9500000"/>
|
||||
{dev}
|
||||
</div>
|
||||
"""
|
||||
|
||||
|
||||
def test_segment_novostroyki_when_development_name_present() -> None:
|
||||
"""Карточка с data-marker='item-development-name' → listing_segment='novostroyki'."""
|
||||
html = f"<html><body>{_serp_card('8043003287', with_dev_name=True)}</body></html>"
|
||||
lots = AvitoScraper()._parse_html(html, "https://www.avito.ru/ekaterinburg/kvartiry")
|
||||
assert len(lots) == 1
|
||||
assert lots[0].listing_segment == "novostroyki"
|
||||
|
||||
|
||||
def test_segment_vtorichka_when_no_development_name() -> None:
|
||||
"""Карточка без маркера ЖК → listing_segment='vtorichka' (даже с /kvartiry/ в URL)."""
|
||||
html = f"<html><body>{_serp_card('8043936560', with_dev_name=False)}</body></html>"
|
||||
lots = AvitoScraper()._parse_html(html, "https://www.avito.ru/ekaterinburg/kvartiry")
|
||||
assert len(lots) == 1
|
||||
assert lots[0].listing_segment == "vtorichka"
|
||||
|
|
|
|||
|
|
@ -462,6 +462,40 @@ async def test_fetch_detail_kitchen_area_none_when_missing(monkeypatch):
|
|||
assert result.kitchen_area_m2 is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_detail_extracts_description(monkeypatch):
|
||||
"""offer.description → DetailEnrichment.description (для persist в listings)."""
|
||||
fake_state = {
|
||||
"offerData": {
|
||||
"offer": {
|
||||
"cianId": 555555,
|
||||
"description": "Продаётся уютная квартира с видом на парк.",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
monkeypatch.setattr(
|
||||
"app.services.scrapers.cian_detail.extract_state",
|
||||
lambda html, mfe, key: fake_state if key == "defaultState" else None,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"app.services.scrapers.cian_detail.extract_all_states",
|
||||
lambda html: {},
|
||||
)
|
||||
|
||||
response = MagicMock()
|
||||
response.status_code = 200
|
||||
response.text = "<html></html>"
|
||||
|
||||
session = MagicMock()
|
||||
session.get = AsyncMock(return_value=response)
|
||||
session.close = AsyncMock()
|
||||
|
||||
result = await fetch_detail("https://ekb.cian.ru/sale/flat/555555/", session=session)
|
||||
assert result is not None
|
||||
assert result.description == "Продаётся уютная квартира с видом на парк."
|
||||
|
||||
|
||||
# ── save_detail_enrichment — kitchen_area_m2 persisted ───────────────────────
|
||||
|
||||
|
||||
|
|
@ -504,6 +538,36 @@ def test_save_detail_enrichment_kitchen_area_none_passthrough():
|
|||
assert update_call_params["ka"] is None
|
||||
|
||||
|
||||
# ── save_detail_enrichment — description + detail_enriched_at stamp ───────────
|
||||
|
||||
|
||||
def test_save_detail_enrichment_writes_description():
|
||||
"""save_detail_enrichment передаёт description в UPDATE listings."""
|
||||
db = MagicMock()
|
||||
db.execute.return_value.fetchone.return_value = None
|
||||
|
||||
enrichment = DetailEnrichment(description="Светлая квартира рядом с парком")
|
||||
|
||||
save_detail_enrichment(db, listing_id=1002, enrichment=enrichment)
|
||||
|
||||
update_call_params = db.execute.call_args_list[0][0][1]
|
||||
assert update_call_params["descr"] == "Светлая квартира рядом с парком"
|
||||
|
||||
|
||||
def test_save_detail_enrichment_stamps_detail_enriched_at():
|
||||
"""UPDATE listings содержит detail_enriched_at = NOW() — иначе строка
|
||||
переобогащается каждый run (detail_enriched_at IS NULL filter)."""
|
||||
db = MagicMock()
|
||||
db.execute.return_value.fetchone.return_value = None
|
||||
|
||||
enrichment = DetailEnrichment(kitchen_area_m2=8.3)
|
||||
|
||||
save_detail_enrichment(db, listing_id=1003, enrichment=enrichment)
|
||||
|
||||
update_sql = str(db.execute.call_args_list[0][0][0])
|
||||
assert "detail_enriched_at = NOW()" in update_sql
|
||||
|
||||
|
||||
# ── fetch_detail browser_fetcher param ───────────────────────────────────────
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -62,6 +62,42 @@ def test_build_query_sources_array():
|
|||
assert args["sources"] == ["avito", "cian"]
|
||||
|
||||
|
||||
def test_build_query_segment_default_vtorichka():
|
||||
"""Без параметра segment — back-compat: canon-предикат вторички (#1186/#1188)."""
|
||||
sql, _ = build_search_query(SearchParams())
|
||||
assert SearchParams().segment == "vtorichka"
|
||||
assert "listing_id IN (SELECT id FROM listings WHERE" in sql
|
||||
assert "(listing_segment IS NULL OR listing_segment = 'vtorichka')" in sql
|
||||
|
||||
|
||||
def test_build_query_segment_novostroyki():
|
||||
"""segment=novostroyki — только первичка, без NULL-вторички."""
|
||||
sql, _ = build_search_query(SearchParams(segment="novostroyki"))
|
||||
assert "listing_segment = 'novostroyki'" in sql
|
||||
# Не должен попасть canon-guard вторички.
|
||||
assert "(listing_segment IS NULL OR listing_segment = 'vtorichka')" not in sql
|
||||
|
||||
|
||||
def test_build_query_segment_all_no_filter():
|
||||
"""segment=all — без фильтрации по сегменту."""
|
||||
sql, _ = build_search_query(SearchParams(segment="all"))
|
||||
assert "listing_segment" not in sql
|
||||
|
||||
|
||||
def test_build_count_query_inherits_segment():
|
||||
"""COUNT-запрос наследует segment-фильтр из build_search_query."""
|
||||
sql, _ = build_count_query(SearchParams(segment="novostroyki"))
|
||||
assert "listing_segment = 'novostroyki'" in sql
|
||||
sql_v, _ = build_count_query(SearchParams())
|
||||
assert "(listing_segment IS NULL OR listing_segment = 'vtorichka')" in sql_v
|
||||
|
||||
|
||||
def test_segment_invalid_rejected():
|
||||
"""Невалидное значение segment отклоняется валидацией."""
|
||||
with pytest.raises(ValueError):
|
||||
SearchParams(segment="kommercia")
|
||||
|
||||
|
||||
def test_build_count_query_strips_limit():
|
||||
sql, args = build_count_query(SearchParams(rooms=2, page=3))
|
||||
assert "count(*)" in sql
|
||||
|
|
|
|||
|
|
@ -182,6 +182,29 @@ def test_entity_to_lot_full_entity():
|
|||
assert lot.raw_payload["kitchen_area_m2"] == pytest.approx(12.7)
|
||||
|
||||
|
||||
def test_entity_to_lot_segment_vtorichka_default():
|
||||
"""Default new_flat='NO' -> listing_segment='vtorichka'."""
|
||||
lot = _entity_to_lot(_ENTITY_FULL)
|
||||
assert lot is not None
|
||||
assert lot.listing_segment == "vtorichka"
|
||||
|
||||
|
||||
def test_entity_to_lot_segment_novostroyki_on_new_flat_yes():
|
||||
"""new_flat='YES' -> listing_segment='novostroyki'."""
|
||||
lot = _entity_to_lot(_ENTITY_FULL, new_flat="YES")
|
||||
assert lot is not None
|
||||
assert lot.listing_segment == "novostroyki"
|
||||
|
||||
|
||||
def test_parse_gate_json_propagates_segment():
|
||||
"""_parse_gate_json threads new_flat down to each lot's listing_segment."""
|
||||
payload = _make_gate_payload([_ENTITY_FULL, _ENTITY_STUDIO])
|
||||
lots_nb = _parse_gate_json(payload, new_flat="YES")
|
||||
assert lots_nb and all(lot.listing_segment == "novostroyki" for lot in lots_nb)
|
||||
lots_vt = _parse_gate_json(payload, new_flat="NO")
|
||||
assert lots_vt and all(lot.listing_segment == "vtorichka" for lot in lots_vt)
|
||||
|
||||
|
||||
def test_entity_to_lot_studio_rooms_zero():
|
||||
lot = _entity_to_lot(_ENTITY_STUDIO)
|
||||
assert lot is not None
|
||||
|
|
@ -350,6 +373,21 @@ def test_build_url_combos_are_unique():
|
|||
assert len(urls) == expected, f"URL collisions: {expected} combos but {len(urls)} unique"
|
||||
|
||||
|
||||
def test_build_url_new_flat_default_no():
|
||||
"""Default new_flat must keep newFlat=NO (vtorichka) — backward compat."""
|
||||
s = YandexRealtyScraper()
|
||||
url = s._build_url(page=1)
|
||||
assert "newFlat=NO" in url
|
||||
assert "newFlat=YES" not in url
|
||||
|
||||
|
||||
def test_build_url_new_flat_yes():
|
||||
"""new_flat='YES' must produce newFlat=YES (novostroyki segment)."""
|
||||
s = YandexRealtyScraper()
|
||||
url = s._build_url(page=1, new_flat="YES")
|
||||
assert "newFlat=YES" in url
|
||||
|
||||
|
||||
def test_default_city():
|
||||
assert DEFAULT_CITY == "ekaterinburg"
|
||||
|
||||
|
|
@ -665,6 +703,81 @@ async def test_fetch_around_multi_room_dedup():
|
|||
assert len(ids) == 3
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_around_multi_room_runs_both_segments_with_shared_dedup():
|
||||
"""segments=['NO','YES'] sweeps both vtorichka+novostroyki; dedup spans both passes.
|
||||
|
||||
Mocks _http_get and inspects requested URLs: both newFlat=NO and newFlat=YES
|
||||
must be requested. The same offerId returned in both passes must be deduped to
|
||||
one lot (single shared `seen` across passes), tagged by the FIRST pass's segment.
|
||||
"""
|
||||
s = YandexRealtyScraper()
|
||||
|
||||
requested_new_flat: list[str] = []
|
||||
|
||||
entity_shared = dict(
|
||||
_ENTITY_FULL, offerId="shared_xseg", url="//realty.yandex.ru/offer/shared_xseg"
|
||||
)
|
||||
entity_vt = dict(_ENTITY_FULL, offerId="vt_only", url="//realty.yandex.ru/offer/vt_only")
|
||||
entity_nb = dict(_ENTITY_FULL, offerId="nb_only", url="//realty.yandex.ru/offer/nb_only")
|
||||
pager = {"page": 0, "pageSize": 20, "totalItems": 2, "totalPages": 1}
|
||||
|
||||
async def fake_http_get(url: str, **kwargs: object) -> _CurlResponse:
|
||||
if "newFlat=YES" in url:
|
||||
requested_new_flat.append("YES")
|
||||
payload = _make_gate_payload([entity_shared, entity_nb], pager)
|
||||
else:
|
||||
requested_new_flat.append("NO")
|
||||
payload = _make_gate_payload([entity_shared, entity_vt], pager)
|
||||
return _CurlResponse(status_code=200, text=json.dumps(payload))
|
||||
|
||||
with patch.object(s, "_http_get", side_effect=fake_http_get):
|
||||
with patch.object(s, "sleep_between_requests", new_callable=AsyncMock):
|
||||
lots = await s.fetch_around_multi_room(
|
||||
lat=0.0,
|
||||
lon=0.0,
|
||||
rooms_list=["1"],
|
||||
price_ranges=[(None, 5_000_000)],
|
||||
segments=["NO", "YES"],
|
||||
max_pages=1,
|
||||
)
|
||||
|
||||
assert "NO" in requested_new_flat
|
||||
assert "YES" in requested_new_flat
|
||||
|
||||
by_id = {lot.source_id: lot for lot in lots}
|
||||
assert set(by_id) == {"shared_xseg", "vt_only", "nb_only"}
|
||||
# Shared offer deduped to ONE lot, tagged by the first pass (NO -> vtorichka).
|
||||
assert by_id["shared_xseg"].listing_segment == "vtorichka"
|
||||
assert by_id["vt_only"].listing_segment == "vtorichka"
|
||||
assert by_id["nb_only"].listing_segment == "novostroyki"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_around_multi_room_default_segment_no_only():
|
||||
"""Without segments arg, only newFlat=NO (vtorichka) is requested (legacy default)."""
|
||||
s = YandexRealtyScraper()
|
||||
requested_new_flat: list[str] = []
|
||||
pager = {"page": 0, "pageSize": 20, "totalItems": 1, "totalPages": 1}
|
||||
|
||||
async def fake_http_get(url: str, **kwargs: object) -> _CurlResponse:
|
||||
requested_new_flat.append("YES" if "newFlat=YES" in url else "NO")
|
||||
body = json.dumps(_make_gate_payload([_ENTITY_FULL], pager))
|
||||
return _CurlResponse(status_code=200, text=body)
|
||||
|
||||
with patch.object(s, "_http_get", side_effect=fake_http_get):
|
||||
with patch.object(s, "sleep_between_requests", new_callable=AsyncMock):
|
||||
await s.fetch_around_multi_room(
|
||||
lat=0.0,
|
||||
lon=0.0,
|
||||
rooms_list=["1"],
|
||||
price_ranges=[(None, 5_000_000)],
|
||||
max_pages=1,
|
||||
)
|
||||
|
||||
assert set(requested_new_flat) == {"NO"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_around_multi_room_legacy_single_sweep():
|
||||
"""Legacy mode (no rooms_list/price_ranges) uses single citywide sweep."""
|
||||
|
|
|
|||
|
|
@ -121,9 +121,9 @@ export interface AvitoImvSummary {
|
|||
// нет сделок. count..period_months — required (бэкенд не отдаёт частичный объект).
|
||||
export interface DkpCorridor {
|
||||
count: number; // число ДКП-сделок в выборке
|
||||
low_ppm2: number; // min ₽/м² по сделкам
|
||||
low_ppm2: number; // P10 ₽/м² по сделкам (робастный коридор)
|
||||
median_ppm2: number; // медиана ₽/м²
|
||||
high_ppm2: number; // max ₽/м²
|
||||
high_ppm2: number; // P90 ₽/м² по сделкам (робастный коридор)
|
||||
period_months: number; // окно поиска сделок
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue