From f1c5056397f353368f9117334067472a85861b52 Mon Sep 17 00:00:00 2001 From: bot-backend Date: Wed, 17 Jun 2026 21:22:49 +0300 Subject: [PATCH 01/15] docs(estimator): DKP corridor described as P10/P90 not min/max in schema+TS (#1520 follow-up) --- tradein-mvp/backend/app/schemas/trade_in.py | 4 ++-- tradein-mvp/frontend/src/types/trade-in.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tradein-mvp/backend/app/schemas/trade_in.py b/tradein-mvp/backend/app/schemas/trade_in.py index e883551f..b51889ea 100644 --- a/tradein-mvp/backend/app/schemas/trade_in.py +++ b/tradein-mvp/backend/app/schemas/trade_in.py @@ -106,9 +106,9 @@ class DkpCorridor(BaseModel): """ count: int # число ДКП-сделок в выборке - low_ppm2: int # min ₽/м² по сделкам (P10-ish — берём минимум) + low_ppm2: int # P10 ₽/м² по сделкам (робастный коридор) median_ppm2: int # медиана ₽/м² - high_ppm2: int # max ₽/м² + high_ppm2: int # P90 ₽/м² по сделкам (робастный коридор) period_months: int # окно поиска сделок diff --git a/tradein-mvp/frontend/src/types/trade-in.ts b/tradein-mvp/frontend/src/types/trade-in.ts index 39a1acf5..3b9c9cd4 100644 --- a/tradein-mvp/frontend/src/types/trade-in.ts +++ b/tradein-mvp/frontend/src/types/trade-in.ts @@ -121,9 +121,9 @@ export interface AvitoImvSummary { // нет сделок. count..period_months — required (бэкенд не отдаёт частичный объект). export interface DkpCorridor { count: number; // число ДКП-сделок в выборке - low_ppm2: number; // min ₽/м² по сделкам + low_ppm2: number; // P10 ₽/м² по сделкам (робастный коридор) median_ppm2: number; // медиана ₽/м² - high_ppm2: number; // max ₽/м² + high_ppm2: number; // P90 ₽/м² по сделкам (робастный коридор) period_months: number; // окно поиска сделок } From bcb348da402844fd9c96b7c092bfbc4410670b1a Mon Sep 17 00:00:00 2001 From: bot-backend Date: Wed, 17 Jun 2026 21:22:53 +0300 Subject: [PATCH 02/15] fix(leads): replace ::interval cast with make_interval (psycopg v3 trap) (#1383 follow-up) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (:bind || ' months')::interval is a psycopg v3 SyntaxError — the driver sees '::' immediately after the bind placeholder token and chokes. Replace all 6 occurrences (admin_leads.py + analytics_queries.py) with make_interval(months => :bind) which is unambiguous to the parser and semantically identical. --- backend/app/api/v1/admin_leads.py | 2 +- backend/app/services/analytics_queries.py | 14 ++++++-------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/backend/app/api/v1/admin_leads.py b/backend/app/api/v1/admin_leads.py index c5a7a89d..f23b839c 100644 --- a/backend/app/api/v1/admin_leads.py +++ b/backend/app/api/v1/admin_leads.py @@ -138,7 +138,7 @@ def leads_stats( WITH window_leads AS ( SELECT * FROM prinzip_leads - WHERE created_at >= NOW() - (:m || ' months')::interval + WHERE created_at >= NOW() - make_interval(months => :m) ) SELECT (SELECT COUNT(*) FROM prinzip_leads) AS leads_total, diff --git a/backend/app/services/analytics_queries.py b/backend/app/services/analytics_queries.py index b542ae66..cd368a89 100644 --- a/backend/app/services/analytics_queries.py +++ b/backend/app/services/analytics_queries.py @@ -184,8 +184,7 @@ def quartirography(db: Session, source: str, region_id: int = 66) -> list[dict[s -- ('2025-07-01' расширял «recent»-окно каждую неделю по мере -- доливки ETL новых report_months → перекос в сторону всё -- более длинной истории). Тот же фикс, что #1203 и _BUCKET_SQL. - AND period_start_date >= NOW() - - (:months_window || ' months')::INTERVAL + AND period_start_date >= NOW() - make_interval(months => :months_window) ), bucketed AS ( SELECT CASE @@ -1169,7 +1168,7 @@ def prinzip_funnel_monthly(db: Session, months: int = 24) -> list[dict[str, Any] """ SELECT month, source, leads, engaged, converted, conv_pct FROM prinzip_funnel_monthly - WHERE month >= (CURRENT_DATE - (:months || ' months')::interval)::date + WHERE month >= (CURRENT_DATE - make_interval(months => :months))::date ORDER BY month DESC, leads DESC """ ), @@ -1203,7 +1202,7 @@ def prinzip_funnel_by_source(db: Session, months: int = 12) -> list[dict[str, An SUM(converted) AS converted, ROUND(100.0 * SUM(converted) / NULLIF(SUM(leads), 0), 2) AS conv_pct FROM prinzip_funnel_monthly - WHERE month >= (CURRENT_DATE - (:months || ' months')::interval)::date + WHERE month >= (CURRENT_DATE - make_interval(months => :months))::date GROUP BY source ORDER BY leads DESC """ @@ -1364,8 +1363,7 @@ _BUCKET_SQL = text( AND deal_count > 0 AND (area / deal_count) BETWEEN 15 AND 200 AND price_per_sqm BETWEEN 30000 AND 1000000 - AND period_start_date >= NOW() - - (:months_window || ' months')::INTERVAL + AND period_start_date >= NOW() - make_interval(months => :months_window) ), bucketed AS ( SELECT CASE @@ -1995,7 +1993,7 @@ def _elasticity_coef( {where_district} {where_class} AND crm.deals_total_avg_price_thousand_rub_per_m2 > 0 - AND crm.report_month >= NOW() - (:ew || ' months')::interval + AND crm.report_month >= NOW() - make_interval(months => :ew) ) SELECT regr_slope(y, x) AS slope, @@ -2086,7 +2084,7 @@ def _elasticity_per_bucket_coef( {where_class} AND crm.deals_total_count > 0 AND crm.deals_total_avg_price_thousand_rub_per_m2 > 0 - AND crm.report_month >= NOW() - (:ew || ' months')::interval + AND crm.report_month >= NOW() - make_interval(months => :ew) ) SELECT bucket, regr_slope(y, x) AS slope, From 51701074058d0f0554acdb2ab34eee8e32a9a9d2 Mon Sep 17 00:00:00 2001 From: bot-backend Date: Wed, 17 Jun 2026 21:27:34 +0300 Subject: [PATCH 03/15] fix(sql): EMISS PK includes period_type so yearly+Q1 coexist (#1606 follow-up) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds period_type TEXT NOT NULL DEFAULT 'unknown' to macro_indicator and widens the PRIMARY KEY to (indicator_type, region, obs_date, period_type). Before: yearly aggregate ('год'→obs_date YYYY-01-01) and Q1 ('I квартал'→ same date) shared the same PK slot → ON CONFLICT DO UPDATE made them overwrite each other despite the in-memory dedup fix in #1687. After: each EmissRow carries period_type from _emiss_period_granularity ('year'/'quarter'/'month'); DB ON CONFLICT targets include it, so yearly+Q1 rows genuinely coexist. Non-EMISS sources (CBR, rosstat open-data, domrf) use period_type='unknown' (literal in SQL) — they are disambiguated by obs_date already, so the wider PK is backward-compatible. All ON CONFLICT clauses in cbr_macro_sync.py and rosstat_macro_sync.py updated accordingly. Migration: 163_emiss_pk_period_type.sql (idempotent BEGIN/COMMIT). Tests: 49 passed (emiss + rosstat suite), ruff clean, py_compile OK. --- .../app/services/scrapers/rosstat_emiss.py | 11 ++- backend/app/workers/tasks/cbr_macro_sync.py | 21 +++-- .../app/workers/tasks/rosstat_macro_sync.py | 55 +++++++----- backend/tests/scrapers/test_emiss_sdmx.py | 13 ++- backend/tests/workers/test_cbr_macro_sync.py | 9 +- .../tests/workers/test_rosstat_macro_sync.py | 42 ++++++--- data/sql/163_emiss_pk_period_type.sql | 86 +++++++++++++++++++ 7 files changed, 191 insertions(+), 46 deletions(-) create mode 100644 data/sql/163_emiss_pk_period_type.sql diff --git a/backend/app/services/scrapers/rosstat_emiss.py b/backend/app/services/scrapers/rosstat_emiss.py index 14b633bf..775a5751 100644 --- a/backend/app/services/scrapers/rosstat_emiss.py +++ b/backend/app/services/scrapers/rosstat_emiss.py @@ -667,7 +667,14 @@ class EmissRow: """Одна готовая к upsert строка macro_indicator из ЕМИСС (source='emiss'). Отдельно от ``MacroRow`` (open-data, source='rosstat', yearly): ЕМИСС-ряды несут - свою frequency (quarterly/monthly) и source — контракт upsert'а у них иной. + свою frequency (quarterly/monthly), source и period_type — контракт upsert'а у + них иной. + + period_type: гранулярность под-периода ('year' | 'quarter' | 'month' | 'unknown'). + Берётся из _emiss_period_granularity(PERIOD). Необходима как часть PK + macro_indicator (migration 163), чтобы годовой агрегат ('год' → 'year') и + Q1 ('I квартал' → 'quarter') за один год не перезаписывали друг друга при + ON CONFLICT DO UPDATE (оба дают obs_date=YYYY-01-01) (#1606). """ indicator_type: str @@ -677,6 +684,7 @@ class EmissRow: unit: str frequency: str comment: str + period_type: str = "unknown" def _emiss_period_to_month(period: str) -> int | None: @@ -817,6 +825,7 @@ def parse_emiss_sdmx(raw: bytes | str, spec: EmissIndicatorSpec) -> list[EmissRo unit=spec.unit, frequency=spec.frequency, comment=spec.comment, + period_type=granularity, ) return [by_key[k] for k in sorted(by_key, key=lambda k: (k[0], k[1]))] diff --git a/backend/app/workers/tasks/cbr_macro_sync.py b/backend/app/workers/tasks/cbr_macro_sync.py index 441612af..92c27a5d 100644 --- a/backend/app/workers/tasks/cbr_macro_sync.py +++ b/backend/app/workers/tasks/cbr_macro_sync.py @@ -41,19 +41,21 @@ from app.workers.celery_app import celery_app logger = logging.getLogger(__name__) # psycopg v3: CAST(:x AS type) — НИКОГДА :x::type (SQLAlchemy+psycopg3 роняет -# синтаксис на ::). Контракт колонок совпадает с macro_indicator (migration 123): -# (indicator_type, region, obs_date, value, source, frequency, unit, comment, -# updated_at) PK (indicator_type, region, obs_date). +# синтаксис на ::). Контракт колонок совпадает с macro_indicator (migration 163): +# (indicator_type, region, obs_date, period_type, value, source, frequency, unit, +# comment, updated_at). PK (indicator_type, region, obs_date, period_type). +# CBR-ряды используют period_type='unknown' (литерал) — они не несут sub-period +# granularity и различаются по obs_date. UPSERT_KEY_RATE_SQL = text( """ INSERT INTO macro_indicator ( - indicator_type, region, obs_date, value, + indicator_type, region, obs_date, period_type, value, source, frequency, unit, comment ) VALUES ( - 'key_rate', 'rf', CAST(:d AS date), CAST(:v AS numeric), + 'key_rate', 'rf', CAST(:d AS date), 'unknown', CAST(:v AS numeric), 'cbr', 'daily', '%', 'CBR key rate' ) - ON CONFLICT (indicator_type, region, obs_date) DO UPDATE SET + ON CONFLICT (indicator_type, region, obs_date, period_type) DO UPDATE SET value = EXCLUDED.value, updated_at = now() """ @@ -61,16 +63,17 @@ UPSERT_KEY_RATE_SQL = text( # Инфляция «% г/г» (ИПЦ YoY): indicator_type='inflation_yoy', monthly, region='rf'. # obs_date уже нормализован к 1-му числу месяца парсером (parse_inflation_xlsx). +# period_type='unknown' — месячный ряд без sub-period granularity. UPSERT_INFLATION_SQL = text( """ INSERT INTO macro_indicator ( - indicator_type, region, obs_date, value, + indicator_type, region, obs_date, period_type, value, source, frequency, unit, comment ) VALUES ( - 'inflation_yoy', 'rf', CAST(:d AS date), CAST(:v AS numeric), + 'inflation_yoy', 'rf', CAST(:d AS date), 'unknown', CAST(:v AS numeric), 'cbr', 'monthly', '%', 'CBR inflation YoY (ИПЦ, % г/г)' ) - ON CONFLICT (indicator_type, region, obs_date) DO UPDATE SET + ON CONFLICT (indicator_type, region, obs_date, period_type) DO UPDATE SET value = EXCLUDED.value, updated_at = now() """ diff --git a/backend/app/workers/tasks/rosstat_macro_sync.py b/backend/app/workers/tasks/rosstat_macro_sync.py index 3cbec22a..cf84c5a3 100644 --- a/backend/app/workers/tasks/rosstat_macro_sync.py +++ b/backend/app/workers/tasks/rosstat_macro_sync.py @@ -47,21 +47,23 @@ from app.workers.celery_app import celery_app logger = logging.getLogger(__name__) # psycopg v3: CAST(:x AS type) — НИКОГДА :x::type (SQLAlchemy+psycopg3 роняет -# синтаксис на ::). Контракт колонок совпадает с macro_indicator (migration 123): -# (indicator_type, region, obs_date, value, source, frequency, unit, comment, -# updated_at) PK (indicator_type, region, obs_date). source='rosstat', -# frequency='yearly' (текущие ряды Росстата — годовые: obs_date = 1 января года). +# синтаксис на ::). Контракт колонок совпадает с macro_indicator (migration 163): +# (indicator_type, region, obs_date, period_type, value, source, frequency, unit, +# comment, updated_at). PK (indicator_type, region, obs_date, period_type). +# Не-ЕМИСС источники (open-data, СМР) используют period_type='unknown' (литерал) — +# они различаются по obs_date без sub-period granularity. +# source='rosstat', frequency='yearly' (текущие ряды Росстата — годовые). UPSERT_ROSSTAT_SQL = text( """ INSERT INTO macro_indicator ( - indicator_type, region, obs_date, value, + indicator_type, region, obs_date, period_type, value, source, frequency, unit, comment ) VALUES ( CAST(:itype AS text), CAST(:region AS text), CAST(:d AS date), - CAST(:v AS numeric), + 'unknown', CAST(:v AS numeric), 'rosstat', 'yearly', CAST(:unit AS text), CAST(:comment AS text) ) - ON CONFLICT (indicator_type, region, obs_date) DO UPDATE SET + ON CONFLICT (indicator_type, region, obs_date, period_type) DO UPDATE SET value = EXCLUDED.value, source = EXCLUDED.source, frequency = EXCLUDED.frequency, @@ -71,20 +73,23 @@ UPSERT_ROSSTAT_SQL = text( """ ) -# ЕМИСС-ряды (source='emiss'): frequency параметризована (quarterly для доходов, -# monthly для ИПЦ когда добавится) — в отличие от open-data, где она фикс 'yearly'. -# Тот же PK и ON CONFLICT-контракт. CAST(:x AS type) — НИКОГДА :x::type (psycopg v3). +# ЕМИСС-ряды (source='emiss'): period_type параметризован — 'year' | 'quarter' | +# 'month' (из _emiss_period_granularity). Это ключевое отличие от не-ЕМИСС источников: +# период ЕМИСС-ряда несёт granularity, необходимую для разделения годового агрегата +# ('год' → 'year') и Q1 ('I квартал' → 'quarter'), оба с obs_date=YYYY-01-01 (#1606). +# frequency параметризована (quarterly для доходов, monthly для ИПЦ когда добавится). +# CAST(:x AS type) — НИКОГДА :x::type (psycopg v3). UPSERT_EMISS_SQL = text( """ INSERT INTO macro_indicator ( - indicator_type, region, obs_date, value, + indicator_type, region, obs_date, period_type, value, source, frequency, unit, comment ) VALUES ( CAST(:itype AS text), CAST(:region AS text), CAST(:d AS date), - CAST(:v AS numeric), + CAST(:period_type AS text), CAST(:v AS numeric), 'emiss', CAST(:freq AS text), CAST(:unit AS text), CAST(:comment AS text) ) - ON CONFLICT (indicator_type, region, obs_date) DO UPDATE SET + ON CONFLICT (indicator_type, region, obs_date, period_type) DO UPDATE SET value = EXCLUDED.value, source = EXCLUDED.source, frequency = EXCLUDED.frequency, @@ -96,19 +101,19 @@ UPSERT_EMISS_SQL = text( # Индекс цен на СМР — открытый xlsx Росстата: source='rosstat' (rosstat.gov.ru-файл), # НО frequency='monthly' (в отличие от open-data демографии — там фикс 'yearly'), -# поэтому frequency параметризована. MacroRow не несёт frequency-поля, подставляем -# литералом в bind. Тот же PK и ON CONFLICT-контракт. CAST(:x AS type) — psycopg v3. +# поэтому frequency параметризована. MacroRow не несёт frequency/period_type-полей; +# подставляем литералами. CAST(:x AS type) — psycopg v3. UPSERT_ROSSTAT_MONTHLY_SQL = text( """ INSERT INTO macro_indicator ( - indicator_type, region, obs_date, value, + indicator_type, region, obs_date, period_type, value, source, frequency, unit, comment ) VALUES ( CAST(:itype AS text), CAST(:region AS text), CAST(:d AS date), - CAST(:v AS numeric), + 'unknown', CAST(:v AS numeric), 'rosstat', 'monthly', CAST(:unit AS text), CAST(:comment AS text) ) - ON CONFLICT (indicator_type, region, obs_date) DO UPDATE SET + ON CONFLICT (indicator_type, region, obs_date, period_type) DO UPDATE SET value = EXCLUDED.value, source = EXCLUDED.source, frequency = EXCLUDED.frequency, @@ -185,9 +190,15 @@ def _upsert_monthly_rows(db: Session, rows: list[MacroRow]) -> int: def _upsert_emiss_rows(db: Session, rows: list[EmissRow]) -> int: - """Апсертит EmissRow в macro_indicator (source='emiss', frequency per-row). + """Апсертит EmissRow в macro_indicator (source='emiss', frequency+period_type per-row). + SAVEPOINT per-row, чтобы один битый ряд не откатывал всю транзакцию. Возвращает - число успешных upsert'ов.""" + число успешных upsert'ов. + + period_type (из r.period_type) — часть нового PK (migration 163): позволяет + годовому агрегату ('year') и Q1 ('quarter') за один год коexist в таблице без + взаимной перезаписи (#1606). + """ upserted = 0 for r in rows: try: @@ -198,6 +209,7 @@ def _upsert_emiss_rows(db: Session, rows: list[EmissRow]) -> int: "itype": r.indicator_type, "region": r.region, "d": r.obs_date, + "period_type": r.period_type, "v": r.value, "freq": r.frequency, "unit": r.unit, @@ -207,10 +219,11 @@ def _upsert_emiss_rows(db: Session, rows: list[EmissRow]) -> int: upserted += 1 except Exception as e: logger.warning( - "upsert emiss %s/%s@%s=%s failed: %s", + "upsert emiss %s/%s@%s[%s]=%s failed: %s", r.indicator_type, r.region, r.obs_date, + r.period_type, r.value, e, ) diff --git a/backend/tests/scrapers/test_emiss_sdmx.py b/backend/tests/scrapers/test_emiss_sdmx.py index ac6947a6..8b356e1d 100644 --- a/backend/tests/scrapers/test_emiss_sdmx.py +++ b/backend/tests/scrapers/test_emiss_sdmx.py @@ -108,19 +108,30 @@ def test_yearly_and_q1_both_survive_dedup() -> None: assert Decimal("42000") in values, "Q1-наблюдение потеряно" # obs_date у обоих одинаковый (это нормально — коллизия теперь на стороне DB-upsert) assert all(r.obs_date == date(2023, 1, 1) for r in rows) + # period_type: годовой → 'year', Q1 → 'quarter' (часть PK в macro_indicator, migration 163) + period_types = {r.period_type for r in rows} + assert period_types == { + "year", + "quarter", + }, f"ожидались period_type year+quarter, получили {period_types}" # ── real-fixture extraction (income id=57039) ───────────────────────────────────── def test_income_extracts_sverdlovsk_only() -> None: - """Из реальной SDMX-выгрузки извлекается ТОЛЬКО Свердл (ОКАТО 65) — РФ/ЦФО/Адыгея нет.""" + """Из реальной SDMX-выгрузки извлекается ТОЛЬКО Свердл (ОКАТО 65) — РФ/ЦФО/Адыгея нет. + Все строки квартальные → period_type='quarter' (часть PK macro_indicator, migration 163). + """ rows = parse_emiss_sdmx(_load("emiss_income_57039.xml"), INCOME_PER_CAPITA_SPEC) assert rows, "ожидались строки по Свердл" assert {r.region for r in rows} == {"sverdl"} assert all(r.indicator_type == "income_per_capita" for r in rows) assert all(r.unit == "руб" for r in rows) assert all(r.frequency == "quarterly" for r in rows) + assert all( + r.period_type == "quarter" for r in rows + ), "квартальные строки должны иметь period_type='quarter'" def test_income_concrete_values_and_dates() -> None: diff --git a/backend/tests/workers/test_cbr_macro_sync.py b/backend/tests/workers/test_cbr_macro_sync.py index 2b09c51f..5b89376d 100644 --- a/backend/tests/workers/test_cbr_macro_sync.py +++ b/backend/tests/workers/test_cbr_macro_sync.py @@ -48,14 +48,16 @@ def test_upsert_sql_contract() -> None: assert "'rf'" in sql assert "'cbr'" in sql assert "'daily'" in sql - # ON CONFLICT по полному PK + обновление value/updated_at - assert "ON CONFLICT (INDICATOR_TYPE, REGION, OBS_DATE) DO UPDATE" in upper + # ON CONFLICT по новому PK включает period_type (migration 163) + assert "ON CONFLICT (INDICATOR_TYPE, REGION, OBS_DATE, PERIOD_TYPE) DO UPDATE" in upper assert "VALUE = EXCLUDED.VALUE" in upper assert "UPDATED_AT = NOW()" in upper # psycopg v3: CAST(:x AS type), НИКОГДА :x::type assert "CAST(:D AS DATE)" in upper assert "CAST(:V AS NUMERIC)" in upper assert "::" not in sql + # CBR-строки используют period_type='unknown' (литерал в SQL) + assert "'unknown'" in sql def test_task_upserts_each_row_with_correct_params() -> None: @@ -192,13 +194,14 @@ def test_upsert_inflation_sql_contract() -> None: assert "'rf'" in sql assert "'cbr'" in sql assert "'monthly'" in sql - assert "ON CONFLICT (INDICATOR_TYPE, REGION, OBS_DATE) DO UPDATE" in upper + assert "ON CONFLICT (INDICATOR_TYPE, REGION, OBS_DATE, PERIOD_TYPE) DO UPDATE" in upper assert "VALUE = EXCLUDED.VALUE" in upper assert "UPDATED_AT = NOW()" in upper # psycopg v3: CAST(:x AS type), НИКОГДА :x::type assert "CAST(:D AS DATE)" in upper assert "CAST(:V AS NUMERIC)" in upper assert "::" not in sql + assert "'unknown'" in sql def test_task_registered_in_beat_weekly() -> None: diff --git a/backend/tests/workers/test_rosstat_macro_sync.py b/backend/tests/workers/test_rosstat_macro_sync.py index 267239c5..c9ca8c35 100644 --- a/backend/tests/workers/test_rosstat_macro_sync.py +++ b/backend/tests/workers/test_rosstat_macro_sync.py @@ -35,7 +35,7 @@ def _make_mock_db() -> tuple[MagicMock, list[dict[str, Any]]]: def test_upsert_sql_contract() -> None: """UPSERT_ROSSTAT_SQL содержит обязательные литералы контракта macro_indicator - и НЕ использует :x::type (psycopg v3).""" + и НЕ использует :x::type (psycopg v3). PK включает period_type (migration 163).""" from app.workers.tasks.rosstat_macro_sync import UPSERT_ROSSTAT_SQL sql = str(UPSERT_ROSSTAT_SQL) @@ -44,7 +44,7 @@ def test_upsert_sql_contract() -> None: assert "INTO MACRO_INDICATOR" in upper assert "'rosstat'" in sql assert "'yearly'" in sql - assert "ON CONFLICT (INDICATOR_TYPE, REGION, OBS_DATE) DO UPDATE" in upper + assert "ON CONFLICT (INDICATOR_TYPE, REGION, OBS_DATE, PERIOD_TYPE) DO UPDATE" in upper assert "VALUE = EXCLUDED.VALUE" in upper assert "UPDATED_AT = NOW()" in upper # psycopg v3: CAST(:x AS type), НИКОГДА :x::type @@ -52,6 +52,8 @@ def test_upsert_sql_contract() -> None: assert "CAST(:D AS DATE)" in upper assert "CAST(:V AS NUMERIC)" in upper assert "::" not in sql + # не-ЕМИСС строки используют period_type='unknown' (литерал в SQL) + assert "'unknown'" in sql def test_task_upserts_each_row_with_correct_params() -> None: @@ -155,15 +157,24 @@ def test_task_emiss_failure_does_not_block_opendata() -> None: def test_task_upserts_emiss_rows_with_correct_params() -> None: - """Каждая EmissRow → execute UPSERT_EMISS_SQL с itype/region/d/v/freq/unit/comment.""" + """Каждая EmissRow → execute UPSERT_EMISS_SQL с itype/region/d/period_type/v/freq/unit/comment. + + period_type передаётся из r.period_type (часть нового PK, migration 163). + """ from app.services.scrapers.rosstat_emiss import EmissRow from app.workers.tasks import rosstat_macro_sync as task_mod db, captured = _make_mock_db() emiss_rows = [ EmissRow( - "income_per_capita", "sverdl", date(2024, 1, 1), - Decimal("54006"), "руб", "quarterly", "c", + "income_per_capita", + "sverdl", + date(2024, 1, 1), + Decimal("54006"), + "руб", + "quarterly", + "c", + period_type="quarter", ), ] @@ -181,6 +192,7 @@ def test_task_upserts_emiss_rows_with_correct_params() -> None: "itype": "income_per_capita", "region": "sverdl", "d": date(2024, 1, 1), + "period_type": "quarter", "v": Decimal("54006"), "freq": "quarterly", "unit": "руб", @@ -190,7 +202,8 @@ def test_task_upserts_emiss_rows_with_correct_params() -> None: def test_upsert_emiss_sql_contract() -> None: - """UPSERT_EMISS_SQL: source='emiss', frequency параметризован, ON CONFLICT, CAST not ::.""" + """UPSERT_EMISS_SQL: source='emiss', frequency+period_type параметризованы, + ON CONFLICT включает period_type (migration 163), CAST not ::.""" from app.workers.tasks.rosstat_macro_sync import UPSERT_EMISS_SQL sql = str(UPSERT_EMISS_SQL) @@ -198,14 +211,16 @@ def test_upsert_emiss_sql_contract() -> None: assert "INTO MACRO_INDICATOR" in upper assert "'emiss'" in sql assert "CAST(:FREQ AS TEXT)" in upper - assert "ON CONFLICT (INDICATOR_TYPE, REGION, OBS_DATE) DO UPDATE" in upper + assert "CAST(:PERIOD_TYPE AS TEXT)" in upper + assert "ON CONFLICT (INDICATOR_TYPE, REGION, OBS_DATE, PERIOD_TYPE) DO UPDATE" in upper assert "CAST(:ITYPE AS TEXT)" in upper assert "CAST(:V AS NUMERIC)" in upper assert "::" not in sql def test_upsert_rosstat_monthly_sql_contract() -> None: - """UPSERT_ROSSTAT_MONTHLY_SQL: source='rosstat', frequency='monthly', CAST not ::.""" + """UPSERT_ROSSTAT_MONTHLY_SQL: source='rosstat', frequency='monthly', period_type='unknown', + ON CONFLICT включает period_type (migration 163), CAST not ::.""" from app.workers.tasks.rosstat_macro_sync import UPSERT_ROSSTAT_MONTHLY_SQL sql = str(UPSERT_ROSSTAT_MONTHLY_SQL) @@ -213,11 +228,12 @@ def test_upsert_rosstat_monthly_sql_contract() -> None: assert "INTO MACRO_INDICATOR" in upper assert "'rosstat'" in sql assert "'monthly'" in sql - assert "ON CONFLICT (INDICATOR_TYPE, REGION, OBS_DATE) DO UPDATE" in upper + assert "ON CONFLICT (INDICATOR_TYPE, REGION, OBS_DATE, PERIOD_TYPE) DO UPDATE" in upper assert "CAST(:ITYPE AS TEXT)" in upper assert "CAST(:D AS DATE)" in upper assert "CAST(:V AS NUMERIC)" in upper assert "::" not in sql + assert "'unknown'" in sql def test_task_upserts_construction_rows_with_correct_params() -> None: @@ -228,8 +244,12 @@ def test_task_upserts_construction_rows_with_correct_params() -> None: db, captured = _make_mock_db() constr_rows = [ MacroRow( - "construction_price_index", "rf", date(2025, 1, 1), - Decimal("100.8"), "%", "smr", + "construction_price_index", + "rf", + date(2025, 1, 1), + Decimal("100.8"), + "%", + "smr", ), ] diff --git a/data/sql/163_emiss_pk_period_type.sql b/data/sql/163_emiss_pk_period_type.sql new file mode 100644 index 00000000..40e6a2ef --- /dev/null +++ b/data/sql/163_emiss_pk_period_type.sql @@ -0,0 +1,86 @@ +-- 163_emiss_pk_period_type.sql +-- #1606 follow-up: расширить PK macro_indicator колонкой period_type, чтобы годовой +-- агрегат ('год' → 'year') и Q1 ('I квартал' → 'quarter') за один год не перезаписывали +-- друг друга при ON CONFLICT DO UPDATE (оба дают obs_date=YYYY-01-01). +-- +-- Контекст: +-- PR #1687 (#1606) исправил in-memory дедупликацию в parse_emiss_sdmx: ключ дедупа +-- стал трёхкомпонентным (region, obs_date, granularity). Но PK таблицы остался +-- (indicator_type, region, obs_date) — при апсерте в БД коллизия сохранялась. +-- Эта миграция фиксирует PK на стороне БД. +-- +-- Что делает: +-- 1. Добавляет колонку period_type TEXT NOT NULL DEFAULT 'unknown'. +-- Все существующие строки (CBR, rosstat open-data, domrf) получают 'unknown' — +-- это корректно: у них нет sub-period granularity (они уже различаются по obs_date). +-- 2. Бэкфилл: строки source='emiss' обновляем до 'quarterly' (единственный +-- активный ЕМИСС-индикатор — income_per_capita, он квартальный; годовых EMISS- +-- строк ещё нет в таблице до этой миграции). +-- 3. Пересоздаёт PRIMARY KEY: старый (indicator_type, region, obs_date) → новый +-- (indicator_type, region, obs_date, period_type). +-- 4. Пересоздаёт сопутствующий индекс idx_macro_type_region_date с тем же составом +-- (расширяется автоматически через новый PK-индекс; отдельный индекс DROP+CREATE). +-- +-- Зависимости: macro_indicator (migration 123_macro_indicator.sql). +-- Нет VIEW/FK зависимостей от macro_indicator PK (проверено через pg_depend). +-- Idempotent: ADD COLUMN IF NOT EXISTS + IF NOT EXISTS в CREATE INDEX. Повторный +-- прогон безопасен: DROP CONSTRAINT / ADD CONSTRAINT могут упасть если PK уже +-- переименован — обёрнуты в DO $$ BEGIN ... EXCEPTION WHEN ... END $$; блоки. +-- +-- Deploy order: SQL migration применяется ПЕРВОЙ (auto-deploy.yml), затем деплоится +-- backend со обновлёнными UPSERT_EMISS_SQL / UPSERT_ROSSTAT_SQL / UPSERT_KEY_RATE_SQL +-- / UPSERT_INFLATION_SQL / UPSERT_ROSSTAT_MONTHLY_SQL (они несут новый конфликт-таргет +-- + period_type в INSERT). + +BEGIN; + +-- ── 1. Добавить колонку period_type (DEFAULT 'unknown' для всех существующих строк) ── +ALTER TABLE macro_indicator + ADD COLUMN IF NOT EXISTS period_type TEXT NOT NULL DEFAULT 'unknown'; + +-- ── 2. Бэкфилл ЕМИСС-строк → 'quarterly' ──────────────────────────────────────────── +-- Все существующие ЕМИСС-строки — quarterly (income_per_capita). Если появятся 'year'- +-- строки (будущие индикаторы) — они придут с корректным period_type из нового скрейпера. +UPDATE macro_indicator + SET period_type = 'quarterly' + WHERE source = 'emiss' + AND period_type = 'unknown'; + +-- ── 3. Пересоздать PRIMARY KEY ──────────────────────────────────────────────────────── +-- PostgreSQL не поддерживает ALTER PRIMARY KEY напрямую — нужно DROP + ADD. +-- Ловим случай когда старый PK уже дропнут (повторный прогон). +DO $$ +BEGIN + ALTER TABLE macro_indicator DROP CONSTRAINT macro_indicator_pkey; +EXCEPTION + WHEN undefined_object THEN + NULL; -- PK уже удалён (повторный прогон) + WHEN feature_not_supported THEN + NULL; +END; +$$; + +-- Новый PK включает period_type. Если уже существует с правильным составом — +-- ADD CONSTRAINT бросит duplicate_table (конвенция Pg: PK → unique index), ловим. +DO $$ +BEGIN + ALTER TABLE macro_indicator + ADD CONSTRAINT macro_indicator_pkey + PRIMARY KEY (indicator_type, region, obs_date, period_type); +EXCEPTION + WHEN duplicate_table THEN + NULL; -- PK уже создан (повторный прогон) + WHEN invalid_table_definition THEN + NULL; -- уже существует тот же constraint +END; +$$; + +-- ── 4. Пересоздать сопутствующий индекс ────────────────────────────────────────────── +-- Старый индекс (без period_type в составе) уже покрыт новым PK-индексом по факту, +-- но явно дропаем и пересоздаём с period_type для консистентности плана запросов. +DROP INDEX IF EXISTS idx_macro_type_region_date; + +CREATE INDEX IF NOT EXISTS idx_macro_type_region_date + ON macro_indicator (indicator_type, region, obs_date DESC, period_type); + +COMMIT; From b64334d7bd871693a6c7e87b79d2d458f7a8dcc2 Mon Sep 17 00:00:00 2001 From: bot-backend Date: Wed, 17 Jun 2026 21:43:01 +0300 Subject: [PATCH 04/15] feat(tradein): segment=vtorichka|novostroyki|all query-param on listings (#1188) --- tradein-mvp/backend/app/schemas/search.py | 21 +++++++++-- .../backend/app/services/search_query.py | 16 +++++++++ tradein-mvp/backend/tests/test_search_api.py | 36 +++++++++++++++++++ 3 files changed, 70 insertions(+), 3 deletions(-) diff --git a/tradein-mvp/backend/app/schemas/search.py b/tradein-mvp/backend/app/schemas/search.py index bf797ab0..b1ecad9a 100644 --- a/tradein-mvp/backend/app/schemas/search.py +++ b/tradein-mvp/backend/app/schemas/search.py @@ -6,9 +6,11 @@ from typing import Literal from pydantic import BaseModel, Field, model_validator -SortKey = Literal[ - "price_asc", "price_desc", "area_desc", "area_asc", "date_desc", "dist_asc" -] +SortKey = Literal["price_asc", "price_desc", "area_desc", "area_asc", "date_desc", "dist_asc"] + +# Сегмент рынка (#1188, поверх canon-предиката #1186). +# NULL listing_segment (legacy вторичка до миграции 011) трактуется как 'vtorichka'. +SegmentKey = Literal["vtorichka", "novostroyki", "all"] class SearchParams(BaseModel): @@ -49,6 +51,19 @@ class SearchParams(BaseModel): address_query: str | None = Field(default=None, max_length=200) description_query: str | None = Field(default=None, max_length=200) + # --- Market segment (#1188) --- + segment: SegmentKey = Field( + default="vtorichka", + description=( + "Сегмент рынка для фильтрации listings. " + "`vtorichka` (по умолчанию, back-compat) — вторичка; " + "строки с listing_segment=NULL (legacy до миграции 011) " + "считаются вторичкой согласно canon-предикату #1186. " + "`novostroyki` — только первичка. " + "`all` — без фильтра по сегменту." + ), + ) + # --- Sort + pagination --- sort: SortKey = "date_desc" page: int = Field(default=1, ge=1, le=1000) diff --git a/tradein-mvp/backend/app/services/search_query.py b/tradein-mvp/backend/app/services/search_query.py index e0687876..b7d06ed7 100644 --- a/tradein-mvp/backend/app/services/search_query.py +++ b/tradein-mvp/backend/app/services/search_query.py @@ -17,6 +17,18 @@ _SORT_SQL: dict[str, str] = { ), } +# Сегмент рынка (#1188). listings_search_mv не несёт колонку listing_segment, +# поэтому фильтруем через подзапрос к базовой таблице listings, переиспользуя +# canon-предикат #1186: NULL = legacy вторичка до миграции 011. +_VTORICHKA_GUARD = "(listing_segment IS NULL OR listing_segment = 'vtorichka')" +_SEGMENT_SQL: dict[str, str | None] = { + "vtorichka": (f"listing_id IN (SELECT id FROM listings WHERE {_VTORICHKA_GUARD})"), + "novostroyki": ( + "listing_id IN (SELECT id FROM listings WHERE listing_segment = 'novostroyki')" + ), + "all": None, +} + def build_search_query(params: SearchParams) -> tuple[str, dict[str, object]]: """Возвращает (sql, args) для SELECT из listings_search_mv.""" @@ -86,6 +98,10 @@ def build_search_query(params: SearchParams) -> tuple[str, dict[str, object]]: if params.has_kadastr: where.append("cadastral_number IS NOT NULL") + segment_clause = _SEGMENT_SQL[params.segment] + if segment_clause is not None: + where.append(segment_clause) + if params.sources: where.append("sources && CAST(:sources AS text[])") args["sources"] = params.sources diff --git a/tradein-mvp/backend/tests/test_search_api.py b/tradein-mvp/backend/tests/test_search_api.py index 0e0512aa..0a7fc383 100644 --- a/tradein-mvp/backend/tests/test_search_api.py +++ b/tradein-mvp/backend/tests/test_search_api.py @@ -62,6 +62,42 @@ def test_build_query_sources_array(): assert args["sources"] == ["avito", "cian"] +def test_build_query_segment_default_vtorichka(): + """Без параметра segment — back-compat: canon-предикат вторички (#1186/#1188).""" + sql, _ = build_search_query(SearchParams()) + assert SearchParams().segment == "vtorichka" + assert "listing_id IN (SELECT id FROM listings WHERE" in sql + assert "(listing_segment IS NULL OR listing_segment = 'vtorichka')" in sql + + +def test_build_query_segment_novostroyki(): + """segment=novostroyki — только первичка, без NULL-вторички.""" + sql, _ = build_search_query(SearchParams(segment="novostroyki")) + assert "listing_segment = 'novostroyki'" in sql + # Не должен попасть canon-guard вторички. + assert "(listing_segment IS NULL OR listing_segment = 'vtorichka')" not in sql + + +def test_build_query_segment_all_no_filter(): + """segment=all — без фильтрации по сегменту.""" + sql, _ = build_search_query(SearchParams(segment="all")) + assert "listing_segment" not in sql + + +def test_build_count_query_inherits_segment(): + """COUNT-запрос наследует segment-фильтр из build_search_query.""" + sql, _ = build_count_query(SearchParams(segment="novostroyki")) + assert "listing_segment = 'novostroyki'" in sql + sql_v, _ = build_count_query(SearchParams()) + assert "(listing_segment IS NULL OR listing_segment = 'vtorichka')" in sql_v + + +def test_segment_invalid_rejected(): + """Невалидное значение segment отклоняется валидацией.""" + with pytest.raises(ValueError): + SearchParams(segment="kommercia") + + def test_build_count_query_strips_limit(): sql, args = build_count_query(SearchParams(rooms=2, page=3)) assert "count(*)" in sql From 26596003742e4280708a0c9d1af170420fc807dd Mon Sep 17 00:00:00 2001 From: bot-backend Date: Wed, 17 Jun 2026 21:45:12 +0300 Subject: [PATCH 05/15] fix(sql): emiss backfill period_type='quarter' to match scraper (#1606 review) --- data/sql/163_emiss_pk_period_type.sql | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/data/sql/163_emiss_pk_period_type.sql b/data/sql/163_emiss_pk_period_type.sql index 40e6a2ef..2aa64851 100644 --- a/data/sql/163_emiss_pk_period_type.sql +++ b/data/sql/163_emiss_pk_period_type.sql @@ -13,9 +13,11 @@ -- 1. Добавляет колонку period_type TEXT NOT NULL DEFAULT 'unknown'. -- Все существующие строки (CBR, rosstat open-data, domrf) получают 'unknown' — -- это корректно: у них нет sub-period granularity (они уже различаются по obs_date). --- 2. Бэкфилл: строки source='emiss' обновляем до 'quarterly' (единственный +-- 2. Бэкфилл: строки source='emiss' обновляем до 'quarter' (единственный -- активный ЕМИСС-индикатор — income_per_capita, он квартальный; годовых EMISS- --- строк ещё нет в таблице до этой миграции). +-- строк ещё нет в таблице до этой миграции). Значение ДОЛЖНО совпадать с тем, +-- что пишет UPSERT_EMISS_SQL (_emiss_period_granularity → 'quarter'/'year'/'month', +-- короткая форма), иначе ON CONFLICT не сматчит → дубли при следующем скрейпе. -- 3. Пересоздаёт PRIMARY KEY: старый (indicator_type, region, obs_date) → новый -- (indicator_type, region, obs_date, period_type). -- 4. Пересоздаёт сопутствующий индекс idx_macro_type_region_date с тем же составом @@ -38,11 +40,15 @@ BEGIN; ALTER TABLE macro_indicator ADD COLUMN IF NOT EXISTS period_type TEXT NOT NULL DEFAULT 'unknown'; --- ── 2. Бэкфилл ЕМИСС-строк → 'quarterly' ──────────────────────────────────────────── --- Все существующие ЕМИСС-строки — quarterly (income_per_capita). Если появятся 'year'- --- строки (будущие индикаторы) — они придут с корректным period_type из нового скрейпера. +-- ── 2. Бэкфилл ЕМИСС-строк → 'quarter' ──────────────────────────────────────────── +-- Все существующие ЕМИСС-строки — квартальные (income_per_capita). Значение 'quarter' +-- (короткая форма) обязано совпадать с UPSERT_EMISS_SQL, который параметризует +-- period_type из _emiss_period_granularity ('quarter'/'year'/'month'). Иначе после +-- деплоя следующий скрейп вставит 'quarter'-строки, не матчащие 'quarterly' на +-- ON CONFLICT → дубли для каждого квартального наблюдения. Если появятся 'year'-строки +-- (будущие индикаторы) — придут с корректным period_type из скрейпера. UPDATE macro_indicator - SET period_type = 'quarterly' + SET period_type = 'quarter' WHERE source = 'emiss' AND period_type = 'unknown'; @@ -60,16 +66,16 @@ EXCEPTION END; $$; --- Новый PK включает period_type. Если уже существует с правильным составом — --- ADD CONSTRAINT бросит duplicate_table (конвенция Pg: PK → unique index), ловим. +-- Новый PK включает period_type. Если constraint с этим именем уже существует — +-- ADD CONSTRAINT бросит duplicate_object (42710, дубль имени constraint), ловим. DO $$ BEGIN ALTER TABLE macro_indicator ADD CONSTRAINT macro_indicator_pkey PRIMARY KEY (indicator_type, region, obs_date, period_type); EXCEPTION - WHEN duplicate_table THEN - NULL; -- PK уже создан (повторный прогон) + WHEN duplicate_object THEN + NULL; -- constraint с таким именем уже создан (повторный прогон) WHEN invalid_table_definition THEN NULL; -- уже существует тот же constraint END; From 96a9c575b44d9566f54711dd2d4d10f906891939 Mon Sep 17 00:00:00 2001 From: bot-backend Date: Wed, 17 Jun 2026 21:48:11 +0300 Subject: [PATCH 06/15] feat(sql): sales-tracker velocity + absorption MVs for Site Finder (#61) --- .../site_finder/sales_tracker_mv_refresh.py | 95 +++++++++ backend/app/workers/beat_schedule.py | 17 ++ backend/app/workers/celery_app.py | 1 + .../workers/tasks/mv_sales_tracker_refresh.py | 52 +++++ ...1_mv_sales_tracker_velocity_absorption.sql | 184 ++++++++++++++++++ 5 files changed, 349 insertions(+) create mode 100644 backend/app/services/site_finder/sales_tracker_mv_refresh.py create mode 100644 backend/app/workers/tasks/mv_sales_tracker_refresh.py create mode 100644 data/sql/161_mv_sales_tracker_velocity_absorption.sql diff --git a/backend/app/services/site_finder/sales_tracker_mv_refresh.py b/backend/app/services/site_finder/sales_tracker_mv_refresh.py new file mode 100644 index 00000000..fa2afed1 --- /dev/null +++ b/backend/app/services/site_finder/sales_tracker_mv_refresh.py @@ -0,0 +1,95 @@ +"""Refresh helper for the sales-tracker MVs (Issue #61). + +Two independent materialized views built from the Объектив sales-tracker +("шахматки") snapshots (objective_lots / objective_lots_history), created by +data/sql/161_mv_sales_tracker_velocity_absorption.sql: + + 1. mv_sales_tracker_velocity_by_district — per (district, month) sold/total/ + avg-sold-price. Feeds the Site Finder Velocity Score (4th scoring criterion). + 2. mv_sales_tracker_absorption_curves — cumulative sold% as f(months from + sales_start_date) per (rooms_int, area_bucket). Foundation for recommend_mix + + sellout forecast. + +The two MVs do not depend on each other, so refresh order is irrelevant; both +are refreshed in the same call. + +Scheduled via Celery beat hardcoded entry in workers/beat_schedule.py +('mv-sales-tracker-refresh-weekly', Mon 04:30 MSK). + +Usage example (manual, via psql-connected shell or admin endpoint): + from app.services.site_finder.sales_tracker_mv_refresh import refresh_sales_tracker_mvs + + counts = refresh_sales_tracker_mvs(db) + # logs: "mv_sales_tracker_velocity_by_district refreshed: 70 rows", etc. +""" + +from __future__ import annotations + +import logging + +from sqlalchemy import text +from sqlalchemy.exc import DatabaseError +from sqlalchemy.orm import Session + +logger = logging.getLogger(__name__) + +_MV_NAMES: tuple[str, ...] = ( + "mv_sales_tracker_velocity_by_district", + "mv_sales_tracker_absorption_curves", +) + + +def _refresh_mv(db: Session, mv_name: str, *, concurrently: bool) -> int: + """Run REFRESH MATERIALIZED VIEW [CONCURRENTLY] , return row count. + + Falls back to non-concurrent on the known "cannot refresh concurrently" + error (MV empty or no UNIQUE index — should not happen in prod since the + migration creates the UNIQUE index and populates the MV, but provides a + safe recovery path for first-run / post-recreation edge cases). + """ + try: + if concurrently: + db.execute(text(f"REFRESH MATERIALIZED VIEW CONCURRENTLY {mv_name}")) + else: + db.execute(text(f"REFRESH MATERIALIZED VIEW {mv_name}")) + db.commit() + except DatabaseError as e: + # PostgreSQL emits "CONCURRENTLY cannot be used when the materialized + # view ... is not populated" (matview.c, SQLSTATE 55000), surfaced by + # psycopg3 as an InternalError (a DatabaseError sibling). + if concurrently and "concurrently cannot be used" in str(e).lower(): + logger.warning( + "%s: CONCURRENTLY failed (MV likely not populated), " + "falling back to non-concurrent refresh", + mv_name, + ) + db.rollback() + db.execute(text(f"REFRESH MATERIALIZED VIEW {mv_name}")) + db.commit() + else: + raise + + row = db.execute(text(f"SELECT COUNT(*) FROM {mv_name}")).first() + count = int(row[0]) if row else 0 + logger.info("%s refreshed: %d rows", mv_name, count) + return count + + +def refresh_sales_tracker_mvs(db: Session, *, concurrently: bool = True) -> dict[str, int]: + """Refresh both sales-tracker MVs. + + Args: + db: SQLAlchemy Session (sync). + concurrently: When True, uses REFRESH CONCURRENTLY (non-blocking — + readers continue). Requires the per-MV UNIQUE indexes + (mv_sales_tracker_velocity_by_district_pk, + mv_sales_tracker_absorption_curves_pk) and the MVs to be already + populated. Pass False only for first populate or after recreation. + + Returns: + Mapping mv_name -> row count after refresh (for observability). + """ + counts: dict[str, int] = {} + for mv_name in _MV_NAMES: + counts[mv_name] = _refresh_mv(db, mv_name, concurrently=concurrently) + return counts diff --git a/backend/app/workers/beat_schedule.py b/backend/app/workers/beat_schedule.py index 0252d9b7..104c6a5c 100644 --- a/backend/app/workers/beat_schedule.py +++ b/backend/app/workers/beat_schedule.py @@ -522,4 +522,21 @@ def build_beat_schedule() -> dict: "options": {"queue": "celery"}, } + # Sales-tracker MVs (#61) — питают Site Finder Velocity Score (4-й критерий) + + # recommend_mix / sellout-forecast. Оба MV (mv_sales_tracker_velocity_by_district, + # mv_sales_tracker_absorption_curves) рефрешатся CONCURRENTLY (non-blocking, требуют + # unique-индексы из миграции 161). Источник — objective_lots / objective_lots_history + # (Объектив-шахматки), наполняются objective_sync (Mon 04:15 МСК по умолчанию). + # + # Понедельник 04:30 МСК (Celery conf.timezone=Europe/Moscow → crontab в МСК, #1233) — + # ПОСЛЕ objective_sync (04:15), чтобы агрегаты считались по свежему снапшоту; в + # окне до тяжёлого monday-кластера site_finder-рефрешей (ird 05:00, gknspecial 05:30, + # supply-layers 06:00). Refresh лёгкий (~6с на 1.1M lots). Техническая infra-задача, + # не в job_settings (как refresh-quarter-price-index / refresh-layout-velocity). + schedule["mv-sales-tracker-refresh-weekly"] = { + "task": "tasks.mv_sales_tracker_refresh.refresh_sales_tracker_mvs", + "schedule": _parse_cron("30 4 * * mon"), # 04:30 MSK, понедельник + "options": {"queue": "celery"}, + } + return schedule diff --git a/backend/app/workers/celery_app.py b/backend/app/workers/celery_app.py index cec38d12..86fa867c 100644 --- a/backend/app/workers/celery_app.py +++ b/backend/app/workers/celery_app.py @@ -82,6 +82,7 @@ celery_app = Celery( "app.workers.tasks.izyatie_ocr_ingest", "app.workers.tasks.developer_registry_refresh", "app.workers.tasks.refresh_layout_velocity", + "app.workers.tasks.mv_sales_tracker_refresh", ], ) celery_app.conf.timezone = "Europe/Moscow" diff --git a/backend/app/workers/tasks/mv_sales_tracker_refresh.py b/backend/app/workers/tasks/mv_sales_tracker_refresh.py new file mode 100644 index 00000000..a38790be --- /dev/null +++ b/backend/app/workers/tasks/mv_sales_tracker_refresh.py @@ -0,0 +1,52 @@ +"""Celery task: refresh the sales-tracker MVs (Issue #61). + +Scheduled via hardcoded beat entry in workers/beat_schedule.py: + 'mv-sales-tracker-refresh-weekly' — weekly on Monday at 04:30 MSK. + +Refreshes (both CONCURRENTLY, non-blocking): + - mv_sales_tracker_velocity_by_district (Site Finder Velocity Score, 4th criterion) + - mv_sales_tracker_absorption_curves (recommend_mix + sellout forecast foundation) + +Both MVs are built from the Объектив sales-tracker ("шахматки") snapshots +(objective_lots / objective_lots_history). Source data refreshes via the +objective_sync beat job, so a weekly MV refresh keeps the aggregates current. + +MV-source migration: data/sql/161_mv_sales_tracker_velocity_absorption.sql. +""" + +from __future__ import annotations + +import logging +from typing import Any + +from app.core.db import SessionLocal +from app.services.site_finder.sales_tracker_mv_refresh import refresh_sales_tracker_mvs +from app.workers.celery_app import celery_app + +logger = logging.getLogger(__name__) + + +@celery_app.task( + bind=True, + name="tasks.mv_sales_tracker_refresh.refresh_sales_tracker_mvs", + max_retries=2, +) +def refresh_sales_tracker_mvs_task(self: Any) -> dict[str, Any]: + """REFRESH both sales-tracker MVs (#61). + + Both MVs are refreshed CONCURRENTLY (non-blocking, require their UNIQUE + indexes created by migration 161); the service falls back to non-concurrent + if an MV is found unpopulated (first-run edge case). + + Returns result dict for the Celery task result store / logging. + """ + db = SessionLocal() + try: + counts = refresh_sales_tracker_mvs(db, concurrently=True) + logger.info("refresh_sales_tracker_mvs: completed, rows=%s", counts) + return {"status": "ok", "rows": counts} + except Exception as e: + logger.exception("refresh_sales_tracker_mvs failed: %s", e) + raise + finally: + db.close() diff --git a/data/sql/161_mv_sales_tracker_velocity_absorption.sql b/data/sql/161_mv_sales_tracker_velocity_absorption.sql new file mode 100644 index 00000000..3ffeea5e --- /dev/null +++ b/data/sql/161_mv_sales_tracker_velocity_absorption.sql @@ -0,0 +1,184 @@ +-- 161_mv_sales_tracker_velocity_absorption.sql +-- Issue #61 — Velocity materialized views for Site Finder Velocity Score (4th scoring +-- criterion) + recommend_mix smart unit-mix. Foundation for sellout forecast. +-- +-- B2-1 data source ("шахматки" / sales-tracker): the Объектив scraper +-- (backend/app/workers/tasks/scrape_objective.py) → tables: +-- objective_lots — 1.12M rows, one row per tracked lot (current state), +-- carries district / rooms_int / area_pd / sales_start_date / +-- is_sold / registration_date / contract_date / price_per_m2_rub. +-- objective_lots_history — 974k rows, daily-ish per-lot snapshots +-- (snapshot_date, is_sold, status, prices). +-- Snapshot history depth (as of 2026-06-17): 3 captures 2026-05-17 / 05-19 / 06-03 (spans +-- >2 weeks, sold count moved 193188->194893 => measurable absorption). Cohort/absorption +-- resolution improves automatically as the weekly scraper accumulates more snapshots. +-- +-- -- MV 1: mv_sales_tracker_velocity_by_district -------------------------------------- +-- Grain: (district, sale_month). One row per district per month. +-- Dedup: a lot appears in multiple snapshots within a month -> we keep that lot's LATEST +-- snapshot within the month (DISTINCT ON lot, snapshot_date DESC) before +-- aggregating, so total_count is lots-tracked-that-month (not snapshot rows). +-- Metrics: total_count, sold_count, avg_sold_price_per_m2, avg_sold_price_total, +-- sold_share (velocity proxy for SF Velocity Score). +-- +-- -- MV 2: mv_sales_tracker_absorption_curves ---------------------------------------- +-- Grain: (rooms_int, area_bucket, months_since_start). Cumulative sold% as f(months +-- from first_seen). "first_seen" = objective_lots.sales_start_date (true sales +-- launch — richer/longer than the 3-snapshot window). Sold-month anchor = +-- COALESCE(registration_date, contract_date). months_since_start clamped >= 0 +-- (712 noise rows have anchor < start). 99.98% of sold lots carry both dates. +-- cohort_size = all lots in (rooms, area_bucket) cohort; cum_sold = sold lots +-- whose months_since_start <= the row's bucket; cum_sold_pct = cum_sold/cohort. +-- This is snapshot-sparsity-independent (driven by registration dates, not snapshots), +-- so the curve is usable today and the foundation for sellout forecast. +-- +-- REFRESH CONCURRENTLY: both MVs get a UNIQUE index on their full grain immediately after +-- creation (on empty MV -> instant), enabling non-blocking weekly REFRESH CONCURRENTLY. +-- Scheduled via Celery beat `mv-sales-tracker-refresh-weekly` (Mon 04:30 MSK) -> +-- task app.workers.tasks.mv_sales_tracker_refresh.refresh_sales_tracker_mvs. +-- +-- Deploy: auto-applied by deploy.yml via _schema_migrations tracking (one-shot, NN order). +-- Dependencies on existing objects: objective_lots, objective_lots_history (read-only). +-- No views depend on these MVs at creation time. +-- +-- WARN: re-apply (DR / lost _schema_migrations / dev local) DROP ... CASCADE снесёт MV + +-- зависимости. После re-apply ПЕРВЫЙ refresh = non-concurrent (CONCURRENTLY падает +-- на пустой/не-populated MV). _schema_migrations нормально предотвращает re-apply. + +BEGIN; + +-- ==================================================================================== +-- MV 1: velocity by district x month +-- ==================================================================================== +DROP MATERIALIZED VIEW IF EXISTS mv_sales_tracker_velocity_by_district CASCADE; + +CREATE MATERIALIZED VIEW mv_sales_tracker_velocity_by_district AS +WITH lot_month AS ( + -- One row per (lot, month): the lot's latest snapshot within that month. + SELECT DISTINCT ON (h.objective_lot_id, date_trunc('month', h.snapshot_date)) + l.district AS district, + date_trunc('month', h.snapshot_date)::date AS sale_month, + h.objective_lot_id, + h.is_sold, + h.price_per_m2_rub, + h.price_calculated_total_rub + FROM objective_lots_history h + JOIN objective_lots l ON l.objective_lot_id = h.objective_lot_id + WHERE l.district IS NOT NULL + ORDER BY h.objective_lot_id, + date_trunc('month', h.snapshot_date), + h.snapshot_date DESC +) +SELECT + district, + sale_month, + count(*)::int AS total_count, + count(*) FILTER (WHERE is_sold)::int AS sold_count, + round( + count(*) FILTER (WHERE is_sold)::numeric + / NULLIF(count(*), 0), 4 + ) AS sold_share, + round(avg(price_per_m2_rub) FILTER (WHERE is_sold), 2) AS avg_sold_price_per_m2, + round(avg(price_calculated_total_rub) FILTER (WHERE is_sold), 2) AS avg_sold_price_total +FROM lot_month +GROUP BY district, sale_month +WITH NO DATA; + +-- UNIQUE index on full grain -> enables REFRESH CONCURRENTLY (created on empty MV = instant) +CREATE UNIQUE INDEX mv_sales_tracker_velocity_by_district_pk + ON mv_sales_tracker_velocity_by_district (district, sale_month); + +CREATE INDEX mv_sales_tracker_velocity_district_idx + ON mv_sales_tracker_velocity_by_district (district); + +REFRESH MATERIALIZED VIEW mv_sales_tracker_velocity_by_district; + +COMMENT ON MATERIALIZED VIEW mv_sales_tracker_velocity_by_district IS + 'Issue #61. Per (district, month) sold/total/avg-sold-price from objective_lots_history ' + 'snapshots (Obektiv shahmatka), deduped to latest snapshot per lot per month. ' + 'Feeds Site Finder Velocity Score. Refresh weekly CONCURRENTLY.'; + +-- ==================================================================================== +-- MV 2: absorption curves by room_count x area_bucket x months-from-first-seen +-- ==================================================================================== +DROP MATERIALIZED VIEW IF EXISTS mv_sales_tracker_absorption_curves CASCADE; + +CREATE MATERIALIZED VIEW mv_sales_tracker_absorption_curves AS +WITH base AS ( + -- One row per lot. area_bucket from area_pd; months_since_start = whole months between + -- sales_start_date and the sold anchor (reg/contract). Unsold lots have NULL anchor. + SELECT + l.rooms_int, + CASE + WHEN l.area_pd < 30 THEN '<30' + WHEN l.area_pd < 45 THEN '30-45' + WHEN l.area_pd < 60 THEN '45-60' + WHEN l.area_pd < 80 THEN '60-80' + ELSE '80+' + END AS area_bucket, + l.is_sold, + CASE + WHEN l.is_sold + AND l.sales_start_date IS NOT NULL + AND COALESCE(l.registration_date, l.contract_date) IS NOT NULL + THEN GREATEST( + 0, + (date_part('year', age(COALESCE(l.registration_date, l.contract_date), + l.sales_start_date)) * 12 + + date_part('month', age(COALESCE(l.registration_date, l.contract_date), + l.sales_start_date)))::int + ) + END AS months_since_start + FROM objective_lots l + WHERE l.rooms_int IS NOT NULL + AND l.area_pd IS NOT NULL + AND l.sales_start_date IS NOT NULL +), +cohort AS ( + SELECT rooms_int, area_bucket, count(*)::int AS cohort_size + FROM base + GROUP BY rooms_int, area_bucket +), +sold_at_month AS ( + SELECT rooms_int, area_bucket, months_since_start, count(*)::int AS sold_in_month + FROM base + WHERE is_sold AND months_since_start IS NOT NULL + GROUP BY rooms_int, area_bucket, months_since_start +) +SELECT + s.rooms_int, + s.area_bucket, + s.months_since_start, + c.cohort_size, + -- cumulative sold up to and including this month-offset (per cohort) + SUM(s.sold_in_month) OVER ( + PARTITION BY s.rooms_int, s.area_bucket + ORDER BY s.months_since_start + ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW + )::int AS cum_sold, + round( + SUM(s.sold_in_month) OVER ( + PARTITION BY s.rooms_int, s.area_bucket + ORDER BY s.months_since_start + ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW + )::numeric / NULLIF(c.cohort_size, 0), 4 + ) AS cum_sold_pct +FROM sold_at_month s +JOIN cohort c ON c.rooms_int = s.rooms_int AND c.area_bucket = s.area_bucket +WITH NO DATA; + +-- UNIQUE index on full grain -> enables REFRESH CONCURRENTLY +CREATE UNIQUE INDEX mv_sales_tracker_absorption_curves_pk + ON mv_sales_tracker_absorption_curves (rooms_int, area_bucket, months_since_start); + +CREATE INDEX mv_sales_tracker_absorption_cohort_idx + ON mv_sales_tracker_absorption_curves (rooms_int, area_bucket); + +REFRESH MATERIALIZED VIEW mv_sales_tracker_absorption_curves; + +COMMENT ON MATERIALIZED VIEW mv_sales_tracker_absorption_curves IS + 'Issue #61. Cumulative sold-pct as f(months from sales_start_date) per (rooms_int, ' + 'area_bucket). Anchor = COALESCE(registration_date, contract_date) from objective_lots. ' + 'Foundation for recommend_mix + sellout forecast. Refresh weekly CONCURRENTLY.'; + +COMMIT; From 7f732ff24deec4cdb11c22b4046259e497ea4afe Mon Sep 17 00:00:00 2001 From: bot-backend Date: Wed, 17 Jun 2026 22:09:01 +0300 Subject: [PATCH 07/15] ci: drop push-on-feature-branch trigger to kill duplicate CI runs (#1709) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Каждый коммит в ветку с открытым PR триггерил ДВА прогона ci.yml на один SHA: push-событие (github.ref=refs/heads/) и pull_request-событие (github.ref=refs/pull//merge). Разный github.ref → разные concurrency- группы → прогоны не отменяли друг друга → 2x нагрузка на и так дефицитные 2 раннера (CI-шторм 2026-06-17, деплой tradein #1699 простоял 17 мин). В bot-пайплайне каждый коммит идёт через PR, поэтому pull_request гейтит его полностью; push-прогон был чистым дублем. Оставляю только pull_request: теперь github.ref стабилен на весь PR и concurrency cancel-in-progress корректно отменяет superseded-прогоны при новом пуше. Refs #1709 --- .forgejo/workflows/ci.yml | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index e228cbbd..78008735 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -15,23 +15,22 @@ name: CI # FUTURE: добавить `postgis/postgis:16-3.4` service + гонять mv_layout — см. # .github/workflows/ci.yml как образец service-блока. on: + # ТОЛЬКО pull_request — НЕТ push-триггера на feature-ветки (CI-шторм #1709). + # WHY: раньше был и push: [feat/**,fix/**,...]. Каждый коммит в ветку с открытым + # PR триггерил ДВА прогона на ОДИН SHA: push-событие (github.ref=refs/heads/) + # и pull_request-событие (github.ref=refs/pull//merge). Разный github.ref → + # разные concurrency-группы (см. ниже) → прогоны НЕ отменяют друг друга → 2× job + # при и так дефицитных раннерах. В bot-пайплайне каждый коммит идёт через PR, так + # что pull_request гейтит его полностью; push-прогон был чистым дублем. + # Trade-off: push в feature-ветку БЕЗ открытого PR не получит CI до открытия PR + # (бот открывает PR сразу после первого push) — приемлемо. pull_request: branches: [main] - push: - branches: - # Mirror the bot-PR / feature-branch flow в .claude/rules/git-pr.md: - # PR получает gate, прямые пуши в feature-ветки — тоже. - - "feat/**" - - "fix/**" - - "refactor/**" - - "chore/**" - - "docs/**" - - "perf/**" - - "test/**" - - "ci/**" - - "hotfix/**" concurrency: + # Теперь, когда остался только pull_request, github.ref стабилен на весь PR + # (refs/pull//merge) → новый push в ветку PR отменяет предыдущий незавершённый + # прогон ЭТОГО PR (cancel-in-progress) вместо накопления параллельных. group: ci-${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true From d44f78c33e434b4b4124c6af6506743e450d45b7 Mon Sep 17 00:00:00 2001 From: bot-backend Date: Wed, 17 Jun 2026 22:22:02 +0300 Subject: [PATCH 08/15] =?UTF-8?q?fix(frontend):=20regen=20api-types.ts=20?= =?UTF-8?q?=E2=80=94=20add=20POI-score=20fields=20(#1709)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit openapi-codegen-check падал СИСТЕМНО на каждом backend-PR: committed frontend/src/lib/api-types.ts отставал от backend OpenAPI на 2 поля, добавленных в POI-score схемы (PR ~#1486 normalize POI weighted score), но не перегенерённых: - PoiScoreItem.score_contribution - PoiScoreResponse.poi_weighted_score Контент взят 1:1 из authoritative CI-диффа (openapi-typescript + prettier). Возвращает gate openapi-codegen-check в зелёное для всех PR. Refs #1709 --- frontend/src/lib/api-types.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/frontend/src/lib/api-types.ts b/frontend/src/lib/api-types.ts index 9d45a9f7..e3e059ed 100644 --- a/frontend/src/lib/api-types.ts +++ b/frontend/src/lib/api-types.ts @@ -4168,6 +4168,8 @@ export interface components { distance_m: number; /** Weight */ weight: number; + /** Score Contribution */ + score_contribution: number; /** Address */ address: string | null; }; @@ -4177,6 +4179,8 @@ export interface components { cad_num: string; /** Radius M */ radius_m: number; + /** Poi Weighted Score */ + poi_weighted_score: number; /** Top Poi */ top_poi: components["schemas"]["PoiScoreItem"][]; }; From bc4c6852a1daea5135d2a704c999d4d069e9d839 Mon Sep 17 00:00:00 2001 From: bot-backend Date: Wed, 17 Jun 2026 22:22:17 +0300 Subject: [PATCH 09/15] feat(yandex): ingest novostroyki via gate-API newFlat=YES --- tradein-mvp/backend/app/services/scheduler.py | 1 + .../backend/app/services/scrape_pipeline.py | 20 +- .../app/services/scrapers/yandex_realty.py | 241 ++++++++++-------- .../backend/tests/test_yandex_realty_serp.py | 113 ++++++++ 4 files changed, 269 insertions(+), 106 deletions(-) diff --git a/tradein-mvp/backend/app/services/scheduler.py b/tradein-mvp/backend/app/services/scheduler.py index 696cf7a7..0555a7b6 100644 --- a/tradein-mvp/backend/app/services/scheduler.py +++ b/tradein-mvp/backend/app/services/scheduler.py @@ -321,6 +321,7 @@ async def trigger_yandex_city_sweep_run(db: Session, schedule_row: dict[str, Any request_delay_sec=float(params.get("request_delay_sec", 9.0)), radius_m=int(params.get("radius_m", 1500)), enrich_address=bool(params.get("enrich_address", True)), + segments=params.get("segments"), ) except Exception: logger.exception("scheduler: run_yandex_city_sweep crashed run_id=%d", run_id) diff --git a/tradein-mvp/backend/app/services/scrape_pipeline.py b/tradein-mvp/backend/app/services/scrape_pipeline.py index 2d02d49c..793a3101 100644 --- a/tradein-mvp/backend/app/services/scrape_pipeline.py +++ b/tradein-mvp/backend/app/services/scrape_pipeline.py @@ -1033,6 +1033,7 @@ async def run_yandex_city_sweep( enrich_address: bool = True, rooms_list: list[str] | None = None, price_ranges: list[tuple[int | None, int | None]] | None = None, + segments: list[str] | None = None, ) -> YandexCitySweepCounters: """Yandex.Недвижимость city sweep: rooms × price combos от центра ЕКБ → save → address-enrich. @@ -1078,6 +1079,11 @@ async def run_yandex_city_sweep( _rooms_list = rooms_list or list(ROOM_PATH.keys()) _price_ranges = price_ranges or DEFAULT_PRICE_RANGES + # newFlat-сегменты: оба прохода (vtorichka + novostroyki) по одним combos. + # Дефолт ["NO", "YES"] — оба; dedup по source_id span'ит оба прохода (один seen + # внутри fetch_around_multi_room). Counters (lots_fetched/inserted/updated) + # агрегируют оба прохода через len(anchor_lots). + _segments = segments or ["NO", "YES"] counters = YandexCitySweepCounters(anchors_total=len(_anchors)) inter_anchor_delay = request_delay_sec if request_delay_sec is not None else 7.0 @@ -1086,16 +1092,21 @@ async def run_yandex_city_sweep( consecutive_failures = 0 # Вычисляем watchdog-таймаут для combos-режима (центр, anchors=None). - # В combos-режиме один "anchor" выполняет num_combos × max_pages fetch'ей — - # каждый занимает примерно _resolved_delay + _YANDEX_COMBOS_PER_FETCH_S секунд. - # Для 30 combos × 3 pages × (9+12) + 300 ≈ 2190s (~37 мин). + # В combos-режиме один "anchor" выполняет num_segments × num_combos × max_pages + # fetch'ей — каждый занимает примерно _resolved_delay + _YANDEX_COMBOS_PER_FETCH_S сек. + # Для 2 segments × 30 combos × 3 pages × (9+12) + 300 ≈ 4080s (~68 мин). + # Множитель len(_segments) масштабирует watchdog пропорционально числу проходов + # (vtorichka + novostroyki ≈ ×2 wall-clock) — иначе второй проход гильотинится. # В explicit-anchor (тест/override) режиме оставляем ANCHOR_TIMEOUT_SEC (240s) — # там каждый anchor небольшой и watchdog работает как старый защитный барьер. _num_combos = len(_rooms_list) * len(_price_ranges) if anchors is None and _num_combos > 0: _sweep_timeout = max( ANCHOR_TIMEOUT_SEC, - _num_combos * pages_per_anchor * (_resolved_delay + _YANDEX_COMBOS_PER_FETCH_S) + len(_segments) + * _num_combos + * pages_per_anchor + * (_resolved_delay + _YANDEX_COMBOS_PER_FETCH_S) + _YANDEX_ADDRESS_ENRICH_BUDGET_S, ) else: @@ -1162,6 +1173,7 @@ async def run_yandex_city_sweep( max_pages=pages_per_anchor, rooms_list=_rooms_list, price_ranges=_price_ranges, + segments=_segments, ) counters.lots_fetched += len(anchor_lots) if anchor_lots: diff --git a/tradein-mvp/backend/app/services/scrapers/yandex_realty.py b/tradein-mvp/backend/app/services/scrapers/yandex_realty.py index d078465c..a9310ce8 100644 --- a/tradein-mvp/backend/app/services/scrapers/yandex_realty.py +++ b/tradein-mvp/backend/app/services/scrapers/yandex_realty.py @@ -129,7 +129,14 @@ def _extract_json_from_content(content: str) -> str | None: return None -def _parse_gate_json(payload: dict[str, Any], page_param: int = 1) -> list[ScrapedLot]: +def _segment_for(new_flat: str) -> str: + """Map a newFlat gate-API value to the listings.listing_segment value.""" + return "novostroyki" if new_flat == "YES" else "vtorichka" + + +def _parse_gate_json( + payload: dict[str, Any], page_param: int = 1, new_flat: str = "NO" +) -> list[ScrapedLot]: """Parse gate-API JSON payload into a list of ScrapedLot. Verified field mapping (from .issdump/yandex_gate_api_sample.json + live prod 2026-06-17): @@ -150,7 +157,7 @@ def _parse_gate_json(payload: dict[str, Any], page_param: int = 1) -> list[Scrap location.geocoderAddress -> address (full, with house number) mainImages[] -> photo_urls (prepend "https:", up to 5) building.siteId -> house_ext_id (JK linkage) - listing_segment = "vtorichka" (all requests use newFlat=NO) + listing_segment -> "novostroyki" (newFlat=YES) | "vtorichka" (newFlat=NO) """ result = _extract_gate_data(payload) if result is None: @@ -160,13 +167,15 @@ def _parse_gate_json(payload: dict[str, Any], page_param: int = 1) -> list[Scrap entities, _pager = result lots: list[ScrapedLot] = [] for entity in entities: - lot = _entity_to_lot(entity, page_param=page_param) + lot = _entity_to_lot(entity, page_param=page_param, new_flat=new_flat) if lot is not None: lots.append(lot) return lots -def _entity_to_lot(entity: dict[str, Any], page_param: int = 1) -> ScrapedLot | None: +def _entity_to_lot( + entity: dict[str, Any], page_param: int = 1, new_flat: str = "NO" +) -> ScrapedLot | None: """Convert one gate-API entity dict to ScrapedLot. None on missing required fields.""" try: offer_id = str(entity.get("offerId") or "") @@ -263,7 +272,7 @@ def _entity_to_lot(entity: dict[str, Any], page_param: int = 1) -> ScrapedLot | house_source=house_source, house_ext_id=house_ext_id, house_url=None, - listing_segment="vtorichka", + listing_segment=_segment_for(new_flat), raw_payload={ "page_param": page_param, "ceiling_height": ceiling_height, @@ -366,13 +375,18 @@ class YandexRealtyScraper(BaseScraper): rooms: str | None = None, price_min: int | None = None, price_max: int | None = None, + new_flat: str = "NO", ) -> str: - """Build gate-API URL. page: 1-based (page=0 -> API error).""" + """Build gate-API URL. page: 1-based (page=0 -> API error). + + new_flat: "NO" -> vtorichka (default, preserves legacy behavior), + "YES" -> novostroyki (primary-sale / reassignment segment). + """ params: dict[str, str | int] = { "rgid": _EKB_RGID, "type": "SELL", "category": "APARTMENT", - "newFlat": "NO", + "newFlat": new_flat, "_pageType": "search", "_providers": "react-search-results-data", "page": page, @@ -391,9 +405,12 @@ class YandexRealtyScraper(BaseScraper): page: int, price_min: int | None, price_max: int | None, + new_flat: str = "NO", ) -> dict[str, Any] | None: """Fetch one gate-API page. Returns parsed JSON dict or None on error/invalid.""" - url = self._build_url(page=page, rooms=rooms, price_min=price_min, price_max=price_max) + url = self._build_url( + page=page, rooms=rooms, price_min=price_min, price_max=price_max, new_flat=new_flat + ) try: resp = await self._http_get(url, timeout=60) except Exception: @@ -421,14 +438,18 @@ class YandexRealtyScraper(BaseScraper): rooms: str | None = None, price_min: int | None = None, price_max: int | None = None, + new_flat: str = "NO", ) -> list[ScrapedLot]: """Fetch ONE page of gate-API results. lat/lon/radius_m ignored (uses rgid). page: 1-based. Tarpit resilience: status_code==0 or JSON error -> rotate IP + retry up to _YANDEX_TARPIT_MAX_RETRIES times. Non-tarpit failures (4xx/5xx) are not retried. + new_flat: "NO" -> vtorichka, "YES" -> novostroyki. """ - url = self._build_url(page=page, rooms=rooms, price_min=price_min, price_max=price_max) + url = self._build_url( + page=page, rooms=rooms, price_min=price_min, price_max=price_max, new_flat=new_flat + ) payload: dict[str, Any] | None = None for attempt in range(1 + _YANDEX_TARPIT_MAX_RETRIES): @@ -494,13 +515,14 @@ class YandexRealtyScraper(BaseScraper): ) return [] - lots = _parse_gate_json(payload, page_param=page) + lots = _parse_gate_json(payload, page_param=page, new_flat=new_flat) logger.info( - "yandex gate page=%d rooms=%s price=[%s-%s]: %d cards", + "yandex gate page=%d rooms=%s price=[%s-%s] newFlat=%s: %d cards", page, rooms or "all", price_min, price_max, + new_flat, len(lots), ) await self.sleep_between_requests() @@ -514,16 +536,20 @@ class YandexRealtyScraper(BaseScraper): max_pages: int = _GATE_MAX_PAGES_CAP, rooms_list: list[str] | None = None, price_ranges: list[tuple[int | None, int | None]] | None = None, + segments: list[str] | None = None, **_legacy_kwargs: Any, ) -> list[ScrapedLot]: - """Fetch via rooms x price-range combos; paginate each combo to totalPages. + """Fetch via segment x rooms x price-range combos; paginate each combo to totalPages. Pagination driven by pager.totalPages from first page response. Each combo iterates page=1 .. min(totalPages, max_pages, _GATE_MAX_PAGES_CAP). - Deduplicates by source_id/source_url across all combos. + Deduplicates by source_id/source_url across ALL combos AND segments (one `seen`). Legacy mode (rooms_list=None, price_ranges=None): single citywide sweep. + segments: list of newFlat values, default ["NO"] (vtorichka only). Pass + ["NO", "YES"] to sweep both vtorichka and novostroyki in one call. """ seen: dict[str, ScrapedLot] = {} + _segments = segments or ["NO"] if rooms_list is None and price_ranges is None: combos: list[tuple[str | None, int | None, int | None]] = [(None, None, None)] @@ -532,112 +558,123 @@ class YandexRealtyScraper(BaseScraper): p_ranges = price_ranges or DEFAULT_PRICE_RANGES combos = [(r, lo, hi) for r in r_list for lo, hi in p_ranges] - for rooms, price_min, price_max in combos: - combo_label = _combo_label(rooms, price_min, price_max) - total_pages: int | None = None + for new_flat in _segments: + _seg = _segment_for(new_flat) + for rooms, price_min, price_max in combos: + combo_label = f"{_seg}/{_combo_label(rooms, price_min, price_max)}" + total_pages: int | None = None - for page in range(1, max_pages + 1): - if page == 1: - payload_p1: dict[str, Any] | None = None - p1_url = self._build_url( - page=1, rooms=rooms, price_min=price_min, price_max=price_max - ) - for _attempt in range(1 + _YANDEX_TARPIT_MAX_RETRIES): - try: - resp = await self._http_get(p1_url, timeout=60) - except Exception: - logger.exception("yandex gate combo fetch failed combo=%s", combo_label) + for page in range(1, max_pages + 1): + if page == 1: + payload_p1: dict[str, Any] | None = None + p1_url = self._build_url( + page=1, + rooms=rooms, + price_min=price_min, + price_max=price_max, + new_flat=new_flat, + ) + for _attempt in range(1 + _YANDEX_TARPIT_MAX_RETRIES): + try: + resp = await self._http_get(p1_url, timeout=60) + except Exception: + logger.exception( + "yandex gate combo fetch failed combo=%s", combo_label + ) + break + if resp.status_code == 0: # type: ignore[union-attr] + logger.warning( + "yandex gate: tarpit combo=%s page=1 -- rotating", combo_label + ) + await self._rotate_ip() + await asyncio.sleep(2) + continue + if resp.status_code != 200: # type: ignore[union-attr] + logger.warning( + "yandex gate: HTTP %d combo=%s page=1", + resp.status_code, + combo_label, # type: ignore[union-attr] + ) + break + try: + payload_p1 = json.loads(resp.text) # type: ignore[union-attr] + except (json.JSONDecodeError, ValueError): + logger.warning( + "yandex gate: JSON parse error combo=%s page=1", combo_label + ) + payload_p1 = None + await self._rotate_ip() + await asyncio.sleep(2) + continue + if _is_gate_error(payload_p1): + logger.warning( + "yandex gate: error payload combo=%s page=1", combo_label + ) + payload_p1 = None + break break - if resp.status_code == 0: # type: ignore[union-attr] - logger.warning( - "yandex gate: tarpit combo=%s page=1 -- rotating", combo_label - ) - await self._rotate_ip() - await asyncio.sleep(2) - continue - if resp.status_code != 200: # type: ignore[union-attr] - logger.warning( - "yandex gate: HTTP %d combo=%s page=1", - resp.status_code, - combo_label, # type: ignore[union-attr] + + if payload_p1 is None: + logger.debug( + "yandex gate combo [%s] page=1: failed -- skipping", combo_label ) break - try: - payload_p1 = json.loads(resp.text) # type: ignore[union-attr] - except (json.JSONDecodeError, ValueError): - logger.warning( - "yandex gate: JSON parse error combo=%s page=1", combo_label - ) - payload_p1 = None - await self._rotate_ip() - await asyncio.sleep(2) - continue - if _is_gate_error(payload_p1): - logger.warning( - "yandex gate: error payload combo=%s page=1", combo_label - ) - payload_p1 = None + + result = _extract_gate_data(payload_p1) + if result is None: break + _entities_p1, pager_p1 = result + total_pages = min( + pager_p1.get("totalPages", 1), + max_pages, + _GATE_MAX_PAGES_CAP, + ) + lots_p1 = _parse_gate_json(payload_p1, page_param=1, new_flat=new_flat) + if not lots_p1: + logger.debug( + "yandex gate combo [%s] page=1: empty -- stopping", combo_label + ) + break + for lot in lots_p1: + key = lot.source_id or lot.source_url + if key and key not in seen: + seen[key] = lot + await self.sleep_between_requests() + if total_pages <= 1: + break + continue + + if total_pages is not None and page > total_pages: break - if payload_p1 is None: + lots = await self.fetch_around( + lat, + lon, + radius_m, + page=page, + rooms=rooms, + price_min=price_min, + price_max=price_max, + new_flat=new_flat, + ) + if not lots: logger.debug( - "yandex gate combo [%s] page=1: failed -- skipping", combo_label + "yandex gate combo [%s] page=%d: empty -- stopping", + combo_label, + page, ) break - - result = _extract_gate_data(payload_p1) - if result is None: - break - _entities_p1, pager_p1 = result - total_pages = min( - pager_p1.get("totalPages", 1), - max_pages, - _GATE_MAX_PAGES_CAP, - ) - lots_p1 = _parse_gate_json(payload_p1, page_param=1) - if not lots_p1: - logger.debug( - "yandex gate combo [%s] page=1: empty -- stopping", combo_label - ) - break - for lot in lots_p1: + for lot in lots: key = lot.source_id or lot.source_url if key and key not in seen: seen[key] = lot - await self.sleep_between_requests() - if total_pages <= 1: - break - continue - - if total_pages is not None and page > total_pages: - break - - lots = await self.fetch_around( - lat, - lon, - radius_m, - page=page, - rooms=rooms, - price_min=price_min, - price_max=price_max, - ) - if not lots: - logger.debug( - "yandex gate combo [%s] page=%d: empty -- stopping", - combo_label, - page, - ) - break - for lot in lots: - key = lot.source_id or lot.source_url - if key and key not in seen: - seen[key] = lot logger.info( - "yandex gate aggregate: %d unique lots (%d combos)", + "yandex gate aggregate: %d unique lots (%d combos x %d segments=%s)", len(seen), len(combos), + len(_segments), + _segments, ) return list(seen.values()) diff --git a/tradein-mvp/backend/tests/test_yandex_realty_serp.py b/tradein-mvp/backend/tests/test_yandex_realty_serp.py index 2a3119ea..2561b0c5 100644 --- a/tradein-mvp/backend/tests/test_yandex_realty_serp.py +++ b/tradein-mvp/backend/tests/test_yandex_realty_serp.py @@ -182,6 +182,29 @@ def test_entity_to_lot_full_entity(): assert lot.raw_payload["kitchen_area_m2"] == pytest.approx(12.7) +def test_entity_to_lot_segment_vtorichka_default(): + """Default new_flat='NO' -> listing_segment='vtorichka'.""" + lot = _entity_to_lot(_ENTITY_FULL) + assert lot is not None + assert lot.listing_segment == "vtorichka" + + +def test_entity_to_lot_segment_novostroyki_on_new_flat_yes(): + """new_flat='YES' -> listing_segment='novostroyki'.""" + lot = _entity_to_lot(_ENTITY_FULL, new_flat="YES") + assert lot is not None + assert lot.listing_segment == "novostroyki" + + +def test_parse_gate_json_propagates_segment(): + """_parse_gate_json threads new_flat down to each lot's listing_segment.""" + payload = _make_gate_payload([_ENTITY_FULL, _ENTITY_STUDIO]) + lots_nb = _parse_gate_json(payload, new_flat="YES") + assert lots_nb and all(lot.listing_segment == "novostroyki" for lot in lots_nb) + lots_vt = _parse_gate_json(payload, new_flat="NO") + assert lots_vt and all(lot.listing_segment == "vtorichka" for lot in lots_vt) + + def test_entity_to_lot_studio_rooms_zero(): lot = _entity_to_lot(_ENTITY_STUDIO) assert lot is not None @@ -350,6 +373,21 @@ def test_build_url_combos_are_unique(): assert len(urls) == expected, f"URL collisions: {expected} combos but {len(urls)} unique" +def test_build_url_new_flat_default_no(): + """Default new_flat must keep newFlat=NO (vtorichka) — backward compat.""" + s = YandexRealtyScraper() + url = s._build_url(page=1) + assert "newFlat=NO" in url + assert "newFlat=YES" not in url + + +def test_build_url_new_flat_yes(): + """new_flat='YES' must produce newFlat=YES (novostroyki segment).""" + s = YandexRealtyScraper() + url = s._build_url(page=1, new_flat="YES") + assert "newFlat=YES" in url + + def test_default_city(): assert DEFAULT_CITY == "ekaterinburg" @@ -665,6 +703,81 @@ async def test_fetch_around_multi_room_dedup(): assert len(ids) == 3 +@pytest.mark.asyncio +async def test_fetch_around_multi_room_runs_both_segments_with_shared_dedup(): + """segments=['NO','YES'] sweeps both vtorichka+novostroyki; dedup spans both passes. + + Mocks _http_get and inspects requested URLs: both newFlat=NO and newFlat=YES + must be requested. The same offerId returned in both passes must be deduped to + one lot (single shared `seen` across passes), tagged by the FIRST pass's segment. + """ + s = YandexRealtyScraper() + + requested_new_flat: list[str] = [] + + entity_shared = dict( + _ENTITY_FULL, offerId="shared_xseg", url="//realty.yandex.ru/offer/shared_xseg" + ) + entity_vt = dict(_ENTITY_FULL, offerId="vt_only", url="//realty.yandex.ru/offer/vt_only") + entity_nb = dict(_ENTITY_FULL, offerId="nb_only", url="//realty.yandex.ru/offer/nb_only") + pager = {"page": 0, "pageSize": 20, "totalItems": 2, "totalPages": 1} + + async def fake_http_get(url: str, **kwargs: object) -> _CurlResponse: + if "newFlat=YES" in url: + requested_new_flat.append("YES") + payload = _make_gate_payload([entity_shared, entity_nb], pager) + else: + requested_new_flat.append("NO") + payload = _make_gate_payload([entity_shared, entity_vt], pager) + return _CurlResponse(status_code=200, text=json.dumps(payload)) + + with patch.object(s, "_http_get", side_effect=fake_http_get): + with patch.object(s, "sleep_between_requests", new_callable=AsyncMock): + lots = await s.fetch_around_multi_room( + lat=0.0, + lon=0.0, + rooms_list=["1"], + price_ranges=[(None, 5_000_000)], + segments=["NO", "YES"], + max_pages=1, + ) + + assert "NO" in requested_new_flat + assert "YES" in requested_new_flat + + by_id = {lot.source_id: lot for lot in lots} + assert set(by_id) == {"shared_xseg", "vt_only", "nb_only"} + # Shared offer deduped to ONE lot, tagged by the first pass (NO -> vtorichka). + assert by_id["shared_xseg"].listing_segment == "vtorichka" + assert by_id["vt_only"].listing_segment == "vtorichka" + assert by_id["nb_only"].listing_segment == "novostroyki" + + +@pytest.mark.asyncio +async def test_fetch_around_multi_room_default_segment_no_only(): + """Without segments arg, only newFlat=NO (vtorichka) is requested (legacy default).""" + s = YandexRealtyScraper() + requested_new_flat: list[str] = [] + pager = {"page": 0, "pageSize": 20, "totalItems": 1, "totalPages": 1} + + async def fake_http_get(url: str, **kwargs: object) -> _CurlResponse: + requested_new_flat.append("YES" if "newFlat=YES" in url else "NO") + body = json.dumps(_make_gate_payload([_ENTITY_FULL], pager)) + return _CurlResponse(status_code=200, text=body) + + with patch.object(s, "_http_get", side_effect=fake_http_get): + with patch.object(s, "sleep_between_requests", new_callable=AsyncMock): + await s.fetch_around_multi_room( + lat=0.0, + lon=0.0, + rooms_list=["1"], + price_ranges=[(None, 5_000_000)], + max_pages=1, + ) + + assert set(requested_new_flat) == {"NO"} + + @pytest.mark.asyncio async def test_fetch_around_multi_room_legacy_single_sweep(): """Legacy mode (no rooms_list/price_ranges) uses single citywide sweep.""" From 22a7a3f0861aa9e38a004c5273a93d8a9cbfa252 Mon Sep 17 00:00:00 2001 From: bot-backend Date: Wed, 17 Jun 2026 22:24:13 +0300 Subject: [PATCH 10/15] =?UTF-8?q?fix(avito):=20classify=20novostroyki=20by?= =?UTF-8?q?=20item-development-name=20marker;=20map=20=D0=BF=D0=B5=D1=80?= =?UTF-8?q?=D0=B5=D1=83=D1=81=D1=82=D1=83=D0=BF=D0=BA=D0=B0=20sale=5Ftype?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../backend/app/services/scrapers/avito.py | 22 ++++-------- .../app/services/scrapers/avito_detail.py | 1 + .../backend/tests/test_avito_detail_parse.py | 19 +++++++++++ .../backend/tests/test_avito_serp_date.py | 34 +++++++++++++++++++ 4 files changed, 61 insertions(+), 15 deletions(-) diff --git a/tradein-mvp/backend/app/services/scrapers/avito.py b/tradein-mvp/backend/app/services/scrapers/avito.py index b500752f..38255dda 100644 --- a/tradein-mvp/backend/app/services/scrapers/avito.py +++ b/tradein-mvp/backend/app/services/scrapers/avito.py @@ -74,15 +74,7 @@ _UNIT_DAYS: dict[str, int] = { # поэтому отдельная ветка с неявным n=1. Юниты в винительном падеже («минуту», # «неделю») плюс именительный («час», «день»). _REL_DATE_SINGULAR_RE = re.compile( - r"(?P" - r"секунду|" - r"минуту|" - r"час|" - r"день|" - r"неделю|" - r"месяц|" - r"год" - r")\s+назад", + r"(?P" r"секунду|" r"минуту|" r"час|" r"день|" r"неделю|" r"месяц|" r"год" r")\s+назад", flags=re.I, ) @@ -751,12 +743,12 @@ class AvitoScraper(BaseScraper): house_ext_id = parts[-1] house_url = urljoin("https://www.avito.ru", h_href) - # listing_segment по item URL pattern - listing_segment: str | None = None - if "/kvartiry/" in href: - listing_segment = "vtorichka" - elif "/novostroyki/" in href: - listing_segment = "novostroyki" + # listing_segment по DOM-маркеру карточки: новостройки несут + # data-marker="item-development-name" (название ЖК/застройщика), + # вторичка — нет. URL-паттерн ненадёжен: каждый avito-URL квартиры + # содержит /kvartiry/ (вкл. новостройки) → ветка всегда vtorichka. + dev_name = card.css_first('[data-marker="item-development-name"]') + listing_segment = "novostroyki" if dev_name is not None else "vtorichka" return ScrapedLot( source="avito", diff --git a/tradein-mvp/backend/app/services/scrapers/avito_detail.py b/tradein-mvp/backend/app/services/scrapers/avito_detail.py index c0e246a3..f7f52ddd 100644 --- a/tradein-mvp/backend/app/services/scrapers/avito_detail.py +++ b/tradein-mvp/backend/app/services/scrapers/avito_detail.py @@ -152,6 +152,7 @@ _SALE_TYPE_MAP: dict[str, str] = { "свободная": "free", "альтернатива": "alternative", "аукцион": "auction", + "переуступка": "assignment", } _HOUSE_TYPE_MAP: dict[str, str] = { diff --git a/tradein-mvp/backend/tests/test_avito_detail_parse.py b/tradein-mvp/backend/tests/test_avito_detail_parse.py index e9e419fb..7a2f6b0e 100644 --- a/tradein-mvp/backend/tests/test_avito_detail_parse.py +++ b/tradein-mvp/backend/tests/test_avito_detail_parse.py @@ -78,6 +78,25 @@ def test_minimal_parse() -> None: assert result.views_today == 12 +def test_sale_type_assignment_pereustupka() -> None: + """«Способ продажи: переуступка» → sale_type='assignment' (ДДУ переуступка, + сигнал новостройки). До фикса map не содержал ключ → None.""" + html = """ + +
№ 8043003287
+ 9 500 000 ₽ +
+
    +
  • Количество комнат: 2
    • +
  • Способ продажи: переуступка
    • +
+
+ + """ + result = parse_detail_html(html, "https://www.avito.ru/test_8043003287") + assert result.sale_type == "assignment" + + def test_metro_extraction() -> None: result = parse_detail_html(MINIMAL_HTML, SOURCE_URL) # "Чкаловская 11-15 мин пешком" → metro_stations diff --git a/tradein-mvp/backend/tests/test_avito_serp_date.py b/tradein-mvp/backend/tests/test_avito_serp_date.py index 870541d4..f6fa868f 100644 --- a/tradein-mvp/backend/tests/test_avito_serp_date.py +++ b/tradein-mvp/backend/tests/test_avito_serp_date.py @@ -60,3 +60,37 @@ def test_lazy_card_date_comes_from_json_not_dom() -> None: lots = AvitoScraper()._parse_html(html, "https://www.avito.ru/x") lazy = next(lot for lot in lots if "8043936560" in (lot.source_url or "")) assert lazy.listing_date == _expected(_TS_LAZY) + + +# ── listing_segment по DOM-маркеру item-development-name ─────────────────────── +# Каждый avito-URL квартиры содержит /kvartiry/ (вкл. новостройки), поэтому +# сегмент определяется не по URL, а по наличию data-marker="item-development-name" +# (название ЖК/застройщика рендерится только у карточек новостроек). + + +def _serp_card(item_id: str, *, with_dev_name: bool) -> str: + dev = '
ЖК «Федерация»
' if with_dev_name else "" + return f""" + + """ + + +def test_segment_novostroyki_when_development_name_present() -> None: + """Карточка с data-marker='item-development-name' → listing_segment='novostroyki'.""" + html = f"{_serp_card('8043003287', with_dev_name=True)}" + lots = AvitoScraper()._parse_html(html, "https://www.avito.ru/ekaterinburg/kvartiry") + assert len(lots) == 1 + assert lots[0].listing_segment == "novostroyki" + + +def test_segment_vtorichka_when_no_development_name() -> None: + """Карточка без маркера ЖК → listing_segment='vtorichka' (даже с /kvartiry/ в URL).""" + html = f"{_serp_card('8043936560', with_dev_name=False)}" + lots = AvitoScraper()._parse_html(html, "https://www.avito.ru/ekaterinburg/kvartiry") + assert len(lots) == 1 + assert lots[0].listing_segment == "vtorichka" From 4e8086cc1f73b9e4f6dde3c91f56bdd033a2fe85 Mon Sep 17 00:00:00 2001 From: bot-backend Date: Wed, 17 Jun 2026 22:29:33 +0300 Subject: [PATCH 11/15] ci: cache uv + skip dev deps to speed up gate jobs (#1709) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit backend-tests и openapi-codegen-check тратят ~90% времени не на работу (дамп схемы + pytest), а на пересборку окружения с нуля каждый прогон. Доминанта — 'uv sync --frozen' тянет весь geo/WeasyPrint-стек заново, т.к. ~/.cache/uv не кэшировался между прогонами (npm уже кэширован setup-node). Изменения: - Cache uv (~/.cache/uv) через actions/cache в обоих job, ключ по backend/uv.lock, restore-keys для частичного хита. continue-on-error → сбой cache-бэкенда раннера не ломает gate. - openapi-codegen-check: 'uv sync --frozen --no-dev' — job только импортирует app.main для дампа OpenAPI, dev-группа (pytest/ruff/coverage) не нужна. Dockerfile тоже --no-dev, поэтому импорт гарантирован. Refs #1709 --- .forgejo/workflows/ci.yml | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index 78008735..ff1270b0 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -88,6 +88,19 @@ jobs: curl -LsSf https://astral.sh/uv/install.sh | sh echo "$HOME/.local/bin" >> "$GITHUB_PATH" + - name: Cache uv packages + # Кросс-прогонный кэш скачанных/собранных wheel'ов (~/.cache/uv по умолч.). + # `uv sync --frozen` без него каждый прогон тянет весь geo-стек заново — + # доминирующая часть времени job (#1709). Ключ по uv.lock; continue-on-error + # чтобы сбой cache-бэкенда раннера НИКОГДА не ронял gate. + uses: actions/cache@v4 + continue-on-error: true + with: + path: ~/.cache/uv + key: uv-${{ runner.os }}-${{ hashFiles('backend/uv.lock') }} + restore-keys: | + uv-${{ runner.os }}- + - name: Install system deps for geo + WeasyPrint # libpq/gdal/proj/geos — geo-стек (geopandas/shapely/pyproj). # libcairo2/libpango* — нативные либы WeasyPrint: с ними PDF-тесты @@ -229,6 +242,16 @@ jobs: curl -LsSf https://astral.sh/uv/install.sh | sh echo "$HOME/.local/bin" >> "$GITHUB_PATH" + - name: Cache uv packages + # См. backend-tests: кросс-прогонный кэш ~/.cache/uv, тот же ключ по uv.lock. + uses: actions/cache@v4 + continue-on-error: true + with: + path: ~/.cache/uv + key: uv-${{ runner.os }}-${{ hashFiles('backend/uv.lock') }} + restore-keys: | + uv-${{ runner.os }}- + - name: Install system deps for geo + WeasyPrint # app.main транзитивно тянет geo/PDF-модули. На macOS-dev импорт схемы # проходит и без этих либ, но на ubuntu ставим как backend-tests @@ -238,9 +261,12 @@ jobs: sudo apt-get install -y libpq-dev libgdal-dev libproj-dev libgeos-dev \ libcairo2 libpango-1.0-0 libpangoft2-1.0-0 - - name: Install backend deps (uv sync --frozen) + - name: Install backend deps (uv sync --frozen --no-dev) working-directory: backend - run: uv sync --frozen + # --no-dev: этот job только дампит app.openapi() (нужен runtime app.main). + # pytest/ruff/coverage не используются → не ставим dev-группу (быстрее). + # Dockerfile тоже собирает с --no-dev → импорт app.main гарантированно ок. + run: uv sync --frozen --no-dev - name: Install frontend deps (npm ci) working-directory: frontend From fe805ea528a20566ae16d9b847759c0c50543be4 Mon Sep 17 00:00:00 2001 From: bot-backend Date: Wed, 17 Jun 2026 22:32:18 +0300 Subject: [PATCH 12/15] fix(cian): persist detail rich fields (description/kitchen/ceiling) + stamp detail_enriched_at --- .../app/services/scrapers/cian_detail.py | 7 +- tradein-mvp/backend/tests/test_cian_detail.py | 64 +++++++++++++++++++ 2 files changed, 70 insertions(+), 1 deletion(-) diff --git a/tradein-mvp/backend/app/services/scrapers/cian_detail.py b/tradein-mvp/backend/app/services/scrapers/cian_detail.py index 55b54092..e7b4749c 100644 --- a/tradein-mvp/backend/app/services/scrapers/cian_detail.py +++ b/tradein-mvp/backend/app/services/scrapers/cian_detail.py @@ -48,6 +48,7 @@ class DetailEnrichment: repair_type: str | None = None # 'cosmetic' / 'design' / 'no' — raw Cian value repair_state: str | None = None # enum: needs_repair/standard/good/excellent kitchen_area_m2: float | None = None # offer.kitchenArea (м²) + description: str | None = None # offer.description (текст объявления) views_total: int | None = None # Cian stats.totalViewsFormattedString → int views_today: int | None = None @@ -144,6 +145,7 @@ async def fetch_detail( repair_type=_extract_repair_type(offer), repair_state=_extract_repair_state(offer), kitchen_area_m2=_parse_float(offer.get("kitchenArea")), + description=offer.get("description") or None, raw_offer=offer, ) @@ -302,8 +304,10 @@ def save_detail_enrichment(db: Session, listing_id: int, enrichment: DetailEnric repair_type = COALESCE(:rt, repair_type), repair_state = COALESCE(:rs, repair_state), kitchen_area_m2 = COALESCE(CAST(:ka AS double precision), kitchen_area_m2), + description = COALESCE(:descr, description), views_total = COALESCE(:vt, views_total), - views_today = COALESCE(:vd, views_today) + views_today = COALESCE(:vd, views_today), + detail_enriched_at = NOW() WHERE id = CAST(:lid AS bigint) """), { @@ -315,6 +319,7 @@ def save_detail_enrichment(db: Session, listing_id: int, enrichment: DetailEnric "rt": enrichment.repair_type, "rs": enrichment.repair_state, "ka": enrichment.kitchen_area_m2, + "descr": enrichment.description, "vt": enrichment.views_total, "vd": enrichment.views_today, }, diff --git a/tradein-mvp/backend/tests/test_cian_detail.py b/tradein-mvp/backend/tests/test_cian_detail.py index b303373f..d084da48 100644 --- a/tradein-mvp/backend/tests/test_cian_detail.py +++ b/tradein-mvp/backend/tests/test_cian_detail.py @@ -462,6 +462,40 @@ async def test_fetch_detail_kitchen_area_none_when_missing(monkeypatch): assert result.kitchen_area_m2 is None +@pytest.mark.asyncio +async def test_fetch_detail_extracts_description(monkeypatch): + """offer.description → DetailEnrichment.description (для persist в listings).""" + fake_state = { + "offerData": { + "offer": { + "cianId": 555555, + "description": "Продаётся уютная квартира с видом на парк.", + } + } + } + + monkeypatch.setattr( + "app.services.scrapers.cian_detail.extract_state", + lambda html, mfe, key: fake_state if key == "defaultState" else None, + ) + monkeypatch.setattr( + "app.services.scrapers.cian_detail.extract_all_states", + lambda html: {}, + ) + + response = MagicMock() + response.status_code = 200 + response.text = "" + + session = MagicMock() + session.get = AsyncMock(return_value=response) + session.close = AsyncMock() + + result = await fetch_detail("https://ekb.cian.ru/sale/flat/555555/", session=session) + assert result is not None + assert result.description == "Продаётся уютная квартира с видом на парк." + + # ── save_detail_enrichment — kitchen_area_m2 persisted ─────────────────────── @@ -504,6 +538,36 @@ def test_save_detail_enrichment_kitchen_area_none_passthrough(): assert update_call_params["ka"] is None +# ── save_detail_enrichment — description + detail_enriched_at stamp ─────────── + + +def test_save_detail_enrichment_writes_description(): + """save_detail_enrichment передаёт description в UPDATE listings.""" + db = MagicMock() + db.execute.return_value.fetchone.return_value = None + + enrichment = DetailEnrichment(description="Светлая квартира рядом с парком") + + save_detail_enrichment(db, listing_id=1002, enrichment=enrichment) + + update_call_params = db.execute.call_args_list[0][0][1] + assert update_call_params["descr"] == "Светлая квартира рядом с парком" + + +def test_save_detail_enrichment_stamps_detail_enriched_at(): + """UPDATE listings содержит detail_enriched_at = NOW() — иначе строка + переобогащается каждый run (detail_enriched_at IS NULL filter).""" + db = MagicMock() + db.execute.return_value.fetchone.return_value = None + + enrichment = DetailEnrichment(kitchen_area_m2=8.3) + + save_detail_enrichment(db, listing_id=1003, enrichment=enrichment) + + update_sql = str(db.execute.call_args_list[0][0][0]) + assert "detail_enriched_at = NOW()" in update_sql + + # ── fetch_detail browser_fetcher param ─────────────────────────────────────── From 24615b96c190e423789247bd488a01a3803c3b6b Mon Sep 17 00:00:00 2001 From: bot-backend Date: Wed, 17 Jun 2026 22:32:52 +0300 Subject: [PATCH 13/15] fix(site-finder): gate velocity on is_reviewed objective_complex_mapping (#307 OBJ-2) --- backend/app/services/site_finder/velocity.py | 30 +++++- backend/tests/test_velocity.py | 33 +++++++ .../165_velocity_mapping_reviewed_gate.sql | 93 +++++++++++++++++++ 3 files changed, 153 insertions(+), 3 deletions(-) create mode 100644 data/sql/165_velocity_mapping_reviewed_gate.sql diff --git a/backend/app/services/site_finder/velocity.py b/backend/app/services/site_finder/velocity.py index e391dee3..75b68784 100644 --- a/backend/app/services/site_finder/velocity.py +++ b/backend/app/services/site_finder/velocity.py @@ -16,9 +16,13 @@ Foundation: domrf_kn_objects (lat/lon, comm_name, obj_class, region_cd), Fallback: rosreestr_deals (quarter_cad_number, deal_count, period_start_date). Linkage: domrf_kn_objects.obj_id - → objective_complex_mapping.domrf_obj_id + → objective_complex_mapping.domrf_obj_id (gated: is_reviewed/manual/score≥0.85) → objective_complex_mapping.objective_complex_name → objective_corpus_room_month.project_name + +OBJ-2 (#307): маппинги фильтруются по confidence (_MAPPING_CONFIDENCE_GATE) — +unreviewed low-score auto-matches (#1331/#1333 backfill) исключаются как +false-positive risk. """ from __future__ import annotations @@ -40,6 +44,24 @@ _EKB_MEDIAN_FALLBACK_SQM_PER_MONTH: float = 4500.0 # пытаемся rosreestr_fallback. _OBJECTIVE_COVERAGE_MIN_RATIO: float = 0.50 +# OBJ-2 (#307): gate objective_complex_mapping by confidence перед использованием +# в velocity. Fuzzy-trgm backfill (#1331/#1333) добавил ~115 auto-matched строк с +# is_reviewed=false и низким match_score (вплоть до 0.50-0.625) — false-positive +# risk, который раздувал/искажал velocity конкурентов. +# +# Принимаем mapping только если: +# - is_reviewed = TRUE (человек подтвердил), ИЛИ +# - match_method = 'manual' (ручной маппинг, score обычно NULL), ИЛИ +# - match_score >= 0.85 (AUTO_ACCEPT_THRESHOLD — high-confidence auto, +# vault: fuzzy_trgm 0.85+ надёжен для auto-use). +# +# Строгий gate только на is_reviewed=true дал бы 2 строки из 303 → обнулил бы +# velocity-покрытие; 0.85-порог сохраняет 264/303 EKB-маппингов, отбрасывая 39 +# низкоуверенных. Совпадает с AUTO_ACCEPT_THRESHOLD из objective_backfill.py. +_MAPPING_CONFIDENCE_GATE: str = ( + "(cm.is_reviewed = TRUE OR cm.match_method = 'manual' OR cm.match_score >= 0.85)" +) + @dataclass(frozen=True) class VelocityResult: @@ -195,7 +217,7 @@ def compute_velocity( sales_rows = ( db.execute( text( - """ + f""" WITH all_competitors AS ( SELECT unnest(CAST(:obj_ids AS int[])) AS obj_id ), @@ -204,6 +226,7 @@ def compute_velocity( cm.objective_complex_name FROM objective_complex_mapping cm WHERE cm.domrf_obj_id = ANY(:obj_ids) + AND {_MAPPING_CONFIDENCE_GATE} ) SELECT ac.obj_id, @@ -311,12 +334,13 @@ def compute_velocity( bucket_rows = ( db.execute( text( - """ + f""" WITH mapped AS ( SELECT cm.domrf_obj_id AS obj_id, cm.objective_complex_name FROM objective_complex_mapping cm WHERE cm.domrf_obj_id = ANY(:obj_ids) + AND {_MAPPING_CONFIDENCE_GATE} ) SELECT m.obj_id, diff --git a/backend/tests/test_velocity.py b/backend/tests/test_velocity.py index aa25fdbb..c5aff67b 100644 --- a/backend/tests/test_velocity.py +++ b/backend/tests/test_velocity.py @@ -327,6 +327,39 @@ def test_by_room_bucket_empty_when_no_bucket_data(): assert result.by_room_bucket == {} +def test_mapping_confidence_gate_in_sales_query(): + """OBJ-2 (#307): sales/bucket queries фильтруют objective_complex_mapping + по confidence — unreviewed low-score auto-matches исключены. + + Проверяем, что SQL, переданный в db.execute для маппинг-CTE, содержит + предикат gate (is_reviewed / manual / match_score >= 0.85). Это гарантирует, + что ~115 fuzzy-trgm строк с is_reviewed=false и score<0.85 не попадают в velocity. + """ + comp_rows = [_comp_row(1), _comp_row(2)] + sales_rows = [ + _sales_row(1, total_sqm=4000.0, months=4), + _sales_row(2, total_sqm=3000.0, months=4), + ] + bucket_rows = [_bucket_row(1, "1", units_sold=20, sqm_sold=900.0)] + db = _make_db(comp_rows=comp_rows, sales_rows=sales_rows, bucket_rows=bucket_rows) + + with patch( + "app.services.site_finder.velocity._get_ekb_median", + return_value=_EKB_MEDIAN_FALLBACK_SQM_PER_MONTH, + ): + compute_velocity(db, parcel_geom_wkt=_PARCEL_WKT) + + # db.execute вызывается 3 раза: comp / sales / bucket. SQL берём из call_args. + executed_sql = [str(call.args[0]) for call in db.execute.call_args_list] + # comp-query НЕ трогает mapping; sales (idx 1) и bucket (idx 2) — должны иметь gate. + mapping_queries = [s for s in executed_sql if "objective_complex_mapping" in s] + assert len(mapping_queries) >= 2, "ожидаются sales + bucket запросы с mapping" + for sql in mapping_queries: + assert "cm.is_reviewed = TRUE" in sql + assert "cm.match_method = 'manual'" in sql + assert "cm.match_score >= 0.85" in sql + + def test_sample_competitors_include_by_room_bucket(): """sample_competitors каждого элемента содержит by_room_bucket.""" comp_rows = [_comp_row(1), _comp_row(2)] diff --git a/data/sql/165_velocity_mapping_reviewed_gate.sql b/data/sql/165_velocity_mapping_reviewed_gate.sql new file mode 100644 index 00000000..1e2bb619 --- /dev/null +++ b/data/sql/165_velocity_mapping_reviewed_gate.sql @@ -0,0 +1,93 @@ +-- 165_velocity_mapping_reviewed_gate.sql +-- #307 OBJ-2 — gate objective_complex_mapping by confidence в mv_layout_velocity. +-- +-- Контекст: fuzzy-trgm backfill (#1331/#1333, миграции 155/156/116/117/150) добавил +-- ~115+ auto-matched строк в objective_complex_mapping с is_reviewed=false и низким +-- match_score (вплоть до 0.50-0.625). Они с высокой вероятностью false-positive +-- (Objective project name ≠ тот же ЖК что domrf_obj_id) и искажали velocity-агрегаты +-- Site Finder, попадая в MV наравне с проверенными маппингами. +-- +-- Fix: redefine MV с confidence-gate в JOIN — принимаем маппинг только если +-- is_reviewed = TRUE (человек подтвердил), ИЛИ +-- match_method = 'manual' (ручной маппинг, score обычно NULL), ИЛИ +-- match_score >= 0.85 (AUTO_ACCEPT_THRESHOLD из objective_backfill.py — +-- high-confidence auto, fuzzy_trgm 0.85+ надёжен). +-- +-- Строгий gate только на is_reviewed=true дал бы 2 строки из 303 EKB-маппингов → +-- обнулил бы velocity. Порог 0.85 сохраняет ~264/303, отбрасывая ~39 низкоуверенных. +-- Тот же предикат применён в runtime-пути backend/app/services/site_finder/velocity.py. +-- +-- Сохраняет weighted-average формулу из #21 (100_fix_mv_layout_velocity_weighted_avg.sql) — +-- меняется ТОЛЬКО WHERE/JOIN-условие на objective_complex_mapping. +-- +-- Dependencies: mv_layout_velocity не имеет зависимых VIEW/MV (проверено 2026-05-17/100_fix). +-- Idempotency: DROP IF EXISTS → безопасен при повторном запуске. +-- Deploy order: SQL migration → app deploy. Auto-applied через _schema_migrations. +-- ВАЖНО: DROP + CREATE + REFRESH (non-concurrent, блокировка ~30 с на ~19 738 source rows). + +BEGIN; + +DROP MATERIALIZED VIEW IF EXISTS mv_layout_velocity CASCADE; + +CREATE MATERIALIZED VIEW mv_layout_velocity AS +WITH last24mo AS ( + SELECT + ocm.project_name, + CASE + WHEN ocm.room_bucket = 'студия' THEN 'studio' + ELSE ocm.room_bucket + END AS room_bucket, + ocm.deals_total_count, + ocm.deals_total_avg_area_m2, + ocm.deals_total_avg_price_thousand_rub_per_m2, + ocm.deals_total_vol_m2, + ocm.report_month + FROM objective_corpus_room_month ocm + WHERE ocm.report_month >= (NOW() - INTERVAL '24 months')::date +) +SELECT + cm.domrf_obj_id AS obj_id, + l.room_bucket, + SUM(l.deals_total_count)::int AS total_deals_24mo, + + -- weighted average (#21) — нули из месяцев без сделок не тянут вниз. + (SUM(l.deals_total_avg_area_m2 * l.deals_total_count) + / NULLIF(SUM(l.deals_total_count), 0))::numeric(10, 2) AS avg_area_m2, + + (SUM(l.deals_total_avg_price_thousand_rub_per_m2 * l.deals_total_count) + / NULLIF(SUM(l.deals_total_count), 0))::numeric(12, 2) AS avg_price_thousand_rub_per_m2, + + SUM(l.deals_total_vol_m2)::numeric(12, 2) AS total_vol_m2, + MIN(l.report_month) AS window_start, + MAX(l.report_month) AS window_end, + COUNT(DISTINCT l.report_month)::int AS months_with_data +FROM last24mo l +JOIN objective_complex_mapping cm + ON cm.objective_complex_name = l.project_name +WHERE l.room_bucket IS NOT NULL + AND cm.domrf_obj_id IS NOT NULL + AND cm.objective_group = 'Екатеринбург' + -- #307 OBJ-2: confidence-gate — исключаем unreviewed low-score auto-matches. + AND (cm.is_reviewed = TRUE OR cm.match_method = 'manual' OR cm.match_score >= 0.85) +GROUP BY cm.domrf_obj_id, l.room_bucket +WITH NO DATA; + +-- UNIQUE index: required for REFRESH CONCURRENTLY (periodic via layout_velocity_refresh.py). +CREATE UNIQUE INDEX mv_layout_velocity_pk + ON mv_layout_velocity (obj_id, room_bucket); + +-- Lookup index for /best-layouts endpoint queries by obj_id. +CREATE INDEX mv_layout_velocity_obj_idx + ON mv_layout_velocity (obj_id); + +-- Initial populate (non-concurrent — MV just created, CONCURRENTLY requires populated MV). +REFRESH MATERIALIZED VIEW mv_layout_velocity; + +COMMENT ON MATERIALIZED VIEW mv_layout_velocity IS + 'Per-(obj_id, room_bucket) deals aggregation за last 24 months. ' + 'WEIGHTED average площади и цены (SF Bug #21). ' + 'Confidence-gated mapping: is_reviewed/manual/score>=0.85 (#307 OBJ-2). ' + 'Source: objective_corpus_room_month × objective_complex_mapping (EKB only). ' + 'Refresh via layout_velocity_refresh.py (CONCURRENTLY после initial populate).'; + +COMMIT; From 2cf6261005e7bb3d234485e5cfb061a4367ad52e Mon Sep 17 00:00:00 2001 From: bot-backend Date: Wed, 17 Jun 2026 22:36:39 +0300 Subject: [PATCH 14/15] =?UTF-8?q?chore(sql):=20renumber=20sales-tracker=20?= =?UTF-8?q?MV=20migration=20161=E2=86=92164=20(avoid=20collision)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/services/site_finder/sales_tracker_mv_refresh.py | 2 +- backend/app/workers/tasks/mv_sales_tracker_refresh.py | 2 +- ...orption.sql => 164_mv_sales_tracker_velocity_absorption.sql} | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) rename data/sql/{161_mv_sales_tracker_velocity_absorption.sql => 164_mv_sales_tracker_velocity_absorption.sql} (99%) diff --git a/backend/app/services/site_finder/sales_tracker_mv_refresh.py b/backend/app/services/site_finder/sales_tracker_mv_refresh.py index fa2afed1..f5943bb9 100644 --- a/backend/app/services/site_finder/sales_tracker_mv_refresh.py +++ b/backend/app/services/site_finder/sales_tracker_mv_refresh.py @@ -2,7 +2,7 @@ Two independent materialized views built from the Объектив sales-tracker ("шахматки") snapshots (objective_lots / objective_lots_history), created by -data/sql/161_mv_sales_tracker_velocity_absorption.sql: +data/sql/164_mv_sales_tracker_velocity_absorption.sql: 1. mv_sales_tracker_velocity_by_district — per (district, month) sold/total/ avg-sold-price. Feeds the Site Finder Velocity Score (4th scoring criterion). diff --git a/backend/app/workers/tasks/mv_sales_tracker_refresh.py b/backend/app/workers/tasks/mv_sales_tracker_refresh.py index a38790be..badee03f 100644 --- a/backend/app/workers/tasks/mv_sales_tracker_refresh.py +++ b/backend/app/workers/tasks/mv_sales_tracker_refresh.py @@ -11,7 +11,7 @@ Both MVs are built from the Объектив sales-tracker ("шахматки") (objective_lots / objective_lots_history). Source data refreshes via the objective_sync beat job, so a weekly MV refresh keeps the aggregates current. -MV-source migration: data/sql/161_mv_sales_tracker_velocity_absorption.sql. +MV-source migration: data/sql/164_mv_sales_tracker_velocity_absorption.sql. """ from __future__ import annotations diff --git a/data/sql/161_mv_sales_tracker_velocity_absorption.sql b/data/sql/164_mv_sales_tracker_velocity_absorption.sql similarity index 99% rename from data/sql/161_mv_sales_tracker_velocity_absorption.sql rename to data/sql/164_mv_sales_tracker_velocity_absorption.sql index 3ffeea5e..5ce58fd8 100644 --- a/data/sql/161_mv_sales_tracker_velocity_absorption.sql +++ b/data/sql/164_mv_sales_tracker_velocity_absorption.sql @@ -1,4 +1,4 @@ --- 161_mv_sales_tracker_velocity_absorption.sql +-- 164_mv_sales_tracker_velocity_absorption.sql -- Issue #61 — Velocity materialized views for Site Finder Velocity Score (4th scoring -- criterion) + recommend_mix smart unit-mix. Foundation for sellout forecast. -- From 65b0c02b85c99dd2b71ddf93c6990c73f58291ad Mon Sep 17 00:00:00 2001 From: bot-backend Date: Wed, 17 Jun 2026 22:38:44 +0300 Subject: [PATCH 15/15] test(etl): fix stale assert in objective_backfill dry_run test (#1709) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit backend-tests падал СИСТЕМНО на каждом PR на test_auto_apply_matches_dry_run_no_inserts (assert 1 == 0). Root cause: аудит-коммит 14f3ef20 (#1660) намеренно изменил auto_apply_matches — dry_run теперь возвращает projected-счётчики (auto_accepted=len(auto), сколько БЫ приняли — смысл preview в admin_etl endpoint'е), но тест в том же коммите не обновили, он остался с assert auto_accepted == 0. Функция корректна (dry_run делает ранний return до db.execute/commit — execute.assert_not_called() валиден). Правлю устаревшую ассерцию: == 0 → == 1 (кандидат 0.95 >= AUTO_ACCEPT_THRESHOLD 0.85), + assert skipped == 0. Полный сьют: 3277 passed, 0 failed. Refs #1709 --- backend/tests/services/test_objective_backfill.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/backend/tests/services/test_objective_backfill.py b/backend/tests/services/test_objective_backfill.py index 3da13880..99197115 100644 --- a/backend/tests/services/test_objective_backfill.py +++ b/backend/tests/services/test_objective_backfill.py @@ -77,7 +77,12 @@ def test_find_match_candidates_returns_candidates() -> None: def test_auto_apply_matches_dry_run_no_inserts() -> None: - """dry_run=True возвращает счётчики без обращения к БД (execute не вызывается).""" + """dry_run=True возвращает projected-счётчики без обращения к БД. + + auto_accepted = сколько кандидатов БЫЛО БЫ принято (preview), а не 0 — + смысл dry-run в admin-endpoint'е именно показать оператору объём перед + реальным insert. execute/commit при этом не вызываются. + """ mock_db = MagicMock() candidates = [ @@ -88,8 +93,9 @@ def test_auto_apply_matches_dry_run_no_inserts() -> None: result = auto_apply_matches(mock_db, candidates, dry_run=True) - assert result["auto_accepted"] == 0 + assert result["auto_accepted"] == 1 # projected: 1 кандидат >= AUTO_ACCEPT_THRESHOLD assert result["review_queue"] == 2 + assert result["skipped"] == 0 mock_db.execute.assert_not_called() mock_db.commit.assert_not_called()