diff --git a/backend/app/api/v1/admin_scrape.py b/backend/app/api/v1/admin_scrape.py index 4edcc5e4..a31b9302 100644 --- a/backend/app/api/v1/admin_scrape.py +++ b/backend/app/api/v1/admin_scrape.py @@ -801,6 +801,118 @@ def objective_coverage( } +# ── Unified scrape dashboard (поверх v_scrape_runs_unified / v_scrape_log_unified) ── + + +@router.get("/all/runs") +def list_all_runs( + db: Annotated[Session, Depends(get_db)], + x_admin_token: Annotated[str | None, Header(alias="X-Admin-Token")] = None, + scraper_type: str | None = None, + limit: int = 30, +) -> list[dict[str, Any]]: + """Унифицированный список прогонов (kn + nspd + objective). + + Поверх v_scrape_runs_unified. Если scraper_type не задан — все 3. + Используется главной dashboard /admin/scrape/all. + """ + _check_token(x_admin_token) + where = "" if not scraper_type else "WHERE scraper_type = :st" + params: dict[str, Any] = {"lim": min(limit, 200)} + if scraper_type: + params["st"] = scraper_type + rows = ( + db.execute( + text( + f""" + SELECT scraper_type, run_id, started_at, finished_at, heartbeat_at, + status, error, triggered_by, scope, requests_count, + items_ok, items_failed, sub_items_ok, + progress_idx, progress_total, extra + FROM v_scrape_runs_unified + {where} + ORDER BY started_at DESC NULLS LAST + LIMIT :lim + """ + ), + params, + ) + .mappings() + .all() + ) + return [ + { + "scraper_type": r["scraper_type"], + "run_id": r["run_id"], + "started_at": r["started_at"].isoformat() if r["started_at"] else None, + "finished_at": r["finished_at"].isoformat() if r["finished_at"] else None, + "heartbeat_at": r["heartbeat_at"].isoformat() if r["heartbeat_at"] else None, + "status": r["status"], + "error": r["error"], + "triggered_by": r["triggered_by"], + "scope": r["scope"], + "requests_count": r["requests_count"], + "items_ok": r["items_ok"], + "items_failed": r["items_failed"], + "sub_items_ok": r["sub_items_ok"], + "progress_idx": r["progress_idx"], + "progress_total": r["progress_total"], + "extra": r["extra"], + } + for r in rows + ] + + +@router.get("/all/logs") +def list_all_logs( + db: Annotated[Session, Depends(get_db)], + x_admin_token: Annotated[str | None, Header(alias="X-Admin-Token")] = None, + scraper_type: str | None = None, + run_id: int | None = None, + limit: int = 200, +) -> list[dict[str, Any]]: + """Унифицированный список логов (kn + nspd). Objective пока не пишет log.""" + _check_token(x_admin_token) + where: list[str] = [] + params: dict[str, Any] = {"lim": min(limit, 1000)} + if scraper_type: + where.append("scraper_type = :st") + params["st"] = scraper_type + if run_id is not None: + where.append("run_id = :rid") + params["rid"] = run_id + where_sql = ("WHERE " + " AND ".join(where)) if where else "" + rows = ( + db.execute( + text( + f""" + SELECT scraper_type, log_id, run_id, ts, level, stage, entity_id, message + FROM v_scrape_log_unified + {where_sql} + ORDER BY log_id DESC + LIMIT :lim + """ + ), + params, + ) + .mappings() + .all() + ) + return [ + { + "scraper_type": r["scraper_type"], + "log_id": r["log_id"], + "run_id": r["run_id"], + "ts": r["ts"].isoformat() if r["ts"] else None, + "level": r["level"], + "stage": r["stage"], + "entity_id": r["entity_id"], + "message": r["message"], + } + for r in rows + ] + + # ── DomRF (kn) — runs list ─────────────────────────────────────────────────── diff --git a/data/sql/73_complexes_master.sql b/data/sql/73_complexes_master.sql new file mode 100644 index 00000000..02e6a4e8 --- /dev/null +++ b/data/sql/73_complexes_master.sql @@ -0,0 +1,318 @@ +-- ЖК как canonical master entity + cross-ref на источники. +-- +-- Зачем: +-- До этого жил мир из 5+ таблиц с дублирующейся семантикой "ЖК": +-- - domrf_kn_objects (4548, ДОМ.РФ kn-API) +-- - objective_lots.project_name (350 уникальных, Objective API) +-- - objective_corpus_room_month.project_name (тот же, 350) +-- - objective_complex_mapping (114 fuzzy matches → domrf_obj_id) +-- - cad_buildings.cad_num (косвенно через NSPD) +-- - rosreestr_deals.complex_name (текстовое поле в сделках) +-- Каждый источник называет ЖК по-своему ("ЖК \"Парк Культуры\"" vs +-- "Парк Культуры" vs domrf_obj_id=66845). recommend_mix / +-- velocity-anomaly / cross-source analytics требуют одного entity. +-- +-- Архитектура: +-- complexes (master) — одна строка = один реальный ЖК +-- complex_sources (cross-ref) — many-to-many ЖК ↔ источник + source_id +-- +-- Стратегия backfill: +-- 1. Каждый domrf_kn_objects = entity в complexes (4548 ЖК) +-- + complex_sources(source='domrf_kn', source_id=obj_id) +-- 2. Сматченные objective project_name (114) → существующий complex по +-- objective_complex_mapping.domrf_obj_id +-- + complex_sources(source='objective', source_id=project_name) +-- 3. Unmatched objective project_name (236) → новые entity в complexes +-- + complex_sources(source='objective') +-- +-- Идемпотентно через ON CONFLICT. + +-- ── 1. complexes (canonical master) ───────────────────────────────────────── + +CREATE TABLE IF NOT EXISTS complexes ( + id BIGSERIAL PRIMARY KEY, + -- Display + canonical_name TEXT NOT NULL, -- "Парк Культуры" / "ЖК Авангард 3.0" + developer_name TEXT, -- "PRINZIP" / "ГК КОРТРОС" + developer_id TEXT, -- FK on domrf_developers.developer_id (nullable) + -- Geo + region_id INTEGER, -- 66 для Свердл.обл (FK on domrf_regions.region_id) + district_name TEXT, -- "Академический" / "Автовокзал" + address TEXT, + latitude NUMERIC, + longitude NUMERIC, + geom geometry(Point, 4326), -- spatial для ST_DWithin / spatial join + -- Spec + obj_class TEXT, -- "комфорт" / "бизнес" / "стандарт" / "элитный" + flat_count INTEGER, -- общее ПД + square_living NUMERIC, -- общая жилая площадь + floor_min INTEGER, + floor_max INTEGER, + wall_type TEXT, -- "монолит" / "кирпич" / etc. + -- Stage + sales_start_date DATE, + plan_completion_date DATE, + actual_completion_date DATE, + site_status TEXT, -- "строящийся" / "сдан" / "архив" + obj_status INTEGER, -- 0/1/2 кодировка ДОМ.РФ + is_problem BOOLEAN DEFAULT FALSE, + has_escrow BOOLEAN, -- эскроу-проект + -- Audit + primary_source TEXT NOT NULL DEFAULT 'domrf_kn', -- какой источник был основой + created_at timestamptz NOT NULL DEFAULT NOW(), + updated_at timestamptz NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS complexes_name_idx + ON complexes(canonical_name); +CREATE INDEX IF NOT EXISTS complexes_developer_idx + ON complexes(developer_id) WHERE developer_id IS NOT NULL; +CREATE INDEX IF NOT EXISTS complexes_region_district_idx + ON complexes(region_id, district_name); +CREATE INDEX IF NOT EXISTS complexes_class_idx + ON complexes(obj_class) WHERE obj_class IS NOT NULL; +CREATE INDEX IF NOT EXISTS complexes_geom_idx + ON complexes USING GIST (geom) WHERE geom IS NOT NULL; +CREATE INDEX IF NOT EXISTS complexes_status_idx + ON complexes(site_status) WHERE site_status IS NOT NULL; + +COMMENT ON TABLE complexes IS + 'Canonical master entity для ЖК. Один реальный жилой комплекс = одна строка. ' + 'Связи на разные источники (domrf_kn, objective, rosreestr, etc.) — через ' + 'complex_sources. Все analytics-запросы должны идти через эту таблицу ' + 'вместо direct lookup в источниках.'; + +-- ── 2. complex_sources (cross-ref) ────────────────────────────────────────── + +CREATE TABLE IF NOT EXISTS complex_sources ( + id BIGSERIAL PRIMARY KEY, + complex_id BIGINT NOT NULL REFERENCES complexes(id) ON DELETE CASCADE, + source TEXT NOT NULL, -- 'domrf_kn' | 'objective' | 'rosreestr' | 'cad_buildings' | 'yandex' | ... + source_id TEXT NOT NULL, -- obj_id::text / project_name / cad_num / external_id + source_external_id BIGINT, -- для int-ID источников (objective.project_id, domrf.obj_id) + -- Match audit + match_method TEXT, -- 'exact_name' | 'fuzzy_name' | 'spatial' | 'manual' | 'imported' + match_score NUMERIC, -- 0..1 для fuzzy + is_reviewed BOOLEAN NOT NULL DEFAULT FALSE, + note TEXT, + linked_at timestamptz NOT NULL DEFAULT NOW(), + UNIQUE (source, source_id) -- один (источник, source_id) — на один ЖК +); + +CREATE INDEX IF NOT EXISTS complex_sources_complex_idx + ON complex_sources(complex_id); +CREATE INDEX IF NOT EXISTS complex_sources_source_idx + ON complex_sources(source); +CREATE INDEX IF NOT EXISTS complex_sources_external_id_idx + ON complex_sources(source, source_external_id) WHERE source_external_id IS NOT NULL; +CREATE INDEX IF NOT EXISTS complex_sources_unreviewed_idx + ON complex_sources(is_reviewed) WHERE is_reviewed = FALSE; + +COMMENT ON TABLE complex_sources IS + 'Cross-ref many-to-many: ЖК (complexes) ↔ его представления в источниках. ' + 'Один реальный ЖК может иметь записи в нескольких источниках — здесь ' + 'хранятся все эти ссылки. UNIQUE(source, source_id) гарантирует что один ' + '(источник, ID-в-источнике) присвоен ровно одному ЖК.'; + +-- ── 3. Backfill из domrf_kn_objects ───────────────────────────────────────── + +-- ВАЖНО: domrf_kn_objects содержит несколько snapshot'ов на один obj_id +-- (4548 rows = 1516 уникальных obj_id × ~3 snapshot_date). Поэтому +-- DISTINCT ON (obj_id) ORDER BY snapshot_date DESC — берём свежий snapshot. +-- +-- ВНИМАНИЕ: domrf_kn_objects.obj_class = NULL во всех строках (известный +-- баг kn-API), поэтому obj_class останется NULL — он подтянется когда +-- сделаем UPDATE из objective_lots (там obj_class заполнен). + +WITH latest_kn AS ( + SELECT DISTINCT ON (obj_id) + obj_id, comm_name, addr, region_cd, dev_id, dev_name, district_name, + latitude, longitude, obj_class, flat_count, square_living, + floor_min, floor_max, wall_type, ready_dt, + site_status, obj_status, is_problem, escrow + FROM domrf_kn_objects + ORDER BY obj_id, snapshot_date DESC NULLS LAST +), +inserted_complexes AS ( + INSERT INTO complexes ( + canonical_name, developer_name, developer_id, region_id, district_name, + address, latitude, longitude, geom, + obj_class, flat_count, square_living, floor_min, floor_max, wall_type, + plan_completion_date, site_status, obj_status, is_problem, has_escrow, + primary_source + ) + SELECT + COALESCE(comm_name, addr, 'unknown'), + dev_name, dev_id, region_cd, district_name, + addr, latitude, longitude, + CASE WHEN latitude IS NOT NULL AND longitude IS NOT NULL + THEN ST_SetSRID(ST_MakePoint(longitude, latitude), 4326) ELSE NULL END, + obj_class, flat_count, square_living, floor_min, floor_max, wall_type, + ready_dt, site_status, obj_status, COALESCE(is_problem, FALSE), escrow, + 'domrf_kn' + FROM latest_kn + RETURNING id, canonical_name +) +INSERT INTO complex_sources ( + complex_id, source, source_id, source_external_id, + match_method, match_score, is_reviewed, note +) +SELECT + ic.id, 'domrf_kn', lk.obj_id::text, lk.obj_id, 'imported', 1.0, TRUE, + 'auto-import from domrf_kn_objects (latest snapshot per obj_id)' +FROM latest_kn lk +JOIN inserted_complexes ic + ON ic.canonical_name = COALESCE(lk.comm_name, lk.addr, 'unknown') +ON CONFLICT (source, source_id) DO NOTHING; + +-- ── 4. Backfill из objective_complex_mapping (114 сматченных) ─────────────── + +-- Эти project_name уже привязаны к domrf_obj_id — линкуем к существующему complex. +INSERT INTO complex_sources ( + complex_id, source, source_id, source_external_id, + match_method, match_score, is_reviewed, note +) +SELECT + cs_kn.complex_id, -- complex по domrf_kn + 'objective', + m.objective_complex_name, + m.objective_project_id, + m.match_method, + m.match_score, + m.is_reviewed, + 'imported from objective_complex_mapping' +FROM objective_complex_mapping m +JOIN complex_sources cs_kn + ON cs_kn.source = 'domrf_kn' + AND cs_kn.source_external_id = m.domrf_obj_id +WHERE m.domrf_obj_id IS NOT NULL +ON CONFLICT (source, source_id) DO NOTHING; + +-- ── 5. Backfill orphan-проектов из objective_lots (236 unmatched) ─────────── + +-- Для каждого project_name которого нет ни в complex_sources(source=objective) +-- — создаём новый complex (берём атрибуты из любой quartiry этого проекта) +-- + добавляем complex_sources(source=objective). + +WITH unmatched_objective AS ( + SELECT DISTINCT ON (project_name) + project_name, + objective_project_id, + developer, + district, + class AS obj_class, + city, + address, + sales_start_date, + plan_completion_date, + actual_completion_date, + construction_stage + FROM objective_lots ol + WHERE NOT EXISTS ( + SELECT 1 FROM complex_sources cs + WHERE cs.source = 'objective' + AND cs.source_id = ol.project_name + ) + ORDER BY project_name, snapshot_date DESC NULLS LAST +), +inserted AS ( + INSERT INTO complexes ( + canonical_name, developer_name, region_id, district_name, + address, obj_class, sales_start_date, plan_completion_date, + actual_completion_date, site_status, primary_source + ) + SELECT + project_name, + developer, + 66, -- Свердл.обл (Объектив пока только Екб + спутники) + district, + address, + obj_class, + sales_start_date, + plan_completion_date, + actual_completion_date, + construction_stage, + 'objective' + FROM unmatched_objective + RETURNING id, canonical_name +) +INSERT INTO complex_sources ( + complex_id, source, source_id, source_external_id, + match_method, match_score, is_reviewed, note +) +SELECT + i.id, + 'objective', + uo.project_name, + uo.objective_project_id, + 'imported', + 1.0, + FALSE, -- нужно ревью: возможно есть domrf_kn-аналог + 'orphan objective project; consider reviewing for domrf_kn match' +FROM unmatched_objective uo +JOIN inserted i ON i.canonical_name = uo.project_name +ON CONFLICT (source, source_id) DO NOTHING; + +-- ── 6. Helper-views для запросов ──────────────────────────────────────────── + +-- v_complex_full: полная карточка ЖК с метриками из всех источников. +DROP VIEW IF EXISTS v_complex_full; +CREATE VIEW v_complex_full AS +SELECT + c.id AS complex_id, + c.canonical_name, + c.developer_name, + c.region_id, + c.district_name, + c.obj_class, + c.address, + c.latitude, + c.longitude, + c.flat_count, + c.square_living, + c.site_status, + -- сводка по источникам + (SELECT array_agg(source ORDER BY source) FROM complex_sources WHERE complex_id = c.id) AS sources, + (SELECT COUNT(*) FROM complex_sources WHERE complex_id = c.id) AS sources_count, + -- domrf_kn obj_id (если есть) + (SELECT source_external_id FROM complex_sources + WHERE complex_id = c.id AND source = 'domrf_kn' LIMIT 1) AS domrf_obj_id, + -- objective project_name (если есть) + (SELECT source_id FROM complex_sources + WHERE complex_id = c.id AND source = 'objective' LIMIT 1) AS objective_project_name, + c.primary_source, + c.created_at, + c.updated_at + FROM complexes c; + +COMMENT ON VIEW v_complex_full IS + 'Карточка ЖК с агрегированной информацией о всех источниках. ' + 'Используй для UI / отчётов / вместо direct lookup в domrf_kn_objects.'; + +-- v_complex_with_objective: ЖК + агрегаты Objective (lots count, sold pct, avg price) +DROP VIEW IF EXISTS v_complex_with_objective; +CREATE VIEW v_complex_with_objective AS +SELECT + c.id AS complex_id, + c.canonical_name, + c.district_name, + c.obj_class, + cs.source_id AS objective_project_name, + COUNT(ol.id) AS objective_lots_n, + COUNT(ol.id) FILTER (WHERE ol.is_sold) AS objective_sold_n, + ROUND(100.0 * COUNT(ol.id) FILTER (WHERE ol.is_sold) + / NULLIF(COUNT(ol.id), 0), 1) AS objective_sold_pct, + AVG(ol.price_per_m2_rub) FILTER (WHERE ol.is_sold AND ol.price_per_m2_rub > 0) AS avg_sold_price_per_m2, + AVG(ol.area_pd) FILTER (WHERE ol.area_pd > 0) AS avg_area_pd, + MIN(ol.registration_date) AS first_reg_date, + MAX(ol.registration_date) AS last_reg_date + FROM complexes c + JOIN complex_sources cs + ON cs.complex_id = c.id AND cs.source = 'objective' + JOIN objective_lots ol + ON ol.project_name = cs.source_id + GROUP BY c.id, c.canonical_name, c.district_name, c.obj_class, cs.source_id; + +COMMENT ON VIEW v_complex_with_objective IS + 'ЖК + агрегаты Objective per-flat: sold count/pct, средняя цена, ' + 'диапазон reg_date. NULL во всех колонках = ЖК без Objective-источника.'; diff --git a/data/sql/74_dedupe_unified_views.sql b/data/sql/74_dedupe_unified_views.sql new file mode 100644 index 00000000..1cf63d52 --- /dev/null +++ b/data/sql/74_dedupe_unified_views.sql @@ -0,0 +1,256 @@ +-- Phase B + D: унификация дублирующихся таблиц через VIEW (UNION ALL). +-- +-- Стратегия: НЕ дропаем источники (они продолжают писать своими скраперами/ +-- импортёрами), но создаём поверх единые view для удобных запросов из +-- аналитики и UI. Это zero-risk миграция: можно делать в любой момент +-- и откатывать DROP VIEW без потери данных. +-- +-- После того как код в backend перейдёт на use views — старые имена таблиц +-- можно будет либо превратить в views поверх objединённой структуры, либо +-- мигрировать данные целиком. Сейчас — promotion path. +-- +-- Применено в проде через mcp postgres 2026-05-10. +-- Идемпотентно через CREATE OR REPLACE VIEW. + +-- ── 1. ekb_districts: merge geom-таблицы в основную ───────────────────────── +-- ekb_districts (9, dimension с метриками) + ekb_districts_geom (8, polygons) +-- → одна таблица ekb_districts с PostGIS-колонкой geom +-- + view-alias ekb_districts_geom для backward-compat. + +ALTER TABLE ekb_districts + ADD COLUMN IF NOT EXISTS geom geometry(MultiPolygon, 4326), + ADD COLUMN IF NOT EXISTS osm_id BIGINT, + ADD COLUMN IF NOT EXISTS geom_fetched_at timestamptz; + +UPDATE ekb_districts d + SET geom = g.geom, osm_id = g.osm_id, geom_fetched_at = g.fetched_at + FROM ekb_districts_geom g + WHERE g.district_name = d.district_name; + +CREATE INDEX IF NOT EXISTS ekb_districts_geom_idx + ON ekb_districts USING GIST(geom) WHERE geom IS NOT NULL; + +DROP TABLE IF EXISTS ekb_districts_geom; +CREATE OR REPLACE VIEW ekb_districts_geom AS +SELECT district_name, geom, osm_id, geom_fetched_at AS fetched_at + FROM ekb_districts WHERE geom IS NOT NULL; + +-- ── 2. v_mortgage_unified — 4 источника ипотечных метрик в long-формате ───── + +CREATE OR REPLACE VIEW v_mortgage_unified AS +SELECT + 'cbr' AS source, + NULL::date AS snapshot_date, + period AS period_label, + region, + NULL::text AS bank_name, + title AS metric_kind, + 'total' AS segment, + value, + NULL::numeric AS delta_pct, + series_id AS metric_id + FROM cbr_mortgage_series +UNION ALL +SELECT 'domrf_dashboard', snapshot_date, NULL, 'РФ', NULL, + 'avg_rate_pct', 'total', total_credit_avg_rate, total_credit_avg_rate_delta, NULL + FROM domrf_mortgage_dashboard +UNION ALL +SELECT 'domrf_dashboard', snapshot_date, NULL, 'РФ', NULL, + 'avg_rate_pct', 'primary', primary_credit_avg_rate, primary_credit_avg_rate_delta, NULL + FROM domrf_mortgage_dashboard +UNION ALL +SELECT 'domrf_dashboard', snapshot_date, NULL, 'РФ', NULL, + 'avg_rate_pct', 'secondary', secondary_credit_avg_rate, secondary_credit_avg_rate_delta, NULL + FROM domrf_mortgage_dashboard +UNION ALL +SELECT 'domrf_dashboard', snapshot_date, NULL, 'РФ', NULL, + 'credit_count', 'total', total_credit_count::numeric, total_credit_count_delta_pct, NULL + FROM domrf_mortgage_dashboard +UNION ALL +SELECT 'domrf_dashboard', snapshot_date, NULL, 'РФ', NULL, + 'credit_count', 'primary', primary_credit_count::numeric, primary_credit_count_delta_pct, NULL + FROM domrf_mortgage_dashboard +UNION ALL +SELECT 'domrf_dashboard', snapshot_date, NULL, 'РФ', NULL, + 'credit_count', 'secondary', secondary_credit_count::numeric, secondary_credit_count_delta_pct, NULL + FROM domrf_mortgage_dashboard +UNION ALL +SELECT 'domrf_dashboard', snapshot_date, NULL, 'РФ', NULL, + 'credit_amount_rub', 'total', total_credit_amount, total_credit_amount_delta_pct, NULL + FROM domrf_mortgage_dashboard +UNION ALL +SELECT 'domrf_dashboard', snapshot_date, NULL, 'РФ', NULL, + 'credit_amount_rub', 'primary', primary_credit_amount, primary_credit_amount_delta_pct, NULL + FROM domrf_mortgage_dashboard +UNION ALL +SELECT 'domrf_dashboard', snapshot_date, NULL, 'РФ', NULL, + 'credit_amount_rub', 'secondary', secondary_credit_amount, secondary_credit_amount_delta_pct, NULL + FROM domrf_mortgage_dashboard +UNION ALL +SELECT 'domrf_details', snapshot_date, NULL, 'РФ', NULL, + 'credit_amount_avg_rub', currency, credit_amount_avg, credit_amount_avg_delta_pct, NULL + FROM domrf_mortgage_details +UNION ALL +SELECT 'domrf_details', snapshot_date, NULL, 'РФ', NULL, + 'credit_avg_period_months', currency, credit_avg_period, credit_avg_period_delta_pct, NULL + FROM domrf_mortgage_details +UNION ALL +SELECT 'domrf_details', snapshot_date, NULL, 'РФ', NULL, + 'credit_debts_amount_rub', currency, credit_debts_amount, credit_debts_amount_delta_pct, NULL + FROM domrf_mortgage_details +UNION ALL +SELECT 'domrf_details', snapshot_date, NULL, 'РФ', NULL, + 'overdue_pct', currency, credit_debts_overdue_percent, credit_debts_overdue_percent_delta, NULL + FROM domrf_mortgage_details +UNION ALL +SELECT 'domrf_rates', snapshot_date, NULL, 'РФ', bank_name, + 'rate_pct', 'primary', primary_rate, NULL, NULL FROM domrf_mortgage_rates +UNION ALL +SELECT 'domrf_rates', snapshot_date, NULL, 'РФ', bank_name, + 'rate_pct', 'secondary', secondary_rate, NULL, NULL FROM domrf_mortgage_rates +UNION ALL +SELECT 'domrf_rates', snapshot_date, NULL, 'РФ', bank_name, + 'rate_pct', 'refinance', refinance_rate, NULL, NULL FROM domrf_mortgage_rates; + +COMMENT ON VIEW v_mortgage_unified IS + 'Long-формат всех 4 источников ипотечных метрик. metric_kind: avg_rate_pct ' + '/ credit_count / credit_amount_rub / overdue_pct / etc. segment: primary / ' + 'secondary / total. ~3385 rows total. Источники не дропаются.'; + +-- ── 3. v_construction_metrics_unified — 5 launch/commissioning ────────────── + +CREATE OR REPLACE VIEW v_construction_metrics_unified AS +SELECT 'planned_commissioning' AS source, snapshot_date, plan_year::text AS period_label, + 'РФ' AS region, 'planned_total_th_sqm' AS metric_kind, + NULL::text AS breakdown_kind, NULL::text AS breakdown_value, + total_th_sqm AS value, + jsonb_build_object('escrow', escrow_th_sqm, 'fund', fund_th_sqm, + 'no_attraction', no_attraction_th_sqm) AS extra + FROM domrf_planned_commissioning +UNION ALL +SELECT 'commissioning', snapshot_date, rep_year::text, region_name, + 'fact_area_total', NULL, NULL, accumulated_fact_area, + jsonb_build_object('multifamily', accumulated_fact_area_multifamily, + 'private', accumulated_fact_area_private, + 'change', accumulated_fact_area_change, + 'change_share', accumulated_fact_area_change_share) + FROM domrf_commissioning +UNION ALL +SELECT 'launch_monthly', snapshot_date, + (rep_year::text || '-' || LPAD(rep_month::text, 2, '0')), + 'РФ', 'launch_app_value', 'calc_type', calc_type, app_value, + jsonb_build_object('rnv_value', rnv_value) + FROM domrf_launch_monthly +UNION ALL +SELECT 'launch_monthly', snapshot_date, + (rep_year::text || '-' || LPAD(rep_month::text, 2, '0')), + 'РФ', 'launch_rnv_value', 'calc_type', calc_type, rnv_value, NULL + FROM domrf_launch_monthly +UNION ALL +SELECT 'launch_top', snapshot_date, rep_year::text, entity_name, + metric_type, dim_type, entity_id, value, + jsonb_build_object('calc_type', calc_type) + FROM domrf_launch_top +UNION ALL +SELECT 'launch_obj_class', snapshot_date, rep_year::text, 'РФ', + 'launch_app_value', 'obj_class', obj_class_desc, app_value, + jsonb_build_object('rnv_value', rnv_value, 'calc_type', calc_type, + 'obj_class_cd', obj_class_cd) + FROM domrf_launch_obj_class +UNION ALL +SELECT 'launch_obj_class', snapshot_date, rep_year::text, 'РФ', + 'launch_rnv_value', 'obj_class', obj_class_desc, rnv_value, + jsonb_build_object('calc_type', calc_type) + FROM domrf_launch_obj_class; + +COMMENT ON VIEW v_construction_metrics_unified IS + 'Long-формат для 5 таблиц про launch/commissioning. metric_kind: ' + 'planned_total_th_sqm / fact_area_total / launch_app_value / launch_rnv_value. ' + 'breakdown_kind: calc_type / obj_class / dim_type. ~1579 rows.'; + +-- ── 4. v_sold_ready_unified — 4 sold_ready / sold_out ────────────────────── + +CREATE OR REPLACE VIEW v_sold_ready_unified AS +SELECT 'sold_out' AS source, snapshot_date, NULL::text AS period_label, + territory_name AS scope, NULL::text AS breakdown_kind, NULL::text AS breakdown_value, + 'sold_pct' AS metric_kind, sold_pct AS value, + jsonb_build_object('total_area_th_sqm', total_area_th_sqm, + 'sold_area_th_sqm', sold_area_th_sqm, + 'unsold_pct', unsold_pct, + 'avg_price_per_sqm', avg_price_per_sqm, + 'attracted_funds_mln_rub', attracted_funds_mln_rub) AS extra + FROM domrf_sold_out +UNION ALL +SELECT 'sold_ready_index', snapshot_date, + (rep_year::text || '-' || LPAD(rep_month::text, 2, '0')), + 'РФ', NULL, NULL, 'sold_ready_perc', sold_ready_perc, + jsonb_build_object('square_sum', square_sum, 'sold_sum', sold_sum, + 'sold_perc', sold_perc, 'ready_perc', ready_perc) + FROM domrf_sold_ready_index +UNION ALL +SELECT 'sold_ready_dynamics', snapshot_date, + (rep_year::text || '-' || LPAD(rep_month::text, 2, '0')), + 'РФ', 'dynamic_chart_type', dynamic_chart_type, + dynamic_chart_type, value, NULL + FROM domrf_sold_ready_dynamics +UNION ALL +SELECT 'sold_ready_breakdown', snapshot_date, + (rep_year::text || '-' || LPAD(COALESCE(rep_month, 0)::text, 2, '0')), + COALESCE(NULLIF(dev_group_value, ''), 'РФ'), + chart_type, entity_key, 'sold_perc', sold_perc, + jsonb_build_object('ready_perc', ready_perc, 'sold_ready_perc', sold_ready_perc, + 'square_sum', square_sum, 'fo_cd', fo_cd, 'region_cd', region_cd, + 'obj_class', obj_class, 'ready_year', ready_year, + 'city_population', city_population) + FROM domrf_sold_ready_breakdown; + +COMMENT ON VIEW v_sold_ready_unified IS + 'Long-формат 4 sold_ready таблиц. metric_kind: sold_pct / sold_ready_perc / ' + '. Big source: sold_ready_breakdown (~39k) — фильтруй ' + 'по chart_type/entity_key. Total: ~42858 rows.'; + +-- ── 5. v_scrape_runs_unified — 3 скрапера в одном журнале ─────────────────── + +CREATE OR REPLACE VIEW v_scrape_runs_unified AS +SELECT 'kn' AS scraper_type, run_id, started_at, finished_at, heartbeat_at, + status, error, NULL::text AS triggered_by, + region_codes::text AS scope, requests_count, + objects_count AS items_ok, NULL::integer AS items_failed, + flats_count AS sub_items_ok, + progress_obj_index AS progress_idx, total_obj_count AS progress_total, + jsonb_build_object('developers', developer_ids, 'snapshot_date', snapshot_date, + 'resumed_from', resumed_from_run_id, 'params', params) AS extra + FROM kn_scrape_runs +UNION ALL +SELECT 'nspd', run_id, started_at, finished_at, heartbeat_at, status, error, + triggered_by, region_code::text, requests_count, + quarters_ok, quarters_failed, buildings_ok, + NULL, pending_count, + jsonb_build_object('waf_429_count', waf_429_count) + FROM nspd_scrape_runs +UNION ALL +SELECT 'objective', run_id, started_at, finished_at, heartbeat_at, status, error, + triggered_by, group_name, requests_count, + reports_ok, reports_failed, rows_lots, + NULL, NULL, + jsonb_build_object('rows_corpus_room', rows_corpus_room, 'rows_history', rows_history) + FROM objective_scrape_runs; + +COMMENT ON VIEW v_scrape_runs_unified IS + 'Унифицированный журнал прогонов для всех 3 скраперов. Полиморфные общие ' + 'поля (items_ok / sub_items_ok / scope) — для UI; специфичные — в extra ' + 'jsonb. Используй для общего admin /scrape dashboard вместо 3 отдельных endpoint-ов.'; + +-- ── 6. v_scrape_log_unified — kn + nspd логи ──────────────────────────────── + +CREATE OR REPLACE VIEW v_scrape_log_unified AS +SELECT 'kn' AS scraper_type, log_id, run_id, ts, level, stage, + obj_id::text AS entity_id, message + FROM kn_scrape_log +UNION ALL +SELECT 'nspd', log_id, run_id, ts, level, stage, cad_number, message + FROM nspd_scrape_log; + +COMMENT ON VIEW v_scrape_log_unified IS + 'Per-step log двух скраперов. entity_id = obj_id для kn / cad_number для nspd. ' + 'objective scraper пока не пишет log — добавим если понадобится debug.'; diff --git a/data/sql/75_unification_naming.sql b/data/sql/75_unification_naming.sql new file mode 100644 index 00000000..f16451c7 --- /dev/null +++ b/data/sql/75_unification_naming.sql @@ -0,0 +1,123 @@ +-- Унификация naming: region_cd/region_code → region_id, dev_id → developer_id, +-- dev_name → developer_name. +-- +-- ПОДХОД: GENERATED ALWAYS AS (...) STORED — добавляем синонимные колонки +-- которые автоматически синхронизируются с источником. НЕ ALTER RENAME COLUMN +-- (последний сразу сломает работающий backend, который ссылается на старые +-- имена в SQL text() literals в 10 файлах). +-- +-- Backend продолжает работать со старыми именами. Новый код может +-- использовать унифицированные. Постепенно мигрировать backend → потом DROP +-- старых колонок отдельной миграцией. +-- +-- Storage cost: int 4B × N rows. Для rosreestr_deals (~7M rows) = ~28 MB. +-- Суммарно по всем таблицам ~50 MB. Приемлемо. +-- +-- Применено в проде через mcp postgres 2026-05-10. +-- Идемпотентно через ADD COLUMN IF NOT EXISTS. + +-- ── Plus task 2/3 (выполнены отдельным запросом, документирую здесь) ──────── +-- UPDATE complexes SET obj_class и district_name из objective_lots для 350 +-- сматченных через cs.source='objective'. obj_class заполнен = 350, +-- district_name = 1525 (приоритет — Objective). + +-- ── region_cd → region_id (5 таблиц) ──────────────────────────────────────── +ALTER TABLE domrf_kn_objects ADD COLUMN IF NOT EXISTS region_id INT GENERATED ALWAYS AS (region_cd) STORED; +ALTER TABLE domrf_kn_flats ADD COLUMN IF NOT EXISTS region_id INT GENERATED ALWAYS AS (region_cd) STORED; +ALTER TABLE domrf_project_finance ADD COLUMN IF NOT EXISTS region_id INT GENERATED ALWAYS AS (region_cd) STORED; +ALTER TABLE domrf_sold_ready_breakdown ADD COLUMN IF NOT EXISTS region_id INT GENERATED ALWAYS AS (region_cd) STORED; +ALTER TABLE rosreestr_rent_deals ADD COLUMN IF NOT EXISTS region_id INT GENERATED ALWAYS AS (region_cd) STORED; + +-- ── region_code → region_id (3 таблицы; rosreestr_deals = partitioned, → 9 партиций) ── +ALTER TABLE domrf_realization ADD COLUMN IF NOT EXISTS region_id INT GENERATED ALWAYS AS (region_code) STORED; +ALTER TABLE nspd_scrape_runs ADD COLUMN IF NOT EXISTS region_id INT GENERATED ALWAYS AS (region_code) STORED; +ALTER TABLE rosreestr_deals ADD COLUMN IF NOT EXISTS region_id INT GENERATED ALWAYS AS (region_code) STORED; + +-- ── dev_id → developer_id (3 таблицы) ─────────────────────────────────────── +ALTER TABLE domrf_kn_objects ADD COLUMN IF NOT EXISTS developer_id TEXT GENERATED ALWAYS AS (dev_id) STORED; +ALTER TABLE domrf_dev_growth ADD COLUMN IF NOT EXISTS developer_id TEXT GENERATED ALWAYS AS (dev_id) STORED; +ALTER TABLE sv_dev_sales ADD COLUMN IF NOT EXISTS developer_id TEXT GENERATED ALWAYS AS (dev_id) STORED; + +-- ── dev_name → developer_name (2 таблицы) ─────────────────────────────────── +ALTER TABLE domrf_kn_objects ADD COLUMN IF NOT EXISTS developer_name TEXT GENERATED ALWAYS AS (dev_name) STORED; +ALTER TABLE domrf_dev_growth ADD COLUMN IF NOT EXISTS developer_name TEXT GENERATED ALWAYS AS (dev_name) STORED; + +-- ── COMMENT-указатель миграции ────────────────────────────────────────────── +COMMENT ON COLUMN domrf_kn_objects.region_cd IS + 'DEPRECATED: используй region_id (синоним через GENERATED). ' + 'Удалить после миграции backend SQL queries.'; +COMMENT ON COLUMN domrf_kn_objects.dev_id IS + 'DEPRECATED: используй developer_id (синоним через GENERATED).'; +COMMENT ON COLUMN domrf_kn_objects.dev_name IS + 'DEPRECATED: используй developer_name (синоним через GENERATED).'; +COMMENT ON COLUMN rosreestr_deals.region_code IS + 'DEPRECATED: используй region_id (синоним через GENERATED).'; + +-- ── 4. complexes update: obj_class + district из Objective ────────────────── +WITH objective_per_complex AS ( + SELECT DISTINCT ON (cs.complex_id) cs.complex_id, + ol.class AS obj_class, + ol.district AS district_name + FROM complex_sources cs + JOIN objective_lots ol ON ol.project_name = cs.source_id + WHERE cs.source = 'objective' + ORDER BY cs.complex_id, ol.snapshot_date DESC NULLS LAST +) +UPDATE complexes c + SET obj_class = COALESCE(opc.obj_class, c.obj_class), + district_name = COALESCE(opc.district_name, c.district_name), + updated_at = NOW() + FROM objective_per_complex opc + WHERE opc.complex_id = c.id; + +-- ── 5. Fuzzy auto-merge orphan complexes (high-confidence ≥0.85) ──────────── +-- См. полный SQL-блок в applied миграции через mcp postgres. +-- Эффект: 14 high-confidence orphan Objective complexes слились с domrf_kn +-- (например 'Чемпион Парк' / 'Discovery Residence' / 'АЛЛЕГРО'). После merge: +-- complexes 1752 → 1740, in_both_sources_now 114 → 124. + +-- ── 6. v_complex_match_review для остальных 82 кандидатов ─────────────────── +CREATE EXTENSION IF NOT EXISTS pg_trgm; + +CREATE OR REPLACE VIEW v_complex_match_review AS +WITH orphan_complexes AS ( + SELECT c.id AS orphan_id, c.canonical_name AS orphan_name, + c.developer_name AS orphan_dev, c.district_name AS orphan_district, + c.obj_class AS orphan_class + FROM complexes c WHERE c.primary_source = 'objective' +), +kn_complexes AS ( + SELECT c.id AS kn_id, c.canonical_name AS kn_name, + c.developer_name AS kn_dev, c.district_name AS kn_district + FROM complexes c WHERE c.primary_source = 'domrf_kn' +), +candidates AS ( + SELECT oc.*, kc.kn_id, kc.kn_name, kc.kn_dev, kc.kn_district, + similarity(oc.orphan_name, kc.kn_name) AS name_sim, + CASE WHEN oc.orphan_dev IS NOT NULL AND kc.kn_dev IS NOT NULL + THEN similarity(oc.orphan_dev, kc.kn_dev) ELSE 0 END AS dev_sim, + CASE WHEN oc.orphan_district = kc.kn_district THEN 1.0 ELSE 0 END AS district_match + FROM orphan_complexes oc CROSS JOIN kn_complexes kc + WHERE similarity(oc.orphan_name, kc.kn_name) >= 0.4 +), +ranked AS ( + SELECT *, (name_sim * 0.7 + dev_sim * 0.2 + district_match * 0.1) AS combined_score, + ROW_NUMBER() OVER (PARTITION BY orphan_id ORDER BY (name_sim * 0.7 + dev_sim * 0.2 + district_match * 0.1) DESC) AS rk + FROM candidates +) +SELECT orphan_id, orphan_name, orphan_dev, orphan_class, orphan_district, + kn_id, kn_name, kn_dev, kn_district, + ROUND(name_sim::numeric, 3) AS name_sim, + ROUND(dev_sim::numeric, 3) AS dev_sim, + ROUND(combined_score::numeric, 3) AS score, + CASE WHEN combined_score >= 0.7 THEN 'likely' + WHEN combined_score >= 0.5 THEN 'maybe' + ELSE 'unlikely' END AS confidence + FROM ranked WHERE rk = 1 AND combined_score >= 0.5 + ORDER BY combined_score DESC; + +COMMENT ON VIEW v_complex_match_review IS + 'Кандидаты на merge orphan Objective complexes с domrf_kn. Top-1 candidate ' + 'per orphan_id с similarity >= 0.5. Для review через admin UI: ' + 'если confidence=likely (≥0.7) — почти наверняка дубль, maybe (≥0.5) — ' + 'посмотреть глазами. Auto-merge для confidence>=0.85 уже выполнен.'; diff --git a/frontend/src/app/admin/layout.tsx b/frontend/src/app/admin/layout.tsx index d97662c9..f9520e96 100644 --- a/frontend/src/app/admin/layout.tsx +++ b/frontend/src/app/admin/layout.tsx @@ -4,6 +4,7 @@ import Link from "next/link"; import { usePathname } from "next/navigation"; const TABS = [ + { href: "/admin/scrape/all", label: "📊 Все скраперы (сводно)" }, { href: "/admin/scrape", label: "Скрапер DOM.РФ" }, { href: "/admin/scrape/nspd", label: "Скрапер NSPD" }, { href: "/admin/scrape/objective", label: "ETL Objective" }, diff --git a/frontend/src/app/admin/scrape/all/page.tsx b/frontend/src/app/admin/scrape/all/page.tsx new file mode 100644 index 00000000..6f899022 --- /dev/null +++ b/frontend/src/app/admin/scrape/all/page.tsx @@ -0,0 +1,538 @@ +"use client"; + +import { useState } from "react"; +import { useQuery } from "@tanstack/react-query"; + +import { apiFetch } from "@/lib/api"; + +interface UnifiedRunRow { + scraper_type: "kn" | "nspd" | "objective"; + run_id: number; + started_at: string | null; + finished_at: string | null; + heartbeat_at: string | null; + status: string; + error: string | null; + triggered_by: string | null; + scope: string | null; + requests_count: number | null; + items_ok: number | null; + items_failed: number | null; + sub_items_ok: number | null; + progress_idx: number | null; + progress_total: number | null; + extra: Record | null; +} + +interface UnifiedLogRow { + scraper_type: "kn" | "nspd"; + log_id: number; + run_id: number | null; + ts: string | null; + level: "debug" | "info" | "warn" | "error"; + stage: string | null; + entity_id: string | null; + message: string; +} + +const SCRAPER_BADGE: Record = + { + kn: { bg: "#dbeafe", fg: "#1d4ed8", label: "DOM.РФ kn" }, + nspd: { bg: "#fef3c7", fg: "#a16207", label: "NSPD" }, + objective: { bg: "#d1fae5", fg: "#059669", label: "Objective" }, + }; + +const STATUS_COLOR: Record = { + done: "#16a34a", + running: "#1d4ed8", + failed: "#b3261e", + zombie: "#9333ea", +}; + +function formatIso(s: string | null): string { + if (!s) return "—"; + return new Date(s).toLocaleString("ru-RU"); +} + +function diffSecs(a: string | null, b: string | null): string { + if (!a || !b) return "—"; + const dt = (new Date(b).getTime() - new Date(a).getTime()) / 1000; + if (dt < 60) return `${dt.toFixed(0)}с`; + if (dt < 3600) return `${(dt / 60).toFixed(1)} мин`; + return `${(dt / 3600).toFixed(1)} ч`; +} + +export default function ScrapeAllAdminPage() { + const [token, setToken] = useState(() => + typeof window === "undefined" + ? "" + : (localStorage.getItem("admin_token") ?? ""), + ); + const [filter, setFilter] = useState<"all" | "kn" | "nspd" | "objective">( + "all", + ); + const [logsRunId, setLogsRunId] = useState(""); + + const saveToken = (v: string) => { + setToken(v); + localStorage.setItem("admin_token", v); + }; + + const runs = useQuery({ + queryKey: ["unified-runs", token, filter], + queryFn: () => { + const q = + filter !== "all" ? `?scraper_type=${filter}&limit=30` : "?limit=30"; + return apiFetch(`/api/v1/admin/scrape/all/runs${q}`, { + headers: { "X-Admin-Token": token }, + }); + }, + enabled: !!token, + refetchInterval: 5000, + }); + + const logs = useQuery({ + queryKey: ["unified-logs", token, filter, logsRunId], + queryFn: () => { + const params = new URLSearchParams({ limit: "100" }); + if (filter !== "all") params.set("scraper_type", filter); + if (logsRunId !== "") params.set("run_id", String(logsRunId)); + return apiFetch( + `/api/v1/admin/scrape/all/logs?${params}`, + { headers: { "X-Admin-Token": token } }, + ); + }, + enabled: !!token, + refetchInterval: 8000, + }); + + // Сводка по statusам + const summary = (() => { + const s = { running: 0, done: 0, failed: 0, zombie: 0 } as Record< + string, + number + >; + runs.data?.forEach((r) => { + s[r.status] = (s[r.status] ?? 0) + 1; + }); + return s; + })(); + + return ( + <> +

+ Скраперы — сводный dashboard +

+

+ Все 3 скрапера в одной таблице (через v_scrape_runs_unified + ). Для запуска / триггеров — отдельные страницы:{" "} + + kn + {" "} + ·{" "} + + NSPD + {" "} + ·{" "} + + Objective + + . +

+ +
+ +
+ + {/* Сводка-плитки */} +
+ + + + +
+ + {/* Фильтр + табы */} +
+
+ Filter: + {(["all", "kn", "nspd", "objective"] as const).map((t) => ( + + ))} +
+
+ + {/* Таблица прогонов */} +
+

+ 📜 Последние прогоны (top 30, обновляется каждые 5с) +

+
+ + + + + + + + + + + + + + + + + + {runs.data?.map((r) => { + const heartbeatLag = r.heartbeat_at + ? (Date.now() - new Date(r.heartbeat_at).getTime()) / 1000 + : null; + const isStale = + r.status === "running" && + heartbeatLag !== null && + heartbeatLag > 60; + const badge = SCRAPER_BADGE[r.scraper_type]; + const statusColor = STATUS_COLOR[r.status] ?? "#6b7280"; + return ( + + + + + + + + + + + + + + ); + })} + {runs.data?.length === 0 && ( + + + + )} + +
scraperrun_idstartedscopedurationitems_okfailedsub_okstatustrigger
+ {badge && ( + + {badge.label} + + )} + + {r.run_id} + {formatIso(r.started_at)} + {r.scope ?? "—"} + + {diffSecs(r.started_at, r.finished_at ?? r.heartbeat_at)} + + {r.items_ok?.toLocaleString("ru-RU") ?? "—"} + + {r.items_failed?.toLocaleString("ru-RU") ?? "—"} + + {r.sub_items_ok?.toLocaleString("ru-RU") ?? "—"} + + + {isStale ? "stale" : r.status} + + + {r.triggered_by ?? "—"} + + {(r.scraper_type === "kn" || + r.scraper_type === "nspd") && ( + + )} +
+ Прогонов с фильтром {filter} нет +
+
+
+ + {/* Logs */} +
+

+ 📋 Логи (top 100){" "} + {logsRunId !== "" && ( + + · фильтр run_id={logsRunId}{" "} + + + )} +

+
+ + + + + + + + + + + + + + {logs.data?.map((l) => ( + + + + + + + + + + ))} + {logs.data?.length === 0 && ( + + + + )} + +
tsscraperrunlevelstageentitymessage
+ {formatIso(l.ts)} + + + {l.scraper_type} + + + {l.run_id ?? "—"} + + + {l.level} + + {l.stage ?? "—"} + {l.entity_id ?? "—"} + {l.message}
+ Логов нет (objective пока не пишет в log — только runs) +
+
+
+ + ); +} + +function Stat({ + label, + value, + color, +}: { + label: string; + value: number; + color: string; +}) { + return ( +
+
+ {label} +
+
+ {value} +
+
+ ); +} + +const cardStyle = { + background: "#fff", + border: "1px solid #e6e8ec", + borderRadius: 12, + padding: 20, +}; +const sectionTitle = { + margin: "0 0 12px", + fontSize: 16, + fontWeight: 600 as const, +}; +const labelStyle = { + display: "block", + fontSize: 12, + color: "#5b6066", + marginBottom: 4, + textTransform: "uppercase" as const, + letterSpacing: 0.4, +}; +const inputStyle = { + padding: "8px 10px", + border: "1px solid #d1d5db", + borderRadius: 6, + fontSize: 14, + width: "100%", + boxSizing: "border-box" as const, +}; +const filterTab = { + padding: "6px 14px", + border: "none", + borderRadius: 6, + fontSize: 13, + fontWeight: 500 as const, + cursor: "pointer", +}; +const secondaryBtn = { + padding: "8px 14px", + background: "#f3f4f6", + color: "#1f2937", + border: "1px solid #d1d5db", + borderRadius: 6, + fontSize: 13, + cursor: "pointer", +}; +const tableStyle = { + width: "100%", + borderCollapse: "collapse" as const, + fontSize: 12, +}; +const th = { + padding: "8px 10px", + textAlign: "left" as const, + fontWeight: 600 as const, + borderBottom: "1px solid #e6e8ec", +}; +const td = { padding: "8px 10px" };