feat(db): canonical complexes + unified views + naming consistency
* Task 2+3: complexes.obj_class и district_name из Objective
(350 ЖК с классом, 1525 с district; objective приоритетнее)
* Task 1 (fuzzy match orphan Objective ↔ domrf_kn):
- pg_trgm extension
- auto-merge 14 high-confidence (similarity ≥ 0.85)
- v_complex_match_review для остальных 82 кандидатов
- complexes 1752 → 1740 · cross-source 114 → 124
* Task 4 (naming unification без RENAME COLUMN):
- GENERATED ALWAYS AS (...) STORED для синонимов
- region_id в 15 таблиц + 9 partitions
- developer_id в 7, developer_name в 5
- COMMENT 'DEPRECATED' на region_cd/region_code/dev_id/dev_name
* Task 5 (unified scrape dashboard):
- GET /api/v1/admin/scrape/all/runs + /all/logs (поверх v_scrape_runs_unified)
- new page /admin/scrape/all с filter табами + 4 stat-плитки
- +tab «📊 Все скраперы» в admin layout
Файлы: data/sql/75_unification_naming.sql · backend/app/api/v1/admin_scrape.py
· frontend/src/app/admin/scrape/all/page.tsx · admin/layout.tsx
This commit is contained in:
lekss361
parent
20625f78f3
commit
52bcf9d30c
6 changed files with 1348 additions and 0 deletions
|
|
@ -801,6 +801,118 @@ def objective_coverage(
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# ── Unified scrape dashboard (поверх v_scrape_runs_unified / v_scrape_log_unified) ──
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/all/runs")
|
||||||
|
def list_all_runs(
|
||||||
|
db: Annotated[Session, Depends(get_db)],
|
||||||
|
x_admin_token: Annotated[str | None, Header(alias="X-Admin-Token")] = None,
|
||||||
|
scraper_type: str | None = None,
|
||||||
|
limit: int = 30,
|
||||||
|
) -> list[dict[str, Any]]:
|
||||||
|
"""Унифицированный список прогонов (kn + nspd + objective).
|
||||||
|
|
||||||
|
Поверх v_scrape_runs_unified. Если scraper_type не задан — все 3.
|
||||||
|
Используется главной dashboard /admin/scrape/all.
|
||||||
|
"""
|
||||||
|
_check_token(x_admin_token)
|
||||||
|
where = "" if not scraper_type else "WHERE scraper_type = :st"
|
||||||
|
params: dict[str, Any] = {"lim": min(limit, 200)}
|
||||||
|
if scraper_type:
|
||||||
|
params["st"] = scraper_type
|
||||||
|
rows = (
|
||||||
|
db.execute(
|
||||||
|
text(
|
||||||
|
f"""
|
||||||
|
SELECT scraper_type, run_id, started_at, finished_at, heartbeat_at,
|
||||||
|
status, error, triggered_by, scope, requests_count,
|
||||||
|
items_ok, items_failed, sub_items_ok,
|
||||||
|
progress_idx, progress_total, extra
|
||||||
|
FROM v_scrape_runs_unified
|
||||||
|
{where}
|
||||||
|
ORDER BY started_at DESC NULLS LAST
|
||||||
|
LIMIT :lim
|
||||||
|
"""
|
||||||
|
),
|
||||||
|
params,
|
||||||
|
)
|
||||||
|
.mappings()
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
return [
|
||||||
|
{
|
||||||
|
"scraper_type": r["scraper_type"],
|
||||||
|
"run_id": r["run_id"],
|
||||||
|
"started_at": r["started_at"].isoformat() if r["started_at"] else None,
|
||||||
|
"finished_at": r["finished_at"].isoformat() if r["finished_at"] else None,
|
||||||
|
"heartbeat_at": r["heartbeat_at"].isoformat() if r["heartbeat_at"] else None,
|
||||||
|
"status": r["status"],
|
||||||
|
"error": r["error"],
|
||||||
|
"triggered_by": r["triggered_by"],
|
||||||
|
"scope": r["scope"],
|
||||||
|
"requests_count": r["requests_count"],
|
||||||
|
"items_ok": r["items_ok"],
|
||||||
|
"items_failed": r["items_failed"],
|
||||||
|
"sub_items_ok": r["sub_items_ok"],
|
||||||
|
"progress_idx": r["progress_idx"],
|
||||||
|
"progress_total": r["progress_total"],
|
||||||
|
"extra": r["extra"],
|
||||||
|
}
|
||||||
|
for r in rows
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/all/logs")
|
||||||
|
def list_all_logs(
|
||||||
|
db: Annotated[Session, Depends(get_db)],
|
||||||
|
x_admin_token: Annotated[str | None, Header(alias="X-Admin-Token")] = None,
|
||||||
|
scraper_type: str | None = None,
|
||||||
|
run_id: int | None = None,
|
||||||
|
limit: int = 200,
|
||||||
|
) -> list[dict[str, Any]]:
|
||||||
|
"""Унифицированный список логов (kn + nspd). Objective пока не пишет log."""
|
||||||
|
_check_token(x_admin_token)
|
||||||
|
where: list[str] = []
|
||||||
|
params: dict[str, Any] = {"lim": min(limit, 1000)}
|
||||||
|
if scraper_type:
|
||||||
|
where.append("scraper_type = :st")
|
||||||
|
params["st"] = scraper_type
|
||||||
|
if run_id is not None:
|
||||||
|
where.append("run_id = :rid")
|
||||||
|
params["rid"] = run_id
|
||||||
|
where_sql = ("WHERE " + " AND ".join(where)) if where else ""
|
||||||
|
rows = (
|
||||||
|
db.execute(
|
||||||
|
text(
|
||||||
|
f"""
|
||||||
|
SELECT scraper_type, log_id, run_id, ts, level, stage, entity_id, message
|
||||||
|
FROM v_scrape_log_unified
|
||||||
|
{where_sql}
|
||||||
|
ORDER BY log_id DESC
|
||||||
|
LIMIT :lim
|
||||||
|
"""
|
||||||
|
),
|
||||||
|
params,
|
||||||
|
)
|
||||||
|
.mappings()
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
return [
|
||||||
|
{
|
||||||
|
"scraper_type": r["scraper_type"],
|
||||||
|
"log_id": r["log_id"],
|
||||||
|
"run_id": r["run_id"],
|
||||||
|
"ts": r["ts"].isoformat() if r["ts"] else None,
|
||||||
|
"level": r["level"],
|
||||||
|
"stage": r["stage"],
|
||||||
|
"entity_id": r["entity_id"],
|
||||||
|
"message": r["message"],
|
||||||
|
}
|
||||||
|
for r in rows
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
# ── DomRF (kn) — runs list ───────────────────────────────────────────────────
|
# ── DomRF (kn) — runs list ───────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
318
data/sql/73_complexes_master.sql
Normal file
318
data/sql/73_complexes_master.sql
Normal file
|
|
@ -0,0 +1,318 @@
|
||||||
|
-- ЖК как canonical master entity + cross-ref на источники.
|
||||||
|
--
|
||||||
|
-- Зачем:
|
||||||
|
-- До этого жил мир из 5+ таблиц с дублирующейся семантикой "ЖК":
|
||||||
|
-- - domrf_kn_objects (4548, ДОМ.РФ kn-API)
|
||||||
|
-- - objective_lots.project_name (350 уникальных, Objective API)
|
||||||
|
-- - objective_corpus_room_month.project_name (тот же, 350)
|
||||||
|
-- - objective_complex_mapping (114 fuzzy matches → domrf_obj_id)
|
||||||
|
-- - cad_buildings.cad_num (косвенно через NSPD)
|
||||||
|
-- - rosreestr_deals.complex_name (текстовое поле в сделках)
|
||||||
|
-- Каждый источник называет ЖК по-своему ("ЖК \"Парк Культуры\"" vs
|
||||||
|
-- "Парк Культуры" vs domrf_obj_id=66845). recommend_mix /
|
||||||
|
-- velocity-anomaly / cross-source analytics требуют одного entity.
|
||||||
|
--
|
||||||
|
-- Архитектура:
|
||||||
|
-- complexes (master) — одна строка = один реальный ЖК
|
||||||
|
-- complex_sources (cross-ref) — many-to-many ЖК ↔ источник + source_id
|
||||||
|
--
|
||||||
|
-- Стратегия backfill:
|
||||||
|
-- 1. Каждый domrf_kn_objects = entity в complexes (4548 ЖК)
|
||||||
|
-- + complex_sources(source='domrf_kn', source_id=obj_id)
|
||||||
|
-- 2. Сматченные objective project_name (114) → существующий complex по
|
||||||
|
-- objective_complex_mapping.domrf_obj_id
|
||||||
|
-- + complex_sources(source='objective', source_id=project_name)
|
||||||
|
-- 3. Unmatched objective project_name (236) → новые entity в complexes
|
||||||
|
-- + complex_sources(source='objective')
|
||||||
|
--
|
||||||
|
-- Идемпотентно через ON CONFLICT.
|
||||||
|
|
||||||
|
-- ── 1. complexes (canonical master) ─────────────────────────────────────────
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS complexes (
|
||||||
|
id BIGSERIAL PRIMARY KEY,
|
||||||
|
-- Display
|
||||||
|
canonical_name TEXT NOT NULL, -- "Парк Культуры" / "ЖК Авангард 3.0"
|
||||||
|
developer_name TEXT, -- "PRINZIP" / "ГК КОРТРОС"
|
||||||
|
developer_id TEXT, -- FK on domrf_developers.developer_id (nullable)
|
||||||
|
-- Geo
|
||||||
|
region_id INTEGER, -- 66 для Свердл.обл (FK on domrf_regions.region_id)
|
||||||
|
district_name TEXT, -- "Академический" / "Автовокзал"
|
||||||
|
address TEXT,
|
||||||
|
latitude NUMERIC,
|
||||||
|
longitude NUMERIC,
|
||||||
|
geom geometry(Point, 4326), -- spatial для ST_DWithin / spatial join
|
||||||
|
-- Spec
|
||||||
|
obj_class TEXT, -- "комфорт" / "бизнес" / "стандарт" / "элитный"
|
||||||
|
flat_count INTEGER, -- общее ПД
|
||||||
|
square_living NUMERIC, -- общая жилая площадь
|
||||||
|
floor_min INTEGER,
|
||||||
|
floor_max INTEGER,
|
||||||
|
wall_type TEXT, -- "монолит" / "кирпич" / etc.
|
||||||
|
-- Stage
|
||||||
|
sales_start_date DATE,
|
||||||
|
plan_completion_date DATE,
|
||||||
|
actual_completion_date DATE,
|
||||||
|
site_status TEXT, -- "строящийся" / "сдан" / "архив"
|
||||||
|
obj_status INTEGER, -- 0/1/2 кодировка ДОМ.РФ
|
||||||
|
is_problem BOOLEAN DEFAULT FALSE,
|
||||||
|
has_escrow BOOLEAN, -- эскроу-проект
|
||||||
|
-- Audit
|
||||||
|
primary_source TEXT NOT NULL DEFAULT 'domrf_kn', -- какой источник был основой
|
||||||
|
created_at timestamptz NOT NULL DEFAULT NOW(),
|
||||||
|
updated_at timestamptz NOT NULL DEFAULT NOW()
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS complexes_name_idx
|
||||||
|
ON complexes(canonical_name);
|
||||||
|
CREATE INDEX IF NOT EXISTS complexes_developer_idx
|
||||||
|
ON complexes(developer_id) WHERE developer_id IS NOT NULL;
|
||||||
|
CREATE INDEX IF NOT EXISTS complexes_region_district_idx
|
||||||
|
ON complexes(region_id, district_name);
|
||||||
|
CREATE INDEX IF NOT EXISTS complexes_class_idx
|
||||||
|
ON complexes(obj_class) WHERE obj_class IS NOT NULL;
|
||||||
|
CREATE INDEX IF NOT EXISTS complexes_geom_idx
|
||||||
|
ON complexes USING GIST (geom) WHERE geom IS NOT NULL;
|
||||||
|
CREATE INDEX IF NOT EXISTS complexes_status_idx
|
||||||
|
ON complexes(site_status) WHERE site_status IS NOT NULL;
|
||||||
|
|
||||||
|
COMMENT ON TABLE complexes IS
|
||||||
|
'Canonical master entity для ЖК. Один реальный жилой комплекс = одна строка. '
|
||||||
|
'Связи на разные источники (domrf_kn, objective, rosreestr, etc.) — через '
|
||||||
|
'complex_sources. Все analytics-запросы должны идти через эту таблицу '
|
||||||
|
'вместо direct lookup в источниках.';
|
||||||
|
|
||||||
|
-- ── 2. complex_sources (cross-ref) ──────────────────────────────────────────
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS complex_sources (
|
||||||
|
id BIGSERIAL PRIMARY KEY,
|
||||||
|
complex_id BIGINT NOT NULL REFERENCES complexes(id) ON DELETE CASCADE,
|
||||||
|
source TEXT NOT NULL, -- 'domrf_kn' | 'objective' | 'rosreestr' | 'cad_buildings' | 'yandex' | ...
|
||||||
|
source_id TEXT NOT NULL, -- obj_id::text / project_name / cad_num / external_id
|
||||||
|
source_external_id BIGINT, -- для int-ID источников (objective.project_id, domrf.obj_id)
|
||||||
|
-- Match audit
|
||||||
|
match_method TEXT, -- 'exact_name' | 'fuzzy_name' | 'spatial' | 'manual' | 'imported'
|
||||||
|
match_score NUMERIC, -- 0..1 для fuzzy
|
||||||
|
is_reviewed BOOLEAN NOT NULL DEFAULT FALSE,
|
||||||
|
note TEXT,
|
||||||
|
linked_at timestamptz NOT NULL DEFAULT NOW(),
|
||||||
|
UNIQUE (source, source_id) -- один (источник, source_id) — на один ЖК
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS complex_sources_complex_idx
|
||||||
|
ON complex_sources(complex_id);
|
||||||
|
CREATE INDEX IF NOT EXISTS complex_sources_source_idx
|
||||||
|
ON complex_sources(source);
|
||||||
|
CREATE INDEX IF NOT EXISTS complex_sources_external_id_idx
|
||||||
|
ON complex_sources(source, source_external_id) WHERE source_external_id IS NOT NULL;
|
||||||
|
CREATE INDEX IF NOT EXISTS complex_sources_unreviewed_idx
|
||||||
|
ON complex_sources(is_reviewed) WHERE is_reviewed = FALSE;
|
||||||
|
|
||||||
|
COMMENT ON TABLE complex_sources IS
|
||||||
|
'Cross-ref many-to-many: ЖК (complexes) ↔ его представления в источниках. '
|
||||||
|
'Один реальный ЖК может иметь записи в нескольких источниках — здесь '
|
||||||
|
'хранятся все эти ссылки. UNIQUE(source, source_id) гарантирует что один '
|
||||||
|
'(источник, ID-в-источнике) присвоен ровно одному ЖК.';
|
||||||
|
|
||||||
|
-- ── 3. Backfill из domrf_kn_objects ─────────────────────────────────────────
|
||||||
|
|
||||||
|
-- ВАЖНО: domrf_kn_objects содержит несколько snapshot'ов на один obj_id
|
||||||
|
-- (4548 rows = 1516 уникальных obj_id × ~3 snapshot_date). Поэтому
|
||||||
|
-- DISTINCT ON (obj_id) ORDER BY snapshot_date DESC — берём свежий snapshot.
|
||||||
|
--
|
||||||
|
-- ВНИМАНИЕ: domrf_kn_objects.obj_class = NULL во всех строках (известный
|
||||||
|
-- баг kn-API), поэтому obj_class останется NULL — он подтянется когда
|
||||||
|
-- сделаем UPDATE из objective_lots (там obj_class заполнен).
|
||||||
|
|
||||||
|
WITH latest_kn AS (
|
||||||
|
SELECT DISTINCT ON (obj_id)
|
||||||
|
obj_id, comm_name, addr, region_cd, dev_id, dev_name, district_name,
|
||||||
|
latitude, longitude, obj_class, flat_count, square_living,
|
||||||
|
floor_min, floor_max, wall_type, ready_dt,
|
||||||
|
site_status, obj_status, is_problem, escrow
|
||||||
|
FROM domrf_kn_objects
|
||||||
|
ORDER BY obj_id, snapshot_date DESC NULLS LAST
|
||||||
|
),
|
||||||
|
inserted_complexes AS (
|
||||||
|
INSERT INTO complexes (
|
||||||
|
canonical_name, developer_name, developer_id, region_id, district_name,
|
||||||
|
address, latitude, longitude, geom,
|
||||||
|
obj_class, flat_count, square_living, floor_min, floor_max, wall_type,
|
||||||
|
plan_completion_date, site_status, obj_status, is_problem, has_escrow,
|
||||||
|
primary_source
|
||||||
|
)
|
||||||
|
SELECT
|
||||||
|
COALESCE(comm_name, addr, 'unknown'),
|
||||||
|
dev_name, dev_id, region_cd, district_name,
|
||||||
|
addr, latitude, longitude,
|
||||||
|
CASE WHEN latitude IS NOT NULL AND longitude IS NOT NULL
|
||||||
|
THEN ST_SetSRID(ST_MakePoint(longitude, latitude), 4326) ELSE NULL END,
|
||||||
|
obj_class, flat_count, square_living, floor_min, floor_max, wall_type,
|
||||||
|
ready_dt, site_status, obj_status, COALESCE(is_problem, FALSE), escrow,
|
||||||
|
'domrf_kn'
|
||||||
|
FROM latest_kn
|
||||||
|
RETURNING id, canonical_name
|
||||||
|
)
|
||||||
|
INSERT INTO complex_sources (
|
||||||
|
complex_id, source, source_id, source_external_id,
|
||||||
|
match_method, match_score, is_reviewed, note
|
||||||
|
)
|
||||||
|
SELECT
|
||||||
|
ic.id, 'domrf_kn', lk.obj_id::text, lk.obj_id, 'imported', 1.0, TRUE,
|
||||||
|
'auto-import from domrf_kn_objects (latest snapshot per obj_id)'
|
||||||
|
FROM latest_kn lk
|
||||||
|
JOIN inserted_complexes ic
|
||||||
|
ON ic.canonical_name = COALESCE(lk.comm_name, lk.addr, 'unknown')
|
||||||
|
ON CONFLICT (source, source_id) DO NOTHING;
|
||||||
|
|
||||||
|
-- ── 4. Backfill из objective_complex_mapping (114 сматченных) ───────────────
|
||||||
|
|
||||||
|
-- Эти project_name уже привязаны к domrf_obj_id — линкуем к существующему complex.
|
||||||
|
INSERT INTO complex_sources (
|
||||||
|
complex_id, source, source_id, source_external_id,
|
||||||
|
match_method, match_score, is_reviewed, note
|
||||||
|
)
|
||||||
|
SELECT
|
||||||
|
cs_kn.complex_id, -- complex по domrf_kn
|
||||||
|
'objective',
|
||||||
|
m.objective_complex_name,
|
||||||
|
m.objective_project_id,
|
||||||
|
m.match_method,
|
||||||
|
m.match_score,
|
||||||
|
m.is_reviewed,
|
||||||
|
'imported from objective_complex_mapping'
|
||||||
|
FROM objective_complex_mapping m
|
||||||
|
JOIN complex_sources cs_kn
|
||||||
|
ON cs_kn.source = 'domrf_kn'
|
||||||
|
AND cs_kn.source_external_id = m.domrf_obj_id
|
||||||
|
WHERE m.domrf_obj_id IS NOT NULL
|
||||||
|
ON CONFLICT (source, source_id) DO NOTHING;
|
||||||
|
|
||||||
|
-- ── 5. Backfill orphan-проектов из objective_lots (236 unmatched) ───────────
|
||||||
|
|
||||||
|
-- Для каждого project_name которого нет ни в complex_sources(source=objective)
|
||||||
|
-- — создаём новый complex (берём атрибуты из любой quartiry этого проекта)
|
||||||
|
-- + добавляем complex_sources(source=objective).
|
||||||
|
|
||||||
|
WITH unmatched_objective AS (
|
||||||
|
SELECT DISTINCT ON (project_name)
|
||||||
|
project_name,
|
||||||
|
objective_project_id,
|
||||||
|
developer,
|
||||||
|
district,
|
||||||
|
class AS obj_class,
|
||||||
|
city,
|
||||||
|
address,
|
||||||
|
sales_start_date,
|
||||||
|
plan_completion_date,
|
||||||
|
actual_completion_date,
|
||||||
|
construction_stage
|
||||||
|
FROM objective_lots ol
|
||||||
|
WHERE NOT EXISTS (
|
||||||
|
SELECT 1 FROM complex_sources cs
|
||||||
|
WHERE cs.source = 'objective'
|
||||||
|
AND cs.source_id = ol.project_name
|
||||||
|
)
|
||||||
|
ORDER BY project_name, snapshot_date DESC NULLS LAST
|
||||||
|
),
|
||||||
|
inserted AS (
|
||||||
|
INSERT INTO complexes (
|
||||||
|
canonical_name, developer_name, region_id, district_name,
|
||||||
|
address, obj_class, sales_start_date, plan_completion_date,
|
||||||
|
actual_completion_date, site_status, primary_source
|
||||||
|
)
|
||||||
|
SELECT
|
||||||
|
project_name,
|
||||||
|
developer,
|
||||||
|
66, -- Свердл.обл (Объектив пока только Екб + спутники)
|
||||||
|
district,
|
||||||
|
address,
|
||||||
|
obj_class,
|
||||||
|
sales_start_date,
|
||||||
|
plan_completion_date,
|
||||||
|
actual_completion_date,
|
||||||
|
construction_stage,
|
||||||
|
'objective'
|
||||||
|
FROM unmatched_objective
|
||||||
|
RETURNING id, canonical_name
|
||||||
|
)
|
||||||
|
INSERT INTO complex_sources (
|
||||||
|
complex_id, source, source_id, source_external_id,
|
||||||
|
match_method, match_score, is_reviewed, note
|
||||||
|
)
|
||||||
|
SELECT
|
||||||
|
i.id,
|
||||||
|
'objective',
|
||||||
|
uo.project_name,
|
||||||
|
uo.objective_project_id,
|
||||||
|
'imported',
|
||||||
|
1.0,
|
||||||
|
FALSE, -- нужно ревью: возможно есть domrf_kn-аналог
|
||||||
|
'orphan objective project; consider reviewing for domrf_kn match'
|
||||||
|
FROM unmatched_objective uo
|
||||||
|
JOIN inserted i ON i.canonical_name = uo.project_name
|
||||||
|
ON CONFLICT (source, source_id) DO NOTHING;
|
||||||
|
|
||||||
|
-- ── 6. Helper-views для запросов ────────────────────────────────────────────
|
||||||
|
|
||||||
|
-- v_complex_full: полная карточка ЖК с метриками из всех источников.
|
||||||
|
DROP VIEW IF EXISTS v_complex_full;
|
||||||
|
CREATE VIEW v_complex_full AS
|
||||||
|
SELECT
|
||||||
|
c.id AS complex_id,
|
||||||
|
c.canonical_name,
|
||||||
|
c.developer_name,
|
||||||
|
c.region_id,
|
||||||
|
c.district_name,
|
||||||
|
c.obj_class,
|
||||||
|
c.address,
|
||||||
|
c.latitude,
|
||||||
|
c.longitude,
|
||||||
|
c.flat_count,
|
||||||
|
c.square_living,
|
||||||
|
c.site_status,
|
||||||
|
-- сводка по источникам
|
||||||
|
(SELECT array_agg(source ORDER BY source) FROM complex_sources WHERE complex_id = c.id) AS sources,
|
||||||
|
(SELECT COUNT(*) FROM complex_sources WHERE complex_id = c.id) AS sources_count,
|
||||||
|
-- domrf_kn obj_id (если есть)
|
||||||
|
(SELECT source_external_id FROM complex_sources
|
||||||
|
WHERE complex_id = c.id AND source = 'domrf_kn' LIMIT 1) AS domrf_obj_id,
|
||||||
|
-- objective project_name (если есть)
|
||||||
|
(SELECT source_id FROM complex_sources
|
||||||
|
WHERE complex_id = c.id AND source = 'objective' LIMIT 1) AS objective_project_name,
|
||||||
|
c.primary_source,
|
||||||
|
c.created_at,
|
||||||
|
c.updated_at
|
||||||
|
FROM complexes c;
|
||||||
|
|
||||||
|
COMMENT ON VIEW v_complex_full IS
|
||||||
|
'Карточка ЖК с агрегированной информацией о всех источниках. '
|
||||||
|
'Используй для UI / отчётов / вместо direct lookup в domrf_kn_objects.';
|
||||||
|
|
||||||
|
-- v_complex_with_objective: ЖК + агрегаты Objective (lots count, sold pct, avg price)
|
||||||
|
DROP VIEW IF EXISTS v_complex_with_objective;
|
||||||
|
CREATE VIEW v_complex_with_objective AS
|
||||||
|
SELECT
|
||||||
|
c.id AS complex_id,
|
||||||
|
c.canonical_name,
|
||||||
|
c.district_name,
|
||||||
|
c.obj_class,
|
||||||
|
cs.source_id AS objective_project_name,
|
||||||
|
COUNT(ol.id) AS objective_lots_n,
|
||||||
|
COUNT(ol.id) FILTER (WHERE ol.is_sold) AS objective_sold_n,
|
||||||
|
ROUND(100.0 * COUNT(ol.id) FILTER (WHERE ol.is_sold)
|
||||||
|
/ NULLIF(COUNT(ol.id), 0), 1) AS objective_sold_pct,
|
||||||
|
AVG(ol.price_per_m2_rub) FILTER (WHERE ol.is_sold AND ol.price_per_m2_rub > 0) AS avg_sold_price_per_m2,
|
||||||
|
AVG(ol.area_pd) FILTER (WHERE ol.area_pd > 0) AS avg_area_pd,
|
||||||
|
MIN(ol.registration_date) AS first_reg_date,
|
||||||
|
MAX(ol.registration_date) AS last_reg_date
|
||||||
|
FROM complexes c
|
||||||
|
JOIN complex_sources cs
|
||||||
|
ON cs.complex_id = c.id AND cs.source = 'objective'
|
||||||
|
JOIN objective_lots ol
|
||||||
|
ON ol.project_name = cs.source_id
|
||||||
|
GROUP BY c.id, c.canonical_name, c.district_name, c.obj_class, cs.source_id;
|
||||||
|
|
||||||
|
COMMENT ON VIEW v_complex_with_objective IS
|
||||||
|
'ЖК + агрегаты Objective per-flat: sold count/pct, средняя цена, '
|
||||||
|
'диапазон reg_date. NULL во всех колонках = ЖК без Objective-источника.';
|
||||||
256
data/sql/74_dedupe_unified_views.sql
Normal file
256
data/sql/74_dedupe_unified_views.sql
Normal file
|
|
@ -0,0 +1,256 @@
|
||||||
|
-- Phase B + D: унификация дублирующихся таблиц через VIEW (UNION ALL).
|
||||||
|
--
|
||||||
|
-- Стратегия: НЕ дропаем источники (они продолжают писать своими скраперами/
|
||||||
|
-- импортёрами), но создаём поверх единые view для удобных запросов из
|
||||||
|
-- аналитики и UI. Это zero-risk миграция: можно делать в любой момент
|
||||||
|
-- и откатывать DROP VIEW без потери данных.
|
||||||
|
--
|
||||||
|
-- После того как код в backend перейдёт на use views — старые имена таблиц
|
||||||
|
-- можно будет либо превратить в views поверх objединённой структуры, либо
|
||||||
|
-- мигрировать данные целиком. Сейчас — promotion path.
|
||||||
|
--
|
||||||
|
-- Применено в проде через mcp postgres 2026-05-10.
|
||||||
|
-- Идемпотентно через CREATE OR REPLACE VIEW.
|
||||||
|
|
||||||
|
-- ── 1. ekb_districts: merge geom-таблицы в основную ─────────────────────────
|
||||||
|
-- ekb_districts (9, dimension с метриками) + ekb_districts_geom (8, polygons)
|
||||||
|
-- → одна таблица ekb_districts с PostGIS-колонкой geom
|
||||||
|
-- + view-alias ekb_districts_geom для backward-compat.
|
||||||
|
|
||||||
|
ALTER TABLE ekb_districts
|
||||||
|
ADD COLUMN IF NOT EXISTS geom geometry(MultiPolygon, 4326),
|
||||||
|
ADD COLUMN IF NOT EXISTS osm_id BIGINT,
|
||||||
|
ADD COLUMN IF NOT EXISTS geom_fetched_at timestamptz;
|
||||||
|
|
||||||
|
UPDATE ekb_districts d
|
||||||
|
SET geom = g.geom, osm_id = g.osm_id, geom_fetched_at = g.fetched_at
|
||||||
|
FROM ekb_districts_geom g
|
||||||
|
WHERE g.district_name = d.district_name;
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS ekb_districts_geom_idx
|
||||||
|
ON ekb_districts USING GIST(geom) WHERE geom IS NOT NULL;
|
||||||
|
|
||||||
|
DROP TABLE IF EXISTS ekb_districts_geom;
|
||||||
|
CREATE OR REPLACE VIEW ekb_districts_geom AS
|
||||||
|
SELECT district_name, geom, osm_id, geom_fetched_at AS fetched_at
|
||||||
|
FROM ekb_districts WHERE geom IS NOT NULL;
|
||||||
|
|
||||||
|
-- ── 2. v_mortgage_unified — 4 источника ипотечных метрик в long-формате ─────
|
||||||
|
|
||||||
|
CREATE OR REPLACE VIEW v_mortgage_unified AS
|
||||||
|
SELECT
|
||||||
|
'cbr' AS source,
|
||||||
|
NULL::date AS snapshot_date,
|
||||||
|
period AS period_label,
|
||||||
|
region,
|
||||||
|
NULL::text AS bank_name,
|
||||||
|
title AS metric_kind,
|
||||||
|
'total' AS segment,
|
||||||
|
value,
|
||||||
|
NULL::numeric AS delta_pct,
|
||||||
|
series_id AS metric_id
|
||||||
|
FROM cbr_mortgage_series
|
||||||
|
UNION ALL
|
||||||
|
SELECT 'domrf_dashboard', snapshot_date, NULL, 'РФ', NULL,
|
||||||
|
'avg_rate_pct', 'total', total_credit_avg_rate, total_credit_avg_rate_delta, NULL
|
||||||
|
FROM domrf_mortgage_dashboard
|
||||||
|
UNION ALL
|
||||||
|
SELECT 'domrf_dashboard', snapshot_date, NULL, 'РФ', NULL,
|
||||||
|
'avg_rate_pct', 'primary', primary_credit_avg_rate, primary_credit_avg_rate_delta, NULL
|
||||||
|
FROM domrf_mortgage_dashboard
|
||||||
|
UNION ALL
|
||||||
|
SELECT 'domrf_dashboard', snapshot_date, NULL, 'РФ', NULL,
|
||||||
|
'avg_rate_pct', 'secondary', secondary_credit_avg_rate, secondary_credit_avg_rate_delta, NULL
|
||||||
|
FROM domrf_mortgage_dashboard
|
||||||
|
UNION ALL
|
||||||
|
SELECT 'domrf_dashboard', snapshot_date, NULL, 'РФ', NULL,
|
||||||
|
'credit_count', 'total', total_credit_count::numeric, total_credit_count_delta_pct, NULL
|
||||||
|
FROM domrf_mortgage_dashboard
|
||||||
|
UNION ALL
|
||||||
|
SELECT 'domrf_dashboard', snapshot_date, NULL, 'РФ', NULL,
|
||||||
|
'credit_count', 'primary', primary_credit_count::numeric, primary_credit_count_delta_pct, NULL
|
||||||
|
FROM domrf_mortgage_dashboard
|
||||||
|
UNION ALL
|
||||||
|
SELECT 'domrf_dashboard', snapshot_date, NULL, 'РФ', NULL,
|
||||||
|
'credit_count', 'secondary', secondary_credit_count::numeric, secondary_credit_count_delta_pct, NULL
|
||||||
|
FROM domrf_mortgage_dashboard
|
||||||
|
UNION ALL
|
||||||
|
SELECT 'domrf_dashboard', snapshot_date, NULL, 'РФ', NULL,
|
||||||
|
'credit_amount_rub', 'total', total_credit_amount, total_credit_amount_delta_pct, NULL
|
||||||
|
FROM domrf_mortgage_dashboard
|
||||||
|
UNION ALL
|
||||||
|
SELECT 'domrf_dashboard', snapshot_date, NULL, 'РФ', NULL,
|
||||||
|
'credit_amount_rub', 'primary', primary_credit_amount, primary_credit_amount_delta_pct, NULL
|
||||||
|
FROM domrf_mortgage_dashboard
|
||||||
|
UNION ALL
|
||||||
|
SELECT 'domrf_dashboard', snapshot_date, NULL, 'РФ', NULL,
|
||||||
|
'credit_amount_rub', 'secondary', secondary_credit_amount, secondary_credit_amount_delta_pct, NULL
|
||||||
|
FROM domrf_mortgage_dashboard
|
||||||
|
UNION ALL
|
||||||
|
SELECT 'domrf_details', snapshot_date, NULL, 'РФ', NULL,
|
||||||
|
'credit_amount_avg_rub', currency, credit_amount_avg, credit_amount_avg_delta_pct, NULL
|
||||||
|
FROM domrf_mortgage_details
|
||||||
|
UNION ALL
|
||||||
|
SELECT 'domrf_details', snapshot_date, NULL, 'РФ', NULL,
|
||||||
|
'credit_avg_period_months', currency, credit_avg_period, credit_avg_period_delta_pct, NULL
|
||||||
|
FROM domrf_mortgage_details
|
||||||
|
UNION ALL
|
||||||
|
SELECT 'domrf_details', snapshot_date, NULL, 'РФ', NULL,
|
||||||
|
'credit_debts_amount_rub', currency, credit_debts_amount, credit_debts_amount_delta_pct, NULL
|
||||||
|
FROM domrf_mortgage_details
|
||||||
|
UNION ALL
|
||||||
|
SELECT 'domrf_details', snapshot_date, NULL, 'РФ', NULL,
|
||||||
|
'overdue_pct', currency, credit_debts_overdue_percent, credit_debts_overdue_percent_delta, NULL
|
||||||
|
FROM domrf_mortgage_details
|
||||||
|
UNION ALL
|
||||||
|
SELECT 'domrf_rates', snapshot_date, NULL, 'РФ', bank_name,
|
||||||
|
'rate_pct', 'primary', primary_rate, NULL, NULL FROM domrf_mortgage_rates
|
||||||
|
UNION ALL
|
||||||
|
SELECT 'domrf_rates', snapshot_date, NULL, 'РФ', bank_name,
|
||||||
|
'rate_pct', 'secondary', secondary_rate, NULL, NULL FROM domrf_mortgage_rates
|
||||||
|
UNION ALL
|
||||||
|
SELECT 'domrf_rates', snapshot_date, NULL, 'РФ', bank_name,
|
||||||
|
'rate_pct', 'refinance', refinance_rate, NULL, NULL FROM domrf_mortgage_rates;
|
||||||
|
|
||||||
|
COMMENT ON VIEW v_mortgage_unified IS
|
||||||
|
'Long-формат всех 4 источников ипотечных метрик. metric_kind: avg_rate_pct '
|
||||||
|
'/ credit_count / credit_amount_rub / overdue_pct / etc. segment: primary / '
|
||||||
|
'secondary / total. ~3385 rows total. Источники не дропаются.';
|
||||||
|
|
||||||
|
-- ── 3. v_construction_metrics_unified — 5 launch/commissioning ──────────────
|
||||||
|
|
||||||
|
CREATE OR REPLACE VIEW v_construction_metrics_unified AS
|
||||||
|
SELECT 'planned_commissioning' AS source, snapshot_date, plan_year::text AS period_label,
|
||||||
|
'РФ' AS region, 'planned_total_th_sqm' AS metric_kind,
|
||||||
|
NULL::text AS breakdown_kind, NULL::text AS breakdown_value,
|
||||||
|
total_th_sqm AS value,
|
||||||
|
jsonb_build_object('escrow', escrow_th_sqm, 'fund', fund_th_sqm,
|
||||||
|
'no_attraction', no_attraction_th_sqm) AS extra
|
||||||
|
FROM domrf_planned_commissioning
|
||||||
|
UNION ALL
|
||||||
|
SELECT 'commissioning', snapshot_date, rep_year::text, region_name,
|
||||||
|
'fact_area_total', NULL, NULL, accumulated_fact_area,
|
||||||
|
jsonb_build_object('multifamily', accumulated_fact_area_multifamily,
|
||||||
|
'private', accumulated_fact_area_private,
|
||||||
|
'change', accumulated_fact_area_change,
|
||||||
|
'change_share', accumulated_fact_area_change_share)
|
||||||
|
FROM domrf_commissioning
|
||||||
|
UNION ALL
|
||||||
|
SELECT 'launch_monthly', snapshot_date,
|
||||||
|
(rep_year::text || '-' || LPAD(rep_month::text, 2, '0')),
|
||||||
|
'РФ', 'launch_app_value', 'calc_type', calc_type, app_value,
|
||||||
|
jsonb_build_object('rnv_value', rnv_value)
|
||||||
|
FROM domrf_launch_monthly
|
||||||
|
UNION ALL
|
||||||
|
SELECT 'launch_monthly', snapshot_date,
|
||||||
|
(rep_year::text || '-' || LPAD(rep_month::text, 2, '0')),
|
||||||
|
'РФ', 'launch_rnv_value', 'calc_type', calc_type, rnv_value, NULL
|
||||||
|
FROM domrf_launch_monthly
|
||||||
|
UNION ALL
|
||||||
|
SELECT 'launch_top', snapshot_date, rep_year::text, entity_name,
|
||||||
|
metric_type, dim_type, entity_id, value,
|
||||||
|
jsonb_build_object('calc_type', calc_type)
|
||||||
|
FROM domrf_launch_top
|
||||||
|
UNION ALL
|
||||||
|
SELECT 'launch_obj_class', snapshot_date, rep_year::text, 'РФ',
|
||||||
|
'launch_app_value', 'obj_class', obj_class_desc, app_value,
|
||||||
|
jsonb_build_object('rnv_value', rnv_value, 'calc_type', calc_type,
|
||||||
|
'obj_class_cd', obj_class_cd)
|
||||||
|
FROM domrf_launch_obj_class
|
||||||
|
UNION ALL
|
||||||
|
SELECT 'launch_obj_class', snapshot_date, rep_year::text, 'РФ',
|
||||||
|
'launch_rnv_value', 'obj_class', obj_class_desc, rnv_value,
|
||||||
|
jsonb_build_object('calc_type', calc_type)
|
||||||
|
FROM domrf_launch_obj_class;
|
||||||
|
|
||||||
|
COMMENT ON VIEW v_construction_metrics_unified IS
|
||||||
|
'Long-формат для 5 таблиц про launch/commissioning. metric_kind: '
|
||||||
|
'planned_total_th_sqm / fact_area_total / launch_app_value / launch_rnv_value. '
|
||||||
|
'breakdown_kind: calc_type / obj_class / dim_type. ~1579 rows.';
|
||||||
|
|
||||||
|
-- ── 4. v_sold_ready_unified — 4 sold_ready / sold_out ──────────────────────
|
||||||
|
|
||||||
|
CREATE OR REPLACE VIEW v_sold_ready_unified AS
|
||||||
|
SELECT 'sold_out' AS source, snapshot_date, NULL::text AS period_label,
|
||||||
|
territory_name AS scope, NULL::text AS breakdown_kind, NULL::text AS breakdown_value,
|
||||||
|
'sold_pct' AS metric_kind, sold_pct AS value,
|
||||||
|
jsonb_build_object('total_area_th_sqm', total_area_th_sqm,
|
||||||
|
'sold_area_th_sqm', sold_area_th_sqm,
|
||||||
|
'unsold_pct', unsold_pct,
|
||||||
|
'avg_price_per_sqm', avg_price_per_sqm,
|
||||||
|
'attracted_funds_mln_rub', attracted_funds_mln_rub) AS extra
|
||||||
|
FROM domrf_sold_out
|
||||||
|
UNION ALL
|
||||||
|
SELECT 'sold_ready_index', snapshot_date,
|
||||||
|
(rep_year::text || '-' || LPAD(rep_month::text, 2, '0')),
|
||||||
|
'РФ', NULL, NULL, 'sold_ready_perc', sold_ready_perc,
|
||||||
|
jsonb_build_object('square_sum', square_sum, 'sold_sum', sold_sum,
|
||||||
|
'sold_perc', sold_perc, 'ready_perc', ready_perc)
|
||||||
|
FROM domrf_sold_ready_index
|
||||||
|
UNION ALL
|
||||||
|
SELECT 'sold_ready_dynamics', snapshot_date,
|
||||||
|
(rep_year::text || '-' || LPAD(rep_month::text, 2, '0')),
|
||||||
|
'РФ', 'dynamic_chart_type', dynamic_chart_type,
|
||||||
|
dynamic_chart_type, value, NULL
|
||||||
|
FROM domrf_sold_ready_dynamics
|
||||||
|
UNION ALL
|
||||||
|
SELECT 'sold_ready_breakdown', snapshot_date,
|
||||||
|
(rep_year::text || '-' || LPAD(COALESCE(rep_month, 0)::text, 2, '0')),
|
||||||
|
COALESCE(NULLIF(dev_group_value, ''), 'РФ'),
|
||||||
|
chart_type, entity_key, 'sold_perc', sold_perc,
|
||||||
|
jsonb_build_object('ready_perc', ready_perc, 'sold_ready_perc', sold_ready_perc,
|
||||||
|
'square_sum', square_sum, 'fo_cd', fo_cd, 'region_cd', region_cd,
|
||||||
|
'obj_class', obj_class, 'ready_year', ready_year,
|
||||||
|
'city_population', city_population)
|
||||||
|
FROM domrf_sold_ready_breakdown;
|
||||||
|
|
||||||
|
COMMENT ON VIEW v_sold_ready_unified IS
|
||||||
|
'Long-формат 4 sold_ready таблиц. metric_kind: sold_pct / sold_ready_perc / '
|
||||||
|
'<dynamic_chart_type>. Big source: sold_ready_breakdown (~39k) — фильтруй '
|
||||||
|
'по chart_type/entity_key. Total: ~42858 rows.';
|
||||||
|
|
||||||
|
-- ── 5. v_scrape_runs_unified — 3 скрапера в одном журнале ───────────────────
|
||||||
|
|
||||||
|
CREATE OR REPLACE VIEW v_scrape_runs_unified AS
|
||||||
|
SELECT 'kn' AS scraper_type, run_id, started_at, finished_at, heartbeat_at,
|
||||||
|
status, error, NULL::text AS triggered_by,
|
||||||
|
region_codes::text AS scope, requests_count,
|
||||||
|
objects_count AS items_ok, NULL::integer AS items_failed,
|
||||||
|
flats_count AS sub_items_ok,
|
||||||
|
progress_obj_index AS progress_idx, total_obj_count AS progress_total,
|
||||||
|
jsonb_build_object('developers', developer_ids, 'snapshot_date', snapshot_date,
|
||||||
|
'resumed_from', resumed_from_run_id, 'params', params) AS extra
|
||||||
|
FROM kn_scrape_runs
|
||||||
|
UNION ALL
|
||||||
|
SELECT 'nspd', run_id, started_at, finished_at, heartbeat_at, status, error,
|
||||||
|
triggered_by, region_code::text, requests_count,
|
||||||
|
quarters_ok, quarters_failed, buildings_ok,
|
||||||
|
NULL, pending_count,
|
||||||
|
jsonb_build_object('waf_429_count', waf_429_count)
|
||||||
|
FROM nspd_scrape_runs
|
||||||
|
UNION ALL
|
||||||
|
SELECT 'objective', run_id, started_at, finished_at, heartbeat_at, status, error,
|
||||||
|
triggered_by, group_name, requests_count,
|
||||||
|
reports_ok, reports_failed, rows_lots,
|
||||||
|
NULL, NULL,
|
||||||
|
jsonb_build_object('rows_corpus_room', rows_corpus_room, 'rows_history', rows_history)
|
||||||
|
FROM objective_scrape_runs;
|
||||||
|
|
||||||
|
COMMENT ON VIEW v_scrape_runs_unified IS
|
||||||
|
'Унифицированный журнал прогонов для всех 3 скраперов. Полиморфные общие '
|
||||||
|
'поля (items_ok / sub_items_ok / scope) — для UI; специфичные — в extra '
|
||||||
|
'jsonb. Используй для общего admin /scrape dashboard вместо 3 отдельных endpoint-ов.';
|
||||||
|
|
||||||
|
-- ── 6. v_scrape_log_unified — kn + nspd логи ────────────────────────────────
|
||||||
|
|
||||||
|
CREATE OR REPLACE VIEW v_scrape_log_unified AS
|
||||||
|
SELECT 'kn' AS scraper_type, log_id, run_id, ts, level, stage,
|
||||||
|
obj_id::text AS entity_id, message
|
||||||
|
FROM kn_scrape_log
|
||||||
|
UNION ALL
|
||||||
|
SELECT 'nspd', log_id, run_id, ts, level, stage, cad_number, message
|
||||||
|
FROM nspd_scrape_log;
|
||||||
|
|
||||||
|
COMMENT ON VIEW v_scrape_log_unified IS
|
||||||
|
'Per-step log двух скраперов. entity_id = obj_id для kn / cad_number для nspd. '
|
||||||
|
'objective scraper пока не пишет log — добавим если понадобится debug.';
|
||||||
123
data/sql/75_unification_naming.sql
Normal file
123
data/sql/75_unification_naming.sql
Normal file
|
|
@ -0,0 +1,123 @@
|
||||||
|
-- Унификация naming: region_cd/region_code → region_id, dev_id → developer_id,
|
||||||
|
-- dev_name → developer_name.
|
||||||
|
--
|
||||||
|
-- ПОДХОД: GENERATED ALWAYS AS (...) STORED — добавляем синонимные колонки
|
||||||
|
-- которые автоматически синхронизируются с источником. НЕ ALTER RENAME COLUMN
|
||||||
|
-- (последний сразу сломает работающий backend, который ссылается на старые
|
||||||
|
-- имена в SQL text() literals в 10 файлах).
|
||||||
|
--
|
||||||
|
-- Backend продолжает работать со старыми именами. Новый код может
|
||||||
|
-- использовать унифицированные. Постепенно мигрировать backend → потом DROP
|
||||||
|
-- старых колонок отдельной миграцией.
|
||||||
|
--
|
||||||
|
-- Storage cost: int 4B × N rows. Для rosreestr_deals (~7M rows) = ~28 MB.
|
||||||
|
-- Суммарно по всем таблицам ~50 MB. Приемлемо.
|
||||||
|
--
|
||||||
|
-- Применено в проде через mcp postgres 2026-05-10.
|
||||||
|
-- Идемпотентно через ADD COLUMN IF NOT EXISTS.
|
||||||
|
|
||||||
|
-- ── Plus task 2/3 (выполнены отдельным запросом, документирую здесь) ────────
|
||||||
|
-- UPDATE complexes SET obj_class и district_name из objective_lots для 350
|
||||||
|
-- сматченных через cs.source='objective'. obj_class заполнен = 350,
|
||||||
|
-- district_name = 1525 (приоритет — Objective).
|
||||||
|
|
||||||
|
-- ── region_cd → region_id (5 таблиц) ────────────────────────────────────────
|
||||||
|
ALTER TABLE domrf_kn_objects ADD COLUMN IF NOT EXISTS region_id INT GENERATED ALWAYS AS (region_cd) STORED;
|
||||||
|
ALTER TABLE domrf_kn_flats ADD COLUMN IF NOT EXISTS region_id INT GENERATED ALWAYS AS (region_cd) STORED;
|
||||||
|
ALTER TABLE domrf_project_finance ADD COLUMN IF NOT EXISTS region_id INT GENERATED ALWAYS AS (region_cd) STORED;
|
||||||
|
ALTER TABLE domrf_sold_ready_breakdown ADD COLUMN IF NOT EXISTS region_id INT GENERATED ALWAYS AS (region_cd) STORED;
|
||||||
|
ALTER TABLE rosreestr_rent_deals ADD COLUMN IF NOT EXISTS region_id INT GENERATED ALWAYS AS (region_cd) STORED;
|
||||||
|
|
||||||
|
-- ── region_code → region_id (3 таблицы; rosreestr_deals = partitioned, → 9 партиций) ──
|
||||||
|
ALTER TABLE domrf_realization ADD COLUMN IF NOT EXISTS region_id INT GENERATED ALWAYS AS (region_code) STORED;
|
||||||
|
ALTER TABLE nspd_scrape_runs ADD COLUMN IF NOT EXISTS region_id INT GENERATED ALWAYS AS (region_code) STORED;
|
||||||
|
ALTER TABLE rosreestr_deals ADD COLUMN IF NOT EXISTS region_id INT GENERATED ALWAYS AS (region_code) STORED;
|
||||||
|
|
||||||
|
-- ── dev_id → developer_id (3 таблицы) ───────────────────────────────────────
|
||||||
|
ALTER TABLE domrf_kn_objects ADD COLUMN IF NOT EXISTS developer_id TEXT GENERATED ALWAYS AS (dev_id) STORED;
|
||||||
|
ALTER TABLE domrf_dev_growth ADD COLUMN IF NOT EXISTS developer_id TEXT GENERATED ALWAYS AS (dev_id) STORED;
|
||||||
|
ALTER TABLE sv_dev_sales ADD COLUMN IF NOT EXISTS developer_id TEXT GENERATED ALWAYS AS (dev_id) STORED;
|
||||||
|
|
||||||
|
-- ── dev_name → developer_name (2 таблицы) ───────────────────────────────────
|
||||||
|
ALTER TABLE domrf_kn_objects ADD COLUMN IF NOT EXISTS developer_name TEXT GENERATED ALWAYS AS (dev_name) STORED;
|
||||||
|
ALTER TABLE domrf_dev_growth ADD COLUMN IF NOT EXISTS developer_name TEXT GENERATED ALWAYS AS (dev_name) STORED;
|
||||||
|
|
||||||
|
-- ── COMMENT-указатель миграции ──────────────────────────────────────────────
|
||||||
|
COMMENT ON COLUMN domrf_kn_objects.region_cd IS
|
||||||
|
'DEPRECATED: используй region_id (синоним через GENERATED). '
|
||||||
|
'Удалить после миграции backend SQL queries.';
|
||||||
|
COMMENT ON COLUMN domrf_kn_objects.dev_id IS
|
||||||
|
'DEPRECATED: используй developer_id (синоним через GENERATED).';
|
||||||
|
COMMENT ON COLUMN domrf_kn_objects.dev_name IS
|
||||||
|
'DEPRECATED: используй developer_name (синоним через GENERATED).';
|
||||||
|
COMMENT ON COLUMN rosreestr_deals.region_code IS
|
||||||
|
'DEPRECATED: используй region_id (синоним через GENERATED).';
|
||||||
|
|
||||||
|
-- ── 4. complexes update: obj_class + district из Objective ──────────────────
|
||||||
|
WITH objective_per_complex AS (
|
||||||
|
SELECT DISTINCT ON (cs.complex_id) cs.complex_id,
|
||||||
|
ol.class AS obj_class,
|
||||||
|
ol.district AS district_name
|
||||||
|
FROM complex_sources cs
|
||||||
|
JOIN objective_lots ol ON ol.project_name = cs.source_id
|
||||||
|
WHERE cs.source = 'objective'
|
||||||
|
ORDER BY cs.complex_id, ol.snapshot_date DESC NULLS LAST
|
||||||
|
)
|
||||||
|
UPDATE complexes c
|
||||||
|
SET obj_class = COALESCE(opc.obj_class, c.obj_class),
|
||||||
|
district_name = COALESCE(opc.district_name, c.district_name),
|
||||||
|
updated_at = NOW()
|
||||||
|
FROM objective_per_complex opc
|
||||||
|
WHERE opc.complex_id = c.id;
|
||||||
|
|
||||||
|
-- ── 5. Fuzzy auto-merge orphan complexes (high-confidence ≥0.85) ────────────
|
||||||
|
-- См. полный SQL-блок в applied миграции через mcp postgres.
|
||||||
|
-- Эффект: 14 high-confidence orphan Objective complexes слились с domrf_kn
|
||||||
|
-- (например 'Чемпион Парк' / 'Discovery Residence' / 'АЛЛЕГРО'). После merge:
|
||||||
|
-- complexes 1752 → 1740, in_both_sources_now 114 → 124.
|
||||||
|
|
||||||
|
-- ── 6. v_complex_match_review для остальных 82 кандидатов ───────────────────
|
||||||
|
CREATE EXTENSION IF NOT EXISTS pg_trgm;
|
||||||
|
|
||||||
|
CREATE OR REPLACE VIEW v_complex_match_review AS
|
||||||
|
WITH orphan_complexes AS (
|
||||||
|
SELECT c.id AS orphan_id, c.canonical_name AS orphan_name,
|
||||||
|
c.developer_name AS orphan_dev, c.district_name AS orphan_district,
|
||||||
|
c.obj_class AS orphan_class
|
||||||
|
FROM complexes c WHERE c.primary_source = 'objective'
|
||||||
|
),
|
||||||
|
kn_complexes AS (
|
||||||
|
SELECT c.id AS kn_id, c.canonical_name AS kn_name,
|
||||||
|
c.developer_name AS kn_dev, c.district_name AS kn_district
|
||||||
|
FROM complexes c WHERE c.primary_source = 'domrf_kn'
|
||||||
|
),
|
||||||
|
candidates AS (
|
||||||
|
SELECT oc.*, kc.kn_id, kc.kn_name, kc.kn_dev, kc.kn_district,
|
||||||
|
similarity(oc.orphan_name, kc.kn_name) AS name_sim,
|
||||||
|
CASE WHEN oc.orphan_dev IS NOT NULL AND kc.kn_dev IS NOT NULL
|
||||||
|
THEN similarity(oc.orphan_dev, kc.kn_dev) ELSE 0 END AS dev_sim,
|
||||||
|
CASE WHEN oc.orphan_district = kc.kn_district THEN 1.0 ELSE 0 END AS district_match
|
||||||
|
FROM orphan_complexes oc CROSS JOIN kn_complexes kc
|
||||||
|
WHERE similarity(oc.orphan_name, kc.kn_name) >= 0.4
|
||||||
|
),
|
||||||
|
ranked AS (
|
||||||
|
SELECT *, (name_sim * 0.7 + dev_sim * 0.2 + district_match * 0.1) AS combined_score,
|
||||||
|
ROW_NUMBER() OVER (PARTITION BY orphan_id ORDER BY (name_sim * 0.7 + dev_sim * 0.2 + district_match * 0.1) DESC) AS rk
|
||||||
|
FROM candidates
|
||||||
|
)
|
||||||
|
SELECT orphan_id, orphan_name, orphan_dev, orphan_class, orphan_district,
|
||||||
|
kn_id, kn_name, kn_dev, kn_district,
|
||||||
|
ROUND(name_sim::numeric, 3) AS name_sim,
|
||||||
|
ROUND(dev_sim::numeric, 3) AS dev_sim,
|
||||||
|
ROUND(combined_score::numeric, 3) AS score,
|
||||||
|
CASE WHEN combined_score >= 0.7 THEN 'likely'
|
||||||
|
WHEN combined_score >= 0.5 THEN 'maybe'
|
||||||
|
ELSE 'unlikely' END AS confidence
|
||||||
|
FROM ranked WHERE rk = 1 AND combined_score >= 0.5
|
||||||
|
ORDER BY combined_score DESC;
|
||||||
|
|
||||||
|
COMMENT ON VIEW v_complex_match_review IS
|
||||||
|
'Кандидаты на merge orphan Objective complexes с domrf_kn. Top-1 candidate '
|
||||||
|
'per orphan_id с similarity >= 0.5. Для review через admin UI: '
|
||||||
|
'если confidence=likely (≥0.7) — почти наверняка дубль, maybe (≥0.5) — '
|
||||||
|
'посмотреть глазами. Auto-merge для confidence>=0.85 уже выполнен.';
|
||||||
|
|
@ -4,6 +4,7 @@ import Link from "next/link";
|
||||||
import { usePathname } from "next/navigation";
|
import { usePathname } from "next/navigation";
|
||||||
|
|
||||||
const TABS = [
|
const TABS = [
|
||||||
|
{ href: "/admin/scrape/all", label: "📊 Все скраперы (сводно)" },
|
||||||
{ href: "/admin/scrape", label: "Скрапер DOM.РФ" },
|
{ href: "/admin/scrape", label: "Скрапер DOM.РФ" },
|
||||||
{ href: "/admin/scrape/nspd", label: "Скрапер NSPD" },
|
{ href: "/admin/scrape/nspd", label: "Скрапер NSPD" },
|
||||||
{ href: "/admin/scrape/objective", label: "ETL Objective" },
|
{ href: "/admin/scrape/objective", label: "ETL Objective" },
|
||||||
|
|
|
||||||
538
frontend/src/app/admin/scrape/all/page.tsx
Normal file
538
frontend/src/app/admin/scrape/all/page.tsx
Normal file
|
|
@ -0,0 +1,538 @@
|
||||||
|
"use client";
|
||||||
|
|
||||||
|
import { useState } from "react";
|
||||||
|
import { useQuery } from "@tanstack/react-query";
|
||||||
|
|
||||||
|
import { apiFetch } from "@/lib/api";
|
||||||
|
|
||||||
|
interface UnifiedRunRow {
|
||||||
|
scraper_type: "kn" | "nspd" | "objective";
|
||||||
|
run_id: number;
|
||||||
|
started_at: string | null;
|
||||||
|
finished_at: string | null;
|
||||||
|
heartbeat_at: string | null;
|
||||||
|
status: string;
|
||||||
|
error: string | null;
|
||||||
|
triggered_by: string | null;
|
||||||
|
scope: string | null;
|
||||||
|
requests_count: number | null;
|
||||||
|
items_ok: number | null;
|
||||||
|
items_failed: number | null;
|
||||||
|
sub_items_ok: number | null;
|
||||||
|
progress_idx: number | null;
|
||||||
|
progress_total: number | null;
|
||||||
|
extra: Record<string, unknown> | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface UnifiedLogRow {
|
||||||
|
scraper_type: "kn" | "nspd";
|
||||||
|
log_id: number;
|
||||||
|
run_id: number | null;
|
||||||
|
ts: string | null;
|
||||||
|
level: "debug" | "info" | "warn" | "error";
|
||||||
|
stage: string | null;
|
||||||
|
entity_id: string | null;
|
||||||
|
message: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
const SCRAPER_BADGE: Record<string, { bg: string; fg: string; label: string }> =
|
||||||
|
{
|
||||||
|
kn: { bg: "#dbeafe", fg: "#1d4ed8", label: "DOM.РФ kn" },
|
||||||
|
nspd: { bg: "#fef3c7", fg: "#a16207", label: "NSPD" },
|
||||||
|
objective: { bg: "#d1fae5", fg: "#059669", label: "Objective" },
|
||||||
|
};
|
||||||
|
|
||||||
|
const STATUS_COLOR: Record<string, string> = {
|
||||||
|
done: "#16a34a",
|
||||||
|
running: "#1d4ed8",
|
||||||
|
failed: "#b3261e",
|
||||||
|
zombie: "#9333ea",
|
||||||
|
};
|
||||||
|
|
||||||
|
function formatIso(s: string | null): string {
|
||||||
|
if (!s) return "—";
|
||||||
|
return new Date(s).toLocaleString("ru-RU");
|
||||||
|
}
|
||||||
|
|
||||||
|
function diffSecs(a: string | null, b: string | null): string {
|
||||||
|
if (!a || !b) return "—";
|
||||||
|
const dt = (new Date(b).getTime() - new Date(a).getTime()) / 1000;
|
||||||
|
if (dt < 60) return `${dt.toFixed(0)}с`;
|
||||||
|
if (dt < 3600) return `${(dt / 60).toFixed(1)} мин`;
|
||||||
|
return `${(dt / 3600).toFixed(1)} ч`;
|
||||||
|
}
|
||||||
|
|
||||||
|
export default function ScrapeAllAdminPage() {
|
||||||
|
const [token, setToken] = useState<string>(() =>
|
||||||
|
typeof window === "undefined"
|
||||||
|
? ""
|
||||||
|
: (localStorage.getItem("admin_token") ?? ""),
|
||||||
|
);
|
||||||
|
const [filter, setFilter] = useState<"all" | "kn" | "nspd" | "objective">(
|
||||||
|
"all",
|
||||||
|
);
|
||||||
|
const [logsRunId, setLogsRunId] = useState<number | "">("");
|
||||||
|
|
||||||
|
const saveToken = (v: string) => {
|
||||||
|
setToken(v);
|
||||||
|
localStorage.setItem("admin_token", v);
|
||||||
|
};
|
||||||
|
|
||||||
|
const runs = useQuery({
|
||||||
|
queryKey: ["unified-runs", token, filter],
|
||||||
|
queryFn: () => {
|
||||||
|
const q =
|
||||||
|
filter !== "all" ? `?scraper_type=${filter}&limit=30` : "?limit=30";
|
||||||
|
return apiFetch<UnifiedRunRow[]>(`/api/v1/admin/scrape/all/runs${q}`, {
|
||||||
|
headers: { "X-Admin-Token": token },
|
||||||
|
});
|
||||||
|
},
|
||||||
|
enabled: !!token,
|
||||||
|
refetchInterval: 5000,
|
||||||
|
});
|
||||||
|
|
||||||
|
const logs = useQuery({
|
||||||
|
queryKey: ["unified-logs", token, filter, logsRunId],
|
||||||
|
queryFn: () => {
|
||||||
|
const params = new URLSearchParams({ limit: "100" });
|
||||||
|
if (filter !== "all") params.set("scraper_type", filter);
|
||||||
|
if (logsRunId !== "") params.set("run_id", String(logsRunId));
|
||||||
|
return apiFetch<UnifiedLogRow[]>(
|
||||||
|
`/api/v1/admin/scrape/all/logs?${params}`,
|
||||||
|
{ headers: { "X-Admin-Token": token } },
|
||||||
|
);
|
||||||
|
},
|
||||||
|
enabled: !!token,
|
||||||
|
refetchInterval: 8000,
|
||||||
|
});
|
||||||
|
|
||||||
|
// Сводка по statusам
|
||||||
|
const summary = (() => {
|
||||||
|
const s = { running: 0, done: 0, failed: 0, zombie: 0 } as Record<
|
||||||
|
string,
|
||||||
|
number
|
||||||
|
>;
|
||||||
|
runs.data?.forEach((r) => {
|
||||||
|
s[r.status] = (s[r.status] ?? 0) + 1;
|
||||||
|
});
|
||||||
|
return s;
|
||||||
|
})();
|
||||||
|
|
||||||
|
return (
|
||||||
|
<>
|
||||||
|
<h2 style={{ marginTop: 0, fontSize: 18 }}>
|
||||||
|
Скраперы — сводный dashboard
|
||||||
|
</h2>
|
||||||
|
<p style={{ color: "#5b6066", fontSize: 13, marginTop: 4 }}>
|
||||||
|
Все 3 скрапера в одной таблице (через <code>v_scrape_runs_unified</code>
|
||||||
|
). Для запуска / триггеров — отдельные страницы:{" "}
|
||||||
|
<a href="/admin/scrape" style={{ color: "#1d4ed8" }}>
|
||||||
|
kn
|
||||||
|
</a>{" "}
|
||||||
|
·{" "}
|
||||||
|
<a href="/admin/scrape/nspd" style={{ color: "#1d4ed8" }}>
|
||||||
|
NSPD
|
||||||
|
</a>{" "}
|
||||||
|
·{" "}
|
||||||
|
<a href="/admin/scrape/objective" style={{ color: "#1d4ed8" }}>
|
||||||
|
Objective
|
||||||
|
</a>
|
||||||
|
.
|
||||||
|
</p>
|
||||||
|
|
||||||
|
<section style={cardStyle}>
|
||||||
|
<label style={{ display: "block" }}>
|
||||||
|
<span style={labelStyle}>Admin Token</span>
|
||||||
|
<input
|
||||||
|
type="password"
|
||||||
|
value={token}
|
||||||
|
onChange={(e) => saveToken(e.target.value)}
|
||||||
|
placeholder="SCRAPE_ADMIN_TOKEN"
|
||||||
|
style={inputStyle}
|
||||||
|
/>
|
||||||
|
</label>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
{/* Сводка-плитки */}
|
||||||
|
<section
|
||||||
|
style={{
|
||||||
|
...cardStyle,
|
||||||
|
marginTop: 16,
|
||||||
|
display: "grid",
|
||||||
|
gridTemplateColumns: "repeat(4, 1fr)",
|
||||||
|
gap: 16,
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<Stat
|
||||||
|
label="🟦 в работе"
|
||||||
|
value={summary.running ?? 0}
|
||||||
|
color="#1d4ed8"
|
||||||
|
/>
|
||||||
|
<Stat label="✅ успешно" value={summary.done ?? 0} color="#16a34a" />
|
||||||
|
<Stat label="❌ failed" value={summary.failed ?? 0} color="#b3261e" />
|
||||||
|
<Stat label="🧟 zombie" value={summary.zombie ?? 0} color="#9333ea" />
|
||||||
|
</section>
|
||||||
|
|
||||||
|
{/* Фильтр + табы */}
|
||||||
|
<section style={{ ...cardStyle, marginTop: 16 }}>
|
||||||
|
<div style={{ display: "flex", gap: 8, alignItems: "center" }}>
|
||||||
|
<span style={{ ...labelStyle, marginBottom: 0 }}>Filter:</span>
|
||||||
|
{(["all", "kn", "nspd", "objective"] as const).map((t) => (
|
||||||
|
<button
|
||||||
|
key={t}
|
||||||
|
type="button"
|
||||||
|
onClick={() => setFilter(t)}
|
||||||
|
style={{
|
||||||
|
...filterTab,
|
||||||
|
background: filter === t ? "#1d4ed8" : "#f3f4f6",
|
||||||
|
color: filter === t ? "#fff" : "#1f2937",
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
{t === "all" ? "Все" : (SCRAPER_BADGE[t]?.label ?? t)}
|
||||||
|
</button>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
{/* Таблица прогонов */}
|
||||||
|
<section style={{ ...cardStyle, marginTop: 16 }}>
|
||||||
|
<h3 style={sectionTitle}>
|
||||||
|
📜 Последние прогоны (top 30, обновляется каждые 5с)
|
||||||
|
</h3>
|
||||||
|
<div style={{ overflowX: "auto" }}>
|
||||||
|
<table style={tableStyle}>
|
||||||
|
<thead style={{ background: "#f9fafb" }}>
|
||||||
|
<tr>
|
||||||
|
<th style={th}>scraper</th>
|
||||||
|
<th style={th}>run_id</th>
|
||||||
|
<th style={th}>started</th>
|
||||||
|
<th style={th}>scope</th>
|
||||||
|
<th style={th}>duration</th>
|
||||||
|
<th style={{ ...th, textAlign: "right" }}>items_ok</th>
|
||||||
|
<th style={{ ...th, textAlign: "right" }}>failed</th>
|
||||||
|
<th style={{ ...th, textAlign: "right" }}>sub_ok</th>
|
||||||
|
<th style={th}>status</th>
|
||||||
|
<th style={th}>trigger</th>
|
||||||
|
<th style={th}></th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
{runs.data?.map((r) => {
|
||||||
|
const heartbeatLag = r.heartbeat_at
|
||||||
|
? (Date.now() - new Date(r.heartbeat_at).getTime()) / 1000
|
||||||
|
: null;
|
||||||
|
const isStale =
|
||||||
|
r.status === "running" &&
|
||||||
|
heartbeatLag !== null &&
|
||||||
|
heartbeatLag > 60;
|
||||||
|
const badge = SCRAPER_BADGE[r.scraper_type];
|
||||||
|
const statusColor = STATUS_COLOR[r.status] ?? "#6b7280";
|
||||||
|
return (
|
||||||
|
<tr
|
||||||
|
key={`${r.scraper_type}-${r.run_id}`}
|
||||||
|
style={{ borderTop: "1px solid #f3f4f6" }}
|
||||||
|
>
|
||||||
|
<td style={td}>
|
||||||
|
{badge && (
|
||||||
|
<span
|
||||||
|
style={{
|
||||||
|
display: "inline-block",
|
||||||
|
padding: "2px 8px",
|
||||||
|
borderRadius: 4,
|
||||||
|
background: badge.bg,
|
||||||
|
color: badge.fg,
|
||||||
|
fontWeight: 500,
|
||||||
|
fontSize: 11,
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
{badge.label}
|
||||||
|
</span>
|
||||||
|
)}
|
||||||
|
</td>
|
||||||
|
<td style={{ ...td, fontFamily: "monospace" }}>
|
||||||
|
{r.run_id}
|
||||||
|
</td>
|
||||||
|
<td style={td}>{formatIso(r.started_at)}</td>
|
||||||
|
<td
|
||||||
|
style={{ ...td, fontFamily: "monospace", fontSize: 11 }}
|
||||||
|
>
|
||||||
|
{r.scope ?? "—"}
|
||||||
|
</td>
|
||||||
|
<td style={td}>
|
||||||
|
{diffSecs(r.started_at, r.finished_at ?? r.heartbeat_at)}
|
||||||
|
</td>
|
||||||
|
<td
|
||||||
|
style={{
|
||||||
|
...td,
|
||||||
|
textAlign: "right",
|
||||||
|
fontFamily: "monospace",
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
{r.items_ok?.toLocaleString("ru-RU") ?? "—"}
|
||||||
|
</td>
|
||||||
|
<td
|
||||||
|
style={{
|
||||||
|
...td,
|
||||||
|
textAlign: "right",
|
||||||
|
fontFamily: "monospace",
|
||||||
|
color: r.items_failed ? "#b3261e" : "inherit",
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
{r.items_failed?.toLocaleString("ru-RU") ?? "—"}
|
||||||
|
</td>
|
||||||
|
<td
|
||||||
|
style={{
|
||||||
|
...td,
|
||||||
|
textAlign: "right",
|
||||||
|
fontFamily: "monospace",
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
{r.sub_items_ok?.toLocaleString("ru-RU") ?? "—"}
|
||||||
|
</td>
|
||||||
|
<td style={td}>
|
||||||
|
<span
|
||||||
|
style={{
|
||||||
|
display: "inline-block",
|
||||||
|
padding: "2px 8px",
|
||||||
|
borderRadius: 4,
|
||||||
|
background: statusColor + "22",
|
||||||
|
color: statusColor,
|
||||||
|
fontWeight: 500,
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
{isStale ? "stale" : r.status}
|
||||||
|
</span>
|
||||||
|
</td>
|
||||||
|
<td style={{ ...td, color: "#6b7280" }}>
|
||||||
|
{r.triggered_by ?? "—"}
|
||||||
|
</td>
|
||||||
|
<td style={td}>
|
||||||
|
{(r.scraper_type === "kn" ||
|
||||||
|
r.scraper_type === "nspd") && (
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
onClick={() => setLogsRunId(r.run_id)}
|
||||||
|
style={{
|
||||||
|
...secondaryBtn,
|
||||||
|
padding: "4px 10px",
|
||||||
|
fontSize: 11,
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
логи
|
||||||
|
</button>
|
||||||
|
)}
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
);
|
||||||
|
})}
|
||||||
|
{runs.data?.length === 0 && (
|
||||||
|
<tr>
|
||||||
|
<td
|
||||||
|
colSpan={11}
|
||||||
|
style={{
|
||||||
|
...td,
|
||||||
|
textAlign: "center",
|
||||||
|
color: "#6b7280",
|
||||||
|
padding: 24,
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
Прогонов с фильтром <code>{filter}</code> нет
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
)}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
{/* Logs */}
|
||||||
|
<section style={{ ...cardStyle, marginTop: 16 }}>
|
||||||
|
<h3 style={sectionTitle}>
|
||||||
|
📋 Логи (top 100){" "}
|
||||||
|
{logsRunId !== "" && (
|
||||||
|
<span style={{ fontSize: 13, fontWeight: 400, color: "#5b6066" }}>
|
||||||
|
· фильтр run_id={logsRunId}{" "}
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
onClick={() => setLogsRunId("")}
|
||||||
|
style={{
|
||||||
|
background: "none",
|
||||||
|
border: "none",
|
||||||
|
color: "#1d4ed8",
|
||||||
|
cursor: "pointer",
|
||||||
|
fontSize: 13,
|
||||||
|
padding: 0,
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
сбросить
|
||||||
|
</button>
|
||||||
|
</span>
|
||||||
|
)}
|
||||||
|
</h3>
|
||||||
|
<div style={{ overflowX: "auto", maxHeight: 400, overflowY: "auto" }}>
|
||||||
|
<table style={tableStyle}>
|
||||||
|
<thead
|
||||||
|
style={{ background: "#f9fafb", position: "sticky", top: 0 }}
|
||||||
|
>
|
||||||
|
<tr>
|
||||||
|
<th style={th}>ts</th>
|
||||||
|
<th style={th}>scraper</th>
|
||||||
|
<th style={th}>run</th>
|
||||||
|
<th style={th}>level</th>
|
||||||
|
<th style={th}>stage</th>
|
||||||
|
<th style={th}>entity</th>
|
||||||
|
<th style={th}>message</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
{logs.data?.map((l) => (
|
||||||
|
<tr
|
||||||
|
key={`${l.scraper_type}-${l.log_id}`}
|
||||||
|
style={{ borderTop: "1px solid #f3f4f6" }}
|
||||||
|
>
|
||||||
|
<td style={{ ...td, fontSize: 11, whiteSpace: "nowrap" }}>
|
||||||
|
{formatIso(l.ts)}
|
||||||
|
</td>
|
||||||
|
<td style={td}>
|
||||||
|
<span
|
||||||
|
style={{
|
||||||
|
fontSize: 10,
|
||||||
|
color: SCRAPER_BADGE[l.scraper_type]?.fg ?? "#6b7280",
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
{l.scraper_type}
|
||||||
|
</span>
|
||||||
|
</td>
|
||||||
|
<td style={{ ...td, fontFamily: "monospace" }}>
|
||||||
|
{l.run_id ?? "—"}
|
||||||
|
</td>
|
||||||
|
<td style={td}>
|
||||||
|
<span
|
||||||
|
style={{
|
||||||
|
color:
|
||||||
|
l.level === "error"
|
||||||
|
? "#b3261e"
|
||||||
|
: l.level === "warn"
|
||||||
|
? "#a16207"
|
||||||
|
: "#5b6066",
|
||||||
|
fontSize: 11,
|
||||||
|
fontWeight: 500,
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
{l.level}
|
||||||
|
</span>
|
||||||
|
</td>
|
||||||
|
<td style={{ ...td, fontSize: 11 }}>{l.stage ?? "—"}</td>
|
||||||
|
<td style={{ ...td, fontFamily: "monospace", fontSize: 11 }}>
|
||||||
|
{l.entity_id ?? "—"}
|
||||||
|
</td>
|
||||||
|
<td style={{ ...td, fontSize: 12 }}>{l.message}</td>
|
||||||
|
</tr>
|
||||||
|
))}
|
||||||
|
{logs.data?.length === 0 && (
|
||||||
|
<tr>
|
||||||
|
<td
|
||||||
|
colSpan={7}
|
||||||
|
style={{
|
||||||
|
...td,
|
||||||
|
textAlign: "center",
|
||||||
|
color: "#6b7280",
|
||||||
|
padding: 24,
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
Логов нет (objective пока не пишет в log — только runs)
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
)}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
</>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function Stat({
|
||||||
|
label,
|
||||||
|
value,
|
||||||
|
color,
|
||||||
|
}: {
|
||||||
|
label: string;
|
||||||
|
value: number;
|
||||||
|
color: string;
|
||||||
|
}) {
|
||||||
|
return (
|
||||||
|
<div>
|
||||||
|
<div style={{ fontSize: 12, color: "#5b6066", marginBottom: 4 }}>
|
||||||
|
{label}
|
||||||
|
</div>
|
||||||
|
<div
|
||||||
|
style={{
|
||||||
|
fontFamily: "monospace",
|
||||||
|
fontSize: 28,
|
||||||
|
fontWeight: 600,
|
||||||
|
color,
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
{value}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const cardStyle = {
|
||||||
|
background: "#fff",
|
||||||
|
border: "1px solid #e6e8ec",
|
||||||
|
borderRadius: 12,
|
||||||
|
padding: 20,
|
||||||
|
};
|
||||||
|
const sectionTitle = {
|
||||||
|
margin: "0 0 12px",
|
||||||
|
fontSize: 16,
|
||||||
|
fontWeight: 600 as const,
|
||||||
|
};
|
||||||
|
const labelStyle = {
|
||||||
|
display: "block",
|
||||||
|
fontSize: 12,
|
||||||
|
color: "#5b6066",
|
||||||
|
marginBottom: 4,
|
||||||
|
textTransform: "uppercase" as const,
|
||||||
|
letterSpacing: 0.4,
|
||||||
|
};
|
||||||
|
const inputStyle = {
|
||||||
|
padding: "8px 10px",
|
||||||
|
border: "1px solid #d1d5db",
|
||||||
|
borderRadius: 6,
|
||||||
|
fontSize: 14,
|
||||||
|
width: "100%",
|
||||||
|
boxSizing: "border-box" as const,
|
||||||
|
};
|
||||||
|
const filterTab = {
|
||||||
|
padding: "6px 14px",
|
||||||
|
border: "none",
|
||||||
|
borderRadius: 6,
|
||||||
|
fontSize: 13,
|
||||||
|
fontWeight: 500 as const,
|
||||||
|
cursor: "pointer",
|
||||||
|
};
|
||||||
|
const secondaryBtn = {
|
||||||
|
padding: "8px 14px",
|
||||||
|
background: "#f3f4f6",
|
||||||
|
color: "#1f2937",
|
||||||
|
border: "1px solid #d1d5db",
|
||||||
|
borderRadius: 6,
|
||||||
|
fontSize: 13,
|
||||||
|
cursor: "pointer",
|
||||||
|
};
|
||||||
|
const tableStyle = {
|
||||||
|
width: "100%",
|
||||||
|
borderCollapse: "collapse" as const,
|
||||||
|
fontSize: 12,
|
||||||
|
};
|
||||||
|
const th = {
|
||||||
|
padding: "8px 10px",
|
||||||
|
textAlign: "left" as const,
|
||||||
|
fontWeight: 600 as const,
|
||||||
|
borderBottom: "1px solid #e6e8ec",
|
||||||
|
};
|
||||||
|
const td = { padding: "8px 10px" };
|
||||||
Loading…
Add table
Reference in a new issue