6d3fa8cbd2
fix Docker build for playwright - Regenerate uv.lock with playwright + tenacity (was missing → uv sync --frozen skipped them, hence playwright not in /app/.venv/bin/). - Dockerfile: explicit Chromium runtime libs in apt (libnss3 + 14 more) + `playwright install chromium` BEFORE apt-lists cleanup. - Schema 51_schema_kn_extras.sql: 5 new tables for time-series sales, current aggregates, POI infrastructure, photo metadata, failure log. - Scraper: 4 new fetch helpers + upserts; photo binary download via Playwright APIRequest; failure-logging with full_url to kn_scrape_failures. - CLI: --no-extras / --download-photos / --no-flats / --probe URL. - Admin UI: extras checkboxes + failures table with copy-URL button. - PRINZIP smoke: 28 objs, 5409 POI, 5150 photos metadata, sales_agg matches probe (Парк Победы 145/291 = 49%).
97 lines
5.8 KiB
SQL
97 lines
5.8 KiB
SQL
-- Extras-эндпойнты наш.дом.рф (sale_graph, sales_agg, infrastructure, photos)
|
|
-- + журнал ошибок per-request для ручной проверки в браузере.
|
|
-- См. план C:\Users\user\.claude\plans\crispy-waddling-swing.md.
|
|
|
|
-- ── Time-series продаж per-ЖК (apartments + parking) ──────────────────────
|
|
-- Источник: GET /сервисы/api/object/{obj_id}/sale_graph?type=apartments|parking
|
|
CREATE TABLE IF NOT EXISTS domrf_kn_sale_graph (
|
|
obj_id BIGINT NOT NULL,
|
|
report_month DATE NOT NULL, -- '01.04.2025' → 2025-04-01
|
|
type TEXT NOT NULL, -- 'apartments' | 'parking'
|
|
realised INT, -- штук реализовано
|
|
contracted INT, -- штук законтрактовано (ДКП)
|
|
area_sq NUMERIC, -- м² за месяц
|
|
price_avg NUMERIC, -- средняя цена ₽/м²
|
|
snapshot_date DATE NOT NULL DEFAULT CURRENT_DATE,
|
|
UNIQUE (obj_id, report_month, type, snapshot_date)
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_sg_obj
|
|
ON domrf_kn_sale_graph (obj_id, snapshot_date);
|
|
CREATE INDEX IF NOT EXISTS idx_sg_month
|
|
ON domrf_kn_sale_graph (report_month, type);
|
|
|
|
-- ── Текущие агрегаты per-ЖК (apartments + nonliv + parking) ───────────────
|
|
-- Источник: GET /сервисы/api/object/{obj_id}/sales_agg
|
|
CREATE TABLE IF NOT EXISTS domrf_kn_sales_agg (
|
|
obj_id BIGINT NOT NULL,
|
|
type TEXT NOT NULL, -- 'apartments'|'nonliv'|'parking'
|
|
name TEXT, -- 'Квартира'|'Нежилое помещение'|'Машино-место'
|
|
total INT, -- всего юнитов в ЖК
|
|
realised INT, -- реализовано
|
|
perc NUMERIC, -- sold % (server-calculated)
|
|
snapshot_date DATE NOT NULL DEFAULT CURRENT_DATE,
|
|
UNIQUE (obj_id, type, snapshot_date)
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_sa_obj
|
|
ON domrf_kn_sales_agg (obj_id, snapshot_date);
|
|
|
|
-- ── POI вокруг ЖК (нормализованная) ───────────────────────────────────────
|
|
-- Источник: GET /сервисы/api/object/{obj_id}/infrastructure
|
|
CREATE TABLE IF NOT EXISTS domrf_kn_infrastructure (
|
|
obj_id BIGINT NOT NULL,
|
|
poi_name TEXT NOT NULL,
|
|
poi_subtitle TEXT,
|
|
poi_category TEXT, -- tabName: Развлечения|Образование|...
|
|
poi_address TEXT,
|
|
poi_lat NUMERIC,
|
|
poi_lon NUMERIC,
|
|
distance_m NUMERIC,
|
|
snapshot_date DATE NOT NULL DEFAULT CURRENT_DATE,
|
|
UNIQUE (obj_id, poi_name, poi_lat, poi_lon, snapshot_date)
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_infra_obj
|
|
ON domrf_kn_infrastructure (obj_id, snapshot_date);
|
|
CREATE INDEX IF NOT EXISTS idx_infra_cat_dist
|
|
ON domrf_kn_infrastructure (poi_category, distance_m);
|
|
|
|
-- ── Фото-метаданные (binary в data/raw/domrf_photos/{obj_id}/{file_id}.png) ─
|
|
-- Источник: GET /сервисы/api/object/construction/progress/photo/{obj_id}
|
|
CREATE TABLE IF NOT EXISTS domrf_kn_photos (
|
|
obj_id BIGINT NOT NULL,
|
|
obj_file_id TEXT NOT NULL, -- DOM.РФ хеш-ID, идемпотентен
|
|
ord_num INT,
|
|
photo_url TEXT, -- абсолютный URL DOM.РФ
|
|
photo_dttm TIMESTAMP,
|
|
period_dt DATE, -- '2026-04-01'
|
|
size_bytes BIGINT,
|
|
photo_name TEXT, -- 'PRINZIP_43_new.jpg'
|
|
ready_desc TEXT, -- 'Строится'/'Сдан'
|
|
build_type TEXT, -- 'Жилое'/'Нежилое'
|
|
hidden BOOLEAN,
|
|
local_path TEXT, -- если binary скачан
|
|
downloaded_at TIMESTAMPTZ,
|
|
UNIQUE (obj_id, obj_file_id)
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_photos_obj
|
|
ON domrf_kn_photos (obj_id, period_dt DESC);
|
|
|
|
-- ── Журнал per-request ошибок для ручной проверки в браузере ──────────────
|
|
CREATE TABLE IF NOT EXISTS kn_scrape_failures (
|
|
failure_id BIGSERIAL PRIMARY KEY,
|
|
run_id BIGINT REFERENCES kn_scrape_runs(run_id) ON DELETE CASCADE,
|
|
obj_id BIGINT,
|
|
endpoint TEXT, -- 'sale_graph'|'sales_agg'|'infra'|'photos'
|
|
full_url TEXT, -- полный URL для копирования в браузер
|
|
http_status INT,
|
|
error TEXT,
|
|
body_preview TEXT, -- первые ~500 байт ответа
|
|
occurred_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_kn_fail_run
|
|
ON kn_scrape_failures (run_id, occurred_at DESC);
|
|
|
|
COMMENT ON TABLE domrf_kn_sale_graph IS 'Time-series продаж per-ЖК. /сервисы/api/object/{id}/sale_graph?type=apartments|parking';
|
|
COMMENT ON TABLE domrf_kn_sales_agg IS 'Текущие агрегаты per-ЖК. /сервисы/api/object/{id}/sales_agg';
|
|
COMMENT ON TABLE domrf_kn_infrastructure IS 'POI вокруг ЖК. /сервисы/api/object/{id}/infrastructure';
|
|
COMMENT ON TABLE domrf_kn_photos IS 'Фото прогресса стройки. /сервисы/api/object/construction/progress/photo/{id}';
|
|
COMMENT ON TABLE kn_scrape_failures IS 'Per-request ошибки парсера для ручной проверки full_url в браузере.';
|