5bcaa67a6d
- data/sql/112_22i_domrf_documents.sql: CREATE TABLE domrf_kn_documents with UNIQUE(obj_id, doc_type, doc_num, file_url) + index (obj_id, doc_type). Renamed from original 100_* to 112 to avoid NN collision (100 = mv_layout weighted_avg PR #290 merged; 111 = 22f obj_checks PR #303). - backend/app/services/scrapers/documents.py: extract_documents() + upsert_documents() with SAVEPOINT per-row + canonical doc_type normalisation + download stub. Payload audit (obj_id=65136): kn/object bulk list does NOT contain documents field. Documents live on per-object endpoint /object/{obj_id}/documents (not yet scraped). Follow-up PR needed to wire endpoint into domrf_kn.py scrape loop. Issue #297 22i.
38 lines
1.7 KiB
PL/PgSQL
38 lines
1.7 KiB
PL/PgSQL
-- Migration 100: domrf_kn_documents — PDF documents scraped from DOM.РФ object pages
|
|
-- Issue #297, sub-task 22i. Phase 3 new tables.
|
|
-- Stores declarations, permits, project docs, reports, misc PDFs per obj_id.
|
|
|
|
BEGIN;
|
|
|
|
CREATE TABLE IF NOT EXISTS domrf_kn_documents (
|
|
id bigserial PRIMARY KEY,
|
|
obj_id bigint NOT NULL,
|
|
doc_type text NOT NULL, -- декларация/разрешение/проектная/отчётность/прочее
|
|
doc_num text,
|
|
posted_at date,
|
|
file_url text NOT NULL,
|
|
size_bytes bigint,
|
|
local_path text,
|
|
downloaded_at timestamptz,
|
|
scraped_at timestamptz NOT NULL DEFAULT NOW(),
|
|
UNIQUE (obj_id, doc_type, doc_num, file_url)
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS domrf_kn_documents_obj_type_idx
|
|
ON domrf_kn_documents (obj_id, doc_type);
|
|
|
|
COMMENT ON TABLE domrf_kn_documents IS
|
|
'PDF docs scraped from DOM.RF obj page. Issue #297 22i.';
|
|
|
|
COMMENT ON COLUMN domrf_kn_documents.doc_type IS
|
|
'Canonical type: декларация / разрешение / проектная / отчётность / прочее';
|
|
COMMENT ON COLUMN domrf_kn_documents.doc_num IS
|
|
'Document number as shown on DOM.RF (e.g. permit number, declaration number)';
|
|
COMMENT ON COLUMN domrf_kn_documents.file_url IS
|
|
'Full URL to the PDF on наш.дом.рф (CDN or api/ext/file/...)';
|
|
COMMENT ON COLUMN domrf_kn_documents.local_path IS
|
|
'Relative path inside data/raw/domrf_docs/ after download (populated by future task)';
|
|
COMMENT ON COLUMN domrf_kn_documents.downloaded_at IS
|
|
'Timestamp when PDF was successfully downloaded locally (NULL = not yet downloaded)';
|
|
|
|
COMMIT;
|