28ddf6ef17
Closes Stage 1 of AvitoScraper_v2 plan. - 009_houses.sql: CREATE TABLE houses (multi-source: avito/cian, geom trigger, 24 cols) - 010_houses_alter.sql: + 15 cols (material_walls, parking_type, rating_distribution, raw_characteristics, ...) - 011_listings_alter.sql: + 24 cols (house linking, kitchen_area_m2, owners_count, owners_at_least, metro_stations, detail_enriched_at, domoteka/Rosreestr badges, 8 partial indexes) - 012_sellers.sql: CREATE TABLE sellers (Avito seller cache, no logos) - 013_listings_alter_seller.sql: listings.seller_id_fk FK - 014_house_reviews.sql: CREATE TABLE house_reviews (textSections main/pros/cons, no images) - 015_scrape_runs.sql: CREATE TABLE scrape_runs (status CHECK enum, heartbeat, zombie partial idx) - 016_listings_snapshots.sql: CREATE TABLE listings_snapshots (append-only, PK listing_id+date) - 017_house_placement_history.sql: CREATE TABLE house_placement_history (removed_date from IMV only) - 018_avito_imv_evaluations.sql: CREATE TABLE + cache_key col + imv_cache_key_idx (TTL 24h lookup) All migrations: BEGIN/COMMIT, IF NOT EXISTS idempotent, PostGIS geom triggers reuse listings_set_geom(). Smoke tested locally: 7 new tables, 34 indexes, 14 listings columns verified. Refs: decisions/Schema_Avito_Houses_Listings_History.md (sec 4.2, 4.3, 4.4, 6, 7, 13, 25, 29) Refs: decisions/AvitoScraper_v2_Implementation_Plan.md Stage 1
51 lines
2.6 KiB
PL/PgSQL
51 lines
2.6 KiB
PL/PgSQL
-- 016_listings_snapshots.sql
|
|
-- CREATE TABLE listings_snapshots — append-only daily price/status history
|
|
--
|
|
-- Dependencies:
|
|
-- 002_core_tables.sql (listings table — FK listing_id)
|
|
-- 015_scrape_runs.sql (scrape_runs table — FK run_id)
|
|
--
|
|
-- Apply after: 015_scrape_runs.sql
|
|
-- Apply before: 017_house_placement_history.sql
|
|
--
|
|
-- Source: decisions/Schema_Avito_Houses_Listings_History.md sec 4.4
|
|
-- Pattern: аналог objective_lots_history из основной GenDesign БД.
|
|
--
|
|
-- ON CONFLICT DO UPDATE: берём последний snapshot за день (если за сутки несколько run'ов).
|
|
-- Cleanup policy: хранить вечно (data moat). Partitioning отложить на 1-2 года.
|
|
--
|
|
-- Идемпотентно: безопасно запускать повторно.
|
|
|
|
BEGIN;
|
|
|
|
CREATE TABLE IF NOT EXISTS listings_snapshots (
|
|
listing_id bigint NOT NULL REFERENCES listings(id) ON DELETE CASCADE,
|
|
snapshot_date date NOT NULL DEFAULT CURRENT_DATE,
|
|
run_id bigint REFERENCES scrape_runs(id),
|
|
|
|
-- Snapshot изменяющихся полей
|
|
price_rub bigint NOT NULL,
|
|
price_per_m2 int,
|
|
position_in_serp int, -- 1..60 — позиция в поисковой выдаче
|
|
status text, -- 'active' / 'closed' / null (unknown)
|
|
|
|
observed_at timestamptz NOT NULL DEFAULT NOW(),
|
|
|
|
-- 1 snapshot в день максимум (ON CONFLICT DO UPDATE в scrapers)
|
|
PRIMARY KEY (listing_id, snapshot_date)
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS listings_snapshots_run_idx ON listings_snapshots (run_id);
|
|
CREATE INDEX IF NOT EXISTS listings_snapshots_listing_date_idx ON listings_snapshots (listing_id, snapshot_date DESC);
|
|
-- Для "что нового/изменилось сегодня" запросов
|
|
CREATE INDEX IF NOT EXISTS listings_snapshots_date_idx ON listings_snapshots (snapshot_date DESC);
|
|
|
|
COMMENT ON TABLE listings_snapshots IS
|
|
'Append-only ежедневные снэпшоты listings. PRIMARY KEY (listing_id, snapshot_date) = max 1/день. '
|
|
'Pattern: objective_lots_history. ON CONFLICT DO UPDATE — берём последний за день.';
|
|
COMMENT ON COLUMN listings_snapshots.position_in_serp IS
|
|
'Позиция в поисковой выдаче (1-60). NULL для snapshot''ов из non-search sources.';
|
|
COMMENT ON COLUMN listings_snapshots.status IS
|
|
'''active'' = видели в run, ''closed'' = не нашли (пропало). NULL = неизвестно.';
|
|
|
|
COMMIT;
|