28ddf6ef17
Closes Stage 1 of AvitoScraper_v2 plan. - 009_houses.sql: CREATE TABLE houses (multi-source: avito/cian, geom trigger, 24 cols) - 010_houses_alter.sql: + 15 cols (material_walls, parking_type, rating_distribution, raw_characteristics, ...) - 011_listings_alter.sql: + 24 cols (house linking, kitchen_area_m2, owners_count, owners_at_least, metro_stations, detail_enriched_at, domoteka/Rosreestr badges, 8 partial indexes) - 012_sellers.sql: CREATE TABLE sellers (Avito seller cache, no logos) - 013_listings_alter_seller.sql: listings.seller_id_fk FK - 014_house_reviews.sql: CREATE TABLE house_reviews (textSections main/pros/cons, no images) - 015_scrape_runs.sql: CREATE TABLE scrape_runs (status CHECK enum, heartbeat, zombie partial idx) - 016_listings_snapshots.sql: CREATE TABLE listings_snapshots (append-only, PK listing_id+date) - 017_house_placement_history.sql: CREATE TABLE house_placement_history (removed_date from IMV only) - 018_avito_imv_evaluations.sql: CREATE TABLE + cache_key col + imv_cache_key_idx (TTL 24h lookup) All migrations: BEGIN/COMMIT, IF NOT EXISTS idempotent, PostGIS geom triggers reuse listings_set_geom(). Smoke tested locally: 7 new tables, 34 indexes, 14 listings columns verified. Refs: decisions/Schema_Avito_Houses_Listings_History.md (sec 4.2, 4.3, 4.4, 6, 7, 13, 25, 29) Refs: decisions/AvitoScraper_v2_Implementation_Plan.md Stage 1
65 lines
3.3 KiB
PL/PgSQL
65 lines
3.3 KiB
PL/PgSQL
-- 015_scrape_runs.sql
|
||
-- CREATE TABLE scrape_runs — логирование scrape-сессий с heartbeat + lifecycle
|
||
--
|
||
-- Dependencies: нет (независимая таблица)
|
||
-- Apply after: 014_house_reviews.sql
|
||
-- Apply before: 016_listings_snapshots.sql
|
||
--
|
||
-- Source: decisions/Schema_Avito_Houses_Listings_History.md sec 4.3
|
||
-- Status enum: running / done / failed / zombie / skipped / banned
|
||
--
|
||
-- Паттерн совпадает с objective_scrape_runs из основной GenDesign БД.
|
||
-- Zombie detection: cron каждые 5 мин помечает runs где heartbeat_at < NOW() - 30 min.
|
||
--
|
||
-- Идемпотентно: безопасно запускать повторно.
|
||
|
||
BEGIN;
|
||
|
||
CREATE TABLE IF NOT EXISTS scrape_runs (
|
||
id bigserial PRIMARY KEY,
|
||
source text NOT NULL, -- 'avito' / 'cian' / ...
|
||
run_type text NOT NULL, -- 'search' / 'house_catalog' / 'detail' / 'imv'
|
||
|
||
-- Scope (для search runs — геометрия поиска)
|
||
anchor_lat double precision,
|
||
anchor_lon double precision,
|
||
radius_m int,
|
||
segment text, -- 'vtorichka' / 'novostroyki'
|
||
|
||
-- Scope (для house_catalog/detail runs)
|
||
target_ext_id text, -- house_id или item_id
|
||
|
||
-- Lifecycle
|
||
started_at timestamptz NOT NULL DEFAULT NOW(),
|
||
heartbeat_at timestamptz NOT NULL DEFAULT NOW(), -- обновляется каждые N сек в process loop
|
||
finished_at timestamptz,
|
||
status text NOT NULL DEFAULT 'running'
|
||
CHECK (status IN ('running','done','failed','zombie','skipped','banned')),
|
||
error_text text,
|
||
|
||
-- Counters (для аналитики и debugging)
|
||
total_seen int DEFAULT 0, -- кол-во объявлений в выдаче
|
||
new_count int DEFAULT 0, -- новых (не было в БД)
|
||
returning_count int DEFAULT 0, -- видели → пропали → снова видим
|
||
disappeared_count int DEFAULT 0, -- пропали (last_seen > cutoff после run)
|
||
|
||
-- HTTP stats (для мониторинга rate limits)
|
||
http_requests int DEFAULT 0,
|
||
http_errors int DEFAULT 0
|
||
);
|
||
|
||
CREATE INDEX IF NOT EXISTS scrape_runs_source_started_idx ON scrape_runs (source, started_at DESC);
|
||
-- Partial index для zombie detection (только 'running' статус)
|
||
CREATE INDEX IF NOT EXISTS scrape_runs_zombies_idx ON scrape_runs (status, heartbeat_at)
|
||
WHERE status = 'running';
|
||
|
||
COMMENT ON TABLE scrape_runs IS
|
||
'Лог scrape-сессий. Heartbeat обновляется в process loop. '
|
||
'Zombie: status=running AND heartbeat_at < NOW() - 30 min — помечается cron.';
|
||
COMMENT ON COLUMN scrape_runs.status IS
|
||
'Enum: running=активен, done=успех, failed=ошибка, zombie=heartbeat устарел, '
|
||
'skipped=нет новых данных, banned=Avito вернул 403/captcha.';
|
||
COMMENT ON COLUMN scrape_runs.returning_count IS
|
||
'Объявления которые пропадали (is_active=false) и снова появились в выдаче.';
|
||
|
||
COMMIT;
|