28 lines
1.3 KiB
PL/PgSQL
28 lines
1.3 KiB
PL/PgSQL
-- 053_scraper_settings.sql
|
|
-- Purpose: Per-source live-config for scraper request delays (anti-ban knob).
|
|
-- Loaded by app at runtime; UI can update without restart.
|
|
-- Dependencies: none (standalone settings table)
|
|
-- Sources: User request 2026-05-23 -- Yandex admin page + global delay knob.
|
|
|
|
BEGIN;
|
|
|
|
CREATE TABLE IF NOT EXISTS scraper_settings (
|
|
source text PRIMARY KEY, -- 'yandex' / 'avito' / 'cian' / 'n1' / 'yandex_detail' / ...
|
|
request_delay_sec numeric(5,2) NOT NULL, -- seconds between requests (anti-ban). Range 1-60.
|
|
updated_at timestamptz NOT NULL DEFAULT NOW(),
|
|
-- Hint about who owns this knob (UI displays, no behavior)
|
|
description text
|
|
);
|
|
|
|
COMMENT ON TABLE scraper_settings IS
|
|
'Per-source runtime config for scrapers. Currently exposes request_delay_sec '
|
|
'(anti-ban delay between HTTP requests). Editable via admin UI; backend reads '
|
|
'with 60s in-process cache. Source key matches BaseScraper.name in code.';
|
|
|
|
-- Seed: Yandex umbrella key applies to all 4 Yandex scrapers
|
|
-- (yandex / yandex_detail / yandex_newbuilding / yandex_valuation read this key).
|
|
INSERT INTO scraper_settings (source, request_delay_sec, description)
|
|
VALUES ('yandex', 5.0, 'Global delay applied to all Yandex Realty scrapers (SERP, detail, newbuilding, valuation)')
|
|
ON CONFLICT (source) DO NOTHING;
|
|
|
|
COMMIT;
|