bfe56e44d6
- SQL migrations 053 (scraper_settings table) + 054 (seed global + per-source rows)
- scraper_settings.py: get_scraper_delay() returns max(per_source, global);
in-memory cache TTL=60s; invalidate_cache() for immediate effect on PUT
- avito/cian/n1 scrapers load delay from DB in __init__ (mirrors yandex pattern)
- Admin API: GET /scraper-settings (list all), PUT /scraper-settings/{source}
with cache invalidation on update; CAST(:d AS numeric) per psycopg v3 rules
- 10 unit tests: global>per_source, per_source>global, global=0, DB error fallback,
cache invalidation, API endpoint smoke
19 lines
772 B
PL/PgSQL
19 lines
772 B
PL/PgSQL
-- 053_scraper_settings.sql
|
|
-- Таблица настроек парсеров: задержки между запросами (anti-ban), extensible.
|
|
-- Одна строка на источник (source); UNIQUE(source).
|
|
BEGIN;
|
|
|
|
CREATE TABLE IF NOT EXISTS scraper_settings (
|
|
id SERIAL PRIMARY KEY,
|
|
source TEXT NOT NULL,
|
|
request_delay_sec NUMERIC(6,2) NOT NULL DEFAULT 5.0,
|
|
description TEXT,
|
|
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
|
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
|
CONSTRAINT scraper_settings_source_uq UNIQUE (source)
|
|
);
|
|
|
|
COMMENT ON TABLE scraper_settings IS
|
|
'Per-source scraper delay settings. Special source=''global'' acts as floor across all scrapers.';
|
|
|
|
COMMIT;
|