lekss361
e79c9cec88
Add global request delay floor + per-source delay management. - 054: seed 'global'=0 + per-source rows (avito=7, cian=5, n1=5, domrf=5, rosreestr=5) using ON CONFLICT DO NOTHING; reuses existing 053 table from #484 - scraper_settings.py: get_scraper_delay() = max(per_source, global); _GLOBAL_KEY, refactored _get_setting_cached(); preserved yandex umbrella alias logic - admin.py: GET /scraper-settings + PUT /scraper-settings/{source} (ge=0.0, le=60.0); cache invalidation on update; CAST(:d AS numeric) per psycopg v3 - avito/cian/n1 scrapers wired (yandex was wired by #484) - 18 unit tests pass (10 original updated for max() + 8 new global delay tests) Closes follow-up gap from #486 (Cian admin slider min=0 now accepted by API).
25 lines
936 B
PL/PgSQL
25 lines
936 B
PL/PgSQL
-- 054_scraper_settings_global.sql
|
|
-- Add global delay row + per-source aliases for all scrapers.
|
|
-- Idempotent — ON CONFLICT DO NOTHING preserves user-edited values.
|
|
BEGIN;
|
|
|
|
-- 'global' row: when > 0, acts as floor (max with per-source delay).
|
|
-- Default 0 = use per-source defaults.
|
|
INSERT INTO scraper_settings (source, request_delay_sec, description)
|
|
VALUES (
|
|
'global',
|
|
0.0,
|
|
'Global delay applied across ALL scrapers (max with per-source). 0 = use per-source defaults.'
|
|
)
|
|
ON CONFLICT (source) DO NOTHING;
|
|
|
|
-- Per-source rows for all scrapers (individual control).
|
|
INSERT INTO scraper_settings (source, request_delay_sec, description) VALUES
|
|
('avito', 7.0, 'Avito SERP delay (anti-ban)'),
|
|
('cian', 5.0, 'Cian SERP delay'),
|
|
('n1', 5.0, 'N1 SERP delay'),
|
|
('domrf', 5.0, 'DOM.RF SERP delay'),
|
|
('rosreestr', 5.0, 'Rosreestr API delay')
|
|
ON CONFLICT (source) DO NOTHING;
|
|
|
|
COMMIT;
|