bfe56e44d6
- SQL migrations 053 (scraper_settings table) + 054 (seed global + per-source rows)
- scraper_settings.py: get_scraper_delay() returns max(per_source, global);
in-memory cache TTL=60s; invalidate_cache() for immediate effect on PUT
- avito/cian/n1 scrapers load delay from DB in __init__ (mirrors yandex pattern)
- Admin API: GET /scraper-settings (list all), PUT /scraper-settings/{source}
with cache invalidation on update; CAST(:d AS numeric) per psycopg v3 rules
- 10 unit tests: global>per_source, per_source>global, global=0, DB error fallback,
cache invalidation, API endpoint smoke
25 lines
936 B
PL/PgSQL
25 lines
936 B
PL/PgSQL
-- 054_scraper_settings_global.sql
|
|
-- Add global delay row + per-source aliases for all scrapers.
|
|
-- Idempotent — ON CONFLICT DO NOTHING preserves user-edited values.
|
|
BEGIN;
|
|
|
|
-- 'global' row: when > 0, acts as floor (max with per-source delay).
|
|
-- Default 0 = use per-source defaults.
|
|
INSERT INTO scraper_settings (source, request_delay_sec, description)
|
|
VALUES (
|
|
'global',
|
|
0.0,
|
|
'Global delay applied across ALL scrapers (max with per-source). 0 = use per-source defaults.'
|
|
)
|
|
ON CONFLICT (source) DO NOTHING;
|
|
|
|
-- Per-source rows for all scrapers (individual control).
|
|
INSERT INTO scraper_settings (source, request_delay_sec, description) VALUES
|
|
('avito', 7.0, 'Avito SERP delay (anti-ban)'),
|
|
('cian', 5.0, 'Cian SERP delay'),
|
|
('n1', 5.0, 'N1 SERP delay'),
|
|
('domrf', 5.0, 'DOM.RF SERP delay'),
|
|
('rosreestr', 5.0, 'Rosreestr API delay')
|
|
ON CONFLICT (source) DO NOTHING;
|
|
|
|
COMMIT;
|