feat(tradein): автоматизация cian-backfill + rosreestr-import через scheduler (#560 #563) #630

Merged
Light1YT merged 2 commits from feat/560-563-scrapers-scheduler into main 2026-05-28 15:31:24 +00:00
2 changed files with 15 additions and 0 deletions
Showing only changes of commit edd50a04fd - Show all commits

View file

@ -0,0 +1,7 @@
-- 122_grant_rosreestr_to_fdw_reader.sql
-- #563: tradein scheduler reads gendesign.rosreestr_deals via FDW; tradein_fdw_reader
-- was REVOKE ALL (100_tradein_fdw_role.sql) → grant SELECT on the partitioned parent.
-- Single parent grant suffices (PG declarative partitioning checks parent ACL).
BEGIN;
GRANT SELECT ON public.rosreestr_deals TO tradein_fdw_reader;
COMMIT;

View file

@ -53,11 +53,17 @@ COMMENT ON FOREIGN TABLE gendesign_rosreestr_deals IS
-- ── Seed rows для scrape_schedules ────────────────────────────────────────────
-- next_run_at = следующее наступление night-window (tomorrow + window_start_hour),
-- иначе при NULL get_due_schedules() выстрелит сразу после деплоя (вне окна,
-- cian — на момент деплоя / до того как ляжет FDW-грант миграции 122).
-- window_*_hour документированы в UTC (см. 052) → AT TIME ZONE 'UTC' фиксирует
-- wall-clock час в UTC независимо от session TimeZone GUC.
INSERT INTO scrape_schedules (
source,
enabled,
window_start_hour,
window_end_hour,
next_run_at,
default_params
)
VALUES
@ -66,6 +72,7 @@ VALUES
true,
2,
5,
((CURRENT_DATE + INTERVAL '1 day') + make_interval(hours => 2)) AT TIME ZONE 'UTC',
'{"batch_size": 100}'::jsonb
),
(
@ -73,6 +80,7 @@ VALUES
true,
4,
6,
((CURRENT_DATE + INTERVAL '1 day') + make_interval(hours => 4)) AT TIME ZONE 'UTC',
'{"since": "2024-01-01", "batch_size": 2000}'::jsonb
)
ON CONFLICT (source) DO NOTHING;