bot-backend
03e43f12b5
- yandex_detail.py: добавить save_detail_enrichment(db, listing_id, enrichment) → bool; UPDATE listings COALESCE по 17 полям (rooms/area_m2/floor/total_floors/address/ description/repair_state/publish_date/views_total_yandex/publish_date_relative/ agency_name/agency_founded_year/agency_objects_count/metro_stations/photo_urls/ newbuilding_url/newbuilding_id); metro/photos → jsonb; detail_enriched_at = NOW() - app/tasks/yandex_detail_backfill.py: новый task run_yandex_detail_backfill; snapshot SELECT source='yandex' AND detail_enriched_at IS NULL; YandexDetailScraper async context manager; consecutive None abort (max_consecutive_blocks=5); budget guard; run lifecycle update_heartbeat/mark_done/mark_failed - scheduler.py: trigger_yandex_detail_backfill_run + elif source=='yandex_detail_backfill' в scheduler_loop; window 12-15 UTC (после avito_detail 09-12 UTC) - migration 113: ADD COLUMN IF NOT EXISTS photo_urls jsonb, newbuilding_url text, newbuilding_id text; seed scrape_schedules yandex_detail_backfill enabled=true - tests: 10 тестов (8 backfill + 2 save_detail_enrichment); 1819 passed 0 failed Closes #1553
49 lines
2.6 KiB
PL/PgSQL
49 lines
2.6 KiB
PL/PgSQL
-- Migration 113: yandex detail backfill — новые колонки + seed расписания.
|
||
--
|
||
-- Добавляет в listings три колонки, которых нет ни в 002_core_tables.sql ни в более ранних
|
||
-- migrations, но которые парсит YandexDetailScraper.DetailEnrichment:
|
||
--
|
||
-- photo_urls jsonb -- список URL фото из Product JSON-LD image[]
|
||
-- newbuilding_url text -- ссылка на ЖК (/kupit/novostrojka/...) если есть
|
||
-- newbuilding_id text -- id ЖК из URL (числовая строка)
|
||
--
|
||
-- Уже существующие колонки (не дублируем):
|
||
-- rooms, area_m2, floor, total_floors, address, description, repair_state,
|
||
-- publish_date, metro_stations, detail_enriched_at -- 002/011_*.sql
|
||
-- views_total_yandex, publish_date_relative, agency_name,
|
||
-- agency_founded_year, agency_objects_count -- 033_listings_alter_yandex.sql
|
||
--
|
||
-- Seed scrape_schedules: source='yandex_detail_backfill', window 12-15 UTC,
|
||
-- разнесено с avito_detail_backfill (09-12 UTC) чтобы не делить трафик.
|
||
-- ON CONFLICT DO NOTHING — идемпотентно.
|
||
|
||
BEGIN;
|
||
|
||
-- ── listings — новые колонки для detail enrichment ────────────────────────────
|
||
|
||
ALTER TABLE listings
|
||
ADD COLUMN IF NOT EXISTS photo_urls jsonb, -- [url, ...] из Product JSON-LD image[]
|
||
ADD COLUMN IF NOT EXISTS newbuilding_url text, -- ссылка на ЖК (/kupit/novostrojka/...)
|
||
ADD COLUMN IF NOT EXISTS newbuilding_id text; -- id ЖК из URL (числовая строка)
|
||
|
||
-- ── scrape_schedules — seed yandex_detail_backfill ────────────────────────────
|
||
|
||
INSERT INTO scrape_schedules
|
||
(source, enabled, window_start_hour, window_end_hour, next_run_at, default_params)
|
||
VALUES (
|
||
'yandex_detail_backfill',
|
||
true,
|
||
12, -- 12:00 UTC = 15:00 МСК
|
||
15, -- 15:00 UTC = 18:00 МСК
|
||
(NOW() + INTERVAL '1 day')::date + TIME '12:00:00' + INTERVAL '0 second',
|
||
'{"batch_size": 800, "budget_sec": 3600, "request_delay_sec": 5, "max_consecutive_blocks": 5}'::jsonb
|
||
)
|
||
ON CONFLICT (source) DO NOTHING;
|
||
|
||
COMMENT ON TABLE scrape_schedules IS
|
||
'Периодические задачи: avito/yandex/n1/cian city sweep, rosreestr import, '
|
||
'listing snapshot, ratio refresh, geocode backfill, deactivate stale, '
|
||
'sber/rosreestr quarter poll, newbuilding enrich, yandex newbuilding sweep, '
|
||
'avito detail backfill, yandex detail backfill.';
|
||
|
||
COMMIT;
|