bot-backend
ab4bc229b2
- _walk_price_range open band: depth guard + progressing anchor (lo+_HIGH_ANCHOR) → forward progress, no infinite recursion / tail-loss on >cap open tail - COUNT: raise DomClickBlockedError on missing snippetsCount (schema regression) - _offer_to_lot: "5+" bucket fallback → rooms=5 (was None) - _parse_page: canary when raw>0 and all geo-dropped (ekb=0) - tests: closed-band bisect + real offset-cap (max offset 1980); studio force rooms=0 over present value; real save_listings-error sweep test - 138 migration header filename 132→138
39 lines
1.9 KiB
PL/PgSQL
39 lines
1.9 KiB
PL/PgSQL
-- 138_update_domclick_sweep_params.sql
|
||
-- Обновление параметров DomClick citywide sweep после рефакторинга на JSON API.
|
||
--
|
||
-- Контекст (2026-06-27):
|
||
-- Старый скрейпер (122_enable_domclick_city_sweep.sql) использовал HTML/camoufox
|
||
-- через domclick.ru/search → 0 листингов (URL redesigned + datacenter IP banned
|
||
-- QRATOR). Новая реализация (Layer A) использует:
|
||
-- GET https://bff-search-web.domclick.ru/api/offers/v1
|
||
-- GET https://bff-search-web.domclick.ru/api/offers/count/v1
|
||
-- Транспорт: BrowserFetcher(source="domclick") → tradein-browser pool → generic
|
||
-- provider → shared RU mobile proxy (не заблокировано, PoC verified 2026-06-27).
|
||
--
|
||
-- Изменения default_params:
|
||
-- rooms: добавлен 5 (токен "5+", ранее отсутствовал).
|
||
-- pages_per_anchor: 100 (= OFFSET_CAP/LIMIT = 2000/20, exhaustive within cap).
|
||
-- request_delay_sec: 3 (JSON endpoint легче camoufox-рендера; QRATOR rate-limit
|
||
-- не обнаружен на datacenter IP с shared proxy; conservative polite delay).
|
||
--
|
||
-- !!! DORMANT BY DESIGN !!! enabled остаётся false.
|
||
-- Включать вручную после prod-smoke:
|
||
-- UPDATE scrape_schedules SET enabled = true WHERE source = 'domclick_city_sweep';
|
||
--
|
||
-- Idempotent: UPDATE с WHERE source = 'domclick_city_sweep' — no-op если строка
|
||
-- отсутствует (не должна: засеяна в 122; guard на случай misapply-порядка).
|
||
--
|
||
-- ЗАВИСИМОСТИ: 122_enable_domclick_city_sweep.sql (INSERT строки в scrape_schedules).
|
||
|
||
BEGIN;
|
||
|
||
UPDATE scrape_schedules
|
||
SET default_params = '{
|
||
"city_id": 4,
|
||
"rooms": [0, 1, 2, 3, 4, 5],
|
||
"pages_per_anchor": 100,
|
||
"request_delay_sec": 3
|
||
}'::jsonb
|
||
WHERE source = 'domclick_city_sweep';
|
||
|
||
COMMIT;
|