From 3853007e8867230c9af0043d89d61f40dc682c16 Mon Sep 17 00:00:00 2001 From: bot-backend Date: Tue, 16 Jun 2026 20:05:21 +0300 Subject: [PATCH] =?UTF-8?q?feat(scrapers):=20enable=20cian=5Fcity=5Fsweep?= =?UTF-8?q?=20=E2=80=94=20proxy=20alive,=20sweep=20is=20cookieless?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit cian_city_sweep was dormant (107, enabled=false) on a stale 'proxy dead' assumption from 2026-05-31. Verified 2026-06-16: all 3 mobile-proxy egresses alive; CianScraper + cian_detail self-warm their curl_cffi sessions (no DB cookies). Prod smoke proved the full sweep works cookieless: run 99 (SERP): 560 fetched / 89 inserted / 471 updated, 0 errors run 100 (detail): detail_enriched=3/3, houses_enriched=4, 0 failures Only cian_history_backfill (source-provided priceChanges) needs the stale cookies — tracked separately for human re-upload. Restores novostroyki + vtorichka cian coverage (was a frozen late-May snapshot: ~28 of 15067 active re-seen within 14d). Nightly 02-05 UTC. Refs #759 --- .../data/sql/118_enable_cian_city_sweep.sql | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 tradein-mvp/backend/data/sql/118_enable_cian_city_sweep.sql diff --git a/tradein-mvp/backend/data/sql/118_enable_cian_city_sweep.sql b/tradein-mvp/backend/data/sql/118_enable_cian_city_sweep.sql new file mode 100644 index 00000000..f4710b39 --- /dev/null +++ b/tradein-mvp/backend/data/sql/118_enable_cian_city_sweep.sql @@ -0,0 +1,49 @@ +-- 118_enable_cian_city_sweep.sql +-- Enable the cian_city_sweep schedule (novostroyki + vtorichka listing coverage). +-- +-- Context: +-- cian_city_sweep was seeded enabled=false in 107_scrape_schedules_seed_cian_city_sweep.sql +-- with the note "включается оператором вручную после восстановления прокси" — it was +-- shipped DORMANT because the cian mobile-proxy egress was assumed dead (last attempt +-- 2026-05-31 aborted on "3 consecutive anchor failures / anchor timeout 240s"). +-- +-- As a result cian listing coverage collapsed: the ~15k active cian rows became a frozen +-- late-May snapshot (only ~28 of 15067 re-seen within 14 days as of 2026-06-16), and +-- the 9659 active novostroyki were never refreshed. +-- +-- Diagnosis (2026-06-16, verified on prod): +-- - All three mobile-proxy egresses (AVITO/CIAN/YANDEX) are ALIVE — the rotate endpoints +-- return status=OK with fresh IPs. The "proxy dead" premise is stale. +-- - cian_city_sweep is COOKIELESS: CianScraper self-warms its curl_cffi (chrome120) +-- session via a homepage GET; cian_detail.fetch_detail opens its own AsyncSession +-- (chrome120 + cian_proxy_url). Neither uses the DB cian_session_cookies. Only the +-- separate cian_history_backfill (source-provided priceChanges backfill) needs those +-- cookies (which are stale since 2026-05-31 — tracked separately for human re-upload). +-- - Smoke runs on prod (run 99 SERP-only, run 100 with detail+houses) succeeded: +-- run 99: lots_fetched=560, lots_inserted=89, lots_updated=471, 0 errors +-- run 100: detail_enriched=3/3, houses_enriched=4, 0 detail failures +-- proving the full sweep (SERP → detail → houses) works cookieless via the live proxy. +-- +-- What this migration does: +-- Enable cian_city_sweep with production params (pages_per_anchor=3, detail_top_n=10, +-- enrich_houses=true, request_delay_sec=5, radius_m=1500), nightly window 02:00-05:00 UTC +-- (same as avito; a separate proxy egress so no contention). next_run_at bootstrapped to +-- tomorrow 02:30 UTC so a fresh deploy does not fire immediately. +-- +-- Idempotent: plain UPDATE keyed on source — safe to re-run (sets the same values). +-- +-- Dependencies: +-- 052_scrape_schedules.sql (table), 107_scrape_schedules_seed_cian_city_sweep.sql (row), +-- scheduler.py trigger_cian_city_sweep_run (already deployed). + +BEGIN; + +UPDATE scrape_schedules +SET enabled = true, + window_start_hour = 2, + window_end_hour = 5, + default_params = '{"radius_m":1500,"detail_top_n":10,"enrich_houses":true,"pages_per_anchor":3,"request_delay_sec":5}'::jsonb, + next_run_at = ((CURRENT_DATE + INTERVAL '1 day') + INTERVAL '2 hours 30 minutes') AT TIME ZONE 'UTC' +WHERE source = 'cian_city_sweep'; + +COMMIT; -- 2.45.3