From b5829fa22613d86eec9809b4a971c948ac57ac1a Mon Sep 17 00:00:00 2001
From: lekss361 <lekss361@gendsgn.local>
Date: Tue, 16 Jun 2026 17:08:47 +0000
Subject: [PATCH] feat(scrapers): enable cian_city_sweep (proxy alive, sweep is
 cookieless) (#1568)

---
 .../data/sql/118_enable_cian_city_sweep.sql   | 49 +++++++++++++++++++
 1 file changed, 49 insertions(+)
 create mode 100644 tradein-mvp/backend/data/sql/118_enable_cian_city_sweep.sql

diff --git a/tradein-mvp/backend/data/sql/118_enable_cian_city_sweep.sql b/tradein-mvp/backend/data/sql/118_enable_cian_city_sweep.sql
new file mode 100644
index 00000000..f4710b39
--- /dev/null
+++ b/tradein-mvp/backend/data/sql/118_enable_cian_city_sweep.sql
@@ -0,0 +1,49 @@
+-- 118_enable_cian_city_sweep.sql
+-- Enable the cian_city_sweep schedule (novostroyki + vtorichka listing coverage).
+--
+-- Context:
+--   cian_city_sweep was seeded enabled=false in 107_scrape_schedules_seed_cian_city_sweep.sql
+--   with the note "включается оператором вручную после восстановления прокси" — it was
+--   shipped DORMANT because the cian mobile-proxy egress was assumed dead (last attempt
+--   2026-05-31 aborted on "3 consecutive anchor failures / anchor timeout 240s").
+--
+--   As a result cian listing coverage collapsed: the ~15k active cian rows became a frozen
+--   late-May snapshot (only ~28 of 15067 re-seen within 14 days as of 2026-06-16), and
+--   the 9659 active novostroyki were never refreshed.
+--
+-- Diagnosis (2026-06-16, verified on prod):
+--   - All three mobile-proxy egresses (AVITO/CIAN/YANDEX) are ALIVE — the rotate endpoints
+--     return status=OK with fresh IPs. The "proxy dead" premise is stale.
+--   - cian_city_sweep is COOKIELESS: CianScraper self-warms its curl_cffi (chrome120)
+--     session via a homepage GET; cian_detail.fetch_detail opens its own AsyncSession
+--     (chrome120 + cian_proxy_url). Neither uses the DB cian_session_cookies. Only the
+--     separate cian_history_backfill (source-provided priceChanges backfill) needs those
+--     cookies (which are stale since 2026-05-31 — tracked separately for human re-upload).
+--   - Smoke runs on prod (run 99 SERP-only, run 100 with detail+houses) succeeded:
+--       run 99: lots_fetched=560, lots_inserted=89, lots_updated=471, 0 errors
+--       run 100: detail_enriched=3/3, houses_enriched=4, 0 detail failures
+--     proving the full sweep (SERP → detail → houses) works cookieless via the live proxy.
+--
+-- What this migration does:
+--   Enable cian_city_sweep with production params (pages_per_anchor=3, detail_top_n=10,
+--   enrich_houses=true, request_delay_sec=5, radius_m=1500), nightly window 02:00-05:00 UTC
+--   (same as avito; a separate proxy egress so no contention). next_run_at bootstrapped to
+--   tomorrow 02:30 UTC so a fresh deploy does not fire immediately.
+--
+-- Idempotent: plain UPDATE keyed on source — safe to re-run (sets the same values).
+--
+-- Dependencies:
+--   052_scrape_schedules.sql (table), 107_scrape_schedules_seed_cian_city_sweep.sql (row),
+--   scheduler.py trigger_cian_city_sweep_run (already deployed).
+
+BEGIN;
+
+UPDATE scrape_schedules
+SET enabled            = true,
+    window_start_hour  = 2,
+    window_end_hour    = 5,
+    default_params     = '{"radius_m":1500,"detail_top_n":10,"enrich_houses":true,"pages_per_anchor":3,"request_delay_sec":5}'::jsonb,
+    next_run_at        = ((CURRENT_DATE + INTERVAL '1 day') + INTERVAL '2 hours 30 minutes') AT TIME ZONE 'UTC'
+WHERE source = 'cian_city_sweep';
+
+COMMIT;