From 4ace87af1443246a64834e463fe09e94b394712c Mon Sep 17 00:00:00 2001 From: lekss361 Date: Fri, 15 May 2026 14:59:41 +0300 Subject: [PATCH] fix(cadastre): exact-match headers to nspd_lite to bypass NSPD WAF (#168) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pilot v2 (job_id=2) failed 50/50 with HTTP 403 WAF block. Comparing nspd_bulk_client.DEFAULT_HEADERS vs legacy nspd_lite.HEADERS (which works on VPS IP since April 2026): PascalCase keys → lowercase keys Chrome/148 UA → Chrome/144 UA No cache-control / pragma → "no-cache" both accept-language ru first → en first No origin → "https://nspd.gov.ru" referer "/map" → "/map?thematic=PKK" NSPD WAF (BotShield-class) likely fingerprints на header order + values combined with TLS fingerprint. Matching legacy exactly minimizes deltas. Test plan: retry pilot job after deploy, expect 0 WAF blocks for first 5 quarters. --- backend/app/scrapers/nspd_bulk_client.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/backend/app/scrapers/nspd_bulk_client.py b/backend/app/scrapers/nspd_bulk_client.py index b51ba3e3..f45c98b9 100644 --- a/backend/app/scrapers/nspd_bulk_client.py +++ b/backend/app/scrapers/nspd_bulk_client.py @@ -48,17 +48,21 @@ _SEMAPHORE = asyncio.Semaphore(3) # Таймаут подключения и чтения DEFAULT_TIMEOUT = httpx.Timeout(30.0, connect=10.0) -# Headers из HAR audit 2026-05-15 +# Headers — exact match to legacy nspd_lite.py (proven to bypass NSPD WAF +# on VPS IP since April 2026). All-lowercase keys + cache-control + Chrome 144 +# UA. Initial HAR-extracted headers (Pascal-Case, Chrome 148, no cache-control) +# were blocked 50/50 в pilot run v2 (job_id=2, all 403 WAF). DEFAULT_HEADERS: dict[str, str] = { - "User-Agent": ( + "accept": "*/*", + "accept-language": "en-US,en;q=0.9,ru-RU;q=0.8,ru;q=0.7,es;q=0.6", + "cache-control": "no-cache", + "pragma": "no-cache", + "referer": "https://nspd.gov.ru/map?thematic=PKK", + "origin": "https://nspd.gov.ru", + "user-agent": ( "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " - "(KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36" + "(KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36" ), - "Referer": "https://nspd.gov.ru/map", - "Accept": "*/*", - "Accept-Language": "ru-RU,ru;q=0.9,en-US;q=0.8,en;q=0.7", - "x-timezone": "Europe/Moscow", - # Origin только для POST endpoints — не включаем для GET (лишний header) } # Retry settings