lekss361/gendesign
1
0
Fork 0
Code Issues 57 Pull requests Projects Releases Packages Wiki Activity Actions

fix(scrapers): disable catalog-object SSR beat-cron — DOM.РФ WAF cooldown (#516)
All checks were successful
Deploy / changes (push) Successful in 5s
Details
Deploy / build-frontend (push) Has been skipped
Details
Deploy / build-backend (push) Successful in 1m22s
Details
Deploy / build-worker (push) Successful in 3m20s
Details
Deploy / deploy (push) Successful in 1m5s
Details

Browse source
This commit is contained in:
lekss361 2026-05-24 12:21:10 +00:00
parent e45ff47aa6
commit ac57b8e905
1 changed files with 12 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
backend/app/workers/beat_schedule.py
View file

@ -240,12 +240,18 @@ def build_beat_schedule() -> dict:
# Catalog-object scrape — наполняет ~25 NULL колонок domrf_kn_objects из SSR-страниц. # Catalog-object scrape — наполняет ~25 NULL колонок domrf_kn_objects из SSR-страниц.
# kn-API не отдаёт wall_type, energy_eff, ceiling_height_m, parking_* и т.д. # kn-API не отдаёт wall_type, energy_eff, ceiling_height_m, parking_* и т.д.
# Вторник 04:00 UTC. batch 300/run → 1532 объекта за ~5 недель полного обновления. # Вторник 04:00 UTC. batch 300/run → 1532 объекта за ~5 недель полного обновления.
schedule["scrape-kn-catalog-objects-weekly"] = { #
"task": "tasks.scrape_kn_catalog_objects.scrape_kn_catalog_objects", # DISABLED 2026-05-24: DOM.РФ WAF дал hard-ban на VPS IP после серии failed
"schedule": _parse_cron("0 4 * * 2"), # Tuesday 04:00 UTC # extras-сессий (run 26/27/28). Catalog SSR использует тот же BrowserSession
"kwargs": {"region_code": 66, "max_objects": 300}, # + те же /сервисы/* paths → следующий beat-tick (вт 26.05 04:00 UTC) насыпет
"options": {"queue": "celery"}, # 300 failed SSR fetches и углубит WAF reputation penalty. Возврат после
} # cooldown 24-48h (проверить через targeted test).
# schedule["scrape-kn-catalog-objects-weekly"] = {
# "task": "tasks.scrape_kn_catalog_objects.scrape_kn_catalog_objects",
# "schedule": _parse_cron("0 4 * * 2"), # Tuesday 04:00 UTC
# "kwargs": {"region_code": 66, "max_objects": 300},
# "options": {"queue": "celery"},
# }
# NSPD quarter dump refresh — re-enabled 2026-05-17 после Sub-PR B (#260) # NSPD quarter dump refresh — re-enabled 2026-05-17 после Sub-PR B (#260)
# переключения search_by_quarter на grid-walk. Foundation (#247) + integration # переключения search_by_quarter на grid-walk. Foundation (#247) + integration