79070bfe99
- Discovered real endpoints via chrome-devtools: /сервисы/api/kn/object (offset/limit/place/objStatus) + /portal-kn/api/sales/portal/table. Server-side filter ignored — local filter by developer.companyGroup. - Pure Playwright in-context fetch (httpx blocked by ServicePipe TLS-fp). - Saved fingerprint state in data/playwright_state.json for server reuse. - Celery beat (configurable cron + jitter) + admin trigger endpoint + /admin/scrape UI page. - Schema migration: UNIQUE(id, snapshot_date) for versioned snapshots, kn_scrape_runs journal. - Smoke-tested on PRINZIP: 28 ЖК / 826 квартир as expected.
52 lines
1.7 KiB
Python
52 lines
1.7 KiB
Python
"""Celery task wrapper for naш.дом.рф kn-API sweeps."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import logging
|
|
import random
|
|
import time
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from app.core.config import settings
|
|
from app.services.scrapers.domrf_kn import run_region_sweep
|
|
from app.workers.celery_app import celery_app
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
@celery_app.task(bind=True, name="tasks.scrape_kn.scrape_kn_region", max_retries=2)
|
|
def scrape_kn_region(
|
|
self: Any,
|
|
region_code: int,
|
|
developers: list[str] | None = None,
|
|
skip_jitter: bool = False,
|
|
) -> dict[str, Any]:
|
|
"""Run a full kn-API sweep for one region. Returns summary dict.
|
|
|
|
For scheduled runs (skip_jitter=False), sleeps a random 0..SCRAPE_KN_JITTER_SECONDS
|
|
interval before starting so multiple workers / repeated weekly cycles don't hit
|
|
DOM.РФ at exactly the same minute. Manual triggers pass skip_jitter=True for
|
|
immediate execution.
|
|
"""
|
|
if not skip_jitter and settings.scrape_kn_jitter_seconds > 0:
|
|
delay = random.uniform(0, settings.scrape_kn_jitter_seconds)
|
|
logger.info("jitter delay %.0fs before scrape", delay)
|
|
time.sleep(delay)
|
|
|
|
state_path: str | None = settings.scrape_kn_state_path or None
|
|
if state_path and not Path(state_path).exists():
|
|
logger.warning("storage_state %s not found — bootstrap will run cold", state_path)
|
|
state_path = None
|
|
|
|
logger.info(
|
|
"scrape_kn_region start region=%s devs=%s state=%s", region_code, developers, state_path
|
|
)
|
|
return asyncio.run(
|
|
run_region_sweep(
|
|
region_code=region_code,
|
|
developers=developers,
|
|
load_state=state_path,
|
|
)
|
|
)
|