79070bfe99
- Discovered real endpoints via chrome-devtools: /сервисы/api/kn/object (offset/limit/place/objStatus) + /portal-kn/api/sales/portal/table. Server-side filter ignored — local filter by developer.companyGroup. - Pure Playwright in-context fetch (httpx blocked by ServicePipe TLS-fp). - Saved fingerprint state in data/playwright_state.json for server reuse. - Celery beat (configurable cron + jitter) + admin trigger endpoint + /admin/scrape UI page. - Schema migration: UNIQUE(id, snapshot_date) for versioned snapshots, kn_scrape_runs journal. - Smoke-tested on PRINZIP: 28 ЖК / 826 квартир as expected.
46 lines
1.5 KiB
Python
46 lines
1.5 KiB
Python
"""Celery app + beat schedule."""
|
|
|
|
from celery import Celery
|
|
from celery.schedules import crontab
|
|
|
|
from app.core.config import settings
|
|
|
|
|
|
def _parse_cron(spec: str) -> crontab:
|
|
"""Parse 'M H DoM Mon DoW' crontab string into Celery crontab. Empty fields default to '*'."""
|
|
parts = spec.strip().split()
|
|
if len(parts) != 5:
|
|
raise ValueError(f"crontab spec must have 5 fields, got: {spec!r}")
|
|
minute, hour, dom, month, dow = parts
|
|
return crontab(
|
|
minute=minute,
|
|
hour=hour,
|
|
day_of_month=dom,
|
|
month_of_year=month,
|
|
day_of_week=dow,
|
|
)
|
|
|
|
|
|
def _default_regions() -> list[int]:
|
|
return [int(x.strip()) for x in settings.scrape_kn_default_regions.split(",") if x.strip()]
|
|
|
|
|
|
celery_app = Celery(
|
|
"gendesign",
|
|
broker=settings.redis_url,
|
|
backend=settings.redis_url,
|
|
include=["app.workers.tasks.scrape_kn"],
|
|
)
|
|
celery_app.conf.timezone = "Europe/Moscow"
|
|
|
|
# Расписание задаётся через SCRAPE_KN_CRON env var (см. app.core.config).
|
|
# Каждый региональный sweep оборачивается случайной задержкой 0..SCRAPE_KN_JITTER_SECONDS
|
|
# внутри самой задачи (см. tasks.scrape_kn) — чтобы не бить ровно в одну минуту.
|
|
celery_app.conf.beat_schedule = {
|
|
f"kn-region-{rc}": {
|
|
"task": "tasks.scrape_kn.scrape_kn_region",
|
|
"schedule": _parse_cron(settings.scrape_kn_cron),
|
|
"args": [rc, None],
|
|
}
|
|
for rc in _default_regions()
|
|
}
|