6d3fa8cbd2
fix Docker build for playwright - Regenerate uv.lock with playwright + tenacity (was missing → uv sync --frozen skipped them, hence playwright not in /app/.venv/bin/). - Dockerfile: explicit Chromium runtime libs in apt (libnss3 + 14 more) + `playwright install chromium` BEFORE apt-lists cleanup. - Schema 51_schema_kn_extras.sql: 5 new tables for time-series sales, current aggregates, POI infrastructure, photo metadata, failure log. - Scraper: 4 new fetch helpers + upserts; photo binary download via Playwright APIRequest; failure-logging with full_url to kn_scrape_failures. - CLI: --no-extras / --download-photos / --no-flats / --probe URL. - Admin UI: extras checkboxes + failures table with copy-URL button. - PRINZIP smoke: 28 objs, 5409 POI, 5150 photos metadata, sales_agg matches probe (Парк Победы 145/291 = 49%).
63 lines
2 KiB
Python
63 lines
2 KiB
Python
"""Celery task wrapper for naш.дом.рф kn-API sweeps."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import logging
|
|
import random
|
|
import time
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from app.core.config import settings
|
|
from app.services.scrapers.domrf_kn import run_region_sweep
|
|
from app.workers.celery_app import celery_app
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
@celery_app.task(bind=True, name="tasks.scrape_kn.scrape_kn_region", max_retries=2)
|
|
def scrape_kn_region(
|
|
self: Any,
|
|
region_code: int,
|
|
developers: list[str] | None = None,
|
|
skip_jitter: bool = False,
|
|
extras: bool = True,
|
|
download_photos: bool = False,
|
|
fetch_flats: bool = True,
|
|
) -> dict[str, Any]:
|
|
"""Run a full kn-API sweep for one region. Returns summary dict.
|
|
|
|
For scheduled runs (skip_jitter=False), sleeps a random 0..SCRAPE_KN_JITTER_SECONDS
|
|
interval before starting so multiple workers / repeated weekly cycles don't hit
|
|
DOM.РФ at exactly the same minute. Manual triggers pass skip_jitter=True for
|
|
immediate execution.
|
|
"""
|
|
if not skip_jitter and settings.scrape_kn_jitter_seconds > 0:
|
|
delay = random.uniform(0, settings.scrape_kn_jitter_seconds)
|
|
logger.info("jitter delay %.0fs before scrape", delay)
|
|
time.sleep(delay)
|
|
|
|
state_path: str | None = settings.scrape_kn_state_path or None
|
|
if state_path and not Path(state_path).exists():
|
|
logger.warning("storage_state %s not found — bootstrap will run cold", state_path)
|
|
state_path = None
|
|
|
|
logger.info(
|
|
"scrape_kn_region start region=%s devs=%s extras=%s photos=%s state=%s",
|
|
region_code,
|
|
developers,
|
|
extras,
|
|
download_photos,
|
|
state_path,
|
|
)
|
|
return asyncio.run(
|
|
run_region_sweep(
|
|
region_code=region_code,
|
|
developers=developers,
|
|
load_state=state_path,
|
|
fetch_flats=fetch_flats,
|
|
extras=extras,
|
|
download_photos_binary=download_photos,
|
|
)
|
|
)
|