79070bfe99
- Discovered real endpoints via chrome-devtools: /сервисы/api/kn/object (offset/limit/place/objStatus) + /portal-kn/api/sales/portal/table. Server-side filter ignored — local filter by developer.companyGroup. - Pure Playwright in-context fetch (httpx blocked by ServicePipe TLS-fp). - Saved fingerprint state in data/playwright_state.json for server reuse. - Celery beat (configurable cron + jitter) + admin trigger endpoint + /admin/scrape UI page. - Schema migration: UNIQUE(id, snapshot_date) for versioned snapshots, kn_scrape_runs journal. - Smoke-tested on PRINZIP: 28 ЖК / 826 квартир as expected.
34 lines
1.5 KiB
Python
34 lines
1.5 KiB
Python
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||
|
||
|
||
class Settings(BaseSettings):
|
||
model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8", extra="ignore")
|
||
|
||
database_url: str = "postgresql+psycopg://gendesign:gendesign@localhost:5432/gendesign"
|
||
redis_url: str = "redis://localhost:6379/0"
|
||
cors_origins: list[str] = ["http://localhost:3000"]
|
||
sentry_dsn: str | None = None
|
||
environment: str = "dev"
|
||
|
||
# External APIs (Stage 2)
|
||
rosreestr_pkk_base_url: str = "https://pkk.rosreestr.ru/api/features/1"
|
||
overpass_url: str = "https://overpass-api.de/api/interpreter"
|
||
|
||
# Scraper schedule (наш.дом.рф kn-API).
|
||
# Crontab format: "minute hour day_of_month month day_of_week" (Celery crontab).
|
||
# Default: каждый понедельник в окне 04:00–05:00 по МСК.
|
||
scrape_kn_cron: str = "15 4 * * mon"
|
||
# Random delay window in seconds added on top of scheduled start so the worker
|
||
# does not hit DOM.РФ at exactly the same minute every cycle. 0 = disabled.
|
||
scrape_kn_jitter_seconds: int = 1800
|
||
# Default region(s) for scheduled sweeps. Comma-separated.
|
||
scrape_kn_default_regions: str = "66"
|
||
# Path to a pre-captured Playwright storage_state.json (committed in repo,
|
||
# used by worker to skip cold-start WAF challenge).
|
||
scrape_kn_state_path: str = "data/playwright_state.json"
|
||
# Token to authorize ad-hoc /api/v1/admin/scrape/* trigger calls.
|
||
# Empty string = endpoint disabled.
|
||
scrape_admin_token: str = ""
|
||
|
||
|
||
settings = Settings()
|