79070bfe99
- Discovered real endpoints via chrome-devtools: /сервисы/api/kn/object (offset/limit/place/objStatus) + /portal-kn/api/sales/portal/table. Server-side filter ignored — local filter by developer.companyGroup. - Pure Playwright in-context fetch (httpx blocked by ServicePipe TLS-fp). - Saved fingerprint state in data/playwright_state.json for server reuse. - Celery beat (configurable cron + jitter) + admin trigger endpoint + /admin/scrape UI page. - Schema migration: UNIQUE(id, snapshot_date) for versioned snapshots, kn_scrape_runs journal. - Smoke-tested on PRINZIP: 28 ЖК / 826 квартир as expected.
94 lines
3 KiB
Python
94 lines
3 KiB
Python
"""CLI for ad-hoc naш.дом.рф kn-API sweeps.
|
|
|
|
Usage:
|
|
PYTHONIOENCODING=utf-8 python data/sql/50_scrape_kn_flats.py --regions 66
|
|
python data/sql/50_scrape_kn_flats.py --regions 66 --developers 6208_0 --headed
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import asyncio
|
|
import logging
|
|
import os
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
# allow `from app.services.scrapers.domrf_kn import ...` when run from repo root
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
sys.path.insert(0, str(ROOT / "backend"))
|
|
|
|
from app.services.scrapers.domrf_kn import run_region_sweep # noqa: E402
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(description="naш.дом.рф kn-API scraper (CLI)")
|
|
parser.add_argument(
|
|
"--regions",
|
|
type=int,
|
|
nargs="+",
|
|
required=True,
|
|
help="Region codes (1..89). e.g. --regions 66 77",
|
|
)
|
|
parser.add_argument(
|
|
"--developers",
|
|
type=str,
|
|
nargs="*",
|
|
help="Optional developer IDs in '<id>_0' format. e.g. --developers 6208_0",
|
|
)
|
|
parser.add_argument("--headed", action="store_true", help="Show Chromium window during bootstrap")
|
|
parser.add_argument(
|
|
"--save-state",
|
|
type=str,
|
|
default=None,
|
|
help="Path to dump Playwright storage_state.json after bootstrap (commit to git for server reuse)",
|
|
)
|
|
parser.add_argument(
|
|
"--load-state",
|
|
type=str,
|
|
default=None,
|
|
help="Path to load Playwright storage_state.json (server uses fingerprint captured locally)",
|
|
)
|
|
parser.add_argument(
|
|
"--bootstrap-only",
|
|
action="store_true",
|
|
help="Just bootstrap a session and exit — useful for capturing storage_state without scraping",
|
|
)
|
|
parser.add_argument("--verbose", "-v", action="store_true")
|
|
args = parser.parse_args()
|
|
|
|
logging.basicConfig(
|
|
level=logging.DEBUG if args.verbose else logging.INFO,
|
|
format="%(asctime)s %(levelname)s %(name)s: %(message)s",
|
|
)
|
|
|
|
if not os.environ.get("DATABASE_URL"):
|
|
print("WARNING: DATABASE_URL not set; using default from app.core.config")
|
|
|
|
async def _run() -> None:
|
|
if args.bootstrap_only:
|
|
from app.services.scrapers.stealth import BrowserSession
|
|
async with BrowserSession(
|
|
region_code=args.regions[0],
|
|
headed=args.headed,
|
|
load_state=args.load_state,
|
|
save_state=args.save_state,
|
|
):
|
|
print(f"bootstrap-only: state saved to {args.save_state}")
|
|
return
|
|
|
|
for region in args.regions:
|
|
summary = await run_region_sweep(
|
|
region_code=region,
|
|
developers=args.developers or None,
|
|
headed=args.headed,
|
|
load_state=args.load_state,
|
|
save_state=args.save_state if region == args.regions[0] else None,
|
|
)
|
|
print(f"region={region} -> {summary}")
|
|
|
|
asyncio.run(_run())
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|