"""CLI for ad-hoc naш.дом.рф kn-API sweeps. Usage: PYTHONIOENCODING=utf-8 python data/sql/50_scrape_kn_flats.py --regions 66 python data/sql/50_scrape_kn_flats.py --regions 66 --developers 6208_0 --headed """ from __future__ import annotations import argparse import asyncio import logging import os import sys from pathlib import Path # allow `from app.services.scrapers.domrf_kn import ...` when run from repo root ROOT = Path(__file__).resolve().parents[2] sys.path.insert(0, str(ROOT / "backend")) from app.services.scrapers.domrf_kn import probe_endpoint, run_region_sweep # noqa: E402 def main() -> None: parser = argparse.ArgumentParser(description="naш.дом.рф kn-API scraper (CLI)") parser.add_argument( "--regions", type=int, nargs="+", help="Region codes (1..89). e.g. --regions 66 77. Required unless --probe.", ) parser.add_argument( "--developers", type=str, nargs="*", help="Optional developer IDs in '_0' format. e.g. --developers 6208_0", ) parser.add_argument( "--headed", action="store_true", help="Show Chromium window during bootstrap" ) parser.add_argument( "--save-state", type=str, default=None, help="Path to dump Playwright storage_state.json after bootstrap", ) parser.add_argument( "--load-state", type=str, default=None, help="Path to load Playwright storage_state.json (server reuse)", ) parser.add_argument( "--bootstrap-only", action="store_true", help="Just bootstrap a session and exit (useful for --save-state)", ) parser.add_argument( "--no-extras", action="store_true", help="Skip sale_graph/sales_agg/infrastructure/photos endpoints", ) parser.add_argument( "--download-photos", action="store_true", help="Also download photo binaries to data/raw/domrf_photos/{obj_id}/", ) parser.add_argument( "--no-flats", action="store_true", help="Skip per-flat fetch (only objects + extras)", ) parser.add_argument( "--probe", type=str, default=None, help="One-shot probe of a URL or path (e.g. '/сервисы/api/object/60160/sale_graph?type=apartments')", ) parser.add_argument("--verbose", "-v", action="store_true") args = parser.parse_args() logging.basicConfig( level=logging.DEBUG if args.verbose else logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s", ) if not os.environ.get("DATABASE_URL"): print("WARNING: DATABASE_URL not set; using default from app.core.config") async def _run() -> None: if args.probe: result = await probe_endpoint( args.probe, region_code=(args.regions or [66])[0], headed=args.headed, load_state=args.load_state, ) print(f"PROBE {result['full_url']}") print(f" status: {result['status']} ({result['content_type']})") print(f" body: {result['body_preview']}") return if args.bootstrap_only: from app.services.scrapers.stealth import BrowserSession async with BrowserSession( region_code=(args.regions or [66])[0], headed=args.headed, load_state=args.load_state, save_state=args.save_state, ): print(f"bootstrap-only: state saved to {args.save_state}") return if not args.regions: parser.error("--regions is required unless --probe / --bootstrap-only") for region in args.regions: summary = await run_region_sweep( region_code=region, developers=args.developers or None, headed=args.headed, load_state=args.load_state, save_state=args.save_state if region == args.regions[0] else None, fetch_flats=not args.no_flats, extras=not args.no_extras, download_photos_binary=args.download_photos, ) print(f"region={region} -> {summary}") asyncio.run(_run()) if __name__ == "__main__": main()