6d3fa8cbd2
fix Docker build for playwright - Regenerate uv.lock with playwright + tenacity (was missing → uv sync --frozen skipped them, hence playwright not in /app/.venv/bin/). - Dockerfile: explicit Chromium runtime libs in apt (libnss3 + 14 more) + `playwright install chromium` BEFORE apt-lists cleanup. - Schema 51_schema_kn_extras.sql: 5 new tables for time-series sales, current aggregates, POI infrastructure, photo metadata, failure log. - Scraper: 4 new fetch helpers + upserts; photo binary download via Playwright APIRequest; failure-logging with full_url to kn_scrape_failures. - CLI: --no-extras / --download-photos / --no-flats / --probe URL. - Admin UI: extras checkboxes + failures table with copy-URL button. - PRINZIP smoke: 28 objs, 5409 POI, 5150 photos metadata, sales_agg matches probe (Парк Победы 145/291 = 49%).
134 lines
4.3 KiB
Python
134 lines
4.3 KiB
Python
"""CLI for ad-hoc naш.дом.рф kn-API sweeps.
|
|
|
|
Usage:
|
|
PYTHONIOENCODING=utf-8 python data/sql/50_scrape_kn_flats.py --regions 66
|
|
python data/sql/50_scrape_kn_flats.py --regions 66 --developers 6208_0 --headed
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import asyncio
|
|
import logging
|
|
import os
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
# allow `from app.services.scrapers.domrf_kn import ...` when run from repo root
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
sys.path.insert(0, str(ROOT / "backend"))
|
|
|
|
from app.services.scrapers.domrf_kn import probe_endpoint, run_region_sweep # noqa: E402
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(description="naш.дом.рф kn-API scraper (CLI)")
|
|
parser.add_argument(
|
|
"--regions",
|
|
type=int,
|
|
nargs="+",
|
|
help="Region codes (1..89). e.g. --regions 66 77. Required unless --probe.",
|
|
)
|
|
parser.add_argument(
|
|
"--developers",
|
|
type=str,
|
|
nargs="*",
|
|
help="Optional developer IDs in '<id>_0' format. e.g. --developers 6208_0",
|
|
)
|
|
parser.add_argument(
|
|
"--headed", action="store_true", help="Show Chromium window during bootstrap"
|
|
)
|
|
parser.add_argument(
|
|
"--save-state",
|
|
type=str,
|
|
default=None,
|
|
help="Path to dump Playwright storage_state.json after bootstrap",
|
|
)
|
|
parser.add_argument(
|
|
"--load-state",
|
|
type=str,
|
|
default=None,
|
|
help="Path to load Playwright storage_state.json (server reuse)",
|
|
)
|
|
parser.add_argument(
|
|
"--bootstrap-only",
|
|
action="store_true",
|
|
help="Just bootstrap a session and exit (useful for --save-state)",
|
|
)
|
|
parser.add_argument(
|
|
"--no-extras",
|
|
action="store_true",
|
|
help="Skip sale_graph/sales_agg/infrastructure/photos endpoints",
|
|
)
|
|
parser.add_argument(
|
|
"--download-photos",
|
|
action="store_true",
|
|
help="Also download photo binaries to data/raw/domrf_photos/{obj_id}/",
|
|
)
|
|
parser.add_argument(
|
|
"--no-flats",
|
|
action="store_true",
|
|
help="Skip per-flat fetch (only objects + extras)",
|
|
)
|
|
parser.add_argument(
|
|
"--probe",
|
|
type=str,
|
|
default=None,
|
|
help="One-shot probe of a URL or path (e.g. '/сервисы/api/object/60160/sale_graph?type=apartments')",
|
|
)
|
|
parser.add_argument("--verbose", "-v", action="store_true")
|
|
args = parser.parse_args()
|
|
|
|
logging.basicConfig(
|
|
level=logging.DEBUG if args.verbose else logging.INFO,
|
|
format="%(asctime)s %(levelname)s %(name)s: %(message)s",
|
|
)
|
|
|
|
if not os.environ.get("DATABASE_URL"):
|
|
print("WARNING: DATABASE_URL not set; using default from app.core.config")
|
|
|
|
async def _run() -> None:
|
|
if args.probe:
|
|
result = await probe_endpoint(
|
|
args.probe,
|
|
region_code=(args.regions or [66])[0],
|
|
headed=args.headed,
|
|
load_state=args.load_state,
|
|
)
|
|
print(f"PROBE {result['full_url']}")
|
|
print(f" status: {result['status']} ({result['content_type']})")
|
|
print(f" body: {result['body_preview']}")
|
|
return
|
|
|
|
if args.bootstrap_only:
|
|
from app.services.scrapers.stealth import BrowserSession
|
|
async with BrowserSession(
|
|
region_code=(args.regions or [66])[0],
|
|
headed=args.headed,
|
|
load_state=args.load_state,
|
|
save_state=args.save_state,
|
|
):
|
|
print(f"bootstrap-only: state saved to {args.save_state}")
|
|
return
|
|
|
|
if not args.regions:
|
|
parser.error("--regions is required unless --probe / --bootstrap-only")
|
|
|
|
for region in args.regions:
|
|
summary = await run_region_sweep(
|
|
region_code=region,
|
|
developers=args.developers or None,
|
|
headed=args.headed,
|
|
load_state=args.load_state,
|
|
save_state=args.save_state if region == args.regions[0] else None,
|
|
fetch_flats=not args.no_flats,
|
|
extras=not args.no_extras,
|
|
download_photos_binary=args.download_photos,
|
|
)
|
|
print(f"region={region} -> {summary}")
|
|
|
|
asyncio.run(_run())
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|