add domrf extras parser (sale_graph + sales_agg + infra + photos) +
fix Docker build for playwright - Regenerate uv.lock with playwright + tenacity (was missing → uv sync --frozen skipped them, hence playwright not in /app/.venv/bin/). - Dockerfile: explicit Chromium runtime libs in apt (libnss3 + 14 more) + `playwright install chromium` BEFORE apt-lists cleanup. - Schema 51_schema_kn_extras.sql: 5 new tables for time-series sales, current aggregates, POI infrastructure, photo metadata, failure log. - Scraper: 4 new fetch helpers + upserts; photo binary download via Playwright APIRequest; failure-logging with full_url to kn_scrape_failures. - CLI: --no-extras / --download-photos / --no-flats / --probe URL. - Admin UI: extras checkboxes + failures table with copy-URL button. - PRINZIP smoke: 28 objs, 5409 POI, 5150 photos metadata, sales_agg matches probe (Парк Победы 145/291 = 49%).
This commit is contained in:
lekss361
parent
621d56c064
commit
6d3fa8cbd2
8 changed files with 977 additions and 13 deletions
|
|
@ -27,6 +27,8 @@ class TriggerKnRequest(BaseModel):
|
||||||
region_code: int = Field(..., ge=1, le=99)
|
region_code: int = Field(..., ge=1, le=99)
|
||||||
developers: list[str] | None = Field(default=None, max_length=20)
|
developers: list[str] | None = Field(default=None, max_length=20)
|
||||||
fetch_flats: bool = True
|
fetch_flats: bool = True
|
||||||
|
extras: bool = True
|
||||||
|
download_photos: bool = False
|
||||||
|
|
||||||
|
|
||||||
def _check_token(x_admin_token: str | None) -> None:
|
def _check_token(x_admin_token: str | None) -> None:
|
||||||
|
|
@ -50,7 +52,12 @@ def trigger_kn_sweep(
|
||||||
|
|
||||||
result = scrape_kn_region.apply_async(
|
result = scrape_kn_region.apply_async(
|
||||||
args=[payload.region_code, payload.developers],
|
args=[payload.region_code, payload.developers],
|
||||||
kwargs={"skip_jitter": True},
|
kwargs={
|
||||||
|
"skip_jitter": True,
|
||||||
|
"extras": payload.extras,
|
||||||
|
"download_photos": payload.download_photos,
|
||||||
|
"fetch_flats": payload.fetch_flats,
|
||||||
|
},
|
||||||
)
|
)
|
||||||
return {
|
return {
|
||||||
"task_id": result.id,
|
"task_id": result.id,
|
||||||
|
|
@ -60,6 +67,53 @@ def trigger_kn_sweep(
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/failures")
|
||||||
|
def list_failures(
|
||||||
|
db: Annotated[Session, Depends(get_db)],
|
||||||
|
x_admin_token: Annotated[str | None, Header(alias="X-Admin-Token")] = None,
|
||||||
|
run_id: int | None = None,
|
||||||
|
limit: int = 50,
|
||||||
|
) -> list[dict[str, Any]]:
|
||||||
|
"""Per-request failure log for manual browser verification."""
|
||||||
|
_check_token(x_admin_token)
|
||||||
|
where = ""
|
||||||
|
params: dict[str, Any] = {"lim": min(limit, 200)}
|
||||||
|
if run_id is not None:
|
||||||
|
where = "WHERE run_id = :rid"
|
||||||
|
params["rid"] = run_id
|
||||||
|
rows = (
|
||||||
|
db.execute(
|
||||||
|
text(
|
||||||
|
f"""
|
||||||
|
SELECT failure_id, run_id, obj_id, endpoint, full_url,
|
||||||
|
http_status, error, body_preview, occurred_at
|
||||||
|
FROM kn_scrape_failures
|
||||||
|
{where}
|
||||||
|
ORDER BY occurred_at DESC
|
||||||
|
LIMIT :lim
|
||||||
|
"""
|
||||||
|
),
|
||||||
|
params,
|
||||||
|
)
|
||||||
|
.mappings()
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
return [
|
||||||
|
{
|
||||||
|
"failure_id": r["failure_id"],
|
||||||
|
"run_id": r["run_id"],
|
||||||
|
"obj_id": r["obj_id"],
|
||||||
|
"endpoint": r["endpoint"],
|
||||||
|
"full_url": r["full_url"],
|
||||||
|
"http_status": r["http_status"],
|
||||||
|
"error": r["error"],
|
||||||
|
"body_preview": r["body_preview"],
|
||||||
|
"occurred_at": r["occurred_at"].isoformat() if r["occurred_at"] else None,
|
||||||
|
}
|
||||||
|
for r in rows
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
@router.get("/runs")
|
@router.get("/runs")
|
||||||
def list_runs(
|
def list_runs(
|
||||||
db: Annotated[Session, Depends(get_db)],
|
db: Annotated[Session, Depends(get_db)],
|
||||||
|
|
|
||||||
|
|
@ -19,20 +19,28 @@ from __future__ import annotations
|
||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
from datetime import date, datetime
|
from datetime import date, datetime
|
||||||
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from sqlalchemy import text
|
from sqlalchemy import text
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from app.core.db import SessionLocal
|
from app.core.db import SessionLocal
|
||||||
from app.services.scrapers.stealth import BrowserSession
|
from app.services.scrapers.stealth import BASE_URL, BrowserSession
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
PATH_OBJECTS = "/сервисы/api/kn/object"
|
PATH_OBJECTS = "/сервисы/api/kn/object"
|
||||||
PATH_FLATS_TABLE = "/portal-kn/api/sales/portal/table"
|
PATH_FLATS_TABLE = "/portal-kn/api/sales/portal/table"
|
||||||
|
PATH_SALE_GRAPH = "/сервисы/api/object/{obj_id}/sale_graph" # ?type=apartments|parking
|
||||||
|
PATH_SALES_AGG = "/сервисы/api/object/{obj_id}/sales_agg"
|
||||||
|
PATH_INFRA = "/сервисы/api/object/{obj_id}/infrastructure"
|
||||||
|
PATH_PHOTOS = "/сервисы/api/object/construction/progress/photo/{obj_id}"
|
||||||
RAW_SECTION = "kn_api"
|
RAW_SECTION = "kn_api"
|
||||||
|
|
||||||
|
SALE_GRAPH_TYPES = ("apartments", "parking")
|
||||||
|
PHOTOS_DIR_DEFAULT = Path("data/raw/domrf_photos")
|
||||||
|
|
||||||
# Three statuses observed in production (Apr 2026):
|
# Three statuses observed in production (Apr 2026):
|
||||||
# 0 — строящиеся ЖК
|
# 0 — строящиеся ЖК
|
||||||
# 1 — небольшая выборка ~2 объекта на регион (legacy/archive flag)
|
# 1 — небольшая выборка ~2 объекта на регион (legacy/archive flag)
|
||||||
|
|
@ -367,6 +375,367 @@ def upsert_flats(
|
||||||
return inserted
|
return inserted
|
||||||
|
|
||||||
|
|
||||||
|
# ── extras (sale_graph, sales_agg, infrastructure, photos) ──────────────────
|
||||||
|
|
||||||
|
UPSERT_SALE_GRAPH_SQL = text(
|
||||||
|
"""
|
||||||
|
INSERT INTO domrf_kn_sale_graph (
|
||||||
|
obj_id, report_month, type, realised, contracted, area_sq, price_avg, snapshot_date
|
||||||
|
) VALUES (
|
||||||
|
:obj_id, :report_month, :type, :realised, :contracted, :area_sq, :price_avg, :snapshot_date
|
||||||
|
)
|
||||||
|
ON CONFLICT (obj_id, report_month, type, snapshot_date) DO UPDATE SET
|
||||||
|
realised = EXCLUDED.realised,
|
||||||
|
contracted = EXCLUDED.contracted,
|
||||||
|
area_sq = EXCLUDED.area_sq,
|
||||||
|
price_avg = EXCLUDED.price_avg
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
|
||||||
|
UPSERT_SALES_AGG_SQL = text(
|
||||||
|
"""
|
||||||
|
INSERT INTO domrf_kn_sales_agg (
|
||||||
|
obj_id, type, name, total, realised, perc, snapshot_date
|
||||||
|
) VALUES (
|
||||||
|
:obj_id, :type, :name, :total, :realised, :perc, :snapshot_date
|
||||||
|
)
|
||||||
|
ON CONFLICT (obj_id, type, snapshot_date) DO UPDATE SET
|
||||||
|
name = EXCLUDED.name,
|
||||||
|
total = EXCLUDED.total,
|
||||||
|
realised = EXCLUDED.realised,
|
||||||
|
perc = EXCLUDED.perc
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
|
||||||
|
UPSERT_INFRA_SQL = text(
|
||||||
|
"""
|
||||||
|
INSERT INTO domrf_kn_infrastructure (
|
||||||
|
obj_id, poi_name, poi_subtitle, poi_category, poi_address,
|
||||||
|
poi_lat, poi_lon, distance_m, snapshot_date
|
||||||
|
) VALUES (
|
||||||
|
:obj_id, :poi_name, :poi_subtitle, :poi_category, :poi_address,
|
||||||
|
:poi_lat, :poi_lon, :distance_m, :snapshot_date
|
||||||
|
)
|
||||||
|
ON CONFLICT (obj_id, poi_name, poi_lat, poi_lon, snapshot_date) DO UPDATE SET
|
||||||
|
poi_subtitle = EXCLUDED.poi_subtitle,
|
||||||
|
poi_category = EXCLUDED.poi_category,
|
||||||
|
poi_address = EXCLUDED.poi_address,
|
||||||
|
distance_m = EXCLUDED.distance_m
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
|
||||||
|
UPSERT_PHOTO_SQL = text(
|
||||||
|
"""
|
||||||
|
INSERT INTO domrf_kn_photos (
|
||||||
|
obj_id, obj_file_id, ord_num, photo_url, photo_dttm, period_dt,
|
||||||
|
size_bytes, photo_name, ready_desc, build_type, hidden,
|
||||||
|
local_path, downloaded_at
|
||||||
|
) VALUES (
|
||||||
|
:obj_id, :obj_file_id, :ord_num, :photo_url, :photo_dttm, :period_dt,
|
||||||
|
:size_bytes, :photo_name, :ready_desc, :build_type, :hidden,
|
||||||
|
:local_path, :downloaded_at
|
||||||
|
)
|
||||||
|
ON CONFLICT (obj_id, obj_file_id) DO UPDATE SET
|
||||||
|
ord_num = EXCLUDED.ord_num,
|
||||||
|
photo_url = EXCLUDED.photo_url,
|
||||||
|
photo_dttm = EXCLUDED.photo_dttm,
|
||||||
|
period_dt = EXCLUDED.period_dt,
|
||||||
|
size_bytes = EXCLUDED.size_bytes,
|
||||||
|
photo_name = EXCLUDED.photo_name,
|
||||||
|
ready_desc = EXCLUDED.ready_desc,
|
||||||
|
hidden = EXCLUDED.hidden,
|
||||||
|
local_path = COALESCE(EXCLUDED.local_path, domrf_kn_photos.local_path),
|
||||||
|
downloaded_at = COALESCE(EXCLUDED.downloaded_at, domrf_kn_photos.downloaded_at)
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
|
||||||
|
INSERT_FAILURE_SQL = text(
|
||||||
|
"""
|
||||||
|
INSERT INTO kn_scrape_failures (
|
||||||
|
run_id, obj_id, endpoint, full_url, http_status, error, body_preview
|
||||||
|
) VALUES (:run_id, :obj_id, :endpoint, :full_url, :http_status, :error, :body_preview)
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _to_dt(s: Any) -> datetime | None:
|
||||||
|
"""Coerce 'DD-MM-YYYY HH:MM:SS' or 'YYYY-MM-DD ...' to datetime."""
|
||||||
|
if not s or not isinstance(s, str):
|
||||||
|
return None
|
||||||
|
for fmt in ("%d-%m-%Y %H:%M:%S", "%Y-%m-%d %H:%M:%S", "%d-%m-%Y", "%Y-%m-%d"):
|
||||||
|
try:
|
||||||
|
return datetime.strptime(s.strip(), fmt)
|
||||||
|
except ValueError:
|
||||||
|
pass
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _to_month_date(s: Any) -> date | None:
|
||||||
|
"""Coerce '01.04.2025' (sale_graph reportMonthDt) to date."""
|
||||||
|
if not s or not isinstance(s, str):
|
||||||
|
return None
|
||||||
|
for fmt in ("%d.%m.%Y", "%Y-%m-%d"):
|
||||||
|
try:
|
||||||
|
return datetime.strptime(s.strip(), fmt).date()
|
||||||
|
except ValueError:
|
||||||
|
pass
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _http_status_of(e: Exception) -> int | None:
|
||||||
|
"""Try to extract HTTP status from exception message like 'http 404: ...'."""
|
||||||
|
msg = str(e)
|
||||||
|
if msg.startswith("http "):
|
||||||
|
try:
|
||||||
|
return int(msg.split()[1].rstrip(":"))
|
||||||
|
except (IndexError, ValueError):
|
||||||
|
return None
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _body_of(e: Exception) -> str:
|
||||||
|
"""Extract body preview from exception message."""
|
||||||
|
msg = str(e)
|
||||||
|
if ":" in msg:
|
||||||
|
return msg.split(":", 1)[1].strip()[:500]
|
||||||
|
return msg[:500]
|
||||||
|
|
||||||
|
|
||||||
|
def _log_failure(
|
||||||
|
db: Session,
|
||||||
|
run_id: int | None,
|
||||||
|
obj_id: int | None,
|
||||||
|
endpoint: str,
|
||||||
|
full_url: str,
|
||||||
|
http_status: int | None,
|
||||||
|
error: str,
|
||||||
|
body_preview: str = "",
|
||||||
|
) -> None:
|
||||||
|
"""Record a per-request failure into kn_scrape_failures.
|
||||||
|
|
||||||
|
Caller commits later — this just stages the INSERT in the current transaction.
|
||||||
|
Uses a SAVEPOINT so a serialisation issue here doesn't poison the outer txn.
|
||||||
|
"""
|
||||||
|
if run_id is None:
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
db.execute(
|
||||||
|
INSERT_FAILURE_SQL,
|
||||||
|
{
|
||||||
|
"run_id": run_id,
|
||||||
|
"obj_id": obj_id,
|
||||||
|
"endpoint": endpoint[:255] if endpoint else None,
|
||||||
|
"full_url": full_url[:1000] if full_url else None,
|
||||||
|
"http_status": http_status,
|
||||||
|
"error": error[:1000] if error else None,
|
||||||
|
"body_preview": body_preview[:500] if body_preview else None,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("failed to log failure: %s", e)
|
||||||
|
db.rollback()
|
||||||
|
|
||||||
|
|
||||||
|
async def fetch_sale_graph(
|
||||||
|
sess: BrowserSession, obj_id: int, type_: str
|
||||||
|
) -> tuple[list[dict[str, Any]], str]:
|
||||||
|
"""Return (rows, full_url). On HTTP errors raises — caller logs."""
|
||||||
|
path = PATH_SALE_GRAPH.format(obj_id=obj_id)
|
||||||
|
payload = await sess.get_json(path, {"type": type_})
|
||||||
|
full_url = f"{BASE_URL}{path}?type={type_}"
|
||||||
|
rows = (payload.get("data") or {}).get("salesGraphDtos") or []
|
||||||
|
return rows, full_url
|
||||||
|
|
||||||
|
|
||||||
|
async def fetch_sales_agg(sess: BrowserSession, obj_id: int) -> tuple[dict[str, Any], str]:
|
||||||
|
path = PATH_SALES_AGG.format(obj_id=obj_id)
|
||||||
|
payload = await sess.get_json(path, {})
|
||||||
|
return (payload.get("data") or {}), f"{BASE_URL}{path}"
|
||||||
|
|
||||||
|
|
||||||
|
async def fetch_infrastructure(
|
||||||
|
sess: BrowserSession, obj_id: int
|
||||||
|
) -> tuple[list[dict[str, Any]], str]:
|
||||||
|
path = PATH_INFRA.format(obj_id=obj_id)
|
||||||
|
payload = await sess.get_json(path, {})
|
||||||
|
return (payload.get("data") or []), f"{BASE_URL}{path}"
|
||||||
|
|
||||||
|
|
||||||
|
async def fetch_photos(sess: BrowserSession, obj_id: int) -> tuple[list[dict[str, Any]], str]:
|
||||||
|
path = PATH_PHOTOS.format(obj_id=obj_id)
|
||||||
|
# /photo endpoint returns top-level array, not {data: ...}
|
||||||
|
payload = await sess.get_json(path, {})
|
||||||
|
full_url = f"{BASE_URL}{path}"
|
||||||
|
if isinstance(payload, list):
|
||||||
|
return payload, full_url
|
||||||
|
return (payload.get("data") or []), full_url
|
||||||
|
|
||||||
|
|
||||||
|
def upsert_sale_graph(
|
||||||
|
db: Session, obj_id: int, type_: str, rows: list[dict[str, Any]], snapshot_date: date
|
||||||
|
) -> int:
|
||||||
|
n = 0
|
||||||
|
for r in rows:
|
||||||
|
rm = _to_month_date(r.get("reportMonthDt"))
|
||||||
|
if rm is None:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
db.execute(
|
||||||
|
UPSERT_SALE_GRAPH_SQL,
|
||||||
|
{
|
||||||
|
"obj_id": obj_id,
|
||||||
|
"report_month": rm,
|
||||||
|
"type": type_,
|
||||||
|
"realised": r.get("realised"),
|
||||||
|
"contracted": r.get("contracted"),
|
||||||
|
"area_sq": r.get("areaSq"),
|
||||||
|
"price_avg": r.get("priceAvg"),
|
||||||
|
"snapshot_date": snapshot_date,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
n += 1
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("upsert sale_graph obj=%s type=%s month=%s: %s", obj_id, type_, rm, e)
|
||||||
|
db.rollback()
|
||||||
|
return n
|
||||||
|
|
||||||
|
|
||||||
|
def upsert_sales_agg(db: Session, obj_id: int, agg: dict[str, Any], snapshot_date: date) -> int:
|
||||||
|
"""agg is {apartmentsAggData: {...}, nonlivAggData: {...}, parkingAggData: {...}}."""
|
||||||
|
type_map = {
|
||||||
|
"apartmentsAggData": "apartments",
|
||||||
|
"nonlivAggData": "nonliv",
|
||||||
|
"parkingAggData": "parking",
|
||||||
|
}
|
||||||
|
n = 0
|
||||||
|
for key, type_ in type_map.items():
|
||||||
|
d = agg.get(key)
|
||||||
|
if not isinstance(d, dict):
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
db.execute(
|
||||||
|
UPSERT_SALES_AGG_SQL,
|
||||||
|
{
|
||||||
|
"obj_id": obj_id,
|
||||||
|
"type": type_,
|
||||||
|
"name": d.get("name"),
|
||||||
|
"total": d.get("total"),
|
||||||
|
"realised": d.get("realised"),
|
||||||
|
"perc": d.get("perc"),
|
||||||
|
"snapshot_date": snapshot_date,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
n += 1
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("upsert sales_agg obj=%s type=%s: %s", obj_id, type_, e)
|
||||||
|
db.rollback()
|
||||||
|
return n
|
||||||
|
|
||||||
|
|
||||||
|
def upsert_infrastructure(
|
||||||
|
db: Session, obj_id: int, pois: list[dict[str, Any]], snapshot_date: date
|
||||||
|
) -> int:
|
||||||
|
n = 0
|
||||||
|
for poi in pois:
|
||||||
|
try:
|
||||||
|
db.execute(
|
||||||
|
UPSERT_INFRA_SQL,
|
||||||
|
{
|
||||||
|
"obj_id": obj_id,
|
||||||
|
"poi_name": (poi.get("name") or "").strip()[:500] or "—",
|
||||||
|
"poi_subtitle": poi.get("subtitle"),
|
||||||
|
"poi_category": poi.get("tabName"),
|
||||||
|
"poi_address": poi.get("address"),
|
||||||
|
"poi_lat": poi.get("latitude"),
|
||||||
|
"poi_lon": poi.get("longitude"),
|
||||||
|
"distance_m": poi.get("dist"),
|
||||||
|
"snapshot_date": snapshot_date,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
n += 1
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("upsert infra obj=%s poi=%s: %s", obj_id, poi.get("name"), e)
|
||||||
|
db.rollback()
|
||||||
|
return n
|
||||||
|
|
||||||
|
|
||||||
|
def upsert_photos(
|
||||||
|
db: Session,
|
||||||
|
obj_id: int,
|
||||||
|
photos: list[dict[str, Any]],
|
||||||
|
local_paths: dict[str, str] | None = None,
|
||||||
|
) -> int:
|
||||||
|
"""Upsert photo metadata. local_paths maps file_id → relative path on disk."""
|
||||||
|
local_paths = local_paths or {}
|
||||||
|
now_dt = datetime.now()
|
||||||
|
n = 0
|
||||||
|
for p in photos:
|
||||||
|
file_id = p.get("objFileId")
|
||||||
|
if not file_id:
|
||||||
|
continue
|
||||||
|
local = local_paths.get(file_id)
|
||||||
|
try:
|
||||||
|
db.execute(
|
||||||
|
UPSERT_PHOTO_SQL,
|
||||||
|
{
|
||||||
|
"obj_id": obj_id,
|
||||||
|
"obj_file_id": file_id,
|
||||||
|
"ord_num": p.get("ordNum"),
|
||||||
|
"photo_url": p.get("objPhotoUrl"),
|
||||||
|
"photo_dttm": _to_dt(p.get("objPhotoDttm")),
|
||||||
|
"period_dt": _to_month_date(p.get("objPeriodDt")),
|
||||||
|
"size_bytes": p.get("objSize"),
|
||||||
|
"photo_name": p.get("objPhotoNm"),
|
||||||
|
"ready_desc": p.get("objReadyDesc"),
|
||||||
|
"build_type": p.get("objBuildTypeShortDesc"),
|
||||||
|
"hidden": _to_bool(p.get("objPhotoHiddenFlg")),
|
||||||
|
"local_path": local,
|
||||||
|
"downloaded_at": now_dt if local else None,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
n += 1
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("upsert photo obj=%s file=%s: %s", obj_id, file_id, e)
|
||||||
|
db.rollback()
|
||||||
|
return n
|
||||||
|
|
||||||
|
|
||||||
|
async def download_photos(
|
||||||
|
sess: BrowserSession,
|
||||||
|
obj_id: int,
|
||||||
|
photos: list[dict[str, Any]],
|
||||||
|
base_dir: Path,
|
||||||
|
) -> dict[str, str]:
|
||||||
|
"""Download photo binaries to base_dir/{obj_id}/{file_id}.png (or .jpg).
|
||||||
|
Returns mapping {file_id: relative_path}. Skips if file already exists with
|
||||||
|
matching size_bytes.
|
||||||
|
"""
|
||||||
|
out: dict[str, str] = {}
|
||||||
|
obj_dir = base_dir / str(obj_id)
|
||||||
|
obj_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
for p in photos:
|
||||||
|
file_id = p.get("objFileId")
|
||||||
|
url = p.get("objPhotoUrl")
|
||||||
|
if not file_id or not url:
|
||||||
|
continue
|
||||||
|
# Extension hint from objPhotoNm or default png
|
||||||
|
name = (p.get("objPhotoNm") or "").lower()
|
||||||
|
ext = ".jpg" if name.endswith((".jpg", ".jpeg")) else ".png"
|
||||||
|
local = obj_dir / f"{file_id}{ext}"
|
||||||
|
# Skip if size matches
|
||||||
|
expected_size = p.get("objSize")
|
||||||
|
if local.exists() and expected_size and local.stat().st_size == expected_size:
|
||||||
|
out[file_id] = str(local)
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
data = await sess.download_binary(url)
|
||||||
|
local.write_bytes(data)
|
||||||
|
out[file_id] = str(local)
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("download photo obj=%s file=%s failed: %s", obj_id, file_id, e)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
# ── orchestrator ─────────────────────────────────────────────────────────────
|
# ── orchestrator ─────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -410,6 +779,55 @@ async def fetch_flats_for_object(sess: BrowserSession, obj_id: int) -> list[dict
|
||||||
return _flatten_table(payload)
|
return _flatten_table(payload)
|
||||||
|
|
||||||
|
|
||||||
|
async def probe_endpoint(
|
||||||
|
url_or_path: str,
|
||||||
|
region_code: int = 66,
|
||||||
|
headed: bool = False,
|
||||||
|
load_state: str | None = None,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""Quick one-shot probe through a real Playwright session for debugging.
|
||||||
|
|
||||||
|
Returns {status, content_type, body_preview, full_url}. Use to verify that
|
||||||
|
a URL works in our session before/after a failing scraper run.
|
||||||
|
"""
|
||||||
|
if url_or_path.startswith("http"):
|
||||||
|
# split into path+query for BrowserSession.get_json
|
||||||
|
from urllib.parse import parse_qsl, urlsplit
|
||||||
|
|
||||||
|
s = urlsplit(url_or_path)
|
||||||
|
path = s.path
|
||||||
|
params = dict(parse_qsl(s.query, keep_blank_values=True))
|
||||||
|
else:
|
||||||
|
path = url_or_path.split("?", 1)[0]
|
||||||
|
params = {}
|
||||||
|
if "?" in url_or_path:
|
||||||
|
from urllib.parse import parse_qsl
|
||||||
|
|
||||||
|
params = dict(parse_qsl(url_or_path.split("?", 1)[1], keep_blank_values=True))
|
||||||
|
|
||||||
|
async with BrowserSession(
|
||||||
|
region_code=region_code, headed=headed, load_state=load_state
|
||||||
|
) as sess:
|
||||||
|
try:
|
||||||
|
payload = await sess.get_json(path, params)
|
||||||
|
body = json.dumps(payload, ensure_ascii=False)[:2000]
|
||||||
|
qs = "&".join(f"{k}={v}" for k, v in params.items())
|
||||||
|
full_url = f"{BASE_URL}{path}" + (f"?{qs}" if qs else "")
|
||||||
|
return {
|
||||||
|
"status": 200,
|
||||||
|
"content_type": "application/json",
|
||||||
|
"body_preview": body,
|
||||||
|
"full_url": full_url,
|
||||||
|
}
|
||||||
|
except Exception as e:
|
||||||
|
return {
|
||||||
|
"status": _http_status_of(e),
|
||||||
|
"content_type": "error",
|
||||||
|
"body_preview": str(e)[:2000],
|
||||||
|
"full_url": f"{BASE_URL}{path}",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
async def run_region_sweep(
|
async def run_region_sweep(
|
||||||
region_code: int,
|
region_code: int,
|
||||||
developers: list[str] | None = None,
|
developers: list[str] | None = None,
|
||||||
|
|
@ -419,6 +837,9 @@ async def run_region_sweep(
|
||||||
save_state: str | None = None,
|
save_state: str | None = None,
|
||||||
statuses: tuple[int, ...] = DEFAULT_STATUSES,
|
statuses: tuple[int, ...] = DEFAULT_STATUSES,
|
||||||
fetch_flats: bool = True,
|
fetch_flats: bool = True,
|
||||||
|
extras: bool = True,
|
||||||
|
download_photos_binary: bool = False,
|
||||||
|
photos_dir: Path | str | None = None,
|
||||||
place_override: str | None = None,
|
place_override: str | None = None,
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
"""Full sweep for one region: bootstrap → fetch objects (each objStatus) → fetch flats.
|
"""Full sweep for one region: bootstrap → fetch objects (each objStatus) → fetch flats.
|
||||||
|
|
@ -489,6 +910,127 @@ async def run_region_sweep(
|
||||||
)
|
)
|
||||||
logger.info("total flats: %d (across %d objs)", len(all_flats), len(all_objects))
|
logger.info("total flats: %d (across %d objs)", len(all_flats), len(all_objects))
|
||||||
|
|
||||||
|
# --- extras (sale_graph, sales_agg, infra, photos) per object ----
|
||||||
|
extras_counts = {
|
||||||
|
"sale_graph_rows": 0,
|
||||||
|
"sales_agg_rows": 0,
|
||||||
|
"infra_rows": 0,
|
||||||
|
"photos_rows": 0,
|
||||||
|
"photos_downloaded": 0,
|
||||||
|
}
|
||||||
|
if extras:
|
||||||
|
pdir = Path(photos_dir) if photos_dir else PHOTOS_DIR_DEFAULT
|
||||||
|
for i, obj in enumerate(all_objects):
|
||||||
|
obj_id = obj.get("objId") or obj.get("obj_id")
|
||||||
|
if not obj_id:
|
||||||
|
continue
|
||||||
|
|
||||||
|
# sale_graph (apartments + parking)
|
||||||
|
for type_ in SALE_GRAPH_TYPES:
|
||||||
|
try:
|
||||||
|
rows, full_url = await fetch_sale_graph(sess, obj_id, type_)
|
||||||
|
extras_counts["sale_graph_rows"] += upsert_sale_graph(
|
||||||
|
db, obj_id, type_, rows, snapshot_date
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
full_url = (
|
||||||
|
f"{BASE_URL}{PATH_SALE_GRAPH.format(obj_id=obj_id)}?type={type_}"
|
||||||
|
)
|
||||||
|
logger.warning(
|
||||||
|
"sale_graph obj=%s type=%s FAILED: %s\n URL: %s",
|
||||||
|
obj_id,
|
||||||
|
type_,
|
||||||
|
e,
|
||||||
|
full_url,
|
||||||
|
)
|
||||||
|
_log_failure(
|
||||||
|
db,
|
||||||
|
run_id,
|
||||||
|
obj_id,
|
||||||
|
f"sale_graph_{type_}",
|
||||||
|
full_url,
|
||||||
|
_http_status_of(e),
|
||||||
|
str(e),
|
||||||
|
_body_of(e),
|
||||||
|
)
|
||||||
|
|
||||||
|
# sales_agg
|
||||||
|
try:
|
||||||
|
agg, full_url = await fetch_sales_agg(sess, obj_id)
|
||||||
|
extras_counts["sales_agg_rows"] += upsert_sales_agg(
|
||||||
|
db, obj_id, agg, snapshot_date
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
full_url = f"{BASE_URL}{PATH_SALES_AGG.format(obj_id=obj_id)}"
|
||||||
|
logger.warning(
|
||||||
|
"sales_agg obj=%s FAILED: %s\n URL: %s", obj_id, e, full_url
|
||||||
|
)
|
||||||
|
_log_failure(
|
||||||
|
db,
|
||||||
|
run_id,
|
||||||
|
obj_id,
|
||||||
|
"sales_agg",
|
||||||
|
full_url,
|
||||||
|
_http_status_of(e),
|
||||||
|
str(e),
|
||||||
|
_body_of(e),
|
||||||
|
)
|
||||||
|
|
||||||
|
# infrastructure
|
||||||
|
try:
|
||||||
|
pois, full_url = await fetch_infrastructure(sess, obj_id)
|
||||||
|
extras_counts["infra_rows"] += upsert_infrastructure(
|
||||||
|
db, obj_id, pois, snapshot_date
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
full_url = f"{BASE_URL}{PATH_INFRA.format(obj_id=obj_id)}"
|
||||||
|
logger.warning("infra obj=%s FAILED: %s\n URL: %s", obj_id, e, full_url)
|
||||||
|
_log_failure(
|
||||||
|
db,
|
||||||
|
run_id,
|
||||||
|
obj_id,
|
||||||
|
"infrastructure",
|
||||||
|
full_url,
|
||||||
|
_http_status_of(e),
|
||||||
|
str(e),
|
||||||
|
_body_of(e),
|
||||||
|
)
|
||||||
|
|
||||||
|
# photos (metadata + optional binary)
|
||||||
|
try:
|
||||||
|
photos, full_url = await fetch_photos(sess, obj_id)
|
||||||
|
local_paths: dict[str, str] = {}
|
||||||
|
if download_photos_binary and photos:
|
||||||
|
local_paths = await download_photos(sess, obj_id, photos, pdir)
|
||||||
|
extras_counts["photos_downloaded"] += len(local_paths)
|
||||||
|
extras_counts["photos_rows"] += upsert_photos(
|
||||||
|
db, obj_id, photos, local_paths
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
full_url = f"{BASE_URL}{PATH_PHOTOS.format(obj_id=obj_id)}"
|
||||||
|
logger.warning("photos obj=%s FAILED: %s\n URL: %s", obj_id, e, full_url)
|
||||||
|
_log_failure(
|
||||||
|
db,
|
||||||
|
run_id,
|
||||||
|
obj_id,
|
||||||
|
"photos",
|
||||||
|
full_url,
|
||||||
|
_http_status_of(e),
|
||||||
|
str(e),
|
||||||
|
_body_of(e),
|
||||||
|
)
|
||||||
|
|
||||||
|
if (i + 1) % 10 == 0:
|
||||||
|
logger.info(
|
||||||
|
"extras progress: %d/%d objs, %s",
|
||||||
|
i + 1,
|
||||||
|
len(all_objects),
|
||||||
|
extras_counts,
|
||||||
|
)
|
||||||
|
# Periodic commit so failures are visible in admin UI even mid-run
|
||||||
|
db.commit()
|
||||||
|
logger.info("extras done: %s", extras_counts)
|
||||||
|
|
||||||
request_count = sess.request_count
|
request_count = sess.request_count
|
||||||
|
|
||||||
# --- write all in one transaction ------------------------------------
|
# --- write all in one transaction ------------------------------------
|
||||||
|
|
@ -517,6 +1059,7 @@ async def run_region_sweep(
|
||||||
"flats": flat_count,
|
"flats": flat_count,
|
||||||
"requests": request_count,
|
"requests": request_count,
|
||||||
"snapshot_date": snapshot_date.isoformat(),
|
"snapshot_date": snapshot_date.isoformat(),
|
||||||
|
**extras_counts,
|
||||||
}
|
}
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.exception("kn sweep failed for region=%d", region_code)
|
logger.exception("kn sweep failed for region=%d", region_code)
|
||||||
|
|
|
||||||
|
|
@ -180,3 +180,22 @@ class BrowserSession:
|
||||||
@property
|
@property
|
||||||
def request_count(self) -> int:
|
def request_count(self) -> int:
|
||||||
return self._request_count
|
return self._request_count
|
||||||
|
|
||||||
|
async def download_binary(self, url: str) -> bytes:
|
||||||
|
"""Download a binary asset (e.g. photo PNG) reusing browser cookies/auth.
|
||||||
|
|
||||||
|
Uses Playwright APIRequest which goes through the browser context — same
|
||||||
|
cookies, same TLS fingerprint as the page itself.
|
||||||
|
"""
|
||||||
|
if self._context is None:
|
||||||
|
raise RuntimeError("BrowserSession not bootstrapped")
|
||||||
|
await jitter_sleep(200, 500) # Lighter throttle for static assets.
|
||||||
|
self._request_count += 1
|
||||||
|
resp = await self._context.request.get(
|
||||||
|
url,
|
||||||
|
headers={"Authorization": self.auth} if self.auth else {},
|
||||||
|
)
|
||||||
|
if resp.status != 200:
|
||||||
|
body = await resp.text()
|
||||||
|
raise RuntimeError(f"binary http {resp.status}: {body[:200]}")
|
||||||
|
return await resp.body()
|
||||||
|
|
|
||||||
|
|
@ -22,6 +22,9 @@ def scrape_kn_region(
|
||||||
region_code: int,
|
region_code: int,
|
||||||
developers: list[str] | None = None,
|
developers: list[str] | None = None,
|
||||||
skip_jitter: bool = False,
|
skip_jitter: bool = False,
|
||||||
|
extras: bool = True,
|
||||||
|
download_photos: bool = False,
|
||||||
|
fetch_flats: bool = True,
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
"""Run a full kn-API sweep for one region. Returns summary dict.
|
"""Run a full kn-API sweep for one region. Returns summary dict.
|
||||||
|
|
||||||
|
|
@ -41,12 +44,20 @@ def scrape_kn_region(
|
||||||
state_path = None
|
state_path = None
|
||||||
|
|
||||||
logger.info(
|
logger.info(
|
||||||
"scrape_kn_region start region=%s devs=%s state=%s", region_code, developers, state_path
|
"scrape_kn_region start region=%s devs=%s extras=%s photos=%s state=%s",
|
||||||
|
region_code,
|
||||||
|
developers,
|
||||||
|
extras,
|
||||||
|
download_photos,
|
||||||
|
state_path,
|
||||||
)
|
)
|
||||||
return asyncio.run(
|
return asyncio.run(
|
||||||
run_region_sweep(
|
run_region_sweep(
|
||||||
region_code=region_code,
|
region_code=region_code,
|
||||||
developers=developers,
|
developers=developers,
|
||||||
load_state=state_path,
|
load_state=state_path,
|
||||||
|
fetch_flats=fetch_flats,
|
||||||
|
extras=extras,
|
||||||
|
download_photos_binary=download_photos,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -18,7 +18,7 @@ from pathlib import Path
|
||||||
ROOT = Path(__file__).resolve().parents[2]
|
ROOT = Path(__file__).resolve().parents[2]
|
||||||
sys.path.insert(0, str(ROOT / "backend"))
|
sys.path.insert(0, str(ROOT / "backend"))
|
||||||
|
|
||||||
from app.services.scrapers.domrf_kn import run_region_sweep # noqa: E402
|
from app.services.scrapers.domrf_kn import probe_endpoint, run_region_sweep # noqa: E402
|
||||||
|
|
||||||
|
|
||||||
def main() -> None:
|
def main() -> None:
|
||||||
|
|
@ -27,8 +27,7 @@ def main() -> None:
|
||||||
"--regions",
|
"--regions",
|
||||||
type=int,
|
type=int,
|
||||||
nargs="+",
|
nargs="+",
|
||||||
required=True,
|
help="Region codes (1..89). e.g. --regions 66 77. Required unless --probe.",
|
||||||
help="Region codes (1..89). e.g. --regions 66 77",
|
|
||||||
)
|
)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--developers",
|
"--developers",
|
||||||
|
|
@ -36,23 +35,46 @@ def main() -> None:
|
||||||
nargs="*",
|
nargs="*",
|
||||||
help="Optional developer IDs in '<id>_0' format. e.g. --developers 6208_0",
|
help="Optional developer IDs in '<id>_0' format. e.g. --developers 6208_0",
|
||||||
)
|
)
|
||||||
parser.add_argument("--headed", action="store_true", help="Show Chromium window during bootstrap")
|
parser.add_argument(
|
||||||
|
"--headed", action="store_true", help="Show Chromium window during bootstrap"
|
||||||
|
)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--save-state",
|
"--save-state",
|
||||||
type=str,
|
type=str,
|
||||||
default=None,
|
default=None,
|
||||||
help="Path to dump Playwright storage_state.json after bootstrap (commit to git for server reuse)",
|
help="Path to dump Playwright storage_state.json after bootstrap",
|
||||||
)
|
)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--load-state",
|
"--load-state",
|
||||||
type=str,
|
type=str,
|
||||||
default=None,
|
default=None,
|
||||||
help="Path to load Playwright storage_state.json (server uses fingerprint captured locally)",
|
help="Path to load Playwright storage_state.json (server reuse)",
|
||||||
)
|
)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--bootstrap-only",
|
"--bootstrap-only",
|
||||||
action="store_true",
|
action="store_true",
|
||||||
help="Just bootstrap a session and exit — useful for capturing storage_state without scraping",
|
help="Just bootstrap a session and exit (useful for --save-state)",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--no-extras",
|
||||||
|
action="store_true",
|
||||||
|
help="Skip sale_graph/sales_agg/infrastructure/photos endpoints",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--download-photos",
|
||||||
|
action="store_true",
|
||||||
|
help="Also download photo binaries to data/raw/domrf_photos/{obj_id}/",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--no-flats",
|
||||||
|
action="store_true",
|
||||||
|
help="Skip per-flat fetch (only objects + extras)",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--probe",
|
||||||
|
type=str,
|
||||||
|
default=None,
|
||||||
|
help="One-shot probe of a URL or path (e.g. '/сервисы/api/object/60160/sale_graph?type=apartments')",
|
||||||
)
|
)
|
||||||
parser.add_argument("--verbose", "-v", action="store_true")
|
parser.add_argument("--verbose", "-v", action="store_true")
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
@ -66,10 +88,22 @@ def main() -> None:
|
||||||
print("WARNING: DATABASE_URL not set; using default from app.core.config")
|
print("WARNING: DATABASE_URL not set; using default from app.core.config")
|
||||||
|
|
||||||
async def _run() -> None:
|
async def _run() -> None:
|
||||||
|
if args.probe:
|
||||||
|
result = await probe_endpoint(
|
||||||
|
args.probe,
|
||||||
|
region_code=(args.regions or [66])[0],
|
||||||
|
headed=args.headed,
|
||||||
|
load_state=args.load_state,
|
||||||
|
)
|
||||||
|
print(f"PROBE {result['full_url']}")
|
||||||
|
print(f" status: {result['status']} ({result['content_type']})")
|
||||||
|
print(f" body: {result['body_preview']}")
|
||||||
|
return
|
||||||
|
|
||||||
if args.bootstrap_only:
|
if args.bootstrap_only:
|
||||||
from app.services.scrapers.stealth import BrowserSession
|
from app.services.scrapers.stealth import BrowserSession
|
||||||
async with BrowserSession(
|
async with BrowserSession(
|
||||||
region_code=args.regions[0],
|
region_code=(args.regions or [66])[0],
|
||||||
headed=args.headed,
|
headed=args.headed,
|
||||||
load_state=args.load_state,
|
load_state=args.load_state,
|
||||||
save_state=args.save_state,
|
save_state=args.save_state,
|
||||||
|
|
@ -77,6 +111,9 @@ def main() -> None:
|
||||||
print(f"bootstrap-only: state saved to {args.save_state}")
|
print(f"bootstrap-only: state saved to {args.save_state}")
|
||||||
return
|
return
|
||||||
|
|
||||||
|
if not args.regions:
|
||||||
|
parser.error("--regions is required unless --probe / --bootstrap-only")
|
||||||
|
|
||||||
for region in args.regions:
|
for region in args.regions:
|
||||||
summary = await run_region_sweep(
|
summary = await run_region_sweep(
|
||||||
region_code=region,
|
region_code=region,
|
||||||
|
|
@ -84,6 +121,9 @@ def main() -> None:
|
||||||
headed=args.headed,
|
headed=args.headed,
|
||||||
load_state=args.load_state,
|
load_state=args.load_state,
|
||||||
save_state=args.save_state if region == args.regions[0] else None,
|
save_state=args.save_state if region == args.regions[0] else None,
|
||||||
|
fetch_flats=not args.no_flats,
|
||||||
|
extras=not args.no_extras,
|
||||||
|
download_photos_binary=args.download_photos,
|
||||||
)
|
)
|
||||||
print(f"region={region} -> {summary}")
|
print(f"region={region} -> {summary}")
|
||||||
|
|
||||||
|
|
|
||||||
97
data/sql/51_schema_kn_extras.sql
Normal file
97
data/sql/51_schema_kn_extras.sql
Normal file
|
|
@ -0,0 +1,97 @@
|
||||||
|
-- Extras-эндпойнты наш.дом.рф (sale_graph, sales_agg, infrastructure, photos)
|
||||||
|
-- + журнал ошибок per-request для ручной проверки в браузере.
|
||||||
|
-- См. план C:\Users\user\.claude\plans\crispy-waddling-swing.md.
|
||||||
|
|
||||||
|
-- ── Time-series продаж per-ЖК (apartments + parking) ──────────────────────
|
||||||
|
-- Источник: GET /сервисы/api/object/{obj_id}/sale_graph?type=apartments|parking
|
||||||
|
CREATE TABLE IF NOT EXISTS domrf_kn_sale_graph (
|
||||||
|
obj_id BIGINT NOT NULL,
|
||||||
|
report_month DATE NOT NULL, -- '01.04.2025' → 2025-04-01
|
||||||
|
type TEXT NOT NULL, -- 'apartments' | 'parking'
|
||||||
|
realised INT, -- штук реализовано
|
||||||
|
contracted INT, -- штук законтрактовано (ДКП)
|
||||||
|
area_sq NUMERIC, -- м² за месяц
|
||||||
|
price_avg NUMERIC, -- средняя цена ₽/м²
|
||||||
|
snapshot_date DATE NOT NULL DEFAULT CURRENT_DATE,
|
||||||
|
UNIQUE (obj_id, report_month, type, snapshot_date)
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_sg_obj
|
||||||
|
ON domrf_kn_sale_graph (obj_id, snapshot_date);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_sg_month
|
||||||
|
ON domrf_kn_sale_graph (report_month, type);
|
||||||
|
|
||||||
|
-- ── Текущие агрегаты per-ЖК (apartments + nonliv + parking) ───────────────
|
||||||
|
-- Источник: GET /сервисы/api/object/{obj_id}/sales_agg
|
||||||
|
CREATE TABLE IF NOT EXISTS domrf_kn_sales_agg (
|
||||||
|
obj_id BIGINT NOT NULL,
|
||||||
|
type TEXT NOT NULL, -- 'apartments'|'nonliv'|'parking'
|
||||||
|
name TEXT, -- 'Квартира'|'Нежилое помещение'|'Машино-место'
|
||||||
|
total INT, -- всего юнитов в ЖК
|
||||||
|
realised INT, -- реализовано
|
||||||
|
perc NUMERIC, -- sold % (server-calculated)
|
||||||
|
snapshot_date DATE NOT NULL DEFAULT CURRENT_DATE,
|
||||||
|
UNIQUE (obj_id, type, snapshot_date)
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_sa_obj
|
||||||
|
ON domrf_kn_sales_agg (obj_id, snapshot_date);
|
||||||
|
|
||||||
|
-- ── POI вокруг ЖК (нормализованная) ───────────────────────────────────────
|
||||||
|
-- Источник: GET /сервисы/api/object/{obj_id}/infrastructure
|
||||||
|
CREATE TABLE IF NOT EXISTS domrf_kn_infrastructure (
|
||||||
|
obj_id BIGINT NOT NULL,
|
||||||
|
poi_name TEXT NOT NULL,
|
||||||
|
poi_subtitle TEXT,
|
||||||
|
poi_category TEXT, -- tabName: Развлечения|Образование|...
|
||||||
|
poi_address TEXT,
|
||||||
|
poi_lat NUMERIC,
|
||||||
|
poi_lon NUMERIC,
|
||||||
|
distance_m NUMERIC,
|
||||||
|
snapshot_date DATE NOT NULL DEFAULT CURRENT_DATE,
|
||||||
|
UNIQUE (obj_id, poi_name, poi_lat, poi_lon, snapshot_date)
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_infra_obj
|
||||||
|
ON domrf_kn_infrastructure (obj_id, snapshot_date);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_infra_cat_dist
|
||||||
|
ON domrf_kn_infrastructure (poi_category, distance_m);
|
||||||
|
|
||||||
|
-- ── Фото-метаданные (binary в data/raw/domrf_photos/{obj_id}/{file_id}.png) ─
|
||||||
|
-- Источник: GET /сервисы/api/object/construction/progress/photo/{obj_id}
|
||||||
|
CREATE TABLE IF NOT EXISTS domrf_kn_photos (
|
||||||
|
obj_id BIGINT NOT NULL,
|
||||||
|
obj_file_id TEXT NOT NULL, -- DOM.РФ хеш-ID, идемпотентен
|
||||||
|
ord_num INT,
|
||||||
|
photo_url TEXT, -- абсолютный URL DOM.РФ
|
||||||
|
photo_dttm TIMESTAMP,
|
||||||
|
period_dt DATE, -- '2026-04-01'
|
||||||
|
size_bytes BIGINT,
|
||||||
|
photo_name TEXT, -- 'PRINZIP_43_new.jpg'
|
||||||
|
ready_desc TEXT, -- 'Строится'/'Сдан'
|
||||||
|
build_type TEXT, -- 'Жилое'/'Нежилое'
|
||||||
|
hidden BOOLEAN,
|
||||||
|
local_path TEXT, -- если binary скачан
|
||||||
|
downloaded_at TIMESTAMPTZ,
|
||||||
|
UNIQUE (obj_id, obj_file_id)
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_photos_obj
|
||||||
|
ON domrf_kn_photos (obj_id, period_dt DESC);
|
||||||
|
|
||||||
|
-- ── Журнал per-request ошибок для ручной проверки в браузере ──────────────
|
||||||
|
CREATE TABLE IF NOT EXISTS kn_scrape_failures (
|
||||||
|
failure_id BIGSERIAL PRIMARY KEY,
|
||||||
|
run_id BIGINT REFERENCES kn_scrape_runs(run_id) ON DELETE CASCADE,
|
||||||
|
obj_id BIGINT,
|
||||||
|
endpoint TEXT, -- 'sale_graph'|'sales_agg'|'infra'|'photos'
|
||||||
|
full_url TEXT, -- полный URL для копирования в браузер
|
||||||
|
http_status INT,
|
||||||
|
error TEXT,
|
||||||
|
body_preview TEXT, -- первые ~500 байт ответа
|
||||||
|
occurred_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_kn_fail_run
|
||||||
|
ON kn_scrape_failures (run_id, occurred_at DESC);
|
||||||
|
|
||||||
|
COMMENT ON TABLE domrf_kn_sale_graph IS 'Time-series продаж per-ЖК. /сервисы/api/object/{id}/sale_graph?type=apartments|parking';
|
||||||
|
COMMENT ON TABLE domrf_kn_sales_agg IS 'Текущие агрегаты per-ЖК. /сервисы/api/object/{id}/sales_agg';
|
||||||
|
COMMENT ON TABLE domrf_kn_infrastructure IS 'POI вокруг ЖК. /сервисы/api/object/{id}/infrastructure';
|
||||||
|
COMMENT ON TABLE domrf_kn_photos IS 'Фото прогресса стройки. /сервисы/api/object/construction/progress/photo/{id}';
|
||||||
|
COMMENT ON TABLE kn_scrape_failures IS 'Per-request ошибки парсера для ручной проверки full_url в браузере.';
|
||||||
|
|
@ -25,6 +25,18 @@ interface TriggerResp {
|
||||||
developers: string[] | null;
|
developers: string[] | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
interface FailureRow {
|
||||||
|
failure_id: number;
|
||||||
|
run_id: number | null;
|
||||||
|
obj_id: number | null;
|
||||||
|
endpoint: string | null;
|
||||||
|
full_url: string | null;
|
||||||
|
http_status: number | null;
|
||||||
|
error: string | null;
|
||||||
|
body_preview: string | null;
|
||||||
|
occurred_at: string | null;
|
||||||
|
}
|
||||||
|
|
||||||
export default function ScrapeAdminPage() {
|
export default function ScrapeAdminPage() {
|
||||||
const [token, setToken] = useState<string>(() =>
|
const [token, setToken] = useState<string>(() =>
|
||||||
typeof window === "undefined"
|
typeof window === "undefined"
|
||||||
|
|
@ -34,6 +46,10 @@ export default function ScrapeAdminPage() {
|
||||||
const [region, setRegion] = useState(66);
|
const [region, setRegion] = useState(66);
|
||||||
const [devs, setDevs] = useState("");
|
const [devs, setDevs] = useState("");
|
||||||
const [fetchFlats, setFetchFlats] = useState(true);
|
const [fetchFlats, setFetchFlats] = useState(true);
|
||||||
|
const [extras, setExtras] = useState(true);
|
||||||
|
const [downloadPhotos, setDownloadPhotos] = useState(false);
|
||||||
|
const [showFailures, setShowFailures] = useState(false);
|
||||||
|
const [failuresRunId, setFailuresRunId] = useState<number | "">("");
|
||||||
const queryClient = useQueryClient();
|
const queryClient = useQueryClient();
|
||||||
|
|
||||||
const runs = useQuery({
|
const runs = useQuery({
|
||||||
|
|
@ -60,12 +76,35 @@ export default function ScrapeAdminPage() {
|
||||||
.filter(Boolean)
|
.filter(Boolean)
|
||||||
: null,
|
: null,
|
||||||
fetch_flats: fetchFlats,
|
fetch_flats: fetchFlats,
|
||||||
|
extras,
|
||||||
|
download_photos: downloadPhotos,
|
||||||
}),
|
}),
|
||||||
}),
|
}),
|
||||||
onSuccess: () =>
|
onSuccess: () =>
|
||||||
queryClient.invalidateQueries({ queryKey: ["scrape-runs"] }),
|
queryClient.invalidateQueries({ queryKey: ["scrape-runs"] }),
|
||||||
});
|
});
|
||||||
|
|
||||||
|
const failures = useQuery({
|
||||||
|
queryKey: ["scrape-failures", failuresRunId, token],
|
||||||
|
queryFn: () => {
|
||||||
|
const q =
|
||||||
|
failuresRunId !== ""
|
||||||
|
? `?run_id=${failuresRunId}&limit=100`
|
||||||
|
: "?limit=100";
|
||||||
|
return apiFetch<FailureRow[]>(`/api/v1/admin/scrape/failures${q}`, {
|
||||||
|
headers: { "X-Admin-Token": token },
|
||||||
|
});
|
||||||
|
},
|
||||||
|
enabled: !!token && showFailures,
|
||||||
|
refetchInterval: showFailures ? 5000 : false,
|
||||||
|
});
|
||||||
|
|
||||||
|
const copyUrl = (url: string) => {
|
||||||
|
if (typeof navigator !== "undefined" && navigator.clipboard) {
|
||||||
|
navigator.clipboard.writeText(url);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
const saveToken = (v: string) => {
|
const saveToken = (v: string) => {
|
||||||
setToken(v);
|
setToken(v);
|
||||||
localStorage.setItem("admin_token", v);
|
localStorage.setItem("admin_token", v);
|
||||||
|
|
@ -124,7 +163,38 @@ export default function ScrapeAdminPage() {
|
||||||
checked={fetchFlats}
|
checked={fetchFlats}
|
||||||
onChange={(e) => setFetchFlats(e.target.checked)}
|
onChange={(e) => setFetchFlats(e.target.checked)}
|
||||||
/>
|
/>
|
||||||
<span>Также собрать квартиры</span>
|
<span>Квартиры</span>
|
||||||
|
</label>
|
||||||
|
<label
|
||||||
|
style={{
|
||||||
|
display: "flex",
|
||||||
|
alignItems: "center",
|
||||||
|
gap: 6,
|
||||||
|
marginTop: 22,
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<input
|
||||||
|
type="checkbox"
|
||||||
|
checked={extras}
|
||||||
|
onChange={(e) => setExtras(e.target.checked)}
|
||||||
|
/>
|
||||||
|
<span>Extras (sale_graph + agg + infra + photos)</span>
|
||||||
|
</label>
|
||||||
|
<label
|
||||||
|
style={{
|
||||||
|
display: "flex",
|
||||||
|
alignItems: "center",
|
||||||
|
gap: 6,
|
||||||
|
marginTop: 22,
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<input
|
||||||
|
type="checkbox"
|
||||||
|
checked={downloadPhotos}
|
||||||
|
onChange={(e) => setDownloadPhotos(e.target.checked)}
|
||||||
|
disabled={!extras}
|
||||||
|
/>
|
||||||
|
<span>Скачать фото (binary)</span>
|
||||||
</label>
|
</label>
|
||||||
</div>
|
</div>
|
||||||
<div style={{ marginTop: 16 }}>
|
<div style={{ marginTop: 16 }}>
|
||||||
|
|
@ -148,6 +218,127 @@ export default function ScrapeAdminPage() {
|
||||||
</div>
|
</div>
|
||||||
</section>
|
</section>
|
||||||
|
|
||||||
|
<section style={{ ...cardStyle, marginTop: 16 }}>
|
||||||
|
<div
|
||||||
|
style={{
|
||||||
|
display: "flex",
|
||||||
|
alignItems: "center",
|
||||||
|
justifyContent: "space-between",
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<h2 style={{ margin: 0, fontSize: 16 }}>
|
||||||
|
Журнал ошибок (per-request)
|
||||||
|
</h2>
|
||||||
|
<div style={{ display: "flex", gap: 8, alignItems: "center" }}>
|
||||||
|
<input
|
||||||
|
type="number"
|
||||||
|
value={failuresRunId}
|
||||||
|
onChange={(e) =>
|
||||||
|
setFailuresRunId(
|
||||||
|
e.target.value === "" ? "" : Number(e.target.value),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
placeholder="run_id (опц.)"
|
||||||
|
style={{ ...inputStyle, width: 130 }}
|
||||||
|
/>
|
||||||
|
<button
|
||||||
|
onClick={() => setShowFailures((v) => !v)}
|
||||||
|
disabled={!token}
|
||||||
|
style={secondaryBtn}
|
||||||
|
>
|
||||||
|
{showFailures ? "Скрыть" : "Показать"}
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
{showFailures ? (
|
||||||
|
failures.isLoading ? (
|
||||||
|
<p style={{ color: "#5b6066" }}>Загрузка…</p>
|
||||||
|
) : failures.isError ? (
|
||||||
|
<p style={{ color: "#b3261e" }}>Ошибка: {String(failures.error)}</p>
|
||||||
|
) : (failures.data?.length ?? 0) === 0 ? (
|
||||||
|
<p style={{ color: "#5b6066" }}>Ошибок нет.</p>
|
||||||
|
) : (
|
||||||
|
<table
|
||||||
|
style={{
|
||||||
|
width: "100%",
|
||||||
|
borderCollapse: "collapse",
|
||||||
|
fontSize: 12,
|
||||||
|
marginTop: 12,
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<thead>
|
||||||
|
<tr style={{ background: "#fef2f2" }}>
|
||||||
|
{[
|
||||||
|
"Когда",
|
||||||
|
"Run",
|
||||||
|
"obj",
|
||||||
|
"endpoint",
|
||||||
|
"Status",
|
||||||
|
"URL",
|
||||||
|
"Body",
|
||||||
|
].map((h) => (
|
||||||
|
<th key={h} style={th}>
|
||||||
|
{h}
|
||||||
|
</th>
|
||||||
|
))}
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
{(failures.data ?? []).map((f) => (
|
||||||
|
<tr
|
||||||
|
key={f.failure_id}
|
||||||
|
style={{ borderBottom: "1px solid #eef0f3" }}
|
||||||
|
>
|
||||||
|
<td style={td}>
|
||||||
|
{f.occurred_at?.slice(0, 19).replace("T", " ")}
|
||||||
|
</td>
|
||||||
|
<td style={td}>{f.run_id ?? "—"}</td>
|
||||||
|
<td style={td}>{f.obj_id ?? "—"}</td>
|
||||||
|
<td style={td}>{f.endpoint ?? "—"}</td>
|
||||||
|
<td style={{ ...td, color: "#b3261e" }}>
|
||||||
|
{f.http_status ?? "—"}
|
||||||
|
</td>
|
||||||
|
<td
|
||||||
|
style={{ ...td, maxWidth: 380, fontFamily: "monospace" }}
|
||||||
|
>
|
||||||
|
{f.full_url ? (
|
||||||
|
<button
|
||||||
|
onClick={() => copyUrl(f.full_url!)}
|
||||||
|
title="Копировать URL для проверки в браузере"
|
||||||
|
style={{
|
||||||
|
...secondaryBtn,
|
||||||
|
padding: "2px 8px",
|
||||||
|
fontSize: 11,
|
||||||
|
cursor: "copy",
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
📋 {f.full_url.slice(0, 60)}…
|
||||||
|
</button>
|
||||||
|
) : (
|
||||||
|
"—"
|
||||||
|
)}
|
||||||
|
</td>
|
||||||
|
<td
|
||||||
|
style={{
|
||||||
|
...td,
|
||||||
|
fontFamily: "monospace",
|
||||||
|
fontSize: 11,
|
||||||
|
color: "#5b6066",
|
||||||
|
maxWidth: 240,
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
{f.body_preview?.slice(0, 80) ??
|
||||||
|
f.error?.slice(0, 80) ??
|
||||||
|
""}
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
))}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
)
|
||||||
|
) : null}
|
||||||
|
</section>
|
||||||
|
|
||||||
<section style={{ ...cardStyle, marginTop: 16 }}>
|
<section style={{ ...cardStyle, marginTop: 16 }}>
|
||||||
<h2 style={{ marginTop: 0, fontSize: 16 }}>Последние прогоны</h2>
|
<h2 style={{ marginTop: 0, fontSize: 16 }}>Последние прогоны</h2>
|
||||||
{!token ? (
|
{!token ? (
|
||||||
|
|
@ -264,6 +455,15 @@ const primaryBtn = {
|
||||||
fontWeight: 600,
|
fontWeight: 600,
|
||||||
cursor: "pointer",
|
cursor: "pointer",
|
||||||
};
|
};
|
||||||
|
const secondaryBtn = {
|
||||||
|
padding: "8px 14px",
|
||||||
|
background: "#f3f4f6",
|
||||||
|
color: "#1f2937",
|
||||||
|
border: "1px solid #d1d5db",
|
||||||
|
borderRadius: 6,
|
||||||
|
fontSize: 13,
|
||||||
|
cursor: "pointer",
|
||||||
|
};
|
||||||
const th = {
|
const th = {
|
||||||
padding: "8px 10px",
|
padding: "8px 10px",
|
||||||
textAlign: "left" as const,
|
textAlign: "left" as const,
|
||||||
|
|
|
||||||
File diff suppressed because one or more lines are too long
Loading…
Add table
Reference in a new issue