diff --git a/backend/app/api/v1/parcels.py b/backend/app/api/v1/parcels.py index f8c407d3..f63ed9a2 100644 --- a/backend/app/api/v1/parcels.py +++ b/backend/app/api/v1/parcels.py @@ -22,7 +22,9 @@ from app.schemas.parcel import ( CompetitorsResponse, ConnectionPointsResponse, OpportunityParcel, + ParcelBboxResponse, ParcelDetail, + ParcelMapMarker, ParcelMeta, ParcelSearchRequest, ParcelSearchResponse, @@ -1007,6 +1009,96 @@ _INLINE_FETCH_WAIT_S = 15 _INLINE_FETCH_POLL_INTERVAL_S = 2 +@router.get("/by-bbox", response_model=ParcelBboxResponse) +async def get_parcels_by_bbox( + min_lat: Annotated[float, Query(ge=-90, le=90, description="Южная граница bbox")], + min_lon: Annotated[float, Query(ge=-180, le=180, description="Западная граница bbox")], + max_lat: Annotated[float, Query(ge=-90, le=90, description="Северная граница bbox")], + max_lon: Annotated[float, Query(ge=-180, le=180, description="Восточная граница bbox")], + limit: Annotated[int, Query(ge=1, le=1000)] = 200, + user_id: Annotated[str | None, Query(description="user_id для overlay статуса")] = None, + db: Annotated[Session, Depends(get_db)] = ..., +) -> ParcelBboxResponse: + """GET /parcels/by-bbox — вернуть участки внутри bounding box для карты. + + Использует GIST-индекс на cad_parcels.geom (ST_Intersects). + Если передан user_id — добавляет статус из parcel_user_status (overlay). + last_analysis_date — placeholder до реализации B2 auth (#307). + """ + if min_lat >= max_lat or min_lon >= max_lon: + raise HTTPException(status_code=400, detail="Некорректный bbox: min >= max") + + # Площадь bbox в км² (приблизительно через формулу сферической трапеции) + lat_mid = (min_lat + max_lat) / 2 + lat_km = (max_lat - min_lat) * 111.32 + lon_km = (max_lon - min_lon) * 111.32 * math.cos(math.radians(lat_mid)) + bbox_area_km2 = round(lat_km * lon_km, 4) + + sql = text(""" + WITH bbox AS ( + SELECT ST_MakeEnvelope( + CAST(:min_lon AS float), + CAST(:min_lat AS float), + CAST(:max_lon AS float), + CAST(:max_lat AS float), + 4326 + ) AS env + ) + SELECT + p.cad_num, + ST_Y(ST_Centroid(p.geom)) AS centroid_lat, + ST_X(ST_Centroid(p.geom)) AS centroid_lon, + p.area_m2, + p.land_category, + pus.status AS user_status + FROM cad_parcels p + CROSS JOIN bbox + LEFT JOIN parcel_user_status pus + ON pus.cad_num = p.cad_num + AND pus.user_id = CAST(:user_id AS text) + WHERE p.geom IS NOT NULL + AND ST_Intersects(p.geom, bbox.env) + ORDER BY p.area_m2 DESC NULLS LAST + LIMIT CAST(:limit AS int) + """) + + rows = ( + db.execute( + sql, + { + "min_lat": min_lat, + "min_lon": min_lon, + "max_lat": max_lat, + "max_lon": max_lon, + "user_id": user_id, + "limit": limit, + }, + ) + .mappings() + .all() + ) + + markers: list[ParcelMapMarker] = [ + ParcelMapMarker( + cad_num=row["cad_num"], + centroid_lat=float(row["centroid_lat"]) if row["centroid_lat"] is not None else 0.0, + centroid_lon=float(row["centroid_lon"]) if row["centroid_lon"] is not None else 0.0, + area_m2=float(row["area_m2"]) if row["area_m2"] is not None else None, + land_category=row["land_category"], + status=row["user_status"] if user_id else None, + last_analysis_date=None, # placeholder, real after B2 auth (#307) + ) + for row in rows + ] + + return ParcelBboxResponse( + parcels=markers, + count=len(markers), + limit=limit, + bbox_area_km2=bbox_area_km2, + ) + + @router.post("/search", response_model=ParcelSearchResponse) async def search_parcels(payload: ParcelSearchRequest) -> ParcelSearchResponse: """Search parcels by filters + scoring. diff --git a/backend/app/api/v1/pilot.py b/backend/app/api/v1/pilot.py index 7ff2b4e4..a925b86f 100644 --- a/backend/app/api/v1/pilot.py +++ b/backend/app/api/v1/pilot.py @@ -11,7 +11,7 @@ import logging from typing import Annotated, Any, Literal from fastapi import APIRouter, Depends, Request -from pydantic import BaseModel, EmailStr, Field +from pydantic import BaseModel, Field from sqlalchemy import text from sqlalchemy.orm import Session @@ -25,7 +25,10 @@ router = APIRouter() class PilotRequestInput(BaseModel): name: str = Field(min_length=2, max_length=200) phone: str | None = Field(default=None, max_length=50) - email: EmailStr | None = None + # Note: was EmailStr but `email-validator` package not in deps → ImportError + # on backend startup → container unhealthy → whole deploy fails. Using plain + # str + minimal regex; full validation can happen on frontend/CRM side. + email: str | None = Field(default=None, max_length=200, pattern=r"^[^@\s]+@[^@\s]+\.[^@\s]+$") company: str | None = Field(default=None, max_length=200) message: str | None = Field(default=None, max_length=2000) source: Literal["landing", "analyze_page", "other"] = "landing" diff --git a/backend/app/schemas/parcel.py b/backend/app/schemas/parcel.py index 41103e55..1e6d3c1a 100644 --- a/backend/app/schemas/parcel.py +++ b/backend/app/schemas/parcel.py @@ -356,3 +356,31 @@ class AnalyzeRequest(BaseModel): "Validated против ALLOWED_CATEGORIES + MIN_WEIGHT/MAX_WEIGHT." ), ) + + +# ── SF-B1: by-bbox map entry schemas (#307) ────────────────────────────────── + + +class ParcelMapMarker(BaseModel): + """Один маркер участка на карте — облегчённый ответ для bbox-запроса. + + status берётся из parcel_user_status если передан user_id, иначе null. + last_analysis_date — placeholder до реализации B2 auth. + """ + + cad_num: str + centroid_lat: float + centroid_lon: float + area_m2: float | None + land_category: str | None + status: str | None # 'in_work' | 'favorite' | 'dismissed' | null + last_analysis_date: str | None # placeholder, real after B2 auth + + +class ParcelBboxResponse(BaseModel): + """Ответ GET /parcels/by-bbox.""" + + parcels: list[ParcelMapMarker] + count: int + limit: int + bbox_area_km2: float diff --git a/backend/app/services/scrapers/domrf_catalog_object.py b/backend/app/services/scrapers/domrf_catalog_object.py new file mode 100644 index 00000000..6f3e5912 --- /dev/null +++ b/backend/app/services/scrapers/domrf_catalog_object.py @@ -0,0 +1,459 @@ +"""DOM.РФ catalog-OBJECT scraper (issue #297 sub-task 22d). + +Fills ~25 NULL columns in domrf_kn_objects from public SSR catalog page: + https://наш.дом.рф/сервисы/каталог-новостроек/объект/{obj_id} + +kn-API не возвращает: wall_type, energy_eff, ceiling_height_m, parking_*, +playground_*, finishing_variants_count, etc. — все эти поля есть в +__NEXT_DATA__ JSON блоке на странице каталога (Next.js SSR). + +Uses BrowserSession from app.services.scrapers.stealth (Playwright + WAF bypass). +Fetches HTML, extracts __NEXT_DATA__ JSON, maps to DB columns, +UPDATE domrf_kn_objects WHERE obj_id = :id (не перетирает kn-API данные). +""" + +from __future__ import annotations + +import asyncio +import json +import logging +import re +from datetime import date +from typing import Any + +from sqlalchemy import text +from sqlalchemy.orm import Session + +from app.services.scrapers.stealth import BASE_URL, BrowserSession, WafBlockedError, jitter_sleep + +logger = logging.getLogger(__name__) + +# URL шаблон страницы объекта в каталоге DOM.РФ. +# Человекочитаемый вид: https://наш.дом.рф/сервисы/каталог-новостроек/объект/{obj_id} +CATALOG_OBJECT_PATH = "/сервисы/каталог-новостроек/объект/{obj_id}" + +# JS snippet — аналог _FETCH_HTML_JS из domrf_catalog.py. +# Выполняется внутри живой Playwright-страницы, возвращает HTML текст. +_FETCH_HTML_JS = """ +async ({url}) => { + try { + const r = await fetch(url, {credentials: 'include'}); + const ctype = r.headers.get('content-type') || ''; + const body = await r.text(); + return {ok: r.ok, status: r.status, body, contentType: ctype}; + } catch (e) { + return {ok: false, status: 0, body: String(e), contentType: ''}; + } +} +""" + +# UPDATE SQL — обновляет только catalog-derived поля. +# COALESCE гарантирует что NULL-значение не перетирает уже заполненное поле. +UPDATE_OBJECT_CATALOG_SQL = text( + """ + UPDATE domrf_kn_objects SET + obj_class = COALESCE(:obj_class, obj_class), + wall_type = COALESCE(:wall_type, wall_type), + energy_eff = COALESCE(:energy_eff, energy_eff), + section_count = COALESCE(:section_count, section_count), + parking_total_slots = COALESCE(:parking_total_slots, parking_total_slots), + guest_parking_inside_count = COALESCE( + :guest_parking_inside_count, guest_parking_inside_count + ), + guest_parking_outside_count = COALESCE( + :guest_parking_outside_count, guest_parking_outside_count + ), + ceiling_height_m = COALESCE(:ceiling_height_m, ceiling_height_m), + finishing_variants_count = COALESCE(:finishing_variants_count, finishing_variants_count), + has_free_planning = COALESCE(:has_free_planning, has_free_planning), + avg_flat_area_m2 = COALESCE(:avg_flat_area_m2, avg_flat_area_m2), + elevators_passenger_count = COALESCE( + :elevators_passenger_count, elevators_passenger_count + ), + elevators_cargo_count = COALESCE(:elevators_cargo_count, elevators_cargo_count), + playground_kids_count = COALESCE(:playground_kids_count, playground_kids_count), + playground_sport_count = COALESCE(:playground_sport_count, playground_sport_count), + has_bike_paths = COALESCE(:has_bike_paths, has_bike_paths), + trash_areas_count = COALESCE(:trash_areas_count, trash_areas_count), + has_ramp = COALESCE(:has_ramp, has_ramp), + has_low_platforms = COALESCE(:has_low_platforms, has_low_platforms), + has_wheelchair_lift = COALESCE(:has_wheelchair_lift, has_wheelchair_lift), + first_floor_type = COALESCE(:first_floor_type, first_floor_type), + parking_provision_pct = COALESCE(:parking_provision_pct, parking_provision_pct), + project_published_at = COALESCE(:project_published_at, project_published_at), + project_declaration_num = COALESCE(:project_declaration_num, project_declaration_num), + domrf_score_infrastructure = COALESCE( + :domrf_score_infrastructure, domrf_score_infrastructure + ), + domrf_score_transport = COALESCE(:domrf_score_transport, domrf_score_transport), + catalog_scraped_at = NOW() + WHERE obj_id = :obj_id + AND snapshot_date = :snapshot_date + """ +) + + +# ── Value helpers ───────────────────────────────────────────────────────────── + + +def _to_numeric_comma(s: Any) -> float | None: + """Конвертировать строку с запятой-десятичным разделителем в float. + + Примеры: "2,7" → 2.7; "2.7" → 2.7; "" → None; None → None. + """ + if s is None: + return None + raw = str(s).strip().replace(",", ".") + if not raw: + return None + try: + return float(raw) + except ValueError: + return None + + +def _to_date_ddmmyyyy(s: Any) -> date | None: + """Конвертировать строку "DD.MM.YYYY" в date. + + Примеры: "31.03.2025" → date(2025, 3, 31); "" → None; invalid → None. + """ + if not s: + return None + raw = str(s).strip() + if not raw: + return None + try: + parts = raw.split(".") + if len(parts) == 3: + return date(int(parts[2]), int(parts[1]), int(parts[0])) + except (ValueError, IndexError): + pass + return None + + +def _to_bool_int(v: Any) -> bool | None: + """Конвертировать 0/1 (или любое int-like) в bool. + + Примеры: 1 → True; 0 → False; None → None; 3 → True (>0). + """ + if v is None: + return None + try: + return int(v) > 0 + except (ValueError, TypeError): + return None + + +def _to_bool_da_net(s: Any) -> bool | None: + """Конвертировать "Да"/"Нет" строку в bool. + + Примеры: "Да" → True; "Нет" → False; "" → None; None → None. + """ + if s is None: + return None + raw = str(s).strip().lower() + if raw == "да": + return True + if raw == "нет": + return False + return None + + +def _safe_int(v: Any) -> int | None: + """Безопасная конвертация в int, None при ошибке.""" + if v is None: + return None + try: + return int(v) + except (ValueError, TypeError): + return None + + +# ── HTML fetching ───────────────────────────────────────────────────────────── + + +async def fetch_catalog_object_html(session: BrowserSession, obj_id: int) -> str: + """Получить SSR-HTML страницы объекта в каталоге DOM.РФ. + + Использует тот же паттерн что fetch_catalog_html из domrf_catalog.py: + fetch() внутри живой Playwright-страницы — WAF-fingerprint идентичен браузеру. + + Raises: + WafBlockedError: если вернулся не-HTML (JS-challenge или JSON). + RuntimeError: при 404 или исчерпании попыток. + """ + if session._page is None: + raise RuntimeError("BrowserSession not bootstrapped") + + url = BASE_URL + CATALOG_OBJECT_PATH.format(obj_id=obj_id) + last_err: Exception | None = None + + for attempt in range(5): + async with session._sem: + await jitter_sleep(800, 1500) + try: + session._request_count += 1 + result = await session._page.evaluate(_FETCH_HTML_JS, {"url": url}) + except Exception as exc: + last_err = exc + logger.warning( + "catalog_object html evaluate err attempt=%d obj_id=%d: %r", + attempt, + obj_id, + exc, + ) + await asyncio.sleep(2**attempt) + continue + + status: int = result.get("status", 0) + body: str = result.get("body", "") + ctype: str = result.get("contentType", "") + + if status in (429,) or status >= 500 or status == 0: + last_err = RuntimeError(f"transient status={status}") + logger.warning( + "catalog_object transient status=%d attempt=%d obj_id=%d, backing off", + status, + attempt, + obj_id, + ) + await asyncio.sleep(2**attempt) + continue + + if status == 404: + raise RuntimeError(f"catalog_object 404 for obj_id={obj_id}") + + if status != 200: + raise RuntimeError(f"catalog_object http {status}: {body[:200]} obj_id={obj_id}") + + # Проверяем что вернулся HTML, а не WAF JS-challenge. + is_html = "text/html" in ctype or " dict[str, Any]: + """Извлечь JSON из тега ', + html, + re.DOTALL, + ) + if not match: + raise ValueError("__NEXT_DATA__ script tag not found in HTML") + + raw_json = match.group(1).strip() + try: + return json.loads(raw_json) # type: ignore[no-any-return] + except json.JSONDecodeError as exc: + raise ValueError(f"__NEXT_DATA__ JSON parse error: {exc}") from exc + + +# ── Field mapping ───────────────────────────────────────────────────────────── + + +def parse_catalog_object(next_data: dict[str, Any]) -> dict[str, Any]: + """Извлечь поля объекта из __NEXT_DATA__ и вернуть dict для UPDATE. + + Все .get() безопасны — partial responses OK, отсутствующие поля = None. + Возвращает dict с bind-параметрами для UPDATE_OBJECT_CATALOG_SQL. + """ + pp: dict[str, Any] = next_data.get("props", {}).get("pageProps", {}) + ai: dict[str, Any] = pp.get("additionalInfo") or {} + quart: dict[str, Any] = pp.get("quartography") or {} + indexes: dict[str, Any] = pp.get("indexes") or {} + decl: dict[str, Any] = pp.get("projectDeclaration") or {} + + # first_floor_type: 1 = нежилой, 0 = жилой + first_floor_raw = quart.get("nonLivFirstFloor") + first_floor_type: str | None = None + if first_floor_raw is not None: + try: + first_floor_type = "нежилой" if int(first_floor_raw) == 1 else "жилой" + except (ValueError, TypeError): + pass + + # elevators_cargo_count = cargoElevatorsCount + cargoPassengerElevatorCount + cargo = _safe_int(ai.get("cargoElevatorsCount")) + cargo_pass = _safe_int(ai.get("cargoPassengerElevatorCount")) + if cargo is not None or cargo_pass is not None: + elevators_cargo_count: int | None = (cargo or 0) + (cargo_pass or 0) + else: + elevators_cargo_count = None + + return { + "obj_class": pp.get("buildingClass"), + "wall_type": pp.get("wallMaterial"), + "energy_eff": pp.get("objEnergyEfficiency"), + "section_count": _safe_int(quart.get("objLivElemEntrCnt")), + "parking_total_slots": _safe_int(pp.get("parkingCount")), + "guest_parking_inside_count": _safe_int(ai.get("objectParkingPlaces")), + "guest_parking_outside_count": _safe_int(ai.get("nearbyParkingPlaces")), + "ceiling_height_m": _to_numeric_comma(ai.get("ceilingHeight")), + "finishing_variants_count": _safe_int(pp.get("finishTypeCount")), + "has_free_planning": _to_bool_da_net(pp.get("freePlan")), + "avg_flat_area_m2": _to_numeric_comma(quart.get("objLivElemSqAvg")), + "elevators_passenger_count": _safe_int(ai.get("passengerElevatorsCount")), + "elevators_cargo_count": elevators_cargo_count, + "playground_kids_count": _safe_int(ai.get("playgroundsCount")), + "playground_sport_count": _safe_int(ai.get("sportsgroundCount")), + "has_bike_paths": _to_bool_int(ai.get("bicycleLane")), + "trash_areas_count": _safe_int(ai.get("trashAreaCount")), + "has_ramp": _to_bool_int(ai.get("ramp")), + "has_low_platforms": _to_bool_int(ai.get("curbLowering")), + "has_wheelchair_lift": _to_bool_int(ai.get("wheelchairElevatorsCount")), + "first_floor_type": first_floor_type, + "parking_provision_pct": _to_numeric_comma(ai.get("parkingAvailabilityPerc")), + "project_published_at": _to_date_ddmmyyyy(pp.get("publicationDate")), + "project_declaration_num": decl.get("number"), + "domrf_score_infrastructure": _safe_int(indexes.get("infrastructure")), + "domrf_score_transport": _safe_int(indexes.get("transport")), + # TODO: obj_checks (6 detailed checks) — separate investigation (task #21). + # pageProps.isChecked (bool), verificationId, verificationFlg available here + # but detailed per-check breakdown requires separate API investigation. + } + + +# ── DB write ────────────────────────────────────────────────────────────────── + + +async def scrape_catalog_object( + db: Session, + session: BrowserSession, + obj_id: int, + snapshot_date: date, +) -> bool: + """Scrape одного объекта: fetch HTML → extract __NEXT_DATA__ → parse → UPDATE. + + Использует SAVEPOINT (begin_nested) для изоляции per-row ошибок. + Логирует результат через logger.info. + + Returns: + True если UPDATE затронул строку, False при ошибке или 0 rows. + """ + logger.info("catalog_object scrape start obj_id=%d snapshot_date=%s", obj_id, snapshot_date) + + try: + html = await fetch_catalog_object_html(session, obj_id) + except WafBlockedError as exc: + logger.warning("catalog_object WAF blocked obj_id=%d: %s", obj_id, exc) + return False + except Exception as exc: + logger.warning("catalog_object fetch failed obj_id=%d: %s", obj_id, exc) + return False + + try: + next_data = extract_next_data(html) + except ValueError as exc: + logger.warning("catalog_object extract_next_data failed obj_id=%d: %s", obj_id, exc) + return False + + try: + data = parse_catalog_object(next_data) + except Exception as exc: + logger.warning("catalog_object parse failed obj_id=%d: %s", obj_id, exc) + return False + + fields_extracted = len([v for v in data.values() if v is not None]) + + params: dict[str, Any] = { + "obj_id": obj_id, + "snapshot_date": snapshot_date, + **data, + } + + try: + with db.begin_nested(): + result = db.execute(UPDATE_OBJECT_CATALOG_SQL, params) + rows_affected: int = result.rowcount or 0 + except Exception as exc: + logger.warning("catalog_object UPDATE failed obj_id=%d: %s", obj_id, exc) + return False + + if rows_affected == 0: + logger.warning( + "catalog_object UPDATE 0 rows obj_id=%d snapshot_date=%s — not in DB?", + obj_id, + snapshot_date, + ) + return False + + logger.info( + "catalog_object scraped obj_id=%d fields=%d rows_updated=%d", + obj_id, + fields_extracted, + rows_affected, + ) + return True + + +async def scrape_catalog_objects( + db: Session, + obj_ids: list[int], + snapshot_date: date, + region_code: int = 66, +) -> dict[str, int]: + """Scrape списка объектов через один BrowserSession. + + Запускает один BrowserSession на весь batch; jitter_sleep (800–1500 мс) + встроен в fetch_catalog_object_html для защиты от rate-limit. + + Returns: + {"processed": N, "succeeded": N, "failed": N, "skipped": N} + """ + stats: dict[str, int] = { + "processed": 0, + "succeeded": 0, + "failed": 0, + "skipped": 0, + } + + if not obj_ids: + logger.info("scrape_catalog_objects: empty list, nothing to do") + return stats + + logger.info( + "scrape_catalog_objects: starting %d objects region=%d snapshot_date=%s", + len(obj_ids), + region_code, + snapshot_date, + ) + + async with BrowserSession( + region_code=region_code, + # Страницы каталога публичные — Basic auth не нужен + auth=None, + ) as session: + for obj_id in obj_ids: + stats["processed"] += 1 + ok = await scrape_catalog_object(db, session, obj_id, snapshot_date) + if ok: + stats["succeeded"] += 1 + else: + stats["failed"] += 1 + + logger.info( + "scrape_catalog_objects done: processed=%d succeeded=%d failed=%d skipped=%d", + stats["processed"], + stats["succeeded"], + stats["failed"], + stats["skipped"], + ) + return stats diff --git a/backend/app/workers/beat_schedule.py b/backend/app/workers/beat_schedule.py index 7aa9c341..f474e88c 100644 --- a/backend/app/workers/beat_schedule.py +++ b/backend/app/workers/beat_schedule.py @@ -237,6 +237,16 @@ def build_beat_schedule() -> dict: "options": {"queue": "celery"}, } + # Catalog-object scrape — наполняет ~25 NULL колонок domrf_kn_objects из SSR-страниц. + # kn-API не отдаёт wall_type, energy_eff, ceiling_height_m, parking_* и т.д. + # Вторник 04:00 UTC. batch 300/run → 1532 объекта за ~5 недель полного обновления. + schedule["scrape-kn-catalog-objects-weekly"] = { + "task": "tasks.scrape_kn_catalog_objects.scrape_kn_catalog_objects", + "schedule": _parse_cron("0 4 * * 2"), # Tuesday 04:00 UTC + "kwargs": {"region_code": 66, "max_objects": 300}, + "options": {"queue": "celery"}, + } + # NSPD quarter dump refresh — re-enabled 2026-05-17 после Sub-PR B (#260) # переключения search_by_quarter на grid-walk. Foundation (#247) + integration # (#260) теперь возвращают полноценные dumps (territorial_zones, ЗОУИТ, risk diff --git a/backend/app/workers/tasks/scrape_kn_catalog_objects.py b/backend/app/workers/tasks/scrape_kn_catalog_objects.py new file mode 100644 index 00000000..c3b96d21 --- /dev/null +++ b/backend/app/workers/tasks/scrape_kn_catalog_objects.py @@ -0,0 +1,157 @@ +"""Celery task: periodic catalog-object scrape для DOM.РФ. + +Дополняет ~25 NULL колонок в domrf_kn_objects из SSR-страниц каталога. +kn-API эти поля не возвращает — они только на публичных страницах объектов. + +Выбирает объекты где catalog_scraped_at IS NULL или устарело (>30 дней). +Ограничивает batch per run чтобы не нагружать сайт. + +Beat schedule: вторник 04:00 UTC (в beat_schedule.py). +""" + +from __future__ import annotations + +import asyncio +import logging +from datetime import date +from typing import Any + +from sqlalchemy import text + +from app.core.db import SessionLocal +from app.workers.celery_app import celery_app + +logger = logging.getLogger(__name__) + +# Запрос для выбора устаревших / ненаполненных объектов. +# Возвращает obj_id + snapshot_date одним запросом, чтобы избежать race condition: +# если между двумя запросами kn-scraper запишет новый snapshot — UPDATE по старой +# snapshot_date не затронет ни одной строки. MAX subquery ограничена тем же +# region_cd чтобы не захватить snapshot другого региона. +_SELECT_STALE_SQL = text( + """ + SELECT obj_id, snapshot_date + FROM domrf_kn_objects + WHERE region_cd = :region_code + AND ( + catalog_scraped_at IS NULL + OR catalog_scraped_at < NOW() - INTERVAL '30 days' + ) + AND snapshot_date = ( + SELECT MAX(snapshot_date) + FROM domrf_kn_objects + WHERE region_cd = :region_code + ) + ORDER BY catalog_scraped_at NULLS FIRST + LIMIT :max_objects + """ +) + +# Лимит по умолчанию если max_objects не задан явно. +_DEFAULT_MAX_OBJECTS = 300 + + +@celery_app.task( + bind=True, + name="tasks.scrape_kn_catalog_objects.scrape_kn_catalog_objects", + time_limit=3600, +) +def scrape_kn_catalog_objects( + self: Any, + region_code: int = 66, + max_objects: int | None = None, +) -> dict[str, Any]: + """Periodic catalog-object scrape. + + Picks objects from domrf_kn_objects where catalog data missing OR stale + (catalog_scraped_at < NOW() - INTERVAL '30 days'). Limits per run. + + Args: + region_code: Код региона (ОКАТО prefix). Default 66 = Свердловская обл. + max_objects: Максимум объектов за один run. Default 300. + + Returns: + dict с ключами: region_code, snapshot_date, obj_ids_count, + processed, succeeded, failed, skipped. + + Concurrency: + No Redis lock — consistent with sibling tasks (scrape_kn_region etc.). + Beat is configured for non-overlapping fire (Tuesday 04:00 UTC, ~5min run), + so concurrent execution is extremely rare. If it occurs: + - UPDATE is idempotent (COALESCE, catalog_scraped_at = NOW()) + - Max risk: 2x WAF load on DOM.РФ for the same batch + - Both tasks complete; second update is no-op (catalog_scraped_at расхождение) + + Add Redis lock if WAF blocks observed or beat schedule changes to overlap. + """ + from app.services.scrapers.domrf_catalog_object import scrape_catalog_objects + + limit = max_objects if max_objects is not None else _DEFAULT_MAX_OBJECTS + + db = SessionLocal() + try: + rows = ( + db.execute( + _SELECT_STALE_SQL, + {"region_code": region_code, "max_objects": limit}, + ) + .mappings() + .all() + ) + obj_ids: list[int] = [int(r["obj_id"]) for r in rows] + except Exception as exc: + logger.error("scrape_kn_catalog_objects: failed to fetch obj_ids: %s", exc) + db.close() + raise + + if not obj_ids: + logger.info( + "scrape_kn_catalog_objects: no stale objects for region=%d, nothing to do", + region_code, + ) + db.close() + return { + "region_code": region_code, + "obj_ids_count": 0, + "processed": 0, + "succeeded": 0, + "failed": 0, + "skipped": 0, + } + + # snapshot_date берётся из первой строки результата — все строки одинаковые + # (WHERE snapshot_date = MAX(snapshot_date)). Это атомарно: один SELECT вместо двух, + # что устраняет race condition с kn-scraper. + snapshot_date_val: date = rows[0]["snapshot_date"] + + logger.info( + "scrape_kn_catalog_objects: region=%d snapshot_date=%s obj_ids=%d (limit=%d)", + region_code, + snapshot_date_val, + len(obj_ids), + limit, + ) + + try: + stats = asyncio.run( + scrape_catalog_objects( + db=db, + obj_ids=obj_ids, + snapshot_date=snapshot_date_val, + region_code=region_code, + ) + ) + except Exception as exc: + logger.error("scrape_kn_catalog_objects: scrape failed: %s", exc) + raise + finally: + db.close() + + result: dict[str, Any] = { + "region_code": region_code, + "snapshot_date": str(snapshot_date_val), + "obj_ids_count": len(obj_ids), + **stats, + } + logger.info("scrape_kn_catalog_objects done: %s", result) + return result diff --git a/backend/tests/api/v1/test_parcel_by_bbox.py b/backend/tests/api/v1/test_parcel_by_bbox.py new file mode 100644 index 00000000..6c60c9fe --- /dev/null +++ b/backend/tests/api/v1/test_parcel_by_bbox.py @@ -0,0 +1,152 @@ +"""Тесты GET /parcels/by-bbox (SF-B1). + +Проверяют: +1. Валидный bbox → 200 + корректная структура ответа. +2. Некорректный bbox (min_lat >= max_lat) → 400. +3. user_id передан → status заполняется из mock DB; без user_id → status=null. +""" + +from __future__ import annotations + +from typing import Any +from unittest.mock import MagicMock, patch + +import pytest +from fastapi.testclient import TestClient + +from app.main import app + + +def _make_db_row( + cad_num: str = "66:41:0204016:10", + centroid_lat: float = 56.83, + centroid_lon: float = 60.64, + area_m2: float = 1200.0, + land_category: str | None = "land_residential", + user_status: str | None = None, +) -> dict[str, Any]: + return { + "cad_num": cad_num, + "centroid_lat": centroid_lat, + "centroid_lon": centroid_lon, + "area_m2": area_m2, + "land_category": land_category, + "user_status": user_status, + } + + +def _build_mock_db(rows: list[dict[str, Any]]) -> MagicMock: + """Сконструировать mock Session, возвращающий rows при execute().mappings().all().""" + mock_db = MagicMock() + mappings_mock = MagicMock() + mappings_mock.all.return_value = rows + execute_result = MagicMock() + execute_result.mappings.return_value = mappings_mock + mock_db.execute.return_value = execute_result + return mock_db + + +@pytest.fixture() +def client() -> TestClient: + return TestClient(app) + + +# ── Test 1: валидный bbox возвращает корректную структуру ────────────────── + + +def test_by_bbox_valid_returns_structure(client: TestClient) -> None: + rows = [_make_db_row()] + mock_db = _build_mock_db(rows) + + with patch("app.api.v1.parcels.get_db", return_value=iter([mock_db])): + resp = client.get( + "/api/v1/parcels/by-bbox", + params={ + "min_lat": 56.80, + "min_lon": 60.60, + "max_lat": 56.90, + "max_lon": 60.70, + "limit": 200, + }, + ) + + assert resp.status_code == 200 + body = resp.json() + assert "parcels" in body + assert "count" in body + assert "limit" in body + assert "bbox_area_km2" in body + assert body["count"] == 1 + assert body["limit"] == 200 + + parcel = body["parcels"][0] + assert parcel["cad_num"] == "66:41:0204016:10" + assert parcel["centroid_lat"] == pytest.approx(56.83, abs=0.01) + assert parcel["centroid_lon"] == pytest.approx(60.64, abs=0.01) + assert parcel["area_m2"] == pytest.approx(1200.0) + assert parcel["land_category"] == "land_residential" + assert parcel["status"] is None # user_id не передан + assert parcel["last_analysis_date"] is None + + +# ── Test 2: некорректный bbox → 400 ─────────────────────────────────────── + + +def test_by_bbox_invalid_bbox_returns_400(client: TestClient) -> None: + resp = client.get( + "/api/v1/parcels/by-bbox", + params={ + "min_lat": 56.90, # min >= max → ошибка + "min_lon": 60.60, + "max_lat": 56.80, + "max_lon": 60.70, + }, + ) + assert resp.status_code == 400 + assert "bbox" in resp.json()["detail"].lower() + + +# ── Test 3: user_id → status из БД; без user_id → status null ───────────── + + +def test_by_bbox_status_overlay_with_user_id(client: TestClient) -> None: + rows = [_make_db_row(user_status="in_work")] + mock_db = _build_mock_db(rows) + + with patch("app.api.v1.parcels.get_db", return_value=iter([mock_db])): + resp = client.get( + "/api/v1/parcels/by-bbox", + params={ + "min_lat": 56.80, + "min_lon": 60.60, + "max_lat": 56.90, + "max_lon": 60.70, + "user_id": "user-abc-123", + }, + ) + + assert resp.status_code == 200 + parcel = resp.json()["parcels"][0] + assert parcel["status"] == "in_work" + + +def test_by_bbox_no_user_id_status_is_null(client: TestClient) -> None: + rows = [_make_db_row(user_status="favorite")] # DB вернёт статус + mock_db = _build_mock_db(rows) + + with patch("app.api.v1.parcels.get_db", return_value=iter([mock_db])): + resp = client.get( + "/api/v1/parcels/by-bbox", + params={ + "min_lat": 56.80, + "min_lon": 60.60, + "max_lat": 56.90, + "max_lon": 60.70, + # user_id не передан + }, + ) + + assert resp.status_code == 200 + parcel = resp.json()["parcels"][0] + # Без user_id статус принудительно null (endpoint не раскрывает чужие статусы) + assert parcel["status"] is None diff --git a/backend/tests/services/scrapers/__init__.py b/backend/tests/services/scrapers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/tests/services/scrapers/test_domrf_catalog_object.py b/backend/tests/services/scrapers/test_domrf_catalog_object.py new file mode 100644 index 00000000..34012edd --- /dev/null +++ b/backend/tests/services/scrapers/test_domrf_catalog_object.py @@ -0,0 +1,290 @@ +"""Тесты для domrf_catalog_object.py (issue #297 sub-task 22d). + +Покрывает: + - extract_next_data — парсинг HTML с __NEXT_DATA__ + - parse_catalog_object — маппинг pageProps → DB columns + - value helpers (_to_numeric_comma, _to_bool_da_net, _to_date_ddmmyyyy) + - partial responses (partial pageProps → all other fields = None, no crash) +""" + +from __future__ import annotations + +from datetime import date +from typing import Any + +import pytest + +from app.services.scrapers.domrf_catalog_object import ( + _to_bool_da_net, + _to_bool_int, + _to_date_ddmmyyyy, + _to_numeric_comma, + extract_next_data, + parse_catalog_object, +) + +# ── extract_next_data ───────────────────────────────────────────────────────── + + +def test_extract_next_data_from_html() -> None: + """Базовый case: тег найден, JSON возвращается как dict.""" + html = ( + "" + '" + "" + ) + result = extract_next_data(html) + assert isinstance(result, dict) + assert result["props"]["pageProps"]["buildingClass"] == "Комфорт" + + +def test_extract_next_data_single_quotes() -> None: + """Тег с одинарными кавычками тоже должен парситься.""" + html = "" + result = extract_next_data(html) + assert "props" in result + + +def test_extract_next_data_not_found_raises() -> None: + """Если тег не найден — ValueError.""" + with pytest.raises(ValueError, match="__NEXT_DATA__"): + extract_next_data("no script here") + + +def test_extract_next_data_invalid_json_raises() -> None: + """Если JSON некорректный — ValueError.""" + html = '' + with pytest.raises(ValueError): + extract_next_data(html) + + +# ── parse_catalog_object — full sample ─────────────────────────────────────── + + +def _make_full_next_data() -> dict[str, Any]: + """Реалистичный full next_data для obj_id=65136 (подтверждён live).""" + return { + "props": { + "pageProps": { + "buildingClass": "Комфорт", + "wallMaterial": "Монолит-кирпич", + "objEnergyEfficiency": "B", + "parkingCount": 246, + "finishTypeCount": 1, + "freePlan": "Нет", + "publicationDate": "31.03.2025", + "additionalInfo": { + "objectParkingPlaces": 43, + "nearbyParkingPlaces": 0, + "ceilingHeight": "2,7", + "passengerElevatorsCount": 0, + "cargoElevatorsCount": 0, + "cargoPassengerElevatorCount": 4, + "playgroundsCount": 6, + "sportsgroundCount": 5, + "bicycleLane": 0, + "trashAreaCount": 3, + "ramp": 0, + "curbLowering": 1, + "wheelchairElevatorsCount": 0, + "parkingAvailabilityPerc": 60, + }, + "quartography": { + "objLivElemEntrCnt": 1, + "objLivElemSqAvg": 46.2, + "nonLivFirstFloor": 1, + }, + "indexes": { + "infrastructure": 10, + "transport": 6, + }, + "projectDeclaration": { + "number": "66-001686", + }, + } + } + } + + +def test_parse_catalog_object_full() -> None: + """Полный sample: все 25+ полей должны быть замаплены корректно.""" + data = parse_catalog_object(_make_full_next_data()) + + assert data["obj_class"] == "Комфорт" + assert data["wall_type"] == "Монолит-кирпич" + assert data["energy_eff"] == "B" + assert data["section_count"] == 1 + assert data["parking_total_slots"] == 246 + assert data["guest_parking_inside_count"] == 43 + assert data["guest_parking_outside_count"] == 0 + assert data["ceiling_height_m"] == pytest.approx(2.7) + assert data["finishing_variants_count"] == 1 + assert data["has_free_planning"] is False + assert data["avg_flat_area_m2"] == pytest.approx(46.2) + assert data["elevators_passenger_count"] == 0 + assert data["elevators_cargo_count"] == 4 # 0 + 4 + assert data["playground_kids_count"] == 6 + assert data["playground_sport_count"] == 5 + assert data["has_bike_paths"] is False # bicycleLane=0 + assert data["trash_areas_count"] == 3 + assert data["has_ramp"] is False # ramp=0 + assert data["has_low_platforms"] is True # curbLowering=1 + assert data["has_wheelchair_lift"] is False # wheelchairElevatorsCount=0 + assert data["first_floor_type"] == "нежилой" # nonLivFirstFloor=1 + assert data["parking_provision_pct"] == 60 + assert data["project_published_at"] == date(2025, 3, 31) + assert data["project_declaration_num"] == "66-001686" + assert data["domrf_score_infrastructure"] == 10 + assert data["domrf_score_transport"] == 6 + + +def test_parse_catalog_object_has_free_planning_da() -> None: + """freePlan='Да' → has_free_planning=True.""" + nd: dict[str, Any] = {"props": {"pageProps": {"freePlan": "Да"}}} + data = parse_catalog_object(nd) + assert data["has_free_planning"] is True + + +def test_parse_catalog_object_first_floor_zhiloj() -> None: + """nonLivFirstFloor=0 → first_floor_type='жилой'.""" + nd: dict[str, Any] = { + "props": { + "pageProps": { + "quartography": {"nonLivFirstFloor": 0}, + } + } + } + data = parse_catalog_object(nd) + assert data["first_floor_type"] == "жилой" + + +def test_parse_catalog_object_elevators_cargo_sum() -> None: + """elevators_cargo_count = cargoElevatorsCount + cargoPassengerElevatorCount.""" + nd: dict[str, Any] = { + "props": { + "pageProps": { + "additionalInfo": { + "cargoElevatorsCount": 2, + "cargoPassengerElevatorCount": 3, + } + } + } + } + data = parse_catalog_object(nd) + assert data["elevators_cargo_count"] == 5 + + +def test_parse_catalog_object_partial() -> None: + """Только buildingClass → остальные поля None, без исключений.""" + nd: dict[str, Any] = {"props": {"pageProps": {"buildingClass": "Бизнес"}}} + data = parse_catalog_object(nd) + assert data["obj_class"] == "Бизнес" + assert data["wall_type"] is None + assert data["energy_eff"] is None + assert data["section_count"] is None + assert data["parking_total_slots"] is None + assert data["ceiling_height_m"] is None + assert data["has_free_planning"] is None + assert data["elevators_cargo_count"] is None + assert data["project_published_at"] is None + assert data["domrf_score_infrastructure"] is None + + +def test_parse_catalog_object_empty() -> None: + """Полностью пустой next_data → все поля None, без исключений.""" + data = parse_catalog_object({}) + for v in data.values(): + assert v is None + + +def test_parking_provision_pct_preserves_float() -> None: + """parking_provision_pct should preserve fractional values (column is numeric(5,1)).""" + next_data: dict[str, Any] = { + "props": {"pageProps": {"id": 65136, "additionalInfo": {"parkingAvailabilityPerc": 60.5}}} + } + result = parse_catalog_object(next_data) + assert result["parking_provision_pct"] == 60.5 + + +# ── _to_numeric_comma ───────────────────────────────────────────────────────── + + +@pytest.mark.parametrize( + "inp,expected", + [ + ("2,7", 2.7), + ("2.7", 2.7), + ("3,50", 3.5), + ("", None), + (None, None), + (" ", None), + ("abc", None), + ], +) +def test_to_numeric_comma(inp: Any, expected: float | None) -> None: + result = _to_numeric_comma(inp) + if expected is None: + assert result is None + else: + assert result == pytest.approx(expected) + + +# ── _to_bool_da_net ─────────────────────────────────────────────────────────── + + +@pytest.mark.parametrize( + "inp,expected", + [ + ("Да", True), + ("да", True), + ("ДА", True), + ("Нет", False), + ("нет", False), + ("НЕТ", False), + ("", None), + (None, None), + ("maybe", None), + ("Yes", None), + ], +) +def test_to_bool_da_net(inp: Any, expected: bool | None) -> None: + assert _to_bool_da_net(inp) == expected + + +# ── _to_bool_int ────────────────────────────────────────────────────────────── + + +@pytest.mark.parametrize( + "inp,expected", + [ + (0, False), + (1, True), + (5, True), + ("1", True), + ("0", False), + (None, None), + ], +) +def test_to_bool_int(inp: Any, expected: bool | None) -> None: + assert _to_bool_int(inp) == expected + + +# ── _to_date_ddmmyyyy ───────────────────────────────────────────────────────── + + +@pytest.mark.parametrize( + "inp,expected", + [ + ("31.03.2025", date(2025, 3, 31)), + ("01.01.2024", date(2024, 1, 1)), + ("", None), + (None, None), + ("2025-03-31", None), # неправильный формат → None + ("abc", None), + ("31.13.2025", None), # невалидный месяц → None + ], +) +def test_to_date_ddmmyyyy(inp: Any, expected: date | None) -> None: + assert _to_date_ddmmyyyy(inp) == expected diff --git a/data/sql/118_22d_catalog_object_scraped_at.sql b/data/sql/118_22d_catalog_object_scraped_at.sql new file mode 100644 index 00000000..3ea522e3 --- /dev/null +++ b/data/sql/118_22d_catalog_object_scraped_at.sql @@ -0,0 +1,15 @@ +-- Migration 118: добавить catalog_scraped_at в domrf_kn_objects +-- Нужна для catalog-object scraper (issue #297, sub-task 22d): +-- scraper обновляет поле после успешного UPDATE, beat task выбирает +-- только объекты где это поле NULL или устарело (> 30 дней). +-- Индекс NULLS FIRST ускоряет ORDER BY catalog_scraped_at NULLS FIRST LIMIT N. + +BEGIN; + +ALTER TABLE domrf_kn_objects + ADD COLUMN IF NOT EXISTS catalog_scraped_at TIMESTAMP; + +CREATE INDEX IF NOT EXISTS idx_domrf_kn_objects_catalog_scraped_at + ON domrf_kn_objects (catalog_scraped_at NULLS FIRST); + +COMMIT; diff --git a/data/sql/119_parcel_user_status.sql b/data/sql/119_parcel_user_status.sql new file mode 100644 index 00000000..97392e43 --- /dev/null +++ b/data/sql/119_parcel_user_status.sql @@ -0,0 +1,21 @@ +BEGIN; + +CREATE TABLE IF NOT EXISTS parcel_user_status ( + user_id text NOT NULL, + cad_num text NOT NULL, + status text NOT NULL CHECK (status IN ('in_work', 'favorite', 'dismissed')), + notes text, + updated_at timestamptz NOT NULL DEFAULT NOW(), + PRIMARY KEY (user_id, cad_num) +); + +CREATE INDEX IF NOT EXISTS parcel_user_status_cad_idx + ON parcel_user_status (cad_num); + +CREATE INDEX IF NOT EXISTS parcel_user_status_user_idx + ON parcel_user_status (user_id, status); + +COMMENT ON TABLE parcel_user_status + IS '#307 SF-B1 user-marked parcels (in_work/favorite/dismissed) для карты.'; + +COMMIT;