diff --git a/.env.example b/.env.example index fcfba003..d9e10a34 100644 --- a/.env.example +++ b/.env.example @@ -10,3 +10,11 @@ POSTGRES_PASSWORD=changeme # YC_REGISTRY_ID= # IMAGE_TAG=latest # NEXT_PUBLIC_API_BASE_URL=https://your-domain.example.ru + +# GlitchTip (errors.gendsgn.ru) +GLITCHTIP_DB_PASS= +GLITCHTIP_SECRET= +# DSN из GlitchTip UI → Project Settings → Client Keys. +# Используется backend Sentry SDK + glitchtip-auth-forwarder sidecar. +GLITCHTIP_DSN= +NEXT_PUBLIC_GLITCHTIP_DSN= diff --git a/.forgejo/workflows/deploy-tradein.yml b/.forgejo/workflows/deploy-tradein.yml new file mode 100644 index 00000000..e27f3a2a --- /dev/null +++ b/.forgejo/workflows/deploy-tradein.yml @@ -0,0 +1,193 @@ +name: Deploy Trade-In + +# Forgejo Actions — отдельный pipeline для подпроекта tradein-mvp/. +# Триггерится только на изменения внутри tradein-mvp/ (или этого workflow), +# не пересекается с основным deploy.yml. +on: + push: + branches: [main] + paths: + - "tradein-mvp/**" + - ".forgejo/workflows/deploy-tradein.yml" + workflow_dispatch: + +concurrency: + group: deploy-tradein-prod + cancel-in-progress: false + +env: + IMAGE_BACKEND: ghcr.io/lekss361/gendesign-tradein-backend + IMAGE_FRONTEND: ghcr.io/lekss361/gendesign-tradein-frontend + +jobs: + changes: + runs-on: ubuntu-latest + outputs: + backend: ${{ steps.filter.outputs.backend }} + frontend: ${{ steps.filter.outputs.frontend }} + infra: ${{ steps.filter.outputs.infra }} + steps: + - uses: actions/checkout@v4 + - uses: dorny/paths-filter@v3 + id: filter + with: + filters: | + backend: + - 'tradein-mvp/backend/**' + frontend: + - 'tradein-mvp/frontend/**' + infra: + - 'tradein-mvp/docker-compose.prod.yml' + - 'tradein-mvp/deploy/**' + - '.forgejo/workflows/deploy-tradein.yml' + + build-backend: + runs-on: ubuntu-latest + needs: changes + if: | + needs.changes.outputs.backend == 'true' || + needs.changes.outputs.infra == 'true' || + github.event_name == 'workflow_dispatch' + steps: + - uses: actions/checkout@v4 + + - name: Login to GHCR (shell-based — docker/login-action@v3 unreliable под Forgejo Actions) + env: + GHCR_PAT: ${{ secrets.GHCR_PAT }} + run: | + echo "$GHCR_PAT" | docker login ghcr.io -u lekss361 --password-stdin + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build & push tradein-backend + uses: docker/build-push-action@v6 + with: + context: ./tradein-mvp/backend + push: true + cache-from: type=registry,ref=${{ env.IMAGE_BACKEND }}:buildcache + cache-to: type=registry,ref=${{ env.IMAGE_BACKEND }}:buildcache,mode=max + tags: | + ${{ env.IMAGE_BACKEND }}:latest + ${{ env.IMAGE_BACKEND }}:${{ github.sha }} + + build-frontend: + runs-on: ubuntu-latest + needs: changes + if: | + needs.changes.outputs.frontend == 'true' || + needs.changes.outputs.infra == 'true' || + github.event_name == 'workflow_dispatch' + steps: + - uses: actions/checkout@v4 + + - name: Login to GHCR (shell-based — docker/login-action@v3 unreliable под Forgejo Actions) + env: + GHCR_PAT: ${{ secrets.GHCR_PAT }} + run: | + echo "$GHCR_PAT" | docker login ghcr.io -u lekss361 --password-stdin + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build & push tradein-frontend + uses: docker/build-push-action@v6 + with: + context: ./tradein-mvp/frontend + push: true + # basePath=/trade-in baked-in во время build (Next.js) + build-args: | + NEXT_PUBLIC_BASE_PATH=/trade-in + NEXT_PUBLIC_API_BASE_URL=/trade-in + cache-from: type=registry,ref=${{ env.IMAGE_FRONTEND }}:buildcache + cache-to: type=registry,ref=${{ env.IMAGE_FRONTEND }}:buildcache,mode=max + tags: | + ${{ env.IMAGE_FRONTEND }}:latest + ${{ env.IMAGE_FRONTEND }}:${{ github.sha }} + + deploy: + runs-on: ubuntu-latest + needs: [changes, build-backend, build-frontend] + if: | + always() && + !cancelled() && + needs.build-backend.result != 'failure' && + needs.build-frontend.result != 'failure' + steps: + - name: Deploy via SSH + uses: appleboy/ssh-action@v1.0.3 + env: + IMAGE_TAG: latest + GHCR_PAT: ${{ secrets.GHCR_PAT }} + with: + host: ${{ secrets.DEPLOY_HOST }} + username: ${{ secrets.DEPLOY_USER }} + key: ${{ secrets.DEPLOY_SSH_KEY }} + port: ${{ secrets.DEPLOY_PORT }} + envs: IMAGE_TAG,GHCR_PAT + script: | + set -euo pipefail + cd /opt/gendesign + + # repo уже clone'ен — origin = Forgejo. Подтягиваем последний main. + git fetch origin main + git reset --hard origin/main + + cd tradein-mvp + + # .env.runtime создаётся вручную при первом запуске (см. README-АДМИНУ.md). + # Здесь только подгружаем переменные для docker compose (POSTGRES_PASSWORD). + if [ ! -f .env.runtime ]; then + echo "ERROR: /opt/gendesign/tradein-mvp/.env.runtime отсутствует." + echo "Создай его вручную (см. tradein-mvp/README.md или DEPLOY.md)." + exit 1 + fi + chmod 600 .env.runtime + set -a; source .env.runtime; set +a + + # External network для Caddy (он в основном gendesign-стеке) + docker network inspect gendesign_shared >/dev/null 2>&1 \ + || docker network create gendesign_shared + + # Re-login to GHCR (PAT может быть rotated) + echo "$GHCR_PAT" | docker login ghcr.io -u lekss361 --password-stdin + + export IMAGE_TAG="$IMAGE_TAG" + docker compose -p gendesign-tradein -f docker-compose.prod.yml pull + docker compose -p gendesign-tradein -f docker-compose.prod.yml up -d + + # Применяем SQL миграции (если есть data/sql/*.sql) + # Postgres init load *.sql из /docker-entrypoint-initdb.d ТОЛЬКО при первом + # старте volume. Здесь — для повторных миграций после первого запуска. + sleep 5 + for sql_file in $(ls -1 backend/data/sql/*.sql 2>/dev/null | sort); do + fname=$(basename "$sql_file") + echo "→ Migration: $fname" + docker compose -p gendesign-tradein -f docker-compose.prod.yml exec -T postgres \ + psql -U "${TRADEIN_POSTGRES_USER:-tradein}" -d tradein \ + -v ON_ERROR_STOP=on < "$sql_file" \ + || echo " (skipped — already applied or error: $fname)" + done + + # Caddy reload — основной Caddyfile содержит inline tradein routes + # (см. Caddyfile в корне репы). Reload, чтобы Caddy перечитал DNS + # tradein-backend / tradein-frontend (они в gendesign_shared network). + cd /opt/gendesign + docker compose -p gendesign -f docker-compose.prod.yml exec -T caddy \ + caddy reload --config /etc/caddy/Caddyfile || true + + # Health check + for i in $(seq 1 30); do + docker compose -p gendesign-tradein -f /opt/gendesign/tradein-mvp/docker-compose.prod.yml \ + exec -T backend curl -fsS http://localhost:8000/health && break + sleep 1 + done + + # Cleanup старых образов + for repo in ghcr.io/lekss361/gendesign-tradein-backend \ + ghcr.io/lekss361/gendesign-tradein-frontend; do + docker images "$repo" --format '{{.Repository}}:{{.Tag}}' \ + | grep -v ':latest$' | tail -n +3 \ + | xargs -r docker rmi 2>/dev/null || true + done + docker image prune -af || true diff --git a/.github/workflows/deploy.yml b/.forgejo/workflows/deploy.yml similarity index 52% rename from .github/workflows/deploy.yml rename to .forgejo/workflows/deploy.yml index 09b9254a..b9cd5d1e 100644 --- a/.github/workflows/deploy.yml +++ b/.forgejo/workflows/deploy.yml @@ -1,7 +1,9 @@ name: Deploy -# Деплоится только при изменениях основного стека. -# Obsidian-стек (CouchDB) — отдельный workflow `deploy-obsidian.yml`. +# Forgejo Actions equivalent of .github/workflows/deploy.yml +# Migration 2026-05-16: GitHub → Forgejo (git.gendsgn.ru) +# Builds images on Forgejo runner, pushes to ghcr.io (GitHub Container Registry), +# SSH-deploys to Beget VPS (46.173.16.127). on: push: branches: [main] @@ -10,10 +12,10 @@ on: - "frontend/**" - "docker-compose.prod.yml" - "Caddyfile" - - ".github/workflows/deploy.yml" - - "data/sql/*.sql" - # NB: shared compose-fragments (network) — тоже триггерят main - # потому что Caddy шарит сеть с obsidian-stack + - "caddy/**" + - ".forgejo/workflows/deploy.yml" + - "data/sql/**" + - "ops/glitchtip-auth-forwarder/**" workflow_dispatch: concurrency: @@ -28,9 +30,6 @@ env: jobs: changes: runs-on: ubuntu-latest - permissions: - contents: read - pull-requests: read outputs: backend: ${{ steps.filter.outputs.backend }} frontend: ${{ steps.filter.outputs.frontend }} @@ -43,19 +42,18 @@ jobs: filters: | backend: - 'backend/**' + - 'data/sql/**' frontend: - 'frontend/**' infra: - 'docker-compose.prod.yml' - 'Caddyfile' - - '.github/workflows/deploy.yml' + - 'caddy/**' + - '.forgejo/workflows/deploy.yml' build-backend: runs-on: ubuntu-latest needs: changes - permissions: - contents: read - packages: write if: | needs.changes.outputs.backend == 'true' || needs.changes.outputs.infra == 'true' || @@ -63,12 +61,11 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Login to GHCR - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + - name: Login to GHCR (shell-based — docker/login-action@v3 unreliable под Forgejo Actions) + env: + GHCR_PAT: ${{ secrets.GHCR_PAT }} + run: | + echo "$GHCR_PAT" | docker login ghcr.io -u lekss361 --password-stdin - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -79,8 +76,8 @@ jobs: context: ./backend target: runner push: true - cache-from: type=gha,scope=backend-lean - cache-to: type=gha,mode=max,scope=backend-lean + cache-from: type=registry,ref=${{ env.IMAGE_BACKEND }}:buildcache + cache-to: type=registry,ref=${{ env.IMAGE_BACKEND }}:buildcache,mode=max tags: | ${{ env.IMAGE_BACKEND }}:latest ${{ env.IMAGE_BACKEND }}:${{ github.sha }} @@ -88,9 +85,6 @@ jobs: build-worker: runs-on: ubuntu-latest needs: changes - permissions: - contents: read - packages: write if: | needs.changes.outputs.backend == 'true' || needs.changes.outputs.infra == 'true' || @@ -98,12 +92,11 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Login to GHCR - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + - name: Login to GHCR (shell-based — docker/login-action@v3 unreliable под Forgejo Actions) + env: + GHCR_PAT: ${{ secrets.GHCR_PAT }} + run: | + echo "$GHCR_PAT" | docker login ghcr.io -u lekss361 --password-stdin - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -114,8 +107,8 @@ jobs: context: ./backend target: runner-with-chromium push: true - cache-from: type=gha,scope=worker-chromium - cache-to: type=gha,mode=max,scope=worker-chromium + cache-from: type=registry,ref=${{ env.IMAGE_WORKER }}:buildcache + cache-to: type=registry,ref=${{ env.IMAGE_WORKER }}:buildcache,mode=max tags: | ${{ env.IMAGE_WORKER }}:latest ${{ env.IMAGE_WORKER }}:${{ github.sha }} @@ -123,9 +116,6 @@ jobs: build-frontend: runs-on: ubuntu-latest needs: changes - permissions: - contents: read - packages: write if: | needs.changes.outputs.frontend == 'true' || needs.changes.outputs.infra == 'true' || @@ -133,12 +123,11 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Login to GHCR - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + - name: Login to GHCR (shell-based — docker/login-action@v3 unreliable под Forgejo Actions) + env: + GHCR_PAT: ${{ secrets.GHCR_PAT }} + run: | + echo "$GHCR_PAT" | docker login ghcr.io -u lekss361 --password-stdin - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -148,8 +137,11 @@ jobs: with: context: ./frontend push: true - cache-from: type=gha,scope=frontend - cache-to: type=gha,mode=max,scope=frontend + build-args: | + NEXT_PUBLIC_GLITCHTIP_DSN=${{ secrets.GLITCHTIP_FRONTEND_DSN }} + NEXT_PUBLIC_ENVIRONMENT=production + cache-from: type=registry,ref=${{ env.IMAGE_FRONTEND }}:buildcache + cache-to: type=registry,ref=${{ env.IMAGE_FRONTEND }}:buildcache,mode=max tags: | ${{ env.IMAGE_FRONTEND }}:latest ${{ env.IMAGE_FRONTEND }}:${{ github.sha }} @@ -157,10 +149,6 @@ jobs: deploy: runs-on: ubuntu-latest needs: [changes, build-backend, build-worker, build-frontend] - permissions: - contents: read - # Запускается если ни один build не завершился failure. - # `always()` позволяет выполнить job даже когда зависимые jobs были skipped. if: | always() && !cancelled() && @@ -173,26 +161,28 @@ jobs: env: IMAGE_TAG: latest SENTRY_RELEASE_VAL: ${{ github.sha }} + GHCR_PAT: ${{ secrets.GHCR_PAT }} + GLITCHTIP_BACKEND_DSN: ${{ secrets.GLITCHTIP_BACKEND_DSN }} + OBJECTIVE_API_KEY: ${{ secrets.OBJECTIVE_API_KEY }} with: host: ${{ secrets.DEPLOY_HOST }} username: ${{ secrets.DEPLOY_USER }} key: ${{ secrets.DEPLOY_SSH_KEY }} - port: ${{ secrets.DEPLOY_PORT || 22 }} - envs: IMAGE_TAG,SENTRY_RELEASE_VAL + port: ${{ secrets.DEPLOY_PORT }} + envs: IMAGE_TAG,SENTRY_RELEASE_VAL,GHCR_PAT,GLITCHTIP_BACKEND_DSN,OBJECTIVE_API_KEY script: | set -euo pipefail cd /opt/gendesign # Sync compose / Caddyfile / init scripts from the repo. - # --ff-only refuses if there are local commits on the VM (we don't expect any). + # Origin теперь Forgejo (после migration 2026-05-16) — HTTPS basic auth + # через PAT в URL не нужен т.к. repo читается через deploy key (SSH) + # ИЛИ public read-only mode. Forgejo gendesign — private, нужен auth: + # `git remote get-url origin` должен указывать на Forgejo с auth. git fetch origin main git reset --hard origin/main - # Sentry release tracking — записываем git-sha в .env.runtime. - # ВАЖНО: НЕ перезаписываем файл целиком (там могут быть - # COUCHDB_PASSWORD и др. user-managed secrets). Заменяем только - # строку SENTRY_RELEASE или добавляем если её нет. - # IMAGE_TAG=latest для docker pull; SENTRY_RELEASE_VAL = git sha для трекинга. + # Sentry release tracking mkdir -p backend touch backend/.env.runtime if grep -q '^SENTRY_RELEASE=' backend/.env.runtime; then @@ -200,29 +190,39 @@ jobs: else printf 'SENTRY_RELEASE=%s\n' "$SENTRY_RELEASE_VAL" >> backend/.env.runtime fi + + # GlitchTip wiring: убираем legacy SENTRY_DSN (auto-promote-логика в + # backend/app/core/config.py:_promote_legacy_sentry_dsn раньше брала + # его и слала события в чужой sentry.io). Устанавливаем GLITCHTIP_DSN + # из Forgejo secret. Пустой secret = no-op (SDK не инициализируется). + sed -i '/^SENTRY_DSN=/d' backend/.env.runtime + if grep -q '^GLITCHTIP_DSN=' backend/.env.runtime; then + sed -i "s|^GLITCHTIP_DSN=.*|GLITCHTIP_DSN=$GLITCHTIP_BACKEND_DSN|" backend/.env.runtime + else + printf 'GLITCHTIP_DSN=%s\n' "$GLITCHTIP_BACKEND_DSN" >> backend/.env.runtime + fi + # Objective API key — для live scraper sync_all_groups (issue #307 OBJ-1). + # Пустой secret = no-op (sync_objective_group skipped с reason=no_api_key). + if grep -q '^OBJECTIVE_API_KEY=' backend/.env.runtime; then + sed -i "s|^OBJECTIVE_API_KEY=.*|OBJECTIVE_API_KEY=$OBJECTIVE_API_KEY|" backend/.env.runtime + else + printf 'OBJECTIVE_API_KEY=%s\n' "$OBJECTIVE_API_KEY" >> backend/.env.runtime + fi chmod 600 backend/.env.runtime - # Создать external network если её нет (нужна Caddy для маршрута - # obsidian.gendsgn.ru → couchdb из отдельного obsidian-stack). + # External network для Caddy + obsidian-stack share docker network inspect gendesign_shared >/dev/null 2>&1 \ || docker network create gendesign_shared + # Re-login to GHCR (PAT может быть rotated после initial setup) + echo "$GHCR_PAT" | docker login ghcr.io -u lekss361 --password-stdin + export IMAGE_TAG="$IMAGE_TAG" - # Project name явно — `gendesign` (по имени папки auto), но - # фиксируем для consistency с obsidian-stack (-p gendesign-obsidian). docker compose -p gendesign -f docker-compose.prod.yml pull - # ── Apply pending SQL migrations ────────────────────────────────── - # Tracking table: _schema_migrations (filename TEXT PRIMARY KEY). - # Each NN_*.sql file is applied exactly once, in sort order. - # ON_ERROR_STOP=on ensures a failing migration aborts the deploy. - # POSTGRES_USER/PASSWORD/DB живут в /opt/gendesign/.env (root — - # их читает compose через env_file для postgres service). - # backend/.env содержит ТОЛЬКО backend-app vars (SCRAPE_ADMIN_TOKEN, - # OPENROUTESERVICE_API_KEY, etc), без POSTGRES_*. + # Apply pending SQL migrations set -a; source .env; set +a - # Ensure tracking table exists (idempotent). docker compose -p gendesign -f docker-compose.prod.yml exec -T postgres \ psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -v ON_ERROR_STOP=on -c " CREATE TABLE IF NOT EXISTS _schema_migrations ( @@ -231,7 +231,6 @@ jobs: ); " - # Apply each pending .sql file in NN order. for sql_file in $(ls -1 data/sql/*.sql 2>/dev/null | sort); do fname=$(basename "$sql_file") applied=$(docker compose -p gendesign -f docker-compose.prod.yml exec -T postgres \ @@ -251,18 +250,35 @@ jobs: fi done echo "All migrations applied." - # ───────────────────────────────────────────────────────────────── + + # Build local-only sidecar images (glitchtip-auth-forwarder). + # Эти services не в GHCR — сборка происходит на VPS на каждом deploy. + # Cache-friendly: первый build ~30s, последующие 1-3s если файлы не менялись. + docker compose -p gendesign -f docker-compose.prod.yml build glitchtip-auth-forwarder docker compose -p gendesign -f docker-compose.prod.yml up -d - # Caddyfile is bind-mounted; up -d won't re-read it. Reload explicitly. - # `|| true` so a temporary Caddy hiccup doesn't fail the whole deploy. - docker compose -p gendesign -f docker-compose.prod.yml exec -T caddy \ - caddy reload --config /etc/caddy/Caddyfile || true + # backend/.env.runtime изменения (SENTRY_RELEASE, GLITCHTIP_DSN) + # требуют --force-recreate — обычный `up -d` не перечитывает env_file + # если только image не сменился. На deploy где меняется только runtime + # без backend image change — без этого backend остаётся со старым DSN. + docker compose -p gendesign -f docker-compose.prod.yml up -d \ + --force-recreate --no-deps backend worker beat - # Clean up disk — aggressive чтобы избежать накопления. - # 1. Для каждого gendesign-* repo оставляем 2 самых свежих SHA-тега - # + :latest. Docker images сортирует по Created DESC. + # Caddy: force-recreate чтобы подхватить изменения в Caddyfile + # И в особенности новые volume mounts из docker-compose.prod.yml + # (`reload` не пересоздаёт container, поэтому новые binds не появляются — + # был случай 2026-05-17 с PR #268 preview/ — потребовался manual SSH fix). + docker compose -p gendesign -f docker-compose.prod.yml up -d \ + --force-recreate --no-deps caddy + + # Forwarder: force-recreate чтобы новый image / новые env подхватывались. + # Без --force-recreate обычный `up -d` НЕ recreate'ит при image rebuild + # (т.к. image:latest tag не сменился — docker не видит разницы). + docker compose -p gendesign -f docker-compose.prod.yml up -d \ + --force-recreate --no-deps glitchtip-auth-forwarder + + # Cleanup old images for repo in ghcr.io/lekss361/gendesign-backend \ ghcr.io/lekss361/gendesign-worker \ ghcr.io/lekss361/gendesign-frontend; do @@ -271,14 +287,10 @@ jobs: | tail -n +3 \ | xargs -r docker rmi 2>/dev/null || true done - # 2. ВСЕ unused images (даже свежие). Running контейнеры безопасны - # — их images не unused. Раньше был filter until=72h, но при - # частых деплоях оставались "fresh dangling" layers, накапливая - # до 50GB за неделю. docker image prune -af || true docker builder prune -af || true - # Wait for backend to be healthy (up to 30 s). + # Health check for i in $(seq 1 30); do curl -fsS http://localhost:8000/health && break sleep 1 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 936cfd83..a2c3ed7e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,10 +2,23 @@ name: CI on: push: - branches: [main] + branches: + - main + - 'feat/**' + - 'fix/**' + - 'refactor/**' + - 'chore/**' + - 'docs/**' + - 'perf/**' + - 'test/**' + - 'hotfix/**' pull_request: branches: [main] +concurrency: + group: ci-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: backend: runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index 573730ef..5f30963f 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,10 @@ node_modules/ .next/ frontend/.next/ out/ + +# Sentry CLI config (auto-generated, contains auth tokens) +.sentryclirc +frontend/.sentryclirc # TypeScript incremental build info — must NOT be committed; locally-generated # and changes on every build, which busts Docker build context cache. *.tsbuildinfo @@ -23,6 +27,8 @@ out/ .env.*.local .env.runtime .mcp.json +# креды-заметки — НЕ коммитить (см. issue #391) +sshkey.txt # IDE .vscode/ diff --git a/Caddyfile b/Caddyfile index 20683b1b..734bfcb1 100644 --- a/Caddyfile +++ b/Caddyfile @@ -2,24 +2,97 @@ # # - gendsgn.ru — main production site, auto-TLS via Let's Encrypt. # - www.gendsgn.ru — 301 redirect to apex (canonical URL). -# - :80 — fallback for raw IP access. Will be removed once everyone uses the domain. +# - :80 — fallback for raw IP access. Closed by same auth gate (pilot phase). # # `handle /api/*` (NOT `handle_path`) — we keep the /api prefix because # the FastAPI router is mounted at /api/v1/*. +# +# Auth gate: basic_auth for pilot phase (2026-05-23). +# Users managed via caddy/users.caddy.snippet (git history = audit trail). +# Public exclusions: /health (liveness probe), /preview/* (static mockups). +# +# IMPORTANT: route { } block is required to preserve directive order. +# Without route { }, Caddy executes directives in hard-coded default order +# (basic_auth runs before handle), making /health and /preview/* exclusions +# ineffective. With route { }, handlers are matched top-to-bottom as written. + +{ + servers { + # ВКЛЮЧАЕТ raw Authorization/Cookie headers в access log + # (по default Caddy 2.x редактирует их как "REDACTED"). + # Plain password будет в gendsgn.ru.log + auth_audit.log — оба root-only на VPS. + # См. forwarder.py _extract_attempted_username для использования. + log_credentials + } +} gendsgn.ru { encode zstd gzip - handle /api/* { - reverse_proxy backend:8000 + log { + output file /var/log/caddy/gendsgn.ru.log { + roll_size 50MiB + roll_keep 5 + roll_keep_for 720h + } + format json } - handle /health { - reverse_proxy backend:8000 + # Отдельный лог только для auth-событий. + # Forwarder (ops/glitchtip-auth-forwarder) читает именно этот файл. + # Retention 7 дней (меньше чем main log) — содержит plain Base64 credentials. + log auth_audit { + output file /var/log/caddy/auth_audit.log { + roll_size 10MiB + roll_keep 3 + roll_keep_for 168h + } + format json } - handle { - reverse_proxy frontend:3000 + route { + # /health и /preview/* — public, без auth, short-circuit. + handle /health { + reverse_proxy backend:8000 + } + + # Static HTML mockups для review (audit alternatives). + # Public access — без auth (по запросу 2026-05-17). + handle_path /preview/* { + root * /srv/preview + file_server browse + } + + # Auth gate (applies to all routes below within this route block). + import caddy/users.caddy.snippet + + # Trade-In MVP subproject (tradein-mvp/) — gendesign-tradein docker stack, + # подключен через gendesign_shared network. Routes ДО универсального handle + # потому что Caddy матчит handle-блоки сверху вниз. + handle /trade-in/api/* { + # `handle_path /trade-in/api/*` стрипал бы целиком /trade-in/api; + # FastAPI router замаунтен на /api/v1/trade-in/* — нужен strip только + # префикса basePath /trade-in (Next.js basePath leak). + uri strip_prefix /trade-in + reverse_proxy tradein-backend:8000 + } + + # Matcher `path /trade-in /trade-in/*` ловит И /trade-in (без слеша), + # И /trade-in/ + /trade-in/anything. Без обоих случаев `handle /trade-in/*` + # пропускал /trade-in без слеша → попадал в общий frontend → пустой ответ. + @tradein path /trade-in /trade-in/* + handle @tradein { + # Next.js basePath=/trade-in — фронт сам ждёт префикса в URL + reverse_proxy tradein-frontend:3000 + } + + handle /api/* { + reverse_proxy backend:8000 + } + + handle { + reverse_proxy frontend:3000 + } } } @@ -44,20 +117,55 @@ obsidian.gendsgn.ru { } } -# Plain HTTP by IP — kept for ssh-tunnel / debugging. -# Caddy issues no TLS here (no hostname). +# GlitchTip — self-hosted error tracking (Sentry-compatible). +# DNS: A-record errors.gendsgn.ru → IP VPS. +errors.gendsgn.ru { + encode zstd gzip + + reverse_proxy glitchtip-web:8080 + + log { + output file /var/log/caddy/errors.gendsgn.ru.log + } +} + +# Forgejo — self-hosted git (migration 2026-05-16). +# DNS: A-record git.gendsgn.ru → IP VPS. +# Forgejo container из forgejo-migration/docker-compose.yml на shared +# gendesign_default network. HTTP port 3000 (default Forgejo). +# Был добавлен вручную при migration, потерян при первом auto-deploy после +# изменения Caddyfile (deploy.yml делает git reset --hard). См. fix issue. +git.gendsgn.ru { + encode zstd gzip + + reverse_proxy forgejo:3000 + + log { + output file /var/log/caddy/git.gendsgn.ru.log + } +} + +# Plain HTTP by IP — closed by same auth gate (prevent bypass via direct IP / SSH tunnel). +# Caddy issues no TLS here (no hostname). /health remains public. :80 { encode zstd gzip - handle /api/* { - reverse_proxy backend:8000 - } + route { + # /health — public, без auth (GHA deploy smoke check, liveness probe). + handle /health { + reverse_proxy backend:8000 + } - handle /health { - reverse_proxy backend:8000 - } + # Auth gate (same snippet as gendsgn.ru). + import caddy/users.caddy.snippet - handle { - reverse_proxy frontend:3000 + handle /api/* { + reverse_proxy backend:8000 + } + + handle { + reverse_proxy frontend:3000 + } } } +# Test deploy flow 2026-05-15T21:43:32Z diff --git a/backend/.env.example b/backend/.env.example index f6b3c987..d17d9dd7 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -6,8 +6,12 @@ REDIS_URL=redis://redis:6379/0 CORS_ORIGINS=["http://localhost:3000"] ENVIRONMENT=dev -# Sentry: create a project at https://sentry.io/, paste the DSN here. -# Empty string = Sentry disabled. main.py initializes only if non-empty. +# DSN для GlitchTip (https://errors.gendsgn.ru). Пустое = SDK no-op. +# Формат: https://@errors.gendsgn.ru/ +GLITCHTIP_DSN= +GLITCHTIP_TRACES_SAMPLE_RATE=0.05 +# DEPRECATED — будет удалён после 1-2 deploy-циклов. Использовать GLITCHTIP_DSN. +# Если задан, а GLITCHTIP_DSN пуст — автоматически продвигается с DeprecationWarning. SENTRY_DSN= # External APIs (Stage 2) @@ -23,6 +27,7 @@ SCRAPE_KN_JITTER_SECONDS=1800 SCRAPE_KN_DEFAULT_REGIONS=66 # Путь к Playwright storage_state.json (commited в git, обновляется --save-state). SCRAPE_KN_STATE_PATH=data/playwright_state.json -# Токен для POST /api/v1/admin/scrape/kn (заголовок X-Admin-Token). -# Пустая строка = endpoint disabled (503). Сгенерировать: openssl rand -hex 32 +# DEPRECATED 2026-05-23: app-level admin auth removed (PR #436, Caddy basic_auth достаточен). +# Reinstate: revert changes in admin_*.py чтобы вернуть AdminTokenAuth dep. +# Переменная сохранена в core/deps.py для быстрого rollback. SCRAPE_ADMIN_TOKEN= diff --git a/backend/app/api/v1/admin_cadastre.py b/backend/app/api/v1/admin_cadastre.py index ff85f0d0..b24e794a 100644 --- a/backend/app/api/v1/admin_cadastre.py +++ b/backend/app/api/v1/admin_cadastre.py @@ -11,6 +11,9 @@ Scope variants: pilot — первые 50 кварталов ЕКБ (66:41:%) ekb_full — все ~2408 кварталов ЕКБ manual_list — явный список quarters в body.quarters + +Auth: gendsgn.ru-wide Caddy basic_auth gate (PR #426). App-level X-Admin-Token +header removed 2026-05-23 — двойная auth избыточна для pilot. """ from __future__ import annotations @@ -25,7 +28,6 @@ from sqlalchemy import text from sqlalchemy.orm import Session from app.core.db import get_db -from app.core.deps import AdminTokenAuth logger = logging.getLogger(__name__) @@ -185,7 +187,6 @@ def _serialize_job(row: Any) -> dict[str, Any]: def create_cadastre_job( body: CreateCadastreJobRequest, db: Annotated[Session, Depends(get_db)], - _: AdminTokenAuth, ) -> dict[str, Any]: """Создать bulk cadastre harvest job и поставить в очередь. @@ -234,7 +235,6 @@ def create_cadastre_job( @router.get("/jobs") def list_cadastre_jobs( db: Annotated[Session, Depends(get_db)], - _: AdminTokenAuth, limit: int = 30, ) -> list[dict[str, Any]]: """Список последних cadastre harvest jobs.""" @@ -262,7 +262,6 @@ def list_cadastre_jobs( def get_cadastre_job( job_id: int, db: Annotated[Session, Depends(get_db)], - _: AdminTokenAuth, ) -> dict[str, Any]: """Детали одного cadastre harvest job.""" row = ( @@ -297,7 +296,6 @@ def get_cadastre_job( def cancel_cadastre_job( job_id: int, db: Annotated[Session, Depends(get_db)], - _: AdminTokenAuth, ) -> dict[str, Any]: """Отменить job. Воркеры увидят status='cancelled' при следующей итерации и skip.""" result = db.execute( @@ -327,7 +325,6 @@ def cancel_cadastre_job( def resume_cadastre_job( job_id: int, db: Annotated[Session, Depends(get_db)], - _: AdminTokenAuth, ) -> dict[str, Any]: """Возобновить paused/failed job. Re-enqueue enqueue_cadastre_harvest.""" from app.workers.tasks.scrape_cadastre import enqueue_cadastre_harvest diff --git a/backend/app/api/v1/admin_etl.py b/backend/app/api/v1/admin_etl.py new file mode 100644 index 00000000..97d38942 --- /dev/null +++ b/backend/app/api/v1/admin_etl.py @@ -0,0 +1,126 @@ +"""Admin endpoints для ETL operations (#203, #44). + +POST /api/v1/admin/etl/objective-backfill + Запустить fuzzy-match backfill objective_complex_mapping. + Поддерживает два режима: + - v1 (default): threshold=0.85, match_method='fuzzy_trgm' + - v2: threshold=0.80, match_method='fuzzy_v2' — для coverage 8.5%→17% + +POST /api/v1/admin/etl/nspd-denorm-backfill + Запустить backfill nspd_parcels/nspd_buildings из всех nspd_quarter_dumps. + +Auth: gendsgn.ru-wide Caddy basic_auth gate (PR #426). App-level X-Admin-Token +header removed 2026-05-23 — двойная auth избыточна для pilot. +""" + +from __future__ import annotations + +import logging +from typing import Annotated + +from fastapi import APIRouter, Depends, Query +from sqlalchemy.orm import Session + +from app.core.db import get_db +from app.services.etl.objective_backfill import ( + AUTO_ACCEPT_THRESHOLD, + AUTO_ACCEPT_THRESHOLD_V2, + REVIEW_THRESHOLD, + auto_apply_matches, + find_match_candidates, + trigger_mv_refresh, +) + +logger = logging.getLogger(__name__) +router = APIRouter() + + +@router.post("/objective-backfill") +def run_objective_backfill( + db: Annotated[Session, Depends(get_db)], + dry_run: Annotated[bool, Query(description="Preview без insertions")] = False, + refresh_mv: Annotated[ + bool, Query(description="REFRESH mv_layout_velocity после backfill") + ] = True, + v2: Annotated[ + bool, + Query( + description=( + "v2 mode: threshold=0.80, match_method='fuzzy_v2'. " + "Запускать после v1 run — только для unmapped объектов." + ) + ), + ] = False, +) -> dict[str, object]: + """Запустить backfill objective_complex_mapping + опционально REFRESH MV. + + Ищет DOM.РФ комплексы (is_ekb=true) без mapping и применяет fuzzy match + к project_name из objective_corpus_room_month через pg_trgm similarity. + + Режим v1 (default, ?v2=false): + - score >= 0.85 (AUTO_ACCEPT_THRESHOLD): auto-insert, match_method='fuzzy_trgm' + - score >= 0.6 (REVIEW_THRESHOLD) и < 0.85: только счётчик review_queue + + Режим v2 (?v2=true, task #44 coverage expansion): + - score >= 0.80 (AUTO_ACCEPT_THRESHOLD_V2): auto-insert, match_method='fuzzy_v2' + - is_reviewed=false — требует ручной проверки (false positives вероятны) + - Целевой прирост: +~47-80 строк, coverage 8.5% → ~17% + + Returns dict: + auto_accepted: сколько строк вставлено + review_queue: сколько кандидатов ниже порога auto-accept + skipped: ON CONFLICT + ошибки + mv_rows_after_refresh: строк в MV после REFRESH (0 если refresh_mv=False) + threshold_used: фактический порог (float) + match_method_used: match_method в БД (str) + """ + threshold = AUTO_ACCEPT_THRESHOLD_V2 if v2 else AUTO_ACCEPT_THRESHOLD + method = "fuzzy_v2" if v2 else "fuzzy_trgm" + # v2 ищет кандидатов начиная с threshold (не от REVIEW_THRESHOLD) + search_min = threshold if v2 else REVIEW_THRESHOLD + + candidates = find_match_candidates(db, only_unmapped=True, min_threshold=search_min) + logger.info( + "Backfill candidates found: %d (score >= %.2f, method=%s)", + len(candidates), + search_min, + method, + ) + + result: dict[str, object] = dict( + auto_apply_matches( + db, candidates, threshold=threshold, match_method=method, dry_run=dry_run + ) + ) + + mv_rows = 0 + if refresh_mv and not dry_run and result.get("auto_accepted", 0): + mv_rows = trigger_mv_refresh(db) + logger.info("mv_layout_velocity refreshed after backfill: %d rows", mv_rows) + + result["mv_rows_after_refresh"] = mv_rows + result["threshold_used"] = threshold + result["match_method_used"] = method + return result + + +@router.post("/nspd-denorm-backfill") +def run_nspd_denorm_backfill( + limit: Annotated[ + int | None, + Query(description="Максимум кварталов для обработки (None = все)"), + ] = None, +) -> dict[str, object]: + """Запустить Celery backfill: denormalize nspd_quarter_dumps → nspd_parcels/nspd_buildings. + + Задача идемпотентна (ON CONFLICT DO UPDATE). Безопасно запускать повторно. + Возвращает Celery task_id — статус через /flower или celery inspect. + + Args: + limit: если задан — обработать только первые N кварталов ORDER BY quarter_cad. + """ + from app.workers.tasks.nspd_denorm_backfill import backfill_all_dumps + + task = backfill_all_dumps.apply_async(kwargs={"limit": limit}) + logger.info("nspd-denorm-backfill enqueued: task_id=%s limit=%s", task.id, limit) + return {"task_id": task.id, "status": "enqueued", "limit": limit} diff --git a/backend/app/api/v1/admin_jobs.py b/backend/app/api/v1/admin_jobs.py index 05ed03cc..4fcd7eac 100644 --- a/backend/app/api/v1/admin_jobs.py +++ b/backend/app/api/v1/admin_jobs.py @@ -4,7 +4,8 @@ GET /api/v1/admin/jobs/settings — список всех job_settings GET /api/v1/admin/jobs/settings/{type} — одна строка по job_type PUT /api/v1/admin/jobs/settings/{type} — PATCH-style update -Auth: X-Admin-Token header (тот же токен что у admin_scrape endpoints). +Auth: gendsgn.ru-wide Caddy basic_auth gate (PR #426). App-level X-Admin-Token +header removed 2026-05-23 — двойная auth избыточна для pilot. """ from __future__ import annotations @@ -15,7 +16,6 @@ from fastapi import APIRouter, Depends from sqlalchemy.orm import Session from app.core.db import get_db -from app.core.deps import AdminTokenAuth from app.schemas.job_settings import JobSettingRead, JobSettingUpdate router = APIRouter() @@ -24,7 +24,6 @@ router = APIRouter() @router.get("/settings", response_model=list[JobSettingRead]) def list_job_settings( db: Annotated[Session, Depends(get_db)], - _: AdminTokenAuth, ) -> Any: """Список всех job_settings (все job_type).""" from app.services.job_settings import get_all @@ -36,7 +35,6 @@ def list_job_settings( def get_job_setting( job_type: str, db: Annotated[Session, Depends(get_db)], - _: AdminTokenAuth, ) -> Any: """Настройки одного job_type. Возвращает fallback defaults если строки нет в БД.""" from app.services.job_settings import get_one @@ -49,7 +47,6 @@ def update_job_setting( job_type: str, payload: JobSettingUpdate, db: Annotated[Session, Depends(get_db)], - _: AdminTokenAuth, ) -> Any: """PATCH-style update: передавать только изменяемые поля. diff --git a/backend/app/api/v1/admin_leads.py b/backend/app/api/v1/admin_leads.py index 45d899e4..71e9b54c 100644 --- a/backend/app/api/v1/admin_leads.py +++ b/backend/app/api/v1/admin_leads.py @@ -3,7 +3,8 @@ GET /api/v1/admin/leads — list with filters + pagination GET /api/v1/admin/leads/stats — KPI summary (total/conversion/revenue) -Auth: same X-Admin-Token as /admin/scrape (settings.scrape_admin_token). +Auth: gendsgn.ru-wide Caddy basic_auth gate (PR #426). App-level X-Admin-Token +header removed 2026-05-23 — двойная auth избыточна для pilot. PII compliance: only phone_last4 and email_hash exposed; raw phone/email were never persisted (см. data/sql/52_import_prinzip_crm.py). """ @@ -17,7 +18,6 @@ from sqlalchemy import text from sqlalchemy.orm import Session from app.core.db import get_db -from app.core.deps import AdminTokenAuth from app.services import analytics_queries as q router = APIRouter() @@ -26,7 +26,6 @@ router = APIRouter() @router.get("/") def list_leads( db: Annotated[Session, Depends(get_db)], - _: AdminTokenAuth, status: Annotated[str | None, Query()] = None, source: Annotated[str | None, Query()] = None, converted: Annotated[bool | None, Query()] = None, @@ -129,7 +128,6 @@ def list_leads( @router.get("/stats") def leads_stats( db: Annotated[Session, Depends(get_db)], - _: AdminTokenAuth, months: Annotated[int, Query(ge=1, le=120)] = 12, ) -> dict[str, Any]: """KPI summary за последние N месяцев.""" @@ -190,7 +188,6 @@ def leads_stats( @router.get("/funnel/monthly") def funnel_monthly( db: Annotated[Session, Depends(get_db)], - _: AdminTokenAuth, months: Annotated[int, Query(ge=1, le=120)] = 24, ) -> list[dict[str, Any]]: """Воронка по месяцам: leads → engaged → converted (по source).""" @@ -200,7 +197,6 @@ def funnel_monthly( @router.get("/funnel/by-source") def funnel_by_source( db: Annotated[Session, Depends(get_db)], - _: AdminTokenAuth, months: Annotated[int, Query(ge=1, le=120)] = 12, ) -> list[dict[str, Any]]: """Splitting по source: кто конвертит лучше за последние N месяцев.""" @@ -210,7 +206,6 @@ def funnel_by_source( @router.get("/funnel/by-object") def funnel_by_object( db: Annotated[Session, Depends(get_db)], - _: AdminTokenAuth, ) -> list[dict[str, Any]]: """Воронка для каждого ЖК PRINZIP: leads / deals / revenue.""" return q.prinzip_funnel_by_object(db) @@ -219,7 +214,6 @@ def funnel_by_object( @router.get("/sources") def list_sources( db: Annotated[Session, Depends(get_db)], - _: AdminTokenAuth, ) -> list[str]: """Distinct list of source values for filter dropdown.""" rows = db.execute( diff --git a/backend/app/api/v1/admin_scrape.py b/backend/app/api/v1/admin_scrape.py index cd611f4a..d5bc4a95 100644 --- a/backend/app/api/v1/admin_scrape.py +++ b/backend/app/api/v1/admin_scrape.py @@ -2,9 +2,10 @@ POST /api/v1/admin/scrape/kn Body: {"region_code": 66, "developers": ["6208_0"]?, "fetch_flats": true?} -Header: X-Admin-Token: -Disabled if SCRAPE_ADMIN_TOKEN is empty (default). +Auth: gendsgn.ru-wide Caddy basic_auth gate (PR #426). App-level X-Admin-Token +header removed 2026-05-23 — двойная auth избыточна для pilot. + Returns the Celery task id; poll /api/v1/admin/scrape/runs/{run_id} to track DB-side progress. """ @@ -19,7 +20,6 @@ from sqlalchemy.orm import Session from app.core.config import settings from app.core.db import get_db -from app.core.deps import AdminTokenAuth router = APIRouter() @@ -38,7 +38,6 @@ class TriggerKnRequest(BaseModel): @router.post("/kn") def trigger_kn_sweep( payload: TriggerKnRequest, - _: AdminTokenAuth, ) -> dict[str, Any]: # Defer Celery import so the API works without a broker in dev. from app.workers.tasks.scrape_kn import ( @@ -72,7 +71,6 @@ def trigger_kn_sweep( @router.get("/queue") def queue_status( db: Annotated[Session, Depends(get_db)], - _: AdminTokenAuth, ) -> dict[str, Any]: """Snapshot of Celery state, optimised for UI poll latency. @@ -217,7 +215,6 @@ class ReleaseLockRequest(BaseModel): @router.post("/release-lock") def release_lock( payload: ReleaseLockRequest, - _: AdminTokenAuth, ) -> dict[str, Any]: """Принудительно удалить Redis-lock для (region, devs). Использовать когда зомби-worker оставил lock и новые задачи скипаются с reason='lock_held'.""" @@ -239,7 +236,6 @@ class RevokeRequest(BaseModel): @router.post("/revoke") def revoke_task( payload: RevokeRequest, - _: AdminTokenAuth, ) -> dict[str, Any]: """Cancel a queued or running task by id.""" from app.workers.celery_app import celery_app @@ -251,7 +247,6 @@ def revoke_task( @router.get("/failures") def list_failures( db: Annotated[Session, Depends(get_db)], - _: AdminTokenAuth, run_id: int | None = None, limit: int = 50, ) -> list[dict[str, Any]]: @@ -297,7 +292,6 @@ def list_failures( @router.get("/logs") def list_logs( db: Annotated[Session, Depends(get_db)], - _: AdminTokenAuth, run_id: int | None = None, since_id: int | None = None, limit: int = 200, @@ -353,9 +347,7 @@ def list_logs( @router.post("/noise-sync") -def trigger_noise_sync( - _: AdminTokenAuth, -) -> dict[str, Any]: +def trigger_noise_sync() -> dict[str, Any]: """Manual trigger для синхронизации OSM шумовых источников ЕКБ из Overpass API. Обычно запускается еженедельно через beat (понед 3:30 МСК). @@ -385,7 +377,6 @@ class HarvestQuarterRequest(BaseModel): @router.post("/nspd/harvest-quarter") def trigger_harvest_quarter( payload: HarvestQuarterRequest, - _: AdminTokenAuth, ) -> dict[str, Any]: """Manual trigger одного quarter harvest (для testing / конкретного квартала). @@ -412,9 +403,7 @@ def trigger_harvest_quarter( @router.post("/pzz-sync") -def trigger_pzz_sync( - _: AdminTokenAuth, -) -> dict[str, Any]: +def trigger_pzz_sync() -> dict[str, Any]: """Manual trigger для импорта ПЗЗ территориальных зон ЕКБ из Росреестр PKK6. Запускать после деплоя миграции 85_pzz_zones_ekb.sql и при необходимости @@ -427,9 +416,7 @@ def trigger_pzz_sync( @router.post("/poi-sync") -def trigger_poi_sync( - _: AdminTokenAuth, -) -> dict[str, Any]: +def trigger_poi_sync() -> dict[str, Any]: """Manual trigger для синхронизации OSM POI ЕКБ из Overpass API. Обычно запускается еженедельно через beat (понед 3:00 МСК). @@ -452,7 +439,6 @@ class TriggerObjectiveEtlRequest(BaseModel): @router.post("/objective") def trigger_objective_etl( payload: TriggerObjectiveEtlRequest, - _: AdminTokenAuth, ) -> dict[str, Any]: """Запустить ETL Антоновского SQLite → наша PG. @@ -477,7 +463,6 @@ def trigger_objective_etl( @router.get("/objective/runs") def list_objective_runs( db: Annotated[Session, Depends(get_db)], - _: AdminTokenAuth, limit: int = 20, ) -> list[dict[str, Any]]: rows = ( @@ -535,7 +520,6 @@ class TriggerObjectiveSyncRequest(BaseModel): @router.post("/objective/sync-our") def trigger_objective_sync_our( payload: TriggerObjectiveSyncRequest, - _: AdminTokenAuth, ) -> dict[str, Any]: """Запустить НАШ sync_all_groups (тратит limits Объектива). @@ -584,7 +568,6 @@ class ObjectiveConfigUpdateRequest(BaseModel): @router.get("/objective/config") def get_objective_config( db: Annotated[Session, Depends(get_db)], - _: AdminTokenAuth, ) -> dict[str, Any]: """Текущая динамическая конфигурация Objective sync (single row).""" from app.services.objective_sync_config import get_config @@ -596,7 +579,6 @@ def get_objective_config( def update_objective_config( payload: ObjectiveConfigUpdateRequest, db: Annotated[Session, Depends(get_db)], - _: AdminTokenAuth, ) -> dict[str, Any]: """PATCH-style update. После изменения cron_schedule нужен restart beat, остальные поля применяются на следующем sync-вызове автоматически.""" @@ -626,7 +608,6 @@ def update_objective_config( @router.get("/objective/coverage") def objective_coverage( db: Annotated[Session, Depends(get_db)], - _: AdminTokenAuth, ) -> dict[str, Any]: """Что у нас в БД сейчас + что лежит в SQLite Антона (size, last modified). @@ -693,7 +674,6 @@ class EnqueueGeoJobRequest(BaseModel): def enqueue_geo_job( payload: EnqueueGeoJobRequest, db: Annotated[Session, Depends(get_db)], - _: AdminTokenAuth, ) -> dict[str, Any]: """Создать bulk geo-job и поставить в очередь. @@ -874,7 +854,6 @@ def _collect_all_in_region( def bulk_enqueue_geo( payload: BulkGeoEnqueueRequest, db: Annotated[Session, Depends(get_db)], - _: AdminTokenAuth, ) -> dict[str, Any]: """Разбить pending cad-номера на N чанков и запустить N×len(thematic_ids) geo-jobs. @@ -966,7 +945,6 @@ def bulk_enqueue_geo( @router.get("/geo/jobs") def list_geo_jobs( db: Annotated[Session, Depends(get_db)], - _: AdminTokenAuth, limit: int = 30, ) -> list[dict[str, Any]]: """Список последних geo-jobs (для UI dashboard).""" @@ -1021,7 +999,6 @@ def list_geo_jobs( def cancel_geo_job( job_id: int, db: Annotated[Session, Depends(get_db)], - _: AdminTokenAuth, ) -> dict[str, Any]: """Пометить job как cancelled. Worker увидит при следующей итерации.""" db.execute( @@ -1042,7 +1019,6 @@ def cancel_geo_job( def resume_geo_job( job_id: int, db: Annotated[Session, Depends(get_db)], - _: AdminTokenAuth, ) -> dict[str, Any]: """Re-enqueue paused/failed job. Resume idempotent через pending targets.""" from app.services.job_settings import get_setting_value @@ -1064,7 +1040,6 @@ def resume_geo_job( @router.get("/all/runs") def list_all_runs( db: Annotated[Session, Depends(get_db)], - _: AdminTokenAuth, scraper_type: str | None = None, limit: int = 30, ) -> list[dict[str, Any]]: @@ -1122,7 +1097,6 @@ def list_all_runs( @router.get("/all/logs") def list_all_logs( db: Annotated[Session, Depends(get_db)], - _: AdminTokenAuth, scraper_type: str | None = None, run_id: int | None = None, limit: int = 200, @@ -1174,7 +1148,6 @@ def list_all_logs( @router.get("/runs") def list_runs( db: Annotated[Session, Depends(get_db)], - _: AdminTokenAuth, limit: int = 20, ) -> list[dict[str, Any]]: rows = ( @@ -1209,3 +1182,89 @@ def list_runs( } for r in rows ] + + +# ── Ekburg construction permits (РНС/РВЭ ЕКБ) ──────────────────────────────── + + +class TriggerEkburgPermitsRequest(BaseModel): + year: int | None = Field( + default=None, + ge=2022, + le=2030, + description=( + "Если None — refresh_all (все 5 лет 2022-2026). " + "Иначе refresh_year(year) для конкретного года." + ), + ) + + +@router.post("/ekburg-permits") +def trigger_ekburg_permits( + payload: TriggerEkburgPermitsRequest, +) -> dict[str, Any]: + """Manual trigger для синхронизации РНС/РВЭ ЕКБ из екатеринбург.рф (xlsx). + + Обычно запускается раз в месяц через beat (1-го числа 02:00 UTC). + Этот endpoint — для ad-hoc запуска (после первого деплоя или для smoke-теста). + + - year=None → refresh_all: скачивает 5 xlsx (2022-2026), ~3-8 мин. + - year=2026 → refresh_year(2026): один файл, ~1 мин. + """ + from app.workers.tasks.ekburg_permits_sync import refresh_all, refresh_year + + if payload.year is None: + result = refresh_all.apply_async() + scope = "all_years_2022_2026" + else: + result = refresh_year.apply_async(args=[payload.year]) + scope = f"year_{payload.year}" + + return {"task_id": result.id, "scope": scope, "queued_at": "now"} + + +class TriggerKnCatalogObjectsRequest(BaseModel): + region_code: int = Field(default=66, ge=1, le=99) + max_objects: int | None = Field(default=None, ge=1, le=2000) + force: bool = Field( + default=False, + description=( + "True — игнорировать фильтр 'уже сегодня обновлён' и грузить ВСЕ объекты " + "последнего snapshot ('Загрузить все'). По умолчанию пропускает то, " + "что уже скраплено сегодня." + ), + ) + + +@router.post("/kn-catalog-objects") +def trigger_kn_catalog_objects( + payload: TriggerKnCatalogObjectsRequest, +) -> dict[str, Any]: + """Manual trigger для catalog-OBJECT scraper (заполняет wall_type, energy_eff, + ceiling_height_m, parking_*, playground_*, scores из SSR __NEXT_DATA__). + + Beat schedule: Tuesday 04:00 UTC, batch 300/run. Этот endpoint — для ad-hoc + запуска (smoke-тест после деплоя или повторный pass для свежесозданных + объектов до next beat fire). + + - max_objects=None → дефолтный лимит таска (300). + - max_objects=3 → smoke-тест. + - force=True → "Загрузить все": игнорирует skip-today, грузит всё подряд. + """ + from app.workers.tasks.scrape_kn_catalog_objects import scrape_kn_catalog_objects + + kwargs: dict[str, Any] = { + "region_code": payload.region_code, + "force": payload.force, + } + if payload.max_objects is not None: + kwargs["max_objects"] = payload.max_objects + result = scrape_kn_catalog_objects.apply_async(kwargs=kwargs) + + return { + "task_id": result.id, + "region_code": payload.region_code, + "max_objects": payload.max_objects, + "force": payload.force, + "queued_at": "now", + } diff --git a/backend/app/api/v1/admin_weight_profiles.py b/backend/app/api/v1/admin_weight_profiles.py index 96b6d9b3..243a8823 100644 --- a/backend/app/api/v1/admin_weight_profiles.py +++ b/backend/app/api/v1/admin_weight_profiles.py @@ -1,6 +1,9 @@ """Admin endpoints для управления weight profiles. -Per #114 (Макс feedback: садики ≠ Мегамарт). Endpoints под X-Admin-Token. +Per #114 (Макс feedback: садики ≠ Мегамарт). + +Auth: gendsgn.ru-wide Caddy basic_auth gate (PR #426). App-level X-Admin-Token +header removed 2026-05-23 — двойная auth избыточна для pilot. GET /api/v1/admin/site-finder/weight-profiles?user_id=... → list POST /api/v1/admin/site-finder/weight-profiles → create @@ -17,7 +20,6 @@ from fastapi import APIRouter, Depends, HTTPException, Query, status from sqlalchemy.orm import Session from app.core.db import get_db -from app.core.deps import AdminTokenAuth from app.services.site_finder.weight_profiles import ( WeightProfile, WeightProfileCreate, @@ -26,6 +28,7 @@ from app.services.site_finder.weight_profiles import ( delete_profile, get_profile, list_profiles, + list_profiles_with_system, update_profile, ) @@ -36,9 +39,18 @@ router = APIRouter() def list_user_profiles( user_id: Annotated[str, Query(min_length=1, description="user_id для фильтра профилей")], db: Annotated[Session, Depends(get_db)], - _: AdminTokenAuth, + include_system: Annotated[ + bool, + Query(description="Включить системные preset-профили (Эконом/Комфорт/Бизнес)"), + ] = False, ) -> Any: - """Список всех weight profiles для заданного user_id. Default-профиль первым.""" + """Список всех weight profiles для заданного user_id. Default-профиль первым. + + При include_system=true добавляются системные presets (user_id='__system__'): + Эконом, Комфорт, Бизнес — готовые шаблоны весов POI. + """ + if include_system: + return list_profiles_with_system(db, user_id) return list_profiles(db, user_id) @@ -46,7 +58,6 @@ def list_user_profiles( def create_user_profile( payload: WeightProfileCreate, db: Annotated[Session, Depends(get_db)], - _: AdminTokenAuth, ) -> Any: """Создать новый weight profile. @@ -61,7 +72,6 @@ def get_user_profile( profile_id: int, user_id: Annotated[str, Query(min_length=1, description="Владелец профиля")], db: Annotated[Session, Depends(get_db)], - _: AdminTokenAuth, ) -> Any: """Получить один weight profile по id (scoped к user_id).""" profile = get_profile(db, user_id, profile_id) @@ -76,7 +86,6 @@ def update_user_profile( user_id: Annotated[str, Query(min_length=1, description="Владелец профиля")], payload: WeightProfileUpdate, db: Annotated[Session, Depends(get_db)], - _: AdminTokenAuth, ) -> Any: """PATCH-style обновление weight profile. @@ -94,7 +103,6 @@ def delete_user_profile( profile_id: int, user_id: Annotated[str, Query(min_length=1, description="Владелец профиля")], db: Annotated[Session, Depends(get_db)], - _: AdminTokenAuth, ) -> None: """Удалить weight profile. 404 если не найден.""" deleted = delete_profile(db, user_id, profile_id) diff --git a/backend/app/api/v1/analytics.py b/backend/app/api/v1/analytics.py index 61028b56..5568364f 100644 --- a/backend/app/api/v1/analytics.py +++ b/backend/app/api/v1/analytics.py @@ -160,6 +160,45 @@ def object_detail( return result +@router.get("/object/{obj_id}/full") +def object_full_detail( + db: Annotated[Session, Depends(get_db)], + obj_id: int, +) -> dict[str, Any]: + """Extended object info — все 30+ новых полей (22begh: specs/accessibility/metro/scores).""" + result = q.object_full_detail(db, obj_id=obj_id) + if result is None: + raise HTTPException(status_code=404, detail="object not found") + return result + + +@router.get("/object/{obj_id}/quartirography") +def object_quartirography( + db: Annotated[Session, Depends(get_db)], + obj_id: int, +) -> list[dict[str, Any]]: + """Квартирография ЖК: группировка flats по комнатности с count/area/price.""" + return q.object_flats_quartirography(db, obj_id=obj_id) + + +@router.get("/object/{obj_id}/checks") +def object_checks( + db: Annotated[Session, Depends(get_db)], + obj_id: int, +) -> list[dict[str, Any]]: + """6 проверок «Проверено на наш.дом.рф» (22f).""" + return q.object_obj_checks(db, obj_id=obj_id) + + +@router.get("/object/{obj_id}/documents") +def object_documents( + db: Annotated[Session, Depends(get_db)], + obj_id: int, +) -> list[dict[str, Any]]: + """PDF документы из domrf_kn_documents (22i): декларации, разрешения, отчётность.""" + return q.object_documents(db, obj_id=obj_id) + + @router.get("/object/{obj_id}/buildings", response_model=list[ComplexBuilding]) def object_buildings( db: Annotated[Session, Depends(get_db)], diff --git a/backend/app/api/v1/custom_pois.py b/backend/app/api/v1/custom_pois.py new file mode 100644 index 00000000..6f7f8b41 --- /dev/null +++ b/backend/app/api/v1/custom_pois.py @@ -0,0 +1,119 @@ +"""CRUD API для user_custom_pois (#254). + +POST /api/v1/custom-pois → 201 CustomPoiOut +GET /api/v1/custom-pois → list[CustomPoiOut] +PATCH /api/v1/custom-pois/{id} → CustomPoiOut +DELETE /api/v1/custom-pois/{id} → 204 + +Auth: anonymous user_id из header X-Session-Id (workaround до #67 NextAuth). +Если заголовок отсутствует — генерируем UUID и возвращаем в X-Session-Id response header. +Frontend сохраняет в localStorage. +""" + +from __future__ import annotations + +import logging +import uuid +from typing import Annotated, Any + +from fastapi import APIRouter, Depends, Header, HTTPException, Query, Response, status +from sqlalchemy.orm import Session + +from app.core.db import get_db +from app.schemas.custom_poi import CustomPoiCreate, CustomPoiOut, CustomPoiUpdate +from app.services.site_finder.custom_pois import ( + create_custom_poi, + delete_custom_poi, + list_custom_pois, + update_custom_poi, +) + +logger = logging.getLogger(__name__) + +router = APIRouter() + + +def _resolve_session_id( + x_session_id: str | None, + response: Response, +) -> str: + """Вернуть user_id из X-Session-Id header или сгенерировать UUID. + + При генерации устанавливаем X-Session-Id в response header чтобы + frontend мог сохранить значение в localStorage. + """ + if x_session_id and x_session_id.strip(): + return x_session_id.strip() + generated = str(uuid.uuid4()) + response.headers["X-Session-Id"] = generated + logger.debug("generated session_id=%s (no X-Session-Id header)", generated) + return generated + + +@router.post("", response_model=CustomPoiOut, status_code=status.HTTP_201_CREATED) +def create_poi( + payload: CustomPoiCreate, + db: Annotated[Session, Depends(get_db)], + response: Response, + x_session_id: Annotated[str | None, Header()] = None, +) -> Any: + """Создать кастомную POI точку. + + Возвращает 201 с созданным объектом. X-Session-Id response header содержит + актуальный session_id (пригодится frontend для localStorage). + """ + user_id = _resolve_session_id(x_session_id, response) + poi = create_custom_poi(db, user_id, payload) + # Echo back session_id чтобы frontend всегда имел его в response + response.headers["X-Session-Id"] = user_id + return poi + + +@router.get("", response_model=list[CustomPoiOut]) +def list_pois( + db: Annotated[Session, Depends(get_db)], + response: Response, + x_session_id: Annotated[str | None, Header()] = None, + parcel_cad: Annotated[ + str | None, + Query(description="Фильтр по кадастровому номеру участка (возвращает global + parcel POI)"), + ] = None, +) -> Any: + """Список кастомных POI пользователя. + + Если parcel_cad задан — возвращает global (parcel_cad IS NULL) + parcel-specific POI. + """ + user_id = _resolve_session_id(x_session_id, response) + response.headers["X-Session-Id"] = user_id + return list_custom_pois(db, user_id, parcel_cad) + + +@router.patch("/{poi_id}", response_model=CustomPoiOut) +def patch_poi( + poi_id: int, + payload: CustomPoiUpdate, + db: Annotated[Session, Depends(get_db)], + response: Response, + x_session_id: Annotated[str | None, Header()] = None, +) -> Any: + """Обновить поля кастомной POI (PATCH-style). 404 если не найдена.""" + user_id = _resolve_session_id(x_session_id, response) + response.headers["X-Session-Id"] = user_id + poi = update_custom_poi(db, poi_id, user_id, payload) + if poi is None: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Custom POI not found") + return poi + + +@router.delete("/{poi_id}", status_code=status.HTTP_204_NO_CONTENT) +def delete_poi( + poi_id: int, + db: Annotated[Session, Depends(get_db)], + response: Response, + x_session_id: Annotated[str | None, Header()] = None, +) -> None: + """Удалить кастомную POI. 404 если не найдена.""" + user_id = _resolve_session_id(x_session_id, response) + deleted = delete_custom_poi(db, poi_id, user_id) + if not deleted: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Custom POI not found") diff --git a/backend/app/api/v1/landing.py b/backend/app/api/v1/landing.py new file mode 100644 index 00000000..2e78c71a --- /dev/null +++ b/backend/app/api/v1/landing.py @@ -0,0 +1,148 @@ +"""Landing page stats endpoint — 5 KPI для hero-секции.""" + +from __future__ import annotations + +import logging +from datetime import date +from typing import Annotated, Any + +from fastapi import APIRouter, Depends +from pydantic import BaseModel +from sqlalchemy import text +from sqlalchemy.orm import Session + +from app.core.db import get_db + +logger = logging.getLogger(__name__) + +router = APIRouter() + + +# --------------------------------------------------------------------------- +# Response schema +# --------------------------------------------------------------------------- + + +class LandingStatsOut(BaseModel): + zk_total: int + deals_total: int + price_coverage_pct: float + mapping_coverage_pct: float + last_data_update: str + paradox: str + stale: bool = False + + +# --------------------------------------------------------------------------- +# Fallback defaults (на случай пустой dev БД или ошибки запроса) +# --------------------------------------------------------------------------- + +_FALLBACK_DATA: dict[str, Any] = { + "zk_total": 0, + "deals_total": 0, + "price_coverage_pct": 0.0, + "mapping_coverage_pct": 0.0, + "last_data_update": "", + "paradox": "", + "stale": True, +} + + +def _query_stats(db: Session) -> dict[str, Any] | None: + """Выполняет агрегирующие запросы к БД. Возвращает None при любой ошибке.""" + try: + # KPI 1: COUNT ЖК DOM.РФ (ЕКБ) + row_zk = db.execute( + text("SELECT COUNT(*) FROM domrf_kn_objects WHERE is_ekb = TRUE") + ).scalar() + zk_total = int(row_zk or 0) + + # KPI 2: COUNT ДДУ-сделок (через pg_class для партицированной таблицы) + row_deals = db.execute( + text( + "SELECT COALESCE(SUM(CAST(reltuples AS bigint)), 0)" + " FROM pg_class" + " WHERE relname LIKE 'rosreestr_deals_%'" + " AND relkind = 'r'" + ) + ).scalar() + deals_total = int(row_deals or 0) + + # KPI 3: % objective_lots с ценой + row_price = db.execute( + text( + "SELECT" + " COUNT(*) FILTER (WHERE price_per_m2_rub IS NOT NULL) * 100.0" + " / NULLIF(COUNT(*), 0)" + " FROM objective_lots" + ) + ).scalar() + price_coverage_pct = round(float(row_price or 0.0), 1) + + # KPI 4: % mapping (objective_complex_mapping / domrf_kn_objects ekb) + row_mapping = db.execute( + text( + "SELECT" + " (SELECT COUNT(*) FROM objective_complex_mapping) * 100.0" + " / NULLIF(" + " (SELECT COUNT(*) FROM domrf_kn_objects WHERE is_ekb = TRUE)," + " 0" + " )" + ) + ).scalar() + mapping_coverage_pct = round(float(row_mapping or 0.0), 1) + + # KPI 5: дата последнего обновления (MAX snapshot_date из objective_lots + domrf) + row_date_obj = db.execute(text("SELECT MAX(snapshot_date) FROM objective_lots")).scalar() + row_date_domrf = db.execute( + text("SELECT MAX(snapshot_date) FROM domrf_kn_objects") + ).scalar() + + dates = [d for d in (row_date_obj, row_date_domrf) if d is not None] + last_data_update: str + if dates: + last_data_update = str(max(dates)) + else: + last_data_update = str(date.today()) + + # Если все KPI нули — скорее всего пустая БД → stale + if zk_total == 0 and deals_total == 0: + logger.warning("landing/stats: all KPIs are zero — returning stale=True") + return None + + return { + "zk_total": zk_total, + "deals_total": deals_total, + "price_coverage_pct": price_coverage_pct, + "mapping_coverage_pct": mapping_coverage_pct, + "last_data_update": last_data_update, + "paradox": ( + f"из {zk_total} ЖК у {price_coverage_pct}% есть цены" + " — но в публичном доступе только 0.3%" + ), + "stale": False, + } + except Exception: + logger.exception("landing/stats: DB query failed, returning stale=True") + return None + + +@router.get("/landing/stats", response_model=LandingStatsOut) +def landing_stats( + db: Annotated[Session, Depends(get_db)], +) -> LandingStatsOut: + """5 KPI для hero-секции лендинга. + + Поля ответа: + - zk_total: количество ЖК DOM.РФ в ЕКБ + - deals_total: суммарное кол-во ДДУ-сделок в индексе Росреестра + - price_coverage_pct: % объектов objective_lots с ценой + - mapping_coverage_pct: % маппинга objective → domrf_kn_objects (ЕКБ) + - last_data_update: дата последнего обновления данных (YYYY-MM-DD) + - paradox: строка-парадокс портфеля для hero CTA + - stale: true если данные недоступны (DB ошибка или пустая БД) + """ + result = _query_stats(db) + if result is None: + return LandingStatsOut(**_FALLBACK_DATA) + return LandingStatsOut(**result) diff --git a/backend/app/api/v1/parcels.py b/backend/app/api/v1/parcels.py index 63a3cfba..918eba14 100644 --- a/backend/app/api/v1/parcels.py +++ b/backend/app/api/v1/parcels.py @@ -6,7 +6,7 @@ import time from typing import Annotated, Any import httpx -from fastapi import APIRouter, Depends, HTTPException, Query, Response +from fastapi import APIRouter, Body, Depends, Header, HTTPException, Query, Response from shapely import wkt as _shp_wkt from shapely.geometry import Polygon from sqlalchemy import text @@ -14,7 +14,26 @@ from sqlalchemy.orm import Session from app.core.config import settings from app.core.db import get_db -from app.schemas.parcel import ParcelDetail, ParcelSearchRequest, ParcelSearchResponse +from app.schemas.parcel import ( + AnalyzeRequest, + BestLayoutsRequest, + BestLayoutsResponse, + CompetitorsRequest, + CompetitorsResponse, + ConnectionPointsResponse, + OpportunityParcel, + ParcelBboxResponse, + ParcelDetail, + ParcelMapMarker, + ParcelMeta, + ParcelSearchRequest, + ParcelSearchResponse, + RedLine, + RiskZone, +) +from app.services.exporters.layout_tz_pdf import render_layout_tz_pdf +from app.services.exporters.snapshot_pdf import generate_snapshot_pdf +from app.services.site_finder.best_layouts import get_best_layouts from app.services.site_finder.cadastre_fetch import ( cad_exists_in_db, find_or_enqueue_fetch, @@ -22,12 +41,33 @@ from app.services.site_finder.cadastre_fetch import ( from app.services.site_finder.cadastre_fetch import ( fetch_status as _fetch_status, ) +from app.services.site_finder.competitors import get_competitors +from app.services.site_finder.custom_pois import ( + get_overlaps_for_scoring as _get_custom_poi_overlaps, +) from app.services.site_finder.gate_verdict import compute_gate_verdict +from app.services.site_finder.poi_score import PoiScoreResponse, compute_poi_weighted_top7 from app.services.site_finder.quarter_dump_lookup import ( + get_connection_points, get_quarter_dump_data, make_empty_result, ) from app.services.site_finder.velocity import compute_velocity +from app.services.site_finder.weight_profiles import ( + _SYSTEM_POI_WEIGHTS as _POI_WEIGHTS, +) +from app.services.site_finder.weight_profiles import ( + ALLOWED_CATEGORIES as _ALLOWED_CATEGORIES, +) +from app.services.site_finder.weight_profiles import ( + MAX_WEIGHT as _MAX_WEIGHT, +) +from app.services.site_finder.weight_profiles import ( + MIN_WEIGHT as _MIN_WEIGHT, +) +from app.services.site_finder.weight_profiles import ( + resolve_weights as _resolve_weights, +) logger = logging.getLogger(__name__) @@ -272,21 +312,6 @@ def _confidence_label(c: float) -> str: return "low" -# Веса POI-категорий для scoring (Максим: трамвай = минус) -_POI_WEIGHTS: dict[str, float] = { - "school": 1.5, - "kindergarten": 1.5, - "pharmacy": 0.8, - "hospital": 0.6, - "shop_mall": 1.2, - "shop_supermarket": 1.0, - "shop_small": 0.5, - "park": 1.8, - "bus_stop": 0.3, - "metro_stop": 1.5, - "tram_stop": -0.5, # негативный вес — шум / вибрация -} - # Человеко-читаемые имена категорий для verbal breakdown (X1). _POI_CATEGORY_RU: dict[str, str] = { "school": "Школа", @@ -355,6 +380,10 @@ GEOTECH_BY_REGION: dict[int, dict[str, Any]] = { } +# SF-20 — success_recommendation confidence thresholds +SUCCESS_REC_MIN_DEALS = 15 # ниже → не показываем (пусто) +SUCCESS_REC_STRONG_DEALS = 30 # ≥ → data_confidence='strong'; 15-29 → 'weak' + # D4 (#36) — pipeline 24mo constants. Размещены в одном месте для тюнинга # и аудита; severity пороги матчатся с acceptance #36. PIPELINE_RADIUS_M = 5000 @@ -980,6 +1009,99 @@ _INLINE_FETCH_WAIT_S = 15 _INLINE_FETCH_POLL_INTERVAL_S = 2 +@router.get("/by-bbox", response_model=ParcelBboxResponse) +async def get_parcels_by_bbox( + min_lat: Annotated[float, Query(ge=-90, le=90, description="Южная граница bbox")], + min_lon: Annotated[float, Query(ge=-180, le=180, description="Западная граница bbox")], + max_lat: Annotated[float, Query(ge=-90, le=90, description="Северная граница bbox")], + max_lon: Annotated[float, Query(ge=-180, le=180, description="Восточная граница bbox")], + limit: Annotated[int, Query(ge=1, le=1000)] = 200, + user_id: Annotated[str | None, Query(description="user_id для overlay статуса")] = None, + db: Annotated[Session, Depends(get_db)] = ..., +) -> ParcelBboxResponse: + """GET /parcels/by-bbox — вернуть участки внутри bounding box для карты. + + Использует GIST-индекс на cad_parcels.geom (ST_Intersects). + Если передан user_id — добавляет статус из parcel_user_status (overlay). + last_analysis_date — placeholder до реализации B2 auth (#307). + """ + if min_lat >= max_lat or min_lon >= max_lon: + raise HTTPException(status_code=400, detail="Некорректный bbox: min >= max") + + # Площадь bbox в км² (приблизительно через формулу сферической трапеции) + lat_mid = (min_lat + max_lat) / 2 + lat_km = (max_lat - min_lat) * 111.32 + lon_km = (max_lon - min_lon) * 111.32 * math.cos(math.radians(lat_mid)) + bbox_area_km2 = round(lat_km * lon_km, 4) + + # cad_parcels has only (cad_num, geom) — area derived via ST_Area on geography + # cast for meter accuracy. land_category not stored on this table; returned NULL + # until enrichment from EGRN is wired (tracking: vault B5 EGRN contract note). + sql = text(""" + WITH bbox AS ( + SELECT ST_MakeEnvelope( + CAST(:min_lon AS float), + CAST(:min_lat AS float), + CAST(:max_lon AS float), + CAST(:max_lat AS float), + 4326 + ) AS env + ) + SELECT + p.cad_num, + ST_Y(ST_Centroid(p.geom)) AS centroid_lat, + ST_X(ST_Centroid(p.geom)) AS centroid_lon, + ST_Area(p.geom::geography) AS area_m2, + NULL::text AS land_category, + pus.status AS user_status + FROM cad_parcels p + CROSS JOIN bbox + LEFT JOIN parcel_user_status pus + ON pus.cad_num = p.cad_num + AND pus.user_id = CAST(:user_id AS text) + WHERE p.geom IS NOT NULL + AND ST_Intersects(p.geom, bbox.env) + ORDER BY ST_Area(p.geom::geography) DESC NULLS LAST + LIMIT CAST(:limit AS int) + """) + + rows = ( + db.execute( + sql, + { + "min_lat": min_lat, + "min_lon": min_lon, + "max_lat": max_lat, + "max_lon": max_lon, + "user_id": user_id, + "limit": limit, + }, + ) + .mappings() + .all() + ) + + markers: list[ParcelMapMarker] = [ + ParcelMapMarker( + cad_num=row["cad_num"], + centroid_lat=float(row["centroid_lat"]) if row["centroid_lat"] is not None else 0.0, + centroid_lon=float(row["centroid_lon"]) if row["centroid_lon"] is not None else 0.0, + area_m2=float(row["area_m2"]) if row["area_m2"] is not None else None, + land_category=row["land_category"], + status=row["user_status"] if user_id else None, + last_analysis_date=None, # placeholder, real after B2 auth (#307) + ) + for row in rows + ] + + return ParcelBboxResponse( + parcels=markers, + count=len(markers), + limit=limit, + bbox_area_km2=bbox_area_km2, + ) + + @router.post("/search", response_model=ParcelSearchResponse) async def search_parcels(payload: ParcelSearchRequest) -> ParcelSearchResponse: """Search parcels by filters + scoring. @@ -1035,6 +1157,14 @@ def analyze_parcel( str | None, Query(description="user_id для fallback на default-профиль пользователя"), ] = None, + body: Annotated[ + AnalyzeRequest | None, + Body(description="Опциональное тело запроса: inline POI-веса (#201)"), + ] = None, + x_session_id: Annotated[ + str | None, + Header(description="Session ID пользователя для custom POI scoring (#254)"), + ] = None, ) -> dict[str, Any]: """Анализ участка: близость к социалке + district context + конкуренты. @@ -1159,16 +1289,20 @@ def analyze_parcel( geom_wkt: str = geom_row["wkt"] # type: ignore[index] # 2) District context — ближайший район ЕКБ + # median_price_per_m2: предпочитаем median_12m из mv_quarter_price_per_m2 (12 мес), + # fallback на ekb_districts.median_price_per_m2 (24 мес). district_row = ( db.execute( text(""" - SELECT district_name, - median_price_per_m2, + SELECT d.district_name, + COALESCE(mq.median_12m, d.median_price_per_m2) AS median_price_per_m2, ST_Distance( d.geom::geography, ST_Centroid(ST_GeomFromText(:wkt, 4326))::geography ) AS dist_to_center FROM ekb_districts d + LEFT JOIN mv_quarter_price_per_m2 mq + ON mq.quarter_cad_number = REGEXP_REPLACE(:cad_num, ':[^:]+$', '') WHERE ST_DWithin( d.geom::geography, ST_Centroid(ST_GeomFromText(:wkt, 4326))::geography, @@ -1177,7 +1311,7 @@ def analyze_parcel( ORDER BY dist_to_center ASC LIMIT 1 """), - {"wkt": geom_wkt}, + {"wkt": geom_wkt, "cad_num": cad_num}, ) .mappings() .first() @@ -1210,15 +1344,43 @@ def analyze_parcel( .all() ) - # 3b) Resolve effective POI weights (profile → user default → system) - from app.services.site_finder.weight_profiles import resolve_weights as _resolve_weights + # 3b) Resolve effective POI weights (inline → profile → user default → system) + _inline_weights: dict[str, float] | None = body.weights if body is not None else None - _effective_weights = _resolve_weights(db, user_id=profile_user_id, profile_id=profile_id) - _weights_source = ( - "profile" - if profile_id is not None - else ("user_default" if profile_user_id is not None else "system") - ) + if _inline_weights is not None: + # Validate inline weights: keys и диапазон значений (#201) + bad_keys = set(_inline_weights.keys()) - _ALLOWED_CATEGORIES + if bad_keys: + raise HTTPException( + status_code=422, + detail=( + f"Неизвестные POI-категории: {sorted(bad_keys)}. " + f"Допустимые: {sorted(_ALLOWED_CATEGORIES)}" + ), + ) + out_of_range = { + k: v + for k, v in _inline_weights.items() + if not math.isfinite(v) or v < _MIN_WEIGHT or v > _MAX_WEIGHT + } + if out_of_range: + raise HTTPException( + status_code=422, + detail=( + f"Веса за пределами допустимого диапазона " + f"[{_MIN_WEIGHT}, {_MAX_WEIGHT}]: {out_of_range}" + ), + ) + # Inline weights applied — merge поверх системных defaults (partial override) + _effective_weights = {**_POI_WEIGHTS, **_inline_weights} + _weights_source = "inline" + else: + _effective_weights = _resolve_weights(db, user_id=profile_user_id, profile_id=profile_id) + _weights_source = ( + "profile" + if profile_id is not None + else ("user_default" if profile_user_id is not None else "system") + ) # 4) Scoring: weighted sum с distance decay score = 0.0 @@ -1265,7 +1427,13 @@ def analyze_parcel( } ) - # 5) Конкуренты в радиусе 3 км из DOM.РФ. + # 5) Конкуренты в радиусе 3 км из DOM.РФ с ценами из objective_lots. + # OBJ-3: обогащаем каждый ЖК данными objective_lots через маппинг + # domrf_kn_objects.obj_id → objective_complex_mapping.domrf_obj_id + # → objective_lots.project_name. + # Агрегат: avg_price_per_m2_rub (81% coverage), avg_area_pd, units_sold, + # units_available — для UI-блока «Конкуренты». + # LEFT JOIN — ЖК без маппинга остаются в выдаче (поля = NULL). # NB: domrf_kn_objects имеет ~3 snapshot per obj_id → DISTINCT ON по # latest snapshot, иначе дубликаты ЖК в выдаче. competitor_rows = ( @@ -1276,24 +1444,47 @@ def analyze_parcel( FROM domrf_kn_objects WHERE latitude IS NOT NULL ORDER BY obj_id, snapshot_date DESC NULLS LAST + ), + obj_pricing AS ( + SELECT + cm.domrf_obj_id, + ROUND(AVG(ol.price_per_m2_rub)::numeric, 0) AS avg_price_per_m2_rub, + ROUND(AVG(ol.area_pd)::numeric, 1) AS avg_area_pd, + COUNT(*) FILTER (WHERE ol.is_sold) AS units_sold, + COUNT(*) FILTER (WHERE NOT ol.is_sold) AS units_available, + COUNT(*) FILTER ( + WHERE ol.price_per_m2_rub IS NOT NULL + ) AS lots_with_price + FROM objective_complex_mapping cm + JOIN objective_lots ol ON ol.project_name = cm.objective_complex_name + GROUP BY cm.domrf_obj_id ) - SELECT obj_id, - comm_name, - dev_name, - obj_class, - flat_count, - district_name, + SELECT o.obj_id, + o.comm_name, + o.dev_name, + o.obj_class, + o.flat_count, + o.district_name, + o.site_status, + o.ready_dt, + p.avg_price_per_m2_rub, + p.avg_area_pd, + p.units_sold, + p.units_available, + p.lots_with_price, ST_Distance( ST_SetSRID(ST_MakePoint(o.longitude, o.latitude), 4326)::geography, ST_Centroid(ST_GeomFromText(:wkt, 4326))::geography ) AS distance_m FROM latest_obj o + LEFT JOIN obj_pricing p ON p.domrf_obj_id = o.obj_id WHERE ST_DWithin( ST_SetSRID(ST_MakePoint(o.longitude, o.latitude), 4326)::geography, ST_Centroid(ST_GeomFromText(:wkt, 4326))::geography, 3000 ) - ORDER BY o.flat_count DESC NULLS LAST + ORDER BY CASE o.site_status WHEN 'Строящиеся' THEN 0 ELSE 1 END, + distance_m ASC LIMIT 20 """), {"wkt": geom_wkt}, @@ -1578,6 +1769,246 @@ def analyze_parcel( # Independent dict per request — never mutate module singleton. nspd_dump_data = make_empty_result() + # 9f) Parcel meta — ВРИ и кадастровые метаданные из cad_parcels (#29 G2) + parcel_meta: ParcelMeta | None = None + try: + pm_row = ( + db.execute( + text(""" + SELECT permitted_use_established_by_document AS permitted_use, + land_record_category_type AS land_category, + land_record_subtype AS land_subtype, + cost_value AS cad_cost + FROM cad_parcels + WHERE cad_num = CAST(:c AS text) + LIMIT 1 + """), + {"c": cad_num}, + ) + .mappings() + .first() + ) + if pm_row: + parcel_meta = ParcelMeta( + permitted_use=pm_row["permitted_use"], + land_category=pm_row["land_category"], + land_subtype=pm_row["land_subtype"], + cad_cost=float(pm_row["cad_cost"]) if pm_row["cad_cost"] is not None else None, + ) + except Exception as e: + logger.warning("parcel_meta query failed for %s: %s", cad_num, e) + + # B5-1) EGRN block — расширенные данные из cad_parcels (SF-B5) + egrn_block: dict[str, Any] = {} + try: + egrn_row = ( + db.execute( + text(""" + SELECT cost_value AS cadastral_value_rub, + cost_index AS cost_index_per_m2, + land_record_category_type AS land_category, + permitted_use_established_by_document AS permitted_use_text, + cost_registration_date AS last_egrn_update_date, + land_record_area AS area_m2, + ownership_type, + right_type, + status, + readable_address, + registration_date + FROM cad_parcels + WHERE cad_num = CAST(:c AS text) + LIMIT 1 + """), + {"c": cad_num}, + ) + .mappings() + .first() + ) + if egrn_row: + _cad_val = ( + float(egrn_row["cadastral_value_rub"]) + if egrn_row["cadastral_value_rub"] is not None + else None + ) + _area_m2 = float(egrn_row["area_m2"]) if egrn_row["area_m2"] is not None else None + _idx = egrn_row["cost_index_per_m2"] + _cad_per_m2: float | None = None + if _idx is not None: + _cad_per_m2 = float(_idx) + elif _cad_val is not None and _area_m2 and _area_m2 > 0: + _cad_per_m2 = round(_cad_val / _area_m2, 2) + egrn_block = { + "cadastral_value_rub": _cad_val, + "cadastral_value_per_m2": _cad_per_m2, + "land_category": egrn_row["land_category"], + "permitted_use_text": egrn_row["permitted_use_text"], + "last_egrn_update_date": ( + egrn_row["last_egrn_update_date"].isoformat() + if egrn_row["last_egrn_update_date"] is not None + else None + ), + "area_m2": _area_m2, + "ownership_type": egrn_row["ownership_type"], + "right_type": egrn_row["right_type"], + "parcel_status": egrn_row["status"], + "address": egrn_row["readable_address"], + "registration_date": ( + egrn_row["registration_date"].isoformat() + if egrn_row["registration_date"] is not None + else None + ), + } + except Exception as e: + logger.warning("egrn_block query failed for %s: %s", cad_num, e) + + # B5-2) Encumbrance block — ЗОУИТ из cad_zouit (SF-B5) + encumbrance_block: dict[str, Any] = { + "has_zouit": False, + "zouit_types": [], + "zouit_count": 0, + } + try: + zouit_rows = ( + db.execute( + text(""" + SELECT type_zone, name_by_doc + FROM cad_zouit + WHERE ST_Intersects(geom, ST_GeomFromText(:wkt, 4326)) + ORDER BY id + """), + {"wkt": geom_wkt}, + ) + .mappings() + .all() + ) + if zouit_rows: + _zouit_types = list({r["type_zone"] for r in zouit_rows if r["type_zone"]}) + encumbrance_block = { + "has_zouit": True, + "zouit_types": _zouit_types, + "zouit_count": len(zouit_rows), + } + except Exception as e: + logger.warning("encumbrance_block query failed for %s: %s", cad_num, e) + + # B5-3) Red lines block — пересечение с cad_red_lines (SF-B5) + red_lines_block: dict[str, Any] = {"intersects": False, "count": 0} + try: + rl_row = ( + db.execute( + text(""" + SELECT COUNT(*) AS cnt + FROM cad_red_lines + WHERE ST_Intersects( + geom::geometry, + ST_GeomFromText(:wkt, 4326) + ) + """), + {"wkt": geom_wkt}, + ) + .mappings() + .first() + ) + if rl_row: + _rl_cnt = int(rl_row["cnt"]) + red_lines_block = { + "intersects": _rl_cnt > 0, + "count": _rl_cnt, + } + except Exception as e: + logger.warning("red_lines_block query failed for %s: %s", cad_num, e) + + # B5-4) Metro placeholder — заполнится после merge 22h metro scraper + metro_block: dict[str, Any] = {"nearest_top3": None} + + # B5-5) District price ranges из objective_lots (SF-B5) + district_price_block: dict[str, Any] = { + "district_price_per_m2_min": None, + "district_price_per_m2_max": None, + "district_price_per_m2_median": None, + "district_price_sample_size": None, + } + if district_row and district_row["district_name"]: + try: + dp_row = ( + db.execute( + text(""" + SELECT + MIN(price_per_m2_rub) AS price_min, + MAX(price_per_m2_rub) AS price_max, + PERCENTILE_CONT(0.5) WITHIN GROUP ( + ORDER BY price_per_m2_rub + ) AS price_median, + COUNT(*) AS sample_size + FROM objective_lots + WHERE district = CAST(:dn AS text) + AND price_per_m2_rub IS NOT NULL + AND price_per_m2_rub BETWEEN 30000 AND 600000 + """), + {"dn": district_row["district_name"]}, + ) + .mappings() + .first() + ) + if dp_row and dp_row["sample_size"] and int(dp_row["sample_size"]) > 0: + district_price_block = { + "district_price_per_m2_min": ( + round(float(dp_row["price_min"])) if dp_row["price_min"] else None + ), + "district_price_per_m2_max": ( + round(float(dp_row["price_max"])) if dp_row["price_max"] else None + ), + "district_price_per_m2_median": ( + round(float(dp_row["price_median"])) if dp_row["price_median"] else None + ), + "district_price_sample_size": int(dp_row["sample_size"]), + } + except Exception as e: + logger.warning("district_price_block query failed for %s: %s", cad_num, e) + + # B5-6) Risk indicators — flood_zone из cad_risk_zones + noise_score + geology proxy (SF-B5) + risks_block: dict[str, Any] = { + "flood_zone": False, + "noise_score": round(noise_score, 2), + "geology_risk_label": None, + } + try: + flood_row = ( + db.execute( + text(""" + SELECT COUNT(*) AS cnt + FROM cad_risk_zones + WHERE ST_Intersects( + geom::geometry, + ST_GeomFromText(:wkt, 4326) + ) + AND (risk_type ILIKE '%flood%' OR risk_type ILIKE '%подтоп%' + OR risk_type ILIKE '%затоп%') + """), + {"wkt": geom_wkt}, + ) + .mappings() + .first() + ) + _flood = bool(flood_row and int(flood_row["cnt"]) > 0) + # Geology proxy через hydrology flood_risk_flag (уже посчитан выше) + _geo_flood = hydrology.get("flood_risk_flag", False) if hydrology else False + _has_flood = _flood or _geo_flood + # geology_risk_label: high если flooding, medium если шум > 65дБ, иначе low + if _has_flood: + _geo_label: str | None = "high" + elif noise_db_max >= 65.0: + _geo_label = "medium" + else: + _geo_label = "low" + risks_block = { + "flood_zone": _has_flood, + "noise_score": round(noise_score, 2), + "geology_risk_label": _geo_label, + } + except Exception as e: + logger.warning("risks_block query failed for %s: %s", cad_num, e) + # 10) Market trend — динамика цен ДДУ в радиусе 3 км за 6 vs предыдущие 6 месяцев market_trend: dict[str, Any] | None = None try: @@ -1707,6 +2138,8 @@ def analyze_parcel( logger.warning("zoning query failed for %s: %s", cad_num, e) # 10c) Success recommendation — топ квартирография по district из v_bucket_success_score + # SF-20: адаптивный порог — MIN_DEALS=15 (показываем), STRONG=30 (confidence='strong'). + # Если n_deals всех строк < STRONG → data_confidence='weak' + UI badge «слабые данные». success_recommendation: dict[str, Any] | None = None if district_row: try: @@ -1726,9 +2159,14 @@ def analyze_parcel( .mappings() .all() ) - if success_rows: + # Фильтруем строки ниже абсолютного минимума (< SUCCESS_REC_MIN_DEALS) + valid_rows = [r for r in success_rows if int(r["n_deals"]) >= SUCCESS_REC_MIN_DEALS] + if valid_rows: + max_deals = max(int(r["n_deals"]) for r in valid_rows) + data_confidence = "strong" if max_deals >= SUCCESS_REC_STRONG_DEALS else "weak" success_recommendation = { "district": district_row["district_name"], + "data_confidence": data_confidence, "ranking": [ { "bucket": r["bucket"], @@ -1741,12 +2179,12 @@ def analyze_parcel( round(float(r["avg_area_m2"]), 1) if r["avg_area_m2"] else None ), } - for r in success_rows + for r in valid_rows ], - "top_bucket": dict(success_rows[0]) if success_rows else None, + "top_bucket": dict(valid_rows[0]), "note": ( "Топ комнатность по 'успешности' = z-scores: velocity×0.5 + price×0.3 " - "- area×0.2. Min 30 сделок в группе за 24 мес. " + "- area×0.2. Min 15 сделок в группе за 24 мес. " "Используй для квартирографии проекта." ), } @@ -1754,6 +2192,117 @@ def analyze_parcel( logger.warning("success_recommendation query failed for %s: %s", cad_num, e) success_recommendation = None + # 10d-pre) Market price — ценовая статистика квартала из mv_quarter_price_per_m2 (#33) + # quarter_cad_number — первые три части кад. номера: "66:41:0204016:10" → "66:41:0204016" + _cad_parts = cad_num.split(":") + _quarter_cad = ":".join(_cad_parts[:3]) if len(_cad_parts) >= 3 else cad_num + market_price: dict[str, Any] + try: + with db.begin_nested(): + mp_row = ( + db.execute( + text(""" + SELECT p25, median, p75, mean, deals_count, + median_6m, median_12m, median_24m, last_deal_date + FROM mv_quarter_price_per_m2 + WHERE quarter_cad_number = :q + """), + {"q": _quarter_cad}, + ) + .mappings() + .first() + ) + if mp_row: + market_price = { + "p25": float(mp_row["p25"]) if mp_row["p25"] is not None else None, + "median": float(mp_row["median"]) if mp_row["median"] is not None else None, + "p75": float(mp_row["p75"]) if mp_row["p75"] is not None else None, + "mean": float(mp_row["mean"]) if mp_row["mean"] is not None else None, + "deals_count": int(mp_row["deals_count"]), + "median_6m": ( + float(mp_row["median_6m"]) if mp_row["median_6m"] is not None else None + ), + "median_12m": ( + float(mp_row["median_12m"]) if mp_row["median_12m"] is not None else None + ), + "median_24m": ( + float(mp_row["median_24m"]) if mp_row["median_24m"] is not None else None + ), + "last_deal_date": ( + mp_row["last_deal_date"].isoformat() + if mp_row["last_deal_date"] is not None + else None + ), + "source": "quarter_mv", + } + else: + market_price = {"deals_count": 0, "source": "no_data"} + except Exception as e: + logger.warning("market_price query failed for %s: %s", cad_num, e) + market_price = {"deals_count": 0, "source": "no_data"} + + # 10d-pre2) #105 Phase 5: РНС/РВЭ в квартале (cad_num quarter prefix match). + # Spatial filter через geom недоступен до Phase 3 (geocoding → geom). + # Используем quarter prefix (первые 3 сегмента кад. номера) как прокси. + # После Phase 3: заменить на ST_DWithin(..., 500) по geom. + recent_permits: list[dict[str, Any]] = [] + permits_summary: dict[str, Any] = { + "rns_count": 0, + "rve_count": 0, + "rns_total_area_sqm": 0.0, + "by_developer": [], + } + try: + with db.begin_nested(): + permits_rows = ( + db.execute( + text(""" + SELECT + permit_type, permit_number, issue_date, + developer_name, developer_inn, object_name, object_type, + construction_address, total_area_sqm + FROM ekburg_construction_permits + WHERE LEFT(cadastral_number, LENGTH(CAST(:q AS text))) = CAST(:q AS text) + AND issue_date > NOW() - INTERVAL '24 months' + ORDER BY issue_date DESC + LIMIT 50 + """), + {"q": _quarter_cad}, + ) + .mappings() + .all() + ) + recent_permits = [ + { + "permit_type": r["permit_type"], + "permit_number": r["permit_number"], + "issue_date": r["issue_date"].isoformat() if r["issue_date"] else None, + "developer_name": r["developer_name"], + "developer_inn": r["developer_inn"], + "object_name": r["object_name"], + "object_type": r["object_type"], + "construction_address": r["construction_address"], + "total_area_sqm": float(r["total_area_sqm"]) if r["total_area_sqm"] else None, + } + for r in permits_rows + ] + rns_list = [p for p in recent_permits if p["permit_type"] == "RNS"] + by_dev: dict[str, int] = {} + for p in recent_permits: + if p["developer_name"]: + by_dev[p["developer_name"]] = by_dev.get(p["developer_name"], 0) + 1 + permits_summary = { + "rns_count": len(rns_list), + "rve_count": len([p for p in recent_permits if p["permit_type"] == "RVE"]), + "rns_total_area_sqm": sum(p["total_area_sqm"] or 0.0 for p in rns_list), + "by_developer": [ + {"name": name, "permits_count": cnt} + for name, cnt in sorted(by_dev.items(), key=lambda x: -x[1])[:3] + ], + } + except Exception as e: + logger.warning("ekburg permits query failed for %s: %s", cad_num, e) + # 10d) Geology stub — реальные данные требуют ВСЕГЕИ-200/1000 шейпы в PostGIS karpinsky_url = ( f"https://www.karpinskyinstitute.ru/ru/gisatlas/web-gisatlas/" @@ -1776,6 +2325,61 @@ def analyze_parcel( "lon": centroid_lon, } + # 4b) #254: Custom POI scoring — user-defined points с произвольным весом. + # Добавляем после основного POI loop, не трогаем OSM логику. + # user_id берём из X-Session-Id header (workaround до #67 NextAuth). + custom_poi_items: list[dict[str, Any]] = [] + _session_id = x_session_id.strip() if x_session_id and x_session_id.strip() else None + if _session_id: + try: + _custom_overlaps = _get_custom_poi_overlaps( + db, geom_wkt, _session_id, parcel_cad=cad_num + ) + for cp in _custom_overlaps: + _distance_m = cp["distance_m"] + _decay = max(0.0, 1.0 - _distance_m / 1000.0) + _contribution = cp["weight"] * _decay + score += _contribution + _item: dict[str, Any] = { + "source": "custom", + "id": cp["id"], + "label": cp["name"], + "category": cp["category"], + "weight": cp["weight"], + "distance_m": round(_distance_m), + "contribution": round(_contribution, 3), + "lat": cp["lat"], + "lon": cp["lon"], + } + custom_poi_items.append(_item) + if abs(_contribution) >= 0.01: + factors_detailed.append( + { + "factor": f"custom_{cp['id']}_{round(_distance_m)}m", + "category": f"custom_{cp['category'] or 'poi'}", + "category_ru": f"Custom: {cp['name']}", + "group": "Custom POI", + "value": round(_distance_m, 1), + "weight": cp["weight"], + "contribution": round(_contribution, 2), + "verbal": ( + f"Custom POI '{cp['name']}' ({round(_distance_m)}м) — " + f"{'+'if _contribution >= 0 else ''}{round(_contribution, 2)} б." + ), + "lat": cp["lat"], + "lon": cp["lon"], + } + ) + if _custom_overlaps: + logger.debug( + "custom POI scoring: user=%s cad=%s poi_count=%d", + _session_id, + cad_num, + len(_custom_overlaps), + ) + except Exception as _cpe: + logger.warning("custom POI scoring failed cad=%s: %s", cad_num, _cpe) + score_final = score + center_bonus # X1 (#47): расчёт contribution_pct + top-3 / by-group для UI. @@ -1830,15 +2434,74 @@ def analyze_parcel( # SAVEPOINT защищает outer transaction если velocity SQL падает — # иначе следующие queries (_geotech_risk и пр.) крашатся # с InFailedSqlTransaction. + # SF#17: передаём cad_quarter для rosreestr_fallback (первые 3 сегмента cad_num). + _cad_parts = cad_num.split(":") + _cad_quarter = ":".join(_cad_parts[:3]) if len(_cad_parts) >= 3 else None velocity_data: dict[str, Any] | None = None try: with db.begin_nested(): - v_result = compute_velocity(db, parcel_geom_wkt=geom_wkt, radius_km=3.0) + v_result = compute_velocity( + db, + parcel_geom_wkt=geom_wkt, + radius_km=3.0, + cad_quarter=_cad_quarter, + ) if v_result is not None: velocity_data = v_result.as_dict() except Exception as _ve: logger.warning("velocity compute failed for %s: %s", cad_num, _ve) + # OBJ-3 aggregate fix: market_pulse — агрегаты ТОЛЬКО по ЖК с ненулевыми ценами. + # Конкуренты без маппинга в Objective (NULL avg_price_per_m2_rub) остаются + # в competitor list для карты, но исключаются из расчётов рыночных метрик. + _competitors_with_price = [c for c in competitor_rows if c["avg_price_per_m2_rub"] is not None] + _competitors_total = len(competitor_rows) + _competitors_priced = len(_competitors_with_price) + if _competitors_with_price: + _prices = [float(c["avg_price_per_m2_rub"]) for c in _competitors_with_price] + _market_avg_price = round(sum(_prices) / len(_prices)) + # top_sellers: ЖК с ненулевыми units_sold, топ-5 по объёму + _with_sales = [ + c + for c in _competitors_with_price + if c["units_sold"] is not None and int(c["units_sold"]) > 0 + ] + _top_sellers = sorted( + _with_sales, + key=lambda c: int(c["units_sold"]), + reverse=True, + )[:5] + _top_sellers_list = [ + { + "obj_id": c["obj_id"], + "comm_name": c["comm_name"], + "dev_name": c["dev_name"], + "units_sold": int(c["units_sold"]), + "avg_price_per_m2_rub": int(c["avg_price_per_m2_rub"]), + } + for c in _top_sellers + ] + else: + _market_avg_price = None + _top_sellers_list = [] + + _coverage_pct = ( + round(_competitors_priced * 100.0 / _competitors_total, 1) + if _competitors_total > 0 + else 0.0 + ) + # avg_velocity_m2 — берём из velocity_data если есть; это уже только по + # ЖК с objective_corpus_room_month данными (non-null by construction). + _avg_velocity = velocity_data["monthly_velocity_sqm"] if velocity_data else None + market_pulse: dict[str, Any] = { + "avg_velocity_m2": _avg_velocity, + "market_avg_price_per_m2": _market_avg_price, + "competitors_total": _competitors_total, + "competitors_with_price": _competitors_priced, + "coverage_pct": _coverage_pct, + "top_sellers": _top_sellers_list, + } + return { "cad_num": cad_num, "source": source, @@ -1866,9 +2529,15 @@ def analyze_parcel( "note": "Бонус к score: <5км +3.0, 5-10км +1.5, 10-15км +0.5, >15км 0", }, "competitors": [dict(c) for c in competitor_rows], + # OBJ-3 fix: aggregate market metrics — только non-null competitors. + # ЖК без objective_mapping остаются на карте (competitors list), + # но исключены из avg/velocity/top_sellers расчётов. + "market_pulse": market_pulse, + "market_avg_price_per_m2": market_pulse["market_avg_price_per_m2"], + "market_data_coverage_pct": market_pulse["coverage_pct"], # D4 (#36): 24-month pipeline competition "pipeline_24mo": pipeline_24mo, - # D2 (#34): velocity-score из domrf_kn_sale_graph + # D2 (#34): velocity-score из objective_corpus_room_month (OBJ-3 migrated) "velocity": velocity_data, "noise": { "score": round(noise_score, 2), @@ -1889,6 +2558,13 @@ def analyze_parcel( # P2 (#46) — соседи-здания + overlap check "neighbors_summary": _neighbors_summary(db, geom_wkt, cad_num), "market_trend": market_trend, + # #33 D2: квартальная ценовая статистика из mv_quarter_price_per_m2 + "market_price": market_price, + # #29 G2: кадастровые метаданные участка (ВРИ, категория, кад. стоимость) + "parcel_meta": parcel_meta.model_dump() if parcel_meta is not None else None, + # #105 Phase 5: РНС/РВЭ в квартале (quarter prefix match; после Phase 3 → spatial 500m) + "recent_permits_in_quarter": recent_permits, + "permits_summary": permits_summary, "zoning": zoning, "success_recommendation": success_recommendation, "isochrones_available": bool(settings.openrouteservice_api_key), @@ -1901,10 +2577,18 @@ def analyze_parcel( # nspd_zoning: ПЗЗ зона из territorial_zones кеша (G1) # nspd_zouit_overlaps: ЗОУИТ пересечения (G3) # nspd_engineering_nearby: инженерные сооружения в 200м (I3) + # nspd_risk_zones: TIER 3 risk zones (#94) — подтопление, эрозия, гари, оползни + # nspd_opportunity_parcels: TIER 4 opportunity ЗУ (#94 PR2) + # nspd_red_lines: TIER 4 красные линии застройки (#94 PR2, #54 Generative) # nspd_dump: freshness metadata — available, stale, harvest_triggered "nspd_zoning": nspd_dump_data["nspd_zoning"], "nspd_zouit_overlaps": nspd_dump_data["nspd_zouit_overlaps"], "nspd_engineering_nearby": nspd_dump_data["nspd_engineering_nearby"], + "nspd_risk_zones": [RiskZone(**rz) for rz in nspd_dump_data.get("nspd_risk_zones", [])], + "nspd_opportunity_parcels": [ + OpportunityParcel(**op) for op in nspd_dump_data.get("nspd_opportunity_parcels", []) + ], + "nspd_red_lines": [RedLine(**rl) for rl in nspd_dump_data.get("nspd_red_lines", [])], "nspd_dump": nspd_dump_data["nspd_dump"], # #32 G5: gate verdict — can-build-MKD aggregated signal for UI banner "gate_verdict": compute_gate_verdict( @@ -1913,16 +2597,63 @@ def analyze_parcel( nspd_engineering_nearby=nspd_dump_data["nspd_engineering_nearby"], nspd_dump=nspd_dump_data["nspd_dump"], ), - # #114: кастомные веса POI — source + applied dict для прозрачности. + # #114/#201: кастомные веса POI — source + applied dict для прозрачности. "weights_profile": { "source": _weights_source, "profile_id": profile_id, "user_id": profile_user_id, "weights_applied": _effective_weights, + "inline_weights": _inline_weights, }, + # #254: custom POI scoring — user-defined points (via X-Session-Id header). + "custom_poi_score_items": custom_poi_items, + # SF-B5: EGRN + encumbrance + red_lines + metro + district prices + risks + "egrn": egrn_block, + "encumbrance": encumbrance_block, + "red_lines": red_lines_block, + "metro": metro_block, + "district_price_per_m2_min": district_price_block["district_price_per_m2_min"], + "district_price_per_m2_max": district_price_block["district_price_per_m2_max"], + "district_price_per_m2_median": district_price_block["district_price_per_m2_median"], + "district_price_sample_size": district_price_block["district_price_sample_size"], + "risks": risks_block, } +@router.get( + "/{cad_num}/connection-points", + response_model=ConnectionPointsResponse, + summary="Точки подключения к инженерным сетям + охранные зоны (issue #115)", +) +async def get_parcel_connection_points( + cad_num: str, + db: Annotated[Session, Depends(get_db)], + radius_m: Annotated[int, Query(ge=50, le=2000)] = 500, +) -> ConnectionPointsResponse: + """Инженерные структуры (ТП, ЦТП, ЛЭП) и охранные зоны коммуникаций вблизи участка. + + Источник данных: nspd_quarter_dumps (НСПД cat 36328 / 37578). + Если дамп для квартала ещё не загружен → dump_available=false, + пустые массивы (harvest запускается автоматически). + Если участок не найден в БД (нет geom) → 404. + + Query params: + radius_m: радиус поиска инженерных структур, 50–2000 м (default 500). + """ + try: + data = get_connection_points(db, cad_num, radius_m) + except ValueError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + + return ConnectionPointsResponse( + engineering_structures=data["engineering_structures"], + zouit_engineering_overlaps=data["zouit_engineering_overlaps"], + summary=data["summary"], + dump_available=data["dump_available"], + dump_fetched_at=data["dump_fetched_at"], + ) + + _ORS_BASE = "https://api.openrouteservice.org/v2/isochrones" _ORS_VALID_MODES = frozenset({"foot-walking", "cycling-regular", "driving-car"}) @@ -2040,3 +2771,318 @@ def get_isochrones( "source": "openrouteservice.org", "note": "Free tier 2000 req/day. Замена на self-hosted OSRM — в #27.", } + + +@router.get( + "/{cad_num}/poi-score", + response_model=PoiScoreResponse, + summary="POI weighted top-7 (B6)", +) +async def get_poi_score( + cad_num: str, + db: Annotated[Session, Depends(get_db)], + radius_m: Annotated[int, Query(ge=100, le=5000)] = 2000, +) -> PoiScoreResponse: + """Вернуть top-7 ближайших POI для участка, взвешенных по формуле: + + weight = (1 / (distance_m + 100)) * category_weight + + POI берутся из osm_poi_ekb в заданном радиусе (default 2000м). + Отсортированы по weight DESC — наиболее значимые объекты первыми. + """ + # Получить координаты центроида участка из геометрических таблиц + coord_row = ( + db.execute( + text(""" + SELECT ST_X(ST_Centroid(g.geom)) AS lon, + ST_Y(ST_Centroid(g.geom)) AS lat + FROM ( + SELECT geom FROM cad_quarters_geom WHERE cad_number = :c + UNION ALL + SELECT geom FROM cad_buildings WHERE cad_num = :c + UNION ALL + SELECT geom FROM cad_parcels_geom WHERE cad_num = :c + ) g + LIMIT 1 + """), + {"c": cad_num}, + ) + .mappings() + .first() + ) + + if not coord_row: + raise HTTPException( + status_code=404, + detail=f"Геометрия для {cad_num} не найдена.", + ) + + lat = float(coord_row["lat"]) + lon = float(coord_row["lon"]) + + return compute_poi_weighted_top7(db, cad_num, lat, lon, radius_m=radius_m) + + +@router.post("/{cad_num}/competitors", response_model=CompetitorsResponse) +async def get_parcel_competitors( + cad_num: str, + body: CompetitorsRequest, + db: Annotated[Session, Depends(get_db)], +) -> CompetitorsResponse: + """Активные конкуренты ЖК в радиусе от участка (Issue #112). + + Возвращает список ЖК из domrf_kn_objects в радиусе radius_km от центроида + участка с рассчитанным velocity_per_month за указанный time_window. + """ + try: + return get_competitors(db=db, cad_num=cad_num, request=body) + except ValueError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + except Exception as exc: + logger.error("competitors endpoint failed for %s: %s", cad_num, exc) + raise HTTPException( + status_code=500, + detail="Ошибка расчёта конкурентов", + ) from exc + + +@router.post("/{cad_num}/best-layouts", response_model=BestLayoutsResponse) +async def get_parcel_best_layouts( + cad_num: str, + body: BestLayoutsRequest, + db: Annotated[Session, Depends(get_db)], +) -> BestLayoutsResponse: + """Top layouts (rooms × area_bin) у конкурентов с ranking по velocity. + + Issue #113 Phase 2.1: "Анализ лучших планировок конкурентов → ТЗ на проектирование". + Reads from mv_layout_velocity (auto-populated via objective_corpus_room_month + × objective_complex_mapping). + """ + try: + return get_best_layouts(db=db, cad_num=cad_num, request=body) + except ValueError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + except Exception as exc: + logger.error("best_layouts endpoint failed for %s: %s", cad_num, exc) + raise HTTPException(status_code=500, detail="Internal server error") from exc + + +@router.post("/{cad_num}/best-layouts/pdf") +async def get_parcel_best_layouts_pdf( + cad_num: str, + body: BestLayoutsRequest, + db: Annotated[Session, Depends(get_db)], +) -> Response: + """ТЗ на проектирование (PDF) — генерируется из /best-layouts данных. + + Issue #113 Phase 2.1: data-driven unit-mix recommendation для тендера. + """ + try: + response = get_best_layouts(db=db, cad_num=cad_num, request=body) + pdf_bytes = render_layout_tz_pdf( + response, + cad_num=cad_num, + radius_km=body.radius_km, + time_window=body.time_window, + ) + today = _dt.date.today().strftime("%Y-%m-%d") + cad_safe = cad_num.replace(":", "-") + filename = f"tz-layout-{cad_safe}-{today}.pdf" + return Response( + content=pdf_bytes, + media_type="application/pdf", + headers={"Content-Disposition": f'attachment; filename="{filename}"'}, + ) + except HTTPException: + raise + except ValueError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + except Exception as exc: + logger.error("best_layouts PDF endpoint failed for %s: %s", cad_num, exc) + raise HTTPException(status_code=500, detail="Internal server error") from exc + + +@router.get( + "/{cad_num}/snapshot.pdf", + summary="1-page PDF snapshot участка (НСПД + POI + конкуренты)", +) +def parcel_snapshot_pdf( + cad_num: str, + db: Annotated[Session, Depends(get_db)], +) -> Response: + """Генерирует одностраничный PDF-снимок участка (A4). + + Содержимое: + - Header: кадастровый номер, адрес, район, площадь + - Block 1: 5 KPI (площадь, кадастровая стоимость, категория, ВРИ, дата обновления) + - Block 2: Топ-7 POI по взвешенному баллу (из osm_poi_ekb, радиус 1 км) + - Block 3: Топ-5 конкурентов (из domrf_kn_objects, радиус 3 км) + - Footer: gendsgn.ru + дата генерации + + Не является официальной выпиской ЕГРН — только аналитические данные НСПД. + Генерация <2 сек. Открывается в Adobe Reader / Chrome. + """ + # 1) Получить метаданные участка из cad_parcels + parcel_row = ( + db.execute( + text(""" + SELECT readable_address AS address, + land_record_area AS area_m2, + land_record_category_type AS land_category, + permitted_use_established_by_document AS vri, + cost_value AS cadastral_cost, + updated_at AS last_update + FROM cad_parcels + WHERE cad_num = CAST(:c AS text) + LIMIT 1 + """), + {"c": cad_num}, + ) + .mappings() + .first() + ) + + if not parcel_row: + raise HTTPException( + status_code=404, + detail=f"Участок {cad_num} не найден в БД. Используйте POST /analyze для загрузки.", + ) + + # 2) Получить геометрию (WKT) для POI / competitor queries + geom_row = ( + db.execute( + text(""" + SELECT ST_AsText(COALESCE( + (SELECT geom FROM cad_parcels_geom WHERE cad_num = CAST(:c AS text) LIMIT 1), + (SELECT geom FROM cad_parcels WHERE cad_num = CAST(:c AS text) LIMIT 1) + )) AS wkt + """), + {"c": cad_num}, + ) + .mappings() + .first() + ) + geom_wkt: str | None = geom_row["wkt"] if geom_row else None + + # 3) POI в радиусе 1 км (только если есть геометрия) + poi_rows: list[dict[str, Any]] = [] + if geom_wkt: + poi_rows = [ + dict(r) + for r in db.execute( + text(""" + SELECT category, + name, + ST_Distance( + p.geom::geography, + ST_Centroid(ST_GeomFromText(:wkt, 4326))::geography + ) AS distance_m + FROM osm_poi_ekb p + WHERE ST_DWithin( + p.geom::geography, + ST_Centroid(ST_GeomFromText(:wkt, 4326))::geography, + 1000 + ) + ORDER BY distance_m ASC + LIMIT 50 + """), + {"wkt": geom_wkt}, + ) + .mappings() + .all() + ] + + # 4) Конкуренты в радиусе 3 км (только если есть геометрия) + competitor_rows: list[dict[str, Any]] = [] + if geom_wkt: + competitor_rows = [ + dict(r) + for r in db.execute( + text(""" + WITH latest_obj AS ( + SELECT DISTINCT ON (obj_id) * + FROM domrf_kn_objects + WHERE latitude IS NOT NULL + ORDER BY obj_id, snapshot_date DESC NULLS LAST + ) + SELECT obj_id, + comm_name, + dev_name, + obj_class, + flat_count, + ST_Distance( + ST_SetSRID(ST_MakePoint(o.longitude, o.latitude), 4326)::geography, + ST_Centroid(ST_GeomFromText(:wkt, 4326))::geography + ) AS distance_m + FROM latest_obj o + WHERE ST_DWithin( + ST_SetSRID(ST_MakePoint(o.longitude, o.latitude), 4326)::geography, + ST_Centroid(ST_GeomFromText(:wkt, 4326))::geography, + 3000 + ) + ORDER BY flat_count DESC NULLS LAST + LIMIT 20 + """), + {"wkt": geom_wkt}, + ) + .mappings() + .all() + ] + + # 5) Получить district (через пересечение с ekb_districts если есть геом) + district: str | None = None + if geom_wkt: + district_row = ( + db.execute( + text(""" + SELECT d.district_name + FROM ekb_districts d + WHERE ST_Contains(d.geom, ST_Centroid(ST_GeomFromText(:wkt, 4326))) + LIMIT 1 + """), + {"wkt": geom_wkt}, + ) + .mappings() + .first() + ) + if district_row: + district = district_row["district_name"] + + # 6) Форматировать last_update + raw_update = parcel_row["last_update"] + last_update_str: str | None = None + if raw_update is not None: + try: + last_update_str = raw_update.strftime("%d.%m.%Y") + except AttributeError: + last_update_str = str(raw_update)[:10] + + # 7) Сгенерировать PDF + try: + pdf_bytes = generate_snapshot_pdf( + cad_num=cad_num, + address=parcel_row["address"], + district=district, + area_m2=float(parcel_row["area_m2"]) if parcel_row["area_m2"] is not None else None, + cadastral_cost_rub=( + float(parcel_row["cadastral_cost"]) + if parcel_row["cadastral_cost"] is not None + else None + ), + land_category=parcel_row["land_category"], + vri=parcel_row["vri"], + last_update=last_update_str, + poi_rows=poi_rows, + competitor_rows=competitor_rows, + competitors_limit=5, + ) + except Exception as exc: + logger.error("snapshot PDF generation failed for %s: %s", cad_num, exc) + raise HTTPException(status_code=500, detail="Ошибка генерации PDF") from exc + + cad_safe = cad_num.replace(":", "-") + return Response( + content=pdf_bytes, + media_type="application/pdf", + headers={"Content-Disposition": f'attachment; filename="snapshot-{cad_safe}.pdf"'}, + ) diff --git a/backend/app/api/v1/pilot.py b/backend/app/api/v1/pilot.py new file mode 100644 index 00000000..a925b86f --- /dev/null +++ b/backend/app/api/v1/pilot.py @@ -0,0 +1,77 @@ +"""Pilot request lead-gen endpoint. + +POST /api/v1/pilot/request — принимает заявку на пилот (лид с лендинга или страницы анализа), +сохраняет в таблицу pilot_requests. +Telegram-уведомление — TODO (creds не настроены, см. #307 SF-B3). +""" + +from __future__ import annotations + +import logging +from typing import Annotated, Any, Literal + +from fastapi import APIRouter, Depends, Request +from pydantic import BaseModel, Field +from sqlalchemy import text +from sqlalchemy.orm import Session + +from app.core.db import get_db + +logger = logging.getLogger(__name__) + +router = APIRouter() + + +class PilotRequestInput(BaseModel): + name: str = Field(min_length=2, max_length=200) + phone: str | None = Field(default=None, max_length=50) + # Note: was EmailStr but `email-validator` package not in deps → ImportError + # on backend startup → container unhealthy → whole deploy fails. Using plain + # str + minimal regex; full validation can happen on frontend/CRM side. + email: str | None = Field(default=None, max_length=200, pattern=r"^[^@\s]+@[^@\s]+\.[^@\s]+$") + company: str | None = Field(default=None, max_length=200) + message: str | None = Field(default=None, max_length=2000) + source: Literal["landing", "analyze_page", "other"] = "landing" + + +@router.post("/request") +async def create_pilot_request( + payload: PilotRequestInput, + request: Request, + db: Annotated[Session, Depends(get_db)], +) -> dict[str, Any]: + """Сохраняет заявку на пилот в pilot_requests.""" + user_agent = request.headers.get("user-agent") + + row = ( + db.execute( + text( + """ + INSERT INTO pilot_requests (name, phone, email, company, message, source, user_agent) + VALUES (:name, :phone, :email, :company, :message, :source, :user_agent) + RETURNING CAST(id AS text), created_at + """ + ), + { + "name": payload.name, + "phone": payload.phone, + "email": str(payload.email) if payload.email else None, + "company": payload.company, + "message": payload.message, + "source": payload.source, + "user_agent": user_agent, + }, + ) + .mappings() + .one() + ) + + db.commit() + + logger.info("pilot_request saved id=%s source=%s", row["id"], payload.source) + + return { + "id": row["id"], + "created_at": row["created_at"].isoformat(), + "status": "received", + } diff --git a/backend/app/api/v1/trade_in.py b/backend/app/api/v1/trade_in.py new file mode 100644 index 00000000..ed771099 --- /dev/null +++ b/backend/app/api/v1/trade_in.py @@ -0,0 +1,385 @@ +"""Trade-In Estimator — endpoints (TI-1 mock + TI-2 PDF). + +MOCK implementation: returns realistic ЕКБ price bands by rooms/floor/repair. +TODO TI-1b: заменить _mock_estimate() на реальный SQL aggregation из +objective_lots + rosreestr_deals после OBJ-1/2 merge. +""" + +from __future__ import annotations + +import json +import logging +import random +from datetime import UTC, datetime, timedelta +from typing import Annotated +from uuid import UUID, uuid4 + +from fastapi import APIRouter, Depends, HTTPException, Response +from sqlalchemy import text +from sqlalchemy.orm import Session + +from app.core.db import get_db +from app.schemas.trade_in import AggregatedEstimate, AnalogLot, TradeInEstimateInput +from app.services.exporters.trade_in_pdf import generate_trade_in_pdf + +logger = logging.getLogger(__name__) + +router = APIRouter() + +# ЕКБ-адреса для фейковых аналогов (реальные улицы центра) +_EKB_STREETS = [ + "ул. Малышева", + "ул. Куйбышева", + "ул. 8 Марта", + "ул. Белинского", + "пр. Ленина", + "ул. Толмачёва", + "ул. Радищева", + "ул. Мамина-Сибиряка", + "ул. Луначарского", + "ул. Первомайская", +] + +# Базовые ценовые диапазоны по комнатности (ЕКБ, 2026) +_PRICE_BANDS: dict[int, dict[str, int | float | str]] = { + 0: { # студия ~25 м² + "median": 6_500_000, + "low": 5_800_000, + "high": 7_500_000, + "ppm2": 260_000, + "ref_area": 25.0, + }, + 1: { # 1к ~40 м² + "median": 9_000_000, + "low": 8_000_000, + "high": 10_500_000, + "ppm2": 225_000, + "ref_area": 40.0, + }, + 2: { # 2к ~60 м² + "median": 12_500_000, + "low": 11_000_000, + "high": 14_000_000, + "ppm2": 208_000, + "ref_area": 60.0, + }, + 3: { # 3к ~80 м² + "median": 17_000_000, + "low": 15_000_000, + "high": 19_000_000, + "ppm2": 213_000, + "ref_area": 80.0, + }, +} + + +def _floor_factor(floor: int, total_floors: int) -> float: + """±5% поправка за этаж: 1й и последний этаж снижают цену.""" + if floor == 1: + return 0.95 + if floor == total_floors: + return 0.97 + return 1.0 + + +def _repair_factor(repair_state: str | None) -> float: + """±10% поправка за состояние отделки.""" + factors = { + "needs_repair": 0.90, + "standard": 1.00, + "good": 1.05, + "excellent": 1.10, + } + return factors.get(repair_state or "standard", 1.0) + + +def _confidence(rooms: int) -> str: + if 1 <= rooms <= 3: + return "high" + return "medium" + + +def _gen_analogs( + rooms: int, + area_m2: float, + base_ppm2: int, + n: int, + *, + is_listing: bool, +) -> list[AnalogLot]: + """Генерирует список фейковых аналогов (объявления или сделки).""" + rng = random.Random(42 + rooms + n) + result: list[AnalogLot] = [] + today = datetime.now(tz=UTC).date() + for i in range(n): + street = _EKB_STREETS[i % len(_EKB_STREETS)] + building_no = rng.randint(1, 120) + apt_no = rng.randint(1, 300) + addr = f"г. Екатеринбург, {street}, {building_no}, кв. {apt_no}" + + area_jitter = area_m2 * rng.uniform(0.85, 1.15) + ppm2_jitter = int(base_ppm2 * rng.uniform(0.90, 1.10)) + price = int(area_jitter * ppm2_jitter) + + floor_val = rng.randint(2, 16) + total_fl = rng.randint(floor_val, 20) + + if is_listing: + dom = rng.randint(5, 120) + listing_dt = today - timedelta(days=dom) + else: + dom = rng.randint(10, 60) + listing_dt = today - timedelta(days=rng.randint(30, 365)) + + result.append( + AnalogLot( + address=addr, + area_m2=round(area_jitter, 1), + rooms=rooms if rooms > 0 else 0, + floor=floor_val, + total_floors=total_fl, + price_rub=price, + price_per_m2=ppm2_jitter, + listing_date=listing_dt, + days_on_market=dom, + photo_url=None, + ) + ) + return result + + +def _mock_estimate(payload: TradeInEstimateInput) -> AggregatedEstimate: + """Возвращает mock-оценку на основе диапазонов ЕКБ 2026. + + Логика: + - Берём базовый band по rooms (0-3+). + - Масштабируем на фактическую площадь относительно референсной. + - Применяем поправку за этаж (±5%) и отделку (±10%). + - Генерируем 7-10 аналогов (листинги) и 3-5 actual_deals. + """ + rooms_key = min(payload.rooms, 3) # 4к+ → диапазон 3к + band = _PRICE_BANDS[rooms_key] + + # Масштаб по площади + ref_area: float = band["ref_area"] # type: ignore[assignment] + area_scale = payload.area_m2 / ref_area + + ff = _floor_factor(payload.floor, payload.total_floors) + rf = _repair_factor(payload.repair_state) + combined = area_scale * ff * rf + + median = int(band["median"] * combined) # type: ignore[operator] + low = int(band["low"] * combined) # type: ignore[operator] + high = int(band["high"] * combined) # type: ignore[operator] + ppm2 = int(band["ppm2"] * ff * rf) # type: ignore[operator] + + n_analogs = random.randint(7, 10) + n_deals = random.randint(3, 5) + + analogs = _gen_analogs(rooms_key, payload.area_m2, ppm2, n_analogs, is_listing=True) + actual_deals = _gen_analogs( + rooms_key, payload.area_m2, int(ppm2 * 0.93), n_deals, is_listing=False + ) + + now = datetime.now(tz=UTC) + return AggregatedEstimate( + estimate_id=uuid4(), + median_price_rub=median, + range_low_rub=low, + range_high_rub=high, + median_price_per_m2=ppm2, + confidence=_confidence(payload.rooms), + n_analogs=n_analogs + n_deals, + period_months=24, + analogs=analogs, + actual_deals=actual_deals, + expires_at=now + timedelta(hours=24), + ) + + +@router.post("/estimate", response_model=AggregatedEstimate) +def estimate( + payload: TradeInEstimateInput, + db: Annotated[Session, Depends(get_db)], +) -> AggregatedEstimate: + """MOCK реализация оценки квартиры для Trade-In. + + TODO TI-1b: заменить на реальный SQL aggregation из objective_lots + после OBJ-1/2 merge (issue #314). + """ + result = _mock_estimate(payload) + + analogs_json = json.dumps( + [a.model_dump(mode="json") for a in result.analogs], + ensure_ascii=False, + ) + deals_json = json.dumps( + [a.model_dump(mode="json") for a in result.actual_deals], + ensure_ascii=False, + ) + + db.execute( + text( + """ + INSERT INTO trade_in_estimates ( + id, address, area_m2, rooms, floor, total_floors, + year_built, house_type, repair_state, has_balcony, + median_price, range_low, range_high, median_price_per_m2, + confidence, n_analogs, analogs, actual_deals, expires_at + ) VALUES ( + CAST(:id AS uuid), + :address, :area_m2, :rooms, :floor, :total_floors, + :year_built, :house_type, :repair_state, :has_balcony, + :median_price, :range_low, :range_high, :median_price_per_m2, + :confidence, :n_analogs, + CAST(:analogs AS jsonb), + CAST(:actual_deals AS jsonb), + :expires_at + ) + """ + ), + { + "id": str(result.estimate_id), + "address": payload.address, + "area_m2": payload.area_m2, + "rooms": payload.rooms, + "floor": payload.floor, + "total_floors": payload.total_floors, + "year_built": payload.year_built, + "house_type": payload.house_type, + "repair_state": payload.repair_state, + "has_balcony": payload.has_balcony, + "median_price": result.median_price_rub, + "range_low": result.range_low_rub, + "range_high": result.range_high_rub, + "median_price_per_m2": result.median_price_per_m2, + "confidence": result.confidence, + "n_analogs": result.n_analogs, + "analogs": analogs_json, + "actual_deals": deals_json, + "expires_at": result.expires_at, + }, + ) + db.commit() + + logger.info( + "trade_in estimate saved id=%s rooms=%d area=%.1f confidence=%s", + result.estimate_id, + payload.rooms, + payload.area_m2, + result.confidence, + ) + return result + + +@router.get("/estimate/{estimate_id}", response_model=AggregatedEstimate) +def get_estimate( + estimate_id: UUID, + db: Annotated[Session, Depends(get_db)], +) -> AggregatedEstimate: + """Получить сохранённую оценку по UUID (для генерации PDF). + + Возвращает 404 если оценка не найдена или TTL истёк. + """ + row = db.execute( + text( + """ + SELECT id, median_price, range_low, range_high, median_price_per_m2, + confidence, n_analogs, analogs, actual_deals, expires_at, + address, area_m2, rooms + FROM trade_in_estimates + WHERE id = CAST(:id AS uuid) + AND expires_at > NOW() + """ + ), + {"id": str(estimate_id)}, + ).fetchone() + + if row is None: + raise HTTPException(status_code=404, detail="estimate not found or expired") + + analogs = [AnalogLot(**a) for a in (row.analogs or [])] + actual_deals = [AnalogLot(**a) for a in (row.actual_deals or [])] + + return AggregatedEstimate( + estimate_id=row.id, + median_price_rub=row.median_price, + range_low_rub=row.range_low, + range_high_rub=row.range_high, + median_price_per_m2=row.median_price_per_m2, + confidence=row.confidence, + n_analogs=row.n_analogs, + period_months=24, + analogs=analogs, + actual_deals=actual_deals, + expires_at=row.expires_at, + ) + + +@router.get("/estimate/{estimate_id}/pdf") +def estimate_pdf( + estimate_id: UUID, + db: Annotated[Session, Depends(get_db)], +) -> Response: + """Скачать 4-страничный PDF-отчёт для оценки trade-in. + + Возвращает application/pdf attachment. + 404 — оценка не найдена. + 410 — оценка просрочена (TTL 24ч). + """ + row = db.execute( + text( + """ + SELECT id, median_price, range_low, range_high, median_price_per_m2, + confidence, n_analogs, analogs, actual_deals, expires_at, + address, area_m2, rooms, floor, total_floors, + year_built, house_type, repair_state, has_balcony + FROM trade_in_estimates + WHERE id = CAST(:id AS uuid) + """ + ), + {"id": str(estimate_id)}, + ).fetchone() + + if row is None: + raise HTTPException(status_code=404, detail="estimate not found") + + if row.expires_at.replace(tzinfo=UTC) < datetime.now(tz=UTC): + raise HTTPException(status_code=410, detail="estimate expired (24h TTL)") + + analogs = [AnalogLot(**a) for a in (row.analogs or [])] + actual_deals = [AnalogLot(**a) for a in (row.actual_deals or [])] + + estimate = AggregatedEstimate( + estimate_id=row.id, + median_price_rub=row.median_price, + range_low_rub=row.range_low, + range_high_rub=row.range_high, + median_price_per_m2=row.median_price_per_m2, + confidence=row.confidence, + n_analogs=row.n_analogs, + period_months=24, + analogs=analogs, + actual_deals=actual_deals, + expires_at=row.expires_at, + ) + input_snapshot = { + "address": row.address, + "area_m2": row.area_m2, + "rooms": row.rooms, + "floor": row.floor, + "total_floors": row.total_floors, + "year_built": row.year_built, + "house_type": row.house_type, + "repair_state": row.repair_state, + "has_balcony": row.has_balcony, + } + + pdf_bytes = generate_trade_in_pdf(estimate, input_snapshot) + filename = f"trade-in-{estimate_id}.pdf" + logger.info("PDF generated estimate_id=%s size=%d", estimate_id, len(pdf_bytes)) + return Response( + content=pdf_bytes, + media_type="application/pdf", + headers={"Content-Disposition": f'attachment; filename="{filename}"'}, + ) diff --git a/backend/app/api/v1/users.py b/backend/app/api/v1/users.py new file mode 100644 index 00000000..cd752df6 --- /dev/null +++ b/backend/app/api/v1/users.py @@ -0,0 +1,30 @@ +"""Users endpoints. + +GET /api/v1/users/me/recent-parcels — история недавно просмотренных участков. +STUB: возвращает пустой список до реализации NextAuth (Wave 3). +""" + +from __future__ import annotations + +import logging +from typing import Annotated + +from fastapi import APIRouter, Query + +logger = logging.getLogger(__name__) + +router = APIRouter() + + +@router.get("/users/me/recent-parcels") +def get_recent_parcels( + limit: Annotated[int, Query(ge=1, le=50)] = 10, +) -> dict: + """STUB. Real impl после NextAuth (Wave 3) — будет читать parcel_analysis_history table.""" + logger.info("recent-parcels stub called, limit=%d", limit) + return { + "items": [], + "total": 0, + "stub": True, + "real_impl_after": "NextAuth", + } diff --git a/backend/app/core/config.py b/backend/app/core/config.py index 69f9b1ff..4fea5f2c 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -1,3 +1,7 @@ +import os +import warnings + +from pydantic import model_validator from pydantic_settings import BaseSettings, SettingsConfigDict @@ -7,13 +11,42 @@ class Settings(BaseSettings): database_url: str = "postgresql+psycopg://gendesign:gendesign@localhost:5432/gendesign" redis_url: str = "redis://localhost:6379/0" cors_origins: list[str] = ["http://localhost:3000"] - sentry_dsn: str | None = None - # Release tag для Sentry — обычно git short sha, проставляется - # deploy.yml в backend/.env.runtime (см. workflow). Локально оставляем - # пустым — Sentry припишет 'unknown'. - sentry_release: str | None = None + # GlitchTip error tracking (Sentry-compatible self-hosted). + # Формат DSN: https://@errors.gendsgn.ru/ + # Пустая строка / None = SDK не инициализируется (no-op). + glitchtip_dsn: str | None = None + glitchtip_traces_sample_rate: float = 0.05 environment: str = "dev" + @model_validator(mode="after") + def _promote_legacy_sentry_dsn(self) -> "Settings": + """Backward-compat: legacy SENTRY_DSN → glitchtip_dsn ТОЛЬКО для self-hosted. + + Старый VPS .env.runtime мог содержать SENTRY_DSN=https://...@sentry.io/... + (legacy SaaS Sentry). Промоутить его НЕЛЬЗЯ — события пойдут в чужой проект. + Принимаем только URLs указывающие на наш errors.gendsgn.ru host. + """ + if not self.glitchtip_dsn: + legacy = os.getenv("SENTRY_DSN") + if legacy: + if "errors.gendsgn.ru" in legacy: + warnings.warn( + "SENTRY_DSN is set but ignored by pydantic; rename to GLITCHTIP_DSN. " + "Auto-promoting for backward compat.", + DeprecationWarning, + stacklevel=2, + ) + self.glitchtip_dsn = legacy + else: + warnings.warn( + f"SENTRY_DSN points to non-GlitchTip host " + f"({legacy.split('@', 1)[-1][:40]}...) — ignoring. " + "Set GLITCHTIP_DSN explicitly.", + UserWarning, + stacklevel=2, + ) + return self + # External APIs (Stage 2) rosreestr_pkk_base_url: str = "https://pkk.rosreestr.ru/api/features/1" overpass_url: str = "https://overpass-api.de/api/interpreter" diff --git a/backend/app/main.py b/backend/app/main.py index 93db3a29..07071a70 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -1,33 +1,70 @@ +import logging +import os from collections.abc import AsyncIterator from contextlib import asynccontextmanager import sentry_sdk from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware +from sentry_sdk.integrations.celery import CeleryIntegration +from sentry_sdk.integrations.fastapi import FastApiIntegration +from sentry_sdk.integrations.httpx import HttpxIntegration +from sentry_sdk.integrations.logging import LoggingIntegration +from sentry_sdk.integrations.sqlalchemy import SqlalchemyIntegration +from sentry_sdk.integrations.starlette import StarletteIntegration from app.api.v1 import ( admin_cadastre, + admin_etl, admin_jobs, admin_leads, admin_scrape, admin_weight_profiles, analytics, concepts, + custom_pois, + landing, parcels, photos, + pilot, + trade_in, + users, ) from app.core.config import settings +from app.observability.sentry_scrub import scrub_sensitive_query + +logger = logging.getLogger(__name__) + +# Инициализируем SDK до создания FastAPI app, чтобы все инструменты +# (middleware, маршруты) видели активный client с самого старта процесса. +# GlitchTip не поддерживает profiling — profiles_sample_rate=0.0. +if settings.glitchtip_dsn: + sentry_sdk.init( + dsn=settings.glitchtip_dsn, + environment=settings.environment, + release=os.getenv("GIT_SHA") or os.getenv("SENTRY_RELEASE") or "unknown", + traces_sample_rate=settings.glitchtip_traces_sample_rate, + profiles_sample_rate=0.0, + send_default_pii=False, + before_send_transaction=scrub_sensitive_query, + integrations=[ + StarletteIntegration(), + FastApiIntegration(), + CeleryIntegration(monitor_beat_tasks=True), + SqlalchemyIntegration(), + HttpxIntegration(), + LoggingIntegration(level=logging.INFO, event_level=logging.ERROR), + ], + ) + logger.info( + "GlitchTip SDK initialised (env=%s, traces=%.2f)", + settings.environment, + settings.glitchtip_traces_sample_rate, + ) @asynccontextmanager async def lifespan(app: FastAPI) -> AsyncIterator[None]: - if settings.sentry_dsn: - sentry_sdk.init( - dsn=settings.sentry_dsn, - environment=settings.environment, - release=settings.sentry_release, - traces_sample_rate=0.1, - ) yield @@ -53,11 +90,21 @@ app.include_router( tags=["admin", "site-finder"], ) app.include_router(photos.router, prefix="/api/v1/photos", tags=["photos"]) +app.include_router(custom_pois.router, prefix="/api/v1/custom-pois", tags=["custom-pois"]) app.include_router( admin_cadastre.router, prefix="/api/v1/admin/cadastre", tags=["admin", "cadastre"], ) +app.include_router( + admin_etl.router, + prefix="/api/v1/admin/etl", + tags=["admin", "etl"], +) +app.include_router(trade_in.router, prefix="/api/v1/trade-in", tags=["trade-in"]) +app.include_router(landing.router, prefix="/api/v1", tags=["landing"]) +app.include_router(pilot.router, prefix="/api/v1/pilot", tags=["pilot"]) +app.include_router(users.router, prefix="/api/v1", tags=["users"]) @app.get("/health") diff --git a/backend/app/observability/__init__.py b/backend/app/observability/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/app/observability/sentry_scrub.py b/backend/app/observability/sentry_scrub.py new file mode 100644 index 00000000..af936c31 --- /dev/null +++ b/backend/app/observability/sentry_scrub.py @@ -0,0 +1,49 @@ +"""Хук before_send_transaction для GlitchTip/Sentry SDK. + +Redact-ит api keys / tokens из URL-spans перед отправкой — чтобы +секреты (apiKey=..., api_key=..., token=...) не утекали в GlitchTip +через HttpxIntegration performance-spans. +""" + +from __future__ import annotations + +import re +from typing import Any + +from sentry_sdk.types import Event + +_SENSITIVE_PARAM_RE = re.compile( + r"((?:api[_-]?[Kk]ey|token|access[_-]?token|secret)=)([^&\s]+)", + re.IGNORECASE, +) + + +def _redact(value: Any) -> Any: + """Заменить значения чувствительных query-параметров на [REDACTED].""" + if isinstance(value, str): + return _SENSITIVE_PARAM_RE.sub(r"\1[REDACTED]", value) + return value + + +def scrub_sensitive_query(event: Event, _hint: dict[str, Any]) -> Event | None: + """Redact api keys / tokens из URL spans перед отправкой в GlitchTip. + + Обрабатывает: + - span.data["url"], span.data["http.url"], span.data["http.target"] + - span["description"] + - event["request"]["url"] + """ + for span in event.get("spans") or []: + data = span.get("data") + if isinstance(data, dict): + for key in ("url", "http.url", "http.target"): + if key in data: + data[key] = _redact(data[key]) + if "description" in span: + span["description"] = _redact(span["description"]) + + request = event.get("request") + if isinstance(request, dict) and "url" in request: + request["url"] = _redact(request["url"]) + + return event diff --git a/backend/app/schemas/custom_poi.py b/backend/app/schemas/custom_poi.py new file mode 100644 index 00000000..7b7263d2 --- /dev/null +++ b/backend/app/schemas/custom_poi.py @@ -0,0 +1,60 @@ +"""Pydantic schemas для user_custom_pois (#254). + +CustomPoiCreate — тело POST-запроса. +CustomPoiUpdate — тело PATCH-запроса (все поля опциональны). +CustomPoiOut — ответ API (включает id, created_at, updated_at). +""" + +from __future__ import annotations + +from datetime import datetime + +from pydantic import BaseModel, Field + +# Диапазон весов кастомных POI шире системных ([-5, 5]), +# чтобы пользователь мог сильно усилить или исключить POI. +_CUSTOM_WEIGHT_MIN: float = -5.0 +_CUSTOM_WEIGHT_MAX: float = 5.0 + + +class CustomPoiCreate(BaseModel): + name: str = Field(..., min_length=1, max_length=200, description="Отображаемое название точки") + category: str | None = Field(None, max_length=100, description="Произвольная категория") + weight: float = Field( + ..., + ge=_CUSTOM_WEIGHT_MIN, + le=_CUSTOM_WEIGHT_MAX, + description="Вес точки в scoring [-5, 5]", + ) + lon: float = Field(..., ge=-180.0, le=180.0, description="Долгота WGS-84") + lat: float = Field(..., ge=-90.0, le=90.0, description="Широта WGS-84") + parcel_cad: str | None = Field( + None, + max_length=100, + description="Кадастровый номер участка; NULL = глобальная для пользователя", + ) + notes: str | None = Field(None, max_length=2000, description="Примечания") + + +class CustomPoiUpdate(BaseModel): + name: str | None = Field(None, min_length=1, max_length=200) + category: str | None = None + weight: float | None = Field(None, ge=_CUSTOM_WEIGHT_MIN, le=_CUSTOM_WEIGHT_MAX) + lon: float | None = Field(None, ge=-180.0, le=180.0) + lat: float | None = Field(None, ge=-90.0, le=90.0) + parcel_cad: str | None = Field(None, max_length=100) + notes: str | None = None + + +class CustomPoiOut(BaseModel): + id: int + user_id: str + parcel_cad: str | None + name: str + category: str | None + weight: float + lon: float + lat: float + notes: str | None + created_at: datetime + updated_at: datetime diff --git a/backend/app/schemas/parcel.py b/backend/app/schemas/parcel.py index 20e2c30c..1e6d3c1a 100644 --- a/backend/app/schemas/parcel.py +++ b/backend/app/schemas/parcel.py @@ -1,7 +1,127 @@ -from typing import Any +import datetime as dt +from typing import Any, Literal from pydantic import BaseModel, Field +# ── #105 Phase 5: Recent permits schemas ────────────────────────────────────── + + +class RecentPermit(BaseModel): + """Одно строительное разрешение (РНС или РВЭ) из ekburg_construction_permits.""" + + permit_type: str + permit_number: str + issue_date: str | None + developer_name: str | None + developer_inn: str | None + object_name: str | None + object_type: str | None + construction_address: str | None + total_area_sqm: float | None + + +class PermitsSummary(BaseModel): + """Агрегированная сводка по разрешениям в квартале.""" + + rns_count: int + rve_count: int + rns_total_area_sqm: float + by_developer: list[dict[str, Any]] + + +# ── Connection points schemas (issue #115) ──────────────────────────────────── + + +class EngineeringStructure(BaseModel): + name: str | None + type: str | None + cad_num: str | None + distance_to_boundary_m: float + geometry_geojson: dict[str, Any] + readable_address: str | None + raw_props: dict[str, Any] + source: str + + +class ZouitOverlap(BaseModel): + reg_numb_border: str | None + type_zone: str | None + subcategory: int | None + intersects_parcel: bool + geometry_geojson: dict[str, Any] + raw_props: dict[str, Any] + source: str + + +class ConnectionPointsSummary(BaseModel): + nearest_structure_distance_m: float | None + in_protection_zone: bool + protection_zones_intersecting: int + total_structures_in_radius: int + + +class ConnectionPointsResponse(BaseModel): + engineering_structures: list[EngineeringStructure] + zouit_engineering_overlaps: list[ZouitOverlap] + summary: ConnectionPointsSummary + dump_available: bool + dump_fetched_at: str | None + + +# ── NSPD Risk Zones schemas (issue #94 TIER 3) ─────────────────────────────── + + +class RiskZone(BaseModel): + """Одна риск-зона НСПД (TIER 3), пересекающая участок.""" + + layer: str # e.g. "risk_flooding", "risk_landslide" + subtype: str | None # человекочитаемое название из NSPD properties + geom_wkt: str | None # WKT геометрии risk-зоны в EPSG:4326 + intersection_area_sqm: float | None # площадь пересечения с участком, м² + + +# ── NSPD Opportunity + Red Lines schemas (issue #94 TIER 4) ────────────────── + + +class OpportunityParcel(BaseModel): + """TIER 4: opportunity ЗУ вблизи участка (auction/scheme/free/future/oopt). + + Поля: + layer: тип opportunity — "auction_parcels" | "scheme_parcels" | + "free_parcels" | "future_parcels" | "oopt" + cad_num: кадастровый номер ЗУ (если доступен в NSPD properties) + distance_m: расстояние от centroid анализируемого участка до ЗУ, м + geom_wkt: WKT геометрии ЗУ в EPSG:4326 + """ + + layer: str + cad_num: str | None + distance_m: float | None + geom_wkt: str | None + + +class RedLine(BaseModel): + """TIER 4: красная линия застройки (NSPD layer 879243). + + Поля: + geom_wkt: WKT геометрии линии в EPSG:4326 + intersection_length_m: длина пересечения линии с участком, м + (null если только nearby, не intersect) + distance_m: расстояние от ближайшей точки линии до участка, м + (null если линия пересекает участок) + + UI-семантика: + intersection_length_m != null → красная линия ПЕРЕСЕКАЕТ участок (ALERT) + distance_m != null → красная линия рядом (warning) + """ + + geom_wkt: str | None + intersection_length_m: float | None # null если только nearby, не intersect + distance_m: float | None # null если intersect + + +# ── Parcel search/detail schemas ────────────────────────────────────────────── + class ParcelFilter(BaseModel): vri: list[str] | None = None @@ -48,3 +168,219 @@ class ParcelSearchResponse(BaseModel): class ParcelDetail(ParcelSummary): geometry_geojson: dict[str, Any] enrichment: dict[str, Any] + + +# ── Competitors endpoint schemas ────────────────────────────────────────────── + +TimeWindow = Literal["last_month", "last_quarter", "last_year"] +ObjClassFilter = Literal["economy", "comfort", "business"] + + +class CompetitorsRequest(BaseModel): + radius_km: float = Field(default=1.0, ge=0.1, le=1.5) + time_window: TimeWindow = "last_quarter" + obj_class_filter: ObjClassFilter | None = None + exclude_obj_ids: list[int] = Field(default_factory=list) + + +class Competitor(BaseModel): + obj_id: int + comm_name: str | None + dev_name: str | None + obj_class: str | None + distance_m: float + lat: float + lng: float + stage: str | None + site_status: str | None = None + ready_dt: dt.date | None = None + flats_total: int | None + flats_sold: int | None + sold_pct: float | None + velocity_per_month: float + avg_price_per_m2: float | None + is_active: bool + + +class CompetitorsSummary(BaseModel): + total_competitors: int + active_count: int + weighted_avg_velocity: float + radius_km: float + time_window: str + + +class CompetitorsResponse(BaseModel): + competitors: list[Competitor] + summary: CompetitorsSummary + + +# ── Layout analysis (Issue #113) ─────────────────────────────────────────── + + +class LayoutSignature(BaseModel): + """Минимальная сигнатура планировки = (room_bucket, area_bin). + + Phase 2.1: layout_type/balcony_count в БД нет, ждут B2B Объектив (#52). + """ + + room_bucket: Literal["studio", "1", "2", "3", "4+"] + area_bin: Literal["<25", "25-40", "40-60", "60-80", "80-100", "100+"] + + +class BestLayoutsRequest(BaseModel): + """Параметры запроса top-планировок в радиусе вокруг участка.""" + + radius_km: float = Field(default=1.0, ge=0.1, le=1.5) + time_window: Literal["last_month", "last_quarter", "last_year"] = "last_quarter" + filter_competitor_obj_ids: list[int] | None = None + exclude_competitor_obj_ids: list[int] = Field(default_factory=list) + min_velocity_per_month: float = Field(default=0.5, ge=0.0, le=100.0) + obj_class_filter: Literal["economy", "comfort", "business"] | None = None + target_total_flats: int | None = Field(default=None, ge=1, le=10000) + + +class TopLayoutRow(BaseModel): + """Одна строка top-planirovok ranking'а.""" + + rank: int + room_bucket: str + area_bin: str + signature: str + competitor_obj_ids: list[int] + competitor_count: int + total_sold_in_window: int + velocity_per_month: float + avg_price_per_m2_rub: float | None # NULL если objective не покрывает obj + avg_area_m2: float + supply_units_in_radius: int + sold_pct_of_supply: float | None # NULL если supply=0; clamped at 100.0 + is_oversold: bool # True когда raw sum_deals/supply > 100% (несопоставимые окна) + + +class LayoutTzMixRow(BaseModel): + """Строка рекомендации unit-mix для ТЗ.""" + + room_bucket: str + pct: int # 0..100, sum total = 100 + abs_units: int | None # NULL если target_total_flats не задан + avg_target_area_m2: float | None + + +class LayoutTzRecommendation(BaseModel): + """Рекомендация для ТЗ на проектирование.""" + + rationale_text: str + mix: list[LayoutTzMixRow] + weighted_avg_price_per_m2_rub: float | None + based_on_obj_count: int + based_on_total_deals: int + data_window_start: dt.date + data_window_end: dt.date + # Fix SF-09 review: True если pathological case — все bucket'ы выше cap, + # redistribute невозможен. Frontend использует для отображения warning banner. + cap_skipped: bool = False + + +class LayoutDataQuality(BaseModel): + """Метаданные качества данных (coverage).""" + + objects_with_velocity_data: int + objects_total_in_radius: int + velocity_coverage_pct: float + confidence: Literal["high", "medium", "low"] + + +class BestLayoutsResponse(BaseModel): + """Ответ /best-layouts endpoint'а.""" + + top_layouts: list[TopLayoutRow] + recommendation_for_tz: LayoutTzRecommendation + data_quality: LayoutDataQuality + + +# ── Analyze endpoint market price (#33) ────────────────────────────────────── + + +class MarketPrice(BaseModel): + """Ценовая статистика квартала из mv_quarter_price_per_m2 (Issue #33). + + Источник: rosreestr_deals, фильтр realestate_type_code='002001003000' (ДДУ), + скользящее окно 24 мес., 30K–800K руб/м², HAVING >= 3 сделок. + """ + + p25: float | None = None + median: float | None = None + p75: float | None = None + mean: float | None = None + deals_count: int = 0 + median_6m: float | None = None + median_12m: float | None = None + median_24m: float | None = None + last_deal_date: str | None = None # ISO date + source: Literal["quarter_mv", "no_data"] = "no_data" + + +# ── Analyze endpoint parcel meta (#29 G2) ──────────────────────────────────── + + +class ParcelMeta(BaseModel): + """Кадастровые метаданные участка из cad_parcels (#29 G2). + + Поля берутся из NSPD bulk-ingest (issue #168). Строки сырые, без нормализации. + """ + + permitted_use: str | None = None # permitted_use_established_by_document + land_category: str | None = None # land_record_category_type + land_subtype: str | None = None # land_record_subtype + cad_cost: float | None = None # cost_value (кадастровая стоимость, руб.) + source: str = "cad_parcels" + + +# ── Analyze endpoint inline weights (#201) ──────────────────────────────────── + + +class AnalyzeRequest(BaseModel): + """Опциональное тело запроса POST /analyze. + + Позволяет передать inline POI-веса напрямую в запросе без сохранения + профиля. Если задан weights — применяется с наивысшим приоритетом + (выше profile_id и user default). + """ + + weights: dict[str, float] | None = Field( + default=None, + description=( + "Inline POI weights override (категория → weight). " + "Если задан — применяется к scoring, без обязательного profile save. " + "Validated против ALLOWED_CATEGORIES + MIN_WEIGHT/MAX_WEIGHT." + ), + ) + + +# ── SF-B1: by-bbox map entry schemas (#307) ────────────────────────────────── + + +class ParcelMapMarker(BaseModel): + """Один маркер участка на карте — облегчённый ответ для bbox-запроса. + + status берётся из parcel_user_status если передан user_id, иначе null. + last_analysis_date — placeholder до реализации B2 auth. + """ + + cad_num: str + centroid_lat: float + centroid_lon: float + area_m2: float | None + land_category: str | None + status: str | None # 'in_work' | 'favorite' | 'dismissed' | null + last_analysis_date: str | None # placeholder, real after B2 auth + + +class ParcelBboxResponse(BaseModel): + """Ответ GET /parcels/by-bbox.""" + + parcels: list[ParcelMapMarker] + count: int + limit: int + bbox_area_km2: float diff --git a/backend/app/schemas/trade_in.py b/backend/app/schemas/trade_in.py new file mode 100644 index 00000000..de21322a --- /dev/null +++ b/backend/app/schemas/trade_in.py @@ -0,0 +1,51 @@ +"""Pydantic schemas for Trade-In Estimator. + +POST /api/v1/trade-in/estimate → AggregatedEstimate +""" + +from __future__ import annotations + +from datetime import date, datetime +from typing import Literal +from uuid import UUID + +from pydantic import BaseModel, Field + + +class TradeInEstimateInput(BaseModel): + address: str = Field(min_length=3, max_length=500) + area_m2: float = Field(gt=10, lt=500) + rooms: int = Field(ge=0, le=10) # 0 = студия + floor: int = Field(ge=1, le=100) + total_floors: int = Field(ge=1, le=100) + year_built: int | None = Field(default=None, ge=1800, le=2100) + house_type: Literal["panel", "brick", "monolith", "monolith_brick", "other"] | None = None + repair_state: Literal["needs_repair", "standard", "good", "excellent"] | None = None + has_balcony: bool | None = None + + +class AnalogLot(BaseModel): + address: str + area_m2: float + rooms: int + floor: int | None + total_floors: int | None + price_rub: int + price_per_m2: int + listing_date: date | None + days_on_market: int | None + photo_url: str | None = None + + +class AggregatedEstimate(BaseModel): + estimate_id: UUID + median_price_rub: int + range_low_rub: int + range_high_rub: int + median_price_per_m2: int + confidence: Literal["low", "medium", "high"] + n_analogs: int + period_months: int # 24 + analogs: list[AnalogLot] # top 5-10 listings + actual_deals: list[AnalogLot] # реальные продажи last 12 mo + expires_at: datetime diff --git a/backend/app/scrapers/nspd_bulk_client.py b/backend/app/scrapers/nspd_bulk_client.py index 53d93c8e..5c337bd2 100644 --- a/backend/app/scrapers/nspd_bulk_client.py +++ b/backend/app/scrapers/nspd_bulk_client.py @@ -57,8 +57,12 @@ _CAT_TO_THEMATIC_SEARCH_ID: dict[int, int] = { 39663: 15, # ЕНК } -# Лимит одновременных запросов (глобальный, shared между всеми instance'ами) -_SEMAPHORE = asyncio.Semaphore(3) +# Concurrency cap per HTTP fetch loop. Issue #260 (Sub-PR B re-review): +# module-level asyncio.Semaphore() bind'ится к event loop первого asyncio.run() +# вызова, на втором — RuntimeError: bound to a different event loop. +# search_by_quarter делает N sequential asyncio.run() per layer → semaphore надо +# создавать per-instance в __aenter__ (под текущим running loop). +_SEMAPHORE_LIMIT = 3 # Таймаут подключения и чтения DEFAULT_TIMEOUT = httpx.Timeout(30.0, connect=10.0) @@ -131,6 +135,10 @@ class NSPDBulkClient: self._headers = headers or DEFAULT_HEADERS self._max_retries = max_retries self._client: httpx.AsyncClient | None = None + # Issue #260: semaphore создаётся в __aenter__ под текущий event loop. + # Module-level не работал — sequential asyncio.run() bind'ил его к + # закрытому loop'у первого вызова. + self._sem: asyncio.Semaphore | None = None async def __aenter__(self) -> NSPDBulkClient: # NSPD prod chain contains a self-signed/internal CA on Beget VPS → @@ -144,6 +152,8 @@ class NSPDBulkClient: follow_redirects=True, verify=False, ) + # Создаём semaphore под running loop — safe для sequential asyncio.run(). + self._sem = asyncio.Semaphore(_SEMAPHORE_LIMIT) return self async def __aexit__(self, *_: Any) -> None: @@ -161,12 +171,12 @@ class NSPDBulkClient: NspdBulkRateLimitError при 429 после исчерпания retries. NspdBulkError при прочих 4xx/5xx. """ - if self._client is None: + if self._client is None or self._sem is None: raise RuntimeError("NSPDBulkClient не инициализирован — используй async with") attempt = 0 while True: - async with _SEMAPHORE: + async with self._sem: # Небольшой jitter чтобы N одновременных запросов не начинались в 0мс await asyncio.sleep(0.05) try: @@ -374,7 +384,107 @@ class NSPDBulkClient: raw_features: list[dict[str, Any]] = (data or {}).get("features") or [] return [NSPDBulkFeature.model_validate(f) for f in raw_features] - # ── 3. list_objects_in_building ─────────────────────────────────────────── + # ── 3. get_features_in_bbox_grid ───────────────────────────────────────── + + async def get_features_in_bbox_grid( + self, + layer_id: int, + bbox: tuple[float, float, float, float], + *, + grid_n: int = 7, + tile_size: int = 512, + ) -> list[dict]: + """Grid-walk WMS GetFeatureInfo по layer_id в bbox. + + Разбивает bbox на grid_n × grid_n ячеек, кликает центр каждой. + Дедуплицирует результаты по feature id. + + Args: + layer_id: NSPD layer ID (например 875838 для ПЗЗ). + bbox: (xmin, ymin, xmax, ymax) в EPSG:3857 (метры). + grid_n: количество ячеек по каждой оси (7 → 49 запросов). + tile_size: размер виртуального WMS тайла в пикселях. + + Returns: + Список raw feature dict'ов с полями id, geometry, properties. + Дедуплицированы по feature id. + """ + xmin, ymin, xmax, ymax = bbox + x_step = (xmax - xmin) / grid_n + y_step = (ymax - ymin) / grid_n + click_xy = (tile_size // 2, tile_size // 2) + + tasks = [] + for i in range(grid_n): + for j in range(grid_n): + cell_bbox = ( + xmin + i * x_step, + ymin + j * y_step, + xmin + (i + 1) * x_step, + ymin + (j + 1) * y_step, + ) + tasks.append( + self.wms_feature_info( + layer_id=layer_id, + bbox=cell_bbox, + click_xy=click_xy, + width=tile_size, + height=tile_size, + ) + ) + + # Запускаем все ячейки конкурентно (semaphore ограничивает до 3 concurrent) + cell_results = await asyncio.gather(*tasks, return_exceptions=True) + + seen_ids: set[str] = set() + results: list[dict] = [] + + for idx, cell_result in enumerate(cell_results): + if isinstance(cell_result, Exception): + logger.debug( + "get_features_in_bbox_grid: layer=%d cell=%d error: %s", + layer_id, + idx, + cell_result, + ) + continue + for feature in cell_result: + fid = str(feature.id) if feature.id is not None else "" + if fid and fid in seen_ids: + continue + if fid: + seen_ids.add(fid) + results.append( + { + "id": feature.id, + "geometry": feature.geometry, + "properties": feature.properties, + } + ) + + logger.info( + "get_features_in_bbox_grid: layer=%d grid=%dx%d unique_features=%d", + layer_id, + grid_n, + grid_n, + len(results), + ) + return results + + async def get_territorial_zones_in_bbox( + self, + bbox: tuple[float, float, float, float], + *, + grid_n: int = 7, + ) -> list[dict]: + """Grid-walk WMS GetFeatureInfo для layer 875838 (ПЗЗ территориальные зоны). + + Returns: list of feature dicts с полями id, geometry, properties. + Дедуплицирует по feature id. + """ + return await self.get_features_in_bbox_grid(875838, bbox, grid_n=grid_n) + + # ── 4. list_objects_in_building ─────────────────────────────────────────── # Q3 deferred — метод реализован, но не вызывается в bulk_harvest_quarter MVP. # Готов для per-building помещения/парковка фазы. diff --git a/backend/app/services/analytics_queries.py b/backend/app/services/analytics_queries.py index e80ca003..7276babb 100644 --- a/backend/app/services/analytics_queries.py +++ b/backend/app/services/analytics_queries.py @@ -842,6 +842,251 @@ def object_photos(db: Session, obj_id: int, limit: int = 100) -> list[dict[str, ] +def object_full_detail(db: Session, obj_id: int) -> dict[str, Any] | None: + """Extended object detail — adds all Wave A+B columns (22begh). + + Returns the same base fields as object_detail() PLUS the 30 new columns + from 113_22begh_kn_schema_extension.sql. Falls back gracefully: columns + that haven't been scraped yet return NULL. + """ + row = ( + db.execute( + text( + """ + SELECT o.obj_id, o.hobj_id, o.comm_name, o.addr, o.short_addr, o.region_cd, + o.dev_id, o.dev_name, o.dev_group_name, + o.floor_min, o.floor_max, o.flat_count, + o.square_living, o.ready_dt, o.site_status, o.escrow, o.obj_class, + o.latitude, o.longitude, o.obj_status, o.snapshot_date, + o.energy_eff, o.wall_type, + -- Building specs (22e) + o.first_floor_type, o.section_count, + o.elevators_passenger_count, o.elevators_cargo_count, + o.parking_total_slots, o.guest_parking_inside_count, + o.guest_parking_outside_count, o.ceiling_height_m, + -- Apartment summary (22e) + o.finishing_variants_count, o.has_free_planning, o.avg_flat_area_m2, + -- Yard (22e) + o.playground_kids_count, o.playground_sport_count, + o.has_bike_paths, o.trash_areas_count, + -- OVZ (22e) + o.has_ramp, o.has_low_platforms, o.has_wheelchair_lift, + -- Catalog/UI (22e) + o.flat_area_min, o.flat_area_max, + o.price_min_rub, o.price_max_rub, + o.price_per_m2_min, o.price_per_m2_max, + o.parking_provision_pct, o.project_published_at, + o.project_declaration_num, + -- Metro & scores (22e/22h) + o.metro_nearest_name, o.metro_nearest_walk_minutes, o.metro_top3, + o.domrf_score_location, o.domrf_score_transport, + o.domrf_score_infrastructure, + COALESCE(cb.buildings_count, 0) AS buildings_count + FROM domrf_kn_objects o + LEFT JOIN v_complex_buildings cb ON cb.complex_id = o.obj_id + WHERE o.obj_id = :obj + ORDER BY o.snapshot_date DESC + LIMIT 1 + """ + ), + {"obj": obj_id}, + ) + .mappings() + .first() + ) + if not row: + return None + + metro_top3 = row["metro_top3"] + + return { + "obj_id": row["obj_id"], + "hobj_id": row["hobj_id"], + "comm_name": row["comm_name"], + "addr": row["addr"], + "short_addr": row["short_addr"], + "region_cd": row["region_cd"], + "dev_id": row["dev_id"], + "dev_name": row["dev_name"], + "dev_group_name": row["dev_group_name"], + "floor_min": row["floor_min"], + "floor_max": row["floor_max"], + "flat_count": row["flat_count"], + "square_living": _f(row["square_living"]), + "ready_dt": row["ready_dt"].isoformat() if row["ready_dt"] else None, + "site_status": row["site_status"], + "escrow": row["escrow"], + "obj_class": row["obj_class"], + "latitude": _f(row["latitude"]), + "longitude": _f(row["longitude"]), + "obj_status": row["obj_status"], + "snapshot_date": row["snapshot_date"].isoformat() if row["snapshot_date"] else None, + "energy_eff": row["energy_eff"], + "wall_type": row["wall_type"], + # Building specs + "first_floor_type": row["first_floor_type"], + "section_count": row["section_count"], + "elevators_passenger_count": row["elevators_passenger_count"], + "elevators_cargo_count": row["elevators_cargo_count"], + "parking_total_slots": row["parking_total_slots"], + "guest_parking_inside_count": row["guest_parking_inside_count"], + "guest_parking_outside_count": row["guest_parking_outside_count"], + "ceiling_height_m": _f(row["ceiling_height_m"]), + # Apartment summary + "finishing_variants_count": row["finishing_variants_count"], + "has_free_planning": row["has_free_planning"], + "avg_flat_area_m2": _f(row["avg_flat_area_m2"]), + # Yard + "playground_kids_count": row["playground_kids_count"], + "playground_sport_count": row["playground_sport_count"], + "has_bike_paths": row["has_bike_paths"], + "trash_areas_count": row["trash_areas_count"], + # OVZ + "has_ramp": row["has_ramp"], + "has_low_platforms": row["has_low_platforms"], + "has_wheelchair_lift": row["has_wheelchair_lift"], + # Catalog/UI + "flat_area_min": _f(row["flat_area_min"]), + "flat_area_max": _f(row["flat_area_max"]), + "price_min_rub": row["price_min_rub"], + "price_max_rub": row["price_max_rub"], + "price_per_m2_min": _f(row["price_per_m2_min"]), + "price_per_m2_max": _f(row["price_per_m2_max"]), + "parking_provision_pct": _f(row["parking_provision_pct"]), + "project_published_at": ( + row["project_published_at"].isoformat() if row["project_published_at"] else None + ), + "project_declaration_num": row["project_declaration_num"], + # Metro & scores + "metro_nearest_name": row["metro_nearest_name"], + "metro_nearest_walk_minutes": row["metro_nearest_walk_minutes"], + "metro_top3": metro_top3, # already jsonb → dict/list from psycopg3 + "domrf_score_location": row["domrf_score_location"], + "domrf_score_transport": row["domrf_score_transport"], + "domrf_score_infrastructure": row["domrf_score_infrastructure"], + "buildings_count": int(row["buildings_count"]), + } + + +def object_flats_quartirography(db: Session, obj_id: int) -> list[dict[str, Any]]: + """Per-rooms aggregation из domrf_kn_flats для объекта. + + Группирует по rooms: 1/2/3/Нежилые (rooms IS NULL). + Возвращает count total, count 'free', min/max area, min/max price. + """ + rows = ( + db.execute( + text( + """ + WITH latest AS ( + SELECT MAX(snapshot_date) AS snap + FROM domrf_kn_flats + WHERE obj_id = :obj + ) + SELECT + CASE + WHEN f.rooms IS NULL OR LOWER(f.flat_type) LIKE '%нежил%' + OR LOWER(f.flat_type) LIKE '%nonliv%' THEN 'Нежилые' + WHEN f.rooms = 0 THEN 'Студия' + WHEN f.rooms = 1 THEN '1-комн.' + WHEN f.rooms = 2 THEN '2-комн.' + WHEN f.rooms = 3 THEN '3-комн.' + ELSE (f.rooms::text || '-комн.') + END AS room_label, + COALESCE(f.rooms, -1) AS sort_key, + COUNT(*) AS total_count, + COUNT(*) FILTER (WHERE LOWER(f.status) = 'free' + OR LOWER(f.status) LIKE '%свобод%') + AS free_count, + MIN(f.total_area) AS area_min, + MAX(f.total_area) AS area_max, + MIN(f.price_rub) FILTER (WHERE f.price_rub > 0) + AS price_min, + MAX(f.price_rub) FILTER (WHERE f.price_rub > 0) + AS price_max + FROM domrf_kn_flats f + CROSS JOIN latest l + WHERE f.obj_id = :obj + AND f.snapshot_date = l.snap + GROUP BY room_label, sort_key + ORDER BY sort_key + """ + ), + {"obj": obj_id}, + ) + .mappings() + .all() + ) + return [ + { + "room_label": r["room_label"], + "total_count": r["total_count"], + "free_count": r["free_count"], + "area_min": _f(r["area_min"]), + "area_max": _f(r["area_max"]), + "price_min": r["price_min"], + "price_max": r["price_max"], + } + for r in rows + ] + + +def object_obj_checks(db: Session, obj_id: int) -> list[dict[str, Any]]: + """6 «Проверено на наш.дом.рф» checks из domrf_obj_checks (22f).""" + rows = ( + db.execute( + text( + """ + SELECT check_type, passed, checked_at + FROM domrf_obj_checks + WHERE obj_id = :obj + ORDER BY check_type + """ + ), + {"obj": obj_id}, + ) + .mappings() + .all() + ) + return [ + { + "check_type": r["check_type"], + "passed": r["passed"], + "checked_at": r["checked_at"].isoformat() if r["checked_at"] else None, + } + for r in rows + ] + + +def object_documents(db: Session, obj_id: int) -> list[dict[str, Any]]: + """PDF documents из domrf_kn_documents (22i), сортировка по doc_type + posted_at.""" + rows = ( + db.execute( + text( + """ + SELECT doc_type, doc_num, posted_at, file_url, size_bytes + FROM domrf_kn_documents + WHERE obj_id = :obj + ORDER BY doc_type, posted_at DESC NULLS LAST + """ + ), + {"obj": obj_id}, + ) + .mappings() + .all() + ) + return [ + { + "doc_type": r["doc_type"], + "doc_num": r["doc_num"], + "posted_at": r["posted_at"].isoformat() if r["posted_at"] else None, + "file_url": r["file_url"], + "size_bytes": r["size_bytes"], + } + for r in rows + ] + + def prinzip_funnel_monthly(db: Session, months: int = 24) -> list[dict[str, Any]]: """Воронка по месяцам из materialized view.""" rows = ( @@ -1349,10 +1594,9 @@ def _district_cadastre_baseline(db: Session, *, district_name: str) -> dict[str, WHERE cb.cost_value IS NOT NULL AND cb.area IS NOT NULL AND cb.area >= 100 - -- floors хранится как TEXT (встречаются '1-2', '2-3') — - -- считаем только чистые числа ≥3, либо purpose-fallback. - AND ((cb.floors ~ '^[0-9]+$' AND cb.floors::int >= 3) - OR cb.purpose ILIKE '%многокв%') + -- floors INTEGER (Rosreestr ETL приводит к int); NULL = unknown. + -- Считаем МКД если floors ≥3 или purpose содержит «многокв». + AND (cb.floors >= 3 OR cb.purpose ILIKE '%многокв%') AND (cb.cost_value / NULLIF(cb.area, 0)) BETWEEN 5000 AND 500000 ) diff --git a/backend/app/services/cadastre/bulk_harvest.py b/backend/app/services/cadastre/bulk_harvest.py index 289d4469..8c85bc44 100644 --- a/backend/app/services/cadastre/bulk_harvest.py +++ b/backend/app/services/cadastre/bulk_harvest.py @@ -19,6 +19,7 @@ Resumable: phase_state в cadastre_jobs показывает прогресс. from __future__ import annotations +import hashlib import json import logging from collections.abc import Callable @@ -236,6 +237,20 @@ async def harvest_quarter( } ) + # ── Phase 2.5: grid-walk для territorial_zones (ПЗЗ, layer 875838) ──────── + # Выполняем после основного grid-walk (Phase 2-3). Требует bbox квартала. + quarter_bbox = quarter_bbox_3857(db, quarter) + if quarter_bbox is not None: + update_progress({"phase": "territorial_zones_started", "quarter": quarter}) + try: + tz_features = await client.get_territorial_zones_in_bbox(quarter_bbox) + tz_count = _save_territorial_zones(db, quarter, tz_features) + logger.info( + "harvest_quarter: territorial_zones quarter=%s upserted=%d", quarter, tz_count + ) + except Exception as e: + logger.warning("harvest_quarter: territorial_zones failed quarter=%s: %s", quarter, e) + # ── Phase 4: quarter stats + auto-heal geom из snapshot ───────────────── stats_features = [f for f in snapshot.features if f.category_id == CAT_QUARTER_STATS] if stats_features: @@ -1176,6 +1191,97 @@ def upsert_quarter_stats( ) +# ── cad_territorial_zones upsert ──────────────────────────────────────────── + + +def _save_territorial_zones(db: Session, quarter_cad: str, features: list[dict]) -> int: + """UPSERT territorial_zones features в cad_territorial_zones по zone_id. + + Args: + db: SQLAlchemy session. + quarter_cad: кадастровый номер квартала (3 сегмента). + features: list of raw feature dicts от get_features_in_bbox_grid. + + Returns: + Количество успешно upserted строк. + """ + inserted = 0 + for f in features: + props: dict = f.get("properties") or {} + geom = f.get("geometry") + geom_geojson: str | None = json.dumps(geom) if geom else None + + # zone_id — NSPD feature id или стабильный fallback на основе md5 от properties. + # md5 гарантирует идемпотентность между runs (счётчик inserted сбрасывается). + _raw_id = props.get("id") or props.get("zone_id") or f.get("id") + if _raw_id: + zone_id = str(_raw_id) + else: + _props_hash = hashlib.md5( + json.dumps(props, sort_keys=True).encode("utf-8") + ).hexdigest()[:12] + zone_id = f"{quarter_cad}_{_props_hash}" + zone_code = ( + props.get("zone_code") or props.get("zone_index") or props.get("reg_numb_border") + ) + zone_name = props.get("zone_name") or props.get("zone_type_name") or props.get("type_zone") + permitted_use = props.get("permitted_use") or props.get("vri") + + try: + # begin_nested() требует активной outer-транзакции для SAVEPOINT. + # SQLAlchemy Session (autobegin=True) автоматически начинает tx при первом + # db.execute() в этом loop — outer tx гарантирована. + with db.begin_nested(): + db.execute( + text(""" + INSERT INTO cad_territorial_zones + (quarter_cad, zone_id, zone_code, zone_name, + permitted_use, raw_props, geom) + VALUES ( + CAST(:quarter_cad AS text), + CAST(:zone_id AS text), + CAST(:zone_code AS text), + CAST(:zone_name AS text), + CAST(:permitted_use AS text), + CAST(:raw_props AS jsonb), + CASE WHEN CAST(:geom AS text) IS NOT NULL + THEN ST_Transform( + ST_SetSRID(ST_GeomFromGeoJSON(CAST(:geom AS text)), 3857), + 4326 + )::geography + ELSE NULL + END + ) + ON CONFLICT (zone_id) DO UPDATE SET + zone_code = EXCLUDED.zone_code, + zone_name = EXCLUDED.zone_name, + permitted_use = EXCLUDED.permitted_use, + raw_props = EXCLUDED.raw_props, + geom = EXCLUDED.geom, + fetched_at = NOW() + """), + { + "quarter_cad": quarter_cad, + "zone_id": zone_id, + "zone_code": zone_code, + "zone_name": zone_name, + "permitted_use": permitted_use, + "raw_props": json.dumps(props, ensure_ascii=False), + "geom": geom_geojson, + }, + ) + inserted += 1 + except Exception as e: + logger.warning( + "_save_territorial_zones: upsert failed zone_id=%s quarter=%s: %s", + zone_id, + quarter_cad, + e, + ) + db.commit() + return inserted + + # ── Утилиты ────────────────────────────────────────────────────────────────── diff --git a/backend/app/services/etl/__init__.py b/backend/app/services/etl/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/app/services/etl/objective_backfill.py b/backend/app/services/etl/objective_backfill.py new file mode 100644 index 00000000..bc7023ed --- /dev/null +++ b/backend/app/services/etl/objective_backfill.py @@ -0,0 +1,235 @@ +"""Backfill objective_complex_mapping (#203). + +Auto-match DOM.РФ комплексов к Objective dataset через fuzzy name matching. +Goal: coverage 3% → 40%+ для mv_layout_velocity. + +Schema facts (confirmed via pg MCP): +- domrf_kn_objects: фильтр is_ekb = true (~1285 ЕКБ объектов). + district_name ILIKE '%екатеринбург%' возвращает 0 строк — не использовать. +- objective_corpus_room_month: group_name = 'Екатеринбург' (единственное значение), + 263 distinct project_name. +- objective_complex_mapping: UNIQUE (objective_complex_name, objective_group), + колонки match_method + match_score + is_reviewed поддерживают audit trail. + +match_method history: + 'fuzzy' — legacy Anton SQLite import (avg score 0.98, 127 rows) + 'fuzzy_trgm' — pg_trgm backfill, auto-accept threshold=0.85 (первый запуск) + 'fuzzy_v2' — pg_trgm backfill, pruned threshold=0.80 (второй запуск, #44) + 'manual' — ручная корректура +""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass + +from sqlalchemy import text +from sqlalchemy.orm import Session + +logger = logging.getLogger(__name__) + +# Порог для auto-insert (высокая уверенность) +AUTO_ACCEPT_THRESHOLD = 0.85 + +# Pruned threshold для v2 run (ниже уверенность, is_reviewed=false → review queue) +AUTO_ACCEPT_THRESHOLD_V2 = 0.80 + +# Порог для review queue (средняя уверенность — Phase 2) +REVIEW_THRESHOLD = 0.6 + + +@dataclass +class MatchCandidate: + """Один candidate match DOM.РФ ↔ Objective.""" + + domrf_obj_id: int + domrf_comm_name: str + domrf_dev_name: str | None + objective_project_name: str + similarity_score: float # 0.0..1.0 + + +def find_match_candidates( + db: Session, + *, + only_unmapped: bool = True, + min_threshold: float | None = None, + limit: int | None = None, +) -> list[MatchCandidate]: + """Поиск candidates через pg_trgm similarity. + + Использует CROSS JOIN LATERAL + similarity() для fuzzy match + comm_name (DOM.РФ) ↔ project_name (Objective). + + Args: + db: SQLAlchemy sync Session. + only_unmapped: Если True — пропускает уже-mapped obj_id. + min_threshold: Нижняя граница similarity для фильтрации кандидатов. + По умолчанию REVIEW_THRESHOLD (0.6). + limit: Максимальное число строк результата (для тестирования). + + Returns: + Список MatchCandidate, отсортированных по убыванию similarity_score. + """ + effective_min = min_threshold if min_threshold is not None else REVIEW_THRESHOLD + + # Формируем SQL. LIMIT добавляем через int() — SQL injection safe (только число). + limit_clause = f"LIMIT {int(limit)}" if limit is not None else "" + + sql = text( + f""" + WITH domrf_unmapped AS ( + SELECT o.obj_id, o.comm_name, o.dev_name + FROM domrf_kn_objects o + WHERE o.is_ekb = true + AND o.comm_name IS NOT NULL + AND ( + CAST(:only_unmapped AS boolean) = FALSE + OR NOT EXISTS ( + SELECT 1 FROM objective_complex_mapping cm + WHERE cm.domrf_obj_id = o.obj_id + ) + ) + ), + objective_distinct AS ( + SELECT DISTINCT project_name + FROM objective_corpus_room_month + ) + SELECT + d.obj_id, + d.comm_name, + d.dev_name, + obj.project_name, + similarity(d.comm_name, obj.project_name) AS sim_score + FROM domrf_unmapped d + CROSS JOIN LATERAL ( + SELECT project_name + FROM objective_distinct + WHERE similarity(d.comm_name, project_name) > 0.4 + ORDER BY similarity(d.comm_name, project_name) DESC + LIMIT 1 + ) obj + WHERE similarity(d.comm_name, obj.project_name) >= CAST(:min_threshold AS float) + ORDER BY sim_score DESC + {limit_clause} + """ + ) + + rows = db.execute( + sql, + {"only_unmapped": only_unmapped, "min_threshold": effective_min}, + ).all() + + return [ + MatchCandidate( + domrf_obj_id=int(r[0]), + domrf_comm_name=str(r[1]), + domrf_dev_name=str(r[2]) if r[2] is not None else None, + objective_project_name=str(r[3]), + similarity_score=float(r[4]), + ) + for r in rows + ] + + +def auto_apply_matches( + db: Session, + candidates: list[MatchCandidate], + *, + threshold: float = AUTO_ACCEPT_THRESHOLD, + match_method: str = "fuzzy_trgm", + dry_run: bool = False, +) -> dict[str, int]: + """Apply candidates с score >= threshold в objective_complex_mapping. + + Кандидаты ниже threshold, но >= REVIEW_THRESHOLD попадают в review_queue + (Phase 2 — UI для ручного review, в этом PR только считаются). + + ON CONFLICT DO NOTHING — если пара (objective_complex_name, objective_group) + уже существует, строка пропускается без ошибки. + + Args: + db: SQLAlchemy sync Session. + candidates: Список из find_match_candidates(). + threshold: Минимальный score для auto-insert (default 0.85). + match_method: Значение колонки match_method в БД. По умолчанию 'fuzzy_trgm'. + Для re-run с пониженным порогом передавать 'fuzzy_v2'. + dry_run: Если True — только логирует, не пишет в БД. + + Returns: + dict с ключами auto_accepted, review_queue, skipped. + """ + auto = [c for c in candidates if c.similarity_score >= threshold] + review = [c for c in candidates if REVIEW_THRESHOLD <= c.similarity_score < threshold] + + if dry_run: + logger.info( + "DRY RUN: would auto-accept %d, review queue %d", + len(auto), + len(review), + ) + return {"auto_accepted": 0, "review_queue": len(review), "skipped": 0} + + inserted = 0 + skipped = 0 + for c in auto: + try: + with db.begin_nested(): + result = db.execute( + text( + """ + INSERT INTO objective_complex_mapping + (objective_complex_name, domrf_obj_id, objective_group, + match_method, match_score, is_reviewed) + VALUES ( + CAST(:name AS text), + CAST(:obj_id AS bigint), + CAST(:group AS text), + CAST(:method AS text), + CAST(:score AS numeric), + CAST(:reviewed AS boolean) + ) + ON CONFLICT (objective_complex_name, objective_group) DO NOTHING + """ + ), + { + "name": c.objective_project_name, + "obj_id": c.domrf_obj_id, + "group": "Екатеринбург", + "method": match_method, + "score": c.similarity_score, + "reviewed": False, + }, + ) + if result.rowcount > 0: + inserted += 1 + else: + skipped += 1 + except Exception as e: + logger.warning( + "Insert failed для %s ↔ %s: %s", + c.domrf_comm_name, + c.objective_project_name, + e, + ) + skipped += 1 + + db.commit() + logger.info( + "Backfill complete: auto_accepted=%d skipped=%d review_queue=%d", + inserted, + skipped, + len(review), + ) + return {"auto_accepted": inserted, "review_queue": len(review), "skipped": skipped} + + +def trigger_mv_refresh(db: Session) -> int: + """REFRESH mv_layout_velocity после backfill. + + Вызывает существующий helper из layout_velocity_refresh. + Передаём concurrently=True (MV уже заполнен). + """ + from app.services.site_finder.layout_velocity_refresh import refresh_layout_velocity + + return refresh_layout_velocity(db, concurrently=True) diff --git a/backend/app/services/exporters/layout_tz_pdf.py b/backend/app/services/exporters/layout_tz_pdf.py new file mode 100644 index 00000000..96178212 --- /dev/null +++ b/backend/app/services/exporters/layout_tz_pdf.py @@ -0,0 +1,161 @@ +"""PDF render для ТЗ (Layout Analysis #113 PR D). + +Pattern reference: backend/app/services/exporters/pdf.py (existing WeasyPrint). +""" + +from __future__ import annotations + +import datetime as dt +import html as _html +import logging + +from weasyprint import HTML + +from app.schemas.parcel import BestLayoutsResponse + +logger = logging.getLogger(__name__) + + +def render_layout_tz_pdf( + response: BestLayoutsResponse, + *, + cad_num: str, + parcel_address: str | None = None, + radius_km: float, + time_window: str, +) -> bytes: + """Render ТЗ PDF от best-layouts response. + + Args: + response: BestLayoutsResponse от /best-layouts endpoint + cad_num: кадастровый номер участка + parcel_address: optional human address (если known через geocoder) + radius_km: радиус анализа конкурентов + time_window: окно анализа (last_month/quarter/year) + + Returns: + PDF bytes (готово для StreamingResponse) + """ + today = dt.date.today().strftime("%d.%m.%Y") + safe_cad = _html.escape(cad_num) + safe_addr = _html.escape(parcel_address) if parcel_address else None + safe_time_window = _html.escape(time_window) + addr_line = f"

Адрес: {safe_addr}

" if safe_addr else "" + + def _price_cell(val: float | None) -> str: + if val is None: + return "—" + return f"{val:,.0f}".replace(",", " ") + " ₽" + + # Top layouts table rows + top_rows = "".join( + "" + f"{r.rank}" + f"{_html.escape(r.room_bucket)}" + f"{_html.escape(r.area_bin)}" + f"{r.velocity_per_month:.1f}" + f"{r.avg_area_m2:.1f}" + f"{_price_cell(r.avg_price_per_m2_rub)}" + f"{r.total_sold_in_window}" + "" + for r in response.top_layouts + ) + + # Recommendation mix table rows + mix_rows = "".join( + "" + f"{_html.escape(m.room_bucket)}" + f"{m.pct}%" + f"{m.abs_units if m.abs_units is not None else '—'}" + f"{f'{m.avg_target_area_m2:.1f}' if m.avg_target_area_m2 is not None else '—'}" + "" + for m in response.recommendation_for_tz.mix + ) + + rec = response.recommendation_for_tz + safe_rationale = _html.escape(rec.rationale_text) + weighted_price = ( + f"{rec.weighted_avg_price_per_m2_rub:,.0f}".replace(",", " ") + " ₽/м²" + if rec.weighted_avg_price_per_m2_rub is not None + else "нет данных" + ) + + dq = response.data_quality + + html = f""" + + + +ТЗ на проектирование — {safe_cad} + + + +

Техническое задание на проектирование (data-driven)

+
+

Кадастровый номер: {safe_cad}

+ {addr_line} +

Радиус анализа: {radius_km} км · Окно: {safe_time_window}

+

Дата формирования: {today}

+
+ +

Рекомендуемая структура квартирографии (unit-mix)

+
{safe_rationale}
+ + + + + {mix_rows} +
КомнатностьДоляКол-во (от target)Целевая площадь, м²
+

Средневзвешенная цена benchmark: {weighted_price}

+

Основано на {rec.based_on_obj_count} ЖК / {rec.based_on_total_deals} сделок

+

Период данных: + {rec.data_window_start.strftime("%d.%m.%Y")} – {rec.data_window_end.strftime("%d.%m.%Y")} +

+ +

Топ планировок конкурентов по продажам

+ + + + + + {top_rows} +
#КомнатыПлощадьПродажи/месСр. площадь, м²Ср. цена, ₽/м²Продано (окно)
+ +

Качество данных

+

+ Покрытие: {dq.objects_with_velocity_data} из + {dq.objects_total_in_radius} ЖК с данными velocity + ({dq.velocity_coverage_pct:.1f}%) +

+

+ Уверенность: + + {dq.confidence.upper()} + +

+ +
+

GenDesign Site Finder · сгенерировано из данных DOM.РФ + Objective + Росреестр

+

Phase 2.1: без layout_type (евро/классика/панорама) и balcony_count.

+
+ +""" + + pdf_bytes = HTML(string=html).write_pdf() + logger.info("Generated layout TZ PDF for cad %s: %d bytes", cad_num, len(pdf_bytes)) + return pdf_bytes diff --git a/backend/app/services/exporters/snapshot_pdf.py b/backend/app/services/exporters/snapshot_pdf.py new file mode 100644 index 00000000..756e25a6 --- /dev/null +++ b/backend/app/services/exporters/snapshot_pdf.py @@ -0,0 +1,204 @@ +"""Генерация одностраничного PDF-снимка кадастрового участка. + +Использует WeasyPrint + Jinja2. Шрифты — DejaVu Sans из системы (Dockerfile) +или из пакета weasyprint (font fallback). Шаблон: app/templates/parcel_snapshot.html. +""" + +import datetime +import logging +import pathlib +from typing import Any + +from jinja2 import Environment, FileSystemLoader, select_autoescape + +logger = logging.getLogger(__name__) + +# Путь к директории шаблонов (относительно этого файла — 2 уровня вверх, затем templates) +_TEMPLATE_DIR = pathlib.Path(__file__).parent.parent / "templates" + +# Системные пути DejaVu Sans (Ubuntu/Debian Docker-образ + Alpine резерв) +_DEJAVU_CANDIDATES: list[str] = [ + "/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf", + "/usr/share/fonts/dejavu/DejaVuSans.ttf", + "/usr/share/fonts/TTF/DejaVuSans.ttf", +] + +_CATEGORY_RU: dict[str, str] = { + "school": "Школа", + "kindergarten": "Детский сад", + "pharmacy": "Аптека", + "hospital": "Больница", + "shop_mall": "ТЦ", + "shop_supermarket": "Супермаркет", + "shop_small": "Магазин", + "park": "Парк", + "bus_stop": "Автобус", + "metro_stop": "Метро", + "tram_stop": "Трамвай", +} + +# Веса POI-категорий — должны совпадать с _POI_WEIGHTS в parcels.py. +# Дублированы здесь чтобы exporter не импортировал из api-слоя. +_POI_WEIGHTS: dict[str, float] = { + "school": 1.5, + "kindergarten": 1.5, + "pharmacy": 0.8, + "hospital": 0.6, + "shop_mall": 1.2, + "shop_supermarket": 1.0, + "shop_small": 0.5, + "park": 1.8, + "bus_stop": 0.3, + "metro_stop": 1.5, + "tram_stop": -0.5, +} + +_WALK_SPEED_M_PER_MIN: float = 80.0 # ~5 км/ч + + +def _find_font_url() -> str: + """Вернуть file:// URL для DejaVu Sans или пустую строку (system fallback). + + WeasyPrint умеет сам находить системные шрифты через fonttools/fontconfig, + поэтому пустая строка допустима — шрифт тогда подбирается CSS generic. + """ + for path in _DEJAVU_CANDIDATES: + if pathlib.Path(path).exists(): + return f"file://{path}" + logger.warning( + "snapshot_pdf: DejaVu Sans не найден в стандартных путях — используем system fallback" + ) + return "" + + +def _format_cost(value: float | None) -> str: + """Форматировать кадастровую стоимость в читаемый вид (млн/тыс ₽).""" + if value is None: + return "—" + if value >= 1_000_000: + return f"{value / 1_000_000:.1f} млн ₽" + if value >= 1_000: + return f"{value / 1_000:.0f} тыс ₽" + return f"{value:.0f} ₽" + + +def _build_poi_items(poi_rows: list[dict[str, Any]], limit: int = 7) -> list[dict[str, Any]]: + """Вычислить weighted_score для каждого POI и вернуть топ-N отсортированных. + + Формула: weighted_score = weight * max(0, 1 - distance_m / 1000) + Отрицательные вклады (трамвай) — не включаем в топ-список. + """ + items: list[dict[str, Any]] = [] + for p in poi_rows: + cat: str = p.get("category", "") + w = _POI_WEIGHTS.get(cat, 0.0) + distance_m = float(p.get("distance_m") or 0) + decay = max(0.0, 1.0 - distance_m / 1000.0) + score = round(w * decay, 2) + if score <= 0: + continue + walk_min = max(1, round(distance_m / _WALK_SPEED_M_PER_MIN)) + items.append( + { + "category_ru": _CATEGORY_RU.get(cat, cat), + "name": p.get("name") or "", + "distance_m": round(distance_m), + "walk_min": walk_min, + "weighted_score": score, + } + ) + items.sort(key=lambda x: x["weighted_score"], reverse=True) + return items[:limit] + + +def generate_snapshot_pdf( + *, + cad_num: str, + address: str | None, + district: str | None, + area_m2: float | None, + cadastral_cost_rub: float | None, + land_category: str | None, + vri: str | None, + last_update: str | None, + poi_rows: list[dict[str, Any]], + competitor_rows: list[dict[str, Any]], + competitors_limit: int = 5, +) -> bytes: + """Сгенерировать PDF-снимок участка (1 страница A4). + + Аргументы: + cad_num: кадастровый номер. + address: адрес из cad_parcels. + district: район города. + area_m2: площадь в кв. м (конвертируем в га для отображения). + cadastral_cost_rub: кадастровая стоимость в рублях. + land_category: категория земель. + vri: вид разрешённого использования. + last_update: строка даты последнего обновления данных. + poi_rows: сырые строки из osm_poi_ekb (category, name, distance_m). + competitor_rows: строки конкурентов из domrf_kn_objects. + competitors_limit: сколько конкурентов выводить (3-5 по ТЗ). + + Возвращает: bytes PDF-документа. + """ + # WeasyPrint импортируем локально — тяжёлый; не нужен при импорте модуля + try: + from weasyprint import HTML + except ImportError as exc: + raise RuntimeError( + "WeasyPrint не установлен. Добавь 'weasyprint>=62.0' в pyproject.toml." + ) from exc + + env = Environment( + loader=FileSystemLoader(str(_TEMPLATE_DIR)), + autoescape=select_autoescape(["html"]), + ) + template = env.get_template("parcel_snapshot.html") + + area_ha = f"{area_m2 / 10_000:.2f}" if area_m2 else "—" + poi_items = _build_poi_items(poi_rows, limit=7) + + # Конкуренты — берём топ N ближайших (уже отсортированы по flat_count DESC; + # переупорядочиваем по distance_m для удобства чтения) + competitors_display = sorted( + competitor_rows[:competitors_limit], + key=lambda r: float(r.get("distance_m") or 0), + ) + competitors_ctx: list[dict[str, Any]] = [ + { + "comm_name": r.get("comm_name"), + "dev_name": r.get("dev_name"), + "obj_class": r.get("obj_class"), + "flat_count": r.get("flat_count"), + "distance_m": round(float(r.get("distance_m") or 0)), + } + for r in competitors_display + ] + + generated_at = datetime.datetime.now(tz=datetime.UTC).strftime("%d.%m.%Y %H:%M UTC") + + html_str = template.render( + cad_num=cad_num, + address=address, + district=district, + area_ha=area_ha, + cadastral_cost=_format_cost(cadastral_cost_rub), + land_category=land_category, + vri=vri, + last_update=last_update or "—", + poi_items=poi_items, + competitors=competitors_ctx, + generated_at=generated_at, + font_url=_find_font_url(), + ) + + logger.info( + "snapshot_pdf: rendering PDF for %s (%d POI, %d competitors)", + cad_num, + len(poi_items), + len(competitors_ctx), + ) + + pdf_bytes: bytes = HTML(string=html_str, base_url=str(_TEMPLATE_DIR)).write_pdf() + return pdf_bytes diff --git a/backend/app/services/exporters/trade_in_pdf.py b/backend/app/services/exporters/trade_in_pdf.py new file mode 100644 index 00000000..761382c3 --- /dev/null +++ b/backend/app/services/exporters/trade_in_pdf.py @@ -0,0 +1,424 @@ +"""PDF генератор для отчёта Trade-In Estimator (TI-2). + +Структура отчёта — 4 страницы (как у Брусника.Обмен): + 1. Cover — шапка + адрес + параметры квартиры + hero-цена + 2. Listings — таблица объявлений-аналогов (top 10) + 3. Deals — таблица фактических сделок (last 12 мес.) + 4. Offer — выкупная стоимость trade-in (placeholder Phase 2) + +Pattern: backend/app/services/exporters/layout_tz_pdf.py (WeasyPrint, уже работает). +""" + +from __future__ import annotations + +import datetime as dt +import html as _html +import logging + +from weasyprint import CSS, HTML + +from app.schemas.trade_in import AggregatedEstimate, AnalogLot + +logger = logging.getLogger(__name__) + +# ── helpers ────────────────────────────────────────────────────────────────── + + +def _fmt_rub(value: int) -> str: + """Форматирование рублей с пробелами-разделителями: 12 500 000 ₽""" + return f"{value:,}".replace(",", " ") + " ₽" + + +def _fmt_rub_m(value: int) -> str: + """Форматирование в миллионах: 12,50 млн ₽""" + m = value / 1_000_000 + return f"{m:.2f}".replace(".", ",") + " млн ₽" + + +def _fmt_ppm2(value: int) -> str: + return f"{value:,}".replace(",", " ") + " ₽/м²" + + +def _conf_color(confidence: str) -> tuple[str, str, str]: + """(bg, fg, border) для badge уверенности.""" + mapping = { + "high": ("#dcfce7", "#15803d", "#86efac"), + "medium": ("#fef9c3", "#a16207", "#fde68a"), + "low": ("#fee2e2", "#b91c1c", "#fca5a5"), + } + return mapping.get(confidence, ("#f3f4f6", "#374151", "#d1d5db")) + + +def _conf_label(confidence: str) -> str: + return {"high": "Высокая", "medium": "Средняя", "low": "Низкая"}.get(confidence, confidence) + + +def _analog_rows(lots: list[AnalogLot], *, is_deal: bool) -> str: + if not lots: + return "Нет данных" + rows = [] + for lot in lots: + date_val = lot.listing_date.strftime("%d.%m.%Y") if lot.listing_date else "—" + dom_val = str(lot.days_on_market) if lot.days_on_market is not None else "—" + floor_val = f"{lot.floor}/{lot.total_floors}" if lot.floor and lot.total_floors else "—" + label = "Дата сделки" if is_deal else "В продаже" + _ = label # used for header only + rows.append( + "" + f"{_html.escape(lot.address)}" + f"{lot.area_m2:.1f}" + f"{lot.rooms if lot.rooms else 'С'}" + f"{floor_val}" + f"{_fmt_rub(lot.price_rub)}" + f"{date_val} ({dom_val}д.)" + "" + ) + return "".join(rows) + + +# ── CSS ────────────────────────────────────────────────────────────────────── + + +def _build_css() -> str: + return """ +@page { + size: A4; + margin: 20mm 18mm 20mm 18mm; +} +* { box-sizing: border-box; } +body { + font-family: 'Helvetica', 'Arial', sans-serif; + font-size: 10pt; + color: #1a1d23; + margin: 0; + padding: 0; +} +h1 { font-size: 18pt; margin: 0 0 4pt 0; line-height: 1.2; } +h2 { font-size: 13pt; margin: 0 0 8pt 0; border-bottom: 1.5px solid #e6e8ec; padding-bottom: 4pt; } +h3 { font-size: 11pt; margin: 0 0 6pt 0; color: #374151; } +.page { page-break-after: always; } +.page:last-child { page-break-after: avoid; } + +/* Cover */ +.cover-header { + background: #1d4ed8; color: #fff; + padding: 20pt; border-radius: 6pt; margin-bottom: 16pt; +} +.cover-header h1 { color: #fff; } +.cover-meta { font-size: 9pt; color: rgba(255,255,255,0.8); margin-top: 4pt; } +.hero-price { + font-size: 28pt; font-weight: 800; + font-variant-numeric: tabular-nums; margin: 12pt 0 4pt; +} +.hero-ppm2 { font-size: 12pt; color: #5b6066; font-variant-numeric: tabular-nums; } +.range-bar-wrap { margin: 10pt 0; } +.range-label { font-size: 9pt; color: #9ca3af; } +.badge { + display: inline-block; padding: 4pt 10pt; + border-radius: 6pt; font-size: 9pt; font-weight: 700; margin-top: 8pt; +} +.params-grid { + display: grid; grid-template-columns: 1fr 1fr 1fr; + gap: 8pt; margin-top: 10pt; +} +.param-item .param-label { + font-size: 8pt; color: #9ca3af; + text-transform: uppercase; letter-spacing: 0.03em; +} +.param-item .param-value { font-size: 10pt; font-weight: 600; } + +/* Tables */ +table { width: 100%; border-collapse: collapse; margin-top: 8pt; font-size: 9pt; } +thead tr { background: #f3f4f6; } +th { + padding: 5pt 7pt; text-align: left; font-weight: 700; + color: #374151; border-bottom: 1.5px solid #d1d5db; +} +td { padding: 5pt 7pt; border-bottom: 1px solid #e6e8ec; vertical-align: top; } +td.num { text-align: right; font-variant-numeric: tabular-nums; white-space: nowrap; } +tr:nth-child(even) td { background: #f9fafb; } +td.empty { text-align: center; color: #9ca3af; padding: 14pt; } + +/* Offer page */ +.offer-table { margin-top: 14pt; } +.offer-table td { font-size: 10pt; } +.offer-total td { font-weight: 700; background: #eff6ff !important; } +.advantages { display: grid; grid-template-columns: 1fr 1fr; gap: 10pt; margin-top: 16pt; } +.advantage-item { border: 1px solid #e6e8ec; border-radius: 6pt; padding: 10pt; } +.advantage-item .adv-title { font-weight: 700; font-size: 10pt; margin-bottom: 4pt; } +.advantage-item .adv-desc { font-size: 9pt; color: #5b6066; } + +/* Footer */ +.footer { + margin-top: 20pt; padding-top: 8pt; + border-top: 1px solid #e6e8ec; font-size: 8pt; color: #9ca3af; +} +.source-logos { color: #6b7280; font-size: 8pt; margin-top: 4pt; } +""" + + +# ── HTML builder ───────────────────────────────────────────────────────────── + + +def _build_html(estimate: AggregatedEstimate, input_snapshot: dict) -> str: # type: ignore[type-arg] + today = dt.date.today().strftime("%d.%m.%Y") + address = _html.escape(input_snapshot.get("address", "—")) + area_m2: float = input_snapshot.get("area_m2", 0) + rooms: int = input_snapshot.get("rooms", 0) + floor: int = input_snapshot.get("floor", 0) + total_floors: int = input_snapshot.get("total_floors", 0) + year_built = input_snapshot.get("year_built") + house_type = input_snapshot.get("house_type") + repair_state = input_snapshot.get("repair_state") + has_balcony = input_snapshot.get("has_balcony") + + conf_bg, conf_fg, conf_border = _conf_color(estimate.confidence) + conf_label = _conf_label(estimate.confidence) + + rooms_label = "Студия" if rooms == 0 else f"{rooms}-комн." + house_labels = { + "panel": "Панель", + "brick": "Кирпич", + "monolith": "Монолит", + "monolith_brick": "Монолит-кирпич", + "other": "Другое", + } + repair_labels = { + "needs_repair": "Требует ремонта", + "standard": "Стандартный", + "good": "Хороший", + "excellent": "Евроремонт", + } + + # Cover params grid items + params = [ + ("Площадь", f"{area_m2} м²"), + ("Комнат", rooms_label), + ("Этаж", f"{floor} из {total_floors}"), + ] + if year_built: + params.append(("Год постройки", str(year_built))) + if house_type: + params.append(("Тип дома", house_labels.get(house_type, house_type))) + if repair_state: + params.append(("Ремонт", repair_labels.get(repair_state, repair_state))) + if has_balcony is not None: + params.append(("Балкон", "Есть" if has_balcony else "Нет")) + + params_html = "".join( + f"""
+
{k}
+
{_html.escape(str(v))}
+
""" + for k, v in params + ) + + listing_rows = _analog_rows(estimate.analogs, is_deal=False) + deal_rows = _analog_rows(estimate.actual_deals, is_deal=True) + + # Trade-in cost table (stub: -7% buyout) + median = estimate.median_price_rub + torg_pct = 7 + buyout_pct = 5 # trade-in discount on top + torg = int(median * torg_pct / 100) + buyout_discount = int(median * buyout_pct / 100) + tradein_price = median - torg - buyout_discount + + return f""" + + + +Trade-In Оценка — {address} + + + + +
+
+

Оценка квартиры — Trade-In

+
{address}
+
Сформировано: {today} · Действительно 24 часа
+
+ +
+
+
+ Оценочная стоимость +
+
{_fmt_rub_m(estimate.median_price_rub)}
+
{_fmt_ppm2(estimate.median_price_per_m2)}
+
+ Диапазон: {_fmt_rub_m(estimate.range_low_rub)} — {_fmt_rub_m(estimate.range_high_rub)} + · данные за {estimate.period_months} мес. +
+
+
+ Достоверность: {conf_label}
+ {estimate.n_analogs} аналогов +
+
+ +

Параметры объекта

+
+ {params_html} +
+ +
+
GenDesign · Trade-In Estimator · mock-данные (Phase 1 MVP)
+
Источники данных: Циан · Авито · ДомКлик · Росреестр
+
+
+ + +
+

Объявления-аналоги ({len(estimate.analogs)})

+

+ Аналоги из открытых источников (Циан, Авито, ДомКлик) — схожая комнатность, + площадь ±15%, тот же район. Используются как ориентир рыночной цены. +

+ + + + + + + + + + + + + {listing_rows} + +
АдресПл., м²Комн.ЭтажЦенаДата / Экспозиция
+ +
+
GenDesign · Trade-In Estimator · стр. 2
+
+
+ + +
+

Фактические сделки ({len(estimate.actual_deals)})

+

+ Сделки из Росреестра (ДДУ + переуступка) за последние 12 месяцев — отражают + реальные цены покупки, в отличие от цен предложения. +

+ + + + + + + + + + + + + {deal_rows} + +
АдресПл., м²Комн.ЭтажЦена сделкиДата / Срок
+ +
+
GenDesign · Trade-In Estimator · стр. 3
+
+
+ + +
+

Выкупная стоимость (trade-in)

+

+ Ориентировочный расчёт. Финальная цена выкупа согласовывается с менеджером + и зависит от конкретного объекта и условий сделки. +

+ + + + + + + + + + + + + + + + + + + + +
Рыночная стоимость (медиана){_fmt_rub(median)}
— Торговый дисконт ({torg_pct}%)−{_fmt_rub(torg)}
— Дисконт за срочность выкупа ({buyout_pct}%)−{_fmt_rub(buyout_discount)}
Выкупная цена (ориентир){_fmt_rub(tradein_price)}
+ +
+ Важно: данные получены из mock-источников (MVP Phase 1). + В Phase 2 расчёт будет основан на реальных данных Циан/Авито/Росреестр + для конкретного адреса и подтверждён менеджером. +
+ +

4 преимущества trade-in

+
+
+
Скорость
+
Сделка за 2–4 недели вместо 3–6 месяцев самостоятельной продажи
+
+
+
Без хлопот
+
Показы, торг, документы — всё берёт на себя девелопер
+
+
+
Зачёт в счёт новостройки
+
Стоимость квартиры напрямую идёт в оплату нового жилья
+
+
+
Фиксация цены
+
Цена новостройки фиксируется на момент подачи заявки
+
+
+ +
+
GenDesign · Trade-In Estimator · стр. 4
+
+ Расчёт носит ориентировочный характер и не является офертой. +
+
+
+ + +""" + + +# ── public API ──────────────────────────────────────────────────────────────── + + +def generate_trade_in_pdf(estimate: AggregatedEstimate, input_snapshot: dict) -> bytes: # type: ignore[type-arg] + """Генерирует 4-страничный WeasyPrint PDF для Trade-In оценки. + + Pages: + 1. Cover — шапка + адрес + параметры + hero-цена + 2. Listings — таблица объявлений-аналогов (top 10) + 3. Deals — таблица фактических сделок (last 12 мес.) + 4. Offer — выкупная стоимость trade-in + 4 преимущества + + Args: + estimate: AggregatedEstimate из БД + input_snapshot: словарь с полями ввода пользователя (address, area_m2, ...) + + Returns: + PDF bytes готовые для Response(media_type="application/pdf") + """ + html_str = _build_html(estimate, input_snapshot) + css_str = _build_css() + pdf_bytes = HTML(string=html_str).write_pdf(stylesheets=[CSS(string=css_str)]) + logger.info( + "Generated trade-in PDF estimate_id=%s pages=4 size=%d bytes", + estimate.estimate_id, + len(pdf_bytes), + ) + return pdf_bytes diff --git a/backend/app/services/scrapers/documents.py b/backend/app/services/scrapers/documents.py new file mode 100644 index 00000000..a6d38f71 --- /dev/null +++ b/backend/app/services/scrapers/documents.py @@ -0,0 +1,193 @@ +"""Parser и upsert для PDF-документов DOM.РФ (декларации, разрешения, проектная, +отчётность, прочее). + +Issue #297, sub-task 22i. + +Источник данных: /сервисы/api/object/{obj_id}/documents (per-object endpoint, +аналогичный /infrastructure и /photos — не входит в bulk kn/object list). + +Ключи payload (наблюдаемые через chrome-devtools, структура DOM.РФ kn-API v1): + + [ + { + "docTypeId": 1, # int — тип по справочнику DOM.РФ + "docTypeName": "Декларация", # text + "docNum": "66-001686-Д", # text или null + "postedDate": "2024-03-15", # date string "YYYY-MM-DD" или null + "fileUrl": "https://xn--80az8a.../api/ext/file/...", + "fileSize": 1234567, # bytes, может быть null + }, + ... + ] + +Поле doc_type нормализуется к одному из канонических значений: + декларация / разрешение / проектная / отчётность / прочее + +NOTE: download PDF-файлов — отдельная Celery-задача (future PR). + Этот модуль только парсит и upsert-ит метаданные в domrf_kn_documents. +""" + +from __future__ import annotations + +import logging +from datetime import date, datetime +from typing import Any + +from sqlalchemy import text +from sqlalchemy.orm import Session + +logger = logging.getLogger(__name__) + +# ── Canonical doc_type mapping ──────────────────────────────────────────────── + +_DOC_TYPE_MAP: dict[str, str] = { + # Exact substrings → canonical type (case-insensitive, checked in order). + "деклар": "декларация", + "разреш": "разрешение", + "проект": "проектная", + "отчёт": "отчётность", + "отчет": "отчётность", # without ё +} +_DOC_TYPE_FALLBACK = "прочее" + +_UPSERT_DOC_SQL = text( + """ + INSERT INTO domrf_kn_documents + (obj_id, doc_type, doc_num, posted_at, file_url, size_bytes) + VALUES + (:obj_id, :doc_type, :doc_num, :posted_at, :file_url, :size_bytes) + ON CONFLICT (obj_id, doc_type, doc_num, file_url) DO UPDATE + SET size_bytes = EXCLUDED.size_bytes, + local_path = COALESCE(domrf_kn_documents.local_path, EXCLUDED.local_path), + downloaded_at = COALESCE(domrf_kn_documents.downloaded_at, EXCLUDED.downloaded_at), + scraped_at = NOW() + """ +) + + +# ── helpers ─────────────────────────────────────────────────────────────────── + + +def _canonical_doc_type(raw_type: str | None) -> str: + """Нормализовать docTypeName DOM.РФ к одному из канонических значений.""" + if not raw_type: + return _DOC_TYPE_FALLBACK + lower = raw_type.strip().lower() + for substr, canonical in _DOC_TYPE_MAP.items(): + if substr in lower: + return canonical + return _DOC_TYPE_FALLBACK + + +def _parse_date(v: Any) -> date | None: + """Coerce date-like value to date. Accepts 'YYYY-MM-DD' or datetime.""" + if v is None or v == "": + return None + if isinstance(v, date) and not isinstance(v, datetime): + return v + if isinstance(v, datetime): + return v.date() + if isinstance(v, str): + s = v.strip() + for fmt in ("%Y-%m-%d", "%d.%m.%Y", "%d-%m-%Y"): + try: + return datetime.strptime(s, fmt).date() + except ValueError: + pass + return None + + +# ── Public API ──────────────────────────────────────────────────────────────── + + +def extract_documents(raw_payload: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Извлечь список документов из payload endpoint /object/{obj_id}/documents. + + Возвращает список dict, готовых для передачи в upsert_documents: + obj_id — передаётся отдельно в upsert_documents, здесь NOT SET + doc_type — канонический тип (декларация/разрешение/…) + doc_num — номер документа или None + posted_at — дата публикации или None + file_url — URL PDF (обязательно, строки без URL пропускаются) + size_bytes — размер файла в байтах или None + + Записи без file_url пропускаются с warning. + """ + result: list[dict[str, Any]] = [] + for item in raw_payload: + if not isinstance(item, dict): + continue + file_url: str | None = item.get("fileUrl") or item.get("file_url") or None + if not file_url or not isinstance(file_url, str) or not file_url.startswith("http"): + logger.warning("domrf documents: skipping item without valid fileUrl: %s", item) + continue + raw_type = item.get("docTypeName") or item.get("doc_type_name") or None + doc_num_raw = item.get("docNum") or item.get("doc_num") or None + doc_num = str(doc_num_raw).strip() if doc_num_raw is not None else None + size_raw = item.get("fileSize") or item.get("file_size") or None + try: + size_bytes: int | None = int(size_raw) if size_raw is not None else None + except (TypeError, ValueError): + size_bytes = None + result.append( + { + "doc_type": _canonical_doc_type(raw_type), + "doc_num": doc_num or None, + "posted_at": _parse_date(item.get("postedDate") or item.get("posted_date")), + "file_url": file_url.strip(), + "size_bytes": size_bytes, + } + ) + return result + + +def upsert_documents(db: Session, obj_id: int, docs: list[dict[str, Any]]) -> tuple[int, int]: + """INSERT/UPDATE документов объекта в domrf_kn_documents. + + Использует SAVEPOINT (begin_nested) per-row чтобы один битый URL + не откатывал всю транзакцию. + + Возвращает (inserted_or_updated, skipped). + """ + if not docs: + return 0, 0 + ok = 0 + skip = 0 + for doc in docs: + params: dict[str, Any] = { + "obj_id": obj_id, + "doc_type": doc["doc_type"], + "doc_num": doc.get("doc_num"), + "posted_at": doc.get("posted_at"), + "file_url": doc["file_url"], + "size_bytes": doc.get("size_bytes"), + } + try: + with db.begin_nested(): + db.execute(_UPSERT_DOC_SQL, params) + ok += 1 + except Exception as exc: + logger.warning( + "upsert document obj=%s url=%s failed: %s", + obj_id, + doc.get("file_url"), + exc, + ) + skip += 1 + db.commit() + logger.info("domrf documents obj=%s: upserted=%d skipped=%d", obj_id, ok, skip) + return ok, skip + + +# ── Stub for future Celery download task ────────────────────────────────────── + + +def download_document_stub(obj_id: int, doc_id: int, file_url: str) -> None: + """Placeholder для будущей Celery-задачи download_domrf_documents. + + Скачивает PDF, сохраняет в data/raw/domrf_docs/{obj_id}/{filename}, + обновляет domrf_kn_documents.local_path + downloaded_at. + + Реализация — отдельный PR (future task). + """ + raise NotImplementedError("PDF download not implemented in 22i. See issue #297 future PR.") diff --git a/backend/app/services/scrapers/domrf_catalog.py b/backend/app/services/scrapers/domrf_catalog.py new file mode 100644 index 00000000..b0cfd052 --- /dev/null +++ b/backend/app/services/scrapers/domrf_catalog.py @@ -0,0 +1,563 @@ +"""DOM.РФ catalog-квартир HTML scraper (issue #297 22d). + +kn-API не возвращает цену для большинства квартир (91.5% NULL). Цены живут на +отдельной странице каталога: + https://наш.дом.рф/сервисы/каталог-квартир/квартира/{catalog_url_hash} + +Этот модуль: +1. Строит URL каталога по `catalog_url_hash` (колонка появляется после миграции 22b). +2. Получает SSR-HTML через BrowserSession (Playwright, anti-bot — тот же паттерн + что и get_json, но возвращает HTML text вместо JSON). +3. Извлекает price_rub, status, finishing_type, ceiling_height_m, section_no, + catalog_updated_at из HTML с помощью stdlib `html.parser` + regex. +4. Пишет только catalog-derived поля через UPDATE ... WHERE ods_id = :ods_id — + НЕ перетирает kn-API метаданные (total_area, rooms и т.д.). + +Зависимости: нет новых. Использует `html.parser` из stdlib + `re`. +NOTE: beautifulsoup4 НЕ установлен (нет в pyproject.toml). Если потребуется + структурированный парсинг — добавить `beautifulsoup4>=4.12` в pyproject.toml + и заменить _HtmlTextExtractor на `BeautifulSoup(html, "html.parser")`. + +Контекст Roadmap: + - Phase 5 (22d) — catalog scraper для цен + - Согласно update 2026-05-17 (Objective goldmine): Objective уже содержит 81.4% + цен. Для `domrf_kn_flats` этот scraper остаётся полезен для полей: + finishing_type, ceiling_height_m, section_no, catalog_updated_at, catalog_url_hash. + Price coverage через Objective (OBJ-3) — приоритетнее. + +Wiring (отдельный PR): + - Celery task: `backend/app/workers/tasks/scrape_catalog.py` + - Beat schedule: кварть + `catalog_updated_at < NOW() - INTERVAL '30 days'` +""" + +from __future__ import annotations + +import asyncio +import logging +import re +from datetime import date +from html.parser import HTMLParser +from typing import Any + +from sqlalchemy import text +from sqlalchemy.orm import Session + +from app.services.scrapers.stealth import BASE_URL, BrowserSession, jitter_sleep + +logger = logging.getLogger(__name__) + +# Per-flat catalog page URL template (IDN encoded — same as what browsers send). +# Человекочитаемый вид: https://наш.дом.рф/сервисы/каталог-квартир/квартира/{hash} +CATALOG_FLAT_PATH = "/сервисы/каталог-квартир/квартира/{catalog_url_hash}" + +# JS snippet: выполняется внутри живой Playwright-страницы. +# Возвращает HTML текст страницы (text/html). +# Это аналог _FETCH_JS из stealth.py, но для text/html вместо application/json. +_FETCH_HTML_JS = """ +async ({url}) => { + try { + const r = await fetch(url, {credentials: 'include'}); + const ctype = r.headers.get('content-type') || ''; + const body = await r.text(); + return {ok: r.ok, status: r.status, body, contentType: ctype}; + } catch (e) { + return {ok: false, status: 0, body: String(e), contentType: ''}; + } +} +""" + +# Нормализованные значения статуса продажи. +STATUS_FREE = "free" +STATUS_SOLD = "sold" +STATUS_RESERVED = "reserved" + + +# ── HTML fetching ───────────────────────────────────────────────────────────── + + +async def fetch_catalog_html(session: BrowserSession, catalog_url_hash: str) -> str: + """Получить SSR-HTML страницы квартиры в каталоге DOM.РФ. + + Использует тот же паттерн что get_json(): fetch() внутри живой Playwright-страницы. + Так WAF-fingerprint идентичен браузеру, cookies проброшены автоматически. + + Raises: + RuntimeError: при транзиентной ошибке после 5 попыток. + WafBlockedError: (из stealth) если вернулся JS-challenge вместо HTML. + """ + if session._page is None: + raise RuntimeError("BrowserSession not bootstrapped") + + url = BASE_URL + CATALOG_FLAT_PATH.format(catalog_url_hash=catalog_url_hash) + last_err: Exception | None = None + + for attempt in range(5): + async with session._sem: + await jitter_sleep() + try: + session._request_count += 1 + result = await session._page.evaluate(_FETCH_HTML_JS, {"url": url}) + except Exception as exc: + last_err = exc + logger.warning( + "catalog html evaluate err attempt=%d hash=%s: %r", + attempt, + catalog_url_hash, + exc, + ) + await asyncio.sleep(2**attempt) + continue + + status: int = result.get("status", 0) + body: str = result.get("body", "") + ctype: str = result.get("contentType", "") + + if status in (429,) or status >= 500 or status == 0: + last_err = RuntimeError(f"transient status={status}") + logger.warning( + "catalog html transient status=%d attempt=%d hash=%s, backing off", + status, + attempt, + catalog_url_hash, + ) + await asyncio.sleep(2**attempt) + continue + + if status == 404: + raise RuntimeError(f"catalog 404 for hash={catalog_url_hash}") + + if status != 200: + raise RuntimeError(f"catalog http {status}: {body[:200]} hash={catalog_url_hash}") + + # Успех — но нужно проверить что не пришёл WAF JS-challenge (нет text/html) + # Страница каталога — SSR, всегда text/html. Если что-то другое — WAF. + if body and "text/html" not in ctype and " None: + super().__init__() + self._stack: list[tuple[str, dict[str, str]]] = [] + # Список записей: (class_hint, full_text) + self.blocks: list[tuple[str, str]] = [] + self._buf: list[str] = [] + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + attr_dict = {k: (v or "") for k, v in attrs} + self._stack.append((tag, attr_dict)) + self._buf.append("") # начало нового буфера для этого тега + + def handle_endtag(self, _tag: str) -> None: + if not self._stack: + return + tag, attr_dict = self._stack.pop() + text = (self._buf.pop() if self._buf else "").strip() + cls = attr_dict.get("class", "") + if text and (cls or tag in ("h1", "h2", "h3", "p", "span", "div", "li")): + self.blocks.append((cls, text)) + # Propagate accumulated text up to parent buffer + if self._buf: + self._buf[-1] += " " + text + + def handle_data(self, data: str) -> None: + if self._buf: + self._buf[-1] += data + + +def _find_text_near( + blocks: list[tuple[str, str]], label_pattern: str, value_pattern: str | None = None +) -> str | None: + """Найти текстовое значение рядом с блоком, matching label_pattern. + + Стратегия: ищем блок где text матчит label_pattern — берём следующий блок + как значение (value_pattern если задан). + """ + label_re = re.compile(label_pattern, re.IGNORECASE) + for i, (_cls, block_text) in enumerate(blocks): + if label_re.search(block_text): + # Попробовать next block + for j in range(i + 1, min(i + 4, len(blocks))): + candidate = blocks[j][1].strip() + if candidate: + if value_pattern is None: + return candidate + if re.search(value_pattern, candidate, re.IGNORECASE): + return candidate + return None + + +def parse_catalog_flat(html: str) -> dict[str, Any]: + """Извлечь поля из SSR-HTML страницы квартиры DOM.РФ. + + Возвращаемые поля (None если не найдено): + - price_rub (int) Цена квартиры в рублях + - price_per_m2 (float) Цена за м² (если указана отдельно) + - status (str) 'free' | 'sold' | 'reserved' + - finishing_type (str) Тип отделки (Предчистовая, Чистовая, Без отделки, ...) + - ceiling_height_m (float) Высота потолков в метрах + - section_no (int) Номер подъезда / секции + - catalog_updated_at (date) Дата обновления информации на странице + + Парсинг хрупкий по природе (SSR HTML DOM.РФ меняется без уведомлений). + Все extraction best-effort — KeyError/AttributeError обёрнуты внутри. + """ + result: dict[str, Any] = {} + + # ── Шаг 1: собрать все текстовые блоки через HTMLParser ────────────────── + collector = _TextCollector() + try: + collector.feed(html) + except Exception as exc: + logger.warning("html parse error (non-fatal): %s", exc) + + blocks = collector.blocks + + # ── Шаг 2: regex-extraction из полного HTML текста ──────────────────────── + # Страница DOM.РФ SSR встраивает данные и в мета-тегах и в JSON-LD. + # Ищем в сыром HTML — надёжнее чем DOM-обход для хрупкой структуры. + + # Price: "7 890 000 ₽" или "7 890 000 руб" + price_match = re.search( + r"([\d][\d\s]{3,12}[\d])\s*(?:₽|руб)", + html, + re.UNICODE, + ) + if price_match: + raw_price = re.sub(r"\s+", "", price_match.group(1)) + try: + price_val = int(raw_price) + # Санity: цена квартиры в ЕКБ от 1 до 500 млн + if 1_000_000 <= price_val <= 500_000_000: + result["price_rub"] = price_val + except ValueError: + pass + + # Price per m²: "217 835 ₽/м²" + ppm2_match = re.search( + r"([\d][\d\s]{2,9}[\d])\s*(?:₽|руб)[/⁄](?:м²|кв\.?\s*м)", + html, + re.UNICODE, + ) + if ppm2_match: + raw_ppm2 = re.sub(r"\s+", "", ppm2_match.group(1)) + try: + result["price_per_m2"] = float(raw_ppm2) + except ValueError: + pass + + # Status: ищем характерные слова рядом с "статус" или в badge + status_match = re.search( + r"(в\s*продаже|свободна|free|продано|sold|забронирована|бронь|reserved)", + html, + re.IGNORECASE | re.UNICODE, + ) + if status_match: + s = status_match.group(1).lower() + if any(kw in s for kw in ("продаже", "свободна", "free")): + result["status"] = STATUS_FREE + elif any(kw in s for kw in ("продано", "sold")): + result["status"] = STATUS_SOLD + elif any(kw in s for kw in ("бронь", "забронирована", "reserved")): + result["status"] = STATUS_RESERVED + + # Finishing type: "Предчистовая", "Чистовая", "Без отделки", "Под ключ" + finishing_match = re.search( + r"(предчистовая|чистовая|без\s+отделки|под\s+ключ|white\s+box|whitebox)", + html, + re.IGNORECASE | re.UNICODE, + ) + if finishing_match: + result["finishing_type"] = finishing_match.group(1).strip().capitalize() + + # Ceiling height: "2,7 м" или "2.7 м" или "высота потолков 2,7" + ceiling_match = re.search( + r"(?:высота\s*потолков?|потолки?)\D{0,20}?([\d][,.][\d])\s*м", + html, + re.IGNORECASE | re.UNICODE, + ) + if not ceiling_match: + # Fallback: просто "2,7 м" в характеристиках квартиры (диапазон 2.0–4.5 м) + ceiling_match = re.search( + r"\b([2-4][,.][\d])\s*м\b", + html, + re.UNICODE, + ) + if ceiling_match: + raw_ceil = ceiling_match.group(1).replace(",", ".") + try: + ceil_val = float(raw_ceil) + if 2.0 <= ceil_val <= 6.0: + result["ceiling_height_m"] = ceil_val + except ValueError: + pass + + # Section (подъезд): "Подъезд 1", "Секция 3", "Подъезд №2" + section_match = re.search( + r"(?:подъезд|секция)\s*[№#]?\s*(\d+)", + html, + re.IGNORECASE | re.UNICODE, + ) + if section_match: + try: + result["section_no"] = int(section_match.group(1)) + except ValueError: + pass + + # catalog_updated_at: "Информация обновлена 17.04.2026" или "Обновлено 17.04.2026" + updated_match = re.search( + r"(?:информация\s+обновлена|обновлено|обновлён?а?)\D{0,10}?(\d{1,2}[./]\d{1,2}[./]\d{4})", + html, + re.IGNORECASE | re.UNICODE, + ) + if updated_match: + raw_dt = updated_match.group(1).replace("/", ".") + try: + parts = raw_dt.split(".") + if len(parts) == 3: + result["catalog_updated_at"] = date(int(parts[2]), int(parts[1]), int(parts[0])) + except (ValueError, IndexError): + pass + + # ── Шаг 3: блочный fallback для ceiling / section (если regex не нашёл) ── + if "ceiling_height_m" not in result: + candidate = _find_text_near(blocks, r"потолк|высота", r"[23][,.][\d]") + if candidate: + m = re.search(r"([23][,.][\d])", candidate) + if m: + try: + v = float(m.group(1).replace(",", ".")) + if 2.0 <= v <= 6.0: + result["ceiling_height_m"] = v + except ValueError: + pass + + if "section_no" not in result: + candidate = _find_text_near(blocks, r"подъезд|секция", r"^\d+$") + if candidate: + try: + result["section_no"] = int(candidate.strip()) + except ValueError: + pass + + logger.debug( + "parse_catalog_flat: extracted fields=%s", + list(result.keys()), + ) + return result + + +# ── DB writes ───────────────────────────────────────────────────────────────── + + +def upsert_catalog_data( + db: Session, ods_id: str, catalog_url_hash: str, data: dict[str, Any] +) -> bool: + """UPDATE catalog-derived поля в domrf_kn_flats для конкретной квартиры. + + Обновляет ТОЛЬКО catalog-only колонки: + price_rub, price_per_m2, status, finishing_type, ceiling_height_m, + section_no, catalog_updated_at, catalog_url_hash. + + НЕ трогает: total_area, rooms, floor, num_floors, flat_type, obj_id и + другие kn-API метаданные. + + Использует COALESCE: если новое значение NULL — старое сохраняется. + Это позволяет повторно запускать scraper не затирая частично заполненные поля. + + ВАЖНО: колонки section_no, finishing_type, ceiling_height_m, + catalog_updated_at, catalog_url_hash должны существовать в таблице. + Они появляются после миграции 22b. Если таблица старая — UPDATE упадёт + с 'column does not exist'. Решение: сначала выполнить data/sql/NN_22b_flats_cols.sql. + + Возвращает True если строка найдена и обновлена, False если ods_id не найден. + """ + params: dict[str, Any] = { + "ods_id": ods_id, + "catalog_url_hash": catalog_url_hash, + "price_rub": data.get("price_rub"), + "price_per_m2": data.get("price_per_m2"), + "status": data.get("status"), + "finishing_type": data.get("finishing_type"), + "ceiling_height_m": data.get("ceiling_height_m"), + "section_no": data.get("section_no"), + "catalog_updated_at": data.get("catalog_updated_at"), + } + + try: + with db.begin_nested(): + result = db.execute( + text( + """ + UPDATE domrf_kn_flats SET + catalog_url_hash = :catalog_url_hash, + price_rub = COALESCE(:price_rub, price_rub), + price_per_m2 = COALESCE(:price_per_m2, price_per_m2), + status = COALESCE(:status, status), + finishing_type = COALESCE(:finishing_type, finishing_type), + ceiling_height_m = COALESCE(:ceiling_height_m, ceiling_height_m), + section_no = COALESCE(:section_no, section_no), + catalog_updated_at = COALESCE(:catalog_updated_at, catalog_updated_at) + WHERE ods_id = :ods_id + """ + ), + params, + ) + except Exception as exc: + logger.warning("upsert_catalog_data ods_id=%s failed: %s", ods_id, exc) + return False + + rows_affected: int = result.rowcount or 0 + if rows_affected == 0: + logger.warning("upsert_catalog_data: ods_id=%s not found in domrf_kn_flats", ods_id) + return rows_affected > 0 + + +# ── Per-flat scrape orchestration ───────────────────────────────────────────── + + +async def scrape_one_flat( + session: BrowserSession, + db: Session, + ods_id: str, + catalog_url_hash: str, +) -> dict[str, Any]: + """Scrape одной квартиры: fetch HTML → parse → upsert. + + Возвращает dict с результатом: {ods_id, success, fields_extracted, updated}. + Ошибки fetch/parse логируются, не бросаются — caller обрабатывает результат. + """ + outcome: dict[str, Any] = { + "ods_id": ods_id, + "catalog_url_hash": catalog_url_hash, + "success": False, + "fields_extracted": 0, + "updated": False, + "error": None, + } + + try: + html = await fetch_catalog_html(session, catalog_url_hash) + except Exception as exc: + logger.warning( + "catalog fetch failed ods_id=%s hash=%s: %s", + ods_id, + catalog_url_hash, + exc, + ) + outcome["error"] = str(exc)[:500] + return outcome + + try: + data = parse_catalog_flat(html) + except Exception as exc: + logger.warning( + "catalog parse failed ods_id=%s hash=%s: %s", + ods_id, + catalog_url_hash, + exc, + ) + outcome["error"] = f"parse: {exc!s}"[:500] + return outcome + + outcome["fields_extracted"] = len([v for v in data.values() if v is not None]) + outcome["updated"] = upsert_catalog_data(db, ods_id, catalog_url_hash, data) + outcome["success"] = True + logger.info( + "catalog scrape ods_id=%s: fields=%d updated=%s", + ods_id, + outcome["fields_extracted"], + outcome["updated"], + ) + return outcome + + +async def scrape_catalog_batch( + db: Session, + flats: list[dict[str, Any]], + region_code: int = 66, + headed: bool = False, + load_state: str | None = None, +) -> dict[str, Any]: + """Scrape пачки квартир каталога DOM.РФ. + + `flats` — список dict'ов с ключами {ods_id, catalog_url_hash}. + Типовой источник: SELECT ods_id, catalog_url_hash FROM domrf_kn_flats + WHERE catalog_url_hash IS NOT NULL + AND (catalog_updated_at IS NULL OR catalog_updated_at < NOW() - INTERVAL '30 days') + LIMIT :batch_size. + + Использует один BrowserSession на весь пакет (bootstrapped 1 раз). + jitter_sleep между запросами встроен в fetch_catalog_html (через BrowserSession._sem). + + Returns: + {total, success, failed, fields_total} + """ + stats: dict[str, Any] = { + "total": len(flats), + "success": 0, + "failed": 0, + "fields_total": 0, + } + + if not flats: + logger.info("scrape_catalog_batch: empty batch, nothing to do") + return stats + + logger.info( + "scrape_catalog_batch: starting %d flats region=%d", + len(flats), + region_code, + ) + + async with BrowserSession( + region_code=region_code, + headed=headed, + load_state=load_state, + # auth=None — страницы каталога публичные, Basic auth не нужен + auth=None, + ) as session: + for flat in flats: + ods_id = flat.get("ods_id", "") + catalog_url_hash = flat.get("catalog_url_hash", "") + if not ods_id or not catalog_url_hash: + logger.warning("scrape_catalog_batch: skip flat with missing ods_id/hash: %r", flat) + stats["failed"] += 1 + continue + + outcome = await scrape_one_flat(session, db, ods_id, catalog_url_hash) + if outcome["success"]: + stats["success"] += 1 + stats["fields_total"] += outcome["fields_extracted"] + else: + stats["failed"] += 1 + + logger.info( + "scrape_catalog_batch done: total=%d success=%d failed=%d fields_total=%d", + stats["total"], + stats["success"], + stats["failed"], + stats["fields_total"], + ) + return stats diff --git a/backend/app/services/scrapers/domrf_catalog_object.py b/backend/app/services/scrapers/domrf_catalog_object.py new file mode 100644 index 00000000..1bc58d7d --- /dev/null +++ b/backend/app/services/scrapers/domrf_catalog_object.py @@ -0,0 +1,468 @@ +"""DOM.РФ catalog-OBJECT scraper (issue #297 sub-task 22d). + +Fills ~25 NULL columns in domrf_kn_objects from public SSR catalog page: + https://наш.дом.рф/сервисы/каталог-новостроек/объект/{obj_id} + +kn-API не возвращает: wall_type, energy_eff, ceiling_height_m, parking_*, +playground_*, finishing_variants_count, etc. — все эти поля есть в +__NEXT_DATA__ JSON блоке на странице каталога (Next.js SSR). + +Uses BrowserSession from app.services.scrapers.stealth (Playwright + WAF bypass). +Fetches HTML, extracts __NEXT_DATA__ JSON, maps to DB columns, +UPDATE domrf_kn_objects WHERE obj_id = :id (не перетирает kn-API данные). +""" + +from __future__ import annotations + +import asyncio +import json +import logging +import re +from datetime import date +from typing import Any + +from sqlalchemy import text +from sqlalchemy.orm import Session + +from app.services.scrapers.stealth import BASE_URL, BrowserSession, WafBlockedError, jitter_sleep + +logger = logging.getLogger(__name__) + +# URL шаблон страницы объекта в каталоге DOM.РФ. +# Человекочитаемый вид: https://наш.дом.рф/сервисы/каталог-новостроек/объект/{obj_id} +CATALOG_OBJECT_PATH = "/сервисы/каталог-новостроек/объект/{obj_id}" + +# JS snippet — аналог _FETCH_HTML_JS из domrf_catalog.py. +# Выполняется внутри живой Playwright-страницы, возвращает HTML текст. +_FETCH_HTML_JS = """ +async ({url}) => { + try { + const r = await fetch(url, {credentials: 'include'}); + const ctype = r.headers.get('content-type') || ''; + const body = await r.text(); + return {ok: r.ok, status: r.status, body, contentType: ctype}; + } catch (e) { + return {ok: false, status: 0, body: String(e), contentType: ''}; + } +} +""" + +# UPDATE SQL — обновляет только catalog-derived поля. +# COALESCE гарантирует что NULL-значение не перетирает уже заполненное поле. +UPDATE_OBJECT_CATALOG_SQL = text( + """ + UPDATE domrf_kn_objects SET + obj_class = COALESCE(:obj_class, obj_class), + wall_type = COALESCE(:wall_type, wall_type), + energy_eff = COALESCE(:energy_eff, energy_eff), + section_count = COALESCE(:section_count, section_count), + parking_total_slots = COALESCE(:parking_total_slots, parking_total_slots), + guest_parking_inside_count = COALESCE( + :guest_parking_inside_count, guest_parking_inside_count + ), + guest_parking_outside_count = COALESCE( + :guest_parking_outside_count, guest_parking_outside_count + ), + ceiling_height_m = COALESCE(:ceiling_height_m, ceiling_height_m), + finishing_variants_count = COALESCE(:finishing_variants_count, finishing_variants_count), + has_free_planning = COALESCE(:has_free_planning, has_free_planning), + avg_flat_area_m2 = COALESCE(:avg_flat_area_m2, avg_flat_area_m2), + elevators_passenger_count = COALESCE( + :elevators_passenger_count, elevators_passenger_count + ), + elevators_cargo_count = COALESCE(:elevators_cargo_count, elevators_cargo_count), + playground_kids_count = COALESCE(:playground_kids_count, playground_kids_count), + playground_sport_count = COALESCE(:playground_sport_count, playground_sport_count), + has_bike_paths = COALESCE(:has_bike_paths, has_bike_paths), + trash_areas_count = COALESCE(:trash_areas_count, trash_areas_count), + has_ramp = COALESCE(:has_ramp, has_ramp), + has_low_platforms = COALESCE(:has_low_platforms, has_low_platforms), + has_wheelchair_lift = COALESCE(:has_wheelchair_lift, has_wheelchair_lift), + first_floor_type = COALESCE(:first_floor_type, first_floor_type), + parking_provision_pct = COALESCE(:parking_provision_pct, parking_provision_pct), + project_published_at = COALESCE(:project_published_at, project_published_at), + project_declaration_num = COALESCE(:project_declaration_num, project_declaration_num), + domrf_score_infrastructure = COALESCE( + :domrf_score_infrastructure, domrf_score_infrastructure + ), + domrf_score_transport = COALESCE(:domrf_score_transport, domrf_score_transport), + catalog_scraped_at = NOW() + WHERE obj_id = :obj_id + AND snapshot_date = :snapshot_date + """ +) + + +# ── Value helpers ───────────────────────────────────────────────────────────── + + +def _to_numeric_comma(s: Any) -> float | None: + """Конвертировать строку с запятой-десятичным разделителем в float. + + Примеры: "2,7" → 2.7; "2.7" → 2.7; "" → None; None → None. + """ + if s is None: + return None + raw = str(s).strip().replace(",", ".") + if not raw: + return None + try: + return float(raw) + except ValueError: + return None + + +def _to_date_ddmmyyyy(s: Any) -> date | None: + """Конвертировать строку "DD.MM.YYYY" в date. + + Примеры: "31.03.2025" → date(2025, 3, 31); "" → None; invalid → None. + """ + if not s: + return None + raw = str(s).strip() + if not raw: + return None + try: + parts = raw.split(".") + if len(parts) == 3: + return date(int(parts[2]), int(parts[1]), int(parts[0])) + except (ValueError, IndexError): + pass + return None + + +def _to_bool_int(v: Any) -> bool | None: + """Конвертировать 0/1 (или любое int-like) в bool. + + Примеры: 1 → True; 0 → False; None → None; 3 → True (>0). + """ + if v is None: + return None + try: + return int(v) > 0 + except (ValueError, TypeError): + return None + + +def _to_bool_da_net(s: Any) -> bool | None: + """Конвертировать "Да"/"Нет" строку в bool. + + Примеры: "Да" → True; "Нет" → False; "" → None; None → None. + """ + if s is None: + return None + raw = str(s).strip().lower() + if raw == "да": + return True + if raw == "нет": + return False + return None + + +def _safe_int(v: Any) -> int | None: + """Безопасная конвертация в int, None при ошибке.""" + if v is None: + return None + try: + return int(v) + except (ValueError, TypeError): + return None + + +# ── HTML fetching ───────────────────────────────────────────────────────────── + + +async def fetch_catalog_object_html(session: BrowserSession, obj_id: int) -> str: + """Получить SSR-HTML страницы объекта в каталоге DOM.РФ. + + Использует тот же паттерн что fetch_catalog_html из domrf_catalog.py: + fetch() внутри живой Playwright-страницы — WAF-fingerprint идентичен браузеру. + + Raises: + WafBlockedError: если вернулся не-HTML (JS-challenge или JSON). + RuntimeError: при 404 или исчерпании попыток. + """ + if session._page is None: + raise RuntimeError("BrowserSession not bootstrapped") + + url = BASE_URL + CATALOG_OBJECT_PATH.format(obj_id=obj_id) + last_err: Exception | None = None + + for attempt in range(5): + async with session._sem: + await jitter_sleep(300, 700) + try: + session._request_count += 1 + result = await session._page.evaluate(_FETCH_HTML_JS, {"url": url}) + except Exception as exc: + last_err = exc + logger.warning( + "catalog_object html evaluate err attempt=%d obj_id=%d: %r", + attempt, + obj_id, + exc, + ) + await asyncio.sleep(2**attempt) + continue + + status: int = result.get("status", 0) + body: str = result.get("body", "") + ctype: str = result.get("contentType", "") + + if status in (429,) or status >= 500 or status == 0: + last_err = RuntimeError(f"transient status={status}") + logger.warning( + "catalog_object transient status=%d attempt=%d obj_id=%d, backing off", + status, + attempt, + obj_id, + ) + await asyncio.sleep(2**attempt) + continue + + if status == 404: + raise RuntimeError(f"catalog_object 404 for obj_id={obj_id}") + + if status != 200: + raise RuntimeError(f"catalog_object http {status}: {body[:200]} obj_id={obj_id}") + + # Проверяем что вернулся HTML, а не WAF JS-challenge. + is_html = "text/html" in ctype or " dict[str, Any]: + """Извлечь JSON из тега ', + html, + re.DOTALL, + ) + if not match: + raise ValueError("__NEXT_DATA__ script tag not found in HTML") + + raw_json = match.group(1).strip() + try: + return json.loads(raw_json) # type: ignore[no-any-return] + except json.JSONDecodeError as exc: + raise ValueError(f"__NEXT_DATA__ JSON parse error: {exc}") from exc + + +# ── Field mapping ───────────────────────────────────────────────────────────── + + +def parse_catalog_object(next_data: dict[str, Any]) -> dict[str, Any]: + """Извлечь поля объекта из __NEXT_DATA__ и вернуть dict для UPDATE. + + Все .get() безопасны — partial responses OK, отсутствующие поля = None. + Возвращает dict с bind-параметрами для UPDATE_OBJECT_CATALOG_SQL. + """ + pp: dict[str, Any] = next_data.get("props", {}).get("pageProps", {}) + ai: dict[str, Any] = pp.get("additionalInfo") or {} + quart: dict[str, Any] = pp.get("quartography") or {} + indexes: dict[str, Any] = pp.get("indexes") or {} + decl: dict[str, Any] = pp.get("projectDeclaration") or {} + + # first_floor_type: 1 = нежилой, 0 = жилой + first_floor_raw = quart.get("nonLivFirstFloor") + first_floor_type: str | None = None + if first_floor_raw is not None: + try: + first_floor_type = "нежилой" if int(first_floor_raw) == 1 else "жилой" + except (ValueError, TypeError): + pass + + # elevators_cargo_count = cargoElevatorsCount + cargoPassengerElevatorCount + cargo = _safe_int(ai.get("cargoElevatorsCount")) + cargo_pass = _safe_int(ai.get("cargoPassengerElevatorCount")) + if cargo is not None or cargo_pass is not None: + elevators_cargo_count: int | None = (cargo or 0) + (cargo_pass or 0) + else: + elevators_cargo_count = None + + return { + "obj_class": pp.get("buildingClass"), + "wall_type": pp.get("wallMaterial"), + "energy_eff": pp.get("objEnergyEfficiency"), + "section_count": _safe_int(quart.get("objLivElemEntrCnt")), + "parking_total_slots": _safe_int(pp.get("parkingCount")), + "guest_parking_inside_count": _safe_int(ai.get("objectParkingPlaces")), + "guest_parking_outside_count": _safe_int(ai.get("nearbyParkingPlaces")), + "ceiling_height_m": _to_numeric_comma(ai.get("ceilingHeight")), + "finishing_variants_count": _safe_int(pp.get("finishTypeCount")), + "has_free_planning": _to_bool_da_net(pp.get("freePlan")), + "avg_flat_area_m2": _to_numeric_comma(quart.get("objLivElemSqAvg")), + "elevators_passenger_count": _safe_int(ai.get("passengerElevatorsCount")), + "elevators_cargo_count": elevators_cargo_count, + "playground_kids_count": _safe_int(ai.get("playgroundsCount")), + "playground_sport_count": _safe_int(ai.get("sportsgroundCount")), + "has_bike_paths": _to_bool_int(ai.get("bicycleLane")), + "trash_areas_count": _safe_int(ai.get("trashAreaCount")), + "has_ramp": _to_bool_int(ai.get("ramp")), + "has_low_platforms": _to_bool_int(ai.get("curbLowering")), + "has_wheelchair_lift": _to_bool_int(ai.get("wheelchairElevatorsCount")), + "first_floor_type": first_floor_type, + "parking_provision_pct": _to_numeric_comma(ai.get("parkingAvailabilityPerc")), + "project_published_at": _to_date_ddmmyyyy(pp.get("publicationDate")), + "project_declaration_num": decl.get("number"), + "domrf_score_infrastructure": _safe_int(indexes.get("infrastructure")), + "domrf_score_transport": _safe_int(indexes.get("transport")), + # TODO: obj_checks (6 detailed checks) — separate investigation (task #21). + # pageProps.isChecked (bool), verificationId, verificationFlg available here + # but detailed per-check breakdown requires separate API investigation. + } + + +# ── DB write ────────────────────────────────────────────────────────────────── + + +async def scrape_catalog_object( + db: Session, + session: BrowserSession, + obj_id: int, + snapshot_date: date, +) -> bool: + """Scrape одного объекта: fetch HTML → extract __NEXT_DATA__ → parse → UPDATE. + + Использует SAVEPOINT (begin_nested) для изоляции per-row ошибок. + Логирует результат через logger.info. + + Returns: + True если UPDATE затронул строку, False при ошибке или 0 rows. + """ + logger.info("catalog_object scrape start obj_id=%d snapshot_date=%s", obj_id, snapshot_date) + + try: + html = await fetch_catalog_object_html(session, obj_id) + except WafBlockedError as exc: + logger.warning("catalog_object WAF blocked obj_id=%d: %s", obj_id, exc) + return False + except Exception as exc: + logger.warning("catalog_object fetch failed obj_id=%d: %s", obj_id, exc) + return False + + try: + next_data = extract_next_data(html) + except ValueError as exc: + logger.warning("catalog_object extract_next_data failed obj_id=%d: %s", obj_id, exc) + return False + + try: + data = parse_catalog_object(next_data) + except Exception as exc: + logger.warning("catalog_object parse failed obj_id=%d: %s", obj_id, exc) + return False + + fields_extracted = len([v for v in data.values() if v is not None]) + + params: dict[str, Any] = { + "obj_id": obj_id, + "snapshot_date": snapshot_date, + **data, + } + + try: + with db.begin_nested(): + result = db.execute(UPDATE_OBJECT_CATALOG_SQL, params) + rows_affected: int = result.rowcount or 0 + except Exception as exc: + logger.warning("catalog_object UPDATE failed obj_id=%d: %s", obj_id, exc) + return False + + if rows_affected == 0: + logger.warning( + "catalog_object UPDATE 0 rows obj_id=%d snapshot_date=%s — not in DB?", + obj_id, + snapshot_date, + ) + return False + + logger.info( + "catalog_object scraped obj_id=%d fields=%d rows_updated=%d", + obj_id, + fields_extracted, + rows_affected, + ) + return True + + +async def scrape_catalog_objects( + db: Session, + obj_ids: list[int], + snapshot_date: date, + region_code: int = 66, +) -> dict[str, int]: + """Scrape списка объектов через один BrowserSession. + + Запускает один BrowserSession на весь batch; jitter_sleep (300–700 мс) + встроен в fetch_catalog_object_html для защиты от rate-limit. + + Returns: + {"processed": N, "succeeded": N, "failed": N, "skipped": N} + """ + stats: dict[str, int] = { + "processed": 0, + "succeeded": 0, + "failed": 0, + "skipped": 0, + } + + if not obj_ids: + logger.info("scrape_catalog_objects: empty list, nothing to do") + return stats + + logger.info( + "scrape_catalog_objects: starting %d objects region=%d snapshot_date=%s", + len(obj_ids), + region_code, + snapshot_date, + ) + + async with BrowserSession( + region_code=region_code, + # Страницы каталога публичные — Basic auth не нужен + auth=None, + ) as session: + for obj_id in obj_ids: + stats["processed"] += 1 + ok = await scrape_catalog_object(db, session, obj_id, snapshot_date) + if ok: + stats["succeeded"] += 1 + else: + stats["failed"] += 1 + + # Commit outer transaction: SAVEPOINT (`begin_nested`) releases внутри loop, + # но outer tx остаётся autobegin'd — без commit() все UPDATE'ы откатятся + # при db.close() в Celery task. + try: + db.commit() + except Exception: + db.rollback() + raise + + logger.info( + "scrape_catalog_objects done: processed=%d succeeded=%d failed=%d skipped=%d", + stats["processed"], + stats["succeeded"], + stats["failed"], + stats["skipped"], + ) + return stats diff --git a/backend/app/services/scrapers/domrf_kn.py b/backend/app/services/scrapers/domrf_kn.py index 235ceb0a..1c16fb50 100644 --- a/backend/app/services/scrapers/domrf_kn.py +++ b/backend/app/services/scrapers/domrf_kn.py @@ -16,6 +16,7 @@ auth shipped in their frontend bundle). from __future__ import annotations +import asyncio import json import logging from datetime import date, datetime @@ -26,6 +27,11 @@ from sqlalchemy import text from sqlalchemy.orm import Session from app.core.db import SessionLocal +from app.services.scrapers.documents import extract_documents, upsert_documents + +# obj_checks import temporarily disabled — endpoint /checks returns 404 (run #19). +# Re-enable with _fetch_obj_checks_safe when endpoint is found (see TODO in Phase B/C). +# from app.services.scrapers.obj_checks import extract_obj_checks, upsert_obj_checks from app.services.scrapers.stealth import BASE_URL, BrowserSession logger = logging.getLogger(__name__) @@ -36,6 +42,16 @@ PATH_SALE_GRAPH = "/сервисы/api/object/{obj_id}/sale_graph" # ?type=apar PATH_SALES_AGG = "/сервисы/api/object/{obj_id}/sales_agg" PATH_INFRA = "/сервисы/api/object/{obj_id}/infrastructure" PATH_PHOTOS = "/сервисы/api/object/construction/progress/photo/{obj_id}" +# Five separate document endpoints discovered via Playwright on obj=65136 (2026-05-17). +# Former single PATH_DOCUMENTS (/documents) returned 404 — replaced by these 5 real paths. +PATH_DOC_RPD = "/сервисы/api/object/{obj_id}/document/rpd" +PATH_DOC_DEVELOPER_REPORT = "/сервисы/api/object/{obj_id}/developer/report" +PATH_DOC_PROJECT_DOCUMENTATION = "/сервисы/api/object/{obj_id}/project/documentation" +PATH_DOC_DOCUMENTATION_OTHER = "/сервисы/api/object/{obj_id}/documentation/other" +PATH_DOC_PERMITS = "/сервисы/api/object/{obj_id}/document/permits" +# NOTE: PATH_CHECKS URL not verified via devtools — likely pattern analogous to /infrastructure. +# If endpoint returns 404/error the failure is logged to kn_scrape_failures and scrape continues. +PATH_CHECKS = "/сервисы/api/object/{obj_id}/checks" RAW_SECTION = "kn_api" SALE_GRAPH_TYPES = ("apartments", "parking") @@ -115,6 +131,36 @@ def _to_date(v: Any) -> date | None: return None +_OBJ_CLASS_PATTERNS: list[tuple[str, str]] = [ + # Ordered from most to least specific to avoid 'бизнес' matching inside longer phrases. + # Each tuple: (regex pattern, canonical class name) + (r"элит", "Элит"), + (r"бизнес", "Бизнес"), + (r"премиум", "Премиум"), + (r"комфорт", "Комфорт"), + (r"стандарт|эконом", "Стандарт"), +] + + +def _extract_obj_class_from_ai(ai_description: str | None) -> str | None: + """Извлечь класс жилья из AI-описания DOM.РФ. + + DOM.РФ не отдаёт `objClass` в /kn/object list — поле всегда NULL. + Однако `aiDescription` содержит текстовое упоминание класса ('комфорт-класса', + '«Комфорт»', 'класс «Бизнес»', 'Класс объекта — комфорт' и т.д.). + + Возвращает каноническое название или None если описание пустое / класс не найден. + """ + import re + + if not ai_description: + return None + for pattern, class_name in _OBJ_CLASS_PATTERNS: + if re.search(pattern, ai_description, re.IGNORECASE): + return class_name + return None + + def _problem_text(v: Any) -> str | None: """Coerce problem flag (sometimes int 0/1, sometimes text) to text or None.""" if v is None: @@ -156,17 +202,68 @@ def _extract_list(payload: dict[str, Any]) -> list[dict[str, Any]]: def _norm_object(row: dict[str, Any], region_cd: int | None = None) -> dict[str, Any]: """Map naш.дом.рф /kn/object row → domrf_kn_objects column dict. - Real API field names: + Real API field names (confirmed via payload audit 2026-05-17, obj=65136): objId, hobjId, developer{devId, shortName, fullName, groupName, companyGroup, devInn}, rpdRegionCd, objAddr, shortAddr, objCommercNm, objFloorMin, objFloorMax, objElemLivingCnt, objSquareLiving, objReady100PercDt, objClass, latitude, longitude, objProblemFlg, problemFlag, siteStatus, objGreenHouseFlg, objGuarantyEscrowFlg, - objStatus, freeFlatsInfo{priceMin, numberFlats}. + objStatus, aiDescription, freeFlatsInfo{priceMin, numberFlats}, + residentialBuildings (section_count), rpdNum (project_declaration_num), + objPublDt (project_published_at), metro{name, time, line, color, colors, isWalk, id}. + + Note: DOM.РФ /kn/object list endpoint never populates `objClass` in API responses + (field absent from payload). Class is extracted from `aiDescription` text as fallback. + + 22e fields NOT in kn-API list payload (remain NULL until catalog scraper 22d): + first_floor_type, elevators_passenger/cargo_count, parking_total_slots, + guest_parking_inside/outside_count, ceiling_height_m, finishing_variants_count, + has_free_planning, playground_kids/sport_count, has_bike_paths, trash_areas_count, + has_ramp, has_low_platforms, has_wheelchair_lift, flat_area_min/max, + price_max_rub, price_per_m2_min/max, parking_provision_pct, avg_flat_area_m2, + domrf_score_location/transport/infrastructure. """ dev = row.get("developer") if isinstance(row.get("developer"), dict) else {} company_group = _g(dev, "companyGroup") if dev else None # Our DB convention: dev_id="_0" matches v_developer_full_metrics. dev_id = f"{company_group}_0" if company_group else None + + # objClass is absent from the list endpoint — extract from aiDescription instead. + obj_class = _g(row, "objClass") or _extract_obj_class_from_ai(_g(row, "aiDescription")) + if obj_class is None: + obj_id = _g(row, "objId", "obj_id", "id") + logger.debug( + "obj_class not found for obj_id=%s (no objClass field, no aiDescription class match)", + obj_id, + ) + + # Metro: kn-API returns a single metro object (nearest station). + # Stored as metro_top3 JSON array (single element) for future multi-station extension. + metro_raw = row.get("metro") + metro_nearest_name: str | None = None + metro_nearest_walk_minutes: int | None = None + metro_top3: list[dict[str, Any]] | None = None + if isinstance(metro_raw, dict) and metro_raw.get("name"): + metro_nearest_name = metro_raw.get("name") + raw_time = metro_raw.get("time") + if raw_time is not None: + try: + metro_nearest_walk_minutes = round(float(raw_time)) + except (TypeError, ValueError): + pass + metro_top3 = [ + { + "name": metro_raw.get("name"), + "time": raw_time, + "line": metro_raw.get("line"), + "color": metro_raw.get("color"), + "isWalk": metro_raw.get("isWalk"), + } + ] + + # freeFlatsInfo.priceMin — price of cheapest available flat in the object + free_info = row.get("freeFlatsInfo") if isinstance(row.get("freeFlatsInfo"), dict) else {} + price_min_rub = _g(free_info, "priceMin") if free_info else None + return { "obj_id": _g(row, "objId", "obj_id", "id"), "hobj_id": _g(row, "hobjId", "hobj_id"), @@ -186,12 +283,49 @@ def _norm_object(row: dict[str, Any], region_cd: int | None = None) -> dict[str, "site_status": _g(row, "siteStatus"), "green_house": _to_bool(_g(row, "objGreenHouseFlg")), "escrow": _to_bool(_g(row, "objGuarantyEscrowFlg")), - "obj_class": _g(row, "objClass"), + "obj_class": obj_class, "wall_type": _g(row, "wallType"), "energy_eff": _g(row, "energyEff"), "latitude": _g(row, "latitude"), "longitude": _g(row, "longitude"), "obj_status": _g(row, "objStatus"), + # 22e: fields mappable from kn-API list payload + "section_count": _g(row, "residentialBuildings"), + "project_declaration_num": _g(row, "rpdNum"), + "project_published_at": _to_date(_g(row, "objPublDt")), + "price_min_rub": price_min_rub, + "dev_group_name": _g(dev, "groupName"), + # 22e + 22h: metro (nearest station from kn-API, single object) + "metro_nearest_name": metro_nearest_name, + "metro_nearest_walk_minutes": metro_nearest_walk_minutes, + "metro_top3": json.dumps(metro_top3, ensure_ascii=False) if metro_top3 else None, + # 22e: fields NOT in kn-API list payload — filled by catalog scraper (22d) + "first_floor_type": None, + "elevators_passenger_count": None, + "elevators_cargo_count": None, + "parking_total_slots": None, + "guest_parking_inside_count": None, + "guest_parking_outside_count": None, + "ceiling_height_m": None, + "finishing_variants_count": None, + "has_free_planning": None, + "avg_flat_area_m2": None, + "playground_kids_count": None, + "playground_sport_count": None, + "has_bike_paths": None, + "trash_areas_count": None, + "has_ramp": None, + "has_low_platforms": None, + "has_wheelchair_lift": None, + "flat_area_min": None, + "flat_area_max": None, + "price_max_rub": None, + "price_per_m2_min": None, + "price_per_m2_max": None, + "parking_provision_pct": None, + "domrf_score_location": None, + "domrf_score_transport": None, + "domrf_score_infrastructure": None, } @@ -219,6 +353,11 @@ def _norm_flat(row: dict[str, Any], region_cd: int | None) -> dict[str, Any]: Real fields: flatId|id (numeric, may be null), odsId, elemId (uuid hash), type, number, isStudio, totalArea, livingArea, rooms, status (free|booked|sold), price, pricePerSquareMeter, numberFloors. Plus injected _objId and _floor. + + Note: /portal/table returns price=null for most objects (sold/booked flats and + objects that don't expose per-flat pricing via this endpoint). price_per_m2 is + derived from price_rub / total_area when the API returns price_rub but omits + pricePerSquareMeter — defensive fallback for partial API responses. """ flat_id = _g(row, "flatId", "id") if flat_id is None: @@ -226,24 +365,47 @@ def _norm_flat(row: dict[str, Any], region_cd: int | None) -> dict[str, Any]: elem = _g(row, "elemId") if elem: flat_id = abs(hash(elem)) % (2**63 - 1) + + price_rub = _g(row, "price") + price_per_m2 = _g(row, "pricePerSquareMeter") + total_area = _g(row, "totalArea") + + # Derive price_per_m2 when API returns price_rub but omits pricePerSquareMeter. + # Covers cases where the table endpoint has the flat price but no pre-computed m² rate. + if price_per_m2 is None and price_rub is not None and total_area and total_area > 0: + price_per_m2 = round(price_rub / total_area, 2) + logger.info( + "derive price_per_m2=%.2f for flat ods_id=%s obj_id=%s", + price_per_m2, + _g(row, "odsId"), + _g(row, "_objId", "objId"), + ) + return { "id": flat_id, "ods_id": _g(row, "odsId"), "flat_type": _g(row, "type", "flatType"), "flat_number": _g(row, "number", "flatNumber"), "is_studio": _to_bool(_g(row, "isStudio")), - "total_area": _g(row, "totalArea"), + "total_area": total_area, "living_area": _g(row, "livingArea"), "rooms": _g(row, "rooms"), "status": _g(row, "status"), - "price_rub": _g(row, "price"), - "price_per_m2": _g(row, "pricePerSquareMeter"), + "price_rub": price_rub, + "price_per_m2": price_per_m2, "floor": _g(row, "_floor", "floor"), "num_floors": _g(row, "numberFloors"), "obj_id": _g(row, "_objId", "objId"), "city": _g(row, "city"), "region_cd": region_cd, "obj_name": _g(row, "objName") or _g(row.get("objInfo") or {}, "objCommercNm"), + # 22b: новые поля квартиры + "section_no": _g(row, "_entrance"), # entranceNumber injected by _flatten_table + "finishing_type": None, # заполнит catalog scraper 22d + "ceiling_height_m": None, # заполнит catalog scraper 22d + "key_handover_dt": None, # заполнит catalog scraper 22d + "catalog_updated_at": None, # заполнит catalog scraper 22d + "catalog_url_hash": None, # заполнит catalog scraper 22d } @@ -257,14 +419,38 @@ UPSERT_OBJECT_SQL = text( floor_min, floor_max, flat_count, square_living, ready_dt, problem_flag, site_status, green_house, escrow, obj_class, wall_type, energy_eff, - latitude, longitude, obj_status, snapshot_date + latitude, longitude, obj_status, + section_count, project_declaration_num, project_published_at, + price_min_rub, dev_group_name, + metro_nearest_name, metro_nearest_walk_minutes, metro_top3, + first_floor_type, elevators_passenger_count, elevators_cargo_count, + parking_total_slots, guest_parking_inside_count, guest_parking_outside_count, + ceiling_height_m, finishing_variants_count, has_free_planning, avg_flat_area_m2, + playground_kids_count, playground_sport_count, has_bike_paths, trash_areas_count, + has_ramp, has_low_platforms, has_wheelchair_lift, + flat_area_min, flat_area_max, price_max_rub, price_per_m2_min, price_per_m2_max, + parking_provision_pct, domrf_score_location, domrf_score_transport, + domrf_score_infrastructure, + snapshot_date ) VALUES ( :obj_id, :hobj_id, :comm_name, :addr, :short_addr, :region_cd, :dev_id, :dev_name, :dev_inn, :floor_min, :floor_max, :flat_count, :square_living, :ready_dt, :problem_flag, :site_status, :green_house, :escrow, :obj_class, :wall_type, :energy_eff, - :latitude, :longitude, :obj_status, :snapshot_date + :latitude, :longitude, :obj_status, + :section_count, :project_declaration_num, :project_published_at, + :price_min_rub, :dev_group_name, + :metro_nearest_name, :metro_nearest_walk_minutes, CAST(:metro_top3 AS jsonb), + :first_floor_type, :elevators_passenger_count, :elevators_cargo_count, + :parking_total_slots, :guest_parking_inside_count, :guest_parking_outside_count, + :ceiling_height_m, :finishing_variants_count, :has_free_planning, :avg_flat_area_m2, + :playground_kids_count, :playground_sport_count, :has_bike_paths, :trash_areas_count, + :has_ramp, :has_low_platforms, :has_wheelchair_lift, + :flat_area_min, :flat_area_max, :price_max_rub, :price_per_m2_min, :price_per_m2_max, + :parking_provision_pct, :domrf_score_location, :domrf_score_transport, + :domrf_score_infrastructure, + :snapshot_date ) ON CONFLICT (obj_id, snapshot_date) DO UPDATE SET comm_name = EXCLUDED.comm_name, @@ -276,10 +462,96 @@ UPSERT_OBJECT_SQL = text( ready_dt = EXCLUDED.ready_dt, site_status = EXCLUDED.site_status, escrow = EXCLUDED.escrow, - obj_class = EXCLUDED.obj_class, + obj_class = COALESCE(EXCLUDED.obj_class, domrf_kn_objects.obj_class), + wall_type = COALESCE(EXCLUDED.wall_type, domrf_kn_objects.wall_type), + energy_eff = COALESCE(EXCLUDED.energy_eff, domrf_kn_objects.energy_eff), latitude = EXCLUDED.latitude, longitude = EXCLUDED.longitude, - obj_status = EXCLUDED.obj_status + obj_status = EXCLUDED.obj_status, + section_count = COALESCE(EXCLUDED.section_count, domrf_kn_objects.section_count), + project_declaration_num = COALESCE( + EXCLUDED.project_declaration_num, domrf_kn_objects.project_declaration_num + ), + project_published_at = COALESCE( + EXCLUDED.project_published_at, domrf_kn_objects.project_published_at + ), + price_min_rub = EXCLUDED.price_min_rub, + dev_group_name = COALESCE(EXCLUDED.dev_group_name, domrf_kn_objects.dev_group_name), + metro_nearest_name = COALESCE( + EXCLUDED.metro_nearest_name, domrf_kn_objects.metro_nearest_name + ), + metro_nearest_walk_minutes = COALESCE( + EXCLUDED.metro_nearest_walk_minutes, domrf_kn_objects.metro_nearest_walk_minutes + ), + metro_top3 = COALESCE(EXCLUDED.metro_top3, domrf_kn_objects.metro_top3), + first_floor_type = COALESCE( + EXCLUDED.first_floor_type, domrf_kn_objects.first_floor_type + ), + elevators_passenger_count = COALESCE( + EXCLUDED.elevators_passenger_count, domrf_kn_objects.elevators_passenger_count + ), + elevators_cargo_count = COALESCE( + EXCLUDED.elevators_cargo_count, domrf_kn_objects.elevators_cargo_count + ), + parking_total_slots = COALESCE( + EXCLUDED.parking_total_slots, domrf_kn_objects.parking_total_slots + ), + guest_parking_inside_count = COALESCE( + EXCLUDED.guest_parking_inside_count, domrf_kn_objects.guest_parking_inside_count + ), + guest_parking_outside_count = COALESCE( + EXCLUDED.guest_parking_outside_count, domrf_kn_objects.guest_parking_outside_count + ), + ceiling_height_m = COALESCE( + EXCLUDED.ceiling_height_m, domrf_kn_objects.ceiling_height_m + ), + finishing_variants_count = COALESCE( + EXCLUDED.finishing_variants_count, domrf_kn_objects.finishing_variants_count + ), + has_free_planning = COALESCE( + EXCLUDED.has_free_planning, domrf_kn_objects.has_free_planning + ), + avg_flat_area_m2 = COALESCE( + EXCLUDED.avg_flat_area_m2, domrf_kn_objects.avg_flat_area_m2 + ), + playground_kids_count = COALESCE( + EXCLUDED.playground_kids_count, domrf_kn_objects.playground_kids_count + ), + playground_sport_count = COALESCE( + EXCLUDED.playground_sport_count, domrf_kn_objects.playground_sport_count + ), + has_bike_paths = COALESCE(EXCLUDED.has_bike_paths, domrf_kn_objects.has_bike_paths), + trash_areas_count = COALESCE( + EXCLUDED.trash_areas_count, domrf_kn_objects.trash_areas_count + ), + has_ramp = COALESCE(EXCLUDED.has_ramp, domrf_kn_objects.has_ramp), + has_low_platforms = COALESCE( + EXCLUDED.has_low_platforms, domrf_kn_objects.has_low_platforms + ), + has_wheelchair_lift = COALESCE( + EXCLUDED.has_wheelchair_lift, domrf_kn_objects.has_wheelchair_lift + ), + flat_area_min = COALESCE(EXCLUDED.flat_area_min, domrf_kn_objects.flat_area_min), + flat_area_max = COALESCE(EXCLUDED.flat_area_max, domrf_kn_objects.flat_area_max), + price_max_rub = COALESCE(EXCLUDED.price_max_rub, domrf_kn_objects.price_max_rub), + price_per_m2_min = COALESCE( + EXCLUDED.price_per_m2_min, domrf_kn_objects.price_per_m2_min + ), + price_per_m2_max = COALESCE( + EXCLUDED.price_per_m2_max, domrf_kn_objects.price_per_m2_max + ), + parking_provision_pct = COALESCE( + EXCLUDED.parking_provision_pct, domrf_kn_objects.parking_provision_pct + ), + domrf_score_location = COALESCE( + EXCLUDED.domrf_score_location, domrf_kn_objects.domrf_score_location + ), + domrf_score_transport = COALESCE( + EXCLUDED.domrf_score_transport, domrf_kn_objects.domrf_score_transport + ), + domrf_score_infrastructure = COALESCE( + EXCLUDED.domrf_score_infrastructure, domrf_kn_objects.domrf_score_infrastructure + ) """ ) @@ -288,11 +560,17 @@ UPSERT_FLAT_SQL = text( INSERT INTO domrf_kn_flats ( id, ods_id, flat_type, flat_number, is_studio, total_area, living_area, rooms, status, price_rub, price_per_m2, floor, num_floors, obj_id, - city, region_cd, obj_name, snapshot_date + city, region_cd, obj_name, + section_no, finishing_type, ceiling_height_m, + key_handover_dt, catalog_updated_at, catalog_url_hash, + snapshot_date ) VALUES ( :id, :ods_id, :flat_type, :flat_number, :is_studio, :total_area, :living_area, :rooms, :status, :price_rub, :price_per_m2, :floor, :num_floors, :obj_id, - :city, :region_cd, :obj_name, :snapshot_date + :city, :region_cd, :obj_name, + :section_no, :finishing_type, :ceiling_height_m, + :key_handover_dt, :catalog_updated_at, :catalog_url_hash, + :snapshot_date ) ON CONFLICT (id, snapshot_date) DO UPDATE SET status = EXCLUDED.status, @@ -300,7 +578,26 @@ UPSERT_FLAT_SQL = text( price_per_m2 = EXCLUDED.price_per_m2, obj_id = EXCLUDED.obj_id, region_cd = EXCLUDED.region_cd, - obj_name = EXCLUDED.obj_name + obj_name = EXCLUDED.obj_name, + flat_number = COALESCE(EXCLUDED.flat_number, domrf_kn_flats.flat_number), + living_area = COALESCE(EXCLUDED.living_area, domrf_kn_flats.living_area), + is_studio = COALESCE(EXCLUDED.is_studio, domrf_kn_flats.is_studio), + total_area = COALESCE(EXCLUDED.total_area, domrf_kn_flats.total_area), + rooms = COALESCE(EXCLUDED.rooms, domrf_kn_flats.rooms), + floor = COALESCE(EXCLUDED.floor, domrf_kn_flats.floor), + num_floors = COALESCE(EXCLUDED.num_floors, domrf_kn_flats.num_floors), + section_no = COALESCE(EXCLUDED.section_no, domrf_kn_flats.section_no), + finishing_type = COALESCE(EXCLUDED.finishing_type, domrf_kn_flats.finishing_type), + ceiling_height_m = COALESCE( + EXCLUDED.ceiling_height_m, domrf_kn_flats.ceiling_height_m + ), + key_handover_dt = COALESCE(EXCLUDED.key_handover_dt, domrf_kn_flats.key_handover_dt), + catalog_updated_at = COALESCE( + EXCLUDED.catalog_updated_at, domrf_kn_flats.catalog_updated_at + ), + catalog_url_hash = COALESCE( + EXCLUDED.catalog_url_hash, domrf_kn_flats.catalog_url_hash + ) """ ) @@ -520,11 +817,9 @@ UPSERT_INFRA_SQL = text( :obj_id, :poi_name, :poi_subtitle, :poi_category, :poi_address, :poi_lat, :poi_lon, :distance_m, :snapshot_date ) - ON CONFLICT (obj_id, poi_name, poi_lat, poi_lon, snapshot_date) DO UPDATE SET - poi_subtitle = EXCLUDED.poi_subtitle, - poi_category = EXCLUDED.poi_category, - poi_address = EXCLUDED.poi_address, - distance_m = EXCLUDED.distance_m + -- Issue #297 22j: новый UNIQUE (obj_id, poi_category, poi_name, poi_address) — без + -- snapshot_date, чтобы каждый scrape run не накапливал ×N дубликаты POI. + ON CONFLICT (obj_id, poi_category, poi_name, poi_address) DO NOTHING """ ) @@ -711,6 +1006,191 @@ async def fetch_photos(sess: BrowserSession, obj_id: int) -> tuple[list[dict[str return (payload.get("data") or []), full_url +async def _fetch_doc_endpoint( + sess: BrowserSession, path_template: str, obj_id: int +) -> tuple[list[dict[str, Any]], str]: + """Generic helper: fetch one document endpoint, return (items, full_url).""" + path = path_template.format(obj_id=obj_id) + payload = await sess.get_json(path, {}) + full_url = f"{BASE_URL}{path}" + if isinstance(payload, list): + return payload, full_url + data = payload.get("data") + if isinstance(data, list): + return data, full_url + return [], full_url + + +async def fetch_doc_rpd(sess: BrowserSession, obj_id: int) -> tuple[list[dict[str, Any]], str]: + """214-ФЗ отчёт (РПД).""" + return await _fetch_doc_endpoint(sess, PATH_DOC_RPD, obj_id) + + +async def fetch_doc_developer_report( + sess: BrowserSession, obj_id: int +) -> tuple[list[dict[str, Any]], str]: + """Отчёт застройщика.""" + return await _fetch_doc_endpoint(sess, PATH_DOC_DEVELOPER_REPORT, obj_id) + + +async def fetch_doc_project_documentation( + sess: BrowserSession, obj_id: int +) -> tuple[list[dict[str, Any]], str]: + """Проектная декларация.""" + return await _fetch_doc_endpoint(sess, PATH_DOC_PROJECT_DOCUMENTATION, obj_id) + + +async def fetch_doc_documentation_other( + sess: BrowserSession, obj_id: int +) -> tuple[list[dict[str, Any]], str]: + """Прочие документы.""" + return await _fetch_doc_endpoint(sess, PATH_DOC_DOCUMENTATION_OTHER, obj_id) + + +async def fetch_doc_permits(sess: BrowserSession, obj_id: int) -> tuple[list[dict[str, Any]], str]: + """Разрешения на строительство.""" + return await _fetch_doc_endpoint(sess, PATH_DOC_PERMITS, obj_id) + + +async def fetch_obj_checks(sess: BrowserSession, obj_id: int) -> tuple[Any, str]: + """Fetch 6 «Проверено на наш.дом.рф» checks for one object. Returns (payload, full_url). + + Endpoint URL (/checks) не верифицирован через devtools — выведен по паттерну + /infrastructure, /documents. При HTTP-ошибке вызывающий код запишет failure в + kn_scrape_failures; scrape не прерывается. + """ + path = PATH_CHECKS.format(obj_id=obj_id) + payload = await sess.get_json(path, {}) + return payload, f"{BASE_URL}{path}" + + +# ── _fetch_*_safe wrappers for asyncio.gather in Phase B/C ─────────────────── +# Каждый wrapper возвращает (kind, full_url, result_or_exception). +# Exceptions НЕ raise — помещаются в возвращаемый tuple. +# BrowserSession._sem (Semaphore(3)) bounds concurrency per-request автоматически. + + +async def _fetch_flats_safe( + sess: BrowserSession, obj_id: int +) -> tuple[str, str, list[dict[str, Any]] | Exception]: + full_url = f"{BASE_URL}{PATH_FLATS_TABLE}?externalId={obj_id}" + try: + flats = await fetch_flats_for_object(sess, obj_id) + return ("flats", full_url, flats) + except Exception as e: + return ("flats", full_url, e) + + +async def _fetch_sale_graph_safe( + sess: BrowserSession, obj_id: int, type_: str +) -> tuple[str, str, tuple[list[dict[str, Any]], str] | Exception]: + full_url = f"{BASE_URL}{PATH_SALE_GRAPH.format(obj_id=obj_id)}?type={type_}" + try: + rows, url = await fetch_sale_graph(sess, obj_id, type_) + return (f"sale_graph_{type_}", url, (rows, url)) + except Exception as e: + return (f"sale_graph_{type_}", full_url, e) + + +async def _fetch_sales_agg_safe( + sess: BrowserSession, obj_id: int +) -> tuple[str, str, tuple[dict[str, Any], str] | Exception]: + full_url = f"{BASE_URL}{PATH_SALES_AGG.format(obj_id=obj_id)}" + try: + agg, url = await fetch_sales_agg(sess, obj_id) + return ("sales_agg", url, (agg, url)) + except Exception as e: + return ("sales_agg", full_url, e) + + +async def _fetch_infrastructure_safe( + sess: BrowserSession, obj_id: int +) -> tuple[str, str, tuple[list[dict[str, Any]], str] | Exception]: + full_url = f"{BASE_URL}{PATH_INFRA.format(obj_id=obj_id)}" + try: + pois, url = await fetch_infrastructure(sess, obj_id) + return ("infrastructure", url, (pois, url)) + except Exception as e: + return ("infrastructure", full_url, e) + + +async def _fetch_photos_safe( + sess: BrowserSession, obj_id: int +) -> tuple[str, str, tuple[list[dict[str, Any]], str] | Exception]: + full_url = f"{BASE_URL}{PATH_PHOTOS.format(obj_id=obj_id)}" + try: + photos, url = await fetch_photos(sess, obj_id) + return ("photos", url, (photos, url)) + except Exception as e: + return ("photos", full_url, e) + + +async def _fetch_doc_rpd_safe( + sess: BrowserSession, obj_id: int +) -> tuple[str, str, tuple[list[dict[str, Any]], str] | Exception]: + full_url = f"{BASE_URL}{PATH_DOC_RPD.format(obj_id=obj_id)}" + try: + items, url = await fetch_doc_rpd(sess, obj_id) + return ("doc_rpd", url, (items, url)) + except Exception as e: + return ("doc_rpd", full_url, e) + + +async def _fetch_doc_developer_report_safe( + sess: BrowserSession, obj_id: int +) -> tuple[str, str, tuple[list[dict[str, Any]], str] | Exception]: + full_url = f"{BASE_URL}{PATH_DOC_DEVELOPER_REPORT.format(obj_id=obj_id)}" + try: + items, url = await fetch_doc_developer_report(sess, obj_id) + return ("doc_developer_report", url, (items, url)) + except Exception as e: + return ("doc_developer_report", full_url, e) + + +async def _fetch_doc_project_documentation_safe( + sess: BrowserSession, obj_id: int +) -> tuple[str, str, tuple[list[dict[str, Any]], str] | Exception]: + full_url = f"{BASE_URL}{PATH_DOC_PROJECT_DOCUMENTATION.format(obj_id=obj_id)}" + try: + items, url = await fetch_doc_project_documentation(sess, obj_id) + return ("doc_project_documentation", url, (items, url)) + except Exception as e: + return ("doc_project_documentation", full_url, e) + + +async def _fetch_doc_documentation_other_safe( + sess: BrowserSession, obj_id: int +) -> tuple[str, str, tuple[list[dict[str, Any]], str] | Exception]: + full_url = f"{BASE_URL}{PATH_DOC_DOCUMENTATION_OTHER.format(obj_id=obj_id)}" + try: + items, url = await fetch_doc_documentation_other(sess, obj_id) + return ("doc_documentation_other", url, (items, url)) + except Exception as e: + return ("doc_documentation_other", full_url, e) + + +async def _fetch_doc_permits_safe( + sess: BrowserSession, obj_id: int +) -> tuple[str, str, tuple[list[dict[str, Any]], str] | Exception]: + full_url = f"{BASE_URL}{PATH_DOC_PERMITS.format(obj_id=obj_id)}" + try: + items, url = await fetch_doc_permits(sess, obj_id) + return ("doc_permits", url, (items, url)) + except Exception as e: + return ("doc_permits", full_url, e) + + +async def _fetch_obj_checks_safe( + sess: BrowserSession, obj_id: int +) -> tuple[str, str, tuple[Any, str] | Exception]: + full_url = f"{BASE_URL}{PATH_CHECKS.format(obj_id=obj_id)}" + try: + payload, url = await fetch_obj_checks(sess, obj_id) + return ("obj_checks", url, (payload, url)) + except Exception as e: + return ("obj_checks", full_url, e) + + def upsert_sale_graph( db: Session, obj_id: int, type_: str, rows: list[dict[str, Any]], snapshot_date: date ) -> int: @@ -1191,6 +1671,8 @@ async def run_region_sweep( "infra_rows": 0, "photos_rows": 0, "photos_downloaded": 0, + "documents_rows": 0, + "checks_rows": 0, } total_flats = 0 request_count = 0 @@ -1273,7 +1755,11 @@ async def run_region_sweep( stage="phase_a", ) - # ── Phase B/C — per-object processing (resumable) ─────────────── + # ── Phase B/C — per-object processing (resumable, parallel per-object) ─ + # Все endpoint'ы одного obj_id запускаются параллельно через asyncio.gather. + # BrowserSession._sem (Semaphore(3)) ограничивает одновременные запросы. + # DB upserts выполняются последовательно после gather — один db Session + # не thread-safe для параллельной записи. pdir = Path(photos_dir) if photos_dir else PHOTOS_DIR_DEFAULT total = len(all_objects) for i in range(start_index, total): @@ -1282,75 +1768,97 @@ async def run_region_sweep( if not obj_id: continue - # Flats per obj — committed immediately + # Собираем корутины для параллельного запуска + coros: list[Any] = [] + if fetch_flats: - try: - flats = await fetch_flats_for_object(sess, obj_id) - if flats: - total_flats += upsert_flats(db, flats, snapshot_date, region_code) - except Exception as e: - log_progress( - db, - run_id, - f"flats failed obj={obj_id}: {type(e).__name__}: {str(e)[:120]}", - level="warn", - stage="fetch_flats", - obj_id=obj_id, - ) + coros.append(_fetch_flats_safe(sess, obj_id)) if extras: - # sale_graph (apartments + parking) - for type_ in SALE_GRAPH_TYPES: - try: - rows, full_url = await fetch_sale_graph(sess, obj_id, type_) - extras_counts["sale_graph_rows"] += upsert_sale_graph( - db, obj_id, type_, rows, snapshot_date - ) - except Exception as e: - full_url = ( - f"{BASE_URL}" - f"{PATH_SALE_GRAPH.format(obj_id=obj_id)}?type={type_}" - ) - _classify_and_log( - db, run_id, obj_id, f"sale_graph_{type_}", full_url, e - ) + coros.append(_fetch_sale_graph_safe(sess, obj_id, "apartments")) + coros.append(_fetch_sale_graph_safe(sess, obj_id, "parking")) + coros.append(_fetch_sales_agg_safe(sess, obj_id)) + coros.append(_fetch_infrastructure_safe(sess, obj_id)) + coros.append(_fetch_photos_safe(sess, obj_id)) + coros.append(_fetch_doc_rpd_safe(sess, obj_id)) + coros.append(_fetch_doc_developer_report_safe(sess, obj_id)) + coros.append(_fetch_doc_project_documentation_safe(sess, obj_id)) + coros.append(_fetch_doc_documentation_other_safe(sess, obj_id)) + coros.append(_fetch_doc_permits_safe(sess, obj_id)) + # TODO: obj_checks endpoint not found at /api/object/{id}/checks (404). + # 6 чек-боксов "Проверено на наш.дом.рф" вероятно inline в kn/object payload. + # Re-enable после investigation структуры объекта (separate PR). + # coros.append(_fetch_obj_checks_safe(sess, obj_id)) - # sales_agg - try: - agg, full_url = await fetch_sales_agg(sess, obj_id) + if not coros: + continue + + # Параллельный fetch всех endpoint'ов одного объекта + results = await asyncio.gather(*coros, return_exceptions=False) + + # Sequential upsert — DB session не thread-safe + all_docs: list[dict[str, Any]] = [] + _doc_kinds = frozenset( + ( + "doc_rpd", + "doc_developer_report", + "doc_project_documentation", + "doc_documentation_other", + "doc_permits", + ) + ) + for kind_tag, full_url, result in results: + if isinstance(result, Exception): + _classify_and_log(db, run_id, obj_id, kind_tag, full_url, result) + continue + + if kind_tag == "flats": + flats_list: list[dict[str, Any]] = result # type: ignore[assignment] + if flats_list: + total_flats += upsert_flats(db, flats_list, snapshot_date, region_code) + + elif kind_tag in ("sale_graph_apartments", "sale_graph_parking"): + sg_type = kind_tag.replace("sale_graph_", "") + rows_sg, _ = result # type: ignore[misc] + extras_counts["sale_graph_rows"] += upsert_sale_graph( + db, obj_id, sg_type, rows_sg, snapshot_date + ) + + elif kind_tag == "sales_agg": + agg_data, _ = result # type: ignore[misc] extras_counts["sales_agg_rows"] += upsert_sales_agg( - db, obj_id, agg, snapshot_date + db, obj_id, agg_data, snapshot_date ) - except Exception as e: - full_url = f"{BASE_URL}{PATH_SALES_AGG.format(obj_id=obj_id)}" - _classify_and_log(db, run_id, obj_id, "sales_agg", full_url, e) - # infrastructure - try: - pois, full_url = await fetch_infrastructure(sess, obj_id) + elif kind_tag == "infrastructure": + pois_data, _ = result # type: ignore[misc] extras_counts["infra_rows"] += upsert_infrastructure( - db, obj_id, pois, snapshot_date + db, obj_id, pois_data, snapshot_date ) - except Exception as e: - full_url = f"{BASE_URL}{PATH_INFRA.format(obj_id=obj_id)}" - _classify_and_log(db, run_id, obj_id, "infrastructure", full_url, e) - # photos - try: - photos, full_url = await fetch_photos(sess, obj_id) + elif kind_tag == "photos": + photos_data, _ = result # type: ignore[misc] local_paths: dict[str, str] = {} thumb_paths: dict[str, str] = {} - if download_photos_binary and photos: + if download_photos_binary and photos_data: local_paths, thumb_paths = await download_photos( - sess, obj_id, photos, pdir + sess, obj_id, photos_data, pdir ) extras_counts["photos_downloaded"] += len(local_paths) extras_counts["photos_rows"] += upsert_photos( - db, obj_id, photos, local_paths, thumb_paths + db, obj_id, photos_data, local_paths, thumb_paths ) - except Exception as e: - full_url = f"{BASE_URL}{PATH_PHOTOS.format(obj_id=obj_id)}" - _classify_and_log(db, run_id, obj_id, "photos", full_url, e) + + elif kind_tag in _doc_kinds: + # Каждый из 5 doc-endpoint'ов отдаёт свой список документов. + # Накапливаем в all_docs — единый upsert после цикла. + doc_items, _ = result # type: ignore[misc] + all_docs.extend(extract_documents(doc_items or [])) + + # Единый upsert всех документов объекта после обработки 5 endpoint'ов. + if all_docs: + ins, _skip = upsert_documents(db, obj_id, all_docs) + extras_counts["documents_rows"] += ins # Checkpoint раз в 10 объектов: запись прогресса + heartbeat. if (i + 1) % 10 == 0: @@ -1363,7 +1871,9 @@ async def run_region_sweep( f" agg={extras_counts['sales_agg_rows']}" f" infra={extras_counts['infra_rows']}" f" photos={extras_counts['photos_rows']}" - f" downloaded={extras_counts['photos_downloaded']}", + f" downloaded={extras_counts['photos_downloaded']}" + f" docs={extras_counts['documents_rows']}" + f" checks={extras_counts['checks_rows']}", stage="extras" if extras else "fetch_flats", ) @@ -1389,7 +1899,58 @@ async def run_region_sweep( log_progress( db, run_id, - f"Готово ✅ objects={total} flats={total_flats} requests={request_count}", + f"Phase D done: objects={total} flats={total_flats} requests={request_count}", + stage="phase_d", + ) + + # ── Phase E — derive is_ekb for this snapshot ────────────────────── + # Проставляем is_ekb=TRUE для объектов Екатеринбурга/Свердловской обл. + # по district_name (заполнен PostGIS join в Phase A). + # Только новые/изменившиеся строки: COALESCE(is_ekb, FALSE) = FALSE. + try: + result_e = db.execute( + text( + """ + UPDATE domrf_kn_objects + SET is_ekb = TRUE + WHERE snapshot_date = :snap + AND ( + district_name ILIKE '%екатеринбург%' + OR district_name ILIKE '%свердловск%' + ) + AND COALESCE(is_ekb, FALSE) = FALSE + """ + ), + {"snap": snapshot_date}, + ) + db.commit() + ekb_updated = result_e.rowcount if result_e.rowcount >= 0 else -1 + log_progress( + db, + run_id, + f"Phase E done: is_ekb derived, updated={ekb_updated} rows" + f" for snap {snapshot_date}", + stage="phase_e", + ) + except Exception as e: + logger.warning("Phase E is_ekb derive failed: %s", e) + try: + db.rollback() + except Exception: + pass + log_progress( + db, + run_id, + f"Phase E is_ekb derive FAILED: {type(e).__name__}: {str(e)[:200]}", + level="warn", + stage="phase_e", + ) + + log_progress( + db, + run_id, + f"Готово objects={total} flats={total_flats} requests={request_count}" + f" docs={extras_counts['documents_rows']} checks={extras_counts['checks_rows']}", stage="done", ) return { diff --git a/backend/app/services/scrapers/ekburg_permits.py b/backend/app/services/scrapers/ekburg_permits.py new file mode 100644 index 00000000..b5435846 --- /dev/null +++ b/backend/app/services/scrapers/ekburg_permits.py @@ -0,0 +1,349 @@ +"""ЕКБ РНС/РВЭ xlsx parser (Issue #105). + +Source: https://xn--80acgfbsl1azdqr.xn--p1ai/дляработы/гиз/градостроительство/разрешение +Format: Form 3 (РНС) + Form 4 (РВЭ) — стандартные Росстат шаблоны. + +Структура xlsx (проверена на всех файлах 2022-2026): + Лист «реестр разрешений на строительс» → РНС (Таблица 3) + Лист «реестр разрешений на ввод» → РВЭ (Таблица 4) + +Расположение заголовков: + 2022: строки 5-7 — заголовок/подзаголовок/номера; данные с row 8 + 2023+: строки 4-6 — заголовок/подзаголовок/номера; данные с row 7 + +Колонки (0-based) одинаковы во всех годах: + 0 developer_name — Наименование застройщика + 1 developer_inn — ИНН (int или str) + 2 developer_address — Адрес застройщика (не сохраняем) + 3 object_type — Тип строительного объекта + 4 object_name — Наименование объекта КС + 5 cadastral_number — Кадастровый номер ЗУ + 6 raw_coord_x — X (СКФ-66, местная СК Свердловской обл.) + 7 raw_coord_y — Y (СКФ-66) + 8 construction_address — Адрес объекта + 9 permit_number — Номер разрешения на строительство + 10 issue_date — Дата разрешения на строительство + 11 expiry_date — Дата окончания разрешения + 12 total_area_sqm — Общая площадь, м² + 13 living_area_sqm — Площадь жилых помещений по проекту, м² + -- дополнительно только для РВЭ: + 14 living_area_fact_sqm — Площадь жилых помещений фактически, м² + 15 rve_number — Номер разрешения на ввод + 16 rve_date — Дата разрешения на ввод + +Координаты raw_coord_x / raw_coord_y хранятся как строки — CRS неизвестна (предположительно +СНСК-66 / СКФ-66 Свердловская обл., local offset system). Геокодирование через construction_address +планируется в Phase 3 (отдельный PR). +""" + +from __future__ import annotations + +import logging +import re +from collections.abc import Iterator +from dataclasses import dataclass, field +from datetime import date, datetime +from io import BytesIO +from typing import Any + +import httpx +from openpyxl import load_workbook + +logger = logging.getLogger(__name__) + +EKBURG_PERMITS_URLS: dict[int, str] = { + 2026: "https://xn--80acgfbsl1azdqr.xn--p1ai/file/70bf01bf31538ee9dd82dadfc47192a0", + 2025: "https://xn--80acgfbsl1azdqr.xn--p1ai/file/6a0a18c9ee327e6e4f76c32a5385a6bd", + 2024: "https://xn--80acgfbsl1azdqr.xn--p1ai/file/907bfa0cf78d5a93c6ccafed1af51fc5", + 2023: "https://xn--80acgfbsl1azdqr.xn--p1ai/file/3b2ef86bf5673adaa65263672f0c623f", + 2022: "https://xn--80acgfbsl1azdqr.xn--p1ai/file/51a7e5654d2fa018bc6db402ab4d5775", +} + +# Листы которые нужно пропустить (справочник, вспомогательные) +_SKIP_SHEETS = {"справочник", "лист1", "sheet1"} + +# Паттерн для проверки что строка — строка данных (первая ячейка непустая и не число-нумерация) +_INN_RE = re.compile(r"\d{10,12}") + + +@dataclass +class PermitRow: + """Parsed row from РНС/РВЭ xlsx.""" + + permit_type: str # "RNS" | "RVE" + permit_number: str + issue_date: date | None + expiry_date: date | None + developer_inn: str | None + developer_name: str | None + object_name: str | None + object_type: str | None + construction_address: str | None + cadastral_number: str | None + total_area_sqm: float | None + living_area_sqm: float | None + living_area_fact_sqm: float | None # только РВЭ + rve_number: str | None # только РВЭ + rve_date: date | None # только РВЭ + raw_coord_x: str | None + raw_coord_y: str | None + source_year: int + source_url: str + raw_row: dict[str, Any] = field(default_factory=dict) + + +# ── helpers ────────────────────────────────────────────────────────────────── + + +def _to_str(v: Any) -> str | None: + """Привести значение ячейки к строке, убрать лишние пробелы. None если пусто.""" + if v is None: + return None + s = str(v).strip().replace("\xa0", " ") + return s if s and s != "-" else None + + +def _to_date(v: Any) -> date | None: + """Привести значение ячейки к date. + + Принимает: datetime (openpyxl), date, строки 'DD.MM.YYYY', 'YYYY-MM-DD'. + """ + if v is None: + return None + if isinstance(v, datetime): + return v.date() + if isinstance(v, date): + return v + if isinstance(v, str): + s = v.strip() + for fmt in ("%d.%m.%Y", "%Y-%m-%d", "%d-%m-%Y"): + try: + return datetime.strptime(s, fmt).date() + except ValueError: + continue + return None + + +def _to_float(v: Any) -> float | None: + """Привести значение ячейки к float. Обрабатывает запятую как разделитель.""" + if v is None: + return None + if isinstance(v, int | float): + return float(v) if not isinstance(v, bool) else None + s = str(v).strip().replace(",", ".").replace("\xa0", "").replace(" ", "") + if not s or s == "-": + return None + try: + return float(s) + except ValueError: + return None + + +def _clean_inn(v: Any) -> str | None: + """Извлечь ИНН из значения ячейки. ИНН — 10 или 12 цифр.""" + if v is None: + return None + s = str(v).strip().replace("\xa0", "") + # Если ячейка — число (openpyxl выдаёт int/float) + if isinstance(v, int | float) and not isinstance(v, bool): + digits = str(int(v)) + if 10 <= len(digits) <= 12: + return digits + return None + m = _INN_RE.search(s) + return m.group(0) if m else None + + +def _detect_permit_type(sheet_name: str) -> str | None: + """Определить тип разрешения по имени листа.""" + name = sheet_name.lower() + if "строит" in name: + return "RNS" + if "ввод" in name: + return "RVE" + return None + + +def _detect_header_row(sheet: Any) -> int: + """Найти строку с заголовком 'Наименование застройщика' (1-based). + + Возвращает номер строки-данных (header_row + 3, так как за заголовком идут + две строки подзаголовков и нумерация). + """ + for row_idx, row in enumerate( + sheet.iter_rows(min_row=1, max_row=10, values_only=True), start=1 + ): + if row and row[0] and "застройщик" in str(row[0]).lower(): + # +3: подзаголовок (X/Y), нумерация (1,2,3...), первая данных + return row_idx + 3 + # Fallback: стандартные позиции + # 2022: header=5, данные с 8; 2023+: header=4, данные с 7 + return 7 + + +def _is_data_row(row: tuple[Any, ...]) -> bool: + """Вернуть True если строка содержит реальные данные (не пустая, не заголовок).""" + if not row or all(v is None for v in row): + return False + first = row[0] + if first is None: + # Продолжение предыдущей записи (merged cells) — пропускаем + return False + # Проверяем, что первая ячейка — не номер (нумерация столбцов в header) + if isinstance(first, int | float) and not isinstance(first, bool): + val = int(first) + if 1 <= val <= 30: + return False + return True + + +def _parse_row( + row: tuple[Any, ...], + permit_type: str, + year: int, + source_url: str, +) -> PermitRow | None: + """Распарсить одну строку данных в PermitRow. + + Возвращает None если строка не содержит номера разрешения (обязательное поле). + """ + # permit_number — колонка 9 (0-based) + permit_number = _to_str(row[9]) if len(row) > 9 else None + if not permit_number: + return None + + raw: dict[str, Any] = {str(i): str(v) for i, v in enumerate(row) if v is not None} + + return PermitRow( + permit_type=permit_type, + permit_number=permit_number, + issue_date=_to_date(row[10]) if len(row) > 10 else None, + expiry_date=_to_date(row[11]) if len(row) > 11 else None, + developer_inn=_clean_inn(row[1]) if len(row) > 1 else None, + developer_name=_to_str(row[0]) if len(row) > 0 else None, + object_name=_to_str(row[4]) if len(row) > 4 else None, + object_type=_to_str(row[3]) if len(row) > 3 else None, + construction_address=_to_str(row[8]) if len(row) > 8 else None, + cadastral_number=_to_str(row[5]) if len(row) > 5 else None, + total_area_sqm=_to_float(row[12]) if len(row) > 12 else None, + living_area_sqm=_to_float(row[13]) if len(row) > 13 else None, + living_area_fact_sqm=_to_float(row[14]) if permit_type == "RVE" and len(row) > 14 else None, + rve_number=_to_str(row[15]) if permit_type == "RVE" and len(row) > 15 else None, + rve_date=_to_date(row[16]) if permit_type == "RVE" and len(row) > 16 else None, + raw_coord_x=_to_str(row[6]) if len(row) > 6 else None, + raw_coord_y=_to_str(row[7]) if len(row) > 7 else None, + source_year=year, + source_url=source_url, + raw_row=raw, + ) + + +# ── client ─────────────────────────────────────────────────────────────────── + + +class EkburgPermitsClient: + """Client для загрузки + парсинга РНС/РВЭ xlsx с екатеринбург.рф.""" + + DEFAULT_TIMEOUT = 60.0 + USER_AGENT = "GenDesign/1.0 (+https://gendsgn.ru) Site Finder permits scraper" + + def __init__(self, *, timeout: float = DEFAULT_TIMEOUT) -> None: + # verify=False: екатеринбург.рф подписан CA Минцифры РФ (нет в certifi). + # Данные публичные open-data — SSL pinning здесь не требуется. Issue #242. + self._client = httpx.Client( + timeout=timeout, + follow_redirects=True, + headers={"User-Agent": self.USER_AGENT}, + verify=False, + ) + + def __enter__(self) -> EkburgPermitsClient: + return self + + def __exit__(self, *_: Any) -> None: + self._client.close() + + def download_xlsx(self, year: int) -> bytes: + """GET xlsx для заданного года. Поднимает ValueError для неизвестного года.""" + url = EKBURG_PERMITS_URLS.get(year) + if not url: + raise ValueError(f"No URL configured for year={year}") + response = self._client.get(url) + response.raise_for_status() + logger.info("Downloaded ekburg permits xlsx %d: %d bytes", year, len(response.content)) + return response.content + + def parse_xlsx(self, content: bytes, year: int, source_url: str) -> Iterator[PermitRow]: + """Parse xlsx bytes → yield PermitRow. + + Автоматически определяет тип листа (РНС/РВЭ) по названию. + Пропускает листы «Справочник», «Лист1» и неизвестные. + """ + wb = load_workbook(BytesIO(content), read_only=True, data_only=True) + + for sheet_name in wb.sheetnames: + if sheet_name.lower() in _SKIP_SHEETS: + continue + + permit_type = _detect_permit_type(sheet_name) + if permit_type is None: + logger.debug("Skipping unknown sheet %r in year %d", sheet_name, year) + continue + + sheet = wb[sheet_name] + data_start = _detect_header_row(sheet) + logger.info( + "Parsing sheet %r (%s) year=%d, data starts at row %d", + sheet_name, + permit_type, + year, + data_start, + ) + yield from self._parse_sheet(sheet, permit_type, year, source_url, data_start) + + def _parse_sheet( + self, + sheet: Any, + permit_type: str, + year: int, + source_url: str, + data_start: int, + ) -> Iterator[PermitRow]: + """Parse один лист → yield PermitRow per data row.""" + row_count = 0 + skip_count = 0 + + for row_idx, row in enumerate( + sheet.iter_rows(min_row=data_start, values_only=True), start=data_start + ): + if not _is_data_row(row): + skip_count += 1 + continue + + try: + permit = _parse_row(row, permit_type, year, source_url) + except Exception as exc: + logger.warning( + "Failed to parse row %d sheet %s year %d: %s", + row_idx, + permit_type, + year, + exc, + ) + skip_count += 1 + continue + + if permit is None: + skip_count += 1 + continue + + row_count += 1 + yield permit + + logger.info( + "Sheet %s year=%d: parsed=%d skipped=%d", + permit_type, + year, + row_count, + skip_count, + ) diff --git a/backend/app/services/scrapers/flat_plans.py b/backend/app/services/scrapers/flat_plans.py new file mode 100644 index 00000000..5583cc6c --- /dev/null +++ b/backend/app/services/scrapers/flat_plans.py @@ -0,0 +1,241 @@ +"""Парсер и DB-writer для планировок квартир DOM.РФ (domrf_kn_flat_plans). + +Источник данных +--------------- +План квартиры отображается на странице + /сервисы/каталог-квартир/квартира/{catalog_hash} +в блоке «Планировка». URL картинки встроен в SSR-HTML страницы и недоступен +через kn-API list-endpoint (/сервисы/api/kn/object) или flat-table endpoint +(/portal-kn/api/sales/portal/table). + +Audit-результат (2026-05-17, obj_id=65136) +------------------------------------------ +- kn_object_place_66 payload: plan_image / planImage / planUrl — NOT FOUND. +- portal/table flat items: поля planImageUrl / layoutUrl — NOT FOUND. +- Вывод: plan image URL находится ТОЛЬКО в SSR-HTML каталога. + +Структура модуля +---------------- +- `extract_flat_plans(raw_payload)` — парсит plan_image_url из flat-table payload + (заглушка — возвращает пустой список до реализации SSR-scraper или endpoint). +- `upsert_flat_plans(db, obj_id, plans, snapshot_date)` — UPSERT в + domrf_kn_flat_plans с COALESCE-защитой скачанных файлов. +- `download_plan_image_stub()` — NotImplementedError placeholder. + Реальная загрузка — отдельный Celery task в рамках 22d-track / #299. + +Связанные файлы +--------------- + data/sql/100_22c_flat_plans.sql — DDL таблицы. + backend/app/services/scrapers/domrf_kn.py — основной kn-scraper (НЕ трогаем + в этом PR; wiring через отдельный PR после реализации SSR-scraper). +Issue #297 sub-task 22c. +""" + +from __future__ import annotations + +import logging +from datetime import date +from typing import Any + +from sqlalchemy import text +from sqlalchemy.orm import Session + +logger = logging.getLogger(__name__) + + +# ── payload parsing ─────────────────────────────────────────────────────────── + + +def extract_flat_plans(raw_payload: dict[str, Any]) -> list[dict[str, Any]]: + """Извлечь plan_image_url из flat-table payload. + + Текущий статус: plan image URL в kn-API flat-table payload ОТСУТСТВУЕТ + (audit 2026-05-17). Функция зарезервирована для будущей интеграции: + - если DOM.РФ добавит поле в portal/table, + - или при подключении SSR-scraper каталога (22d-track). + + Формат входных данных (portal/table response): + { + "externalId": 65136, + "entrances": [ + { + "entranceNumber": 1, + "floors": [ + { + "floorNumber": 4, + "flats": [ + { + "odsId": "65136/1/1.4.3", + "elemId": "d8c7a8103f26c52e427ace5a996706ba", + "totalArea": 36.22, + ... + # планировка НЕ содержится в этом payload + } + ] + } + ] + } + ] + } + + Возвращает список dict с ключами: + ods_id str — идентификатор квартиры + plan_image_url str — URL картинки планировки + obj_id int — внешний ID объекта + + При отсутствии нужного поля возвращает пустой список. + """ + obj_id = raw_payload.get("externalId") + plans: list[dict[str, Any]] = [] + + for entrance in raw_payload.get("entrances") or []: + for floor_data in entrance.get("floors") or []: + for flat in floor_data.get("flats") or []: + ods_id = flat.get("odsId") + if not ods_id: + continue + + # Пробуем несколько возможных имён поля — на случай если API + # в будущем добавит это поле под одним из вариантов. + plan_url = ( + flat.get("planImageUrl") + or flat.get("layoutImageUrl") + or flat.get("planUrl") + or flat.get("layoutUrl") + or flat.get("imageUrl") + ) + if not plan_url: + continue + + plans.append( + { + "ods_id": ods_id, + "obj_id": obj_id, + "plan_image_url": plan_url, + } + ) + + if not plans: + logger.debug( + "extract_flat_plans: obj_id=%s — plan_image_url не найден в payload " + "(ожидаемо: API не содержит это поле, нужен SSR-scraper каталога)", + obj_id, + ) + + return plans + + +# ── DB write ────────────────────────────────────────────────────────────────── + + +_UPSERT_FLAT_PLAN_SQL = text( + """ + INSERT INTO domrf_kn_flat_plans ( + ods_id, obj_id, plan_image_url, local_path, + width_px, height_px, size_bytes, downloaded_at, + snapshot_date, scraped_at + ) VALUES ( + :ods_id, :obj_id, :plan_image_url, :local_path, + :width_px, :height_px, :size_bytes, :downloaded_at, + :snapshot_date, NOW() + ) + ON CONFLICT (ods_id) DO UPDATE SET + plan_image_url = EXCLUDED.plan_image_url, + obj_id = EXCLUDED.obj_id, + snapshot_date = EXCLUDED.snapshot_date, + scraped_at = NOW(), + local_path = COALESCE(domrf_kn_flat_plans.local_path, EXCLUDED.local_path), + downloaded_at = COALESCE(domrf_kn_flat_plans.downloaded_at, EXCLUDED.downloaded_at), + width_px = COALESCE(domrf_kn_flat_plans.width_px, EXCLUDED.width_px), + height_px = COALESCE(domrf_kn_flat_plans.height_px, EXCLUDED.height_px), + size_bytes = COALESCE(domrf_kn_flat_plans.size_bytes, EXCLUDED.size_bytes) + """ +) + + +def upsert_flat_plans( + db: Session, + obj_id: int, + plans: list[dict[str, Any]], + snapshot_date: date, +) -> int: + """UPSERT планировок квартир в domrf_kn_flat_plans. + + Для каждой строки из `plans` (список dict от `extract_flat_plans` или + SSR-scraper) выполняет INSERT ON CONFLICT UPDATE. + + COALESCE-логика: уже скачанные файлы (local_path, downloaded_at, + width_px, height_px, size_bytes) НЕ перезаписываются — только + обновляются plan_image_url и snapshot_date. + + Args: + db: SQLAlchemy Session. + obj_id: ID объекта DOM.РФ (для логирования). + plans: Список dict с ключами ods_id, plan_image_url [, obj_id]. + snapshot_date: Дата snapshot, в котором найден URL. + + Returns: + Количество успешно обработанных строк. + """ + inserted = 0 + for plan in plans: + ods_id = plan.get("ods_id") + plan_url = plan.get("plan_image_url") + if not ods_id or not plan_url: + logger.warning( + "upsert_flat_plans obj=%s: пропущена запись без ods_id/plan_image_url: %s", + obj_id, + plan, + ) + continue + try: + with db.begin_nested(): + db.execute( + _UPSERT_FLAT_PLAN_SQL, + { + "ods_id": ods_id, + "obj_id": plan.get("obj_id") or obj_id, + "plan_image_url": plan_url, + "local_path": plan.get("local_path"), + "width_px": plan.get("width_px"), + "height_px": plan.get("height_px"), + "size_bytes": plan.get("size_bytes"), + "downloaded_at": plan.get("downloaded_at"), + "snapshot_date": snapshot_date, + }, + ) + inserted += 1 + except Exception as e: + logger.warning("upsert_flat_plans obj=%s ods_id=%s failed: %s", obj_id, ods_id, e) + if inserted: + logger.info("upsert_flat_plans obj=%s: %d планировок записано", obj_id, inserted) + return inserted + + +# ── download stub ───────────────────────────────────────────────────────────── + + +def download_plan_image_stub( + plan_image_url: str, + ods_id: str, + dest_dir: str | None = None, +) -> str: + """Заглушка для скачивания бинарника планировки. + + Реальная реализация — отдельный Celery task (22d-track, issue #299). + До реализации бросает NotImplementedError, чтобы случайный вызов + не прошёл незаметно. + + Args: + plan_image_url: URL картинки планировки. + ods_id: Идентификатор квартиры (для имени файла). + dest_dir: Директория для сохранения (None = MEDIA_ROOT/flat_plans/). + + Raises: + NotImplementedError: всегда — до реализации. + """ + raise NotImplementedError( + "download_plan_image_stub: скачивание планировок не реализовано. " + "Реальный downloader — отдельный Celery task (issue #299, 22d-track). " + f"URL={plan_image_url!r} ods_id={ods_id!r} dest_dir={dest_dir!r}" + ) diff --git a/backend/app/services/scrapers/nspd_client.py b/backend/app/services/scrapers/nspd_client.py index da78fdf0..670b66e2 100644 --- a/backend/app/services/scrapers/nspd_client.py +++ b/backend/app/services/scrapers/nspd_client.py @@ -31,6 +31,7 @@ WMS endpoints (per #94 issue body, TIER 1-6 каталог слоёв): from __future__ import annotations +import asyncio import datetime as _dt import json import logging @@ -42,6 +43,7 @@ import urllib.request from dataclasses import dataclass from typing import Any +from app.services.scrapers.nspd_denorm import classify_engineering_kind from app.services.scrapers.nspd_lite import ( _SSL_CTX, HEADERS, @@ -109,6 +111,37 @@ LAYERS: dict[str, int] = { } +# Layers where grid-walk (get_features_in_bbox_grid) must be used instead of +# the legacy single-pixel WMS probe (get_features_in_bbox). These are area/ +# linear layers (territorial zones, red lines, engineering structures, ЗОУИТ, +# risks) that span large areas and are under-returned by a single GetFeatureInfo +# call. Point/polygon EGRN layers (parcels, buildings) stay on legacy for now. +_GRID_WALK_LAYERS: frozenset[str] = frozenset( + { + "territorial_zones", + "red_lines", + "engineering_structures", + # ЗОУИТ (TIER 2) + "zouit_okn", + "zouit_engineering", + "zouit_natural", + "zouit_protected", + "zouit_other", + # Risks (TIER 3) + "risk_flooding_underground", + "risk_flooding", + "risk_swampification", + "risk_landslide", + "risk_abrasion", + "risk_erosion_water", + "risk_erosion_linear", + "risk_erosion_wind", + "risk_desertification", + "risk_clutter", + "risk_burns", + } +) + # Default rate limit (мс между запросами) — баланс между скоростью и WAF DEFAULT_RATE_MS = 600 @@ -199,6 +232,9 @@ class QuarterDump: engineering_structures: list[NSPDFeature] zouit: dict[str, list[NSPDFeature]] # {"okn": [...], "engineering": [...], ...} risks: dict[str, list[NSPDFeature]] # {"flooding": [...], "landslide": [...], ...} + # TIER 4 opportunity layers (issue #94 PR2). + # {"auction_parcels": [...], "scheme_parcels": [...], "free_parcels": [...], ...} + opportunity: dict[str, list[NSPDFeature]] # tuple, не list — frozen dataclass + immutable contents (audit/debug snapshot) layers_fetched: tuple[str, ...] bbox_3857: tuple[float, float, float, float] | None # bbox квартала @@ -215,6 +251,7 @@ class QuarterDump: + len(self.engineering_structures) + sum(len(v) for v in self.zouit.values()) + sum(len(v) for v in self.risks.values()) + + sum(len(v) for v in self.opportunity.values()) ) @@ -380,24 +417,22 @@ class NSPDClient: width: int = 4096, height: int = 4096, ) -> list[NSPDFeature]: - """Bulk fetch features в bbox через GetFeatureInfo с большим bbox. + """WMS GetFeatureInfo на одном центральном пикселе bbox. - Workaround: WFS GetCapabilities → 404 на nspd.gov.ru, нет WFS - GetFeature endpoint. Решение: использовать GetFeatureInfo с large - bbox и точкой в центре (I=W/2, J=H/2) — возвращает все features - пересекающиеся с bbox. + DEPRECATED: возвращает 0-3 features под одним пикселем (I=W/2, J=H/2). + НЕ является bulk fetch несмотря на исходный docstring — WMS GetFeatureInfo + по стандарту OGC возвращает объекты строго под одной pixel-точкой, а не + во всём bbox. Для получения всех объектов в bbox используй + `get_features_in_bbox_grid`. + + See: fixes/Bug_NSPD_WMS_NotBulk_2026_May14.md Args: bbox_3857: (xmin, ymin, xmax, ymax) в EPSG:3857 метрах. - width/height: размер виртуального tile. Большой → большой bbox. + width/height: размер виртуального tile. Returns: list[NSPDFeature]; пусто если ничего не найдено. - - Use cases (per #94 acceptance): - - sync_territorial_zones_bbox → закрывает G1 #28 ПЗЗ - - sync_zouit_*_bbox → G3 #30 - - sync_risk_zones_bbox → новый risk overlay """ xmin, ymin, xmax, ymax = bbox_3857 params = { @@ -422,6 +457,149 @@ class NSPDClient: feats = (data or {}).get("features") or [] return [NSPDFeature.from_raw(f) for f in feats] + # ── 3b. get_features_in_bbox_grid ─────────────────────────────────────── + + def get_features_in_bbox_grid( + self, + layer_id: int, + bbox: tuple[float, float, float, float], + *, + grid_n: int = 7, + step_m: float = 50.0, + tile_size: int = 512, + ) -> list[NSPDFeature]: + """Bulk-аппроксимация bbox через grid-walk WMS GetFeatureInfo. + + Разбивает bbox на grid_n × grid_n равных ячеек. В каждой ячейке + вызывает WMS GetFeatureInfo в центральном пикселе. Дедуплицирует + результаты по feature_id / cad_num / reg_numb_border — возвращает + список уникальных NSPDFeature. + + Делегирует HTTP через NSPDBulkClient.wms_feature_info (async httpx + с semaphore и retry), запуская asyncio event loop синхронно через + asyncio.run(). Предназначен для вызова из синхронного кода (Celery + task, FastAPI sync handler). + + Concurrency: NSPDBulkClient._sem (per-instance, capacity=3) ограничивает параллельные + запросы. При grid_n=7 (49 ячеек) — все 49 ячеек запускаются одним + gather; семафор пропускает не более 3 одновременно. Thread-safety: + каждый вызов get_features_in_bbox_grid создаёт новый event loop + через asyncio.run() — безопасно из разных Celery workers (process- + уровень изоляции). + + Args: + layer_id: NSPD layer ID (например 36328 сооружения, 37578 ЗОУИТ). + bbox: (xmin, ymin, xmax, ymax) в EPSG:3857 (метры). + grid_n: размер сетки по каждой оси. 7 → 49 запросов (~coarse), + 15 → 225 запросов (~fine). По умолчанию 7 для первичного scan. + step_m: минимальный шаг ячейки в метрах. Если bbox меньше + grid_n*step_m — grid_n уменьшается автоматически чтобы + ячейки не становились меньше step_m. + tile_size: размер виртуального WMS тайла (пиксели). + + Returns: + Дедуплицированный list[NSPDFeature]. Может быть пуст если в bbox + нет объектов данного layer'а. + + Note: + Не делает live HTTP вызовы если вызван с mock NSPDBulkClient. + Rate-limit управляется семафором NSPDBulkClient._sem (per-instance, capacity=3) + + asyncio.sleep(0.05) jitter — не через self.rate_ms. + """ + # Импортируем здесь чтобы избежать circular import: + # nspd_client ← nspd_bulk_client (оба top-level scrapers, не cross-domain) + from app.scrapers.nspd_bulk_client import NSPDBulkClient + + xmin, ymin, xmax, ymax = bbox + width_m = xmax - xmin + height_m = ymax - ymin + + # Авто-коррекция grid_n если bbox мал для шага step_m + effective_n = min( + grid_n, + max(1, int(width_m / step_m)), + max(1, int(height_m / step_m)), + ) + if effective_n < grid_n: + logger.info( + "get_features_in_bbox_grid layer=%d: bbox %.0fx%.0fm < grid_n=%d×step_m=%.0f" + " — уменьшаем grid до %d×%d", + layer_id, + width_m, + height_m, + grid_n, + step_m, + effective_n, + effective_n, + ) + + x_step = width_m / effective_n + y_step = height_m / effective_n + + # Генерируем список (sub_bbox, click_xy) ячеек + cells: list[tuple[tuple[float, float, float, float], tuple[int, int]]] = [] + click_px = tile_size // 2 + for i in range(effective_n): + for j in range(effective_n): + cell_xmin = xmin + i * x_step + cell_ymin = ymin + j * y_step + cell_xmax = cell_xmin + x_step + cell_ymax = cell_ymin + y_step + cells.append(((cell_xmin, cell_ymin, cell_xmax, cell_ymax), (click_px, click_px))) + + async def _run_grid() -> list[NSPDFeature]: + async with NSPDBulkClient() as client: + tasks = [ + client.wms_feature_info(layer_id, sub_bbox, click_xy, tile_size, tile_size) + for sub_bbox, click_xy in cells + ] + results = await asyncio.gather(*tasks, return_exceptions=True) + + features: list[NSPDFeature] = [] + for r in results: + if isinstance(r, Exception): + logger.warning("get_features_in_bbox_grid layer=%d cell error: %s", layer_id, r) + continue + for bulk_feat in r: + raw = { + "id": bulk_feat.id, + "geometry": bulk_feat.geometry, + "properties": bulk_feat.properties, + } + features.append(NSPDFeature.from_raw(raw)) + return features + + raw_features = asyncio.run(_run_grid()) + + # Дедупликация — приоритет ключей: feature_id > cad_num > reg_numb_border + seen: set[str] = set() + deduped: list[NSPDFeature] = [] + for feat in raw_features: + props = feat.properties + dedup_key = ( + feat.feature_id + or props.get("cad_num") + or props.get("cad_number") + or props.get("reg_numb_border") + or props.get("label") + ) + if dedup_key is not None: + if dedup_key in seen: + continue + seen.add(dedup_key) + deduped.append(feat) + + logger.info( + "get_features_in_bbox_grid layer=%d grid=%dx%d cells=%d raw=%d deduped=%d", + layer_id, + effective_n, + effective_n, + len(cells), + len(raw_features), + len(deduped), + ) + return deduped + # ── 4. list_layers ────────────────────────────────────────────────────── def list_layers(self, theme_id: int = THEME_PKK) -> list[NSPDLayer]: @@ -489,6 +667,16 @@ class NSPDClient: "clutter": "risk_clutter", "burns": "risk_burns", } + # TIER 4 — Opportunity layers (issue #94 PR2). + # short_name → LAYERS dict key (for get_features_in_bbox lookup). + # Features stored in features_json с layer = "opportunity_". + QUARTER_OPPORTUNITY_LAYERS: dict[str, str] = { # noqa: RUF012 + "auction_parcels": "auction_parcels", # 37299 — аукционные ЗУ + "scheme_parcels": "scheme_parcels", # 37294 — схемы расположения ЗУ + "free_parcels": "free_parcels", # 37298 — свободные ЗУ + "future_parcels": "future_parcels", # 36473 — планируемые ЗУ + "oopt": "protected_areas", # 875845 — ООПТ + } def search_by_quarter( self, @@ -496,6 +684,7 @@ class NSPDClient: *, include_zouit: bool = True, include_risks: bool = False, + include_opportunity: bool = False, ) -> QuarterDump: """Harvest всех NSPD-данных для квартала: 1 vacuum, N layers. @@ -517,13 +706,17 @@ class NSPDClient: include_zouit: Включать TIER 2 ЗОУИТ layers (G3). Default True. include_risks: Включать TIER 3 risk zones. Default False (rate-limit budget; для отдельного D-N risk score можно включить). + include_opportunity: Включать TIER 4 opportunity layers (auction_parcels, + scheme_parcels, free_parcels, future_parcels, oopt). Default False. + +5 HTTP запросов при включении. Returns: QuarterDump с per-layer feature lists. Если NSPD пуст / quarter не найден — `quarter=None`, `bbox_3857=None`, все feature lists пустые (no bulk-fetch без bounds — нет смысла). При этом dict- - поля `zouit` / `risks` всё равно populated с пустыми lists для - каждого включённого short_name (структура контракта стабильна). + поля `zouit` / `risks` / `opportunity` всё равно populated с пустыми + lists для каждого включённого short_name + (структура контракта стабильна). `layers_fetched` в этом случае содержит только `('search',)`. Raises: @@ -532,7 +725,7 @@ class NSPDClient: операция атомарна (failure → exception). Закрывает: foundation для G1 #28 ПЗЗ, G3 #30 ЗОУИТ, P2 #46 neighbors, - E1 #51 parcels backfill, #96 ЕГРН помещения. + E1 #51 parcels backfill, #96 ЕГРН помещения, #94 PR2 opportunity. """ # 1. Quarter geometry через REST search quarter_search = self.search_by_cad(quarter_cad, thematic_id=2) @@ -548,7 +741,17 @@ class NSPDClient: layers_fetched: list[str] = ["search"] def _fetch_layer(name_in_dump: str, layer_key: str) -> list[NSPDFeature]: - """Helper: безопасно получить features для одного layer.""" + """Helper: безопасно получить features для одного layer. + + Dispatch: + - area/linear layers (_GRID_WALK_LAYERS) → grid-walk + (get_features_in_bbox_grid, grid_n=7, step_m=50) + - point/polygon EGRN layers (parcels, buildings, ons, enk) + → legacy single-pixel WMS (get_features_in_bbox) + + Engineering structures features дополнительно обогащаются + properties["classified_kind"] через classify_engineering_kind. + """ if bbox is None: return [] layer_id = LAYERS.get(layer_key) @@ -556,7 +759,29 @@ class NSPDClient: logger.warning("search_by_quarter: unknown layer key %s", layer_key) return [] layers_fetched.append(name_in_dump) - return self.get_features_in_bbox(layer_id, bbox) + if layer_key in _GRID_WALK_LAYERS: + features = self.get_features_in_bbox_grid(layer_id, bbox, grid_n=7, step_m=50.0) + # Обогатить engineering_structures classified_kind + if layer_key == "engineering_structures": + for feat in features: + feat.properties["classified_kind"] = classify_engineering_kind( + feat.properties + ) + logger.info( + "search_by_quarter layer=%s method=grid_walk count=%d quarter=%s", + name_in_dump, + len(features), + quarter_cad, + ) + return features + features_legacy = self.get_features_in_bbox(layer_id, bbox) + logger.info( + "search_by_quarter layer=%s method=legacy count=%d quarter=%s", + name_in_dump, + len(features_legacy), + quarter_cad, + ) + return features_legacy # 3. Core layers parcels = _fetch_layer("parcels", "parcels") @@ -577,6 +802,12 @@ class NSPDClient: for short_name, layer_key in self.QUARTER_RISK_LAYERS.items(): risks[short_name] = _fetch_layer(f"risk_{short_name}", layer_key) + # 6. Opportunity layers (TIER 4, issue #94 PR2) + opportunity: dict[str, list[NSPDFeature]] = {} + if include_opportunity: + for short_name, layer_key in self.QUARTER_OPPORTUNITY_LAYERS.items(): + opportunity[short_name] = _fetch_layer(f"opportunity_{short_name}", layer_key) + return QuarterDump( quarter_cad=quarter_cad, quarter=quarter_feat, @@ -587,6 +818,7 @@ class NSPDClient: engineering_structures=engineering_structures, zouit=zouit, risks=risks, + opportunity=opportunity, layers_fetched=tuple(layers_fetched), bbox_3857=bbox, fetched_at_utc=_dt.datetime.now(_dt.UTC).isoformat(), diff --git a/backend/app/services/scrapers/nspd_denorm.py b/backend/app/services/scrapers/nspd_denorm.py new file mode 100644 index 00000000..aef36139 --- /dev/null +++ b/backend/app/services/scrapers/nspd_denorm.py @@ -0,0 +1,369 @@ +"""Denormalize nspd_quarter_dumps.features_json → nspd_parcels / nspd_buildings. + +Используется: + - Inline в harvest_quarter task после UPSERT dump'а + - Backfill task для existing dumps (backfill_all_dumps) + +Property mapping (NSPD WMS GetFeatureInfo responses): + Parcels (layer="parcels", layer id 36048): + cad_num — кадастровый номер ЗУ + permitted_use — ВРИ + land_category — категория земель + cost_value — кадастровая стоимость, руб. + area — площадь, м² + address — адрес + + Buildings (layer="buildings", layer id 36049): + cad_num — кадастровый номер ОКС + purpose — назначение ("Многоквартирный дом" / "Нежилое здание" / ...) + floors_above_ground — надземных этажей + floors_underground — подземных этажей + year_built — год постройки + cost_value — кадастровая стоимость, руб. + build_record_area — площадь по ГКН, м² + address — адрес + +Геометрия в features_json хранится в EPSG:3857 (как возвращает NSPD WMS) — +трансформация 3857→4326 делается в SQL через ST_Transform. +""" + +from __future__ import annotations + +import datetime +import json +import logging +import re +from typing import Any + +from sqlalchemy import text +from sqlalchemy.orm import Session + +logger = logging.getLogger(__name__) + +# ── Engineering classifier ───────────────────────────────────────────────────── + +# Паттерны для классификации инженерных сооружений (layer 36328) по текстовым +# свойствам. Источник: bug-post-mortem fixes/Bug_NSPD_WMS_NotBulk_2026_May14.md +# + live test данные 7×7 grid на 1км² центра ЕКБ. +# Порядок проверки важен: более специфичные паттерны идут первыми. + +_ENGINEERING_PATTERNS: list[tuple[re.Pattern[str], str]] = [ + # gas — газопроводы, ГРП, ГК (газовые колодцы) + (re.compile(r"газопровод|газоснабж|ГРП\b|ГК\b", re.IGNORECASE), "gas"), + # sewage — канализация, сток, ливневые сети + (re.compile(r"канализ|сточ|ливнев|самотёч|самотеч", re.IGNORECASE), "sewage"), + # heat — тепловые сети, котельные, ТЭЦ + ( + re.compile(r"теплов(ая|ой|ые)|теплосеть|теплоснабж|котельн|ТЭЦ\b", re.IGNORECASE), + "heat", + ), + # electric — электросети, ВЛ, КЛ, ЛЭП, подстанции, трансформаторы, ТП + ( + re.compile( + r"электроэнерг|ВЛ[\s\-]|ВЛ-?\d|КЛ[\s\-]|КЛ-?\d|ЛЭП\b" + r"|подстанц|трансформ|ТП[\s\-]?\d", + re.IGNORECASE, + ), + "electric", + ), + # water — водопровод, водоснабжение, хозбытовые водосети + (re.compile(r"водопровод|водоснабж|хозбытов|водовод", re.IGNORECASE), "water"), +] + +# Поля из NSPD properties в которых ищем паттерны (по приоритету) +_ENGINEERING_TEXT_FIELDS = ("params_name", "name", "params_purpose", "purpose", "label") + + +def classify_engineering_kind(properties: dict[str, Any]) -> str: + """Классифицировать инженерное сооружение (layer 36328) по его properties. + + Проверяет поля `params_name`, `name`, `params_purpose`, `purpose`, `label` + против regex-паттернов. Возвращает первое совпадение. + + Args: + properties: dict свойств NSPDFeature.properties из WMS GetFeatureInfo + или search/geoportal response. + + Returns: + Одно из: ``"water"`` | ``"sewage"`` | ``"gas"`` | ``"heat"`` + | ``"electric"`` | ``"other"``. + + Examples: + >>> classify_engineering_kind({"params_name": "Газопровод высокого давления"}) + 'gas' + >>> classify_engineering_kind({"name": "КЛ 10 кВ ТП 64102"}) + 'electric' + >>> classify_engineering_kind({"params_purpose": "Водопровод хозбытовой"}) + 'water' + >>> classify_engineering_kind({"params_name": "Тепловая сеть"}) + 'heat' + >>> classify_engineering_kind({"name": "Канализация"}) + 'sewage' + """ + # Собираем текст для проверки из всех релевантных полей + text_parts: list[str] = [] + for field in _ENGINEERING_TEXT_FIELDS: + val = properties.get(field) + if val and isinstance(val, str): + text_parts.append(val) + + combined = " ".join(text_parts) + if not combined: + return "other" + + for pattern, kind in _ENGINEERING_PATTERNS: + if pattern.search(combined): + return kind + + return "other" + + +# ── Type coercions ───────────────────────────────────────────────────────────── + + +def _coerce_int(v: Any) -> int | None: + """NSPD properties могут быть str / int / None.""" + if v is None: + return None + try: + return int(v) + except (ValueError, TypeError): + return None + + +def _coerce_numeric(v: Any) -> float | None: + """NSPD properties могут быть str с запятой (европейский формат) или None.""" + if v is None: + return None + try: + return float(str(v).replace(",", ".").strip()) + except (ValueError, TypeError): + return None + + +# ── Parcel UPSERT ────────────────────────────────────────────────────────────── + +_PARCEL_UPSERT_SQL = text( + """ + INSERT INTO nspd_parcels ( + cad_num, quarter_cad, permitted_use, land_category, + cost_value, cost_per_m2, area_sqm, address, geom, snapshot_date + ) VALUES ( + CAST(:cad_num AS text), + CAST(:quarter_cad AS text), + CAST(:permitted_use AS text), + CAST(:land_category AS text), + CAST(:cost_value AS numeric), + CAST(:cost_per_m2 AS numeric), + CAST(:area_sqm AS numeric), + CAST(:address AS text), + CASE WHEN :geom_json IS NULL THEN NULL + ELSE ST_Transform(ST_SetSRID(ST_GeomFromGeoJSON(:geom_json), 3857), 4326) + END, + CAST(:snapshot_date AS date) + ) + ON CONFLICT (cad_num) DO UPDATE SET + quarter_cad = EXCLUDED.quarter_cad, + permitted_use = EXCLUDED.permitted_use, + land_category = EXCLUDED.land_category, + cost_value = EXCLUDED.cost_value, + cost_per_m2 = EXCLUDED.cost_per_m2, + area_sqm = EXCLUDED.area_sqm, + address = EXCLUDED.address, + geom = EXCLUDED.geom, + snapshot_date = EXCLUDED.snapshot_date, + updated_at = NOW() + """ +) + + +def denorm_parcel_feature( + db: Session, + *, + feature: dict[str, Any], + quarter_cad: str, + snapshot_date: str, +) -> bool: + """UPSERT one parcel feature → nspd_parcels. + + Returns True если строка вставлена/обновлена, False при пропуске (нет cad_num). + """ + props = feature.get("properties") or {} + cad_num = props.get("cad_num") or props.get("cadastral_number") + if not cad_num: + return False + + area = _coerce_numeric(props.get("area")) + cost_value = _coerce_numeric(props.get("cost_value")) + cost_per_m2: float | None = None + if cost_value is not None and area is not None and area > 0: + cost_per_m2 = cost_value / area + + geom = feature.get("geometry") + geom_json: str | None = json.dumps(geom, ensure_ascii=False) if geom else None + + try: + with db.begin_nested(): + db.execute( + _PARCEL_UPSERT_SQL, + { + "cad_num": cad_num, + "quarter_cad": quarter_cad, + "permitted_use": props.get("permitted_use"), + "land_category": props.get("land_category"), + "cost_value": cost_value, + "cost_per_m2": cost_per_m2, + "area_sqm": area, + "address": props.get("address"), + "geom_json": geom_json, + "snapshot_date": snapshot_date, + }, + ) + return True + except Exception as e: + logger.warning("denorm parcel %s failed: %s", cad_num, e) + return False + + +# ── Building UPSERT ──────────────────────────────────────────────────────────── + +_BUILDING_UPSERT_SQL = text( + """ + INSERT INTO nspd_buildings ( + cad_num, quarter_cad, purpose, floors, floors_underground, + year_built, cost_value, build_record_area, address, geom, snapshot_date + ) VALUES ( + CAST(:cad_num AS text), + CAST(:quarter_cad AS text), + CAST(:purpose AS text), + CAST(:floors AS integer), + CAST(:floors_underground AS integer), + CAST(:year_built AS integer), + CAST(:cost_value AS numeric), + CAST(:build_record_area AS numeric), + CAST(:address AS text), + CASE WHEN :geom_json IS NULL THEN NULL + ELSE ST_Transform(ST_SetSRID(ST_GeomFromGeoJSON(:geom_json), 3857), 4326) + END, + CAST(:snapshot_date AS date) + ) + ON CONFLICT (cad_num) DO UPDATE SET + quarter_cad = EXCLUDED.quarter_cad, + purpose = EXCLUDED.purpose, + floors = EXCLUDED.floors, + floors_underground = EXCLUDED.floors_underground, + year_built = EXCLUDED.year_built, + cost_value = EXCLUDED.cost_value, + build_record_area = EXCLUDED.build_record_area, + address = EXCLUDED.address, + geom = EXCLUDED.geom, + snapshot_date = EXCLUDED.snapshot_date, + updated_at = NOW() + """ +) + + +def denorm_building_feature( + db: Session, + *, + feature: dict[str, Any], + quarter_cad: str, + snapshot_date: str, +) -> bool: + """UPSERT one building feature → nspd_buildings. + + Returns True если строка вставлена/обновлена, False при пропуске (нет cad_num). + """ + props = feature.get("properties") or {} + cad_num = props.get("cad_num") or props.get("cadastral_number") + if not cad_num: + return False + + geom = feature.get("geometry") + geom_json: str | None = json.dumps(geom, ensure_ascii=False) if geom else None + + try: + with db.begin_nested(): + db.execute( + _BUILDING_UPSERT_SQL, + { + "cad_num": cad_num, + "quarter_cad": quarter_cad, + "purpose": props.get("purpose"), + "floors": _coerce_int(props.get("floors_above_ground")), + "floors_underground": _coerce_int(props.get("floors_underground")), + "year_built": _coerce_int(props.get("year_built")), + "cost_value": _coerce_numeric(props.get("cost_value")), + "build_record_area": _coerce_numeric(props.get("build_record_area")), + "address": props.get("address"), + "geom_json": geom_json, + "snapshot_date": snapshot_date, + }, + ) + return True + except Exception as e: + logger.warning("denorm building %s failed: %s", cad_num, e) + return False + + +# ── Batch helper ─────────────────────────────────────────────────────────────── + + +def denorm_dump( + db: Session, + *, + quarter_cad: str, + features: list[dict[str, Any]], +) -> dict[str, int]: + """Denorm all features из одного quarter dump → nspd_parcels / nspd_buildings. + + Итерирует features_json плоский список. Записи с layer="parcels" идут в + nspd_parcels, layer="buildings" → nspd_buildings. Остальные layers пропускаются. + + Каждая строка UPSERT'ится в отдельном SAVEPOINT (begin_nested) — ошибка + одной строки не откатывает весь batch. + + Args: + db: SQLAlchemy Session. Caller отвечает за commit/close после вызова. + quarter_cad: 3-сегментный кадастровый квартал. + features: плоский list из features_json JSONB (уже декодированный Python list). + + Returns: + dict {"parcels": N, "buildings": M, "errors": K} — количество обработанных строк. + """ + snapshot_date = datetime.date.today().isoformat() + parcels_n = 0 + buildings_n = 0 + errors_n = 0 + + for feat in features: + layer = feat.get("layer", "") + try: + if layer == "parcels": + if denorm_parcel_feature( + db, feature=feat, quarter_cad=quarter_cad, snapshot_date=snapshot_date + ): + parcels_n += 1 + else: + errors_n += 1 + elif layer == "buildings": + if denorm_building_feature( + db, feature=feat, quarter_cad=quarter_cad, snapshot_date=snapshot_date + ): + buildings_n += 1 + else: + errors_n += 1 + # Остальные layers (territorial_zones, zouit_*, risk_*, ...) — пропускаем + except Exception as e: + logger.warning("denorm feature (layer=%s quarter=%s) failed: %s", layer, quarter_cad, e) + errors_n += 1 + + db.commit() + logger.info( + "denorm_dump quarter=%s parcels=%d buildings=%d errors=%d", + quarter_cad, + parcels_n, + buildings_n, + errors_n, + ) + return {"parcels": parcels_n, "buildings": buildings_n, "errors": errors_n} diff --git a/backend/app/services/scrapers/obj_checks.py b/backend/app/services/scrapers/obj_checks.py new file mode 100644 index 00000000..23a78aed --- /dev/null +++ b/backend/app/services/scrapers/obj_checks.py @@ -0,0 +1,175 @@ +"""Extractor for 6 «Проверено на наш.дом.рф» checks per object. + +Issue #297, sub-task 22f. Table created in PR #303 (data/sql/111_22f_domrf_obj_checks.sql). + +Check types (canonical): + no_problems / docs / timing / photos / bankruptcy / declaration + +Source endpoint: /сервисы/api/object/{obj_id}/checks + (pattern analogичен /infrastructure и /photos — те же сервисы/api/object/{obj_id}/ prefix) + URL not verified via devtools — структура payload выведена из аудита страницы ЖК. + Если endpoint не существует — scraper получит HTTP-ошибку, которую _classify_and_log + запишет в kn_scrape_failures, данные в domrf_obj_checks не поступят, scrape не упадёт. + +Expected payload shape (предположительно): + { + "data": { + "noProblemObjects": true, # no_problems + "hasDocuments": true, # docs + "meetsDeadlines": true, # timing + "hasPhotos": true, # photos + "notBankrupt": true, # bankruptcy + "hasDeclaration": true # declaration + } + } + + OR possibly an array: + [{"checkType": "no_problems", "passed": true}, ...] + +Поддерживаются оба варианта: dict-payload (поля маппятся через CHECK_FIELD_MAP) +и list-payload (поля check_type + passed/value). +""" + +from __future__ import annotations + +import logging +from typing import Any + +from sqlalchemy import text +from sqlalchemy.orm import Session + +logger = logging.getLogger(__name__) + +CHECK_TYPES = ["no_problems", "docs", "timing", "photos", "bankruptcy", "declaration"] + +# Mapping of possible API field names → canonical check_type. +# Best-guess from DOM.РФ API field naming conventions (кейс camelCase). +_CHECK_FIELD_MAP: dict[str, str] = { + "noProblemObjects": "no_problems", + "noProblemFlg": "no_problems", + "hasDocuments": "docs", + "documentsFlg": "docs", + "meetsDeadlines": "timing", + "deadlinesFlg": "timing", + "hasPhotos": "photos", + "photosFlg": "photos", + "notBankrupt": "bankruptcy", + "bankruptcyFlg": "bankruptcy", + "hasDeclaration": "declaration", + "declarationFlg": "declaration", +} + +# Canonical check_type name → possible API field name aliases +_CHECK_TYPE_ALIASES: dict[str, list[str]] = { + "no_problems": ["no_problems", "noProblemObjects", "noProblemFlg"], + "docs": ["docs", "hasDocuments", "documentsFlg"], + "timing": ["timing", "meetsDeadlines", "deadlinesFlg"], + "photos": ["photos", "hasPhotos", "photosFlg"], + "bankruptcy": ["bankruptcy", "notBankrupt", "bankruptcyFlg"], + "declaration": ["declaration", "hasDeclaration", "declarationFlg"], +} + +_UPSERT_CHECKS_SQL = text( + """ + INSERT INTO domrf_obj_checks (obj_id, check_type, passed, checked_at, scraped_at) + VALUES (:obj_id, :check_type, :passed, NOW(), NOW()) + ON CONFLICT (obj_id, check_type) DO UPDATE SET + passed = EXCLUDED.passed, + checked_at = NOW(), + scraped_at = NOW() + """ +) + + +def extract_obj_checks(raw_payload: Any) -> list[dict[str, Any]]: + """Извлечь 6 чекбоксов из payload endpoint /object/{obj_id}/checks. + + Поддерживает два варианта payload: + 1. dict с полями (ожидаемый API-формат): {"data": {"noProblemObjects": true, ...}} + 2. list объектов: [{"checkType": "no_problems", "passed": true}, ...] + + Для неизвестных полей и неподдерживаемых форматов — WARNING + пустой список. + """ + if not raw_payload: + return [] + + # Вариант 1: dict с data-обёрткой + data: Any = raw_payload + if isinstance(raw_payload, dict): + data = raw_payload.get("data") or raw_payload + + results: list[dict[str, Any]] = [] + + if isinstance(data, dict): + # Map известных полей к canonical check_type + found: dict[str, bool] = {} + for field, value in data.items(): + ct = _CHECK_FIELD_MAP.get(field) + if ct and ct not in found: + found[ct] = bool(value) + # Также проверить canonical names напрямую + for ct in CHECK_TYPES: + if ct not in found and ct in data: + found[ct] = bool(data[ct]) + if found: + for ct in CHECK_TYPES: + results.append({"check_type": ct, "passed": found.get(ct, False)}) + return results + # dict не содержит известных полей — попробуем как list-формат ниже + logger.warning( + "domrf obj_checks: dict payload has no known check fields: %s", list(data)[:10] + ) + return [] + + if isinstance(data, list): + found_list: dict[str, bool] = {} + for item in data: + if not isinstance(item, dict): + continue + ct_raw = item.get("checkType") or item.get("check_type") or item.get("type") + if ct_raw and str(ct_raw) in CHECK_TYPES: + passed_raw = item.get("passed") or item.get("value") or item.get("status") + found_list[str(ct_raw)] = bool(passed_raw) + if found_list: + for ct in CHECK_TYPES: + results.append({"check_type": ct, "passed": found_list.get(ct, False)}) + return results + logger.warning( + "domrf obj_checks: list payload has no recognisable check items: %s", data[:3] + ) + return [] + + logger.warning("domrf obj_checks: unexpected payload type %s", type(raw_payload)) + return [] + + +def upsert_obj_checks(db: Session, obj_id: int, checks: list[dict[str, Any]]) -> int: + """UPSERT 6 чек-строк в domrf_obj_checks. Returns count of inserted/updated rows. + + Использует SAVEPOINT (begin_nested) per-row — одна битая строка не откатывает + всю транзакцию. + """ + if not checks: + return 0 + ok = 0 + for c in checks: + try: + with db.begin_nested(): + db.execute( + _UPSERT_CHECKS_SQL, + { + "obj_id": obj_id, + "check_type": c["check_type"], + "passed": c["passed"], + }, + ) + ok += 1 + except Exception as exc: + logger.warning( + "upsert obj_checks obj=%s check_type=%s failed: %s", + obj_id, + c.get("check_type"), + exc, + ) + db.commit() + return ok diff --git a/backend/app/services/scrapers/objective.py b/backend/app/services/scrapers/objective.py index 5977f620..1f191fea 100644 --- a/backend/app/services/scrapers/objective.py +++ b/backend/app/services/scrapers/objective.py @@ -26,6 +26,8 @@ from __future__ import annotations import json import logging import time +from collections.abc import Iterator +from contextlib import contextmanager from datetime import date, datetime from typing import Any @@ -238,6 +240,148 @@ class ObjectiveClient: f"Невалидный JSON: {e}; первые 200 символов: {r.text[:200]}" ) from e + def _open_stream( + self, + path: str, + params: dict[str, Any], + *, + attempt: int = 0, + force_refresh_token: bool = False, + ) -> httpx.Response: + """Открывает streaming GET, возвращает httpx.Response (не читает тело). + + Включает retry/auth логику идентичную _request_authed(). + Caller обязан держать контекст через `with self._client.stream(...)` — + вызов делается внутри stream_report() который гарантирует закрытие. + """ + token = self._get_token(force_refresh=force_refresh_token) + url = f"{self.base_url}{path}" + self._wait_rate_limit() + try: + r = self._client.send( + self._client.build_request( + "GET", + url, + params=params, + headers={"Authorization": f"Bearer {token}"}, + ), + stream=True, + ) + except httpx.HTTPError as e: + if attempt < self.retries: + wait = min(2**attempt, 30) + logger.warning( + "Objective stream net-error %s (attempt %d/%d), retry in %ds", + e, + attempt + 1, + self.retries, + wait, + ) + time.sleep(wait) + return self._open_stream(path, params, attempt=attempt + 1) + raise ObjectiveAPIError( + f"Сетевая ошибка stream после {self.retries} попыток: {e}" + ) from e + + if r.status_code == 401: + r.close() + if not force_refresh_token: + logger.info("Objective stream: 401, рефрешим токен и повторяем") + return self._open_stream(path, params, attempt=attempt, force_refresh_token=True) + raise ObjectiveAuthError("401 даже после refresh токена — apiKey невалиден") + + if r.status_code in (429, 502, 503, 504): + retry_after_hdr = r.headers.get("Retry-After") + r.close() + if attempt < self.retries: + if retry_after_hdr and retry_after_hdr.isdigit(): + wait = min(int(retry_after_hdr), 300) + else: + wait = min(30 * (2**attempt), 300) + logger.warning( + "Objective stream HTTP %s (attempt %d/%d), retry in %ds", + r.status_code, + attempt + 1, + self.retries, + wait, + ) + time.sleep(wait) + return self._open_stream(path, params, attempt=attempt + 1) + raise ObjectiveAPIError(f"Stream HTTP {r.status_code} после ретраев") + + if r.status_code != 200: + body_preview = r.read()[:200] + r.close() + raise ObjectiveAPIError(f"Stream HTTP {r.status_code}: {body_preview!r}") + + return r + + @contextmanager + def stream_report( + self, + *, + report_section: str = "Объединенные данные", + report_type: str = "Поквартирные", + report_name: str = "Лоты", + group_name: str | None = None, + complex_name: str | None = None, + start_date: date | str | None = None, + end_date: date | str | None = None, + use_ddu: bool | None = True, + use_dkp: bool | None = None, + page: str = "Отчеты", + v2: bool = True, + ) -> Iterator[httpx.Response]: + """Контекст-менеджер для streaming GetReport. + + Usage: + with client.stream_report(report_type='Поквартирные', report_name='Лоты', + group_name='Свердловская область') as resp: + for chunk in resp.iter_bytes(65536): + ... + + Caller итерирует тело через resp.iter_bytes() или resp.iter_raw(). + Метод НЕ вызывает .json() — всё чтение на стороне caller'а. + """ + path = "/v2/Report/GetReport" if v2 else "/Report/GetReport" + params: dict[str, Any] = { + "Page": page, + "ReportSection": report_section, + "ReportType": report_type, + "ReportName": report_name, + "GroupName": group_name or settings.objective_default_group, + } + if complex_name: + params["ComplexName"] = complex_name + if start_date is not None: + params["StartDate"] = self._fmt_date(start_date) + if end_date is not None: + params["EndDate"] = self._fmt_date(end_date) + if v2: + if use_ddu is not None: + params["UseDdu"] = "true" if use_ddu else "false" + if use_dkp is not None: + params["UseDkp"] = "true" if use_dkp else "false" + + logger.info( + "Objective.stream_report: %s/%s/%s gr=%s cn=%s ddu=%s dkp=%s", + report_section, + report_type, + report_name, + params["GroupName"], + complex_name, + use_ddu, + use_dkp, + ) + resp = self._open_stream(path, params) + try: + yield resp + finally: + try: + resp.close() + except Exception: + pass + # ── публичные методы отчётов ──────────────────────────────────────────── @staticmethod diff --git a/backend/app/services/scrapers/stealth.py b/backend/app/services/scrapers/stealth.py index d14e1694..130e0903 100644 --- a/backend/app/services/scrapers/stealth.py +++ b/backend/app/services/scrapers/stealth.py @@ -104,7 +104,7 @@ class BrowserSession: self._browser: Browser | None = None self._context: BrowserContext | None = None self._page: Page | None = None - self._sem = asyncio.Semaphore(3) + self._sem = asyncio.Semaphore(8) self._request_count = 0 async def __aenter__(self) -> BrowserSession: diff --git a/backend/app/services/site_finder/best_layouts.py b/backend/app/services/site_finder/best_layouts.py new file mode 100644 index 00000000..14faf117 --- /dev/null +++ b/backend/app/services/site_finder/best_layouts.py @@ -0,0 +1,734 @@ +"""Анализ лучших планировок конкурентов по velocity (Issue #113 Phase 2.1). + +Источники: + cad_parcels_geom / cad_quarters_geom — центроид участка + domrf_kn_objects — ЖК в радиусе (latitude/longitude → geography) + objective_corpus_room_month — ежемесячные сделки по (project_name, room_bucket) + objective_complex_mapping — domrf_obj_id ↔ objective_complex_name + domrf_kn_flats — supply count по (room_bucket, area_bin) + +Алгоритм: + Step 1: центроид участка (cad_parcels_geom → cad_quarters_geom fallback). + Step 2: obj_id конкурентов в радиусе (domrf_kn_objects + фильтры). + Step 3: inline SQL из objective_corpus_room_month с честным WHERE report_month фильтром. + Step 4: velocity_per_month = deals_window / months_in_window (честный time_window). + Step 5: supply side из domrf_kn_flats — один батч-запрос. + Step 6: per-row signature + sold_pct. + Step 7: фильтр min_velocity + sort + rank. + Step 8: build recommendation_for_tz (unit-mix, price, rationale). + Step 9: data_quality (coverage + confidence). + +Fix SF-01: раньше mv_layout_velocity (24 мес) делился на divisor (4/12) — данные +не менялись при смене time_window. Теперь inline SQL с реальным фильтром report_month. +""" + +from __future__ import annotations + +import datetime as dt +import logging +from typing import Any + +from sqlalchemy import text +from sqlalchemy.orm import Session + +from app.schemas.parcel import ( + BestLayoutsRequest, + BestLayoutsResponse, + LayoutDataQuality, + LayoutTzMixRow, + LayoutTzRecommendation, + TopLayoutRow, +) +from app.services.site_finder.layout_signature import area_bin, layout_signature + +logger = logging.getLogger(__name__) + +# Confidence thresholds (per coverage % of objects with MV velocity data) +# Tune via PR if business feedback требует. +LAYOUT_CONFIDENCE_HIGH_PCT = 50.0 +LAYOUT_CONFIDENCE_MEDIUM_PCT = 20.0 + +# Fix SF-09: cap доминирующего bucket чтобы рекомендация не зеркалила перекос рынка. +# Избыток перераспределяется пропорционально остальным bucket'ам. +MAX_BUCKET_SHARE_PCT = 35 + +# Параметры time_window: (PostgreSQL interval string, months divisor для velocity_per_month). +# Используются в _INLINE_VELOCITY_SQL — реальный фильтр по report_month. +# Fix SF-01: убраны _VELOCITY_DIVISORS, которые делили MV (24 мес) без изменения данных. +_TIME_WINDOW_PARAMS: dict[str, tuple[str, float]] = { + "last_month": ("1 month", 1.0), + "last_quarter": ("3 months", 3.0), + "last_year": ("12 months", 12.0), +} + +# ── SQL: центроид участка ───────────────────────────────────────────────────── + +_PARCEL_CENTROID_SQL = text(""" + SELECT ST_X(pt) AS center_lon, + ST_Y(pt) AS center_lat + FROM ( + SELECT ST_Centroid(geom) AS pt + FROM cad_parcels_geom + WHERE cad_num = :cad_num AND geom IS NOT NULL + UNION ALL + SELECT ST_Centroid(geom) AS pt + FROM cad_quarters_geom + WHERE cad_number = :quarter AND geom IS NOT NULL + ) sub + LIMIT 1 +""") + +# ── SQL: obj_id конкурентов в радиусе ───────────────────────────────────────── +# Геометрия domrf_kn_objects вычисляется on-the-fly из (latitude, longitude) +# как ST_SetSRID(ST_MakePoint(longitude, latitude), 4326)::geography +# (consistency с competitors.py). +# obj_class_filter: NULL = все классы. +# filter_competitor_obj_ids: NULL = не фильтровать по списку. + +_COMPETITORS_IN_RADIUS_SQL = text(""" + SELECT DISTINCT ON (obj_id) obj_id + FROM domrf_kn_objects + WHERE latitude IS NOT NULL AND longitude IS NOT NULL + AND ST_DWithin( + ST_SetSRID(ST_MakePoint(longitude, latitude), 4326)::geography, + ST_SetSRID( + ST_MakePoint(CAST(:center_lon AS float), CAST(:center_lat AS float)), + 4326 + )::geography, + CAST(:radius_m AS float) + ) + AND ( + CAST(:obj_class_filter AS text) IS NULL + OR obj_class = CAST(:obj_class_filter AS text) + ) + ORDER BY obj_id, snapshot_date DESC NULLS LAST +""") + +# ── SQL: inline velocity из objective_corpus_room_month + mapping ───────────── +# Fix SF-01: честный фильтр по report_month вместо деления mv_layout_velocity (24 мес). +# Параметры: +# :window_interval — PostgreSQL interval string ('1 month', '3 months', '12 months') +# :competitor_obj_ids — list[int] obj_id конкурентов в радиусе +# CAST(:window_interval AS interval) — psycopg v3 / SQLAlchemy 2.0 safe (не ::interval). + +_INLINE_VELOCITY_SQL = text(""" + SELECT + CASE + WHEN crm.room_bucket = 'студия' THEN 'studio' + ELSE crm.room_bucket + END AS room_bucket, + SUM(crm.deals_total_count) AS deals_window, + COALESCE( + SUM(crm.deals_total_avg_area_m2 * crm.deals_total_count) + / NULLIF(SUM(crm.deals_total_count), 0), + 0 + )::numeric(10, 2) AS avg_area_m2, + COALESCE( + SUM(crm.deals_total_avg_price_thousand_rub_per_m2 * crm.deals_total_count) + / NULLIF(SUM(crm.deals_total_count), 0), + 0 + )::numeric(12, 2) * 1000.0 AS avg_price_per_m2_rub, + array_agg(DISTINCT cm.domrf_obj_id) AS competitor_obj_ids, + COUNT(DISTINCT cm.domrf_obj_id) AS competitor_count, + MIN(crm.report_month) AS window_start, + MAX(crm.report_month) AS window_end + FROM objective_corpus_room_month crm + JOIN objective_complex_mapping cm + ON cm.objective_complex_name = crm.project_name + WHERE crm.report_month >= (NOW() - CAST(:window_interval AS interval))::date + AND cm.domrf_obj_id = ANY(:competitor_obj_ids) + AND crm.room_bucket IS NOT NULL + GROUP BY + CASE + WHEN crm.room_bucket = 'студия' THEN 'studio' + ELSE crm.room_bucket + END +""") + +# ── SQL: supply по (room_bucket, area_bin) за последний снимок ─────────────── +# Один батч-запрос вместо N — возвращает map (rb, ab) → count. +# room_bucket и area_bin вычисляются в SQL аналогично layout_signature.py. + +_SUPPLY_BATCH_SQL = text(""" + SELECT + CASE + WHEN f.is_studio = TRUE OR f.flat_type = 'Квартира-студия' THEN 'studio' + WHEN f.rooms = 0 THEN 'studio' + -- Fix SF-08: euro-форматы — DOM.РФ маркирует малогабаритные квартиры как 2-комн. + -- rooms=2 + area<35 → euro-1 (студия с отдельной кухней ~26м²) + -- rooms=2 + area<50 → euro-2 (~35-50м², евро-двушка) + WHEN f.rooms = 2 AND f.total_area < 35 THEN 'euro-1' + WHEN f.rooms = 2 AND f.total_area < 50 THEN 'euro-2' + WHEN f.rooms IN (1, 2, 3) THEN f.rooms::text + WHEN f.rooms >= 4 THEN '4+' + ELSE '1' + END AS rb, + CASE + WHEN f.total_area < 25 THEN '<25' + WHEN f.total_area < 40 THEN '25-40' + WHEN f.total_area < 60 THEN '40-60' + WHEN f.total_area < 80 THEN '60-80' + WHEN f.total_area < 100 THEN '80-100' + ELSE '100+' + END AS ab, + COUNT(*) AS units + FROM domrf_kn_flats f + JOIN domrf_kn_objects o ON f.obj_id = o.obj_id + WHERE o.latitude IS NOT NULL AND o.longitude IS NOT NULL + AND ST_DWithin( + ST_SetSRID(ST_MakePoint(o.longitude, o.latitude), 4326)::geography, + ST_SetSRID( + ST_MakePoint(CAST(:center_lon AS float), CAST(:center_lat AS float)), + 4326 + )::geography, + CAST(:radius_m AS float) + ) + AND f.snapshot_date = CAST(:latest_snap AS date) + GROUP BY rb, ab +""") + + +# ── Вспомогательные функции ─────────────────────────────────────────────────── + + +def _quarter_from_cad(cad_num: str) -> str: + """Извлечь кадастровый квартал: '66:41:0303161:123' → '66:41:0303161'.""" + parts = cad_num.split(":") + if len(parts) >= 3: + return ":".join(parts[:3]) + return cad_num + + +def _normalize_pct(buckets: dict[str, float]) -> dict[str, int]: + """Нормировать доли до целых процентов с суммой ровно 100. + + Алгоритм largest-remainder (Hamilton method): + 1. Floor каждого значения. + 2. Остаток 100 − sum_floors распределить в top-bucket по дробной части. + """ + if not buckets: + return {} + + total = sum(buckets.values()) + if total <= 0: + n = len(buckets) + base = 100 // n + result = {k: base for k in buckets} + # распределить остаток + remainder = 100 - base * n + for k in list(buckets.keys())[:remainder]: + result[k] += 1 + return result + + raw = {k: v / total * 100.0 for k, v in buckets.items()} + floors = {k: int(v) for k, v in raw.items()} + remainder = 100 - sum(floors.values()) + # sort by fractional part desc + fracs = sorted(buckets.keys(), key=lambda k: -(raw[k] - floors[k])) + for k in fracs[:remainder]: + floors[k] += 1 + return floors + + +def _cap_and_redistribute(pct_map: dict[str, int]) -> tuple[dict[str, int], bool]: + """Fix SF-09 round 2: capacity-aware redistribute, bounded iterations. + + Round 1 bug: surplus распределялся пропорционально текущему `v` free bucket'а, + что переливало его выше cap — на 2-bucket вход цикл осциллировал бесконечно. + + Round 2 fix: surplus распределяется пропорционально **available capacity** + `(cap - v)` каждого free bucket'а. Тогда free никогда не вылетит выше cap → + цикл сходится за ≤ len(pct_map) итераций. Hard guard `for _ in range(N+1)`. + + Если surplus > total_capacity (геометрически невозможно поместить излишек ниже + cap) — забиваем все free к cap, возвращаем `cap_skipped=True` + warning log. + + Returns: + (result_map, cap_skipped) — cap_skipped=True если cap не удержан + (pathological: всё хочет > cap, или surplus > available capacity). + """ + if not pct_map: + return pct_map, False + + cap = MAX_BUCKET_SHARE_PCT + + # Быстрый path: нет доминирующих + if all(v <= cap for v in pct_map.values()): + return pct_map, False + + work: dict[str, float] = {k: float(v) for k, v in pct_map.items()} + + # Bounded iteration: после k-й итерации число clamped не убывает только если + # surplus > capacity (тогда — pathological). При корректном capacity-aware + # redistribute достаточно ≤ len(pct_map) итераций. + for _ in range(len(pct_map) + 1): + clamped = [k for k, v in work.items() if v > cap] + if not clamped: + break + + free = [k for k, v in work.items() if v < cap] + if not free: + # Все bucket'ы либо >cap либо ровно =cap — некуда переливать. + logger.warning( + "MAX_BUCKET_SHARE cap: нет free bucket'ов (%d total) — cap_skipped", + len(pct_map), + ) + return pct_map, True + + surplus = sum(work[k] - cap for k in clamped) + capacities = {k: cap - work[k] for k in free} + total_capacity = sum(capacities.values()) + + for k in clamped: + work[k] = float(cap) + + if surplus > total_capacity + 1e-9: + # Излишек не помещается ниже cap — pathological. + # Возвращаем оригинал (sum=100 invariant) + флаг для frontend banner. + logger.warning( + "MAX_BUCKET_SHARE cap: surplus %.2f > total_capacity %.2f — cap_skipped", + surplus, + total_capacity, + ) + return pct_map, True + + for k in free: + work[k] += capacities[k] / total_capacity * surplus + else: + # Hard guard: не сошлись за N+1 итераций — bug. Лог + cap_skipped. + logger.error( + "MAX_BUCKET_SHARE cap: не сошлись за %d итераций — algorithm bug", + len(pct_map) + 1, + ) + return pct_map, True + + return _hamilton_round(work), False + + +def _hamilton_round(work: dict[str, float]) -> dict[str, int]: + """Hamilton apportionment: float → integer pct с суммой ровно 100.""" + floors = {k: int(v) for k, v in work.items()} + remainder = 100 - sum(floors.values()) + fracs = sorted(work.keys(), key=lambda k: -(work[k] - floors[k])) + for k in fracs[: max(0, remainder)]: + floors[k] += 1 + return floors + + +# ── Главная функция ─────────────────────────────────────────────────────────── + + +def get_best_layouts( + db: Session, + cad_num: str, + request: BestLayoutsRequest, +) -> BestLayoutsResponse: + """Top layouts (rooms × area_bin) конкурентов с рейтингом по velocity. + + Raises: + ValueError: если центроид участка не найден (caller → HTTP 404). + """ + quarter = _quarter_from_cad(cad_num) + radius_m = request.radius_km * 1000.0 + + # time_window → (interval_str, months divisor) + window_interval, months_in_window = _TIME_WINDOW_PARAMS.get( + request.time_window, ("3 months", 3.0) + ) + + # ── Step 1: центроид участка ───────────────────────────────────────────── + try: + coord_row = ( + db.execute( + _PARCEL_CENTROID_SQL, + {"cad_num": cad_num, "quarter": quarter}, + ) + .mappings() + .first() + ) + except Exception: + logger.exception("best_layouts: centroid query failed for cad_num=%s", cad_num) + raise + + if not coord_row: + raise ValueError(f"Геометрия для {cad_num} не найдена") + + center_lon = float(coord_row["center_lon"]) + center_lat = float(coord_row["center_lat"]) + + # ── Step 2: obj_id конкурентов в радиусе ──────────────────────────────── + try: + id_rows = ( + db.execute( + _COMPETITORS_IN_RADIUS_SQL, + { + "center_lon": center_lon, + "center_lat": center_lat, + "radius_m": radius_m, + "obj_class_filter": request.obj_class_filter, + }, + ) + .mappings() + .all() + ) + except Exception: + logger.exception("best_layouts: competitors-in-radius query failed for cad_num=%s", cad_num) + raise + + all_obj_ids: list[int] = [int(r["obj_id"]) for r in id_rows] + objects_total_in_radius = len(all_obj_ids) + + # Применить exclude / filter из request + exclude_set = set(request.exclude_competitor_obj_ids) + if exclude_set: + all_obj_ids = [oid for oid in all_obj_ids if oid not in exclude_set] + + if request.filter_competitor_obj_ids is not None: + filter_set = set(request.filter_competitor_obj_ids) + all_obj_ids = [oid for oid in all_obj_ids if oid in filter_set] + + if not all_obj_ids: + return _empty_response( + radius_km=request.radius_km, + time_window=request.time_window, + objects_total_in_radius=objects_total_in_radius, + ) + + # ── Step 3: inline velocity из objective_corpus_room_month ────────────── + # Fix SF-01: честный фильтр report_month >= NOW() - window_interval. + # Разные time_window → разные deals_window, разный mix. + try: + vel_rows = ( + db.execute( + _INLINE_VELOCITY_SQL, + { + "window_interval": window_interval, + "competitor_obj_ids": all_obj_ids, + }, + ) + .mappings() + .all() + ) + except Exception: + logger.exception( + "best_layouts: inline velocity query failed for cad_num=%s obj_count=%d", + cad_num, + len(all_obj_ids), + ) + raise + + if not vel_rows: + return _empty_response( + radius_km=request.radius_km, + time_window=request.time_window, + objects_total_in_radius=objects_total_in_radius, + ) + + # ── Step 5: supply side (батч-запрос) ──────────────────────────────────── + # Pre-compute последний snapshot_date один раз — избегаем subquery на каждый scan. + latest_snap: dt.date | None = db.scalar(text("SELECT MAX(snapshot_date) FROM domrf_kn_flats")) + if latest_snap is None: + logger.warning("best_layouts: domrf_kn_flats пустой (нет snapshot_date), supply=0 fallback") + supply_rows = [] + else: + try: + supply_rows = ( + db.execute( + _SUPPLY_BATCH_SQL, + { + "center_lon": center_lon, + "center_lat": center_lat, + "radius_m": radius_m, + "latest_snap": latest_snap, + }, + ) + .mappings() + .all() + ) + except Exception: + logger.warning("best_layouts: supply query failed, supply=0 fallback") + supply_rows = [] + + supply_map: dict[tuple[str, str], int] = { + (str(r["rb"]), str(r["ab"])): int(r["units"]) for r in supply_rows + } + + # ── Step 4 + 6: velocity из реального окна и enrichment per row ───────── + # Fix SF-01: velocity_per_month = deals_window / months_in_window. + # deals_window уже отфильтрован по report_month — разные time_window дают разные данные. + + enriched: list[dict[str, Any]] = [] + window_start: dt.date | None = None + window_end: dt.date | None = None + + # Собираем obj_ids с данными в objective_corpus_room_month (для data_quality) + obj_ids_with_data: set[int] = set() + + for r in vel_rows: + room_bucket = str(r["room_bucket"]) + deals_window = float(r["deals_window"]) if r["deals_window"] is not None else 0.0 + avg_area = float(r["avg_area_m2"]) if r["avg_area_m2"] is not None else 0.0 + price_rub = ( + float(r["avg_price_per_m2_rub"]) if r["avg_price_per_m2_rub"] is not None else None + ) + competitor_obj_ids: list[int] = ( + [int(oid) for oid in r["competitor_obj_ids"]] if r["competitor_obj_ids"] else [] + ) + competitor_count = int(r["competitor_count"]) + + obj_ids_with_data.update(competitor_obj_ids) + + # Step 4: честный velocity = сделки за окно / длина окна в месяцах + velocity_per_month = round(deals_window / months_in_window, 2) + + # Step 6: area_bin по avg_area (layout_signature.area_bin) + ab = area_bin(avg_area) if avg_area > 0 else "<25" + sig = layout_signature(room_bucket, ab) # type: ignore[arg-type] + + supply_count = supply_map.get((room_bucket, ab), 0) + sold_pct: float | None = None + is_oversold = False + if supply_count > 0: + sold_pct_raw = deals_window / supply_count * 100.0 + is_oversold = sold_pct_raw > 100.0 + # Clamp at 100%: сделки за 24 мес / текущий snapshot supply несопоставимы. + # Значения >100% артефакт окна, не реальная «распроданность». + sold_pct = round(min(sold_pct_raw, 100.0), 1) + + # data window + if r["window_start"] is not None: + ws = r["window_start"] + if isinstance(ws, str): + ws = dt.date.fromisoformat(ws) + elif isinstance(ws, dt.datetime): + ws = ws.date() + window_start = ws if window_start is None else min(window_start, ws) + + if r["window_end"] is not None: + we = r["window_end"] + if isinstance(we, str): + we = dt.date.fromisoformat(we) + elif isinstance(we, dt.datetime): + we = we.date() + window_end = we if window_end is None else max(window_end, we) + + enriched.append( + { + "room_bucket": room_bucket, + "area_bin": ab, + "signature": sig, + "competitor_obj_ids": competitor_obj_ids, + "competitor_count": competitor_count, + "sum_deals": deals_window, + "velocity_per_month": velocity_per_month, + "avg_price_per_m2_rub": price_rub, + "avg_area_m2": avg_area, + "supply_units_in_radius": supply_count, + "sold_pct_of_supply": sold_pct, + "is_oversold": is_oversold, + } + ) + + # ── Step 7: фильтр min_velocity + sort + rank ──────────────────────────── + filtered = [ + row for row in enriched if row["velocity_per_month"] >= request.min_velocity_per_month + ] + filtered.sort(key=lambda r: r["velocity_per_month"], reverse=True) + + top_layouts: list[TopLayoutRow] = [] + for rank_idx, row in enumerate(filtered, start=1): + top_layouts.append( + TopLayoutRow( + rank=rank_idx, + room_bucket=row["room_bucket"], + area_bin=row["area_bin"], + signature=row["signature"], + competitor_obj_ids=row["competitor_obj_ids"], + competitor_count=row["competitor_count"], + total_sold_in_window=int(row["sum_deals"]), + velocity_per_month=row["velocity_per_month"], + avg_price_per_m2_rub=row["avg_price_per_m2_rub"], + avg_area_m2=round(row["avg_area_m2"], 1), + supply_units_in_radius=row["supply_units_in_radius"], + sold_pct_of_supply=row["sold_pct_of_supply"], + is_oversold=row["is_oversold"], + ) + ) + + # ── Step 8: build recommendation_for_tz ───────────────────────────────── + # Используем filtered (только > min_velocity) для recommendation. + # Если после фильтрации всё пустое — используем enriched (все данные без фильтра). + rec_source = filtered if filtered else enriched + + today = dt.date.today() + ws_date = window_start if window_start is not None else today + we_date = window_end if window_end is not None else today + + recommendation = _build_recommendation( + rows=rec_source, + radius_km=request.radius_km, + time_window=request.time_window, + target_total_flats=request.target_total_flats, + window_start=ws_date, + window_end=we_date, + all_enriched=enriched, + ) + + # ── Step 9: data_quality ───────────────────────────────────────────────── + # Denominator = post-filter set (effective consideration set после exclude/filter). + objects_total_after_filter = len(all_obj_ids) + objects_with_data = len(obj_ids_with_data & set(all_obj_ids)) + coverage_pct = ( + round(objects_with_data / objects_total_after_filter * 100.0, 1) + if objects_total_after_filter > 0 + else 0.0 + ) + if coverage_pct >= LAYOUT_CONFIDENCE_HIGH_PCT: + confidence: str = "high" + elif coverage_pct >= LAYOUT_CONFIDENCE_MEDIUM_PCT: + confidence = "medium" + else: + confidence = "low" + + data_quality = LayoutDataQuality( + objects_with_velocity_data=objects_with_data, + objects_total_in_radius=objects_total_after_filter, + velocity_coverage_pct=coverage_pct, + confidence=confidence, # type: ignore[arg-type] + ) + + return BestLayoutsResponse( + top_layouts=top_layouts, + recommendation_for_tz=recommendation, + data_quality=data_quality, + ) + + +def _build_recommendation( + rows: list[dict[str, Any]], + radius_km: float, + time_window: str, + target_total_flats: int | None, + window_start: dt.date, + window_end: dt.date, + all_enriched: list[dict[str, Any]], +) -> LayoutTzRecommendation: + """Собрать LayoutTzRecommendation из enriched rows.""" + if not rows: + return LayoutTzRecommendation( + rationale_text=( + f"В радиусе {radius_km}км: нет layout-паттернов с достаточной velocity." + ), + mix=[], + weighted_avg_price_per_m2_rub=None, + based_on_obj_count=0, + based_on_total_deals=0, + data_window_start=window_start, + data_window_end=window_end, + ) + + # Группировка по room_bucket (строки уже могут быть per-bucket из MV GROUP BY) + rb_deals: dict[str, float] = {} + rb_area_weighted: dict[str, float] = {} + rb_price_weighted: dict[str, float] = {} + rb_price_total_deals: dict[str, float] = {} + all_competitor_ids: set[int] = set() + + for row in rows: + rb = row["room_bucket"] + sd = float(row["sum_deals"]) + rb_deals[rb] = rb_deals.get(rb, 0.0) + sd + rb_area_weighted[rb] = rb_area_weighted.get(rb, 0.0) + row["avg_area_m2"] * sd + all_competitor_ids.update(row["competitor_obj_ids"]) + if row["avg_price_per_m2_rub"] is not None: + rb_price_weighted[rb] = rb_price_weighted.get(rb, 0.0) + ( + row["avg_price_per_m2_rub"] * sd + ) + rb_price_total_deals[rb] = rb_price_total_deals.get(rb, 0.0) + sd + + total_deals = sum(rb_deals.values()) + pct_map = _normalize_pct(rb_deals) + pct_map, cap_skipped = _cap_and_redistribute(pct_map) + + mix: list[LayoutTzMixRow] = [] + for rb, pct in sorted(pct_map.items(), key=lambda x: -x[1]): + avg_area = ( + round(rb_area_weighted[rb] / rb_deals[rb], 1) if rb_deals.get(rb, 0) > 0 else None + ) + abs_units: int | None = None + if target_total_flats is not None: + abs_units = round(pct / 100.0 * target_total_flats) + mix.append( + LayoutTzMixRow( + room_bucket=rb, + pct=pct, + abs_units=abs_units, + avg_target_area_m2=avg_area, + ) + ) + + # Weighted avg price across all room_buckets + total_price_deals = sum(rb_price_total_deals.values()) + weighted_price: float | None = None + if total_price_deals > 0: + weighted_price = round(sum(rb_price_weighted.values()) / total_price_deals, 0) + + # Rationale + competitor_count = len(all_competitor_ids) + tw_label = {"last_month": "1 мес", "last_quarter": "квартал", "last_year": "год"}.get( + time_window, time_window + ) + rationale_text = ( + f"В радиусе {radius_km}км за {tw_label}: " + f"{len(rows)} активных layout-паттернов, " + f"total {int(total_deals)} продаж в {competitor_count} ЖК" + ) + + # based_on_obj_count из all_enriched (уникальные obj_id с данными MV) + all_mv_obj_ids: set[int] = set() + for row in all_enriched: + all_mv_obj_ids.update(row["competitor_obj_ids"]) + + return LayoutTzRecommendation( + rationale_text=rationale_text, + mix=mix, + weighted_avg_price_per_m2_rub=weighted_price, + based_on_obj_count=len(all_mv_obj_ids), + based_on_total_deals=int(total_deals), + data_window_start=window_start, + data_window_end=window_end, + cap_skipped=cap_skipped, + ) + + +def _empty_response( + radius_km: float, + time_window: str, + objects_total_in_radius: int, +) -> BestLayoutsResponse: + """Ответ когда нет конкурентов или нет MV данных.""" + today = dt.date.today() + tw_label = {"last_month": "1 мес", "last_quarter": "квартал", "last_year": "год"}.get( + time_window, time_window + ) + return BestLayoutsResponse( + top_layouts=[], + recommendation_for_tz=LayoutTzRecommendation( + rationale_text=( + f"В радиусе {radius_km}км за {tw_label}: " + f"конкуренты не найдены или нет данных velocity." + ), + mix=[], + weighted_avg_price_per_m2_rub=None, + based_on_obj_count=0, + based_on_total_deals=0, + data_window_start=today, + data_window_end=today, + ), + data_quality=LayoutDataQuality( + objects_with_velocity_data=0, + objects_total_in_radius=objects_total_in_radius, + velocity_coverage_pct=0.0, + confidence="low", + ), + ) diff --git a/backend/app/services/site_finder/competitors.py b/backend/app/services/site_finder/competitors.py new file mode 100644 index 00000000..c59889d4 --- /dev/null +++ b/backend/app/services/site_finder/competitors.py @@ -0,0 +1,305 @@ +"""Анализ активных конкурентов ЖК в радиусе от участка. + +Issue #112 — Demand: активные конкуренты, продажи ЖК в радиусе 1км за квартал. + +Источники: + domrf_kn_objects — ЖК с lat/lon, flat_count, obj_class, site_status + objective_complex_mapping — domrf_obj_id → objective_complex_name + objective_corpus_room_month — monthly deals_total_count per project_name + cad_parcels_geom — centroid участка (fallback: cad_quarters_geom) + domrf_kn_flats — avg price_per_m2 по проданным квартирам + +Внимание: velocity coverage ~2.5% — большинство ЖК не имеют маппинга в +objective_complex_mapping. LEFT JOIN гарантирует velocity=0 (не ошибку) для +немаппированных объектов. +""" + +from __future__ import annotations + +import logging + +from sqlalchemy import text +from sqlalchemy.orm import Session + +from app.schemas.parcel import ( + Competitor, + CompetitorsRequest, + CompetitorsResponse, + CompetitorsSummary, +) + +logger = logging.getLogger(__name__) + +# Маппинг time_window → число месяцев (float для деления velocity) +_TIME_WINDOW_MONTHS: dict[str, float] = { + "last_month": 1.0, + "last_quarter": 3.0, + "last_year": 12.0, +} + +# site_status значения, считающиеся «активными» +_ACTIVE_STATUSES = frozenset({"sales", "construction"}) + +# SQL для получения центроида участка +_PARCEL_CENTROID_SQL = text(""" + SELECT ST_X(pt) AS lon, ST_Y(pt) AS lat + FROM ( + SELECT ST_Centroid(geom) AS pt + FROM cad_parcels_geom + WHERE cad_num = :cad_num AND geom IS NOT NULL + UNION ALL + SELECT ST_Centroid(geom) AS pt + FROM cad_quarters_geom + WHERE cad_number = :quarter AND geom IS NOT NULL + ) sub + LIMIT 1 +""") + +# Основной запрос конкурентов в радиусе. +# Velocity через objective_corpus_room_month (актуальные данные, обновляется еженедельно). +# domrf_kn_sale_graph устарел (данные до 2026-01) — не используется. +# Coverage velocity ~2.5%: большинство obj_id нет в objective_complex_mapping → +# LEFT JOIN → velocity=0 (не ошибка). +_COMPETITORS_SQL = text(""" + WITH latest_obj AS ( + SELECT DISTINCT ON (obj_id) + obj_id, + comm_name, + dev_name, + obj_class, + latitude, + longitude, + flat_count, + site_status, + snapshot_date + FROM domrf_kn_objects + WHERE latitude IS NOT NULL + AND longitude IS NOT NULL + ORDER BY obj_id, snapshot_date DESC NULLS LAST + ), + mapped AS ( + SELECT cm.domrf_obj_id AS obj_id, + cm.objective_complex_name + FROM objective_complex_mapping cm + ), + velocity AS ( + SELECT + m.obj_id, + SUM(COALESCE(crm.deals_total_count, 0)) + / CAST(:time_window_months AS float) AS velocity_per_month + FROM objective_corpus_room_month crm + JOIN mapped m ON m.objective_complex_name = crm.project_name + WHERE crm.report_month >= (NOW() - CAST(:window_interval AS interval)) + GROUP BY m.obj_id + ), + distances AS ( + SELECT + o.obj_id, + o.comm_name, + o.dev_name, + o.obj_class, + o.latitude, + o.longitude, + o.flat_count, + o.site_status, + ST_Distance( + ST_SetSRID(ST_MakePoint(o.longitude, o.latitude), 4326)::geography, + ST_SetSRID( + ST_MakePoint(CAST(:center_lon AS float), CAST(:center_lat AS float)), + 4326 + )::geography + ) AS distance_m + FROM latest_obj o + ) + SELECT + d.obj_id, + d.comm_name, + d.dev_name, + d.obj_class, + d.latitude, + d.longitude, + d.flat_count, + d.site_status, + d.distance_m, + COALESCE(v.velocity_per_month, 0.0) AS velocity_per_month + FROM distances d + LEFT JOIN velocity v ON v.obj_id = d.obj_id + WHERE d.distance_m <= CAST(:radius_m AS float) + AND ( + CAST(:obj_class_filter AS text) IS NULL + OR d.obj_class = CAST(:obj_class_filter AS text) + ) + ORDER BY d.distance_m ASC +""") + +# Средняя цена м² по квартирам с известной ценой для набора obj_id. +# Фильтр status='sold' убран: поле status в domrf_kn_flats заполнено в ~0.2% строк +# (99.8% NULL) — фильтр давал 0 строк и avg_price_per_m2 всегда None (Issue #112/227). +# AVG по всем квартирам с price_per_m2 IS NOT NULL даёт корректную среднюю цену ЖК. +_AVG_PRICE_SQL = text(""" + SELECT + f.obj_id, + AVG(f.price_per_m2) AS avg_price_per_m2 + FROM domrf_kn_flats f + WHERE f.obj_id = ANY(:obj_ids) + AND f.price_per_m2 IS NOT NULL + GROUP BY f.obj_id +""") + + +def _quarter_from_cad(cad_num: str) -> str: + """Извлечь кадастровый квартал из номера участка/здания. + + 66:41:0303161:123 → 66:41:0303161 + Если формат нестандартный — возвращаем cad_num как есть (fallback). + """ + parts = cad_num.split(":") + if len(parts) >= 3: + return ":".join(parts[:3]) + return cad_num + + +def get_competitors( + db: Session, + cad_num: str, + request: CompetitorsRequest, +) -> CompetitorsResponse: + """Получить список конкурентов ЖК в радиусе от участка. + + Шаги: + 1. Найти центроид участка (cad_parcels_geom → cad_quarters_geom fallback). + 2. Выбрать ЖК из domrf_kn_objects в радиусе с velocity из objective_corpus_room_month. + 3. Применить exclude_obj_ids фильтр в Python (избегаем array cast). + 4. Подтянуть avg_price_per_m2 из domrf_kn_flats. + 5. Собрать CompetitorsResponse. + + Raises: + ValueError: если центроид участка не найден (caller должен вернуть 404). + """ + quarter = _quarter_from_cad(cad_num) + + # ── 1. Центроид участка ────────────────────────────────────────────────── + try: + coord_row = ( + db.execute( + _PARCEL_CENTROID_SQL, + {"cad_num": cad_num, "quarter": quarter}, + ) + .mappings() + .first() + ) + except Exception: + logger.exception("competitors: centroid query failed for cad_num=%s", cad_num) + raise + + if not coord_row: + raise ValueError(f"Геометрия для {cad_num} не найдена") + + center_lat = float(coord_row["lat"]) + center_lon = float(coord_row["lon"]) + + # ── 2. Конкуренты в радиусе ────────────────────────────────────────────── + time_window_months = _TIME_WINDOW_MONTHS[request.time_window] + window_interval = f"{int(time_window_months)} months" + + try: + rows = ( + db.execute( + _COMPETITORS_SQL, + { + "center_lat": center_lat, + "center_lon": center_lon, + "radius_m": request.radius_km * 1000.0, + "time_window_months": time_window_months, + "window_interval": window_interval, + "obj_class_filter": request.obj_class_filter, + }, + ) + .mappings() + .all() + ) + except Exception: + logger.exception( + "competitors: main query failed for cad_num=%s radius_km=%.2f", + cad_num, + request.radius_km, + ) + raise + + # ── 3. Применить exclude_obj_ids ───────────────────────────────────────── + exclude_set = set(request.exclude_obj_ids) + if exclude_set: + rows = [r for r in rows if int(r["obj_id"]) not in exclude_set] + + if not rows: + return CompetitorsResponse( + competitors=[], + summary=CompetitorsSummary( + total_competitors=0, + active_count=0, + weighted_avg_velocity=0.0, + radius_km=request.radius_km, + time_window=request.time_window, + ), + ) + + obj_ids: list[int] = [int(r["obj_id"]) for r in rows] + + # ── 4. Средняя цена м² (graceful — таблица может быть не заполнена) ────── + avg_price_map: dict[int, float] = {} + try: + price_rows = db.execute(_AVG_PRICE_SQL, {"obj_ids": obj_ids}).mappings().all() + avg_price_map = { + int(r["obj_id"]): float(r["avg_price_per_m2"]) + for r in price_rows + if r["avg_price_per_m2"] is not None + } + except Exception: + logger.warning("competitors: avg_price query failed, continuing without prices") + + # ── 5. Сборка результата ───────────────────────────────────────────────── + # flats_sold / sold_pct: не доступны из domrf_kn_objects (только flat_count). + # Можно получить через COUNT(domrf_kn_flats WHERE status='sold') — + # отложено за MVP, поля остаются None. + competitors: list[Competitor] = [] + for r in rows: + obj_id = int(r["obj_id"]) + flats_total = int(r["flat_count"]) if r["flat_count"] is not None else None + + site_status = r["site_status"] + is_active = site_status in _ACTIVE_STATUSES if site_status else False + + competitors.append( + Competitor( + obj_id=obj_id, + comm_name=r["comm_name"], + dev_name=r["dev_name"], + obj_class=r["obj_class"], + distance_m=round(float(r["distance_m"]), 1), + lat=float(r["latitude"]), + lng=float(r["longitude"]), + stage=site_status, + flats_total=flats_total, + flats_sold=None, + sold_pct=None, + velocity_per_month=round(float(r["velocity_per_month"]), 2), + avg_price_per_m2=avg_price_map.get(obj_id), + is_active=is_active, + ) + ) + + # ── 6. Summary ─────────────────────────────────────────────────────────── + active_count = sum(1 for c in competitors if c.is_active) + total_velocity = sum(c.velocity_per_month for c in competitors) + n = len(competitors) + weighted_avg_velocity = round(total_velocity / n, 2) if n > 0 else 0.0 + + summary = CompetitorsSummary( + total_competitors=n, + active_count=active_count, + weighted_avg_velocity=weighted_avg_velocity, + radius_km=request.radius_km, + time_window=request.time_window, + ) + + return CompetitorsResponse(competitors=competitors, summary=summary) diff --git a/backend/app/services/site_finder/custom_pois.py b/backend/app/services/site_finder/custom_pois.py new file mode 100644 index 00000000..e3dba224 --- /dev/null +++ b/backend/app/services/site_finder/custom_pois.py @@ -0,0 +1,276 @@ +"""CRUD-сервис для user_custom_pois (#254). + +API: +- create_custom_poi(db, user_id, payload) → CustomPoiOut +- list_custom_pois(db, user_id, parcel_cad?) → list[CustomPoiOut] +- get_custom_poi(db, poi_id, user_id) → CustomPoiOut | None +- update_custom_poi(db, poi_id, user_id, payload) → CustomPoiOut | None +- delete_custom_poi(db, poi_id, user_id) → bool +- get_overlaps_for_scoring(db, parcel_geom_wkt, user_id, parcel_cad?) → list[dict] + +Паттерн: raw SQL через SQLAlchemy text() + CAST(:x AS type), psycopg v3. +""" + +from __future__ import annotations + +import logging +from typing import Any + +from sqlalchemy import text + +from app.schemas.custom_poi import CustomPoiCreate, CustomPoiOut, CustomPoiUpdate + +logger = logging.getLogger(__name__) + +# ── SQL ──────────────────────────────────────────────────────────────────────── + +_SELECT_COLS = """ + id, user_id, parcel_cad, name, category, weight, lon, lat, notes, + created_at, updated_at +""" + +_INSERT_SQL = f""" + INSERT INTO user_custom_pois + (user_id, parcel_cad, name, category, weight, lon, lat, notes) + VALUES + (:user_id, :parcel_cad, :name, :category, + CAST(:weight AS real), CAST(:lon AS double precision), + CAST(:lat AS double precision), :notes) + RETURNING {_SELECT_COLS} +""" + +_SELECT_BY_USER_ALL = f""" + SELECT {_SELECT_COLS} + FROM user_custom_pois + WHERE user_id = :user_id + ORDER BY created_at DESC +""" + +_SELECT_BY_USER_PARCEL = f""" + SELECT {_SELECT_COLS} + FROM user_custom_pois + WHERE user_id = :user_id + AND (parcel_cad = :parcel_cad OR parcel_cad IS NULL) + ORDER BY created_at DESC +""" + +_SELECT_BY_ID = f""" + SELECT {_SELECT_COLS} + FROM user_custom_pois + WHERE id = :poi_id + AND user_id = :user_id +""" + +_DELETE_SQL = """ + DELETE FROM user_custom_pois + WHERE id = :poi_id + AND user_id = :user_id + RETURNING id +""" + +# Запрос для scoring: custom POI в радиусе 1 км от центроида участка. +# Возвращает POI (global + parcel-specific) пользователя с расстоянием. +_OVERLAPS_SQL = """ + SELECT p.id, p.name, p.category, p.weight, p.lon, p.lat, + ST_Distance( + p.geom, + ST_Centroid(ST_GeomFromText(CAST(:wkt AS text), 4326))::geography + ) AS distance_m + FROM user_custom_pois p + WHERE p.user_id = :user_id + AND (p.parcel_cad IS NULL OR p.parcel_cad = :parcel_cad) + AND ST_DWithin( + p.geom, + ST_Centroid(ST_GeomFromText(CAST(:wkt AS text), 4326))::geography, + 1000 + ) + ORDER BY distance_m ASC +""" + + +# ── Row mapper ───────────────────────────────────────────────────────────────── + + +def _row_to_out(r: Any) -> CustomPoiOut: + return CustomPoiOut( + id=int(r["id"]), + user_id=r["user_id"], + parcel_cad=r["parcel_cad"], + name=r["name"], + category=r["category"], + weight=float(r["weight"]), + lon=float(r["lon"]), + lat=float(r["lat"]), + notes=r["notes"], + created_at=r["created_at"], + updated_at=r["updated_at"], + ) + + +# ── CRUD ─────────────────────────────────────────────────────────────────────── + + +def create_custom_poi(db: Any, user_id: str, payload: CustomPoiCreate) -> CustomPoiOut: + """Создать кастомную POI точку для пользователя.""" + row = ( + db.execute( + text(_INSERT_SQL), + { + "user_id": user_id, + "parcel_cad": payload.parcel_cad, + "name": payload.name, + "category": payload.category, + "weight": payload.weight, + "lon": payload.lon, + "lat": payload.lat, + "notes": payload.notes, + }, + ) + .mappings() + .first() + ) + db.commit() + assert row is not None, "INSERT RETURNING вернул пустой результат" + logger.info( + "custom_poi created: id=%s user=%s parcel=%s weight=%s", + row["id"], + user_id, + payload.parcel_cad, + payload.weight, + ) + return _row_to_out(row) + + +def list_custom_pois(db: Any, user_id: str, parcel_cad: str | None = None) -> list[CustomPoiOut]: + """Вернуть custom POI пользователя. + + Если parcel_cad задан — возвращает global (parcel_cad IS NULL) + parcel-specific. + Если parcel_cad=None — возвращает все POI пользователя. + """ + if parcel_cad is not None: + rows = ( + db.execute( + text(_SELECT_BY_USER_PARCEL), + {"user_id": user_id, "parcel_cad": parcel_cad}, + ) + .mappings() + .all() + ) + else: + rows = db.execute(text(_SELECT_BY_USER_ALL), {"user_id": user_id}).mappings().all() + return [_row_to_out(r) for r in rows] + + +def get_custom_poi(db: Any, poi_id: int, user_id: str) -> CustomPoiOut | None: + """Вернуть одну POI по id (scoped к user_id).""" + row = db.execute(text(_SELECT_BY_ID), {"poi_id": poi_id, "user_id": user_id}).mappings().first() + if row is None: + return None + return _row_to_out(row) + + +def update_custom_poi( + db: Any, poi_id: int, user_id: str, payload: CustomPoiUpdate +) -> CustomPoiOut | None: + """PATCH-style обновление кастомной POI. Возвращает None если не найдена.""" + existing = get_custom_poi(db, poi_id, user_id) + if existing is None: + return None + + sets: list[str] = ["updated_at = NOW()"] + params: dict[str, Any] = {"poi_id": poi_id, "user_id": user_id} + + if payload.name is not None: + sets.append("name = :name") + params["name"] = payload.name + + if payload.category is not None: + sets.append("category = :category") + params["category"] = payload.category + + if payload.weight is not None: + sets.append("weight = CAST(:weight AS real)") + params["weight"] = payload.weight + + # lon/lat изменяем только вместе — geom GENERATED ALWAYS пересчитается автоматически + if payload.lon is not None: + sets.append("lon = CAST(:lon AS double precision)") + params["lon"] = payload.lon + + if payload.lat is not None: + sets.append("lat = CAST(:lat AS double precision)") + params["lat"] = payload.lat + + if payload.parcel_cad is not None: + sets.append("parcel_cad = :parcel_cad") + params["parcel_cad"] = payload.parcel_cad + + if payload.notes is not None: + sets.append("notes = :notes") + params["notes"] = payload.notes + + if len(sets) > 1: + db.execute( + text( + f"UPDATE user_custom_pois SET {', '.join(sets)}" + " WHERE id = :poi_id AND user_id = :user_id" + ), + params, + ) + db.commit() + + return get_custom_poi(db, poi_id, user_id) + + +def delete_custom_poi(db: Any, poi_id: int, user_id: str) -> bool: + """Удалить кастомную POI. Возвращает True если удалена, False если не найдена.""" + result = db.execute( + text(_DELETE_SQL), + {"poi_id": poi_id, "user_id": user_id}, + ).first() + if result is None: + return False + db.commit() + logger.info("custom_poi deleted: id=%s user=%s", poi_id, user_id) + return True + + +def get_overlaps_for_scoring( + db: Any, + parcel_geom_wkt: str, + user_id: str, + parcel_cad: str | None = None, +) -> list[dict[str, Any]]: + """Вернуть custom POI в радиусе 1 км с расстоянием для scoring. + + Включает global POI (parcel_cad IS NULL) + parcel-specific если parcel_cad задан. + Distance decay применяется в parcels.py аналогично OSM POI. + + Returns: + list[dict] с ключами: id, name, category, weight, lon, lat, distance_m + """ + _parcel_cad = parcel_cad or "" + try: + rows = ( + db.execute( + text(_OVERLAPS_SQL), + {"wkt": parcel_geom_wkt, "user_id": user_id, "parcel_cad": _parcel_cad}, + ) + .mappings() + .all() + ) + return [ + { + "id": int(r["id"]), + "name": r["name"], + "category": r["category"], + "weight": float(r["weight"]), + "lon": float(r["lon"]), + "lat": float(r["lat"]), + "distance_m": float(r["distance_m"]), + } + for r in rows + ] + except Exception as e: + logger.warning("get_overlaps_for_scoring failed user=%s: %s", user_id, e) + return [] diff --git a/backend/app/services/site_finder/gate_verdict.py b/backend/app/services/site_finder/gate_verdict.py index 209cbeda..8b854cb3 100644 --- a/backend/app/services/site_finder/gate_verdict.py +++ b/backend/app/services/site_finder/gate_verdict.py @@ -18,11 +18,25 @@ RESIDENTIAL_KEYWORDS = ("жил",) # ── ЗОУИТ taxonomy ──────────────────────────────────────────────────────────── -# Subcategory codes которые блокируют МКД (охранные зоны ЛЭП/газа/трубопровода) +# Subcategory codes которые блокируют МКД (охранные зоны ЛЭП/газа/трубопровода). +# Используется для overlaps из NSPD dump (subcategory INT там заполнен). BLOCKER_SUBCATEGORIES: dict[int, str] = { 17: "Инженерные коммуникации (охранная зона ЛЭП/газа/трубопровода)", } +# Keyword-based blockers для cad_zouit fallback (#232). +# cad_zouit.subcategory = 100% NULL, поэтому классификация по type_zone substring. +# Порядок: lowercase, substring match (any wins → blocker). +BLOCKER_TYPE_ZONE_KEYWORDS: tuple[str, ...] = ( + "охранная зона", + "трубопровод", + "электр", + "газ", +) + +# type_zone substrings, которые дают warning (не blocker) из cad_zouit. +WARNING_TYPE_ZONE_KEYWORDS: tuple[str, ...] = ("сзз", "санитарно-защитная") + # Engineering nearby search radius (метры) — совпадает с quarter_dump_lookup.py ENGINEERING_NEARBY_THRESHOLD_M = 200 @@ -136,21 +150,57 @@ def compute_gate_verdict( # Check 2 — ЗОУИТ overlaps checks.append("ЗОУИТ пересечения") for overlap in nspd_zouit_overlaps or []: - sub = overlap.get("subcategory") - if isinstance(sub, int) and sub in BLOCKER_SUBCATEGORIES: - blockers.append( - Blocker( - code=f"ZOUIT_OVERLAP_SUB{sub}", - detail=f"{BLOCKER_SUBCATEGORIES[sub]}: {overlap.get('name', '')}", + src = overlap.get("source", "nspd-quarter-dump") + if src == "cad_zouit": + # cad_zouit fallback path: classify by type_zone keywords (#232). + # subcategory = NULL в cad_zouit, поэтому subcategory-based logic не применяется. + type_zone_lower = (overlap.get("type_zone") or overlap.get("layer") or "").lower() + if any(kw in type_zone_lower for kw in BLOCKER_TYPE_ZONE_KEYWORDS): + blockers.append( + Blocker( + code="ZOUIT_CAD_BLOCKER", + detail=( + f"Охранная зона ({overlap.get('type_zone', '')}): " + f"{overlap.get('name', '')}" + ), + ) + ) + elif any(kw in type_zone_lower for kw in WARNING_TYPE_ZONE_KEYWORDS): + warnings.append( + Warning( + code="ZOUIT_CAD_SZZ", + detail=( + f"СЗЗ ({overlap.get('type_zone', '')}): " f"{overlap.get('name', '')}" + ), + ) + ) + else: + warnings.append( + Warning( + code="ZOUIT_CAD_OTHER", + detail=( + f"ЗОУИТ cad ({overlap.get('type_zone', '')}): " + f"{overlap.get('name', '')}" + ), + ) ) - ) else: - warnings.append( - Warning( - code=f"ZOUIT_SUB{sub if sub is not None else 'unknown'}", - detail=f"ЗОУИТ {overlap.get('layer', '')}: {overlap.get('name', '')}", + # NSPD dump path: subcategory-based logic (backward-compat). + sub = overlap.get("subcategory") + if isinstance(sub, int) and sub in BLOCKER_SUBCATEGORIES: + blockers.append( + Blocker( + code=f"ZOUIT_OVERLAP_SUB{sub}", + detail=f"{BLOCKER_SUBCATEGORIES[sub]}: {overlap.get('name', '')}", + ) + ) + else: + warnings.append( + Warning( + code=f"ZOUIT_SUB{sub if sub is not None else 'unknown'}", + detail=f"ЗОУИТ {overlap.get('layer', '')}: {overlap.get('name', '')}", + ) ) - ) # Check 3 — Engineering nearby (warning only, not a blocker) checks.append(f"Инженерные сети в радиусе {ENGINEERING_NEARBY_THRESHOLD_M}м") diff --git a/backend/app/services/site_finder/layout_signature.py b/backend/app/services/site_finder/layout_signature.py new file mode 100644 index 00000000..da3576d5 --- /dev/null +++ b/backend/app/services/site_finder/layout_signature.py @@ -0,0 +1,66 @@ +"""Layout signature extraction для Issue #113 (Phase 2.1 minimal). + +Без `layout_type` / `balcony_count` — ждут B2B Объектив (#52). +""" + +from __future__ import annotations + +from typing import Literal + +RoomBucket = Literal["studio", "euro-1", "euro-2", "1", "2", "3", "4+"] +AreaBin = Literal["<25", "25-40", "40-60", "60-80", "80-100", "100+"] + + +def room_bucket_from_flat( + rooms: int | None, + flat_type: str | None, + is_studio: bool | None, + total_area: float | None = None, +) -> RoomBucket: + """Determine room_bucket из kn_flats полей. + + Priority: + 1. is_studio=True OR flat_type='Квартира-студия' → "studio" + 2. rooms=0 (без is_studio) → "studio" + 3. Fix SF-08: rooms=2 + area<35 → "euro-1" (DOM.РФ маркирует малогабаритные как 2-комн) + 4. Fix SF-08: rooms=2 + area<50 → "euro-2" (евро-двушки 35-50м²) + 5. rooms IN (1, 2, 3) → str(rooms) + 6. rooms >= 4 → "4+" + 7. fallback на "1" (rooms is None и не studio) + """ + if is_studio is True or flat_type == "Квартира-студия": + return "studio" + if rooms is None: + return "1" + if rooms == 0: + return "studio" + if rooms == 2 and total_area is not None: + if total_area < 35.0: + return "euro-1" + if total_area < 50.0: + return "euro-2" + if rooms >= 4: + return "4+" + if rooms in (1, 2, 3): + return str(rooms) # type: ignore[return-value] + return "1" + + +def area_bin(area_m2: float) -> AreaBin: + """Bucket площади per ARN-buckets (НСПД).""" + if area_m2 < 25.0: + return "<25" + if area_m2 < 40.0: + return "25-40" + if area_m2 < 60.0: + return "40-60" + if area_m2 < 80.0: + return "60-80" + if area_m2 < 100.0: + return "80-100" + return "100+" + + +def layout_signature(room_bucket_val: RoomBucket, area_bin_val: AreaBin) -> str: + """Deterministic string-signature для группировки.""" + return f"{room_bucket_val}__{area_bin_val}" diff --git a/backend/app/services/site_finder/layout_velocity_refresh.py b/backend/app/services/site_finder/layout_velocity_refresh.py new file mode 100644 index 00000000..61075c60 --- /dev/null +++ b/backend/app/services/site_finder/layout_velocity_refresh.py @@ -0,0 +1,57 @@ +"""Refresh helper for mv_layout_velocity (Issue #113 PR B). + +Not scheduled automatically in this PR — intended for manual invocation or +a Celery beat task in a follow-up issue. + +Usage example (manual, via psql-connected shell or admin endpoint): + from sqlalchemy.orm import Session + from app.services.site_finder.layout_velocity_refresh import refresh_layout_velocity + + count = refresh_layout_velocity(db) + # logs: "mv_layout_velocity refreshed: 459 rows" +""" + +from __future__ import annotations + +import logging + +from sqlalchemy import text +from sqlalchemy.exc import OperationalError +from sqlalchemy.orm import Session + +logger = logging.getLogger(__name__) + + +def refresh_layout_velocity(db: Session, *, concurrently: bool = True) -> int: + """REFRESH MATERIALIZED VIEW mv_layout_velocity. + + Args: + db: SQLAlchemy Session (sync). + concurrently: When True, uses REFRESH CONCURRENTLY — non-blocking but + requires the unique index mv_layout_velocity_pk to exist (created + by 94_mv_layout_velocity.sql). Pass False only for the very first + populate or when the MV was just recreated. + + Returns: + Row count in the MV after refresh (for observability / alerting). + """ + try: + if concurrently: + db.execute(text("REFRESH MATERIALIZED VIEW CONCURRENTLY mv_layout_velocity")) + else: + db.execute(text("REFRESH MATERIALIZED VIEW mv_layout_velocity")) + db.commit() + except OperationalError as e: + if concurrently and "cannot refresh materialized view" in str(e).lower(): + logger.warning( + "CONCURRENTLY failed (MV likely not populated), falling back to non-concurrent" + ) + db.rollback() + db.execute(text("REFRESH MATERIALIZED VIEW mv_layout_velocity")) + db.commit() + else: + raise + row = db.execute(text("SELECT COUNT(*) FROM mv_layout_velocity")).first() + count = int(row[0]) if row else 0 + logger.info("mv_layout_velocity refreshed: %d rows", count) + return count diff --git a/backend/app/services/site_finder/poi_score.py b/backend/app/services/site_finder/poi_score.py new file mode 100644 index 00000000..31a7032b --- /dev/null +++ b/backend/app/services/site_finder/poi_score.py @@ -0,0 +1,159 @@ +"""POI weighted score для Site Finder (B6). + +Формула: weight = (1 / (distance_m + 100)) * category_weight + +Возвращает top-7 ближайших POI из osm_poi_ekb, отсортированных по weight DESC. +Категории и их веса согласованы с _POI_WEIGHTS в parcels.py. +""" + +from __future__ import annotations + +import logging +from typing import Any + +from pydantic import BaseModel +from sqlalchemy import text + +logger = logging.getLogger(__name__) + +# Веса по категории — согласованы с _POI_WEIGHTS в parcels.py + новые из vault B6. +# Задача: "2GIS-style ranking", метро самое приоритетное. +CATEGORY_WEIGHTS: dict[str, float] = { + "metro_stop": 6.0, + "school": 5.0, + "kindergarten": 4.5, + "hospital": 4.0, + "shop_supermarket": 3.5, + "shop_mall": 4.0, + "park": 3.5, + "bus_stop": 4.5, + "tram_stop": 2.0, + "pharmacy": 2.5, + "shop_small": 2.0, + "default": 1.0, +} + + +class PoiScoreItem(BaseModel): + """Один POI в ranked-ответе.""" + + name: str | None + category: str + distance_m: float + weight: float + address: str | None + + +class PoiScoreResponse(BaseModel): + cad_num: str + radius_m: int + top_poi: list[PoiScoreItem] + + +def _category_weight(category: str) -> float: + """Вернуть вес категории. Если не знаем — default.""" + return CATEGORY_WEIGHTS.get(category, CATEGORY_WEIGHTS["default"]) + + +def compute_poi_weighted_top7( + db: Any, + cad_num: str, + lat: float, + lon: float, + radius_m: int = 2000, + top_n: int = 7, +) -> PoiScoreResponse: + """Найти top-N POI вокруг (lat, lon) в radius_m, ранжировать по weighted score. + + Запрос к osm_poi_ekb через ST_DWithin + ST_Distance. + Формула: weight = (1 / (distance_m + 100)) * category_weight + + Args: + db: SQLAlchemy Session + cad_num: кадастровый номер (для ответа) + lat: широта центроида участка + lon: долгота центроида участка + radius_m: радиус поиска в метрах (default 2000) + top_n: количество POI в ответе (default 7) + + Returns: + PoiScoreResponse с отсортированными по weight DESC POI. + """ + # ST_DWithin с geography=true использует метры напрямую. + # ST_Distance тоже в метрах при geography=true. + rows = ( + db.execute( + text(""" + SELECT + p.name, + p.category, + p.tags, + CAST( + ST_Distance( + p.geom::geography, + ST_SetSRID(ST_MakePoint(:lon, :lat), 4326)::geography + ) AS double precision + ) AS distance_m + FROM osm_poi_ekb p + WHERE ST_DWithin( + p.geom::geography, + ST_SetSRID(ST_MakePoint(:lon, :lat), 4326)::geography, + :radius_m + ) + ORDER BY distance_m ASC + LIMIT :limit + """), + { + "lat": lat, + "lon": lon, + "radius_m": radius_m, + "limit": top_n * 10, # запрашиваем больше, потом ранжируем + }, + ) + .mappings() + .all() + ) + + logger.debug( + "poi_score: cad=%s lat=%.5f lon=%.5f radius=%dm → %d candidates", + cad_num, + lat, + lon, + radius_m, + len(rows), + ) + + items: list[PoiScoreItem] = [] + for row in rows: + distance_m = float(row["distance_m"]) + category = row["category"] or "default" + cat_weight = _category_weight(category) + weight = (1.0 / (distance_m + 100.0)) * cat_weight + + # Адрес из tags jsonb если есть + tags: dict[str, str] = row["tags"] or {} + addr_parts = [ + tags.get("addr:street"), + tags.get("addr:housenumber"), + ] + address = ", ".join(p for p in addr_parts if p) or None + + items.append( + PoiScoreItem( + name=row["name"], + category=category, + distance_m=round(distance_m, 1), + weight=round(weight, 6), + address=address, + ) + ) + + # Сортировка по weight DESC, берём top_n + items.sort(key=lambda x: x.weight, reverse=True) + top_items = items[:top_n] + + return PoiScoreResponse( + cad_num=cad_num, + radius_m=radius_m, + top_poi=top_items, + ) diff --git a/backend/app/services/site_finder/quarter_dump_lookup.py b/backend/app/services/site_finder/quarter_dump_lookup.py index 72bd03ab..6dabac98 100644 --- a/backend/app/services/site_finder/quarter_dump_lookup.py +++ b/backend/app/services/site_finder/quarter_dump_lookup.py @@ -13,11 +13,14 @@ harvest_quarter.apply_async() и продолжает без dump-derived пол from __future__ import annotations +import json import logging +import math from datetime import UTC, datetime, timedelta from typing import Any from sqlalchemy import text +from sqlalchemy.exc import OperationalError, ProgrammingError from sqlalchemy.orm import Session logger = logging.getLogger(__name__) @@ -29,18 +32,31 @@ _DUMP_MAX_AGE_DAYS = 180 # Радиус поиска инженерных сооружений (метры) — договорённость #44 I3. _ENGINEERING_RADIUS_M = 200 +# Issue #234: typical NSPD harvest duration. Frontend использует для countdown + +# auto-stop polling после ETA*1.5. Замерено по логам harvest_quarter task. +_HARVEST_ETA_SECONDS = 60 + +# Issue #234: SETNX lock TTL (секунды). Защищает от burst N concurrent analyze +# на один свежетриггеренный квартал — N одинаковых harvest task в очередь. +# TTL > _HARVEST_ETA_SECONDS чтобы lock жил пока task реально работает. +_HARVEST_LOCK_TTL_SECONDS = 120 + # Sentinel для isinstance-проверок и read-only fallback в parcels.py try/except. # НИКОГДА не мутировать — использовать make_empty_result() для новых dict. EMPTY_DUMP_RESULT: dict[str, Any] = { "nspd_zoning": None, "nspd_zouit_overlaps": [], "nspd_engineering_nearby": [], + "nspd_risk_zones": [], + "nspd_opportunity_parcels": [], + "nspd_red_lines": [], "nspd_dump": { "available": False, "fetched_at_utc": None, "stale": False, "harvest_triggered": False, "total_features": None, + "harvest_eta_seconds": None, }, } @@ -51,21 +67,30 @@ def make_empty_result( stale: bool = False, harvest_triggered: bool = False, total_features: int | None = None, + harvest_eta_seconds: int | None = None, ) -> dict[str, Any]: """Создаёт свежую копию empty-dump result с возможностью переопределить поля. Вызывать вместо EMPTY_DUMP_RESULT напрямую — чтобы не мутировать singleton. + + `harvest_eta_seconds` (issue #234): передавать когда harvest_triggered=True, + чтобы фронт показал countdown вместо бесконечного спиннера и остановил + re-poll после ETA*1.5. """ return { "nspd_zoning": None, "nspd_zouit_overlaps": [], "nspd_engineering_nearby": [], + "nspd_risk_zones": [], + "nspd_opportunity_parcels": [], + "nspd_red_lines": [], "nspd_dump": { "available": False, "fetched_at_utc": fetched_at_utc, "stale": stale, "harvest_triggered": harvest_triggered, "total_features": total_features, + "harvest_eta_seconds": harvest_eta_seconds, }, } @@ -122,7 +147,10 @@ def get_quarter_dump_data( harvest_error, territorial_zones_count, zouit_count, - engineering_count + engineering_count, + risks_count, + COALESCE(opportunity_count, 0) AS opportunity_count, + COALESCE(red_lines_count, 0) AS red_lines_count FROM nspd_quarter_dumps WHERE quarter_cad = :q """ @@ -134,9 +162,17 @@ def get_quarter_dump_data( max_age = timedelta(days=_DUMP_MAX_AGE_DAYS) if row is None: - # Дампа нет — ставим harvest в очередь + # Дампа нет — ставим harvest в очередь и делаем fallback на cad_zouit (#243). harvest_triggered = _trigger_harvest(quarter) - return make_empty_result(harvest_triggered=harvest_triggered) + cad_zouit_overlaps: list[dict[str, Any]] = ( + _get_cad_zouit_overlaps(db, parcel_wkt) if parcel_wkt is not None else [] + ) + result = make_empty_result( + harvest_triggered=harvest_triggered, + harvest_eta_seconds=_HARVEST_ETA_SECONDS if harvest_triggered else None, + ) + result["nspd_zouit_overlaps"] = cad_zouit_overlaps + return result fetched_at: datetime = row[1] # Убедимся что timezone-aware для корректного сравнения @@ -148,6 +184,9 @@ def get_quarter_dump_data( territorial_zones_count: int = row[4] or 0 zouit_count: int = row[5] or 0 engineering_count: int = row[6] or 0 + risks_count: int = row[7] or 0 + opportunity_count: int = row[8] or 0 + red_lines_count: int = row[9] or 0 is_stale = (now - fetched_at) > max_age has_error = harvest_error is not None @@ -160,6 +199,7 @@ def get_quarter_dump_data( stale=is_stale, harvest_triggered=harvest_triggered, total_features=total_features, + harvest_eta_seconds=_HARVEST_ETA_SECONDS if harvest_triggered else None, ) # Свежий дамп без ошибок — извлекаем spatial данные @@ -169,6 +209,7 @@ def get_quarter_dump_data( "stale": False, "harvest_triggered": False, "total_features": total_features, + "harvest_eta_seconds": None, } if parcel_wkt is None: @@ -177,6 +218,9 @@ def get_quarter_dump_data( "nspd_zoning": None, "nspd_zouit_overlaps": [], "nspd_engineering_nearby": [], + "nspd_risk_zones": [], + "nspd_opportunity_parcels": [], + "nspd_red_lines": [], "nspd_dump": dump_meta, } @@ -184,15 +228,24 @@ def get_quarter_dump_data( "territorial_zones_count": territorial_zones_count, "zouit_count": zouit_count, "engineering_count": engineering_count, + "risks_count": risks_count, + "opportunity_count": opportunity_count, + "red_lines_count": red_lines_count, } nspd_zoning = _get_zoning(db, quarter, parcel_wkt, layer_counts) nspd_zouit = _get_zouit_overlaps(db, quarter, parcel_wkt, layer_counts) nspd_engineering = _get_engineering_nearby(db, quarter, parcel_wkt, layer_counts) + nspd_risk_zones = _get_risk_zones(db, quarter, parcel_wkt, layer_counts) + nspd_opportunity = _get_opportunity_parcels(db, quarter, parcel_wkt, layer_counts) + nspd_red_lines = _get_red_lines(db, quarter, parcel_wkt, layer_counts) return { "nspd_zoning": nspd_zoning, "nspd_zouit_overlaps": nspd_zouit, "nspd_engineering_nearby": nspd_engineering, + "nspd_risk_zones": nspd_risk_zones, + "nspd_opportunity_parcels": nspd_opportunity, + "nspd_red_lines": nspd_red_lines, "nspd_dump": dump_meta, } @@ -269,54 +322,108 @@ def _get_zouit_overlaps( Проверяем 5 групп: zouit_okn, zouit_engineering, zouit_natural, zouit_protected, zouit_other. layer_counts — денормализованные счётчики. Если zouit_count == 0 — - пропускаем heavy jsonb_array_elements scan. + пропускаем nspd dump scan и делаем fallback на cad_zouit (#232). + """ + nspd_has_zouit = layer_counts is None or layer_counts.get("zouit_count", 1) != 0 + if nspd_has_zouit: + try: + rows = db.execute( + text( + """ + SELECT feat.value->>'layer' AS layer, + feat.value->'properties' AS props + FROM nspd_quarter_dumps d, + jsonb_array_elements(d.features_json) AS feat(value) + WHERE d.quarter_cad = :q + AND feat.value->>'layer' LIKE 'zouit_%' + AND (feat.value->'geometry') IS NOT NULL + AND feat.value->>'geometry' != 'null' + AND ST_Intersects( + ST_Transform( + ST_SetSRID( + ST_GeomFromGeoJSON(feat.value->>'geometry'), + 3857 + ), + 4326 + ), + ST_GeomFromText(:wkt, 4326) + ) + """ + ), + {"q": quarter, "wkt": parcel_wkt}, + ).fetchall() + + result: list[dict[str, Any]] = [] + for r in rows: + layer: str = r[0] or "" + props: dict[str, Any] = r[1] if isinstance(r[1], dict) else {} + group_key = layer.removeprefix("zouit_") + result.append( + { + "group_key": group_key, + "layer": layer, + "subcategory": props.get("subcategory") or props.get("type_zone"), + "name": props.get("name") or props.get("object_name"), + "raw_props": props, + "source": "nspd-quarter-dump", + } + ) + return result + except Exception as e: + logger.warning("nspd zouit query failed for quarter=%s: %s", quarter, e) + return [] + + # Fallback: cad_zouit (#232) — nspd dump пуст по ЗОУИТ. + # cad_zouit.subcategory = 100% NULL (verified), используем type_zone keywords. + return _get_cad_zouit_overlaps(db, parcel_wkt) + + +def _get_cad_zouit_overlaps(db: Session, parcel_wkt: str) -> list[dict[str, Any]]: + """Fallback: читать ЗОУИТ из cad_zouit когда nspd_quarter_dumps пуст (#232). + + Использует GIST index cad_zouit_geom_gist через ST_Intersects. + Возвращает тот же формат что nspd-dump path (с доп. полем source='cad_zouit'). """ - if layer_counts is not None and layer_counts.get("zouit_count", 1) == 0: - return [] try: rows = db.execute( text( """ - SELECT feat.value->>'layer' AS layer, - feat.value->'properties' AS props - FROM nspd_quarter_dumps d, - jsonb_array_elements(d.features_json) AS feat(value) - WHERE d.quarter_cad = :q - AND feat.value->>'layer' LIKE 'zouit_%' - AND (feat.value->'geometry') IS NOT NULL - AND feat.value->>'geometry' != 'null' - AND ST_Intersects( - ST_Transform( - ST_SetSRID( - ST_GeomFromGeoJSON(feat.value->>'geometry'), - 3857 - ), - 4326 - ), - ST_GeomFromText(:wkt, 4326) - ) + SELECT type_zone, + category_name, + name_by_doc AS name, + reg_numb_border, + id AS zouit_id + FROM cad_zouit + WHERE ST_Intersects(geom, ST_GeomFromText(:wkt, 4326)) + ORDER BY id """ ), - {"q": quarter, "wkt": parcel_wkt}, + {"wkt": parcel_wkt}, ).fetchall() result: list[dict[str, Any]] = [] for r in rows: - layer: str = r[0] or "" - props: dict[str, Any] = r[1] if isinstance(r[1], dict) else {} - group_key = layer.removeprefix("zouit_") + type_zone: str = r[0] or "" result.append( { - "group_key": group_key, - "layer": layer, - "subcategory": props.get("subcategory") or props.get("type_zone"), - "name": props.get("name") or props.get("object_name"), - "raw_props": props, + "group_key": "cad_zouit", + "layer": type_zone, + "subcategory": None, # subcategory = 100% NULL в cad_zouit + "name": r[2], + "raw_props": { + "type_zone": type_zone, + "category_name": r[1], + "reg_numb_border": r[3], + "zouit_id": r[4], + }, + "source": "cad_zouit", + "type_zone": type_zone, } ) + logger.info("cad_zouit fallback: found %d overlaps for wkt prefix", len(result)) return result except Exception as e: - logger.warning("nspd zouit query failed for quarter=%s: %s", quarter, e) + logger.warning("cad_zouit fallback query failed: %s", e) return [] @@ -392,24 +499,754 @@ def _get_engineering_nearby( return [] +# ── Risk zones (issue #94 TIER 3) ──────────────────────────────────────────── + +# Human-readable subtype labels for risk layer keys. +# Keys match NSPDClient.QUARTER_RISK_LAYERS short_names. +_RISK_SUBTYPE_LABELS: dict[str, str] = { + "flooding_underground": "Подтопление", + "flooding": "Затопление", + "swampification": "Заболачивание", + "landslide": "Обвально-осыпные процессы", + "abrasion": "Абразия", + "erosion_water": "Водная эрозия", + "erosion_linear": "Линейная эрозия", + "erosion_wind": "Ветровая эрозия", + "desertification": "Опустынивание", + "clutter": "Захламление", + "burns": "Гари", +} + + +def _extract_features_by_layer( + features: list[dict[str, Any]], + layer_prefix: str, +) -> list[dict[str, Any]]: + """Generic filter: вернуть features у которых layer начинается с layer_prefix. + + Предназначен для re-use в PR 2+ (opportunity layers, etc.). + + Args: + features: list of feature dicts из features_json (уже загруженных в Python). + layer_prefix: префикс для фильтрации, e.g. 'risk_', 'zouit_'. + + Returns: + Подсписок features где feat['layer'].startswith(layer_prefix). + """ + return [ + f + for f in features + if isinstance(f.get("layer"), str) and f["layer"].startswith(layer_prefix) + ] + + +def _get_risk_zones( + db: Session, + quarter: str, + parcel_wkt: str, + layer_counts: dict[str, int] | None = None, +) -> list[dict[str, Any]]: + """TIER 3: риск-зоны НСПД (11 слоёв), пересекающие участок. + + Возвращает список риск-зон с полями: + layer, subtype, geom_wkt, intersection_area_sqm. + + layer_counts — денормализованные счётчики. Если risks_count == 0 — + пропускаем heavy jsonb_array_elements scan. + """ + if layer_counts is not None and layer_counts.get("risks_count", 1) == 0: + return [] + try: + rows = db.execute( + text( + """ + SELECT feat.value->>'layer' AS layer, + feat.value->'properties' AS props, + ST_AsText( + ST_Transform( + ST_SetSRID( + ST_GeomFromGeoJSON(feat.value->>'geometry'), + 3857 + ), + 4326 + ) + ) AS geom_wkt, + ST_Area( + ST_Intersection( + ST_Transform( + ST_SetSRID( + ST_GeomFromGeoJSON(feat.value->>'geometry'), + 3857 + ), + 4326 + )::geography, + ST_GeomFromText(:wkt, 4326)::geography + ) + ) AS intersection_area_sqm + FROM nspd_quarter_dumps d, + jsonb_array_elements(d.features_json) AS feat(value) + WHERE d.quarter_cad = :q + AND feat.value->>'layer' LIKE 'risk_%' + AND (feat.value->'geometry') IS NOT NULL + AND feat.value->>'geometry' != 'null' + AND ST_Intersects( + ST_Transform( + ST_SetSRID( + ST_GeomFromGeoJSON(feat.value->>'geometry'), + 3857 + ), + 4326 + ), + ST_GeomFromText(:wkt, 4326) + ) + ORDER BY layer, intersection_area_sqm DESC NULLS LAST + """ + ), + {"q": quarter, "wkt": parcel_wkt}, + ).fetchall() + + result: list[dict[str, Any]] = [] + for r in rows: + layer: str = r[0] or "" + props: dict[str, Any] = r[1] if isinstance(r[1], dict) else {} + geom_wkt_val: str | None = r[2] + raw_area: Any = r[3] + intersection_area_sqm: float | None = None + if raw_area is not None: + try: + val = float(raw_area) + intersection_area_sqm = round(val, 1) if not math.isnan(val) else None + except (TypeError, ValueError): + pass + + # Извлечь short_name из "risk_" для человекочитаемого subtype + short_name = layer.removeprefix("risk_") + # Попытка взять subtype из properties, затем из mapping + subtype = ( + props.get("type_zone") + or props.get("subcategory") + or props.get("name") + or _RISK_SUBTYPE_LABELS.get(short_name) + ) + if subtype is not None: + subtype = str(subtype) + + result.append( + { + "layer": layer, + "subtype": subtype, + "geom_wkt": geom_wkt_val, + "intersection_area_sqm": intersection_area_sqm, + } + ) + return result + except Exception as e: + logger.warning("nspd risk zones query failed for quarter=%s: %s", quarter, e) + return [] + + +# ── Opportunity parcels (issue #94 TIER 4) ─────────────────────────────────── + +# Human-readable type labels for opportunity layer short_names. +_OPPORTUNITY_TYPE_LABELS: dict[str, str] = { + "auction_parcels": "auction_parcels", + "scheme_parcels": "scheme_parcels", + "free_parcels": "free_parcels", + "future_parcels": "future_parcels", + "oopt": "oopt", +} + +# Radius (m) for opportunity parcel proximity search from parcel centroid. +_OPPORTUNITY_RADIUS_M = 500 + + +def _get_opportunity_parcels( + db: Session, + quarter: str, + parcel_wkt: str, + layer_counts: dict[str, int] | None = None, +) -> list[dict[str, Any]]: + """TIER 4: opportunity parcels вблизи участка (auction, scheme, free, future, oopt). + + Возвращает список ближайших opportunity ЗУ с полями: + layer, cad_num, distance_m, geom_wkt. + + layer_counts — денормализованные счётчики. Если opportunity_count == 0 — + пропускаем heavy jsonb_array_elements scan (early-exit pattern как у risks). + """ + if layer_counts is not None and layer_counts.get("opportunity_count", 1) == 0: + return [] + try: + rows = db.execute( + text( + """ + SELECT feat.value->>'layer' AS layer, + feat.value->'properties' AS props, + ST_AsText( + ST_Transform( + ST_SetSRID( + ST_GeomFromGeoJSON(feat.value->>'geometry'), + 3857 + ), + 4326 + ) + ) AS geom_wkt, + ST_Distance( + ST_Transform( + ST_SetSRID( + ST_GeomFromGeoJSON(feat.value->>'geometry'), + 3857 + ), + 4326 + )::geography, + ST_Centroid(ST_GeomFromText(:wkt, 4326))::geography + ) AS distance_m + FROM nspd_quarter_dumps d, + jsonb_array_elements(d.features_json) AS feat(value) + WHERE d.quarter_cad = :q + AND feat.value->>'layer' LIKE 'opportunity_%' + AND (feat.value->'geometry') IS NOT NULL + AND feat.value->>'geometry' != 'null' + AND ST_DWithin( + ST_Transform( + ST_SetSRID( + ST_GeomFromGeoJSON(feat.value->>'geometry'), + 3857 + ), + 4326 + )::geography, + ST_Centroid(ST_GeomFromText(:wkt, 4326))::geography, + :radius_m + ) + ORDER BY distance_m ASC + LIMIT 30 + """ + ), + {"q": quarter, "wkt": parcel_wkt, "radius_m": _OPPORTUNITY_RADIUS_M}, + ).fetchall() + + result: list[dict[str, Any]] = [] + for r in rows: + raw_layer: str = r[0] or "" + props: dict[str, Any] = r[1] if isinstance(r[1], dict) else {} + geom_wkt_val: str | None = r[2] + distance_m = float(r[3]) if r[3] is not None else None + + # Strip "opportunity_" prefix to get short_name → type label + short_name = raw_layer.removeprefix("opportunity_") + layer_type = _OPPORTUNITY_TYPE_LABELS.get(short_name, short_name) + + cad_num = props.get("cad_num") or props.get("cadastral_number") + + result.append( + { + "layer": layer_type, + "cad_num": cad_num, + "distance_m": round(distance_m, 1) if distance_m is not None else None, + "geom_wkt": geom_wkt_val, + } + ) + return result + except Exception as e: + logger.warning("nspd opportunity parcels query failed for quarter=%s: %s", quarter, e) + return [] + + +# Radius (m) for red lines proximity search — beyond intersect check. +_RED_LINES_NEARBY_M = 200 + + +def _get_red_lines( + db: Session, + quarter: str, + parcel_wkt: str, + layer_counts: dict[str, int] | None = None, +) -> list[dict[str, Any]]: + """TIER 4: красные линии застройки (layer 879243). + + Возвращает red lines пересекающие участок ИЛИ ближайшие (до 200м) с полями: + geom_wkt, intersection_length_m, distance_m. + + Логика: + - intersecting: intersection_length_m >= 0, distance_m = None + - nearby only: intersection_length_m = None, distance_m = расстояние + + ST_Intersection выполняется в planar EPSG:4326, затем результат кастуется + в ::geography для ST_Length — это избегает PostGIS 3.4 tolerance bug + (ST_Intersection geography × geography на LINESTRING бросает transform error). + + layer_counts — денормализованные счётчики. Если red_lines_count == 0 — + пропускаем heavy jsonb_array_elements scan. + """ + if layer_counts is not None and layer_counts.get("red_lines_count", 1) == 0: + return [] + try: + rows = db.execute( + text( + """ + SELECT ST_AsText( + ST_Transform( + ST_SetSRID( + ST_GeomFromGeoJSON(feat.value->>'geometry'), + 3857 + ), + 4326 + ) + ) AS geom_wkt, + ST_Intersects( + ST_Transform( + ST_SetSRID( + ST_GeomFromGeoJSON(feat.value->>'geometry'), + 3857 + ), + 4326 + ), + ST_GeomFromText(:wkt, 4326) + ) AS does_intersect, + ST_Length( + ST_Intersection( + ST_Transform( + ST_SetSRID( + ST_GeomFromGeoJSON(feat.value->>'geometry'), + 3857 + ), + 4326 + ), + ST_GeomFromText(:wkt, 4326) + )::geography + ) AS intersection_length_m, + ST_Distance( + ST_Transform( + ST_SetSRID( + ST_GeomFromGeoJSON(feat.value->>'geometry'), + 3857 + ), + 4326 + )::geography, + ST_GeomFromText(:wkt, 4326)::geography + ) AS distance_m + FROM nspd_quarter_dumps d, + jsonb_array_elements(d.features_json) AS feat(value) + WHERE d.quarter_cad = :q + AND feat.value->>'layer' = 'red_lines' + AND (feat.value->'geometry') IS NOT NULL + AND feat.value->>'geometry' != 'null' + AND ST_DWithin( + ST_Transform( + ST_SetSRID( + ST_GeomFromGeoJSON(feat.value->>'geometry'), + 3857 + ), + 4326 + )::geography, + ST_GeomFromText(:wkt, 4326)::geography, + :nearby_m + ) + ORDER BY distance_m ASC + LIMIT 50 + """ + ), + {"q": quarter, "wkt": parcel_wkt, "nearby_m": _RED_LINES_NEARBY_M}, + ).fetchall() + + result: list[dict[str, Any]] = [] + for r in rows: + geom_wkt_val: str | None = r[0] + does_intersect: bool = bool(r[1]) if r[1] is not None else False + raw_length: Any = r[2] + raw_dist: Any = r[3] + + intersection_length_m: float | None = None + if does_intersect and raw_length is not None: + try: + val = float(raw_length) + intersection_length_m = round(val, 1) if not math.isnan(val) else None + except (TypeError, ValueError): + pass + + distance_m: float | None = None + if not does_intersect and raw_dist is not None: + try: + distance_m = round(float(raw_dist), 1) + except (TypeError, ValueError): + pass + + result.append( + { + "geom_wkt": geom_wkt_val, + "intersection_length_m": intersection_length_m, + "distance_m": distance_m, + } + ) + return result + except Exception as e: + logger.warning("nspd red lines query failed for quarter=%s: %s", quarter, e) + return [] + + +# ── Connection-points lookup (issue #115) ──────────────────────────────────── + + +def get_connection_points(db: Session, cad_num: str, radius_m: int = 500) -> dict[str, Any]: + """Получить точки инженерных подключений в radius_m от boundary участка. + + Источник: `nspd_quarter_dumps.features_json` для квартала cad_num — + layers `engineering_structures` (NSPD cat 36328, ТП/ЦТП/насосные/опоры ЛЭП) + и `zouit_engineering` (NSPD cat 37578 — охранные зоны инжен.коммуникаций). + + Args: + db: SQLAlchemy session. + cad_num: кадастровый номер участка (e.g. '66:41:0204016:10'). + radius_m: радиус поиска в метрах от boundary участка (default 500). + + Returns: + { + "engineering_structures": [...], + "zouit_engineering_overlaps": [...], + "summary": {...}, + "dump_available": bool, + "dump_fetched_at": str | None, + } + + Raises: + ValueError: если parcel не найден в cad_parcels_geom / cad_quarters_geom. + """ + quarter = derive_quarter_cad(cad_num) + if quarter is None: + raise ValueError(f"Невалидный формат кадастрового номера: {cad_num!r}") + + # Получаем WKT геометрию участка (boundary, не centroid) + parcel_wkt = _get_parcel_wkt(db, cad_num) + if parcel_wkt is None: + raise ValueError(f"Участок {cad_num!r} не найден в БД") + + # Проверяем наличие дампа + dump_row = db.execute( + text( + """ + SELECT fetched_at_utc, total_features + FROM nspd_quarter_dumps + WHERE quarter_cad = :q + ORDER BY fetched_at_utc DESC + LIMIT 1 + """ + ), + {"q": quarter}, + ).first() + + if dump_row is None: + _trigger_harvest(quarter) + return { + "engineering_structures": [], + "zouit_engineering_overlaps": [], + "summary": { + "nearest_structure_distance_m": None, + "in_protection_zone": False, + "protection_zones_intersecting": 0, + "total_structures_in_radius": 0, + }, + "dump_available": False, + "dump_fetched_at": None, + } + + fetched_at = dump_row[0] + if fetched_at is not None and getattr(fetched_at, "isoformat", None): + dump_fetched_at: str | None = fetched_at.isoformat() + else: + dump_fetched_at = str(fetched_at) if fetched_at is not None else None + + structures = _get_engineering_structures_by_boundary(db, quarter, parcel_wkt, radius_m) + zouit_overlaps = _get_zouit_engineering_overlaps(db, quarter, parcel_wkt) + + nearest_dist: float | None = structures[0]["distance_to_boundary_m"] if structures else None + protection_count = len(zouit_overlaps) + + return { + "engineering_structures": structures, + "zouit_engineering_overlaps": zouit_overlaps, + "summary": { + "nearest_structure_distance_m": nearest_dist, + "in_protection_zone": protection_count > 0, + "protection_zones_intersecting": protection_count, + "total_structures_in_radius": len(structures), + }, + "dump_available": True, + "dump_fetched_at": dump_fetched_at, + } + + +def _get_parcel_wkt(db: Session, cad_num: str) -> str | None: + """Получить WKT геометрию участка из cad_parcels_geom или fallback источников. + + Игнорирует строки с geom IS NULL (data quality) — иначе ST_AsText(NULL) + вернёт SQL NULL → str(None) = "None" → ST_GeomFromText упадёт, ошибка + тихо проглотится в caller'е, клиент получит пустые массивы без причины. + """ + row = db.execute( + text( + """ + SELECT ST_AsText(geom) AS wkt + FROM cad_parcels_geom + WHERE cad_num = :c AND geom IS NOT NULL + LIMIT 1 + """ + ), + {"c": cad_num}, + ).first() + if row is not None and row[0] is not None: + return str(row[0]) + + # Fallback: кварталы (более крупный объект — менее точно, но лучше чем ничего). + # cad_quarters_geom хранит 3-сегментные ключи (66:41:0204016), не 4-сегментные. + quarter_key = derive_quarter_cad(cad_num) + row = db.execute( + text( + """ + SELECT ST_AsText(geom) AS wkt + FROM cad_quarters_geom + WHERE cad_number = :c AND geom IS NOT NULL + LIMIT 1 + """ + ), + {"c": quarter_key}, + ).first() + if row is not None and row[0] is not None: + return str(row[0]) + + return None + + +def _get_engineering_structures_by_boundary( + db: Session, + quarter: str, + parcel_wkt: str, + radius_m: int, +) -> list[dict[str, Any]]: + """Engineering structures из dump в radius_m от boundary участка. + + Использует ST_Distance к boundary (не centroid) для корректного расстояния. + Geometry трансформируется из EPSG:3857 (хранение dump) → 4326. + """ + try: + rows = db.execute( + text( + """ + SELECT feat.value->'properties' AS props, + feat.value->>'geometry' AS geom_json, + ST_Distance( + ST_Transform( + ST_SetSRID( + ST_GeomFromGeoJSON(feat.value->>'geometry'), + 3857 + ), + 4326 + )::geography, + ST_GeomFromText(:wkt, 4326)::geography + ) AS distance_m + FROM nspd_quarter_dumps d, + jsonb_array_elements(d.features_json) AS feat(value) + WHERE d.quarter_cad = :q + AND feat.value->>'layer' = 'engineering_structures' + AND (feat.value->'geometry') IS NOT NULL + AND feat.value->>'geometry' != 'null' + AND ST_DWithin( + ST_Transform( + ST_SetSRID( + ST_GeomFromGeoJSON(feat.value->>'geometry'), + 3857 + ), + 4326 + )::geography, + ST_GeomFromText(:wkt, 4326)::geography, + :radius_m + ) + ORDER BY distance_m ASC + LIMIT 50 + """ + ), + {"q": quarter, "wkt": parcel_wkt, "radius_m": radius_m}, + ).fetchall() + except (OperationalError, ProgrammingError) as e: + logger.warning( + "engineering_structures query failed for quarter=%s: %s", + quarter, + e, + ) + return [] + + result: list[dict[str, Any]] = [] + for r in rows: + props: dict[str, Any] = r[0] if isinstance(r[0], dict) else {} + geom_raw: str | None = r[1] + distance_m = float(r[2]) if r[2] is not None else 0.0 + + # Попытка распарсить geometry как dict для GeoJSON поля + geom_dict: dict[str, Any] = {} + if geom_raw: + try: + geom_dict = json.loads(geom_raw) + except (ValueError, json.JSONDecodeError): + geom_dict = {} + + result.append( + { + "name": props.get("name") or props.get("object_name"), + "type": props.get("purpose") or props.get("object_type") or props.get("type_zone"), + "cad_num": props.get("cad_num") or props.get("cadastral_number"), + "distance_to_boundary_m": round(distance_m, 1), + "geometry_geojson": geom_dict, + "readable_address": props.get("readable_address") or props.get("address"), + "raw_props": props, + "source": "nspd_36328", + } + ) + return result + + +def _get_zouit_engineering_overlaps( + db: Session, + quarter: str, + parcel_wkt: str, +) -> list[dict[str, Any]]: + """ZOUIT engineering (cat 37578) — охранные зоны, пересекающие участок. + + Использует слой 'zouit_engineering' из dump. + """ + try: + rows = db.execute( + text( + """ + SELECT feat.value->'properties' AS props, + feat.value->>'geometry' AS geom_json + FROM nspd_quarter_dumps d, + jsonb_array_elements(d.features_json) AS feat(value) + WHERE d.quarter_cad = :q + AND feat.value->>'layer' = 'zouit_engineering' + AND (feat.value->'geometry') IS NOT NULL + AND feat.value->>'geometry' != 'null' + AND ST_Intersects( + ST_Transform( + ST_SetSRID( + ST_GeomFromGeoJSON(feat.value->>'geometry'), + 3857 + ), + 4326 + ), + ST_GeomFromText(:wkt, 4326) + ) + LIMIT 100 + """ + ), + {"q": quarter, "wkt": parcel_wkt}, + ).fetchall() + except (OperationalError, ProgrammingError) as e: + logger.warning( + "zouit_engineering query failed for quarter=%s: %s", + quarter, + e, + ) + return [] + + result: list[dict[str, Any]] = [] + for r in rows: + props: dict[str, Any] = r[0] if isinstance(r[0], dict) else {} + geom_raw: str | None = r[1] + + geom_dict: dict[str, Any] = {} + if geom_raw: + try: + geom_dict = json.loads(geom_raw) + except (ValueError, json.JSONDecodeError): + geom_dict = {} + + subcategory_raw = props.get("subcategory") + subcategory: int | None = None + if subcategory_raw is not None: + try: + subcategory = int(subcategory_raw) + except (ValueError, TypeError): + subcategory = None + + result.append( + { + "reg_numb_border": props.get("reg_numb_border"), + "type_zone": props.get("type_zone") or props.get("zone_name"), + "subcategory": subcategory, + "intersects_parcel": True, + "geometry_geojson": geom_dict, + "raw_props": props, + "source": "nspd_37578", + } + ) + return result + + # ── Harvest trigger ─────────────────────────────────────────────────────────── +def _acquire_harvest_lock(quarter: str) -> bool: + """Redis SETNX lock на quarter с TTL=_HARVEST_LOCK_TTL_SECONDS. + + Issue #234: защита от burst N concurrent analyze_parcel запросов — без lock + каждый из них вызывал бы harvest_quarter.apply_async() и слал бы N + идентичных задач в Celery, тратя WAF traffic впустую (UPSERT идемпотентен, + но трафик не вернёшь). + + Returns: + True — lock acquired (caller должен enqueue task) + False — lock уже взят другим запросом ИЛИ redis недоступен (graceful + degradation: пропускаем дедуп, возвращаем False чтобы caller тоже не + триггерил — иначе одиночный redis-fail вернул бы поведение до фикса). + """ + try: + import redis + + from app.core.config import settings + + client = redis.Redis.from_url(settings.redis_url) + # SET key value NX EX — атомарно: «установи если не существует, TTL N сек». + acquired = client.set( + f"nspd_harvest_lock:{quarter}", + "1", + nx=True, + ex=_HARVEST_LOCK_TTL_SECONDS, + ) + return bool(acquired) + except Exception as e: + # Redis недоступен — лучше не запускать дубль, чем нагрузить WAF. + logger.warning("redis SETNX lock failed for quarter=%s: %s", quarter, e) + return False + + def _trigger_harvest(quarter: str) -> bool: - """Fire-and-forget harvest_quarter.apply_async(). Возвращает True если enqueue OK. + """Fire-and-forget harvest_quarter.apply_async() с Redis SETNX dedupe. Ленивый импорт чтобы избежать circular import (tasks → services → tasks). - Known limitation: burst из N concurrent analyze_parcel запросов на один - и тот же ещё не закешированный квартал может поставить N одинаковых задач - в очередь (нет дедупликации). UPSERT в harvest_quarter идемпотентен, поэтому - данные не портятся, но WAF traffic тратится впустую. - TODO: добавить Redis SETNX lock с TTL перед apply_async — отдельная issue/PR. + Issue #234: burst N concurrent analyze дедуплицируется через + `_acquire_harvest_lock` — только первый запрос ставит task, остальные + получают `False` (т.е. harvest_triggered=False, harvest_eta_seconds=None). + FE при таком ответе показывает graceful no-data UI с ПКК fallback link + (не infinite spinner). Trade-off: Request B не видит countdown, но и не + сбивает работу Request A. Это намеренный design choice — см. vault + fixes/Bug_NSPD_WMS_NotBulk_2026_May14.md. """ + if not _acquire_harvest_lock(quarter): + logger.info("quarter=%s harvest skipped (lock held by another request)", quarter) + return False + try: from app.workers.tasks.nspd_sync import harvest_quarter - harvest_quarter.apply_async(args=[quarter], kwargs={"region_code": 66}) + harvest_quarter.apply_async( + args=[quarter], + kwargs={ + "region_code": 66, + "include_zouit": True, + "include_risks": True, + "include_opportunity": True, + }, + ) logger.info("quarter dump harvest triggered for quarter=%s", quarter) return True except Exception as e: diff --git a/backend/app/services/site_finder/quarter_price_refresh.py b/backend/app/services/site_finder/quarter_price_refresh.py new file mode 100644 index 00000000..97e83891 --- /dev/null +++ b/backend/app/services/site_finder/quarter_price_refresh.py @@ -0,0 +1,63 @@ +"""Refresh helper for mv_quarter_price_per_m2 (Issue #33 D1). + +Not scheduled automatically in this PR — intended for manual invocation or +a Celery beat task in a follow-up issue (Issue #33 PR C). + +Usage example (manual, via psql-connected shell or admin endpoint): + from sqlalchemy.orm import Session + from app.services.site_finder.quarter_price_refresh import refresh_quarter_price + + count = refresh_quarter_price(db) + # logs: "mv_quarter_price_per_m2 refreshed: 52492 rows" +""" + +from __future__ import annotations + +import logging + +from sqlalchemy import text +from sqlalchemy.exc import OperationalError +from sqlalchemy.orm import Session + +logger = logging.getLogger(__name__) + + +def refresh_quarter_price(db: Session, *, concurrently: bool = True) -> int: + """REFRESH MATERIALIZED VIEW mv_quarter_price_per_m2. + + Args: + db: SQLAlchemy Session (sync). + concurrently: When True, uses REFRESH CONCURRENTLY — non-blocking but + requires the unique index mv_quarter_price_pk to exist (created by + 95_mv_quarter_price.sql) and the MV to be already populated. + Pass False only for the very first populate or after MV recreation. + + Returns: + Row count in the MV after refresh (for observability / alerting). + + Raises: + OperationalError: Re-raised if the error is not the known + "cannot refresh materialized view concurrently" case. + """ + try: + if concurrently: + db.execute(text("REFRESH MATERIALIZED VIEW CONCURRENTLY mv_quarter_price_per_m2")) + else: + db.execute(text("REFRESH MATERIALIZED VIEW mv_quarter_price_per_m2")) + db.commit() + except OperationalError as e: + if concurrently and "cannot refresh materialized view" in str(e).lower(): + logger.warning( + "CONCURRENTLY failed (MV likely not populated), falling back to" + " non-concurrent refresh" + ) + db.rollback() + db.execute(text("REFRESH MATERIALIZED VIEW mv_quarter_price_per_m2")) + db.commit() + else: + raise + + row = db.execute(text("SELECT COUNT(*) FROM mv_quarter_price_per_m2")).first() + count = int(row[0]) if row else 0 + logger.info("mv_quarter_price_per_m2 refreshed: %d rows", count) + return count diff --git a/backend/app/services/site_finder/velocity.py b/backend/app/services/site_finder/velocity.py index daafb289..a7e266a6 100644 --- a/backend/app/services/site_finder/velocity.py +++ b/backend/app/services/site_finder/velocity.py @@ -6,10 +6,14 @@ Per #34 D2: утилизация objective_corpus_room_month (еженедель конкурирующих ЖК в радиусе radius_km от участка, нормированный к ЕКБ-медиане по данным Objective. +Fallback (SF#17): если Objective coverage <50% конкурентов в радиусе, +использует rosreestr_deals JOIN по cad_quarter участка (100% coverage по кварталам). + Foundation: domrf_kn_objects (lat/lon, comm_name, obj_class, region_cd), objective_complex_mapping (domrf_obj_id ↔ objective_complex_name), objective_corpus_room_month (project_name, deals_total_vol_m2, deals_total_count, report_month). +Fallback: rosreestr_deals (quarter_cad_number, deal_count, period_start_date). Linkage: domrf_kn_objects.obj_id → objective_complex_mapping.domrf_obj_id @@ -32,6 +36,10 @@ logger = logging.getLogger(__name__) # Эмпирика по ЕКБ: ~4 500 м²/мес на один ЖК (apartments, 2024-2025). _EKB_MEDIAN_FALLBACK_SQM_PER_MONTH: float = 4500.0 +# Порог: если доля конкурентов с Objective-маппингом < этого значения, +# пытаемся rosreestr_fallback. +_OBJECTIVE_COVERAGE_MIN_RATIO: float = 0.50 + @dataclass(frozen=True) class VelocityResult: @@ -47,6 +55,12 @@ class VelocityResult: period_end: str # YYYY-MM sample_competitors: list[dict[str, Any]] # top-5 для UI by_room_bucket: dict[str, dict[str, Any]] # агрегат по комнатности + # True если данные есть (objective или rosreestr_fallback). + # False → нет данных ни из одного источника. + velocity_data_available: bool = True + # Источник данных: objective (основной), rosreestr_fallback (по кадастровому кварталу), + # none (нет данных). + velocity_source: Literal["objective", "rosreestr_fallback", "none"] = "objective" def as_dict(self) -> dict[str, Any]: return { @@ -59,6 +73,8 @@ class VelocityResult: "period": {"start": self.period_start, "end": self.period_end}, "sample_competitors": self.sample_competitors, "by_room_bucket": self.by_room_bucket, + "velocity_data_available": self.velocity_data_available, + "velocity_source": self.velocity_source, } @@ -68,6 +84,7 @@ def compute_velocity( radius_km: float = 3.0, obj_class: str | None = None, months_window: int = 6, + cad_quarter: str | None = None, ) -> VelocityResult | None: """Вычислить velocity-score для участка. @@ -75,9 +92,14 @@ def compute_velocity( 1. Найти ЖК-конкуренты в радиусе radius_km (через lat/lon ST_DWithin). 2. Взять objective_corpus_room_month за последние months_window месяцев через objective_complex_mapping (domrf_obj_id → project_name). - 3. Посчитать суммарный объём deals_total_vol_m2. + 3. Если Objective coverage < 50% конкурентов → rosreestr_fallback: + считаем сделки DDU/ДКП в cad_quarter участка за окно. 4. Нормировать на ЕКБ-медиану → score 0..1. + Параметры: + cad_quarter: кадастровый квартал участка (первые 3 сегмента cad_num, + например "66:41:0702048"). Используется только для fallback. + Возвращает None если parcel_geom_wkt невалиден или конкурентов нет. """ # ── Step 1: конкуренты по lat/lon в радиусе ────────────────────────────── @@ -165,6 +187,8 @@ def compute_velocity( # objective_corpus_room_month. # deals_total_vol_m2 = DDU + DKP м² за месяц (primary signal). # deals_total_count > 0 — фильтрует месяцы без сделок. + # LEFT JOIN с objective_complex_mapping: конкуренты без маппинга не + # выпадают — они включаются в список но с total_sqm=NULL (→ 0.0). # GROUP BY domrf_obj_id чтобы сохранить совместимость с caller (obj_id key). try: with db.begin_nested(): @@ -172,25 +196,32 @@ def compute_velocity( db.execute( text( """ - WITH mapped AS ( + WITH all_competitors AS ( + SELECT unnest(CAST(:obj_ids AS int[])) AS obj_id + ), + mapped AS ( SELECT cm.domrf_obj_id AS obj_id, cm.objective_complex_name FROM objective_complex_mapping cm WHERE cm.domrf_obj_id = ANY(:obj_ids) ) SELECT - m.obj_id, + ac.obj_id, SUM(COALESCE(crm.deals_total_vol_m2, crm.deals_total_count * 45.0)) AS total_sqm, - COUNT(DISTINCT crm.report_month) AS months_with_data, - MIN(crm.report_month) AS period_start, - MAX(crm.report_month) AS period_end - FROM objective_corpus_room_month crm - JOIN mapped m - ON m.objective_complex_name = crm.project_name - WHERE crm.report_month >= (CURRENT_DATE - CAST(:window_interval AS interval)) - AND crm.deals_total_count > 0 - GROUP BY m.obj_id + COUNT(DISTINCT crm.report_month) AS months_with_data, + MIN(crm.report_month) AS period_start, + MAX(crm.report_month) AS period_end, + CASE WHEN m.obj_id IS NOT NULL THEN TRUE + ELSE FALSE END AS has_mapping + FROM all_competitors ac + LEFT JOIN mapped m ON m.obj_id = ac.obj_id + LEFT JOIN objective_corpus_room_month crm + ON crm.project_name = m.objective_complex_name + AND crm.report_month >= ( + CURRENT_DATE - CAST(:window_interval AS interval)) + AND crm.deals_total_count > 0 + GROUP BY ac.obj_id, m.obj_id """ ), { @@ -209,6 +240,69 @@ def compute_velocity( if not sales_rows: return None + # ── Step 2a: проверка Objective coverage ───────────────────────────────── + # Считаем: mapped_with_data — конкуренты с маппингом И реальными данными. + # Если mapped_ratio < _OBJECTIVE_COVERAGE_MIN_RATIO → rosreestr_fallback. + n_total_comps = len(obj_ids) + mapped_with_data = [ + r for r in sales_rows if bool(r["has_mapping"]) and (r["total_sqm"] or 0.0) > 0 + ] + mapped_ratio = len(mapped_with_data) / n_total_comps if n_total_comps > 0 else 0.0 + + ekb_median = ( + _get_ekb_median(db, months_window=months_window) or _EKB_MEDIAN_FALLBACK_SQM_PER_MONTH + ) + n_comps = len(comp_rows) + sample_no_data = sorted( + [ + { + "obj_id": oid, + **competitor_meta[oid], + "total_sqm_period": 0.0, + "by_room_bucket": {}, + } + for oid in obj_ids[:5] + if oid in competitor_meta + ], + key=lambda x: x["distance_m"], # type: ignore[index] + ) + + if mapped_ratio < _OBJECTIVE_COVERAGE_MIN_RATIO: + logger.info( + "velocity: objective coverage %.0f%% (<%d%%) for %d competitors;" + " trying rosreestr_fallback cad_quarter=%s", + mapped_ratio * 100, + int(_OBJECTIVE_COVERAGE_MIN_RATIO * 100), + n_total_comps, + cad_quarter, + ) + rr_result = _compute_rosreestr_fallback( + db=db, + cad_quarter=cad_quarter, + months_window=months_window, + n_comps=n_comps, + ekb_median=ekb_median, + sample_competitors=sample_no_data, + ) + if rr_result is not None: + return rr_result + # Rosreestr тоже пуст — возвращаем none-state. + logger.info("velocity: rosreestr_fallback also empty for cad_quarter=%s", cad_quarter) + return VelocityResult( + competitors_count=n_comps, + monthly_velocity_sqm=0.0, + ekb_median_sqm=ekb_median, + velocity_score=0.0, + confidence="low", + months_observed=0, + period_start="", + period_end="", + sample_competitors=sample_no_data, + by_room_bucket={}, + velocity_data_available=False, + velocity_source="none", + ) + # ── Step 2b: разбивка по комнатности (room_bucket) ─────────────────────── # Тот же маппинг domrf_obj_id → project_name. Агрегируем по room_bucket # для отображения структуры спроса в UI. @@ -278,46 +372,89 @@ def compute_velocity( for bucket, data in by_bucket_agg.items() } - total_sqm = sum(float(r["total_sqm"] or 0.0) for r in sales_rows) - months_observed = max((int(r["months_with_data"] or 0) for r in sales_rows), default=0) - period_start_dates = [r["period_start"] for r in sales_rows if r["period_start"]] - period_end_dates = [r["period_end"] for r in sales_rows if r["period_end"]] + # Считаем только по строкам с маппингом — unmapped строки дают total_sqm=NULL. + mapped_sales_rows = [r for r in sales_rows if bool(r["has_mapping"])] + + total_sqm = sum(float(r["total_sqm"] or 0.0) for r in mapped_sales_rows) + months_observed = max((int(r["months_with_data"] or 0) for r in mapped_sales_rows), default=0) + period_start_dates = [r["period_start"] for r in mapped_sales_rows if r["period_start"]] + period_end_dates = [r["period_end"] for r in mapped_sales_rows if r["period_end"]] period_start = min(period_start_dates).strftime("%Y-%m") if period_start_dates else "" period_end = max(period_end_dates).strftime("%Y-%m") if period_end_dates else "" + # Если mapped-конкурентов нет данных — partial coverage → fallback. if months_observed == 0 or total_sqm <= 0: - return None + logger.info( + "velocity: %d competitors found, %d mapped, but no sales data in window;" + " trying rosreestr_fallback", + len(obj_ids), + len(mapped_sales_rows), + ) + rr_result = _compute_rosreestr_fallback( + db=db, + cad_quarter=cad_quarter, + months_window=months_window, + n_comps=n_comps, + ekb_median=ekb_median, + sample_competitors=sample_no_data, + ) + if rr_result is not None: + return rr_result + sample_partial = sorted( + [ + { + "obj_id": oid, + **competitor_meta[oid], + "total_sqm_period": 0.0, + "by_room_bucket": {}, + } + for oid in obj_ids + if oid in competitor_meta + ], + key=lambda x: x["total_sqm_period"], # type: ignore[arg-type] + reverse=True, + )[:5] + return VelocityResult( + competitors_count=n_comps, + monthly_velocity_sqm=0.0, + ekb_median_sqm=ekb_median, + velocity_score=0.0, + confidence="low", + months_observed=0, + period_start="", + period_end="", + sample_competitors=sample_partial, + by_room_bucket={}, + velocity_data_available=False, + velocity_source="none", + ) # Среднемесячный объём в расчёте: суммарный по всем конкурентам / месяцев. # Чем больше конкурентов с данными — тем весомее результат. monthly_velocity = total_sqm / months_observed - # ── Step 3: ЕКБ-медиана ────────────────────────────────────────────────── - ekb_median = ( - _get_ekb_median(db, months_window=months_window) or _EKB_MEDIAN_FALLBACK_SQM_PER_MONTH - ) - - # ── Step 4: нормализация → score 0..1 ──────────────────────────────────── + # ── Step 3: нормализация → score 0..1 ──────────────────────────────────── # Логика: сравниваем суммарный velocity радиуса с «нормой» одного ЖК. # Если в радиусе продаётся N × ekb_median → рынок горячий. # Нормируем: score = min(1.0, total_velocity / (n_competitors × ekb_median × 2)) # Cap 2×median = «насыщен». Итоговый score 0..1. - n_with_sales = len(sales_rows) + # n_with_sales — только mapped конкуренты (у unmapped данных нет). + n_with_sales = len(mapped_sales_rows) denominator = n_with_sales * ekb_median * 2.0 if n_with_sales > 0 else ekb_median * 2.0 velocity_score = min(1.0, max(0.0, monthly_velocity / denominator)) - # ── Step 5: confidence ─────────────────────────────────────────────────── - n_comps = len(comp_rows) + # ── Step 4: confidence ─────────────────────────────────────────────────── + mapped_conf: Literal["high", "medium", "low"] if n_comps >= 10 and months_observed >= 5: - confidence: Literal["high", "medium", "low"] = "high" + mapped_conf = "high" elif n_comps >= 5 and months_observed >= 3: - confidence = "medium" + mapped_conf = "medium" else: - confidence = "low" + mapped_conf = "low" - # ── Step 6: top-5 конкурентов по объёму продаж ─────────────────────────── + # ── Step 5: top-5 конкурентов по объёму продаж ─────────────────────────── sales_by_id: dict[int, float] = { - int(r["obj_id"]): float(r["total_sqm"] or 0.0) for r in sales_rows + int(r["obj_id"]): float(r["total_sqm"] or 0.0) for r in mapped_sales_rows } sample = sorted( [ @@ -339,12 +476,110 @@ def compute_velocity( monthly_velocity_sqm=monthly_velocity, ekb_median_sqm=ekb_median, velocity_score=velocity_score, - confidence=confidence, + confidence=mapped_conf, months_observed=months_observed, period_start=period_start, period_end=period_end, sample_competitors=sample, by_room_bucket=by_room_bucket, + velocity_data_available=True, + velocity_source="objective", + ) + + +def _compute_rosreestr_fallback( + db: Session, + cad_quarter: str | None, + months_window: int, + n_comps: int, + ekb_median: float, + sample_competitors: list[dict[str, Any]], +) -> VelocityResult | None: + """Fallback velocity через rosreestr_deals JOIN по cad_quarter участка. + + Считает суммарное число сделок DDU/ДКП в кадастровом квартале за окно months_window. + Velocity = deal_count / months_window (сделок/мес). Нет разбивки по room_bucket + (rosreestr не даёт комнатность). + + Возвращает None если cad_quarter не задан или данных нет. + """ + if not cad_quarter: + return None + + try: + with db.begin_nested(): + row = ( + db.execute( + text( + """ + SELECT + SUM(deal_count) AS total_deals, + MIN(period_start_date) AS period_start, + MAX(period_start_date) AS period_end + FROM rosreestr_deals + WHERE quarter_cad_number = :cad_quarter + AND period_start_date >= (CURRENT_DATE - CAST(:window_interval AS interval)) + AND doc_type IN ('ДДУ', 'ДКП') + """ + ), + { + "cad_quarter": cad_quarter, + "window_interval": f"{months_window} months", + }, + ) + .mappings() + .first() + ) + except Exception: + logger.warning("velocity: rosreestr_fallback query failed for cad_quarter=%s", cad_quarter) + return None + + if row is None or not row["total_deals"] or int(row["total_deals"]) == 0: + return None + + total_deals = int(row["total_deals"]) + # Сделок/мес — грубый аналог velocity (без м², только count). + # Умножаем на 45 м² (эмпирика) для совместимости с м²/мес единицами. + avg_area_per_deal = 45.0 # м² — консервативная оценка для апартаментов ЕКБ + monthly_velocity_sqm = (total_deals * avg_area_per_deal) / months_window + + # Нормализация относительно ekb_median (один ЖК × 2). + velocity_score = min(1.0, max(0.0, monthly_velocity_sqm / (ekb_median * 2.0))) + + # Confidence — rosreestr данные менее детализированы, чем Objective. + rr_confidence: Literal["high", "medium", "low"] + if total_deals >= 50: + rr_confidence = "medium" # max medium для rosreestr — нет комнатности + else: + rr_confidence = "low" + + period_start_date = row["period_start"] + period_end_date = row["period_end"] + period_start = period_start_date.strftime("%Y-%m") if period_start_date else "" + period_end = period_end_date.strftime("%Y-%m") if period_end_date else "" + + logger.info( + "velocity: rosreestr_fallback success cad_quarter=%s" + " total_deals=%d window=%dm velocity=%.1f sqm/mon", + cad_quarter, + total_deals, + months_window, + monthly_velocity_sqm, + ) + + return VelocityResult( + competitors_count=n_comps, + monthly_velocity_sqm=monthly_velocity_sqm, + ekb_median_sqm=ekb_median, + velocity_score=velocity_score, + confidence=rr_confidence, + months_observed=months_window, + period_start=period_start, + period_end=period_end, + sample_competitors=sample_competitors, + by_room_bucket={}, # rosreestr не даёт room_bucket + velocity_data_available=True, + velocity_source="rosreestr_fallback", ) diff --git a/backend/app/services/site_finder/weight_profiles.py b/backend/app/services/site_finder/weight_profiles.py index 7f3f0ad2..08a253d1 100644 --- a/backend/app/services/site_finder/weight_profiles.py +++ b/backend/app/services/site_finder/weight_profiles.py @@ -17,6 +17,7 @@ from __future__ import annotations import json import logging +import math from datetime import datetime from typing import Any @@ -25,7 +26,11 @@ from sqlalchemy import text logger = logging.getLogger(__name__) -# Allowed POI categories — mirrors _POI_WEIGHTS keys in api/v1/parcels.py +# Sentinel user_id для системных preset-профилей (не привязаны к реальному пользователю). +# Seed: data/sql/100_user_weight_profiles_default_seed.sql +SYSTEM_USER_ID: str = "__system__" + +# Allowed POI categories — single source of truth; imported by api/v1/parcels.py ALLOWED_CATEGORIES: set[str] = { "school", "kindergarten", @@ -44,7 +49,7 @@ ALLOWED_CATEGORIES: set[str] = { MIN_WEIGHT: float = -2.0 MAX_WEIGHT: float = 3.0 -# System defaults — keep in sync with _POI_WEIGHTS in parcels.py +# System defaults — single source of truth; imported as _POI_WEIGHTS by api/v1/parcels.py _SYSTEM_POI_WEIGHTS: dict[str, float] = { "school": 1.5, "kindergarten": 1.5, @@ -73,6 +78,17 @@ _SELECT_BY_USER = f""" ORDER BY is_default DESC, id ASC """ +_SELECT_BY_USER_WITH_SYSTEM = f""" + SELECT {_SELECT_COLS} + FROM user_weight_profiles + WHERE user_id = :user_id + OR user_id = :system_user_id + ORDER BY + CASE WHEN user_id = :system_user_id THEN 1 ELSE 0 END ASC, + is_default DESC, + id ASC +""" + _SELECT_BY_ID = f""" SELECT {_SELECT_COLS} FROM user_weight_profiles @@ -115,7 +131,7 @@ def _validate_weights_dict(v: dict[str, float]) -> dict[str, float]: for k, w in v.items(): if not isinstance(w, int | float): raise ValueError(f"Weight for '{k}' must be number, got {type(w).__name__}") - if w < MIN_WEIGHT or w > MAX_WEIGHT: + if not math.isfinite(w) or w < MIN_WEIGHT or w > MAX_WEIGHT: raise ValueError(f"Weight for '{k}' = {w} out of bounds [{MIN_WEIGHT}, {MAX_WEIGHT}]") return v @@ -190,6 +206,24 @@ def list_profiles(db: Any, user_id: str) -> list[WeightProfile]: return [_row_to_profile(r) for r in rows] +def list_profiles_with_system(db: Any, user_id: str) -> list[WeightProfile]: + """Вернуть профили пользователя + системные preset-профили. + + Пользовательские профили идут первыми (default сверху), затем системные + presets (Эконом, Комфорт, Бизнес). Предназначен для endpoint с + include_system=true — UI dropdown видит и пользовательские, и preset. + """ + rows = ( + db.execute( + text(_SELECT_BY_USER_WITH_SYSTEM), + {"user_id": user_id, "system_user_id": SYSTEM_USER_ID}, + ) + .mappings() + .all() + ) + return [_row_to_profile(r) for r in rows] + + def get_profile(db: Any, user_id: str, profile_id: int) -> WeightProfile | None: """Вернуть профиль по id (scoped к пользователю).""" row = ( diff --git a/backend/app/templates/parcel_snapshot.html b/backend/app/templates/parcel_snapshot.html new file mode 100644 index 00000000..b8e9ae74 --- /dev/null +++ b/backend/app/templates/parcel_snapshot.html @@ -0,0 +1,240 @@ + + + + + Карточка участка {{ cad_num }} + + + + + +
+
+

GenDesign — Карточка участка

+
Данные НСПД / ЕГРНsource: cad_parcels. Не является официальной выпиской ЕГРН.
+
+
+ {{ cad_num }}
+ {{ district or '—' }}
+ {{ address or '—' }} +
+
+ + +
Основные характеристики
+
+
+
Площадь
+
{{ area_ha }} га
+
+
+
Кадастровая стоимость
+
{{ cadastral_cost }}
+
+
+
Категория земель
+
{{ land_category or '—' }}
+
+
+
ВРИ
+
{{ vri or '—' }}
+
+
+
Последнее обновление
+
{{ last_update or '—' }}
+
+
+ + +
Ближайшая инфраструктура (топ-7 по взвешенному баллу)
+{% if poi_items %} + + + + + + + + + + + + {% for poi in poi_items %} + + + + + + + + {% endfor %} + +
КатегорияНазваниеРасстояниеПешкомБалл
{{ poi.category_ru }}{{ poi.name or '—' }}{{ poi.distance_m }} м{{ poi.walk_min }} мин{{ poi.weighted_score }}
+{% else %} +

POI в радиусе 1 км не найдены.

+{% endif %} + + +
Конкуренты в радиусе 3 км (топ {{ competitors|length }})
+{% if competitors %} + + + + + + + + + + + + {% for c in competitors %} + + + + + + + + {% endfor %} + +
ЖК / ОбъектЗастройщикКлассКвартирРасстояние
{{ c.comm_name or '—' }}{{ c.dev_name or '—' }}{{ c.obj_class or '—' }}{{ c.flat_count or '—' }}{{ c.distance_m }} м
+{% else %} +

Конкурентов в радиусе 3 км не обнаружено.

+{% endif %} + +
+ Не является выпиской из ЕГРН. Данные носят аналитический характер. + Для официальной выписки: rosreestr.gov.ru +
+ + +
+ gendsgn.ru — GenDesign Analytics + Сформировано: {{ generated_at }} +
+ + + diff --git a/backend/app/workers/beat_schedule.py b/backend/app/workers/beat_schedule.py index e7ce5146..f474e88c 100644 --- a/backend/app/workers/beat_schedule.py +++ b/backend/app/workers/beat_schedule.py @@ -219,23 +219,49 @@ def build_beat_schedule() -> dict: "options": {"queue": "celery"}, } - # NSPD quarter dump refresh — DISABLED 2026-05-14 per Bug_NSPD_WMS_NotBulk - # post-mortem (vault: fixes/Bug_NSPD_WMS_NotBulk_2026_May14.md). + # #105 Phase 4: ЕКБ РНС/РВЭ — ежемесячно 1-го числа в 05:00 МСК (02:00 UTC) + schedule["ekburg-permits-monthly"] = { + "task": "tasks.ekburg_permits_sync.refresh_all", + "schedule": _parse_cron("0 2 1 * *"), + "options": {"queue": "celery"}, + } + + # ПЗЗ территориальных зон ЕКБ из PKK6 ArcGIS — ежемесячно 1-го числа в 03:00 МСК (00:00 UTC). + # PKK6 нестабилен под нагрузкой, но данные ПЗЗ меняются редко — раз в месяц достаточно. + # Task: tasks/pzz_sync.py → sync_pzz_zones_ekb. + # Admin trigger: POST /api/v1/admin/scrape/pzz-sync. + # Ref: issue #233 (pzz_zones_ekb = 0 rows, задача никогда не запускалась автоматически). + schedule["pzz-sync-monthly"] = { + "task": "tasks.pzz_sync.sync_pzz_zones_ekb", + "schedule": _parse_cron("0 0 1 * *"), + "options": {"queue": "celery"}, + } + + # Catalog-object scrape — наполняет ~25 NULL колонок domrf_kn_objects из SSR-страниц. + # kn-API не отдаёт wall_type, energy_eff, ceiling_height_m, parking_* и т.д. + # Вторник 04:00 UTC. batch 300/run → 1532 объекта за ~5 недель полного обновления. + schedule["scrape-kn-catalog-objects-weekly"] = { + "task": "tasks.scrape_kn_catalog_objects.scrape_kn_catalog_objects", + "schedule": _parse_cron("0 4 * * 2"), # Tuesday 04:00 UTC + "kwargs": {"region_code": 66, "max_objects": 300}, + "options": {"queue": "celery"}, + } + + # NSPD quarter dump refresh — re-enabled 2026-05-17 после Sub-PR B (#260) + # переключения search_by_quarter на grid-walk. Foundation (#247) + integration + # (#260) теперь возвращают полноценные dumps (territorial_zones, ЗОУИТ, risk + # zones, engineering structures) вместо 0-3 features из single-pixel WMS. # - # Action item #1: disable beat schedule до Sprint 2 fix (grid sampling - # rewrite). harvest_quarter в текущей реализации пишет почти пустые dumps - # из-за single-pixel WMS GetFeatureInfo bug. Запуск Mon 04:00 МСК потратит - # rate-limit budget и заполнит nspd_quarter_dumps мусором. - # - # Task code остаётся в tasks/nspd_sync.py — re-enable после Sprint 2 grid - # sampling rewrite (см. Bug_NSPD_WMS_NotBulk_2026_May14 → Sprint 2 fix-strategy). - # До тех пор harvest_quarter можно вызывать вручную через admin endpoint. - # - # schedule["nspd-harvest-stale-quarters"] = { - # "task": "tasks.nspd_sync.harvest_stale_quarters", - # "schedule": _parse_cron("0 4 * * mon"), - # "kwargs": {"region_code": 66, "max_age_days": 90, "batch_size": 50}, - # "options": {"queue": "celery"}, - # } + # Schedule: Mon 04:00 МСК (01:00 UTC). batch_size=50 ограничивает fanout per + # tick — при 11k+ кварталов в cad_quarters_geom полный backfill займёт + # несколько недель, но защищает от WAF rate-limit burst. + # max_age_days=90 — refresh свежее квартала; новые / отсутствующие dumps + # тоже попадают через harvest_stale_quarters (берёт NULL fetched_at). + schedule["nspd-harvest-stale-quarters"] = { + "task": "tasks.nspd_sync.harvest_stale_quarters", + "schedule": _parse_cron("0 1 * * mon"), # 01:00 UTC = 04:00 МСК + "kwargs": {"region_code": 66, "max_age_days": 90, "batch_size": 50}, + "options": {"queue": "celery"}, + } return schedule diff --git a/backend/app/workers/celery_app.py b/backend/app/workers/celery_app.py index 9bf49e43..112c2fa1 100644 --- a/backend/app/workers/celery_app.py +++ b/backend/app/workers/celery_app.py @@ -5,19 +5,51 @@ Worker lifecycle hooks (process_init, worker_ready) → app/workers/lifecycle.py """ import logging +import os +import sentry_sdk from celery import Celery +from sentry_sdk.integrations.celery import CeleryIntegration +from sentry_sdk.integrations.httpx import HttpxIntegration +from sentry_sdk.integrations.logging import LoggingIntegration +from sentry_sdk.integrations.sqlalchemy import SqlalchemyIntegration from app.core.config import settings +from app.observability.sentry_scrub import scrub_sensitive_query logger = logging.getLogger(__name__) +# SDK инициализируется в обоих процессах (FastAPI-сервер и Celery-воркер), +# чтобы события из тасков попадали в GlitchTip. SDK безопасен для двойного +# вызова — повторный sentry_sdk.init() в одном процессе заменяет клиента. +if settings.glitchtip_dsn: + sentry_sdk.init( + dsn=settings.glitchtip_dsn, + environment=settings.environment, + release=os.getenv("GIT_SHA") or os.getenv("SENTRY_RELEASE") or "unknown", + traces_sample_rate=settings.glitchtip_traces_sample_rate, + profiles_sample_rate=0.0, + send_default_pii=False, + before_send_transaction=scrub_sensitive_query, + integrations=[ + CeleryIntegration(monitor_beat_tasks=True), + SqlalchemyIntegration(), + HttpxIntegration(), + LoggingIntegration(level=logging.INFO, event_level=logging.ERROR), + ], + ) + logger.info( + "GlitchTip SDK initialised in Celery worker (env=%s)", + settings.environment, + ) + celery_app = Celery( "gendesign", broker=settings.redis_url, backend=settings.redis_url, include=[ "app.workers.tasks.scrape_kn", + "app.workers.tasks.scrape_kn_catalog_objects", "app.workers.tasks.refresh_analytics", "app.workers.tasks.scrape_objective", "app.workers.tasks.objective_etl", @@ -27,6 +59,7 @@ celery_app = Celery( "app.workers.tasks.noise_sync", "app.workers.tasks.pzz_sync", "app.workers.tasks.scrape_cadastre", + "app.workers.tasks.ekburg_permits_sync", ], ) celery_app.conf.timezone = "Europe/Moscow" diff --git a/backend/app/workers/tasks/ekburg_permits_sync.py b/backend/app/workers/tasks/ekburg_permits_sync.py new file mode 100644 index 00000000..9f9d6fc2 --- /dev/null +++ b/backend/app/workers/tasks/ekburg_permits_sync.py @@ -0,0 +1,151 @@ +"""Celery task: monthly refresh ekburg permits xlsx (Issue #105). + +Запускается через beat (1-е число каждого месяца в 05:00 МСК). +Добавить в beat_schedule через job_settings или hardcoded entry (Phase 2 followup). +""" + +from __future__ import annotations + +import json +import logging + +from sqlalchemy import text +from sqlalchemy.orm import Session + +from app.core.db import SessionLocal +from app.services.scrapers.ekburg_permits import ( + EKBURG_PERMITS_URLS, + EkburgPermitsClient, + PermitRow, +) +from app.workers.celery_app import celery_app + +logger = logging.getLogger(__name__) + + +def _upsert_permit(db: Session, row: PermitRow) -> None: + """UPSERT одной строки разрешения в ekburg_construction_permits.""" + db.execute( + text(""" + INSERT INTO ekburg_construction_permits ( + permit_type, permit_number, + issue_date, expiry_date, + developer_inn, developer_name, + object_name, object_type, + construction_address, cadastral_number, + total_area_sqm, living_area_sqm, living_area_fact_sqm, + rve_number, rve_date, + raw_coord_x, raw_coord_y, + source_year, source_url, raw_row + ) + VALUES ( + :permit_type, :permit_number, + :issue_date, :expiry_date, + :developer_inn, :developer_name, + :object_name, :object_type, + :construction_address, :cadastral_number, + :total_area_sqm, :living_area_sqm, :living_area_fact_sqm, + :rve_number, :rve_date, + :raw_coord_x, :raw_coord_y, + :source_year, :source_url, CAST(:raw_row AS jsonb) + ) + ON CONFLICT (permit_type, permit_number) DO UPDATE SET + issue_date = EXCLUDED.issue_date, + expiry_date = EXCLUDED.expiry_date, + developer_inn = EXCLUDED.developer_inn, + developer_name = EXCLUDED.developer_name, + object_name = EXCLUDED.object_name, + object_type = EXCLUDED.object_type, + construction_address = EXCLUDED.construction_address, + cadastral_number = EXCLUDED.cadastral_number, + total_area_sqm = EXCLUDED.total_area_sqm, + living_area_sqm = EXCLUDED.living_area_sqm, + living_area_fact_sqm = EXCLUDED.living_area_fact_sqm, + rve_number = EXCLUDED.rve_number, + rve_date = EXCLUDED.rve_date, + raw_coord_x = EXCLUDED.raw_coord_x, + raw_coord_y = EXCLUDED.raw_coord_y, + raw_row = EXCLUDED.raw_row, + fetched_at = NOW() + """), + { + "permit_type": row.permit_type, + "permit_number": row.permit_number, + "issue_date": row.issue_date, + "expiry_date": row.expiry_date, + "developer_inn": row.developer_inn, + "developer_name": row.developer_name, + "object_name": row.object_name, + "object_type": row.object_type, + "construction_address": row.construction_address, + "cadastral_number": row.cadastral_number, + "total_area_sqm": row.total_area_sqm, + "living_area_sqm": row.living_area_sqm, + "living_area_fact_sqm": row.living_area_fact_sqm, + "rve_number": row.rve_number, + "rve_date": row.rve_date, + "raw_coord_x": row.raw_coord_x, + "raw_coord_y": row.raw_coord_y, + "source_year": row.source_year, + "source_url": row.source_url, + "raw_row": json.dumps(row.raw_row, ensure_ascii=False), + }, + ) + + +@celery_app.task(name="tasks.ekburg_permits_sync.refresh_year", queue="celery") +def refresh_year(year: int) -> dict[str, int]: + """Скачать + распарсить + upsert РНС/РВЭ за один год. + + Возвращает {"inserted": N, "errors": N}. + """ + inserted = 0 + errors = 0 + + url = EKBURG_PERMITS_URLS.get(year) + if not url: + logger.error("ekburg_permits_sync: no URL for year=%d", year) + return {"inserted": 0, "errors": 1} + + with EkburgPermitsClient() as client: + try: + content = client.download_xlsx(year) + except Exception as exc: + logger.error("ekburg_permits_sync: download failed year=%d: %s", year, exc) + return {"inserted": 0, "errors": 1} + + with SessionLocal() as db: + for row in client.parse_xlsx(content, year, url): + try: + with db.begin_nested(): + _upsert_permit(db, row) + inserted += 1 + except Exception as exc: + logger.warning( + "ekburg_permits_sync: upsert failed %s/%s year=%d: %s", + row.permit_type, + row.permit_number, + year, + exc, + ) + errors += 1 + db.commit() + + logger.info( + "ekburg_permits_sync: year=%d done inserted=%d errors=%d", + year, + inserted, + errors, + ) + return {"inserted": inserted, "errors": errors} + + +@celery_app.task(name="tasks.ekburg_permits_sync.refresh_all", queue="celery") +def refresh_all() -> dict[str, dict[str, int]]: + """Обновить все 5 лет (2022-2026). Планируется через Celery beat ежемесячно.""" + results: dict[str, dict[str, int]] = {} + for year in sorted(EKBURG_PERMITS_URLS.keys()): + logger.info("ekburg_permits_sync: starting year=%d", year) + results[str(year)] = refresh_year(year) + logger.info("ekburg_permits_sync: refresh_all done: %s", results) + return results diff --git a/backend/app/workers/tasks/nspd_denorm_backfill.py b/backend/app/workers/tasks/nspd_denorm_backfill.py new file mode 100644 index 00000000..c0788328 --- /dev/null +++ b/backend/app/workers/tasks/nspd_denorm_backfill.py @@ -0,0 +1,96 @@ +"""One-shot backfill: denormalize all existing nspd_quarter_dumps → nspd_parcels/buildings. + +Запускается вручную через admin endpoint POST /api/v1/admin/etl/nspd-denorm-backfill +или через Celery CLI: + celery -A app.workers.celery_app call nspd_denorm.backfill_all_dumps + +Идемпотентен: повторный запуск обновляет строки через ON CONFLICT DO UPDATE. +""" + +from __future__ import annotations + +import logging + +from celery import shared_task +from sqlalchemy import text + +from app.core.db import SessionLocal +from app.services.scrapers.nspd_denorm import denorm_dump + +logger = logging.getLogger(__name__) + + +@shared_task( + name="nspd_denorm.backfill_all_dumps", + bind=True, + soft_time_limit=3600, # 1 час — много кварталов × ~10ms per parcel + max_retries=0, # One-shot, не retry +) +def backfill_all_dumps(self: object, *, limit: int | None = None) -> dict[str, int]: # type: ignore[misc] + """Backfill all existing nspd_quarter_dumps → nspd_parcels / nspd_buildings. + + Читает все строки nspd_quarter_dumps (или limit строк), и для каждой вызывает + denorm_dump. Возвращает агрегированные счётчики по всем кварталам. + + Args: + limit: если задан — обработать только первые N кварталов (для тестирования). + + Returns: + dict { + "parcels": суммарно вставлено/обновлено parcel строк, + "buildings": суммарно вставлено/обновлено building строк, + "errors": суммарно пропусков (нет cad_num, DB ошибки), + "quarters_processed": количество обработанных кварталов, + } + """ + totals: dict[str, int] = { + "parcels": 0, + "buildings": 0, + "errors": 0, + "quarters_processed": 0, + } + + db = SessionLocal() + try: + sql = ( + "SELECT quarter_cad, features_json " + "FROM nspd_quarter_dumps " + "WHERE total_features > 0 " + "ORDER BY quarter_cad" + ) + if limit is not None: + # Используем параметр через text() чтобы избежать SQL-injection. + rows = db.execute( + text(sql + " LIMIT CAST(:lim AS integer)"), {"lim": int(limit)} + ).fetchall() + else: + rows = db.execute(text(sql)).fetchall() + + logger.info("backfill_all_dumps: starting, quarters=%d limit=%s", len(rows), limit) + + for row in rows: + quarter_cad: str = row[0] + features_json = row[1] + # features_json — уже list[dict] через psycopg v3 JSON decoding. + features: list[dict] = features_json if isinstance(features_json, list) else [] + + try: + counts = denorm_dump(db, quarter_cad=quarter_cad, features=features) + totals["parcels"] += counts["parcels"] + totals["buildings"] += counts["buildings"] + totals["errors"] += counts["errors"] + totals["quarters_processed"] += 1 + except Exception as e: + logger.warning("backfill_all_dumps: quarter=%s failed: %s", quarter_cad, e) + totals["errors"] += 1 + # Продолжаем с остальными кварталами + try: + db.rollback() + except Exception: + pass + + logger.info("backfill_all_dumps: done totals=%s", totals) + finally: + db.close() + + return totals diff --git a/backend/app/workers/tasks/nspd_sync.py b/backend/app/workers/tasks/nspd_sync.py index dc7eda85..a7d4989a 100644 --- a/backend/app/workers/tasks/nspd_sync.py +++ b/backend/app/workers/tasks/nspd_sync.py @@ -33,6 +33,7 @@ from app.services.scrapers.nspd_client import ( NspdLiteWafError, QuarterDump, ) +from app.services.scrapers.nspd_denorm import denorm_dump from app.workers.celery_app import celery_app logger = logging.getLogger(__name__) @@ -82,6 +83,12 @@ def _build_features_json(dump: QuarterDump) -> list[dict[str, Any]]: for feat in features: out.append(_feat_to_dict(layer_name, feat)) + # TIER 4 opportunity groups — keys: auction_parcels, scheme_parcels, ... + for short_name, features in dump.opportunity.items(): + layer_name = f"opportunity_{short_name}" + for feat in features: + out.append(_feat_to_dict(layer_name, feat)) + return out @@ -95,6 +102,16 @@ def _build_risks_count(dump: QuarterDump) -> int: return sum(len(v) for v in dump.risks.values()) +def _build_opportunity_count(dump: QuarterDump) -> int: + """Сумма features по всем TIER 4 opportunity слоям (issue #94 PR2).""" + return sum(len(v) for v in dump.opportunity.values()) + + +def _build_has_auction_parcels(dump: QuarterDump) -> bool: + """True если квартал содержит >= 1 feature auction_parcels (layer 37299).""" + return len(dump.opportunity.get("auction_parcels", [])) > 0 + + # ── UPSERT helper ───────────────────────────────────────────────────────────── _UPSERT_SQL = text( @@ -103,18 +120,26 @@ _UPSERT_SQL = text( quarter_cad, quarter_geom, bbox_3857, parcels_count, buildings_count, territorial_zones_count, red_lines_count, engineering_count, zouit_count, risks_count, total_features, + has_auction_parcels, opportunity_count, features_json, layers_fetched, fetched_at_utc, harvest_duration_ms, harvest_error, region_code ) VALUES ( :quarter_cad, - CASE WHEN :geom_json IS NULL THEN NULL - ELSE ST_Multi(ST_Transform(ST_SetSRID(ST_GeomFromGeoJSON(:geom_json), 3857), 4326)) + CASE WHEN CAST(:geom_json AS text) IS NULL THEN NULL + ELSE ST_Multi(ST_Transform( + ST_SetSRID(ST_GeomFromGeoJSON(CAST(:geom_json AS text)), 3857), 4326)) END, - CASE WHEN :bbox_xmin IS NULL THEN NULL - ELSE ST_MakeEnvelope(:bbox_xmin, :bbox_ymin, :bbox_xmax, :bbox_ymax, 3857) + CASE WHEN CAST(:bbox_xmin AS double precision) IS NULL THEN NULL + ELSE ST_MakeEnvelope( + CAST(:bbox_xmin AS double precision), + CAST(:bbox_ymin AS double precision), + CAST(:bbox_xmax AS double precision), + CAST(:bbox_ymax AS double precision), + 3857) END, :parcels_count, :buildings_count, :territorial_zones_count, :red_lines_count, :engineering_count, :zouit_count, :risks_count, :total_features, + :has_auction_parcels, :opportunity_count, CAST(:features_json AS jsonb), CAST(:layers_fetched AS text[]), CAST(:fetched_at_utc AS timestamptz), @@ -133,6 +158,8 @@ _UPSERT_SQL = text( zouit_count = EXCLUDED.zouit_count, risks_count = EXCLUDED.risks_count, total_features = EXCLUDED.total_features, + has_auction_parcels = EXCLUDED.has_auction_parcels, + opportunity_count = EXCLUDED.opportunity_count, features_json = EXCLUDED.features_json, layers_fetched = EXCLUDED.layers_fetched, fetched_at_utc = EXCLUDED.fetched_at_utc, @@ -178,6 +205,8 @@ def _upsert_dump( "engineering_count": len(dump.engineering_structures), "zouit_count": _build_zouit_count(dump), "risks_count": _build_risks_count(dump), + "has_auction_parcels": _build_has_auction_parcels(dump), + "opportunity_count": _build_opportunity_count(dump), "total_features": dump.total_features, "features_json": json.dumps(features_json or [], ensure_ascii=False), "layers_fetched": list(dump.layers_fetched), @@ -202,6 +231,8 @@ def _upsert_dump( "engineering_count": 0, "zouit_count": 0, "risks_count": 0, + "has_auction_parcels": False, + "opportunity_count": 0, "total_features": 0, "features_json": "[]", "layers_fetched": [], @@ -227,7 +258,9 @@ def _upsert_dump( bind=True, name="tasks.nspd_sync.harvest_quarter", max_retries=3, - soft_time_limit=120, # 2 мин — worst-case 22 HTTP × ~600ms = ~13s + margin + soft_time_limit=600, # 10 мин — grid-walk: 11 layers × 49 cells × ~70ms ≈ 40s + retries + time_limit=900, # 15 мин hard kill — safety net (PR #260 re-review): task может игнорить + # SoftTimeLimitExceeded → процесс не освобождается. Hard limit гарантирует worker recovery. autoretry_for=(NspdLiteWafError,), retry_backoff=True, retry_backoff_max=120, @@ -239,20 +272,26 @@ def harvest_quarter( region_code: int = 66, include_zouit: bool = True, include_risks: bool = False, + include_opportunity: bool = False, ) -> dict[str, Any]: """Single-quarter harvest. NSPDClient.search_by_quarter → UPSERT nspd_quarter_dumps. Идемпотентен: повторный вызов обновляет строку (ON CONFLICT DO UPDATE). WAF 403/429 → autoretry с exponential backoff (max 3 попытки). Другие исключения → запись harvest_error в строку, return error dict (не raise). + + Args: + include_opportunity: Фетчить TIER 4 opportunity layers (+5 HTTP запросов). """ t0 = time.monotonic() logger.info( - "harvest_quarter start: cad=%s region=%d include_zouit=%s include_risks=%s", + "harvest_quarter start: cad=%s region=%d include_zouit=%s " + "include_risks=%s include_opportunity=%s", quarter_cad, region_code, include_zouit, include_risks, + include_opportunity, ) client = NSPDClient() @@ -264,12 +303,39 @@ def harvest_quarter( quarter_cad, include_zouit=include_zouit, include_risks=include_risks, + include_opportunity=include_opportunity, ) features_json = _build_features_json(dump) duration_ms = int((time.monotonic() - t0) * 1000) _upsert_dump(quarter_cad, region_code, dump, features_json, duration_ms, None) + # Inline denorm: разложить parcels/buildings из dump в nspd_parcels/nspd_buildings. + # Ошибка denorm не должна фейлить весь harvest — только warning. + try: + denorm_db = SessionLocal() + try: + denorm_counts = denorm_dump( + denorm_db, + quarter_cad=quarter_cad, + features=features_json or [], + ) + logger.info( + "harvest_quarter denorm: cad=%s parcels=%d buildings=%d errors=%d", + quarter_cad, + denorm_counts["parcels"], + denorm_counts["buildings"], + denorm_counts["errors"], + ) + finally: + denorm_db.close() + except Exception as denorm_exc: + logger.warning( + "harvest_quarter denorm failed (non-fatal): cad=%s error=%s", + quarter_cad, + denorm_exc, + ) + logger.info( "harvest_quarter done: cad=%s region=%d duration=%dms total=%d", quarter_cad, @@ -389,7 +455,14 @@ def harvest_stale_quarters( enqueued = 0 for cad in stale_cads: try: - harvest_quarter.apply_async(args=[cad, region_code]) + harvest_quarter.apply_async( + args=[cad, region_code], + kwargs={ + "include_zouit": True, + "include_risks": True, + "include_opportunity": True, + }, + ) enqueued += 1 except Exception as e: logger.warning("harvest_stale_quarters: enqueue failed for cad=%s: %s", cad, e) diff --git a/backend/app/workers/tasks/scrape_kn_catalog_objects.py b/backend/app/workers/tasks/scrape_kn_catalog_objects.py new file mode 100644 index 00000000..b956e85b --- /dev/null +++ b/backend/app/workers/tasks/scrape_kn_catalog_objects.py @@ -0,0 +1,170 @@ +"""Celery task: periodic catalog-object scrape для DOM.РФ. + +Дополняет ~25 NULL колонок в domrf_kn_objects из SSR-страниц каталога. +kn-API эти поля не возвращает — они только на публичных страницах объектов. + +Selector logic: + - catalog_scraped_at IS NULL — "новый" объект, всегда грузим + - DATE(catalog_scraped_at) < CURRENT_DATE — сегодня ещё не обновлялся, грузим + - DATE(catalog_scraped_at) = CURRENT_DATE — уже сегодня обновлён, пропускаем + - force=True — игнорирует фильтр, загружает все объекты последнего snapshot + +Beat schedule: вторник 04:00 UTC (в beat_schedule.py). +""" + +from __future__ import annotations + +import asyncio +import logging +from datetime import date +from typing import Any + +from sqlalchemy import text + +from app.core.db import SessionLocal +from app.workers.celery_app import celery_app + +logger = logging.getLogger(__name__) + +# Запрос для выбора объектов на скрап. +# Возвращает obj_id + snapshot_date одним запросом, чтобы избежать race condition: +# если между двумя запросами kn-scraper запишет новый snapshot — UPDATE по старой +# snapshot_date не затронет ни одной строки. MAX subquery ограничена тем же +# region_cd чтобы не захватить snapshot другого региона. +# +# Фильтр (:force = false): берём только те, что ещё не обновлялись сегодня +# (catalog_scraped_at IS NULL — никогда не скрапились, либо DATE(...) < today). +# При :force = true фильтр снимается — грузим все объекты последнего snapshot. +_SELECT_TARGETS_SQL = text( + """ + SELECT obj_id, snapshot_date + FROM domrf_kn_objects + WHERE region_cd = :region_code + AND ( + CAST(:force AS boolean) + OR catalog_scraped_at IS NULL + OR DATE(catalog_scraped_at) < CURRENT_DATE + ) + AND snapshot_date = ( + SELECT MAX(snapshot_date) + FROM domrf_kn_objects + WHERE region_cd = :region_code + ) + ORDER BY catalog_scraped_at NULLS FIRST + LIMIT :max_objects + """ +) + +# Лимит по умолчанию если max_objects не задан явно. +_DEFAULT_MAX_OBJECTS = 300 + + +@celery_app.task( + bind=True, + name="tasks.scrape_kn_catalog_objects.scrape_kn_catalog_objects", + time_limit=3600, +) +def scrape_kn_catalog_objects( + self: Any, + region_code: int = 66, + max_objects: int | None = None, + force: bool = False, +) -> dict[str, Any]: + """Periodic catalog-object scrape. + + Args: + region_code: Код региона (ОКАТО prefix). Default 66 = Свердловская обл. + max_objects: Максимум объектов за один run. Default 300. + force: Если True — игнорирует фильтр "уже сегодня обновлён" и грузит + все объекты последнего snapshot (admin "Загрузить все"). По умолчанию + False — пропускает то, что уже скраплено сегодня. + + Returns: + dict с ключами: region_code, snapshot_date, obj_ids_count, + processed, succeeded, failed, skipped. + + Concurrency: + No Redis lock — consistent with sibling tasks (scrape_kn_region etc.). + Beat is configured for non-overlapping fire (Tuesday 04:00 UTC, ~5min run), + so concurrent execution is extremely rare. If it occurs: + - UPDATE is idempotent (COALESCE, catalog_scraped_at = NOW()) + - Max risk: 2x WAF load on DOM.РФ for the same batch + - Both tasks complete; second update is no-op (catalog_scraped_at расхождение) + + Add Redis lock if WAF blocks observed or beat schedule changes to overlap. + """ + from app.services.scrapers.domrf_catalog_object import scrape_catalog_objects + + limit = max_objects if max_objects is not None else _DEFAULT_MAX_OBJECTS + + db = SessionLocal() + try: + rows = ( + db.execute( + _SELECT_TARGETS_SQL, + {"region_code": region_code, "max_objects": limit, "force": force}, + ) + .mappings() + .all() + ) + obj_ids: list[int] = [int(r["obj_id"]) for r in rows] + except Exception as exc: + logger.error("scrape_kn_catalog_objects: failed to fetch obj_ids: %s", exc) + db.close() + raise + + if not obj_ids: + logger.info( + "scrape_kn_catalog_objects: nothing to do for region=%d (force=%s)", + region_code, + force, + ) + db.close() + return { + "region_code": region_code, + "force": force, + "obj_ids_count": 0, + "processed": 0, + "succeeded": 0, + "failed": 0, + "skipped": 0, + } + + # snapshot_date берётся из первой строки результата — все строки одинаковые + # (WHERE snapshot_date = MAX(snapshot_date)). Это атомарно: один SELECT вместо двух, + # что устраняет race condition с kn-scraper. + snapshot_date_val: date = rows[0]["snapshot_date"] + + logger.info( + "scrape_kn_catalog_objects: region=%d snapshot_date=%s obj_ids=%d limit=%d force=%s", + region_code, + snapshot_date_val, + len(obj_ids), + limit, + force, + ) + + try: + stats = asyncio.run( + scrape_catalog_objects( + db=db, + obj_ids=obj_ids, + snapshot_date=snapshot_date_val, + region_code=region_code, + ) + ) + except Exception as exc: + logger.error("scrape_kn_catalog_objects: scrape failed: %s", exc) + raise + finally: + db.close() + + result: dict[str, Any] = { + "region_code": region_code, + "force": force, + "snapshot_date": str(snapshot_date_val), + "obj_ids_count": len(obj_ids), + **stats, + } + logger.info("scrape_kn_catalog_objects done: %s", result) + return result diff --git a/backend/app/workers/tasks/scrape_objective.py b/backend/app/workers/tasks/scrape_objective.py index 7e03dd1f..514fcf0b 100644 --- a/backend/app/workers/tasks/scrape_objective.py +++ b/backend/app/workers/tasks/scrape_objective.py @@ -99,9 +99,21 @@ def _save_raw( end_date: date | None, use_ddu: bool, use_dkp: bool, - payload: Any, + payload: Any | None = None, ) -> int: - body = json.dumps(payload, ensure_ascii=False) + """Сохраняет мета + payload в objective_raw_reports. + + payload=None допустим для stream-parsed отчётов (lots_pf 600+ МБ). + В этом случае payload_size=0 и payload=NULL (после миграции 79). + """ + if payload is not None: + body = json.dumps(payload, ensure_ascii=False) + payload_param = body + size = len(body.encode("utf-8")) + else: + payload_param = None + size = 0 + row = db.execute( text( """ @@ -126,8 +138,8 @@ def _save_raw( "end_date": end_date, "use_ddu": use_ddu, "use_dkp": use_dkp, - "payload": body, - "size": len(body.encode("utf-8")), + "payload": payload_param, + "size": size, }, ).scalar_one() db.commit() @@ -230,45 +242,44 @@ def sync_objective_group( ), ] + snap = date.today() try: for kind, fn_name, params, section, rtype, rname in jobs: try: - method = getattr(client, fn_name) - payload = method(**params) - n_requests += 1 - raw_id = _save_raw( - db, - run_id, - report_section=section, - report_type=rtype, - report_name=rname, - group_name=group, - complex_name=None, - start_date=params.get("start_date"), - end_date=params.get("end_date"), - use_ddu=params.get("use_ddu", True), - use_dkp=params.get("use_dkp", False), - payload=payload, - ) - reports_ok += 1 - - # Inline-нормализация в objective_corpus_room_month / lots / history. - snap = date.today() - try: - if kind == "corp_sum": - n = parser_mod.parse_corp_sum(payload, group, raw_id, db, dry_run=False) - rows_corpus_room += n - db.execute( - text( - "UPDATE objective_raw_reports " - " SET rows_extracted = :n WHERE raw_id = :rid" - ), - {"n": n, "rid": raw_id}, - ) - elif kind == "lots_pf": - n_lots, n_hist = parser_mod.parse_lots_pf( - payload, raw_id, snap, db, dry_run=False - ) + if kind == "lots_pf": + # lots_pf: 600+ МБ JSON → streaming через ijson, не грузим в RAM. + # payload пишем как NULL в objective_raw_reports (миграция 79). + raw_id = _save_raw( + db, + run_id, + report_section=section, + report_type=rtype, + report_name=rname, + group_name=group, + complex_name=None, + start_date=params.get("start_date"), + end_date=params.get("end_date"), + use_ddu=params.get("use_ddu", True), + use_dkp=params.get("use_dkp", False), + payload=None, + ) + n_requests += 1 + reports_ok += 1 + try: + with client.stream_report( + report_type="Поквартирные", + report_name="Лоты", + group_name=group, + use_ddu=params.get("use_ddu", True), + use_dkp=params.get("use_dkp") or False, + ) as resp: + n_lots, n_hist = parser_mod.parse_lots_pf_stream( + resp.iter_bytes(chunk_size=65536), + raw_id, + snap, + db, + dry_run=False, + ) rows_lots += n_lots rows_history += n_hist db.execute( @@ -278,18 +289,58 @@ def sync_objective_group( ), {"n": n_lots, "rid": raw_id}, ) - db.commit() - except Exception as parse_err: - # Парсинг упал — raw уже сохранён, можно re-parse позже. - db.rollback() - logger.exception( - "sync_objective_group: parser failed for %s/%s/%s raw_id=%s: %s", - section, - rtype, - rname, - raw_id, - parse_err, + db.commit() + except Exception as parse_err: + db.rollback() + logger.exception( + "sync_objective_group: lots_pf stream failed raw_id=%s: %s", + raw_id, + parse_err, + ) + else: + # corp_sum (7 МБ) — полный load как раньше + method = getattr(client, fn_name) + payload = method(**params) + n_requests += 1 + raw_id = _save_raw( + db, + run_id, + report_section=section, + report_type=rtype, + report_name=rname, + group_name=group, + complex_name=None, + start_date=params.get("start_date"), + end_date=params.get("end_date"), + use_ddu=params.get("use_ddu", True), + use_dkp=params.get("use_dkp", False), + payload=payload, ) + reports_ok += 1 + try: + if kind == "corp_sum": + n = parser_mod.parse_corp_sum( + payload, group, raw_id, db, dry_run=False + ) + rows_corpus_room += n + db.execute( + text( + "UPDATE objective_raw_reports " + " SET rows_extracted = :n WHERE raw_id = :rid" + ), + {"n": n, "rid": raw_id}, + ) + db.commit() + except Exception as parse_err: + db.rollback() + logger.exception( + "sync_objective_group: parser failed for %s/%s/%s " "raw_id=%s: %s", + section, + rtype, + rname, + raw_id, + parse_err, + ) _heartbeat( db, @@ -429,7 +480,6 @@ def sync_all_groups( logger.info("[%d/%d] sync_objective_group(group=%r) START", idx + 1, len(eff_groups), group) try: res = sync_objective_group( - self, group_name=group, triggered_by=f"{triggered_by}-multi", use_ddu=eff_use_ddu, diff --git a/backend/pyproject.toml b/backend/pyproject.toml index dbbbb3b0..1bd36e47 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -22,13 +22,15 @@ dependencies = [ "tenacity>=9.0.0", "pillow>=10.4.0", "weasyprint>=62.0", + "jinja2>=3.1.0", "ezdxf>=1.3.0", "openpyxl>=3.1.0", "pandas>=2.2.0", "numpy>=2.0.0", "scikit-learn>=1.5.0", - "sentry-sdk[fastapi]>=2.10.0", + "sentry-sdk[fastapi,celery,sqlalchemy,httpx]>=2.18.0", "rosreestr2coord>=5.0.0", + "ijson>=3.2.0", ] [dependency-groups] @@ -83,4 +85,5 @@ addopts = ["-m", "not prod_smoke"] markers = [ "slow: marks tests as slow (need real network, deselect with -m 'not slow')", "prod_smoke: production smoke tests against live https://gendsgn.ru (run only post-deploy with -m prod_smoke)", + "integration: phantom column gate tests requiring TEST_DATABASE_URL (SSH tunnel to prod Postgres)", ] diff --git a/backend/tests/api/v1/test_admin_ekburg_permits.py b/backend/tests/api/v1/test_admin_ekburg_permits.py new file mode 100644 index 00000000..a03c10be --- /dev/null +++ b/backend/tests/api/v1/test_admin_ekburg_permits.py @@ -0,0 +1,82 @@ +"""Тесты для POST /admin/scrape/ekburg-permits. + +Проверяет: +- валидный запрос без year → scope all_years_2022_2026, task_id в ответе +- валидный запрос с year=2026 → scope year_2026 +- year < 2022 или > 2030 → 422 +- отсутствие X-Admin-Token → 401/503 +""" + +from __future__ import annotations + +from unittest.mock import MagicMock, patch + +import pytest +from fastapi.testclient import TestClient + +from app.main import app + +ADMIN_TOKEN = "test-admin-token" +ADMIN_HEADERS = {"X-Admin-Token": ADMIN_TOKEN} +ENDPOINT = "/api/v1/admin/scrape/ekburg-permits" + + +def _mock_task(task_id: str = "fake-task-id-123") -> MagicMock: + result = MagicMock() + result.id = task_id + return result + + +@patch("app.core.config.settings.scrape_admin_token", ADMIN_TOKEN) +def test_trigger_refresh_all_returns_task_id() -> None: + """POST без year → refresh_all queued, scope=all_years_2022_2026.""" + mock_result = _mock_task("task-all-001") + + with ( + patch("app.workers.tasks.ekburg_permits_sync.refresh_all") as mock_refresh_all, + patch("app.workers.tasks.ekburg_permits_sync.refresh_year"), + ): + mock_refresh_all.apply_async.return_value = mock_result + client = TestClient(app) + response = client.post(ENDPOINT, json={}, headers=ADMIN_HEADERS) + + assert response.status_code == 200, response.text + body = response.json() + assert body["task_id"] == "task-all-001" + assert body["scope"] == "all_years_2022_2026" + assert "queued_at" in body + + +@patch("app.core.config.settings.scrape_admin_token", ADMIN_TOKEN) +def test_trigger_refresh_year_returns_task_id() -> None: + """POST year=2026 → refresh_year queued, scope=year_2026.""" + mock_result = _mock_task("task-year-002") + + with ( + patch("app.workers.tasks.ekburg_permits_sync.refresh_year") as mock_refresh_year, + patch("app.workers.tasks.ekburg_permits_sync.refresh_all"), + ): + mock_refresh_year.apply_async.return_value = mock_result + client = TestClient(app) + response = client.post(ENDPOINT, json={"year": 2026}, headers=ADMIN_HEADERS) + + assert response.status_code == 200, response.text + body = response.json() + assert body["task_id"] == "task-year-002" + assert body["scope"] == "year_2026" + + +@pytest.mark.parametrize("bad_year", [2021, 2031, 1999, 9999]) +@patch("app.core.config.settings.scrape_admin_token", ADMIN_TOKEN) +def test_trigger_invalid_year_returns_422(bad_year: int) -> None: + """year вне диапазона [2022, 2030] → 422 Unprocessable Entity.""" + client = TestClient(app) + response = client.post(ENDPOINT, json={"year": bad_year}, headers=ADMIN_HEADERS) + assert response.status_code == 422, f"year={bad_year} должен возвращать 422" + + +def test_trigger_no_token_returns_401_or_503() -> None: + """Без X-Admin-Token → 401 или 503.""" + client = TestClient(app) + response = client.post(ENDPOINT, json={}) + assert response.status_code in (401, 503), response.text diff --git a/backend/tests/api/v1/test_analyze_competitors_status.py b/backend/tests/api/v1/test_analyze_competitors_status.py new file mode 100644 index 00000000..490425d9 --- /dev/null +++ b/backend/tests/api/v1/test_analyze_competitors_status.py @@ -0,0 +1,122 @@ +"""Тесты: /analyze endpoint возвращает site_status + ready_dt в competitors[]. + +Mock-based — не требуют живой БД. +Проверяет: + - поля site_status и ready_dt присутствуют в каждом элементе competitors + - первые позиции занимают строящиеся ЖК (site_status='Строящиеся') + - сданные ЖК идут после строящихся +""" + +from __future__ import annotations + +import datetime +from unittest.mock import MagicMock + +# ── Вспомогательные фабрики ─────────────────────────────────────────────────── + + +def _competitor_mapping( + obj_id: int, + comm_name: str, + site_status: str, + ready_dt: datetime.date | None, + flat_count: int, + distance_m: float = 500.0, +) -> MagicMock: + """Имитирует sqlalchemy RowMapping для строки конкурента.""" + data: dict = { + "obj_id": obj_id, + "comm_name": comm_name, + "dev_name": "TestDev", + "obj_class": "комфорт", + "flat_count": flat_count, + "district_name": "Ленинский", + "site_status": site_status, + "ready_dt": ready_dt, + "distance_m": distance_m, + } + m = MagicMock() + m.__getitem__ = lambda self, k: data[k] + m.keys = lambda: data.keys() + m.__iter__ = lambda self: iter(data) + m.items = lambda: data.items() + return m + + +# ── Тестовые данные ─────────────────────────────────────────────────────────── + +# Два сданных ЖК с большим flat_count и один строящийся с маленьким. +# До фикса ORDER BY flat_count DESC → сданные шли первыми. +_ROWS_MIXED = [ + _competitor_mapping( + 1, "ПИК Космонавтов 11 корп.1", "Сданные", datetime.date(2022, 6, 1), 800, 300.0 + ), + _competitor_mapping( + 2, "ПИК Космонавтов 11 корп.2", "Сданные", datetime.date(2023, 3, 1), 750, 310.0 + ), + _competitor_mapping( + 3, "Новый ЖК Строящийся", "Строящиеся", datetime.date(2026, 9, 1), 200, 400.0 + ), +] + + +# ── Тесты ───────────────────────────────────────────────────────────────────── + + +class TestCompetitorsHaveStatusFields: + """site_status и ready_dt должны присутствовать в competitors[].""" + + def test_fields_present(self) -> None: + """Каждый конкурент содержит site_status и ready_dt.""" + competitors = [dict(r.items()) for r in _ROWS_MIXED] + for c in competitors: + assert "site_status" in c, f"site_status отсутствует в {c}" + assert "ready_dt" in c, f"ready_dt отсутствует в {c}" + + def test_site_status_values(self) -> None: + """site_status принимает ожидаемые значения.""" + competitors = [dict(r.items()) for r in _ROWS_MIXED] + statuses = {c["site_status"] for c in competitors} + assert "Строящиеся" in statuses + assert "Сданные" in statuses + + def test_ready_dt_is_date_or_none(self) -> None: + """ready_dt — datetime.date или None.""" + competitors = [dict(r.items()) for r in _ROWS_MIXED] + for c in competitors: + val = c["ready_dt"] + assert val is None or isinstance( + val, datetime.date + ), f"ready_dt имеет неожиданный тип {type(val)}: {val}" + + +class TestCompetitorsSortOrder: + """Строящиеся ЖК должны идти первыми независимо от flat_count.""" + + def test_stroyashchiesya_first(self) -> None: + """Строящийся ЖК с flat_count=200 должен быть раньше сданных с flat_count=800.""" + + # Симулируем SQL ORDER BY: + # CASE site_status WHEN 'Строящиеся' THEN 0 ELSE 1 END, distance_m ASC + def _sort_key(r: MagicMock) -> tuple: + data = dict(r.items()) + status_order = 0 if data["site_status"] == "Строящиеся" else 1 + return (status_order, data["distance_m"]) + + sorted_rows = sorted(_ROWS_MIXED, key=_sort_key) + first = dict(sorted_rows[0].items()) + assert first["site_status"] == "Строящиеся", ( + f"Первый конкурент должен быть 'Строящиеся', " f"но получили '{first['site_status']}'" + ) + + def test_flat_count_desc_would_break_order(self) -> None: + """Демонстрирует, что старый ORDER BY flat_count DESC ставил сданные первыми.""" + sorted_by_flat = sorted( + _ROWS_MIXED, + key=lambda r: dict(r.items())["flat_count"], + reverse=True, + ) + first_old = dict(sorted_by_flat[0].items()) + # Старая логика: первым шёл ЖК с flat_count=800 (Сданные) + assert first_old["flat_count"] == 800 + assert first_old["site_status"] == "Сданные" diff --git a/backend/tests/api/v1/test_analyze_inline_weights.py b/backend/tests/api/v1/test_analyze_inline_weights.py new file mode 100644 index 00000000..14b18bb5 --- /dev/null +++ b/backend/tests/api/v1/test_analyze_inline_weights.py @@ -0,0 +1,331 @@ +"""Тесты для inline POI-weights в POST /api/v1/parcels/{cad_num}/analyze (#201). + +Покрывает: +1. POST /analyze без body → system defaults (no regression) +2. POST /analyze с inline weights → applied (source = "inline") +3. POST /analyze с невалидной категорией → 422 +4. POST /analyze с весом вне диапазона → 422 +5. POST /analyze с body.weights + profile_id → body.weights wins (priority) + +Стратегия mock: DB патчим через dependency_overrides, тяжёлые service-функции +(weather, velocity, dump и т.д.) патчим через unittest.mock.patch — чтобы не +дублировать все 18 db.execute call'ов в каждом тесте. +""" + +from __future__ import annotations + +from typing import Any +from unittest.mock import MagicMock, patch + +import pytest +from fastapi.testclient import TestClient + +from app.main import app + +# ── Константы ───────────────────────────────────────────────────────────────── + +_CAD = "66:41:0204016:10" +_WKT = "POLYGON((60.6 56.838, 60.61 56.838, 60.61 56.845, 60.6 56.845, 60.6 56.838))" +_GEOJSON = '{"type":"Polygon","coordinates":[[[60.6,56.838],[60.61,56.838]]]}' + + +# ── Mock factories ───────────────────────────────────────────────────────────── + + +def _make_mapping(data: dict[str, Any]) -> MagicMock: + """Создать mock-строку (mapping) с __getitem__ + .get() для dict-like доступа.""" + m = MagicMock() + m.__getitem__ = lambda self, k: data[k] + m.get = lambda k, default=None: data.get(k, default) + return m + + +def _make_db_for_analyze( + geom_found: bool = True, + district_found: bool = True, + poi_rows: list[Any] | None = None, +) -> MagicMock: + """Сконструировать mock DB Session для analyze_parcel. + + Порядок db.execute calls в analyze_parcel: + 0. UNION ALL geom + source → .mappings().first() + 1. WKT query → .mappings().first() + 2. District → .mappings().first() + 3. POI rows → .mappings().all() + 4. Competitor rows → .mappings().all() + 5. Pipeline rows → .mappings().all() + 6. Centroid lat/lon → .mappings().first() + 7. Noise rows → .mappings().all() + 8. Hydrology → .mappings().all() + 9. Utilities → .mappings().all() + 10. parcel_meta (cad_parcels) → .mappings().first() ← #29 G2 + 11. Market trend → .mappings().first() + 12. Zoning (begin_nested) → .mappings().first() + 13. Success recommendation (begin_nested) → .mappings().all() + 14. Market price (begin_nested) → .mappings().first() + 15. Recent permits (begin_nested) → .mappings().all() ← #105 Phase 5 + 16. _geotech_risk (industrial count) → .scalar() + 17. _neighbors_summary (neighbor_rows) → .mappings().all() + 18. _neighbors_summary (overlap_row) → .mappings().first() + + begin_nested() — возвращаем context manager чтобы поддержать `with` statement. + """ + db = MagicMock() + + geom_row = ( + _make_mapping({"geom_geojson": _GEOJSON, "geom_wkb": None, "source": "cad_quarter"}) + if geom_found + else None + ) + wkt_row = _make_mapping({"wkt": _WKT}) if geom_found else None + district_row = ( + _make_mapping( + { + "district_name": "Октябрьский", + "median_price_per_m2": 120000, + "dist_to_center": 1500.0, + } + ) + if district_found + else None + ) + centroid_row = _make_mapping({"lat": 56.84, "lon": 60.605}) + + _poi_rows = poi_rows or [] + + # Счётчик вызовов execute — разводим first() / all() / scalar() по очерёдности + call_idx = [0] + # Ответы в порядке вызовов: + responses: list[Any] = [ + ("first", geom_row), # 0: geom UNION ALL + ("first", wkt_row), # 1: WKT + ("first", district_row), # 2: district + ("all", _poi_rows), # 3: POI rows + ("all", []), # 4: competitor rows + ("all", []), # 5: pipeline rows + ("first", centroid_row), # 6: centroid + ("all", []), # 7: noise rows + ("all", []), # 8: hydrology rows + ("all", []), # 9: utilities rows + ("first", None), # 10: parcel_meta (cad_parcels) ← #29 G2 + ("first", None), # 11: market trend + ("first", None), # 12: zoning (inside begin_nested) + ("all", []), # 13: success recommendation (inside begin_nested) + ("scalar", 0), # 14: geotech_risk industrial count + ("all", []), # 15: neighbors + ("first", None), # 16: overlap + ] + + def _execute_side_effect(*args: Any, **kwargs: Any) -> MagicMock: + idx = call_idx[0] + call_idx[0] += 1 + if idx >= len(responses): + # Безопасный fallback для непредусмотренных вызовов + r = MagicMock() + r.mappings.return_value.first.return_value = None + r.mappings.return_value.all.return_value = [] + r.scalar.return_value = 0 + return r + kind, data = responses[idx] + r = MagicMock() + r.mappings.return_value.first.return_value = data + r.mappings.return_value.all.return_value = data if isinstance(data, list) else [] + r.scalar.return_value = data if kind == "scalar" else 0 + return r + + db.execute.side_effect = _execute_side_effect + + # begin_nested() → context manager, остальные execute внутри него проходят + # через тот же side_effect (because db.execute is the same mock). + ctx = MagicMock() + ctx.__enter__ = MagicMock(return_value=ctx) + ctx.__exit__ = MagicMock(return_value=False) + db.begin_nested.return_value = ctx + + return db + + +def _override_db(db: MagicMock): + def _get_db_override(): + yield db + + return _get_db_override + + +# Патчим тяжёлые внешние вызовы (weather / velocity / nspd-dump), +# чтобы тесты не зависели от сети и не требовали полного mock DB. +_PATCHES = [ + patch("app.api.v1.parcels._fetch_air_quality_sync", return_value=None), + patch("app.api.v1.parcels._fetch_weather_sync", return_value=None), + patch("app.api.v1.parcels._fetch_seasonal_weather_sync", return_value=None), + patch( + "app.api.v1.parcels.get_quarter_dump_data", + return_value={ + "nspd_zoning": None, + "nspd_zouit_overlaps": [], + "nspd_engineering_nearby": [], + "nspd_dump": {"available": False, "stale": False, "harvest_triggered": False}, + }, + ), + patch("app.api.v1.parcels.compute_velocity", return_value=None), + patch("app.api.v1.parcels.compute_gate_verdict", return_value={"verdict": "unknown"}), +] + + +def _start_patches() -> list[Any]: + started = [p.start() for p in _PATCHES] + return started + + +def _stop_patches() -> None: + for p in _PATCHES: + p.stop() + + +# ── Тесты ───────────────────────────────────────────────────────────────────── + + +def test_analyze_no_body_uses_system_defaults() -> None: + """POST /analyze без body → source = 'system', нет регрессии.""" + from app.core.db import get_db + + db = _make_db_for_analyze() + app.dependency_overrides[get_db] = _override_db(db) + _start_patches() + try: + client = TestClient(app) + resp = client.post(f"/api/v1/parcels/{_CAD}/analyze") + assert resp.status_code == 200, resp.text + body = resp.json() + assert "weights_profile" in body + assert body["weights_profile"]["source"] == "system" + assert body["weights_profile"]["inline_weights"] is None + finally: + app.dependency_overrides.clear() + _stop_patches() + + +def test_analyze_inline_weights_applied() -> None: + """POST /analyze с body.weights → source = 'inline', веса применены.""" + from app.core.db import get_db + + db = _make_db_for_analyze() + app.dependency_overrides[get_db] = _override_db(db) + _start_patches() + try: + client = TestClient(app) + resp = client.post( + f"/api/v1/parcels/{_CAD}/analyze", + json={"weights": {"kindergarten": 2.5}}, + ) + assert resp.status_code == 200, resp.text + body = resp.json() + wp = body["weights_profile"] + assert wp["source"] == "inline" + assert wp["inline_weights"] == {"kindergarten": 2.5} + # applied weights содержат inline override поверх defaults + assert wp["weights_applied"]["kindergarten"] == pytest.approx(2.5) + finally: + app.dependency_overrides.clear() + _stop_patches() + + +def test_analyze_invalid_category_returns_422() -> None: + """POST /analyze с невалидной POI-категорией → 422.""" + from app.core.db import get_db + + db = _make_db_for_analyze() + app.dependency_overrides[get_db] = _override_db(db) + _start_patches() + try: + client = TestClient(app) + resp = client.post( + f"/api/v1/parcels/{_CAD}/analyze", + json={"weights": {"nonexistent_category": 1.0}}, + ) + assert resp.status_code == 422, resp.text + detail = resp.json()["detail"] + assert "nonexistent_category" in detail + finally: + app.dependency_overrides.clear() + _stop_patches() + + +def test_analyze_weight_out_of_range_returns_422() -> None: + """POST /analyze с весом вне [-2, 3] → 422.""" + from app.core.db import get_db + + db = _make_db_for_analyze() + app.dependency_overrides[get_db] = _override_db(db) + _start_patches() + try: + client = TestClient(app) + # Слишком большой вес + resp = client.post( + f"/api/v1/parcels/{_CAD}/analyze", + json={"weights": {"school": 99.9}}, + ) + assert resp.status_code == 422, resp.text + detail = resp.json()["detail"] + assert "school" in detail + + # Слишком маленький вес + resp2 = client.post( + f"/api/v1/parcels/{_CAD}/analyze", + json={"weights": {"park": -5.0}}, + ) + assert resp2.status_code == 422, resp2.text + assert "park" in resp2.json()["detail"] + finally: + app.dependency_overrides.clear() + _stop_patches() + + +def test_inline_weights_rejects_nan() -> None: + """NaN weight должен вернуть 422, а не propagate в score.""" + from app.core.db import get_db + + db = _make_db_for_analyze() + app.dependency_overrides[get_db] = _override_db(db) + _start_patches() + try: + client = TestClient(app) + # Отправляем raw JSON с NaN — httpx.Client не умеет encode float('nan'), + # поэтому используем content= с явным bytes-телом. + raw_body = b'{"weights": {"school": NaN}}' + resp = client.post( + f"/api/v1/parcels/{_CAD}/analyze", + content=raw_body, + headers={"Content-Type": "application/json"}, + ) + assert ( + resp.status_code == 422 + ), f"Ожидали 422 для NaN-weight, получили {resp.status_code}: {resp.text}" + finally: + app.dependency_overrides.clear() + _stop_patches() + + +def test_analyze_inline_weights_beats_profile_id() -> None: + """body.weights + profile_id → body.weights имеет приоритет (source = 'inline').""" + from app.core.db import get_db + + db = _make_db_for_analyze() + app.dependency_overrides[get_db] = _override_db(db) + _start_patches() + try: + client = TestClient(app) + # Передаём и profile_id=1, и inline weights — inline должен победить + resp = client.post( + f"/api/v1/parcels/{_CAD}/analyze?profile_id=1", + json={"weights": {"metro_stop": 2.0}}, + ) + assert resp.status_code == 200, resp.text + wp = resp.json()["weights_profile"] + assert wp["source"] == "inline", f"Ожидали source='inline', получили '{wp['source']}'" + assert wp["weights_applied"]["metro_stop"] == pytest.approx(2.0) + # profile_id всё ещё присутствует в ответе для трассировки + assert wp["profile_id"] == 1 + finally: + app.dependency_overrides.clear() + _stop_patches() diff --git a/backend/tests/api/v1/test_analyze_market_price.py b/backend/tests/api/v1/test_analyze_market_price.py new file mode 100644 index 00000000..4fbe9081 --- /dev/null +++ b/backend/tests/api/v1/test_analyze_market_price.py @@ -0,0 +1,267 @@ +"""Тесты для market_price в POST /api/v1/parcels/{cad_num}/analyze (#33). + +Покрывает: +1. analyze с known quarter в mv_quarter_price_per_m2 → возвращает median/p25/p75/source='quarter_mv' +2. analyze с quarter которого нет в MV → deals_count=0, source='no_data' +3. analyze с invalid cad → 404 (no regression) + +Стратегия mock: аналогична test_analyze_inline_weights.py — DB mock через +dependency_overrides, тяжёлые сервисы патчим через unittest.mock.patch. + +Порядок db.execute calls в analyze_parcel (v3.7 + #33 + #29 G2): + 0. UNION ALL geom + source → .mappings().first() + 1. WKT query → .mappings().first() + 2. District → .mappings().first() + 3. POI rows → .mappings().all() + 4. Competitor rows → .mappings().all() + 5. Pipeline rows → .mappings().all() + 6. Centroid lat/lon → .mappings().first() + 7. Noise rows → .mappings().all() + 8. Hydrology → .mappings().all() + 9. Utilities → .mappings().all() + 10. parcel_meta (cad_parcels) → .mappings().first() ← #29 G2 + 11. Market trend → .mappings().first() + 12. Zoning (begin_nested) → .mappings().first() + 13. Success recommendation (begin_nested) → .mappings().all() + 14. Market price (begin_nested) → .mappings().first() ← #33 + 15. Recent permits (begin_nested) → .mappings().all() ← #105 Phase 5 + 16. _geotech_risk (industrial count) → .scalar() + 17. _neighbors_summary (neighbor_rows) → .mappings().all() + 18. _neighbors_summary (overlap_row) → .mappings().first() +""" + +from __future__ import annotations + +import datetime as dt +from decimal import Decimal +from typing import Any +from unittest.mock import MagicMock, patch + +import pytest +from fastapi.testclient import TestClient + +from app.main import app + +# ── Константы ───────────────────────────────────────────────────────────────── + +_CAD = "66:41:0204016:10" # cad_num с 4 частями — quarter = "66:41:0204016" +_CAD_3PARTS = "66:41:0204016" # cad_num уже является quarter +_WKT = "POLYGON((60.6 56.838, 60.61 56.838, 60.61 56.845, 60.6 56.845, 60.6 56.838))" +_GEOJSON = '{"type":"Polygon","coordinates":[[[60.6,56.838],[60.61,56.838]]]}' + +# ── Mock factories ───────────────────────────────────────────────────────────── + + +def _make_mapping(data: dict[str, Any]) -> MagicMock: + """Создать mock-строку (mapping) с __getitem__ + .get() для dict-like доступа.""" + m = MagicMock() + m.__getitem__ = lambda self, k: data[k] + m.get = lambda k, default=None: data.get(k, default) + return m + + +def _make_db_for_analyze( + geom_found: bool = True, + district_found: bool = True, + market_price_row: dict[str, Any] | None = None, +) -> MagicMock: + """Сконструировать mock DB Session для analyze_parcel. + + market_price_row=None → имитирует "нет данных в MV" (mp_row is None → source='no_data'). + market_price_row={...} → имитирует найденную строку в MV. + """ + db = MagicMock() + + geom_row = ( + _make_mapping({"geom_geojson": _GEOJSON, "geom_wkb": None, "source": "cad_quarter"}) + if geom_found + else None + ) + wkt_row = _make_mapping({"wkt": _WKT}) if geom_found else None + district_row = ( + _make_mapping( + { + "district_name": "Октябрьский", + "median_price_per_m2": 120000, + "dist_to_center": 1500.0, + } + ) + if district_found + else None + ) + centroid_row = _make_mapping({"lat": 56.84, "lon": 60.605}) + + mp_mock = _make_mapping(market_price_row) if market_price_row is not None else None + + call_idx = [0] + responses: list[Any] = [ + ("first", geom_row), # 0: geom UNION ALL + ("first", wkt_row), # 1: WKT + ("first", district_row), # 2: district + ("all", []), # 3: POI rows + ("all", []), # 4: competitor rows + ("all", []), # 5: pipeline rows + ("first", centroid_row), # 6: centroid + ("all", []), # 7: noise rows + ("all", []), # 8: hydrology rows + ("all", []), # 9: utilities rows + ("first", None), # 10: parcel_meta (cad_parcels) ← #29 G2 + ("first", None), # 11: market trend + ("first", None), # 12: zoning (begin_nested) + ("all", []), # 13: success recommendation (begin_nested) + ("first", mp_mock), # 14: market price (begin_nested) ← #33 + ("all", []), # 15: recent permits (begin_nested) + ("scalar", 0), # 16: geotech_risk + ("all", []), # 17: neighbors + ("first", None), # 18: overlap + ] + + def _execute_side_effect(*args: Any, **kwargs: Any) -> MagicMock: + idx = call_idx[0] + call_idx[0] += 1 + if idx >= len(responses): + r = MagicMock() + r.mappings.return_value.first.return_value = None + r.mappings.return_value.all.return_value = [] + r.scalar.return_value = 0 + return r + kind, data = responses[idx] + r = MagicMock() + r.mappings.return_value.first.return_value = data + r.mappings.return_value.all.return_value = data if isinstance(data, list) else [] + r.scalar.return_value = data if kind == "scalar" else 0 + return r + + db.execute.side_effect = _execute_side_effect + + ctx = MagicMock() + ctx.__enter__ = MagicMock(return_value=ctx) + ctx.__exit__ = MagicMock(return_value=False) + db.begin_nested.return_value = ctx + + return db + + +def _override_db(db: MagicMock): + def _get_db_override(): + yield db + + return _get_db_override + + +_PATCHES = [ + patch("app.api.v1.parcels._fetch_air_quality_sync", return_value=None), + patch("app.api.v1.parcels._fetch_weather_sync", return_value=None), + patch("app.api.v1.parcels._fetch_seasonal_weather_sync", return_value=None), + patch( + "app.api.v1.parcels.get_quarter_dump_data", + return_value={ + "nspd_zoning": None, + "nspd_zouit_overlaps": [], + "nspd_engineering_nearby": [], + "nspd_dump": {"available": False, "stale": False, "harvest_triggered": False}, + }, + ), + patch("app.api.v1.parcels.compute_velocity", return_value=None), + patch("app.api.v1.parcels.compute_gate_verdict", return_value={"verdict": "unknown"}), +] + + +def _start_patches() -> None: + for p in _PATCHES: + p.start() + + +def _stop_patches() -> None: + for p in _PATCHES: + p.stop() + + +# ── Тесты ───────────────────────────────────────────────────────────────────── + + +def test_market_price_found_in_mv() -> None: + """analyze с known quarter → market_price содержит median/p25/p75, source='quarter_mv'.""" + from app.core.db import get_db + + mv_data: dict[str, Any] = { + "p25": Decimal("85000.00"), + "median": Decimal("102000.00"), + "p75": Decimal("118000.00"), + "mean": Decimal("103500.00"), + "deals_count": 47, + "median_6m": Decimal("105000.00"), + "median_12m": Decimal("100000.00"), + "median_24m": Decimal("102000.00"), + "last_deal_date": dt.date(2026, 3, 15), + } + db = _make_db_for_analyze(market_price_row=mv_data) + app.dependency_overrides[get_db] = _override_db(db) + _start_patches() + try: + client = TestClient(app) + resp = client.post(f"/api/v1/parcels/{_CAD}/analyze") + assert resp.status_code == 200, resp.text + body = resp.json() + assert "market_price" in body, "market_price отсутствует в ответе" + mp = body["market_price"] + assert mp["source"] == "quarter_mv" + assert mp["deals_count"] == 47 + assert mp["median"] == pytest.approx(102000.0) + assert mp["p25"] == pytest.approx(85000.0) + assert mp["p75"] == pytest.approx(118000.0) + assert mp["mean"] == pytest.approx(103500.0) + assert mp["median_6m"] == pytest.approx(105000.0) + assert mp["median_12m"] == pytest.approx(100000.0) + assert mp["median_24m"] == pytest.approx(102000.0) + assert mp["last_deal_date"] == "2026-03-15" + finally: + app.dependency_overrides.clear() + _stop_patches() + + +def test_market_price_quarter_not_in_mv() -> None: + """analyze с quarter которого нет в MV → deals_count=0, source='no_data'.""" + from app.core.db import get_db + + # market_price_row=None → mp_mock=None → ветка else {"deals_count": 0, "source": "no_data"} + db = _make_db_for_analyze(market_price_row=None) + app.dependency_overrides[get_db] = _override_db(db) + _start_patches() + try: + client = TestClient(app) + resp = client.post(f"/api/v1/parcels/{_CAD}/analyze") + assert resp.status_code == 200, resp.text + body = resp.json() + assert "market_price" in body + mp = body["market_price"] + assert mp["source"] == "no_data" + assert mp["deals_count"] == 0 + # price fields absent или None + assert mp.get("median") is None + assert mp.get("p25") is None + finally: + app.dependency_overrides.clear() + _stop_patches() + + +def test_market_price_invalid_cad_returns_404() -> None: + """analyze с несуществующим cad → 404 (no regression).""" + from app.core.db import get_db + + # geom_found=False → geom_row=None → endpoint вернёт 202 (inline fetch) или 404 + # В тестовой среде on-demand fetch задизейблен (нет Celery/Redis) → 404 expected + db = _make_db_for_analyze(geom_found=False) + app.dependency_overrides[get_db] = _override_db(db) + _start_patches() + try: + client = TestClient(app) + resp = client.post("/api/v1/parcels/00:00:0000000:0/analyze") + # При отсутствии геометрии возвращается 202 (on-demand fetch enqueue) или 404 + assert resp.status_code in ( + 202, + 404, + ), f"Ожидали 202 или 404 для неизвестного cad, получили {resp.status_code}: {resp.text}" + finally: + app.dependency_overrides.clear() + _stop_patches() diff --git a/backend/tests/api/v1/test_analyze_parcel_meta.py b/backend/tests/api/v1/test_analyze_parcel_meta.py new file mode 100644 index 00000000..7dfb9e76 --- /dev/null +++ b/backend/tests/api/v1/test_analyze_parcel_meta.py @@ -0,0 +1,210 @@ +"""Тесты для parcel_meta в POST /api/v1/parcels/{cad_num}/analyze (#29 G2). + +Покрывает: +1. analyze когда cad_parcels has row → parcel_meta содержит permitted_use/land_category/cad_cost +2. analyze когда cad_parcels row отсутствует → parcel_meta == None + +Стратегия mock: аналогична test_analyze_market_price.py — DB mock через +dependency_overrides, тяжёлые сервисы патчим через unittest.mock.patch. + +Порядок db.execute calls в analyze_parcel (с #29 G2): + 0. UNION ALL geom + source → .mappings().first() + 1. WKT query → .mappings().first() + 2. District → .mappings().first() + 3. POI rows → .mappings().all() + 4. Competitor rows → .mappings().all() + 5. Pipeline rows → .mappings().all() + 6. Centroid lat/lon → .mappings().first() + 7. Noise rows → .mappings().all() + 8. Hydrology → .mappings().all() + 9. Utilities → .mappings().all() + 10. parcel_meta (cad_parcels) → .mappings().first() ← NEW #29 G2 + 11. Market trend → .mappings().first() + 12. Zoning (begin_nested) → .mappings().first() + 13. Success recommendation (begin_nested) → .mappings().all() + 14. Market price (begin_nested) → .mappings().first() + 15. Recent permits (begin_nested) → .mappings().all() + 16. _geotech_risk (industrial count) → .scalar() + 17. _neighbors_summary (neighbor_rows) → .mappings().all() + 18. _neighbors_summary (overlap_row) → .mappings().first() +""" + +from __future__ import annotations + +from typing import Any +from unittest.mock import MagicMock, patch + +from fastapi.testclient import TestClient + +from app.main import app + +_CAD = "66:41:0204016:10" +_WKT = "POLYGON((60.6 56.838, 60.61 56.838, 60.61 56.845, 60.6 56.845, 60.6 56.838))" +_GEOJSON = '{"type":"Polygon","coordinates":[[[60.6,56.838],[60.61,56.838]]]}' + + +def _make_mapping(data: dict[str, Any]) -> MagicMock: + m = MagicMock() + m.__getitem__ = lambda self, k: data[k] + m.get = lambda k, default=None: data.get(k, default) + return m + + +def _make_db_for_analyze( + parcel_meta_row: dict[str, Any] | None = None, +) -> MagicMock: + """Сконструировать mock DB Session для analyze_parcel с фокусом на parcel_meta. + + parcel_meta_row=None → имитирует "строки нет в cad_parcels" → parcel_meta=None. + parcel_meta_row={...} → имитирует найденную строку. + """ + db = MagicMock() + + geom_row = _make_mapping({"geom_geojson": _GEOJSON, "geom_wkb": None, "source": "cad_quarter"}) + wkt_row = _make_mapping({"wkt": _WKT}) + district_row = _make_mapping( + { + "district_name": "Октябрьский", + "median_price_per_m2": 120000, + "dist_to_center": 1500.0, + } + ) + centroid_row = _make_mapping({"lat": 56.84, "lon": 60.605}) + + pm_mock = _make_mapping(parcel_meta_row) if parcel_meta_row is not None else None + + call_idx = [0] + responses: list[Any] = [ + ("first", geom_row), # 0: geom UNION ALL + ("first", wkt_row), # 1: WKT + ("first", district_row), # 2: district + ("all", []), # 3: POI rows + ("all", []), # 4: competitor rows + ("all", []), # 5: pipeline rows + ("first", centroid_row), # 6: centroid + ("all", []), # 7: noise rows + ("all", []), # 8: hydrology rows + ("all", []), # 9: utilities rows + ("first", pm_mock), # 10: parcel_meta ← #29 G2 + ("first", None), # 11: market trend + ("first", None), # 12: zoning (begin_nested) + ("all", []), # 13: success recommendation (begin_nested) + ("first", None), # 14: market price (begin_nested) + ("all", []), # 15: recent permits (begin_nested) + ("scalar", 0), # 16: geotech_risk + ("all", []), # 17: neighbors + ("first", None), # 18: overlap + ] + + def _execute_side_effect(*args: Any, **kwargs: Any) -> MagicMock: + idx = call_idx[0] + call_idx[0] += 1 + if idx >= len(responses): + r = MagicMock() + r.mappings.return_value.first.return_value = None + r.mappings.return_value.all.return_value = [] + r.scalar.return_value = 0 + return r + kind, data = responses[idx] + r = MagicMock() + r.mappings.return_value.first.return_value = data + r.mappings.return_value.all.return_value = data if isinstance(data, list) else [] + r.scalar.return_value = data if kind == "scalar" else 0 + return r + + db.execute.side_effect = _execute_side_effect + + ctx = MagicMock() + ctx.__enter__ = MagicMock(return_value=ctx) + ctx.__exit__ = MagicMock(return_value=False) + db.begin_nested.return_value = ctx + + return db + + +def _override_db(db: MagicMock): + def _get_db_override(): + yield db + + return _get_db_override + + +_PATCHES = [ + patch("app.api.v1.parcels._fetch_air_quality_sync", return_value=None), + patch("app.api.v1.parcels._fetch_weather_sync", return_value=None), + patch("app.api.v1.parcels._fetch_seasonal_weather_sync", return_value=None), + patch( + "app.api.v1.parcels.get_quarter_dump_data", + return_value={ + "nspd_zoning": None, + "nspd_zouit_overlaps": [], + "nspd_engineering_nearby": [], + "nspd_risk_zones": [], + "nspd_opportunity_parcels": [], + "nspd_red_lines": [], + "nspd_dump": {"available": False, "stale": False, "harvest_triggered": False}, + }, + ), + patch("app.api.v1.parcels.compute_velocity", return_value=None), + patch("app.api.v1.parcels.compute_gate_verdict", return_value={"verdict": "unknown"}), +] + + +def _start_patches() -> None: + for p in _PATCHES: + p.start() + + +def _stop_patches() -> None: + for p in _PATCHES: + p.stop() + + +def test_parcel_meta_found_in_cad_parcels() -> None: + """analyze → cad_parcels has row → parcel_meta содержит permitted_use/land_category/cad_cost.""" + from app.core.db import get_db + + pm_data: dict[str, Any] = { + "permitted_use": "многоквартирный дом", + "land_category": "Земли населённых пунктов", + "land_subtype": "жилая застройка", + "cad_cost": 5_000_000.0, + } + db = _make_db_for_analyze(parcel_meta_row=pm_data) + app.dependency_overrides[get_db] = _override_db(db) + _start_patches() + try: + client = TestClient(app) + resp = client.post(f"/api/v1/parcels/{_CAD}/analyze") + assert resp.status_code == 200, resp.text + body = resp.json() + assert "parcel_meta" in body, "parcel_meta отсутствует в ответе" + pm = body["parcel_meta"] + assert pm is not None, "parcel_meta должен быть не None при наличии строки" + assert pm["permitted_use"] == "многоквартирный дом" + assert pm["land_category"] == "Земли населённых пунктов" + assert pm["land_subtype"] == "жилая застройка" + assert pm["cad_cost"] == 5_000_000.0 + assert pm["source"] == "cad_parcels" + finally: + app.dependency_overrides.clear() + _stop_patches() + + +def test_parcel_meta_none_when_cad_parcels_missing() -> None: + """analyze → cad_parcels row отсутствует → parcel_meta == None.""" + from app.core.db import get_db + + db = _make_db_for_analyze(parcel_meta_row=None) + app.dependency_overrides[get_db] = _override_db(db) + _start_patches() + try: + client = TestClient(app) + resp = client.post(f"/api/v1/parcels/{_CAD}/analyze") + assert resp.status_code == 200, resp.text + body = resp.json() + assert "parcel_meta" in body, "parcel_meta ключ должен присутствовать в ответе" + assert body["parcel_meta"] is None, "parcel_meta должен быть None при отсутствии строки" + finally: + app.dependency_overrides.clear() + _stop_patches() diff --git a/backend/tests/api/v1/test_analyze_recent_permits.py b/backend/tests/api/v1/test_analyze_recent_permits.py new file mode 100644 index 00000000..e2db9849 --- /dev/null +++ b/backend/tests/api/v1/test_analyze_recent_permits.py @@ -0,0 +1,283 @@ +"""Тесты для recent_permits_in_quarter в POST /api/v1/parcels/{cad_num}/analyze (#105 Phase 5). + +Покрывает: +1. analyze с quarter где есть РНС/РВЭ → recent_permits non-empty + summary calculated +2. analyze с quarter без permits → recent_permits=[], summary={rns_count=0, ...} +3. analyze 404 на invalid cad (no regression) + +Порядок db.execute calls в analyze_parcel (после #105 Phase 5 + #29 G2): + 0. UNION ALL geom + source → .mappings().first() + 1. WKT query → .mappings().first() + 2. District → .mappings().first() + 3. POI rows → .mappings().all() + 4. Competitor rows → .mappings().all() + 5. Pipeline rows → .mappings().all() + 6. Centroid lat/lon → .mappings().first() + 7. Noise rows → .mappings().all() + 8. Hydrology → .mappings().all() + 9. Utilities → .mappings().all() + 10. parcel_meta (cad_parcels) → .mappings().first() ← #29 G2 + 11. Market trend → .mappings().first() + 12. Zoning (begin_nested) → .mappings().first() + 13. Success recommendation (begin_nested) → .mappings().all() + 14. Market price (begin_nested) → .mappings().first() + 15. Recent permits (begin_nested) → .mappings().all() ← #105 + 16. _geotech_risk (industrial count) → .scalar() + 17. _neighbors_summary (neighbor_rows) → .mappings().all() + 18. _neighbors_summary (overlap_row) → .mappings().first() +""" + +from __future__ import annotations + +import datetime as dt +from typing import Any +from unittest.mock import MagicMock, patch + +import pytest +from fastapi.testclient import TestClient + +from app.main import app + +# ── Константы ───────────────────────────────────────────────────────────────── + +_CAD = "66:41:0204016:10" # cad_num с 4 частями — quarter prefix = "66:41:0204016" +_WKT = "POLYGON((60.6 56.838, 60.61 56.838, 60.61 56.845, 60.6 56.845, 60.6 56.838))" +_GEOJSON = '{"type":"Polygon","coordinates":[[[60.6,56.838],[60.61,56.838]]]}' + + +# ── Mock factories ───────────────────────────────────────────────────────────── + + +def _make_mapping(data: dict[str, Any]) -> MagicMock: + """Создать mock-строку (mapping) с __getitem__ + .get() для dict-like доступа.""" + m = MagicMock() + m.__getitem__ = lambda self, k: data[k] + m.get = lambda k, default=None: data.get(k, default) + return m + + +def _make_db_for_analyze( + geom_found: bool = True, + permits_rows: list[dict[str, Any]] | None = None, +) -> MagicMock: + """Сконструировать mock DB Session для analyze_parcel. + + permits_rows=None → пустой список (нет разрешений в квартале). + permits_rows=[...] → список разрешений для агрегации. + """ + db = MagicMock() + + geom_row = ( + _make_mapping({"geom_geojson": _GEOJSON, "geom_wkb": None, "source": "cad_quarter"}) + if geom_found + else None + ) + wkt_row = _make_mapping({"wkt": _WKT}) if geom_found else None + district_row = _make_mapping( + { + "district_name": "Октябрьский", + "median_price_per_m2": 120000, + "dist_to_center": 1500.0, + } + ) + centroid_row = _make_mapping({"lat": 56.84, "lon": 60.605}) + + raw_permits = [_make_mapping(p) for p in (permits_rows or [])] + + call_idx = [0] + responses: list[Any] = [ + ("first", geom_row), # 0: geom UNION ALL + ("first", wkt_row), # 1: WKT + ("first", district_row), # 2: district + ("all", []), # 3: POI rows + ("all", []), # 4: competitor rows + ("all", []), # 5: pipeline rows + ("first", centroid_row), # 6: centroid + ("all", []), # 7: noise rows + ("all", []), # 8: hydrology rows + ("all", []), # 9: utilities rows + ("first", None), # 10: parcel_meta (cad_parcels) ← #29 G2 + ("first", None), # 11: market trend + ("first", None), # 12: zoning (begin_nested) + ("all", []), # 13: success recommendation (begin_nested) + ("first", None), # 14: market price (begin_nested) + ("all", raw_permits), # 15: recent permits (begin_nested) ← #105 + ("scalar", 0), # 16: geotech_risk + ("all", []), # 17: neighbors + ("first", None), # 18: overlap + ] + + def _execute_side_effect(*args: Any, **kwargs: Any) -> MagicMock: + idx = call_idx[0] + call_idx[0] += 1 + if idx >= len(responses): + r = MagicMock() + r.mappings.return_value.first.return_value = None + r.mappings.return_value.all.return_value = [] + r.scalar.return_value = 0 + return r + kind, data = responses[idx] + r = MagicMock() + r.mappings.return_value.first.return_value = data + r.mappings.return_value.all.return_value = data if isinstance(data, list) else [] + r.scalar.return_value = data if kind == "scalar" else 0 + return r + + db.execute.side_effect = _execute_side_effect + + ctx = MagicMock() + ctx.__enter__ = MagicMock(return_value=ctx) + ctx.__exit__ = MagicMock(return_value=False) + db.begin_nested.return_value = ctx + + return db + + +def _override_db(db: MagicMock): + def _get_db_override(): + yield db + + return _get_db_override + + +_PATCHES = [ + patch("app.api.v1.parcels._fetch_air_quality_sync", return_value=None), + patch("app.api.v1.parcels._fetch_weather_sync", return_value=None), + patch("app.api.v1.parcels._fetch_seasonal_weather_sync", return_value=None), + patch( + "app.api.v1.parcels.get_quarter_dump_data", + return_value={ + "nspd_zoning": None, + "nspd_zouit_overlaps": [], + "nspd_engineering_nearby": [], + "nspd_dump": {"available": False, "stale": False, "harvest_triggered": False}, + }, + ), + patch("app.api.v1.parcels.compute_velocity", return_value=None), + patch("app.api.v1.parcels.compute_gate_verdict", return_value={"verdict": "unknown"}), +] + + +def _start_patches() -> None: + for p in _PATCHES: + p.start() + + +def _stop_patches() -> None: + for p in _PATCHES: + p.stop() + + +# ── Тесты ───────────────────────────────────────────────────────────────────── + + +def test_recent_permits_found_in_quarter() -> None: + """analyze с квартала где есть РНС/РВЭ → recent_permits non-empty + summary calculated.""" + from app.core.db import get_db + + rns_permit: dict[str, Any] = { + "permit_type": "RNS", + "permit_number": "66-RNS-2024-001", + "issue_date": dt.date(2024, 6, 15), + "developer_name": "Атомстрой", + "developer_inn": "6671234567", + "object_name": "МКД ЖК Тест", + "object_type": "МКД", + "construction_address": "г. Екатеринбург, ул. Ленина, 1", + "total_area_sqm": 45000.0, + } + rve_permit: dict[str, Any] = { + "permit_type": "RVE", + "permit_number": "66-RVE-2025-002", + "issue_date": dt.date(2025, 3, 10), + "developer_name": "Атомстрой", + "developer_inn": "6671234567", + "object_name": "МКД ЖК Тест (1 оч.)", + "object_type": "МКД", + "construction_address": "г. Екатеринбург, ул. Ленина, 1", + "total_area_sqm": None, + } + + db = _make_db_for_analyze(permits_rows=[rns_permit, rve_permit]) + app.dependency_overrides[get_db] = _override_db(db) + _start_patches() + try: + client = TestClient(app) + resp = client.post(f"/api/v1/parcels/{_CAD}/analyze") + assert resp.status_code == 200, resp.text + body = resp.json() + + assert "recent_permits_in_quarter" in body + assert "permits_summary" in body + + permits = body["recent_permits_in_quarter"] + assert len(permits) == 2 + + rns = next(p for p in permits if p["permit_type"] == "RNS") + assert rns["permit_number"] == "66-RNS-2024-001" + assert rns["developer_name"] == "Атомстрой" + assert rns["issue_date"] == "2024-06-15" + assert rns["total_area_sqm"] == pytest.approx(45000.0) + + rve = next(p for p in permits if p["permit_type"] == "RVE") + assert rve["issue_date"] == "2025-03-10" + assert "units_count" not in rve + + summary = body["permits_summary"] + assert summary["rns_count"] == 1 + assert summary["rve_count"] == 1 + assert summary["rns_total_area_sqm"] == pytest.approx(45000.0) + assert "rve_total_units" not in summary + assert len(summary["by_developer"]) == 1 + assert summary["by_developer"][0]["name"] == "Атомстрой" + assert summary["by_developer"][0]["permits_count"] == 2 + finally: + app.dependency_overrides.clear() + _stop_patches() + + +def test_recent_permits_empty_quarter() -> None: + """analyze с quarter без permits → recent_permits=[], summary с нулями.""" + from app.core.db import get_db + + db = _make_db_for_analyze(permits_rows=[]) + app.dependency_overrides[get_db] = _override_db(db) + _start_patches() + try: + client = TestClient(app) + resp = client.post(f"/api/v1/parcels/{_CAD}/analyze") + assert resp.status_code == 200, resp.text + body = resp.json() + + assert "recent_permits_in_quarter" in body + assert body["recent_permits_in_quarter"] == [] + + assert "permits_summary" in body + summary = body["permits_summary"] + assert summary["rns_count"] == 0 + assert summary["rve_count"] == 0 + assert summary["rns_total_area_sqm"] == pytest.approx(0.0) + assert "rve_total_units" not in summary + assert summary["by_developer"] == [] + finally: + app.dependency_overrides.clear() + _stop_patches() + + +def test_recent_permits_invalid_cad_no_regression() -> None: + """analyze с несуществующим cad → 202 или 404 (no regression от Phase 5).""" + from app.core.db import get_db + + db = _make_db_for_analyze(geom_found=False) + app.dependency_overrides[get_db] = _override_db(db) + _start_patches() + try: + client = TestClient(app) + resp = client.post("/api/v1/parcels/00:00:0000000:0/analyze") + assert resp.status_code in ( + 202, + 404, + ), f"Ожидали 202 или 404 для неизвестного cad, получили {resp.status_code}: {resp.text}" + finally: + app.dependency_overrides.clear() + _stop_patches() diff --git a/backend/tests/api/v1/test_custom_pois.py b/backend/tests/api/v1/test_custom_pois.py new file mode 100644 index 00000000..a579ff47 --- /dev/null +++ b/backend/tests/api/v1/test_custom_pois.py @@ -0,0 +1,595 @@ +"""Тесты для custom POI CRUD + scoring integration (#254). + +Покрывает: +1. POST /api/v1/custom-pois → 201, correct response, X-Session-Id в response header +2. GET /api/v1/custom-pois → list для user (изоляция по user_id) +3. PATCH /api/v1/custom-pois/{id} → updated, 404 для чужой +4. DELETE /api/v1/custom-pois/{id} → 204, 404 для чужой +5. Validation: weight outside [-5,5] → 422; lon/lat outside range → 422 +6. GET /api/v1/custom-pois?parcel_cad=... → filter works +7. Scoring integration: custom POI в 500м с weight=+2 увеличивает score +8. No X-Session-Id → auto-generated UUID в response header +9. Scoring absent when no X-Session-Id header +10. Service-level: db.commit() вызывается в create/update/delete (#261 regression guard) + +Стратегия mock: сервисные функции патчим через unittest.mock.patch, +DB — через dependency_overrides (аналогично test_analyze_inline_weights.py). +Service commit tests (#261): MagicMock db + assert db.commit.assert_called_once(). +""" + +from __future__ import annotations + +import uuid +from datetime import UTC, datetime +from typing import Any +from unittest.mock import MagicMock, patch + +import pytest +from fastapi.testclient import TestClient + +from app.main import app +from app.schemas.custom_poi import CustomPoiCreate, CustomPoiOut, CustomPoiUpdate + +# ── Константы ───────────────────────────────────────────────────────────────── + +_CAD = "66:41:0204016:10" +_WKT = "POLYGON((60.6 56.838, 60.61 56.838, 60.61 56.845, 60.6 56.845, 60.6 56.838))" +_GEOJSON = '{"type":"Polygon","coordinates":[[[60.6,56.838],[60.61,56.838]]]}' +_SESSION = "test-session-abc123" +_TS = datetime(2026, 5, 17, 10, 0, 0, tzinfo=UTC) + + +# ── Helpers ──────────────────────────────────────────────────────────────────── + + +def _make_poi_out( + poi_id: int = 1, + user_id: str = _SESSION, + name: str = "Парк Маяковского", + weight: float = 2.0, + lon: float = 60.605, + lat: float = 56.838, + parcel_cad: str | None = None, +) -> CustomPoiOut: + return CustomPoiOut( + id=poi_id, + user_id=user_id, + parcel_cad=parcel_cad, + name=name, + category="park", + weight=weight, + lon=lon, + lat=lat, + notes=None, + created_at=_TS, + updated_at=_TS, + ) + + +def _make_mapping(data: dict[str, Any]) -> MagicMock: + m = MagicMock() + m.__getitem__ = lambda self, k: data[k] + m.get = lambda k, default=None: data.get(k, default) + return m + + +# ── Tests: CRUD ──────────────────────────────────────────────────────────────── + + +def test_create_poi_returns_201() -> None: + """POST /custom-pois → 201 + корректный тело + X-Session-Id в header.""" + expected = _make_poi_out() + with patch("app.api.v1.custom_pois.create_custom_poi", return_value=expected) as mock_create: + client = TestClient(app) + resp = client.post( + "/api/v1/custom-pois", + json={ + "name": "Парк Маяковского", + "category": "park", + "weight": 2.0, + "lon": 60.605, + "lat": 56.838, + }, + headers={"X-Session-Id": _SESSION}, + ) + assert resp.status_code == 201, resp.text + body = resp.json() + assert body["name"] == "Парк Маяковского" + assert body["weight"] == pytest.approx(2.0) + assert body["id"] == 1 + assert resp.headers.get("x-session-id") == _SESSION + mock_create.assert_called_once() + + +def test_create_poi_auto_generates_session_id() -> None: + """POST без X-Session-Id → 201 + авто-UUID в X-Session-Id header.""" + expected = _make_poi_out(user_id="some-uuid") + with patch("app.api.v1.custom_pois.create_custom_poi", return_value=expected): + client = TestClient(app) + resp = client.post( + "/api/v1/custom-pois", + json={"name": "Test", "weight": 1.0, "lon": 60.0, "lat": 56.0}, + ) + assert resp.status_code == 201, resp.text + sid = resp.headers.get("x-session-id") + assert sid is not None, "Ожидали X-Session-Id в response headers" + # Должен быть валидным UUID + try: + uuid.UUID(sid) + except ValueError: + pytest.fail(f"X-Session-Id '{sid}' не является валидным UUID") + + +def test_create_poi_weight_out_of_range_returns_422() -> None: + """weight > 5 или < -5 → 422 от Pydantic (FastAPI body validation).""" + client = TestClient(app) + resp = client.post( + "/api/v1/custom-pois", + json={"name": "Test", "weight": 10.0, "lon": 60.0, "lat": 56.0}, + headers={"X-Session-Id": _SESSION}, + ) + assert resp.status_code == 422, resp.text + + resp2 = client.post( + "/api/v1/custom-pois", + json={"name": "Test", "weight": -6.0, "lon": 60.0, "lat": 56.0}, + headers={"X-Session-Id": _SESSION}, + ) + assert resp2.status_code == 422, resp2.text + + +def test_create_poi_lon_out_of_range_returns_422() -> None: + """lon > 180 → 422.""" + client = TestClient(app) + resp = client.post( + "/api/v1/custom-pois", + json={"name": "Test", "weight": 1.0, "lon": 200.0, "lat": 56.0}, + headers={"X-Session-Id": _SESSION}, + ) + assert resp.status_code == 422, resp.text + + +def test_create_poi_lat_out_of_range_returns_422() -> None: + """lat > 90 → 422.""" + client = TestClient(app) + resp = client.post( + "/api/v1/custom-pois", + json={"name": "Test", "weight": 1.0, "lon": 60.0, "lat": 100.0}, + headers={"X-Session-Id": _SESSION}, + ) + assert resp.status_code == 422, resp.text + + +def test_list_pois_returns_user_items() -> None: + """GET /custom-pois → список POI пользователя.""" + pois = [_make_poi_out(1), _make_poi_out(2, name="Школа №5", weight=1.5)] + with patch("app.api.v1.custom_pois.list_custom_pois", return_value=pois): + client = TestClient(app) + resp = client.get("/api/v1/custom-pois", headers={"X-Session-Id": _SESSION}) + assert resp.status_code == 200, resp.text + body = resp.json() + assert len(body) == 2 + assert body[0]["id"] == 1 + assert body[1]["name"] == "Школа №5" + + +def test_list_pois_filter_by_parcel_cad() -> None: + """GET /custom-pois?parcel_cad=... → вызывает list_custom_pois с parcel_cad.""" + with patch("app.api.v1.custom_pois.list_custom_pois", return_value=[]) as mock_list: + client = TestClient(app) + resp = client.get( + f"/api/v1/custom-pois?parcel_cad={_CAD}", + headers={"X-Session-Id": _SESSION}, + ) + assert resp.status_code == 200, resp.text + # Проверяем что parcel_cad передан в сервис + call_kwargs = mock_list.call_args + assert call_kwargs[1].get("parcel_cad") == _CAD or ( + len(call_kwargs[0]) >= 3 and call_kwargs[0][2] == _CAD + ) + + +def test_patch_poi_returns_updated() -> None: + """PATCH /custom-pois/{id} → 200 с обновлёнными данными.""" + updated = _make_poi_out(weight=3.0, name="Обновлённый парк") + with patch("app.api.v1.custom_pois.update_custom_poi", return_value=updated): + client = TestClient(app) + resp = client.patch( + "/api/v1/custom-pois/1", + json={"weight": 3.0, "name": "Обновлённый парк"}, + headers={"X-Session-Id": _SESSION}, + ) + assert resp.status_code == 200, resp.text + body = resp.json() + assert body["weight"] == pytest.approx(3.0) + assert body["name"] == "Обновлённый парк" + + +def test_patch_poi_not_found_returns_404() -> None: + """PATCH для несуществующей/чужой POI → 404.""" + with patch("app.api.v1.custom_pois.update_custom_poi", return_value=None): + client = TestClient(app) + resp = client.patch( + "/api/v1/custom-pois/999", + json={"weight": 1.0}, + headers={"X-Session-Id": _SESSION}, + ) + assert resp.status_code == 404, resp.text + + +def test_delete_poi_returns_204() -> None: + """DELETE /custom-pois/{id} → 204 при успешном удалении.""" + with patch("app.api.v1.custom_pois.delete_custom_poi", return_value=True): + client = TestClient(app) + resp = client.delete("/api/v1/custom-pois/1", headers={"X-Session-Id": _SESSION}) + assert resp.status_code == 204, resp.text + + +def test_delete_poi_not_found_returns_404() -> None: + """DELETE для несуществующей POI → 404.""" + with patch("app.api.v1.custom_pois.delete_custom_poi", return_value=False): + client = TestClient(app) + resp = client.delete("/api/v1/custom-pois/999", headers={"X-Session-Id": _SESSION}) + assert resp.status_code == 404, resp.text + + +# ── Tests: Scoring integration ───────────────────────────────────────────────── + +# Вспомогательный mock DB для analyze_parcel (аналогично test_analyze_inline_weights.py). +# Порядок db.execute calls включает #254 custom POI via get_overlaps_for_scoring, +# который вызывается уже после POI loop — ВНУТРИ analyze_parcel через импортированную +# функцию. Мы патчим её через patch() — DB mock остаётся прежним. + + +def _make_mapping_analyze(data: dict[str, Any]) -> MagicMock: + m = MagicMock() + m.__getitem__ = lambda self, k: data[k] + m.get = lambda k, default=None: data.get(k, default) + return m + + +def _make_db_for_analyze() -> MagicMock: + """Mock DB Session для analyze_parcel (без custom POI вызовов — они пропатчены).""" + db = MagicMock() + + geom_row = _make_mapping_analyze( + {"geom_geojson": _GEOJSON, "geom_wkb": None, "source": "cad_quarter"} + ) + wkt_row = _make_mapping_analyze({"wkt": _WKT}) + district_row = _make_mapping_analyze( + { + "district_name": "Октябрьский", + "median_price_per_m2": 120000, + "dist_to_center": 1500.0, + } + ) + centroid_row = _make_mapping_analyze({"lat": 56.84, "lon": 60.605}) + + call_idx = [0] + responses: list[Any] = [ + ("first", geom_row), # 0: geom UNION ALL + ("first", wkt_row), # 1: WKT + ("first", district_row), # 2: district + ("all", []), # 3: POI rows + ("all", []), # 4: competitor rows + ("all", []), # 5: pipeline rows + ("first", centroid_row), # 6: centroid + ("all", []), # 7: noise rows + ("all", []), # 8: hydrology rows + ("all", []), # 9: utilities rows + ("first", None), # 10: parcel_meta + ("first", None), # 11: market trend + ("first", None), # 12: zoning (begin_nested) + ("all", []), # 13: success recommendation (begin_nested) + ("first", None), # 14: market price (begin_nested) + ("all", []), # 15: recent permits (begin_nested) + ("scalar", 0), # 16: geotech_risk + ("all", []), # 17: neighbors + ("first", None), # 18: overlap + ] + + def _execute_side_effect(*args: Any, **kwargs: Any) -> MagicMock: + idx = call_idx[0] + call_idx[0] += 1 + if idx >= len(responses): + r = MagicMock() + r.mappings.return_value.first.return_value = None + r.mappings.return_value.all.return_value = [] + r.scalar.return_value = 0 + return r + kind, data = responses[idx] + r = MagicMock() + r.mappings.return_value.first.return_value = data + r.mappings.return_value.all.return_value = data if isinstance(data, list) else [] + r.scalar.return_value = data if kind == "scalar" else 0 + return r + + db.execute.side_effect = _execute_side_effect + + ctx = MagicMock() + ctx.__enter__ = MagicMock(return_value=ctx) + ctx.__exit__ = MagicMock(return_value=False) + db.begin_nested.return_value = ctx + + return db + + +def _override_db(db: MagicMock): + def _get_db_override(): + yield db + + return _get_db_override + + +_ANALYZE_PATCHES = [ + patch("app.api.v1.parcels._fetch_air_quality_sync", return_value=None), + patch("app.api.v1.parcels._fetch_weather_sync", return_value=None), + patch("app.api.v1.parcels._fetch_seasonal_weather_sync", return_value=None), + patch( + "app.api.v1.parcels.get_quarter_dump_data", + return_value={ + "nspd_zoning": None, + "nspd_zouit_overlaps": [], + "nspd_engineering_nearby": [], + "nspd_dump": {"available": False, "stale": False, "harvest_triggered": False}, + }, + ), + patch("app.api.v1.parcels.compute_velocity", return_value=None), + patch("app.api.v1.parcels.compute_gate_verdict", return_value={"verdict": "unknown"}), +] + + +def _start_analyze_patches() -> None: + for p in _ANALYZE_PATCHES: + p.start() + + +def _stop_analyze_patches() -> None: + for p in _ANALYZE_PATCHES: + p.stop() + + +def test_analyze_custom_poi_increases_score() -> None: + """Custom POI с weight=+2 в 500м → score повышается, custom_poi_score_items непустой.""" + from app.core.db import get_db + + # Симулируем custom POI в 500м + _custom_overlap = { + "id": 42, + "name": "Мой парк", + "category": "park", + "weight": 2.0, + "lon": 60.607, + "lat": 56.840, + "distance_m": 500.0, + } + # decay = 1 - 500/1000 = 0.5; contribution = 2.0 * 0.5 = 1.0 + + db = _make_db_for_analyze() + app.dependency_overrides[get_db] = _override_db(db) + _start_analyze_patches() + + try: + with patch("app.api.v1.parcels._get_custom_poi_overlaps", return_value=[_custom_overlap]): + client = TestClient(app) + resp = client.post( + f"/api/v1/parcels/{_CAD}/analyze", + headers={"X-Session-Id": _SESSION}, + ) + assert resp.status_code == 200, resp.text + body = resp.json() + + # custom_poi_score_items должен содержать нашу точку + assert "custom_poi_score_items" in body + items = body["custom_poi_score_items"] + assert len(items) == 1 + assert items[0]["label"] == "Мой парк" + assert items[0]["weight"] == pytest.approx(2.0) + assert items[0]["distance_m"] == 500 + # contribution = 2.0 * 0.5 = 1.0 + assert items[0]["contribution"] == pytest.approx(1.0, abs=0.01) + + # score должен учитывать contribution custom POI + # (базовый score без POI = center_bonus из 1500м → 1.5; + custom 1.0 = 2.5) + assert body["score"] == pytest.approx(2.5, abs=0.1) + finally: + app.dependency_overrides.clear() + _stop_analyze_patches() + + +def test_analyze_without_session_id_no_custom_poi() -> None: + """POST /analyze без X-Session-Id → custom_poi_score_items пустой.""" + from app.core.db import get_db + + db = _make_db_for_analyze() + app.dependency_overrides[get_db] = _override_db(db) + _start_analyze_patches() + + try: + with patch("app.api.v1.parcels._get_custom_poi_overlaps", return_value=[]) as mock_overlaps: + client = TestClient(app) + resp = client.post(f"/api/v1/parcels/{_CAD}/analyze") + assert resp.status_code == 200, resp.text + body = resp.json() + # Без session_id custom_poi_score_items пустой (не вызываем get_overlaps) + assert body["custom_poi_score_items"] == [] + mock_overlaps.assert_not_called() + finally: + app.dependency_overrides.clear() + _stop_analyze_patches() + + +def test_analyze_custom_poi_negative_weight_decreases_score() -> None: + """Custom POI с weight=-3 в 500м → score уменьшается.""" + from app.core.db import get_db + + _custom_overlap = { + "id": 7, + "name": "Промзона", + "category": "industrial", + "weight": -3.0, + "lon": 60.607, + "lat": 56.840, + "distance_m": 500.0, + } + # contribution = -3.0 * 0.5 = -1.5 + + db = _make_db_for_analyze() + app.dependency_overrides[get_db] = _override_db(db) + _start_analyze_patches() + + try: + with patch("app.api.v1.parcels._get_custom_poi_overlaps", return_value=[_custom_overlap]): + client = TestClient(app) + resp = client.post( + f"/api/v1/parcels/{_CAD}/analyze", + headers={"X-Session-Id": _SESSION}, + ) + assert resp.status_code == 200, resp.text + body = resp.json() + items = body["custom_poi_score_items"] + assert len(items) == 1 + assert items[0]["contribution"] == pytest.approx(-1.5, abs=0.01) + # center_bonus = 1.5 (1500м), custom = -1.5 → итого ≈ 0 + assert body["score"] == pytest.approx(0.0, abs=0.1) + finally: + app.dependency_overrides.clear() + _stop_analyze_patches() + + +# ── Service commit regression tests (#261) ──────────────────────────────────── +# +# Проверяем что service-функции вызывают db.commit() после каждой мутации. +# Если db.commit() убрать из сервиса — тест упадёт на assert_called_once(). +# +# Паттерн "два сеанса": write_db отдельный от последующего чтения. +# Эмулирует независимый SELECT после HTTP request close: +# get_db.finally → db.close() без commit → rollback pending tx, +# поэтому commit должен быть явным внутри каждой service-функции. + + +def _make_row_data( + poi_id: int = 1, + user_id: str = _SESSION, + name: str = "Тест", + weight: float = 1.5, + lon: float = 60.605, + lat: float = 56.838, +) -> dict[str, Any]: + return { + "id": poi_id, + "user_id": user_id, + "parcel_cad": None, + "name": name, + "category": "park", + "weight": weight, + "lon": lon, + "lat": lat, + "notes": None, + "created_at": _TS, + "updated_at": _TS, + } + + +def _make_db_with_row(row_data: dict[str, Any] | None) -> MagicMock: + """Вернуть MagicMock-сессию где execute().mappings().first() → row_data mapping.""" + db = MagicMock() + row_mock: MagicMock | None = None + if row_data is not None: + row_mock = MagicMock() + row_mock.__getitem__ = lambda self, k: row_data[k] + row_mock.get = lambda k, default=None: row_data.get(k, default) + + mapping_mock = MagicMock() + mapping_mock.first.return_value = row_mock + + exec_mock = MagicMock() + exec_mock.mappings.return_value = mapping_mock + exec_mock.first.return_value = row_mock # для DELETE RETURNING id (не mappings) + + db.execute.return_value = exec_mock + return db + + +def test_service_create_custom_poi_commits() -> None: + """create_custom_poi вызывает db.commit() — regression guard #261. + + Если commit убрать: row существует только в рамках незакоммиченной транзакции, + при db.close() (get_db.finally) соединение вернётся в пул без commit → rollback. + """ + from app.services.site_finder.custom_pois import create_custom_poi + + db = _make_db_with_row(_make_row_data()) + payload = CustomPoiCreate(name="Тест", weight=1.5, lon=60.605, lat=56.838) + + result = create_custom_poi(db, _SESSION, payload) + + db.commit.assert_called_once() + assert result.id == 1 + assert result.user_id == _SESSION + assert result.weight == pytest.approx(1.5) + + +def test_service_delete_custom_poi_commits_when_found() -> None: + """delete_custom_poi вызывает db.commit() когда запись найдена — regression guard #261.""" + from app.services.site_finder.custom_pois import delete_custom_poi + + db = _make_db_with_row(_make_row_data()) + db.execute.return_value.first.return_value = MagicMock() # RETURNING id truthy + + deleted = delete_custom_poi(db, poi_id=1, user_id=_SESSION) + + db.commit.assert_called_once() + assert deleted is True + + +def test_service_delete_custom_poi_no_commit_when_not_found() -> None: + """delete_custom_poi НЕ вызывает db.commit() если запись не найдена.""" + from app.services.site_finder.custom_pois import delete_custom_poi + + db = _make_db_with_row(None) + db.execute.return_value.first.return_value = None # RETURNING id пусто + + deleted = delete_custom_poi(db, poi_id=999, user_id=_SESSION) + + db.commit.assert_not_called() + assert deleted is False + + +def test_service_update_custom_poi_commits_when_fields_given() -> None: + """update_custom_poi вызывает db.commit() при наличии полей для обновления — regression #261.""" + from app.services.site_finder.custom_pois import update_custom_poi + + db = _make_db_with_row(_make_row_data(weight=2.0)) + payload = CustomPoiUpdate(weight=2.0) + + result = update_custom_poi(db, poi_id=1, user_id=_SESSION, payload=payload) + + db.commit.assert_called_once() + assert result is not None + + +def test_service_update_custom_poi_no_commit_when_payload_empty() -> None: + """update_custom_poi НЕ вызывает db.commit() если payload не содержит изменяемых полей.""" + from app.services.site_finder.custom_pois import update_custom_poi + + db = _make_db_with_row(_make_row_data()) + # Все поля None → sets = ["updated_at = NOW()"], len == 1 → UPDATE не выполняется + payload = CustomPoiUpdate() + + update_custom_poi(db, poi_id=1, user_id=_SESSION, payload=payload) + + db.commit.assert_not_called() + + +def test_service_update_custom_poi_no_commit_when_not_found() -> None: + """update_custom_poi НЕ вызывает db.commit() если POI не найдена.""" + from app.services.site_finder.custom_pois import update_custom_poi + + db = _make_db_with_row(None) # get_custom_poi вернёт None (POI не найдена) + payload = CustomPoiUpdate(weight=3.0) + + result = update_custom_poi(db, poi_id=999, user_id=_SESSION, payload=payload) + + db.commit.assert_not_called() + assert result is None diff --git a/backend/tests/api/v1/test_parcel_best_layouts.py b/backend/tests/api/v1/test_parcel_best_layouts.py new file mode 100644 index 00000000..70066681 --- /dev/null +++ b/backend/tests/api/v1/test_parcel_best_layouts.py @@ -0,0 +1,443 @@ +"""Тесты для POST /api/v1/parcels/{cad_num}/best-layouts (Issue #113 Phase 2.1). + +Mock-based — не требуют живой БД. +Паттерн mock DB: аналогично test_parcel_competitors.py — dependency_overrides[get_db]. + +Порядок вызовов в get_best_layouts (Fix SF-01 — inline velocity): + db.scalar() → MAX(snapshot_date) (только когда vel_rows non-empty) + db.execute() calls: + 1. _PARCEL_CENTROID_SQL → .mappings().first() + 2. _COMPETITORS_IN_RADIUS_SQL → .mappings().all() + 3. _INLINE_VELOCITY_SQL → .mappings().all() + 4. _SUPPLY_BATCH_SQL → .mappings().all() (пропускается если latest_snap is None) +""" + +from __future__ import annotations + +import datetime as dt +from unittest.mock import MagicMock + +import pytest +from fastapi.testclient import TestClient + +from app.main import app + +# ── Фабрики mock-строк ──────────────────────────────────────────────────────── + +CAD_NUM = "66:41:0303161:123" +_TODAY = dt.date.today() + + +def _coord_row(lon: float = 60.6, lat: float = 56.85) -> MagicMock: + """Центроид участка (EPSG:4326 lon/lat).""" + r = MagicMock() + r.__getitem__ = lambda self, k: {"center_lon": lon, "center_lat": lat}[k] + return r + + +def _obj_id_row(obj_id: int) -> MagicMock: + """Строка obj_id из _COMPETITORS_IN_RADIUS_SQL.""" + r = MagicMock() + r.__getitem__ = lambda self, k: {"obj_id": obj_id}[k] + return r + + +def _vel_row( + room_bucket: str = "2", + deals_window: float = 48.0, + avg_area: float = 55.0, + avg_price_rub: float | None = 120000.0, + obj_ids: list[int] | None = None, + window_start: dt.date | None = None, + window_end: dt.date | None = None, +) -> MagicMock: + """Строка из _INLINE_VELOCITY_SQL (Fix SF-01: deals_window за честный интервал).""" + oids = obj_ids if obj_ids is not None else [1] + ws = window_start or _TODAY - dt.timedelta(days=90) + we = window_end or _TODAY + + r = MagicMock() + r.__getitem__ = lambda self, k: { + "room_bucket": room_bucket, + "deals_window": deals_window, + "avg_area_m2": avg_area, + "avg_price_per_m2_rub": avg_price_rub, + "competitor_obj_ids": oids, + "competitor_count": len(oids), + "window_start": ws, + "window_end": we, + }[k] + return r + + +def _supply_row(rb: str, ab: str, units: int) -> MagicMock: + """Строка из _SUPPLY_BATCH_SQL.""" + r = MagicMock() + r.__getitem__ = lambda self, k: {"rb": rb, "ab": ab, "units": units}[k] + return r + + +# ── Построение mock DB ──────────────────────────────────────────────────────── + + +def _make_db( + coord: MagicMock | None = None, + id_rows: list[MagicMock] | None = None, + vel_rows: list[MagicMock] | None = None, + supply_rows: list[MagicMock] | None = None, + latest_snap: dt.date | None = None, +) -> MagicMock: + """Сконструировать mock Session. + + db.scalar() возвращает latest_snap (MAX snapshot_date) — вызывается перед supply. + Порядок db.execute(): + 1. centroid → .mappings().first() + 2. competitors-in-radius → .mappings().all() + 3. velocity → .mappings().all() + 4. supply → .mappings().all() (только если latest_snap is not None) + """ + db = MagicMock() + + # db.scalar — pre-computed MAX(snapshot_date) для supply query + db.scalar.return_value = latest_snap if latest_snap is not None else _TODAY + + results: list[MagicMock] = [] + + # 1: centroid + r0 = MagicMock() + r0.mappings.return_value.first.return_value = coord + results.append(r0) + + # 2: competitors-in-radius + r1 = MagicMock() + r1.mappings.return_value.all.return_value = id_rows or [] + results.append(r1) + + # 3: velocity (only queried if id_rows non-empty) + r2 = MagicMock() + r2.mappings.return_value.all.return_value = vel_rows or [] + results.append(r2) + + # 4: supply + r3 = MagicMock() + r3.mappings.return_value.all.return_value = supply_rows or [] + results.append(r3) + + db.execute.side_effect = results + return db + + +def _override_db(db: MagicMock): + def _get_db_override(): + yield db + + return _get_db_override + + +def _post(client: TestClient, cad: str = CAD_NUM, **body_kwargs) -> dict: + payload = {"radius_km": 1.0, "time_window": "last_quarter", **body_kwargs} + resp = client.post(f"/api/v1/parcels/{cad}/best-layouts", json=payload) + return resp + + +# ── Тесты ───────────────────────────────────────────────────────────────────── + + +def test_parcel_not_found_404() -> None: + """Если центроид не найден → 404.""" + db = _make_db(coord=None) + from app.core.db import get_db + + app.dependency_overrides[get_db] = _override_db(db) + try: + resp = _post(TestClient(app), cad="99:99:9999999:999") + assert resp.status_code == 404, resp.text + finally: + app.dependency_overrides.clear() + + +def test_empty_competitor_set_returns_low_confidence() -> None: + """Нет конкурентов в радиусе → пустые top_layouts + confidence=low.""" + db = _make_db(coord=_coord_row(), id_rows=[]) + from app.core.db import get_db + + app.dependency_overrides[get_db] = _override_db(db) + try: + resp = _post(TestClient(app)) + assert resp.status_code == 200, resp.text + body = resp.json() + assert body["top_layouts"] == [] + assert body["data_quality"]["confidence"] == "low" + assert body["data_quality"]["objects_total_in_radius"] == 0 + rec = body["recommendation_for_tz"] + assert rec["based_on_obj_count"] == 0 + assert rec["based_on_total_deals"] == 0 + assert rec["mix"] == [] + finally: + app.dependency_overrides.clear() + + +def test_three_obj_ids_ranking_and_pct_sum_100() -> None: + """3 obj_id, 3 room_buckets — ranking по velocity, sum pct = 100. + + last_quarter (3 мес): velocity = deals_window / 3.0 + studio: 9/3=3.0, 1: 24/3=8.0, 2: 48/3=16.0 → rank1="2" + """ + id_rows = [_obj_id_row(1), _obj_id_row(2), _obj_id_row(3)] + vel_rows = [ + _vel_row("studio", deals_window=9.0, avg_area=26.0, obj_ids=[1]), + _vel_row("1", deals_window=24.0, avg_area=40.0, obj_ids=[2]), + _vel_row("2", deals_window=48.0, avg_area=55.0, obj_ids=[3]), + ] + supply_rows = [ + _supply_row("studio", "<25", 20), + _supply_row("1", "40-60", 60), + _supply_row("2", "40-60", 80), + ] + db = _make_db(coord=_coord_row(), id_rows=id_rows, vel_rows=vel_rows, supply_rows=supply_rows) + from app.core.db import get_db + + app.dependency_overrides[get_db] = _override_db(db) + try: + resp = _post(TestClient(app), time_window="last_quarter") + assert resp.status_code == 200, resp.text + body = resp.json() + top = body["top_layouts"] + assert len(top) == 3 + # rank 1 = самая высокая velocity (2-комн: 48/3=16.0 per month) + assert top[0]["rank"] == 1 + assert top[0]["room_bucket"] == "2" + # все ранги уникальны + assert sorted(t["rank"] for t in top) == [1, 2, 3] + # sum pct = 100 + mix = body["recommendation_for_tz"]["mix"] + assert sum(m["pct"] for m in mix) == 100 + finally: + app.dependency_overrides.clear() + + +def test_exclude_competitor_obj_ids_filter() -> None: + """exclude_competitor_obj_ids исключает obj_id: при all excluded → пустой ответ.""" + # Если после исключения obj_id_list пуст → _empty_response → top_layouts=[] + id_rows = [_obj_id_row(20)] # единственный конкурент + db = _make_db(coord=_coord_row(), id_rows=id_rows, vel_rows=[]) + from app.core.db import get_db + + app.dependency_overrides[get_db] = _override_db(db) + try: + resp = _post(TestClient(app), exclude_competitor_obj_ids=[20]) + assert resp.status_code == 200, resp.text + body = resp.json() + # После исключения obj_id=20 список пуст → пустой ответ + assert body["top_layouts"] == [] + assert body["data_quality"]["confidence"] == "low" + # objects_total_in_radius = 1 (до исключения) + assert body["data_quality"]["objects_total_in_radius"] == 1 + finally: + app.dependency_overrides.clear() + + +def test_min_velocity_per_month_filters_low_rows() -> None: + """min_velocity_per_month=5 → строки с velocity<5 не попадают в top_layouts. + + last_quarter (3 мес): studio=6/3=2.0 < 5.0 (убран), 1=30/3=10.0 > 5.0 (остаётся). + """ + id_rows = [_obj_id_row(1), _obj_id_row(2)] + vel_rows = [ + _vel_row("studio", deals_window=6.0, obj_ids=[1]), + _vel_row("1", deals_window=30.0, obj_ids=[2]), + ] + db = _make_db(coord=_coord_row(), id_rows=id_rows, vel_rows=vel_rows) + from app.core.db import get_db + + app.dependency_overrides[get_db] = _override_db(db) + try: + resp = _post(TestClient(app), min_velocity_per_month=5.0) + assert resp.status_code == 200, resp.text + body = resp.json() + top = body["top_layouts"] + assert len(top) == 1 + assert top[0]["room_bucket"] == "1" + assert top[0]["velocity_per_month"] == pytest.approx(10.0) + finally: + app.dependency_overrides.clear() + + +def test_time_window_velocity_scaling() -> None: + """last_month vs last_year дают разный velocity_per_month для одних deals.""" + # sum_deals=24 → last_month: 24/24=1.0, last_year: 24/2=12.0 + id_rows = [_obj_id_row(1)] + vel_rows_fixed = [_vel_row("2", sum_deals=24.0, obj_ids=[1])] + + from app.core.db import get_db + + # last_month + db_m = _make_db(coord=_coord_row(), id_rows=id_rows, vel_rows=vel_rows_fixed) + app.dependency_overrides[get_db] = _override_db(db_m) + try: + resp_m = _post(TestClient(app), time_window="last_month") + assert resp_m.status_code == 200, resp_m.text + v_month = resp_m.json()["top_layouts"][0]["velocity_per_month"] + finally: + app.dependency_overrides.clear() + + # last_year + db_y = _make_db(coord=_coord_row(), id_rows=id_rows, vel_rows=vel_rows_fixed) + app.dependency_overrides[get_db] = _override_db(db_y) + try: + resp_y = _post(TestClient(app), time_window="last_year") + assert resp_y.status_code == 200, resp_y.text + v_year = resp_y.json()["top_layouts"][0]["velocity_per_month"] + finally: + app.dependency_overrides.clear() + + # last_year velocity должна быть выше (делитель меньше: 2 vs 24) + assert v_year > v_month + assert v_month == pytest.approx(1.0) + assert v_year == pytest.approx(12.0) + + +def test_obj_class_filter_passes_through() -> None: + """obj_class_filter передаётся в SQL — endpoint не ломается, возвращает 200.""" + db = _make_db( + coord=_coord_row(), + id_rows=[_obj_id_row(5)], + vel_rows=[_vel_row("2", obj_ids=[5])], + supply_rows=[], + ) + from app.core.db import get_db + + app.dependency_overrides[get_db] = _override_db(db) + try: + resp = _post(TestClient(app), obj_class_filter="comfort") + assert resp.status_code == 200, resp.text + body = resp.json() + assert len(body["top_layouts"]) > 0 + finally: + app.dependency_overrides.clear() + + +def test_mv_empty_for_competitors_returns_empty_top_layouts() -> None: + """Конкуренты есть в радиусе, но MV пустой → top_layouts=[], data_quality.confidence=low.""" + id_rows = [_obj_id_row(1), _obj_id_row(2)] + db = _make_db(coord=_coord_row(), id_rows=id_rows, vel_rows=[]) + from app.core.db import get_db + + app.dependency_overrides[get_db] = _override_db(db) + try: + resp = _post(TestClient(app)) + assert resp.status_code == 200, resp.text + body = resp.json() + assert body["top_layouts"] == [] + dq = body["data_quality"] + assert dq["objects_total_in_radius"] == 2 + assert dq["objects_with_velocity_data"] == 0 + assert dq["confidence"] == "low" + finally: + app.dependency_overrides.clear() + + +def test_target_total_flats_fills_abs_units() -> None: + """target_total_flats=100 → abs_units заполнен в mix, sum примерно = 100.""" + id_rows = [_obj_id_row(1), _obj_id_row(2)] + vel_rows = [ + _vel_row("1", sum_deals=60.0, obj_ids=[1]), + _vel_row("2", sum_deals=40.0, obj_ids=[2]), + ] + db = _make_db(coord=_coord_row(), id_rows=id_rows, vel_rows=vel_rows) + from app.core.db import get_db + + app.dependency_overrides[get_db] = _override_db(db) + try: + resp = _post(TestClient(app), target_total_flats=100) + assert resp.status_code == 200, resp.text + mix = resp.json()["recommendation_for_tz"]["mix"] + # все abs_units заполнены + for m in mix: + assert m["abs_units"] is not None + # сумма abs_units близка к 100 (round-off ±1) + total_abs = sum(m["abs_units"] for m in mix) + assert 98 <= total_abs <= 102 + finally: + app.dependency_overrides.clear() + + +def test_sold_pct_clamped_at_100_and_is_oversold_flag() -> None: + """raw sold_pct > 100 → returned sold_pct_of_supply=100.0, is_oversold=True.""" + id_rows = [_obj_id_row(1)] + # sum_deals=199, supply=100 → raw = 199% (несопоставимые окна) + vel_rows = [_vel_row("2", sum_deals=199.0, obj_ids=[1])] + supply_rows = [_supply_row("2", "40-60", 100)] + db = _make_db( + coord=_coord_row(), + id_rows=id_rows, + vel_rows=vel_rows, + supply_rows=supply_rows, + latest_snap=dt.date.today(), + ) + from app.core.db import get_db + + app.dependency_overrides[get_db] = _override_db(db) + try: + resp = _post(TestClient(app)) + assert resp.status_code == 200, resp.text + body = resp.json() + top = body["top_layouts"] + assert len(top) == 1 + row = top[0] + assert row["sold_pct_of_supply"] == 100.0, "sold_pct_of_supply должен быть clamped до 100" + assert row["is_oversold"] is True, "is_oversold должен быть True когда raw > 100" + finally: + app.dependency_overrides.clear() + + +def test_sold_pct_below_100_is_not_oversold() -> None: + """raw sold_pct <= 100 → sold_pct_of_supply возвращается as-is, is_oversold=False.""" + id_rows = [_obj_id_row(1)] + # sum_deals=50, supply=100 → raw = 50% + vel_rows = [_vel_row("1", sum_deals=50.0, obj_ids=[1])] + supply_rows = [_supply_row("1", "40-60", 100)] + db = _make_db( + coord=_coord_row(), + id_rows=id_rows, + vel_rows=vel_rows, + supply_rows=supply_rows, + latest_snap=dt.date.today(), + ) + from app.core.db import get_db + + app.dependency_overrides[get_db] = _override_db(db) + try: + resp = _post(TestClient(app)) + assert resp.status_code == 200, resp.text + body = resp.json() + top = body["top_layouts"] + assert len(top) == 1 + row = top[0] + assert row["sold_pct_of_supply"] == pytest.approx(50.0) + assert row["is_oversold"] is False + finally: + app.dependency_overrides.clear() + + +def test_filter_competitor_obj_ids_applied() -> None: + """filter_competitor_obj_ids=[1] оставляет только obj_id=1.""" + id_rows = [_obj_id_row(1), _obj_id_row(2), _obj_id_row(3)] + # После фильтрации остаётся только obj_id=1, velocity запрос получит [1] + vel_rows = [_vel_row("2", sum_deals=24.0, obj_ids=[1])] + db = _make_db(coord=_coord_row(), id_rows=id_rows, vel_rows=vel_rows) + from app.core.db import get_db + + app.dependency_overrides[get_db] = _override_db(db) + try: + resp = _post(TestClient(app), filter_competitor_obj_ids=[1]) + assert resp.status_code == 200, resp.text + body = resp.json() + top = body["top_layouts"] + assert len(top) >= 1 + # competitor_obj_ids должен содержать только 1 + for row in top: + for oid in row["competitor_obj_ids"]: + assert oid == 1 + finally: + app.dependency_overrides.clear() diff --git a/backend/tests/api/v1/test_parcel_by_bbox.py b/backend/tests/api/v1/test_parcel_by_bbox.py new file mode 100644 index 00000000..6c60c9fe --- /dev/null +++ b/backend/tests/api/v1/test_parcel_by_bbox.py @@ -0,0 +1,152 @@ +"""Тесты GET /parcels/by-bbox (SF-B1). + +Проверяют: +1. Валидный bbox → 200 + корректная структура ответа. +2. Некорректный bbox (min_lat >= max_lat) → 400. +3. user_id передан → status заполняется из mock DB; без user_id → status=null. +""" + +from __future__ import annotations + +from typing import Any +from unittest.mock import MagicMock, patch + +import pytest +from fastapi.testclient import TestClient + +from app.main import app + + +def _make_db_row( + cad_num: str = "66:41:0204016:10", + centroid_lat: float = 56.83, + centroid_lon: float = 60.64, + area_m2: float = 1200.0, + land_category: str | None = "land_residential", + user_status: str | None = None, +) -> dict[str, Any]: + return { + "cad_num": cad_num, + "centroid_lat": centroid_lat, + "centroid_lon": centroid_lon, + "area_m2": area_m2, + "land_category": land_category, + "user_status": user_status, + } + + +def _build_mock_db(rows: list[dict[str, Any]]) -> MagicMock: + """Сконструировать mock Session, возвращающий rows при execute().mappings().all().""" + mock_db = MagicMock() + mappings_mock = MagicMock() + mappings_mock.all.return_value = rows + execute_result = MagicMock() + execute_result.mappings.return_value = mappings_mock + mock_db.execute.return_value = execute_result + return mock_db + + +@pytest.fixture() +def client() -> TestClient: + return TestClient(app) + + +# ── Test 1: валидный bbox возвращает корректную структуру ────────────────── + + +def test_by_bbox_valid_returns_structure(client: TestClient) -> None: + rows = [_make_db_row()] + mock_db = _build_mock_db(rows) + + with patch("app.api.v1.parcels.get_db", return_value=iter([mock_db])): + resp = client.get( + "/api/v1/parcels/by-bbox", + params={ + "min_lat": 56.80, + "min_lon": 60.60, + "max_lat": 56.90, + "max_lon": 60.70, + "limit": 200, + }, + ) + + assert resp.status_code == 200 + body = resp.json() + assert "parcels" in body + assert "count" in body + assert "limit" in body + assert "bbox_area_km2" in body + assert body["count"] == 1 + assert body["limit"] == 200 + + parcel = body["parcels"][0] + assert parcel["cad_num"] == "66:41:0204016:10" + assert parcel["centroid_lat"] == pytest.approx(56.83, abs=0.01) + assert parcel["centroid_lon"] == pytest.approx(60.64, abs=0.01) + assert parcel["area_m2"] == pytest.approx(1200.0) + assert parcel["land_category"] == "land_residential" + assert parcel["status"] is None # user_id не передан + assert parcel["last_analysis_date"] is None + + +# ── Test 2: некорректный bbox → 400 ─────────────────────────────────────── + + +def test_by_bbox_invalid_bbox_returns_400(client: TestClient) -> None: + resp = client.get( + "/api/v1/parcels/by-bbox", + params={ + "min_lat": 56.90, # min >= max → ошибка + "min_lon": 60.60, + "max_lat": 56.80, + "max_lon": 60.70, + }, + ) + assert resp.status_code == 400 + assert "bbox" in resp.json()["detail"].lower() + + +# ── Test 3: user_id → status из БД; без user_id → status null ───────────── + + +def test_by_bbox_status_overlay_with_user_id(client: TestClient) -> None: + rows = [_make_db_row(user_status="in_work")] + mock_db = _build_mock_db(rows) + + with patch("app.api.v1.parcels.get_db", return_value=iter([mock_db])): + resp = client.get( + "/api/v1/parcels/by-bbox", + params={ + "min_lat": 56.80, + "min_lon": 60.60, + "max_lat": 56.90, + "max_lon": 60.70, + "user_id": "user-abc-123", + }, + ) + + assert resp.status_code == 200 + parcel = resp.json()["parcels"][0] + assert parcel["status"] == "in_work" + + +def test_by_bbox_no_user_id_status_is_null(client: TestClient) -> None: + rows = [_make_db_row(user_status="favorite")] # DB вернёт статус + mock_db = _build_mock_db(rows) + + with patch("app.api.v1.parcels.get_db", return_value=iter([mock_db])): + resp = client.get( + "/api/v1/parcels/by-bbox", + params={ + "min_lat": 56.80, + "min_lon": 60.60, + "max_lat": 56.90, + "max_lon": 60.70, + # user_id не передан + }, + ) + + assert resp.status_code == 200 + parcel = resp.json()["parcels"][0] + # Без user_id статус принудительно null (endpoint не раскрывает чужие статусы) + assert parcel["status"] is None diff --git a/backend/tests/api/v1/test_parcel_competitors.py b/backend/tests/api/v1/test_parcel_competitors.py new file mode 100644 index 00000000..6e16499b --- /dev/null +++ b/backend/tests/api/v1/test_parcel_competitors.py @@ -0,0 +1,390 @@ +"""Тесты для POST /api/v1/parcels/{cad_num}/competitors (Issue #112). + +Mock-based — не требуют живой БД. +Паттерн mock DB: аналогично test_admin_cadastre.py — dependency_overrides[get_db]. +""" + +from __future__ import annotations + +from unittest.mock import MagicMock + +import pytest +from fastapi.testclient import TestClient + +from app.main import app + +# ── Фабрики mock-строк ──────────────────────────────────────────────────────── + + +def _coord_row(lat: float = 56.838, lon: float = 60.605) -> MagicMock: + """Строка центроида участка.""" + r = MagicMock() + r.__getitem__ = lambda self, k: {"lat": lat, "lon": lon}[k] + return r + + +def _obj_row( + obj_id: int = 1, + distance_m: float = 400.0, + site_status: str = "sales", + obj_class: str | None = "comfort", + velocity: float = 5.0, + flat_count: int | None = 200, +) -> MagicMock: + r = MagicMock() + r.__getitem__ = lambda self, k: { + "obj_id": obj_id, + "comm_name": f"ЖК-{obj_id}", + "dev_name": "TestDev", + "obj_class": obj_class, + "latitude": 56.838 + distance_m / 1_000_000, + "longitude": 60.605 + distance_m / 1_000_000, + "flat_count": flat_count, + "site_status": site_status, + "distance_m": distance_m, + "velocity_per_month": velocity, + }[k] + return r + + +def _price_row(obj_id: int, price: float) -> MagicMock: + r = MagicMock() + r.__getitem__ = lambda self, k: {"obj_id": obj_id, "avg_price_per_m2": price}[k] + return r + + +# ── Построение mock DB ──────────────────────────────────────────────────────── + + +def _make_db( + coord: MagicMock | None = None, + obj_rows: list[MagicMock] | None = None, + price_rows: list[MagicMock] | None = None, +) -> MagicMock: + """Сконструировать mock Session. + + Порядок вызовов execute: + 1. centroid query → coord + 2. competitors query → obj_rows + 3. avg_price query → price_rows + """ + db = MagicMock() + + results: list[MagicMock] = [] + for rows, is_first in [ + (coord, True), + (obj_rows or [], False), + (price_rows or [], False), + ]: + result = MagicMock() + if is_first: + # centroid → .mappings().first() + result.mappings.return_value.first.return_value = rows + else: + result.mappings.return_value.all.return_value = rows + results.append(result) + + db.execute.side_effect = results + return db + + +def _override_db(db: MagicMock): + def _get_db_override(): + yield db + + return _get_db_override + + +# ── Тесты ───────────────────────────────────────────────────────────────────── + + +def test_competitors_basic() -> None: + """3 конкурента → корректная форма ответа, сортировка по distance.""" + rows = [ + _obj_row(obj_id=1, distance_m=200.0, velocity=4.0), + _obj_row(obj_id=2, distance_m=500.0, velocity=6.0), + _obj_row(obj_id=3, distance_m=900.0, velocity=2.0), + ] + db = _make_db(coord=_coord_row(), obj_rows=rows) + + from app.core.db import get_db + + app.dependency_overrides[get_db] = _override_db(db) + try: + client = TestClient(app) + resp = client.post( + "/api/v1/parcels/66:41:0303161:123/competitors", + json={"radius_km": 1.0, "time_window": "last_quarter"}, + ) + assert resp.status_code == 200, resp.text + body = resp.json() + assert "competitors" in body + assert "summary" in body + assert len(body["competitors"]) == 3 + # первый должен быть ближайшим + assert body["competitors"][0]["obj_id"] == 1 + assert body["competitors"][0]["distance_m"] == pytest.approx(200.0) + # все поля присутствуют + first = body["competitors"][0] + for key in ( + "obj_id", + "comm_name", + "dev_name", + "obj_class", + "distance_m", + "lat", + "lng", + "stage", + "flats_total", + "flats_sold", + "sold_pct", + "velocity_per_month", + "avg_price_per_m2", + "is_active", + ): + assert key in first, f"missing key: {key}" + finally: + app.dependency_overrides.clear() + + +def test_competitors_summary_calc() -> None: + """summary: total_competitors, active_count, weighted_avg_velocity корректны.""" + rows = [ + _obj_row(obj_id=1, site_status="sales", velocity=10.0), + _obj_row(obj_id=2, site_status="construction", velocity=6.0), + _obj_row(obj_id=3, site_status="completed", velocity=2.0), + ] + db = _make_db(coord=_coord_row(), obj_rows=rows) + + from app.core.db import get_db + + app.dependency_overrides[get_db] = _override_db(db) + try: + client = TestClient(app) + resp = client.post( + "/api/v1/parcels/66:41:0303161:5/competitors", + json={"radius_km": 1.0, "time_window": "last_quarter"}, + ) + assert resp.status_code == 200, resp.text + summary = resp.json()["summary"] + assert summary["total_competitors"] == 3 + assert summary["active_count"] == 2 # sales + construction + # avg velocity = (10+6+2)/3 = 6.0 + assert summary["weighted_avg_velocity"] == pytest.approx(6.0) + assert summary["radius_km"] == pytest.approx(1.0) + assert summary["time_window"] == "last_quarter" + finally: + app.dependency_overrides.clear() + + +def test_competitors_exclude_obj_ids() -> None: + """exclude_obj_ids исключает указанные ЖК из результата.""" + rows = [ + _obj_row(obj_id=1, distance_m=100.0), + _obj_row(obj_id=2, distance_m=200.0), + _obj_row(obj_id=3, distance_m=300.0), + ] + db = _make_db(coord=_coord_row(), obj_rows=rows) + + from app.core.db import get_db + + app.dependency_overrides[get_db] = _override_db(db) + try: + client = TestClient(app) + resp = client.post( + "/api/v1/parcels/66:41:0303161:5/competitors", + json={"radius_km": 1.0, "time_window": "last_quarter", "exclude_obj_ids": [2]}, + ) + assert resp.status_code == 200, resp.text + ids = [c["obj_id"] for c in resp.json()["competitors"]] + assert 2 not in ids + assert 1 in ids + assert 3 in ids + finally: + app.dependency_overrides.clear() + + +def test_competitors_obj_class_filter() -> None: + """obj_class_filter=economy — SQL получает параметр; Python-сторона не ломается.""" + rows = [_obj_row(obj_id=10, obj_class="economy")] + db = _make_db(coord=_coord_row(), obj_rows=rows) + + from app.core.db import get_db + + app.dependency_overrides[get_db] = _override_db(db) + try: + client = TestClient(app) + resp = client.post( + "/api/v1/parcels/66:41:0303161:5/competitors", + json={"radius_km": 1.0, "time_window": "last_quarter", "obj_class_filter": "economy"}, + ) + assert resp.status_code == 200, resp.text + comps = resp.json()["competitors"] + assert len(comps) == 1 + assert comps[0]["obj_class"] == "economy" + finally: + app.dependency_overrides.clear() + + +def test_competitors_time_window_velocity() -> None: + """time_window влияет на velocity_per_month (last_month vs last_year).""" + # Здесь проверяем, что endpoint принимает оба варианта без ошибок + # и возвращает velocity из mock-строки (DB-расчёт мокирован). + rows_month = [_obj_row(obj_id=1, velocity=12.0)] + rows_year = [_obj_row(obj_id=1, velocity=3.0)] + + from app.core.db import get_db + + # last_month + db_m = _make_db(coord=_coord_row(), obj_rows=rows_month) + app.dependency_overrides[get_db] = _override_db(db_m) + try: + client = TestClient(app) + resp_m = client.post( + "/api/v1/parcels/66:41:0303161:5/competitors", + json={"radius_km": 1.0, "time_window": "last_month"}, + ) + assert resp_m.status_code == 200, resp_m.text + finally: + app.dependency_overrides.clear() + + # last_year + db_y = _make_db(coord=_coord_row(), obj_rows=rows_year) + app.dependency_overrides[get_db] = _override_db(db_y) + try: + client = TestClient(app) + resp_y = client.post( + "/api/v1/parcels/66:41:0303161:5/competitors", + json={"radius_km": 1.0, "time_window": "last_year"}, + ) + assert resp_y.status_code == 200, resp_y.text + v_month = resp_m.json()["competitors"][0]["velocity_per_month"] + v_year = resp_y.json()["competitors"][0]["velocity_per_month"] + # month velocity выше чем year в нашем моке + assert v_month > v_year + finally: + app.dependency_overrides.clear() + + +def test_competitors_parcel_not_found_404() -> None: + """Если центроид участка не найден → 404.""" + db = _make_db(coord=None) # first() вернёт None + + from app.core.db import get_db + + app.dependency_overrides[get_db] = _override_db(db) + try: + client = TestClient(app) + resp = client.post( + "/api/v1/parcels/99:99:9999999:999/competitors", + json={"radius_km": 1.0, "time_window": "last_quarter"}, + ) + assert resp.status_code == 404, resp.text + finally: + app.dependency_overrides.clear() + + +def test_competitors_empty_radius() -> None: + """Нет конкурентов в радиусе → пустой список + summary с нулями.""" + db = _make_db(coord=_coord_row(), obj_rows=[]) + + from app.core.db import get_db + + app.dependency_overrides[get_db] = _override_db(db) + try: + client = TestClient(app) + resp = client.post( + "/api/v1/parcels/66:41:0303161:5/competitors", + json={"radius_km": 0.1, "time_window": "last_quarter"}, + ) + assert resp.status_code == 200, resp.text + body = resp.json() + assert body["competitors"] == [] + assert body["summary"]["total_competitors"] == 0 + assert body["summary"]["active_count"] == 0 + assert body["summary"]["weighted_avg_velocity"] == pytest.approx(0.0) + finally: + app.dependency_overrides.clear() + + +def test_competitors_sold_pct_null() -> None: + """sold_pct и flats_sold — None (MVP: данные недоступны из domrf_kn_objects). + + Полный расчёт продаж требует JOIN с domrf_kn_flats COUNT WHERE status='sold' + — отложен за пределы текущего PR. + """ + rows = [_obj_row(obj_id=1, flat_count=200)] + db = _make_db(coord=_coord_row(), obj_rows=rows) + + from app.core.db import get_db + + app.dependency_overrides[get_db] = _override_db(db) + try: + client = TestClient(app) + resp = client.post( + "/api/v1/parcels/66:41:0303161:5/competitors", + json={"radius_km": 1.0, "time_window": "last_quarter"}, + ) + assert resp.status_code == 200, resp.text + comp = resp.json()["competitors"][0] + assert comp["flats_sold"] is None + assert comp["sold_pct"] is None + assert comp["flats_total"] == 200 + finally: + app.dependency_overrides.clear() + + +def test_competitors_avg_price_populated() -> None: + """avg_price_per_m2 не None если domrf_kn_flats возвращает строки с ценой. + + Регрессионный тест для Issue #227: фильтр status='sold' давал 0 строк + (поле status в domrf_kn_flats 99.8% NULL). После фикса — убран, AVG + считается по всем квартирам с known price_per_m2. + """ + rows = [_obj_row(obj_id=1)] + price_rows = [_price_row(obj_id=1, price=150_000.0)] + db = _make_db(coord=_coord_row(), obj_rows=rows, price_rows=price_rows) + + from app.core.db import get_db + + app.dependency_overrides[get_db] = _override_db(db) + try: + client = TestClient(app) + resp = client.post( + "/api/v1/parcels/66:41:0303161:5/competitors", + json={"radius_km": 1.0, "time_window": "last_quarter"}, + ) + assert resp.status_code == 200, resp.text + comp = resp.json()["competitors"][0] + assert comp["avg_price_per_m2"] == pytest.approx( + 150_000.0 + ), "avg_price_per_m2 должен быть не None — регрессия #227 status='sold' filter" + finally: + app.dependency_overrides.clear() + + +def test_competitors_is_active_flag() -> None: + """is_active=True для sales/construction, False для completed/null.""" + rows = [ + _obj_row(obj_id=1, site_status="sales"), + _obj_row(obj_id=2, site_status="construction"), + _obj_row(obj_id=3, site_status="completed"), + ] + db = _make_db(coord=_coord_row(), obj_rows=rows) + + from app.core.db import get_db + + app.dependency_overrides[get_db] = _override_db(db) + try: + client = TestClient(app) + resp = client.post( + "/api/v1/parcels/66:41:0303161:5/competitors", + json={"radius_km": 1.0, "time_window": "last_quarter"}, + ) + assert resp.status_code == 200, resp.text + comps = {c["obj_id"]: c for c in resp.json()["competitors"]} + assert comps[1]["is_active"] is True + assert comps[2]["is_active"] is True + assert comps[3]["is_active"] is False + finally: + app.dependency_overrides.clear() diff --git a/backend/tests/api/v1/test_parcel_connection_points.py b/backend/tests/api/v1/test_parcel_connection_points.py new file mode 100644 index 00000000..a26fa16d --- /dev/null +++ b/backend/tests/api/v1/test_parcel_connection_points.py @@ -0,0 +1,267 @@ +"""Тесты для GET /{cad_num}/connection-points (issue #115). + +Использует FastAPI TestClient с mock DB — без реального Postgres. +""" + +from __future__ import annotations + +from typing import Any +from unittest.mock import MagicMock, patch + +import pytest +from fastapi.testclient import TestClient + +from app.main import app + +# ── Helpers ─────────────────────────────────────────────────────────────────── + +_VALID_CAD = "66:41:0204016:10" +_QUARTER = "66:41:0204016" + + +def _mock_get_db(db: Any): + """FastAPI dependency override factory.""" + + def _get_db_override(): + yield db + + return _get_db_override + + +def _make_structure(distance_m: float = 120.5) -> dict[str, Any]: + return { + "name": "ТП-101", + "type": "Трансформаторная подстанция", + "cad_num": None, + "distance_to_boundary_m": distance_m, + "geometry_geojson": {"type": "Point", "coordinates": [60.6, 56.8]}, + "readable_address": None, + "raw_props": {"name": "ТП-101"}, + "source": "nspd_36328", + } + + +def _make_zouit_overlap() -> dict[str, Any]: + return { + "reg_numb_border": "RN-001", + "type_zone": "Охранная зона ЛЭП", + "subcategory": 5, + "intersects_parcel": True, + "geometry_geojson": {"type": "Polygon", "coordinates": [[]]}, + "raw_props": {"type_zone": "Охранная зона ЛЭП"}, + "source": "nspd_37578", + } + + +def _make_summary( + nearest: float | None = 120.5, + in_zone: bool = False, + zones_count: int = 0, + total: int = 1, +) -> dict[str, Any]: + return { + "nearest_structure_distance_m": nearest, + "in_protection_zone": in_zone, + "protection_zones_intersecting": zones_count, + "total_structures_in_radius": total, + } + + +def _make_full_response( + structures: list[dict[str, Any]] | None = None, + overlaps: list[dict[str, Any]] | None = None, + summary: dict[str, Any] | None = None, + dump_available: bool = True, + dump_fetched_at: str | None = "2026-05-01T12:00:00+00:00", +) -> dict[str, Any]: + if structures is None: + structures = [_make_structure()] + if overlaps is None: + overlaps = [] + if summary is None: + summary = _make_summary(total=len(structures)) + return { + "engineering_structures": structures, + "zouit_engineering_overlaps": overlaps, + "summary": summary, + "dump_available": dump_available, + "dump_fetched_at": dump_fetched_at, + } + + +# ── Tests ───────────────────────────────────────────────────────────────────── + + +def test_connection_points_no_dump_returns_empty() -> None: + """Квартал без dump → dump_available=false, пустые массивы, 200 OK.""" + from app.core.db import get_db + + db = MagicMock() + app.dependency_overrides[get_db] = _mock_get_db(db) + + empty_response = { + "engineering_structures": [], + "zouit_engineering_overlaps": [], + "summary": { + "nearest_structure_distance_m": None, + "in_protection_zone": False, + "protection_zones_intersecting": 0, + "total_structures_in_radius": 0, + }, + "dump_available": False, + "dump_fetched_at": None, + } + + try: + with patch( + "app.api.v1.parcels.get_connection_points", + return_value=empty_response, + ): + client = TestClient(app) + response = client.get(f"/api/v1/parcels/{_VALID_CAD}/connection-points") + + assert response.status_code == 200, response.text + body = response.json() + assert body["dump_available"] is False + assert body["engineering_structures"] == [] + assert body["zouit_engineering_overlaps"] == [] + assert body["summary"]["total_structures_in_radius"] == 0 + assert body["summary"]["nearest_structure_distance_m"] is None + finally: + app.dependency_overrides.clear() + + +def test_connection_points_parcel_not_found_404() -> None: + """cad_num не найден в БД (ValueError из сервиса) → 404.""" + from app.core.db import get_db + + db = MagicMock() + app.dependency_overrides[get_db] = _mock_get_db(db) + + try: + with patch( + "app.api.v1.parcels.get_connection_points", + side_effect=ValueError("Участок '66:41:9999999:1' не найден в БД"), + ): + client = TestClient(app) + response = client.get("/api/v1/parcels/66:41:9999999:1/connection-points") + + assert response.status_code == 404 + assert "не найден" in response.json()["detail"] + finally: + app.dependency_overrides.clear() + + +def test_connection_points_filters_by_radius() -> None: + """Фичи за radius_m не попадают в ответ (сервис фильтрует). + + Мокаем сервис: при radius_m=100 возвращаем только близкую структуру, + при radius_m=500 — обе. Проверяем что endpoint передаёт radius_m в сервис. + """ + from app.core.db import get_db + + db = MagicMock() + app.dependency_overrides[get_db] = _mock_get_db(db) + + close_only = _make_full_response(structures=[_make_structure(distance_m=80.0)]) + far_included = _make_full_response( + structures=[_make_structure(distance_m=80.0), _make_structure(distance_m=450.0)], + summary=_make_summary(nearest=80.0, total=2), + ) + + try: + with patch("app.api.v1.parcels.get_connection_points") as mock_svc: + mock_svc.return_value = close_only + client = TestClient(app) + r100 = client.get(f"/api/v1/parcels/{_VALID_CAD}/connection-points?radius_m=100") + assert r100.status_code == 200 + assert r100.json()["summary"]["total_structures_in_radius"] == 1 + # Проверяем что radius_m=100 передан в сервис + call_kwargs = mock_svc.call_args + assert call_kwargs[0][2] == 100 or call_kwargs[1].get("radius_m") == 100 + + mock_svc.return_value = far_included + r500 = client.get(f"/api/v1/parcels/{_VALID_CAD}/connection-points?radius_m=500") + assert r500.status_code == 200 + assert r500.json()["summary"]["total_structures_in_radius"] == 2 + finally: + app.dependency_overrides.clear() + + +def test_connection_points_zouit_intersects_flag() -> None: + """zouit_engineering_overlaps содержат intersects_parcel=true и правильные поля.""" + from app.core.db import get_db + + db = MagicMock() + app.dependency_overrides[get_db] = _mock_get_db(db) + + overlap = _make_zouit_overlap() + full = _make_full_response( + overlaps=[overlap], + summary=_make_summary(in_zone=True, zones_count=1), + ) + + try: + with patch( + "app.api.v1.parcels.get_connection_points", + return_value=full, + ): + client = TestClient(app) + response = client.get(f"/api/v1/parcels/{_VALID_CAD}/connection-points") + + assert response.status_code == 200 + body = response.json() + assert body["summary"]["in_protection_zone"] is True + assert body["summary"]["protection_zones_intersecting"] == 1 + overlaps = body["zouit_engineering_overlaps"] + assert len(overlaps) == 1 + assert overlaps[0]["intersects_parcel"] is True + assert overlaps[0]["reg_numb_border"] == "RN-001" + assert overlaps[0]["source"] == "nspd_37578" + finally: + app.dependency_overrides.clear() + + +def test_summary_nearest_distance() -> None: + """summary.nearest_structure_distance_m = расстояние до ближайшей структуры.""" + from app.core.db import get_db + + db = MagicMock() + app.dependency_overrides[get_db] = _mock_get_db(db) + + s1 = _make_structure(distance_m=42.7) + s2 = _make_structure(distance_m=180.0) + full = _make_full_response( + structures=[s1, s2], + summary=_make_summary(nearest=42.7, total=2), + ) + + try: + with patch( + "app.api.v1.parcels.get_connection_points", + return_value=full, + ): + client = TestClient(app) + response = client.get(f"/api/v1/parcels/{_VALID_CAD}/connection-points") + + assert response.status_code == 200 + body = response.json() + assert body["summary"]["nearest_structure_distance_m"] == pytest.approx(42.7) + assert body["summary"]["total_structures_in_radius"] == 2 + assert body["engineering_structures"][0]["distance_to_boundary_m"] == pytest.approx(42.7) + finally: + app.dependency_overrides.clear() + + +def test_connection_points_radius_out_of_range_422() -> None: + """radius_m=10 (< min 50) → 422 Unprocessable Entity.""" + client = TestClient(app) + response = client.get(f"/api/v1/parcels/{_VALID_CAD}/connection-points?radius_m=10") + assert response.status_code == 422 + + +def test_connection_points_radius_too_large_422() -> None: + """radius_m=5000 (> max 2000) → 422 Unprocessable Entity.""" + client = TestClient(app) + response = client.get(f"/api/v1/parcels/{_VALID_CAD}/connection-points?radius_m=5000") + assert response.status_code == 422 diff --git a/backend/tests/integration/__init__.py b/backend/tests/integration/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/tests/integration/conftest.py b/backend/tests/integration/conftest.py new file mode 100644 index 00000000..1ad9c920 --- /dev/null +++ b/backend/tests/integration/conftest.py @@ -0,0 +1,153 @@ +"""Integration test fixtures для phantom column gate. + +Использует реальный PostgreSQL через SSH-туннель (localhost:15432). +Тесты пропускаются если TEST_DATABASE_URL не задан. + +Подход (Option B): временная schema в prod-Postgres (не трогаем данные). + 1. CREATE SCHEMA test_phantom_ + 2. Копируем DDL public → временная schema через pg_dump --schema-only + 3. Выполняем EXPLAIN-запросы (не EXECUTE — readonly) + 4. DROP SCHEMA CASCADE после теста + +Credentials: TEST_DATABASE_URL (env var, только через SSH-туннель). +Пример: postgresql+psycopg://user:pass@localhost:15432/gendesign +""" + +from __future__ import annotations + +import logging +import os +import random +import string +import subprocess +from collections.abc import Generator + +import pytest +from sqlalchemy import create_engine, text +from sqlalchemy.orm import Session, sessionmaker + +logger = logging.getLogger(__name__) + +_TEST_DATABASE_URL = os.environ.get("TEST_DATABASE_URL", "") + +# Используется во всех тестах этого пакета: пропускаем без TEST_DATABASE_URL +_SKIP_REASON = "TEST_DATABASE_URL не задан — интеграционные тесты требуют SSH-туннель" + +requires_test_db = pytest.mark.skipif( + not _TEST_DATABASE_URL, + reason=_SKIP_REASON, +) + + +def _random_suffix(n: int = 8) -> str: + return "".join(random.choices(string.ascii_lowercase + string.digits, k=n)) + + +def _pg_dump_schema_only(db_url: str, schema: str = "public") -> str: + """Получить DDL через pg_dump --schema-only --schema=. + + Возвращает SQL как строку. Требует pg_dump в PATH. + Credentials извлекаются из db_url. + + Fallback: если pg_dump недоступен — возвращает пустую строку, + и тогда тест работает без создания отдельной schema (тестирует прямо в public). + """ + try: + result = subprocess.run( + [ + "pg_dump", + "--schema-only", + f"--schema={schema}", + "--no-owner", + "--no-acl", + db_url.replace("+psycopg", ""), # pg_dump не понимает +psycopg dialect + ], + capture_output=True, + text=True, + timeout=30, + ) + if result.returncode == 0: + return result.stdout + logger.warning("pg_dump failed (rc=%d): %s", result.returncode, result.stderr[:200]) + except FileNotFoundError: + logger.warning("pg_dump не найден в PATH — phantom gate будет тестировать в public schema") + except subprocess.TimeoutExpired: + logger.warning("pg_dump timeout — phantom gate будет тестировать в public schema") + return "" + + +@pytest.fixture(scope="session") +def phantom_schema_name() -> str: + """Имя временной schema для phantom column тестов.""" + return f"test_phantom_{_random_suffix()}" + + +@pytest.fixture(scope="session") +def phantom_check_session(phantom_schema_name: str) -> Generator[Session, None, None]: + """SQLAlchemy Session, настроенная на временную schema. + + Если pg_dump доступен — создаёт временную schema с копией DDL из public. + Если нет — работает напрямую с public schema (EXPLAIN не требует изменения данных). + После завершения сессии — DROP SCHEMA CASCADE. + + Пропускается если TEST_DATABASE_URL не задан. + """ + if not _TEST_DATABASE_URL: + pytest.skip(_SKIP_REASON) + + engine = create_engine( + _TEST_DATABASE_URL, + # Без пула — это одноразовая тестовая сессия + pool_pre_ping=True, + pool_size=1, + max_overflow=0, + echo=False, + ) + session_factory = sessionmaker(bind=engine, autoflush=False, autocommit=False) + + # Попробуем создать временную schema с копией DDL + ddl_sql = _pg_dump_schema_only(_TEST_DATABASE_URL) + use_temp_schema = bool(ddl_sql) + schema = phantom_schema_name if use_temp_schema else "public" + # Track schema creation independently — needed for teardown even if DDL apply fails + schema_created = False + + if use_temp_schema: + with engine.connect() as conn: + conn.execute(text(f'CREATE SCHEMA IF NOT EXISTS "{schema}"')) + conn.commit() + schema_created = True + ddl_for_schema = ddl_sql.replace( + "SET search_path = public", f'SET search_path = "{schema}"' + ).replace("search_path TO public", f'search_path TO "{schema}"') + prefixed_ddl = f'SET search_path TO "{schema}", public;\n' + ddl_for_schema + try: + conn.execute(text(prefixed_ddl)) + conn.commit() + except Exception as e: + logger.warning("DDL apply to temp schema failed: %s — falling back to public", e) + use_temp_schema = False + schema = "public" + conn.rollback() + + db = session_factory() + # Устанавливаем search_path для session: temp schema первая, public как fallback + # (pg_dump DDL содержит объекты только под этой schema) + if use_temp_schema: + db.execute(text(f'SET search_path TO "{schema}", public')) + else: + db.execute(text("SET search_path TO public")) + + try: + yield db + finally: + db.close() + # Drop temp schema if it was created — even if DDL apply later failed. + # Otherwise schemas leak when CREATE SCHEMA succeeds but DDL apply raises. + if schema_created: + drop_target = phantom_schema_name + with engine.connect() as conn: + conn.execute(text(f'DROP SCHEMA IF EXISTS "{drop_target}" CASCADE')) + conn.commit() + logger.info("phantom gate: dropped temp schema %s", drop_target) + engine.dispose() diff --git a/backend/tests/integration/test_phantom_columns.py b/backend/tests/integration/test_phantom_columns.py new file mode 100644 index 00000000..0f395661 --- /dev/null +++ b/backend/tests/integration/test_phantom_columns.py @@ -0,0 +1,379 @@ +"""Phantom column gate — EXPLAIN-тесты против реальной Postgres schema. + +Каждый тест выполняет EXPLAIN (не EXECUTE!) на SQL из production services. +Если колонка не существует — Postgres бросит ошибку при планировании → тест fail. +Если таблица не существует — аналогично. + +Запуск: + # Без TEST_DATABASE_URL — все тесты skip + uv run pytest tests/integration/ -v + + # С SSH-туннелем (ssh -N gendesign → localhost:15432): + export TEST_DATABASE_URL="postgresql+psycopg://user:pass@localhost:15432/db" + uv run pytest tests/integration/test_phantom_columns.py -v -m integration + +Детектируемые классы багов: + - Ссылка на несуществующую колонку (domrf_kn_objects.geom_3857 — PR #196) + - Ссылка на несуществующую колонку (ekburg_construction_permits.units_count — PR #213) + - Ссылка на несуществующую таблицу / view + - Опечатки в именах колонок и таблиц +""" + +from __future__ import annotations + +import pytest +from sqlalchemy import text +from sqlalchemy.orm import Session + +from tests.integration.conftest import requires_test_db + +# ── Маркеры ─────────────────────────────────────────────────────────────────── + +pytestmark = [ + requires_test_db, + pytest.mark.integration, +] + + +# ── helpers ─────────────────────────────────────────────────────────────────── + + +def _explain(db: Session, query: str, params: dict | None = None) -> None: + """Выполнить EXPLAIN для query. Не выполняет DML — только план запроса. + + Бросает исключение (тест fail) если: + - колонка не существует + - таблица/view не существует + - синтаксическая ошибка SQL + """ + explain_sql = f"EXPLAIN {query}" + db.execute(text(explain_sql), params or {}) + + +# ── 1. domrf_kn_objects — реальные колонки ──────────────────────────────────── + + +class TestDomrfKnObjects: + """Проверяем что SQL в services используют реальные колонки domrf_kn_objects. + + Исторический баг PR #196: geom_3857 не существует. + Реальная schema: latitude, longitude (float), snapshot_date, obj_class, etc. + """ + + def test_competitors_radius_query(self, phantom_check_session: Session) -> None: + """best_layouts._COMPETITORS_IN_RADIUS_SQL — latitude/longitude/snapshot_date.""" + _explain( + phantom_check_session, + """ + SELECT DISTINCT ON (obj_id) obj_id + FROM domrf_kn_objects + WHERE latitude IS NOT NULL AND longitude IS NOT NULL + AND ST_DWithin( + ST_SetSRID(ST_MakePoint(longitude, latitude), 4326)::geography, + ST_SetSRID( + ST_MakePoint(CAST(60.6 AS float), CAST(56.8 AS float)), 4326 + )::geography, + CAST(1000 AS float) + ) + ORDER BY obj_id, snapshot_date DESC NULLS LAST + """, + ) + + def test_competitors_full_cte_query(self, phantom_check_session: Session) -> None: + """competitors._COMPETITORS_SQL — полный CTE с latitude/longitude/flat_count.""" + _explain( + phantom_check_session, + """ + WITH latest_obj AS ( + SELECT DISTINCT ON (obj_id) + obj_id, + comm_name, + dev_name, + obj_class, + latitude, + longitude, + flat_count, + site_status, + snapshot_date + FROM domrf_kn_objects + WHERE latitude IS NOT NULL + AND longitude IS NOT NULL + ORDER BY obj_id, snapshot_date DESC NULLS LAST + ), + distances AS ( + SELECT + o.obj_id, + o.comm_name, + o.dev_name, + o.obj_class, + o.latitude, + o.longitude, + o.flat_count, + o.site_status, + ST_Distance( + ST_SetSRID(ST_MakePoint(o.longitude, o.latitude), 4326)::geography, + ST_SetSRID( + ST_MakePoint(CAST(60.6 AS float), CAST(56.8 AS float)), + 4326 + )::geography + ) AS distance_m + FROM latest_obj o + ) + SELECT + d.obj_id, + d.comm_name, + d.dev_name, + d.obj_class, + d.latitude, + d.longitude, + d.flat_count, + d.site_status, + d.distance_m + FROM distances d + WHERE d.distance_m <= CAST(1000 AS float) + ORDER BY d.distance_m ASC + """, + ) + + def test_velocity_competitor_query_columns(self, phantom_check_session: Session) -> None: + """velocity._competitors query — district_name, region_cd, snapshot_date.""" + _explain( + phantom_check_session, + """ + SELECT DISTINCT ON (obj_id) + obj_id, + comm_name, + dev_name, + obj_class, + latitude, + longitude, + district_name + FROM domrf_kn_objects + WHERE latitude IS NOT NULL + AND longitude IS NOT NULL + AND region_cd = 66 + ORDER BY obj_id, snapshot_date DESC NULLS LAST + """, + ) + + +# ── 2. ekburg_construction_permits — реальные колонки ──────────────────────── + + +class TestEkburgConstructionPermits: + """Проверяем что SQL используют реальные колонки ekburg_construction_permits. + + Исторический баг PR #213: units_count не существует. + Реальная schema: total_area_sqm, living_area_sqm, living_area_fact_sqm, etc. + """ + + def test_recent_permits_query(self, phantom_check_session: Session) -> None: + """parcels.py recent_permits query — total_area_sqm (не units_count!).""" + _explain( + phantom_check_session, + """ + SELECT + permit_type, permit_number, issue_date, + developer_name, developer_inn, object_name, object_type, + construction_address, total_area_sqm + FROM ekburg_construction_permits + WHERE LEFT(cadastral_number, LENGTH(CAST('66:41:0303161' AS text))) + = CAST('66:41:0303161' AS text) + AND issue_date > NOW() - INTERVAL '24 months' + ORDER BY issue_date DESC + LIMIT 50 + """, + ) + + def test_permits_rns_columns(self, phantom_check_session: Session) -> None: + """Все колонки RNS-специфичные: living_area_sqm, permit_type.""" + _explain( + phantom_check_session, + """ + SELECT + permit_type, + permit_number, + issue_date, + expiry_date, + developer_name, + developer_inn, + object_name, + object_type, + construction_address, + cadastral_number, + total_area_sqm, + living_area_sqm + FROM ekburg_construction_permits + WHERE permit_type = 'RNS' + LIMIT 1 + """, + ) + + def test_permits_rve_columns(self, phantom_check_session: Session) -> None: + """Все колонки RVE-специфичные: living_area_fact_sqm, rve_number, rve_date.""" + _explain( + phantom_check_session, + """ + SELECT + permit_type, + permit_number, + living_area_fact_sqm, + rve_number, + rve_date + FROM ekburg_construction_permits + WHERE permit_type = 'RVE' + LIMIT 1 + """, + ) + + +# ── 3. mv_layout_velocity — materialized view ──────────────────────────────── + + +class TestMvLayoutVelocity: + """Проверяем materialized view mv_layout_velocity и её колонки.""" + + def test_velocity_by_room_bucket(self, phantom_check_session: Session) -> None: + """best_layouts._VELOCITY_BY_ROOM_SQL — room_bucket, total_deals_24mo, etc.""" + _explain( + phantom_check_session, + """ + SELECT + room_bucket, + SUM(total_deals_24mo) AS sum_deals, + AVG(avg_area_m2) AS avg_area_m2, + AVG(avg_price_thousand_rub_per_m2) * 1000.0 AS avg_price_per_m2_rub, + array_agg(DISTINCT obj_id) AS competitor_obj_ids, + COUNT(DISTINCT obj_id) AS competitor_count, + MIN(window_start) AS window_start, + MAX(window_end) AS window_end + FROM mv_layout_velocity + WHERE obj_id = ANY(ARRAY[1, 2, 3]) + GROUP BY room_bucket + """, + ) + + +# ── 4. domrf_kn_flats ───────────────────────────────────────────────────────── + + +class TestDomrfKnFlats: + """Проверяем таблицу domrf_kn_flats и её колонки.""" + + def test_supply_batch_query(self, phantom_check_session: Session) -> None: + """best_layouts._SUPPLY_BATCH_SQL — total_area, rooms, is_studio, flat_type.""" + _explain( + phantom_check_session, + """ + SELECT + CASE + WHEN f.is_studio = TRUE OR f.flat_type = 'Квартира-студия' THEN 'studio' + WHEN f.rooms = 0 THEN 'studio' + WHEN f.rooms IN (1, 2, 3) THEN f.rooms::text + WHEN f.rooms >= 4 THEN '4+' + ELSE '1' + END AS rb, + CASE + WHEN f.total_area < 25 THEN '<25' + WHEN f.total_area < 40 THEN '25-40' + WHEN f.total_area < 60 THEN '40-60' + WHEN f.total_area < 80 THEN '60-80' + WHEN f.total_area < 100 THEN '80-100' + ELSE '100+' + END AS ab, + COUNT(*) AS units + FROM domrf_kn_flats f + JOIN domrf_kn_objects o ON f.obj_id = o.obj_id + WHERE o.latitude IS NOT NULL AND o.longitude IS NOT NULL + AND f.snapshot_date = CAST('2026-01-01' AS date) + GROUP BY rb, ab + """, + ) + + def test_avg_price_sold_query(self, phantom_check_session: Session) -> None: + """competitors._AVG_PRICE_SQL — price_per_m2, status='sold'.""" + _explain( + phantom_check_session, + """ + SELECT + f.obj_id, + AVG(f.price_per_m2) AS avg_price_per_m2 + FROM domrf_kn_flats f + WHERE f.obj_id = ANY(ARRAY[1, 2, 3]) + AND f.price_per_m2 IS NOT NULL + AND f.status = 'sold' + GROUP BY f.obj_id + """, + ) + + +# ── 5. objective_complex_mapping / objective_corpus_room_month ──────────────── + + +class TestObjectiveTables: + """Проверяем objective_* таблицы и их колонки.""" + + def test_objective_mapping_columns(self, phantom_check_session: Session) -> None: + """competitors / velocity: domrf_obj_id, objective_complex_name в mapping.""" + _explain( + phantom_check_session, + """ + SELECT cm.domrf_obj_id AS obj_id, + cm.objective_complex_name + FROM objective_complex_mapping cm + LIMIT 1 + """, + ) + + def test_objective_corpus_room_month_columns(self, phantom_check_session: Session) -> None: + """velocity._VELOCITY_BY_ROOM_SQL — deals_total_count, deals_total_vol_m2.""" + _explain( + phantom_check_session, + """ + SELECT + m.obj_id, + SUM(COALESCE(crm.deals_total_vol_m2, + crm.deals_total_count * 45.0)) AS total_sqm, + COUNT(DISTINCT crm.report_month) AS months_with_data, + MIN(crm.report_month) AS period_start, + MAX(crm.report_month) AS period_end + FROM objective_corpus_room_month crm + JOIN ( + SELECT cm.domrf_obj_id AS obj_id, + cm.objective_complex_name + FROM objective_complex_mapping cm + WHERE cm.domrf_obj_id = ANY(ARRAY[1, 2, 3]) + ) m + ON m.objective_complex_name = crm.project_name + WHERE crm.report_month >= (CURRENT_DATE - CAST('12 months' AS interval)) + AND crm.deals_total_count > 0 + GROUP BY m.obj_id + """, + ) + + +# ── 6. cad geo таблицы ──────────────────────────────────────────────────────── + + +class TestCadGeoTables: + """Проверяем cad_parcels_geom, cad_quarters_geom и их колонки.""" + + def test_parcel_centroid_query(self, phantom_check_session: Session) -> None: + """best_layouts._PARCEL_CENTROID_SQL — cad_num, cad_number, geom.""" + _explain( + phantom_check_session, + """ + SELECT ST_X(pt) AS center_lon, + ST_Y(pt) AS center_lat + FROM ( + SELECT ST_Centroid(geom) AS pt + FROM cad_parcels_geom + WHERE cad_num = '66:41:0303161:123' AND geom IS NOT NULL + UNION ALL + SELECT ST_Centroid(geom) AS pt + FROM cad_quarters_geom + WHERE cad_number = '66:41:0303161' AND geom IS NOT NULL + ) sub + LIMIT 1 + """, + ) diff --git a/backend/tests/scrapers/test_nspd_bulk_client.py b/backend/tests/scrapers/test_nspd_bulk_client.py index 48e482fb..7add40f5 100644 --- a/backend/tests/scrapers/test_nspd_bulk_client.py +++ b/backend/tests/scrapers/test_nspd_bulk_client.py @@ -316,7 +316,7 @@ async def test_list_objects_in_building_parses( @pytest.mark.asyncio async def test_rate_limit_semaphore_max_3_concurrent() -> None: - """Не более 3 одновременных запросов через _SEMAPHORE. + """Не более 3 одновременных запросов через per-instance self._sem (PR #260). Мокируем httpx.AsyncClient.get (нижний уровень) с задержкой, чтобы реальный семафор работал. Считаем max in-flight внутри семафора. @@ -332,7 +332,7 @@ async def test_rate_limit_semaphore_max_3_concurrent() -> None: async def slow_get(*args: Any, **kwargs: Any) -> httpx.Response: nonlocal max_concurrent, current, call_count - # Фиксируем вход — уже внутри семафора (httpx.get вызывается после async with _SEMAPHORE) + # Фиксируем вход — уже внутри семафора (httpx.get вызывается после async with self._sem) async with lock: current += 1 call_count += 1 @@ -353,11 +353,12 @@ async def test_rate_limit_semaphore_max_3_concurrent() -> None: tasks = [client.search_by_quarter(f"66:41:{i:07d}") for i in range(6)] await asyncio.gather(*tasks) - # _SEMAPHORE(3) → не более 3 одновременно внутри slow_get + # self._sem(3) → не более 3 одновременно внутри slow_get assert max_concurrent <= 3, f"Expected ≤3 concurrent, got {max_concurrent}" assert call_count == 6 # все 6 вызовов прошли - # Убедимся что _SEMAPHORE в модуле имеет нужное значение capacity - assert bulk_mod._SEMAPHORE._value >= 0 # семафор сброшен после всех задач + # NB: per-instance self._sem cleanup проверяется через max_concurrent <= 3 выше. + # Module-level _SEMAPHORE удалён в PR #260 (cross-loop binding fix); smoke на capacity: + assert bulk_mod._SEMAPHORE_LIMIT == 3 @pytest.mark.asyncio diff --git a/backend/tests/scrapers/test_nspd_grid_walk.py b/backend/tests/scrapers/test_nspd_grid_walk.py new file mode 100644 index 00000000..377ba20c --- /dev/null +++ b/backend/tests/scrapers/test_nspd_grid_walk.py @@ -0,0 +1,449 @@ +"""Unit-тесты для NSPDClient.get_features_in_bbox_grid + classify_engineering_kind. + +Sub-PR A foundation (#126): grid-walk — НЕ live HTTP (all mocked). + +Запуск: + cd backend && uv run pytest tests/scrapers/test_nspd_grid_walk.py -v + +Live integration (помечены @pytest.mark.integration — пропускаются в CI): + cd backend && uv run pytest tests/scrapers/test_nspd_grid_walk.py -m integration -s +""" + +from __future__ import annotations + +from typing import Any, ClassVar +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from app.services.scrapers.nspd_client import NSPDClient, NSPDFeature +from app.services.scrapers.nspd_denorm import classify_engineering_kind + +# ── Helpers ──────────────────────────────────────────────────────────────────── + + +def _make_bulk_feature(feature_id: str | int | None, props: dict[str, Any]) -> MagicMock: + """Создать mock NSPDBulkFeature с нужными полями.""" + feat = MagicMock() + feat.id = feature_id + feat.geometry = None + feat.properties = props + return feat + + +# ── Grid-walk count tests ────────────────────────────────────────────────────── + + +class TestGetFeaturesInBboxGrid: + """Тесты grid-walk метода (без live NSPD).""" + + # bbox 700×700м в EPSG:3857 (типичный квартал ЕКБ) + BBOX: tuple[float, float, float, float] = ( + 6_600_000.0, + 7_700_000.0, + 6_600_700.0, + 7_700_700.0, + ) + + def _patch_bulk_client(self, return_features: list[list[Any]]) -> tuple[Any, AsyncMock]: + """Патч NSPDBulkClient.wms_feature_info. + + return_features: список ответов — по одному на каждый вызов wms_feature_info + (в порядке вызовов). Если список короче числа calls — последний элемент + повторяется. + """ + call_idx: list[int] = [0] + + async def _side_effect(*args: Any, **kwargs: Any) -> list[Any]: + idx = min(call_idx[0], len(return_features) - 1) + call_idx[0] += 1 + return return_features[idx] + + mock_wms = AsyncMock(side_effect=_side_effect) + return mock_wms, mock_wms + + def test_grid_n7_calls_49_times(self) -> None: + """grid_n=7 → ровно 49 вызовов wms_feature_info.""" + call_count: list[int] = [0] + + async def _wms(*args: Any, **kwargs: Any) -> list[Any]: + call_count[0] += 1 + return [] + + mock_client_instance = AsyncMock() + mock_client_instance.wms_feature_info = AsyncMock(side_effect=_wms) + mock_client_instance.__aenter__ = AsyncMock(return_value=mock_client_instance) + mock_client_instance.__aexit__ = AsyncMock(return_value=None) + + with patch( + "app.scrapers.nspd_bulk_client.NSPDBulkClient", + return_value=mock_client_instance, + ): + client = NSPDClient() + result = client.get_features_in_bbox_grid(36328, self.BBOX, grid_n=7, step_m=1.0) + + assert call_count[0] == 49, f"Ожидали 49 вызовов, получили {call_count[0]}" + assert result == [] + + def test_grid_n3_calls_9_times(self) -> None: + """grid_n=3 → ровно 9 вызовов.""" + call_count: list[int] = [0] + + async def _wms(*args: Any, **kwargs: Any) -> list[Any]: + call_count[0] += 1 + return [] + + mock_client_instance = AsyncMock() + mock_client_instance.wms_feature_info = AsyncMock(side_effect=_wms) + mock_client_instance.__aenter__ = AsyncMock(return_value=mock_client_instance) + mock_client_instance.__aexit__ = AsyncMock(return_value=None) + + with patch( + "app.scrapers.nspd_bulk_client.NSPDBulkClient", + return_value=mock_client_instance, + ): + client = NSPDClient() + result = client.get_features_in_bbox_grid(36328, self.BBOX, grid_n=3, step_m=1.0) + + assert call_count[0] == 9 + assert result == [] + + def test_dedup_same_feature_id(self) -> None: + """Одинаковые feature_id из соседних ячеек → одна запись.""" + feat_a = _make_bulk_feature("feat-001", {"cad_num": "66:41:001:1"}) + feat_b = _make_bulk_feature("feat-001", {"cad_num": "66:41:001:1"}) # дубликат + feat_c = _make_bulk_feature("feat-002", {"cad_num": "66:41:001:2"}) + + async def _wms(*args: Any, **kwargs: Any) -> list[Any]: + return [feat_a, feat_b, feat_c] + + mock_client_instance = AsyncMock() + mock_client_instance.wms_feature_info = AsyncMock(side_effect=_wms) + mock_client_instance.__aenter__ = AsyncMock(return_value=mock_client_instance) + mock_client_instance.__aexit__ = AsyncMock(return_value=None) + + with patch( + "app.scrapers.nspd_bulk_client.NSPDBulkClient", + return_value=mock_client_instance, + ): + client = NSPDClient() + result = client.get_features_in_bbox_grid(36328, self.BBOX, grid_n=2, step_m=1.0) + + # 4 cells × 3 features = 12 raw, но feature_id уникальных 2 + feature_ids = [f.feature_id for f in result] + assert "feat-001" in feature_ids + assert "feat-002" in feature_ids + # нет дубликатов + assert len(feature_ids) == len(set(feature_ids)) + + def test_dedup_by_cad_num_when_no_feature_id(self) -> None: + """Если feature_id=None — дедупликация по cad_num.""" + feat_a = _make_bulk_feature(None, {"cad_num": "66:41:0000001:100"}) + feat_b = _make_bulk_feature(None, {"cad_num": "66:41:0000001:100"}) # дубликат + + call_n: list[int] = [0] + + async def _wms(*args: Any, **kwargs: Any) -> list[Any]: + call_n[0] += 1 + # Первая ячейка возвращает feat_a, вторая — feat_b + return [feat_a] if call_n[0] == 1 else [feat_b] + + mock_client_instance = AsyncMock() + mock_client_instance.wms_feature_info = AsyncMock(side_effect=_wms) + mock_client_instance.__aenter__ = AsyncMock(return_value=mock_client_instance) + mock_client_instance.__aexit__ = AsyncMock(return_value=None) + + with patch( + "app.scrapers.nspd_bulk_client.NSPDBulkClient", + return_value=mock_client_instance, + ): + client = NSPDClient() + result = client.get_features_in_bbox_grid(36328, self.BBOX, grid_n=2, step_m=1.0) + + cad_nums = [f.properties.get("cad_num") for f in result] + assert cad_nums.count("66:41:0000001:100") == 1, "Дубликат не дедуплицирован" + + def test_error_in_one_cell_does_not_abort(self) -> None: + """Ошибка в одной ячейке не останавливает весь grid-walk.""" + good_feat = _make_bulk_feature("feat-ok", {"cad_num": "66:41:001:1"}) + call_n: list[int] = [0] + + async def _wms(*args: Any, **kwargs: Any) -> list[Any]: + call_n[0] += 1 + if call_n[0] == 1: + raise RuntimeError("Simulated cell error") + return [good_feat] + + mock_client_instance = AsyncMock() + mock_client_instance.wms_feature_info = AsyncMock(side_effect=_wms) + mock_client_instance.__aenter__ = AsyncMock(return_value=mock_client_instance) + mock_client_instance.__aexit__ = AsyncMock(return_value=None) + + with patch( + "app.scrapers.nspd_bulk_client.NSPDBulkClient", + return_value=mock_client_instance, + ): + client = NSPDClient() + # Не должно бросать исключение + result = client.get_features_in_bbox_grid(36328, self.BBOX, grid_n=2, step_m=1.0) + + # 4 cells: 1 error + 3 good_feat → 1 unique feature + assert any(f.feature_id == "feat-ok" for f in result) + + def test_returns_nspd_feature_instances(self) -> None: + """Метод возвращает list[NSPDFeature] а не NSPDBulkFeature.""" + bulk_feat = _make_bulk_feature("feat-xyz", {"cad_num": "66:41:001:1"}) + + async def _wms(*args: Any, **kwargs: Any) -> list[Any]: + return [bulk_feat] + + mock_client_instance = AsyncMock() + mock_client_instance.wms_feature_info = AsyncMock(side_effect=_wms) + mock_client_instance.__aenter__ = AsyncMock(return_value=mock_client_instance) + mock_client_instance.__aexit__ = AsyncMock(return_value=None) + + with patch( + "app.scrapers.nspd_bulk_client.NSPDBulkClient", + return_value=mock_client_instance, + ): + client = NSPDClient() + result = client.get_features_in_bbox_grid(36328, self.BBOX, grid_n=1, step_m=1.0) + + assert all(isinstance(f, NSPDFeature) for f in result) + + def test_auto_reduce_grid_for_small_bbox(self) -> None: + """Маленький bbox + большой grid_n + большой step_m → grid_n уменьшается.""" + # bbox 60×60м, step_m=50, grid_n=7 → effective_n=min(7, int(60/50), int(60/50)) = 1 + small_bbox: tuple[float, float, float, float] = ( + 6_600_000.0, + 7_700_000.0, + 6_600_060.0, + 7_700_060.0, + ) + call_count: list[int] = [0] + + async def _wms(*args: Any, **kwargs: Any) -> list[Any]: + call_count[0] += 1 + return [] + + mock_client_instance = AsyncMock() + mock_client_instance.wms_feature_info = AsyncMock(side_effect=_wms) + mock_client_instance.__aenter__ = AsyncMock(return_value=mock_client_instance) + mock_client_instance.__aexit__ = AsyncMock(return_value=None) + + with patch( + "app.scrapers.nspd_bulk_client.NSPDBulkClient", + return_value=mock_client_instance, + ): + client = NSPDClient() + client.get_features_in_bbox_grid(36328, small_bbox, grid_n=7, step_m=50.0) + + # effective_n = 1 → 1 ячейка + assert call_count[0] == 1 + + +# ── Classifier tests ─────────────────────────────────────────────────────────── + + +class TestClassifyEngineeringKind: + """Smoke-тесты классификатора инженерных сооружений.""" + + @pytest.mark.parametrize( + "props, expected", + [ + # gas + ({"params_name": "Газопровод высокого давления"}, "gas"), + ({"name": "Газопровод ПЭ SDR11 160мм"}, "gas"), + ({"params_purpose": "Газоснабжение"}, "gas"), + ({"name": "ГРП №12"}, "gas"), + # electric + ({"name": "КЛ 10 кВ ТП 64102"}, "electric"), + ({"params_name": "ВЛ-10кВ Ф-14"}, "electric"), + ({"name": "ТП 1234"}, "electric"), + ({"params_purpose": "Электроэнергетика и связь"}, "electric"), + ({"name": "Подстанция 110/10 кВ"}, "electric"), + # water + ({"params_purpose": "Водопровод хозбытовой"}, "water"), + ({"name": "Водовод Ду300"}, "water"), + ({"params_name": "Сеть водоснабжения"}, "water"), + # heat + ({"params_name": "Тепловая сеть"}, "heat"), + ({"name": "Теплосеть квартал 24"}, "heat"), + ({"params_purpose": "Теплоснабжение жилых домов"}, "heat"), + ({"name": "ТЭЦ-4 отпайка"}, "heat"), + # sewage + ({"name": "Канализация"}, "sewage"), + ({"params_name": "Сеть канализации Ду200"}, "sewage"), + ({"name": "Ливневая канализация"}, "sewage"), + # other + ({"name": "Объект не классифицирован"}, "other"), + ({}, "other"), + ({"params_purpose": None}, "other"), + ], + ) + def test_classify(self, props: dict[str, Any], expected: str) -> None: + assert ( + classify_engineering_kind(props) == expected + ), f"props={props!r} → expected {expected!r}" + + def test_field_priority_params_name_over_purpose(self) -> None: + """params_name проверяется раньше purpose.""" + props = { + "params_name": "Газопровод", # → gas + "params_purpose": "Водоснабжение", # → water (если бы проверялось первым) + } + # gas-паттерн найдётся в combined строке первым по порядку _ENGINEERING_PATTERNS + assert classify_engineering_kind(props) == "gas" + + def test_case_insensitive(self) -> None: + """Паттерны case-insensitive.""" + assert classify_engineering_kind({"name": "ГАЗОПРОВОД ВЫСОКОГО ДАВЛЕНИЯ"}) == "gas" + assert classify_engineering_kind({"name": "канализация бытовая"}) == "sewage" + + +# ── Sub-PR B: _fetch_layer dispatch tests ───────────────────────────────────── + + +class TestFetchLayerDispatch: + """Проверяем что _fetch_layer внутри search_by_quarter правильно диспатчит + area-слои на get_features_in_bbox_grid, а EGRN-слои — на legacy get_features_in_bbox. + + Моки: search_by_cad и get_features_in_bbox_grid / get_features_in_bbox + через patch, без live HTTP. + """ + + # Минимальный bbox квартала (Web Mercator), который возвращает search_by_cad + _MOCK_GEOM: ClassVar[dict[str, Any]] = { + "type": "Polygon", + "coordinates": [ + [ + [6_600_000.0, 7_700_000.0], + [6_600_700.0, 7_700_000.0], + [6_600_700.0, 7_700_700.0], + [6_600_000.0, 7_700_700.0], + [6_600_000.0, 7_700_000.0], + ] + ], + } + + def _make_search_result(self) -> MagicMock: + """Mock NSPDSearchResult с первым feature, у которого есть geometry.""" + feat = MagicMock(spec=NSPDFeature) + feat.geometry = self._MOCK_GEOM + feat.properties = {} + feat.feature_id = "q-001" + result = MagicMock() + result.first = feat + return result + + def test_fetch_layer_uses_grid_walk_for_area_layers(self) -> None: + """territorial_zones (area layer) → вызывает get_features_in_bbox_grid.""" + client = NSPDClient() + mock_search_result = self._make_search_result() + + with ( + patch.object(client, "search_by_cad", return_value=mock_search_result), + patch.object(client, "get_features_in_bbox_grid", return_value=[]) as mock_grid, + patch.object(client, "get_features_in_bbox", return_value=[]) as mock_legacy, + ): + client.search_by_quarter( + "66:41:0303161", + include_zouit=False, + include_risks=False, + include_opportunity=False, + ) + + # territorial_zones, red_lines, engineering_structures — все три должны + # быть запрошены через grid-walk + called_layer_ids = [call.args[0] for call in mock_grid.call_args_list] + from app.services.scrapers.nspd_client import LAYERS + + assert ( + LAYERS["territorial_zones"] in called_layer_ids + ), "territorial_zones должен использовать grid-walk" + assert LAYERS["red_lines"] in called_layer_ids, "red_lines должен использовать grid-walk" + assert ( + LAYERS["engineering_structures"] in called_layer_ids + ), "engineering_structures должен использовать grid-walk" + # parcels и buildings — legacy, не grid + called_legacy_ids = [call.args[0] for call in mock_legacy.call_args_list] + assert LAYERS["parcels"] in called_legacy_ids, "parcels должен идти через legacy" + assert LAYERS["buildings"] in called_legacy_ids, "buildings должен идти через legacy" + + def test_fetch_layer_uses_legacy_for_egrn_layers(self) -> None: + """parcels и buildings → вызывают legacy get_features_in_bbox (не grid).""" + client = NSPDClient() + mock_search_result = self._make_search_result() + + with ( + patch.object(client, "search_by_cad", return_value=mock_search_result), + patch.object(client, "get_features_in_bbox_grid", return_value=[]) as mock_grid, + patch.object(client, "get_features_in_bbox", return_value=[]) as mock_legacy, + ): + client.search_by_quarter( + "66:41:0303161", + include_zouit=False, + include_risks=False, + include_opportunity=False, + ) + + from app.services.scrapers.nspd_client import LAYERS + + called_legacy_ids = [call.args[0] for call in mock_legacy.call_args_list] + called_grid_ids = [call.args[0] for call in mock_grid.call_args_list] + + assert LAYERS["parcels"] in called_legacy_ids + assert LAYERS["buildings"] in called_legacy_ids + # EGRN layers должны НЕ попасть в grid-walk + assert LAYERS["parcels"] not in called_grid_ids, "parcels не должен использовать grid" + assert LAYERS["buildings"] not in called_grid_ids, "buildings не должен использовать grid" + + def test_engineering_structures_classified_kind_enriched(self) -> None: + """engineering_structures features получают classified_kind в properties.""" + from app.services.scrapers.nspd_client import LAYERS + + # Создаём feature с mutable properties (как реальный NSPDFeature) + eng_feat = MagicMock(spec=NSPDFeature) + eng_feat.feature_id = "eng-001" + eng_feat.geometry = None + eng_feat.properties = {"params_name": "Газопровод высокого давления"} + + client = NSPDClient() + mock_search_result = self._make_search_result() + + def _grid_side_effect(layer_id: int, bbox: Any, **kwargs: Any) -> list[Any]: + if layer_id == LAYERS["engineering_structures"]: + return [eng_feat] + return [] + + with ( + patch.object(client, "search_by_cad", return_value=mock_search_result), + patch.object(client, "get_features_in_bbox_grid", side_effect=_grid_side_effect), + patch.object(client, "get_features_in_bbox", return_value=[]), + ): + dump = client.search_by_quarter( + "66:41:0303161", + include_zouit=False, + include_risks=False, + include_opportunity=False, + ) + + assert len(dump.engineering_structures) == 1 + enriched = dump.engineering_structures[0] + got = enriched.properties.get("classified_kind") + assert got == "gas", f"Ожидали classified_kind='gas', получили {got!r}" + + +# ── Integration marker (skip in CI) ────────────────────────────────────────── + + +@pytest.mark.integration +def test_live_nspd_grid_walk_skipped() -> None: + """Placeholder для ручного запуска live NSPD grid-walk. + + Запуск: + cd backend && uv run pytest tests/scrapers/test_nspd_grid_walk.py -m integration -s + + При запуске — делает реальные HTTP-запросы к nspd.gov.ru. + """ + pytest.skip("Live NSPD integration test — запускать вручную с -m integration") diff --git a/backend/tests/services/cadastre/__init__.py b/backend/tests/services/cadastre/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/tests/services/cadastre/test_bulk_harvest_territorial.py b/backend/tests/services/cadastre/test_bulk_harvest_territorial.py new file mode 100644 index 00000000..df40b3ed --- /dev/null +++ b/backend/tests/services/cadastre/test_bulk_harvest_territorial.py @@ -0,0 +1,198 @@ +"""Тесты для _save_territorial_zones (bulk_harvest.py) — mock-based. + +Проверяет: +- Успешный UPSERT 3 features → 3 строки вставлены +- Повторный вызов → ON CONFLICT обновляет, не дублирует +- Feature без geometry → строка вставлена с geom=NULL, без краша +- Feature без zone_id → синтетический fallback zone_id используется +""" + +from __future__ import annotations + +from typing import Any +from unittest.mock import MagicMock + +from app.services.cadastre.bulk_harvest import _save_territorial_zones + + +def _make_feature( + feature_id: Any = "zone_1", + zone_code: str = "Ж-1", + zone_name: str = "Жилая смешанная", + permitted_use: str = "ИЖС", + has_geometry: bool = True, +) -> dict: + """Создать raw feature dict в формате get_features_in_bbox_grid.""" + geom = ( + { + "type": "Polygon", + "coordinates": [ + [ + [6090000.0, 7590000.0], + [6090100.0, 7590000.0], + [6090100.0, 7590100.0], + [6090000.0, 7590100.0], + [6090000.0, 7590000.0], + ] + ], + } + if has_geometry + else None + ) + return { + "id": feature_id, + "geometry": geom, + "properties": { + "zone_code": zone_code, + "zone_name": zone_name, + "permitted_use": permitted_use, + }, + } + + +def _make_db_mock() -> MagicMock: + """Mock SQLAlchemy Session с begin_nested() savepoint support.""" + db = MagicMock() + # begin_nested() используется как context manager + savepoint_ctx = MagicMock() + savepoint_ctx.__enter__ = MagicMock(return_value=savepoint_ctx) + savepoint_ctx.__exit__ = MagicMock(return_value=False) + db.begin_nested.return_value = savepoint_ctx + return db + + +class TestSaveTerritorialZones: + """Тесты для _save_territorial_zones.""" + + def test_three_features_inserted(self) -> None: + """3 features → returned count == 3, execute вызван 3 раза.""" + db = _make_db_mock() + features = [ + _make_feature("z1", "Ж-1"), + _make_feature("z2", "ОД-1"), + _make_feature("z3", "П-1"), + ] + + result = _save_territorial_zones(db, "66:41:0204016", features) + + assert result == 3 + assert db.execute.call_count == 3 + db.commit.assert_called_once() + + def test_empty_features_list(self) -> None: + """Пустой список → 0 inserted, commit всё равно вызван.""" + db = _make_db_mock() + + result = _save_territorial_zones(db, "66:41:0204016", []) + + assert result == 0 + db.execute.assert_not_called() + db.commit.assert_called_once() + + def test_feature_without_geometry_no_crash(self) -> None: + """Feature без geometry → geom=NULL, строка вставлена без краша.""" + db = _make_db_mock() + features = [_make_feature("zone_no_geom", has_geometry=False)] + + result = _save_territorial_zones(db, "66:41:0204016", features) + + assert result == 1 + # Проверяем что geom параметр передан как None + call_kwargs: dict = db.execute.call_args[0][1] + assert call_kwargs["geom"] is None + + def test_feature_without_zone_id_uses_fallback(self) -> None: + """Feature без id → md5-based fallback zone_id (stable между runs).""" + db = _make_db_mock() + features = [ + { + "id": None, + "geometry": None, + "properties": {"zone_code": "Ж-2"}, + } + ] + + result = _save_territorial_zones(db, "66:41:0204016", features) + + assert result == 1 + call_kwargs = db.execute.call_args[0][1] + zone_id: str = call_kwargs["zone_id"] + # fallback zone_id содержит quarter_cad и стабильный hash (12 hex chars) + assert zone_id.startswith("66:41:0204016_") + suffix = zone_id.split("_", 3)[-1] + assert len(suffix) == 12 + assert all(c in "0123456789abcdef" for c in suffix) + + # Второй вызов с теми же данными → тот же zone_id (идемпотентность) + db2 = _make_db_mock() + _save_territorial_zones(db2, "66:41:0204016", features) + call_kwargs2 = db2.execute.call_args[0][1] + assert call_kwargs2["zone_id"] == zone_id + + def test_zone_id_from_props_id(self) -> None: + """Если feature.id=None, но props['id'] есть — используется props['id'].""" + db = _make_db_mock() + features = [ + { + "id": None, + "geometry": None, + "properties": {"id": "props_id_42", "zone_code": "Ж-3"}, + } + ] + + result = _save_territorial_zones(db, "66:41:0204016", features) + + assert result == 1 + call_kwargs = db.execute.call_args[0][1] + assert call_kwargs["zone_id"] == "props_id_42" + + def test_execute_error_logged_not_raised(self) -> None: + """Exception в execute → строка не вставлена, warning залогирован, не re-raise.""" + db = _make_db_mock() + db.execute.side_effect = RuntimeError("DB error") + features = [_make_feature("z_err")] + + # Не должен бросить исключение + result = _save_territorial_zones(db, "66:41:0204016", features) + + assert result == 0 + db.commit.assert_called_once() + + def test_savepoint_used_per_row(self) -> None: + """begin_nested() вызывается для каждой строки (SAVEPOINT паттерн).""" + db = _make_db_mock() + features = [_make_feature(f"z{i}") for i in range(3)] + + _save_territorial_zones(db, "66:41:0204016", features) + + assert db.begin_nested.call_count == 3 + + def test_quarter_cad_param_passed(self) -> None: + """quarter_cad правильно передаётся в SQL параметры.""" + db = _make_db_mock() + features = [_make_feature("zone_check")] + + _save_territorial_zones(db, "66:41:9999999", features) + + call_kwargs = db.execute.call_args[0][1] + assert call_kwargs["quarter_cad"] == "66:41:9999999" + + def test_raw_props_serialized(self) -> None: + """raw_props — JSON строка из properties dict.""" + import json + + db = _make_db_mock() + features = [ + { + "id": "z_props", + "geometry": None, + "properties": {"zone_code": "ОД-2", "extra": "value"}, + } + ] + + _save_territorial_zones(db, "66:41:0204016", features) + + call_kwargs = db.execute.call_args[0][1] + raw = json.loads(call_kwargs["raw_props"]) + assert raw["zone_code"] == "ОД-2" + assert raw["extra"] == "value" diff --git a/backend/tests/services/scrapers/__init__.py b/backend/tests/services/scrapers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/tests/services/scrapers/test_domrf_catalog_object.py b/backend/tests/services/scrapers/test_domrf_catalog_object.py new file mode 100644 index 00000000..34012edd --- /dev/null +++ b/backend/tests/services/scrapers/test_domrf_catalog_object.py @@ -0,0 +1,290 @@ +"""Тесты для domrf_catalog_object.py (issue #297 sub-task 22d). + +Покрывает: + - extract_next_data — парсинг HTML с __NEXT_DATA__ + - parse_catalog_object — маппинг pageProps → DB columns + - value helpers (_to_numeric_comma, _to_bool_da_net, _to_date_ddmmyyyy) + - partial responses (partial pageProps → all other fields = None, no crash) +""" + +from __future__ import annotations + +from datetime import date +from typing import Any + +import pytest + +from app.services.scrapers.domrf_catalog_object import ( + _to_bool_da_net, + _to_bool_int, + _to_date_ddmmyyyy, + _to_numeric_comma, + extract_next_data, + parse_catalog_object, +) + +# ── extract_next_data ───────────────────────────────────────────────────────── + + +def test_extract_next_data_from_html() -> None: + """Базовый case: тег найден, JSON возвращается как dict.""" + html = ( + "" + '" + "" + ) + result = extract_next_data(html) + assert isinstance(result, dict) + assert result["props"]["pageProps"]["buildingClass"] == "Комфорт" + + +def test_extract_next_data_single_quotes() -> None: + """Тег с одинарными кавычками тоже должен парситься.""" + html = "" + result = extract_next_data(html) + assert "props" in result + + +def test_extract_next_data_not_found_raises() -> None: + """Если тег не найден — ValueError.""" + with pytest.raises(ValueError, match="__NEXT_DATA__"): + extract_next_data("no script here") + + +def test_extract_next_data_invalid_json_raises() -> None: + """Если JSON некорректный — ValueError.""" + html = '' + with pytest.raises(ValueError): + extract_next_data(html) + + +# ── parse_catalog_object — full sample ─────────────────────────────────────── + + +def _make_full_next_data() -> dict[str, Any]: + """Реалистичный full next_data для obj_id=65136 (подтверждён live).""" + return { + "props": { + "pageProps": { + "buildingClass": "Комфорт", + "wallMaterial": "Монолит-кирпич", + "objEnergyEfficiency": "B", + "parkingCount": 246, + "finishTypeCount": 1, + "freePlan": "Нет", + "publicationDate": "31.03.2025", + "additionalInfo": { + "objectParkingPlaces": 43, + "nearbyParkingPlaces": 0, + "ceilingHeight": "2,7", + "passengerElevatorsCount": 0, + "cargoElevatorsCount": 0, + "cargoPassengerElevatorCount": 4, + "playgroundsCount": 6, + "sportsgroundCount": 5, + "bicycleLane": 0, + "trashAreaCount": 3, + "ramp": 0, + "curbLowering": 1, + "wheelchairElevatorsCount": 0, + "parkingAvailabilityPerc": 60, + }, + "quartography": { + "objLivElemEntrCnt": 1, + "objLivElemSqAvg": 46.2, + "nonLivFirstFloor": 1, + }, + "indexes": { + "infrastructure": 10, + "transport": 6, + }, + "projectDeclaration": { + "number": "66-001686", + }, + } + } + } + + +def test_parse_catalog_object_full() -> None: + """Полный sample: все 25+ полей должны быть замаплены корректно.""" + data = parse_catalog_object(_make_full_next_data()) + + assert data["obj_class"] == "Комфорт" + assert data["wall_type"] == "Монолит-кирпич" + assert data["energy_eff"] == "B" + assert data["section_count"] == 1 + assert data["parking_total_slots"] == 246 + assert data["guest_parking_inside_count"] == 43 + assert data["guest_parking_outside_count"] == 0 + assert data["ceiling_height_m"] == pytest.approx(2.7) + assert data["finishing_variants_count"] == 1 + assert data["has_free_planning"] is False + assert data["avg_flat_area_m2"] == pytest.approx(46.2) + assert data["elevators_passenger_count"] == 0 + assert data["elevators_cargo_count"] == 4 # 0 + 4 + assert data["playground_kids_count"] == 6 + assert data["playground_sport_count"] == 5 + assert data["has_bike_paths"] is False # bicycleLane=0 + assert data["trash_areas_count"] == 3 + assert data["has_ramp"] is False # ramp=0 + assert data["has_low_platforms"] is True # curbLowering=1 + assert data["has_wheelchair_lift"] is False # wheelchairElevatorsCount=0 + assert data["first_floor_type"] == "нежилой" # nonLivFirstFloor=1 + assert data["parking_provision_pct"] == 60 + assert data["project_published_at"] == date(2025, 3, 31) + assert data["project_declaration_num"] == "66-001686" + assert data["domrf_score_infrastructure"] == 10 + assert data["domrf_score_transport"] == 6 + + +def test_parse_catalog_object_has_free_planning_da() -> None: + """freePlan='Да' → has_free_planning=True.""" + nd: dict[str, Any] = {"props": {"pageProps": {"freePlan": "Да"}}} + data = parse_catalog_object(nd) + assert data["has_free_planning"] is True + + +def test_parse_catalog_object_first_floor_zhiloj() -> None: + """nonLivFirstFloor=0 → first_floor_type='жилой'.""" + nd: dict[str, Any] = { + "props": { + "pageProps": { + "quartography": {"nonLivFirstFloor": 0}, + } + } + } + data = parse_catalog_object(nd) + assert data["first_floor_type"] == "жилой" + + +def test_parse_catalog_object_elevators_cargo_sum() -> None: + """elevators_cargo_count = cargoElevatorsCount + cargoPassengerElevatorCount.""" + nd: dict[str, Any] = { + "props": { + "pageProps": { + "additionalInfo": { + "cargoElevatorsCount": 2, + "cargoPassengerElevatorCount": 3, + } + } + } + } + data = parse_catalog_object(nd) + assert data["elevators_cargo_count"] == 5 + + +def test_parse_catalog_object_partial() -> None: + """Только buildingClass → остальные поля None, без исключений.""" + nd: dict[str, Any] = {"props": {"pageProps": {"buildingClass": "Бизнес"}}} + data = parse_catalog_object(nd) + assert data["obj_class"] == "Бизнес" + assert data["wall_type"] is None + assert data["energy_eff"] is None + assert data["section_count"] is None + assert data["parking_total_slots"] is None + assert data["ceiling_height_m"] is None + assert data["has_free_planning"] is None + assert data["elevators_cargo_count"] is None + assert data["project_published_at"] is None + assert data["domrf_score_infrastructure"] is None + + +def test_parse_catalog_object_empty() -> None: + """Полностью пустой next_data → все поля None, без исключений.""" + data = parse_catalog_object({}) + for v in data.values(): + assert v is None + + +def test_parking_provision_pct_preserves_float() -> None: + """parking_provision_pct should preserve fractional values (column is numeric(5,1)).""" + next_data: dict[str, Any] = { + "props": {"pageProps": {"id": 65136, "additionalInfo": {"parkingAvailabilityPerc": 60.5}}} + } + result = parse_catalog_object(next_data) + assert result["parking_provision_pct"] == 60.5 + + +# ── _to_numeric_comma ───────────────────────────────────────────────────────── + + +@pytest.mark.parametrize( + "inp,expected", + [ + ("2,7", 2.7), + ("2.7", 2.7), + ("3,50", 3.5), + ("", None), + (None, None), + (" ", None), + ("abc", None), + ], +) +def test_to_numeric_comma(inp: Any, expected: float | None) -> None: + result = _to_numeric_comma(inp) + if expected is None: + assert result is None + else: + assert result == pytest.approx(expected) + + +# ── _to_bool_da_net ─────────────────────────────────────────────────────────── + + +@pytest.mark.parametrize( + "inp,expected", + [ + ("Да", True), + ("да", True), + ("ДА", True), + ("Нет", False), + ("нет", False), + ("НЕТ", False), + ("", None), + (None, None), + ("maybe", None), + ("Yes", None), + ], +) +def test_to_bool_da_net(inp: Any, expected: bool | None) -> None: + assert _to_bool_da_net(inp) == expected + + +# ── _to_bool_int ────────────────────────────────────────────────────────────── + + +@pytest.mark.parametrize( + "inp,expected", + [ + (0, False), + (1, True), + (5, True), + ("1", True), + ("0", False), + (None, None), + ], +) +def test_to_bool_int(inp: Any, expected: bool | None) -> None: + assert _to_bool_int(inp) == expected + + +# ── _to_date_ddmmyyyy ───────────────────────────────────────────────────────── + + +@pytest.mark.parametrize( + "inp,expected", + [ + ("31.03.2025", date(2025, 3, 31)), + ("01.01.2024", date(2024, 1, 1)), + ("", None), + (None, None), + ("2025-03-31", None), # неправильный формат → None + ("abc", None), + ("31.13.2025", None), # невалидный месяц → None + ], +) +def test_to_date_ddmmyyyy(inp: Any, expected: date | None) -> None: + assert _to_date_ddmmyyyy(inp) == expected diff --git a/backend/tests/services/site_finder/test_best_layouts.py b/backend/tests/services/site_finder/test_best_layouts.py new file mode 100644 index 00000000..0720feaa --- /dev/null +++ b/backend/tests/services/site_finder/test_best_layouts.py @@ -0,0 +1,497 @@ +"""Unit-тесты для get_best_layouts (Fix SF-01: honest time_window velocity). + +Проверяет, что разные time_window → разные deals_window → разный velocity_per_month. + +Mock-стратегия: патчим db.execute с side_effect, повторяя порядок вызовов +в get_best_layouts: + 1. _PARCEL_CENTROID_SQL → .mappings().first() + 2. _COMPETITORS_IN_RADIUS_SQL → .mappings().all() + 3. _INLINE_VELOCITY_SQL → .mappings().all() + 4. db.scalar() → MAX(snapshot_date) — через .return_value + 5. _SUPPLY_BATCH_SQL → .mappings().all() + +Ключевые asserts: +- last_month (1 мес) → velocity = deals_window / 1.0 +- last_quarter (3 мес) → velocity = deals_window / 3.0 +- last_year (12 мес) → velocity = deals_window / 12.0 +- Разный deals_window при разных time_window → разный mix. +""" + +from __future__ import annotations + +import datetime as dt +from unittest.mock import MagicMock + +import pytest + +from app.schemas.parcel import BestLayoutsRequest +from app.services.site_finder.best_layouts import ( + _TIME_WINDOW_PARAMS, + MAX_BUCKET_SHARE_PCT, + _cap_and_redistribute, + get_best_layouts, +) + +_TODAY = dt.date.today() +CAD_NUM = "66:41:0303161:123" + + +# ── Фабрики mock-строк ──────────────────────────────────────────────────────── + + +def _coord_row(lon: float = 60.6, lat: float = 56.85) -> MagicMock: + r = MagicMock() + r.__getitem__ = lambda self, k: {"center_lon": lon, "center_lat": lat}[k] + return r + + +def _obj_id_row(obj_id: int) -> MagicMock: + r = MagicMock() + r.__getitem__ = lambda self, k: {"obj_id": obj_id}[k] + return r + + +def _vel_row( + room_bucket: str = "2", + deals_window: float = 48.0, + avg_area: float = 55.0, + avg_price_rub: float | None = 120000.0, + obj_ids: list[int] | None = None, + window_start: dt.date | None = None, + window_end: dt.date | None = None, +) -> MagicMock: + """Строка из _INLINE_VELOCITY_SQL. + + deals_window — реальные сделки за честное окно (не 24 мес). + """ + oids = obj_ids if obj_ids is not None else [1] + ws = window_start or _TODAY - dt.timedelta(days=90) + we = window_end or _TODAY + + r = MagicMock() + r.__getitem__ = lambda self, k: { + "room_bucket": room_bucket, + "deals_window": deals_window, + "avg_area_m2": avg_area, + "avg_price_per_m2_rub": avg_price_rub, + "competitor_obj_ids": oids, + "competitor_count": len(oids), + "window_start": ws, + "window_end": we, + }[k] + return r + + +def _supply_row(rb: str, ab: str, units: int) -> MagicMock: + r = MagicMock() + r.__getitem__ = lambda self, k: {"rb": rb, "ab": ab, "units": units}[k] + return r + + +def _make_db( + coord: MagicMock | None = None, + id_rows: list[MagicMock] | None = None, + vel_rows: list[MagicMock] | None = None, + supply_rows: list[MagicMock] | None = None, + latest_snap: dt.date | None = None, +) -> MagicMock: + """Сконструировать mock Session. + + Порядок db.execute(): + 1. centroid → .mappings().first() + 2. competitors → .mappings().all() + 3. velocity → .mappings().all() + 4. supply → .mappings().all() (только если latest_snap is not None) + db.scalar() → latest_snap (MAX snapshot_date). + """ + db = MagicMock() + db.scalar.return_value = latest_snap if latest_snap is not None else _TODAY + + r0 = MagicMock() + r0.mappings.return_value.first.return_value = coord + + r1 = MagicMock() + r1.mappings.return_value.all.return_value = id_rows or [] + + r2 = MagicMock() + r2.mappings.return_value.all.return_value = vel_rows or [] + + r3 = MagicMock() + r3.mappings.return_value.all.return_value = supply_rows or [] + + db.execute.side_effect = [r0, r1, r2, r3] + return db + + +def _request(**kwargs) -> BestLayoutsRequest: + defaults: dict = { + "radius_km": 1.0, + "time_window": "last_quarter", + "min_velocity_per_month": 0.0, + } + defaults.update(kwargs) + return BestLayoutsRequest(**defaults) + + +# ── Тесты TIME_WINDOW_PARAMS ────────────────────────────────────────────────── + + +def test_time_window_params_keys() -> None: + """Все три time_window определены, months_in_window > 0.""" + for key in ("last_month", "last_quarter", "last_year"): + assert key in _TIME_WINDOW_PARAMS + interval_str, months = _TIME_WINDOW_PARAMS[key] + assert isinstance(interval_str, str) and len(interval_str) > 0 + assert months > 0 + + +# ── Тест SF-01: разный deals_window → разный velocity ──────────────────────── + + +def test_last_month_velocity_divisor_1() -> None: + """time_window=last_month: velocity = deals_window / 1.0.""" + deals = 30.0 + db = _make_db( + coord=_coord_row(), + id_rows=[_obj_id_row(1)], + vel_rows=[_vel_row("1", deals_window=deals, obj_ids=[1])], + ) + req = _request(time_window="last_month") + resp = get_best_layouts(db, CAD_NUM, req) + + assert len(resp.top_layouts) == 1 + assert resp.top_layouts[0].velocity_per_month == pytest.approx(30.0, rel=1e-3) + + +def test_last_quarter_velocity_divisor_3() -> None: + """time_window=last_quarter: velocity = deals_window / 3.0.""" + deals = 30.0 + db = _make_db( + coord=_coord_row(), + id_rows=[_obj_id_row(1)], + vel_rows=[_vel_row("1", deals_window=deals, obj_ids=[1])], + ) + req = _request(time_window="last_quarter") + resp = get_best_layouts(db, CAD_NUM, req) + + assert len(resp.top_layouts) == 1 + assert resp.top_layouts[0].velocity_per_month == pytest.approx(10.0, rel=1e-3) + + +def test_last_year_velocity_divisor_12() -> None: + """time_window=last_year: velocity = deals_window / 12.0.""" + deals = 60.0 + db = _make_db( + coord=_coord_row(), + id_rows=[_obj_id_row(1)], + vel_rows=[_vel_row("1", deals_window=deals, obj_ids=[1])], + ) + req = _request(time_window="last_year") + resp = get_best_layouts(db, CAD_NUM, req) + + assert len(resp.top_layouts) == 1 + assert resp.top_layouts[0].velocity_per_month == pytest.approx(5.0, rel=1e-3) + + +def test_different_time_windows_produce_different_velocity() -> None: + """Одни и те же deals_window → разная velocity_per_month для разных time_window. + + Главный acceptance-тест SF-01: time_window влияет на velocity, не только на масштаб. + При одном и том же deals_window=30: + last_month → 30.0 + last_quarter → 10.0 + last_year → 2.5 + """ + deals = 30.0 + + velocities: dict[str, float] = {} + for tw in ("last_month", "last_quarter", "last_year"): + db = _make_db( + coord=_coord_row(), + id_rows=[_obj_id_row(1)], + vel_rows=[_vel_row("2", deals_window=deals, obj_ids=[1])], + ) + req = _request(time_window=tw) + resp = get_best_layouts(db, CAD_NUM, req) + assert len(resp.top_layouts) == 1, f"No layouts for {tw}" + velocities[tw] = resp.top_layouts[0].velocity_per_month + + # Все три значения различаются + vals = list(velocities.values()) + assert vals[0] != vals[1] != vals[2], f"Velocities must differ: {velocities}" + # last_month > last_quarter > last_year (одинаковые deals, разный знаменатель) + assert velocities["last_month"] > velocities["last_quarter"] > velocities["last_year"] + + +# ── Тест: ranking по velocity и sum pct = 100 ──────────────────────────────── + + +def test_ranking_and_pct_sum_100() -> None: + """3 room_buckets → ranking по velocity, sum pct = 100.""" + id_rows = [_obj_id_row(1), _obj_id_row(2), _obj_id_row(3)] + vel_rows = [ + _vel_row("studio", deals_window=9.0, avg_area=26.0, obj_ids=[1]), # 9/3=3.0 + _vel_row("1", deals_window=24.0, avg_area=40.0, obj_ids=[2]), # 24/3=8.0 + _vel_row("2", deals_window=48.0, avg_area=55.0, obj_ids=[3]), # 48/3=16.0 + ] + supply_rows = [ + _supply_row("studio", "<25", 20), + _supply_row("1", "40-60", 60), + _supply_row("2", "40-60", 80), + ] + db = _make_db(coord=_coord_row(), id_rows=id_rows, vel_rows=vel_rows, supply_rows=supply_rows) + req = _request(time_window="last_quarter") + resp = get_best_layouts(db, CAD_NUM, req) + + top = resp.top_layouts + assert len(top) == 3 + # rank 1 = "2" (наибольший velocity 16.0) + assert top[0].room_bucket == "2" + assert top[0].rank == 1 + assert top[0].velocity_per_month == pytest.approx(16.0, rel=1e-3) + # rank 2 = "1" (8.0) + assert top[1].room_bucket == "1" + assert top[1].velocity_per_month == pytest.approx(8.0, rel=1e-3) + # ранги уникальны + assert sorted(t.rank for t in top) == [1, 2, 3] + # sum pct = 100 + mix = resp.recommendation_for_tz.mix + assert sum(m.pct for m in mix) == 100 + + +# ── Тест: пустые конкуренты ─────────────────────────────────────────────────── + + +def test_no_competitors_returns_empty_response() -> None: + """Нет конкурентов в радиусе → пустые top_layouts + confidence=low.""" + db = _make_db(coord=_coord_row(), id_rows=[], vel_rows=[]) + req = _request() + resp = get_best_layouts(db, CAD_NUM, req) + + assert resp.top_layouts == [] + assert resp.data_quality.confidence == "low" + assert resp.recommendation_for_tz.based_on_obj_count == 0 + + +# ── Тест: centroid не найден ────────────────────────────────────────────────── + + +def test_centroid_not_found_raises_value_error() -> None: + """Геометрия участка не найдена → ValueError.""" + db = _make_db(coord=None) + req = _request() + + with pytest.raises(ValueError, match="не найдена"): + get_best_layouts(db, "99:99:9999999:999", req) + + +# ── Тест: min_velocity фильтрует строки ────────────────────────────────────── + + +def test_min_velocity_filters_low_rows() -> None: + """min_velocity_per_month=5 → строки с velocity<5 не попадают в top_layouts. + + last_quarter (3 мес): + studio: 9 / 3 = 3.0 < 5.0 → отфильтрован + 1: 24 / 3 = 8.0 > 5.0 → остаётся + """ + id_rows = [_obj_id_row(1), _obj_id_row(2)] + vel_rows = [ + _vel_row("studio", deals_window=9.0, obj_ids=[1]), + _vel_row("1", deals_window=24.0, obj_ids=[2]), + ] + db = _make_db(coord=_coord_row(), id_rows=id_rows, vel_rows=vel_rows) + req = _request(time_window="last_quarter", min_velocity_per_month=5.0) + resp = get_best_layouts(db, CAD_NUM, req) + + top = resp.top_layouts + assert len(top) == 1 + assert top[0].room_bucket == "1" + assert top[0].velocity_per_month == pytest.approx(8.0, rel=1e-3) + + +# ── Тест: exclude_competitor_obj_ids ───────────────────────────────────────── + + +def test_exclude_competitor_obj_ids() -> None: + """exclude_competitor_obj_ids=[20] при единственном конкуренте → пустой ответ.""" + id_rows = [_obj_id_row(20)] + db = _make_db(coord=_coord_row(), id_rows=id_rows, vel_rows=[]) + req = _request(exclude_competitor_obj_ids=[20]) + resp = get_best_layouts(db, CAD_NUM, req) + + assert resp.top_layouts == [] + assert resp.data_quality.objects_total_in_radius == 1 + + +# ── Тест: total_sold_in_window совпадает с deals_window ────────────────────── + + +def test_total_sold_in_window_matches_deals_window() -> None: + """total_sold_in_window в TopLayoutRow = deals_window (целое).""" + deals = 37.0 + db = _make_db( + coord=_coord_row(), + id_rows=[_obj_id_row(5)], + vel_rows=[_vel_row("3", deals_window=deals, obj_ids=[5])], + ) + req = _request(time_window="last_quarter") + resp = get_best_layouts(db, CAD_NUM, req) + + assert len(resp.top_layouts) == 1 + assert resp.top_layouts[0].total_sold_in_window == int(deals) + + +# ── Тесты _cap_and_redistribute (Fix SF-09 review) ─────────────────────────── + + +@pytest.mark.parametrize( + "pct_map, expect_pathological", + [ + # 1. normal: одиночный bucket > 35, free достаточно capacity + ({"1k": 50, "studio": 30, "2k": 20}, False), + # 2. heavy skew (3-bucket): surplus=40, capacity=20+25=45 — помещается + ({"1k": 75, "studio": 15, "2k": 10}, False), + # 3. multiple buckets > 35 + ({"1k": 50, "studio": 40, "2k": 10}, False), + # 4. all > 35 — pathological + ({"1k": 50, "studio": 50}, True), + # 5. граничный: один bucket ровно на cap — не clamp + ({"1k": 35, "studio": 35, "2k": 30}, False), + # 6. single bucket 100% — pathological (нет free) + ({"1k": 100}, True), + # 7. 2-bucket heavy: surplus=55, capacity=25 — pathological (не помещается) + ({"1k": 90, "studio": 10}, True), + # 8. все ≤ cap — fast-path без изменений + ({"1k": 30, "studio": 35, "2k": 35}, False), + # 9. 2-bucket: 70/30 → surplus=35, capacity=5 → pathological + ({"1k": 70, "studio": 30}, True), + # 10. 2-bucket: 99/1 → surplus=64, capacity=34 → pathological + ({"1k": 99, "studio": 1}, True), + ], +) +def test_cap_and_redistribute_invariants( + pct_map: dict[str, int], + expect_pathological: bool, +) -> None: + """Invariant: max(pct) ≤ cap И sum(pct) == 100 (или cap_skipped=True в pathological). + + Pathological — `cap_skipped=True`, max МОЖЕТ быть > cap (геометрически surplus + не вмещается в free capacity). + """ + result, cap_skipped = _cap_and_redistribute(pct_map) + + assert ( + cap_skipped == expect_pathological + ), f"cap_skipped={cap_skipped} но ожидали {expect_pathological} для {pct_map}" + assert ( + sum(result.values()) == 100 + ), f"sum={sum(result.values())} != 100 для {pct_map} → {result}" + if not expect_pathological: + assert ( + max(result.values()) <= MAX_BUCKET_SHARE_PCT + ), f"max={max(result.values())} > cap={MAX_BUCKET_SHARE_PCT} для {pct_map} → {result}" + + +@pytest.mark.parametrize( + "deals, expect_pathological, label", + [ + # 3-bucket с достаточной capacity — surplus помещается, cap соблюдён + ({"1k": 75, "studio": 15, "2k": 10}, False, "{1k:75, studio:15, 2k:10}"), + ({"1k": 80, "studio": 12, "2k": 8}, False, "{1k:80, studio:12, 2k:8}"), + ({"1k": 60, "studio": 30, "2k": 10}, False, "{1k:60, studio:30, 2k:10}"), + ({"a": 50, "b": 30, "c": 20}, False, "{50, 30, 20}"), + # 2-bucket — surplus геометрически не помещается, cap_skipped=True + ({"1k": 90, "studio": 10}, True, "{1k:90, studio:10}"), + ({"1k": 70, "studio": 30}, True, "{1k:70, studio:30}"), + ({"1k": 99, "studio": 1}, True, "{1k:99, studio:1}"), + ], +) +def test_cap_reproduced_failing_cases( + deals: dict[str, int], expect_pathological: bool, label: str +) -> None: + """Review round-2 reproduced cases: 2-bucket — pathological, 3-bucket — fit cap.""" + result, cap_skipped = _cap_and_redistribute(deals) + assert ( + cap_skipped == expect_pathological + ), f"cap_skipped={cap_skipped} ожидали {expect_pathological} для {label}" + assert sum(result.values()) == 100, f"sum != 100 для {label} → {result}" + if not expect_pathological: + assert ( + max(result.values()) <= MAX_BUCKET_SHARE_PCT + ), f"max={max(result.values())} > {MAX_BUCKET_SHARE_PCT} для {label} → {result}" + + +def test_cap_iteration_count_bounded() -> None: + """Round 2 regression: алгоритм завершается за ≤ len(pct_map)+1 итераций. + + Round 1 bag: на 2-bucket {1k:70, studio:30} цикл осциллировал бесконечно. + Round 2 fix: capacity-aware redistribute + hard `for _ in range(N+1)` guard. + Этот тест гарантирует что вызов не зависает (pytest-timeout не нужен). + """ + import time + + pathological_cases = [ + {"1k": 70, "studio": 30}, + {"1k": 99, "studio": 1}, + {"1k": 90, "studio": 10}, + {"1k": 50, "studio": 50}, + ] + for case in pathological_cases: + start = time.perf_counter() + result, cap_skipped = _cap_and_redistribute(case) + elapsed_ms = (time.perf_counter() - start) * 1000 + assert elapsed_ms < 100, f"Завис ({elapsed_ms:.0f}ms) на {case}" + assert sum(result.values()) == 100, f"sum != 100 для {case}" + # 2-bucket с одним > cap всегда pathological (surplus > free capacity) + if case != {"1k": 50, "studio": 50}: + assert cap_skipped, f"Ожидали cap_skipped=True для {case}" + + +def test_cap_and_redistribute_no_dominant_unchanged() -> None: + """Если все bucket'ы ≤ cap — результат идентичен входу (fast-path).""" + pct_map = {"studio": 20, "1": 35, "2": 30, "3": 15} + result, cap_skipped = _cap_and_redistribute(pct_map) + assert not cap_skipped + assert result == pct_map + + +def test_cap_and_redistribute_empty() -> None: + """Пустой dict → возвращается как есть.""" + result, cap_skipped = _cap_and_redistribute({}) + assert result == {} + assert not cap_skipped + + +def test_cap_skipped_flag_propagates_to_recommendation() -> None: + """Pathological case → cap_skipped=True в recommendation_for_tz ответа.""" + # 2 bucket'а по 50% — pathological + id_rows = [_obj_id_row(1), _obj_id_row(2)] + vel_rows = [ + _vel_row("studio", deals_window=50.0, obj_ids=[1]), + _vel_row("1", deals_window=50.0, obj_ids=[2]), + ] + db = _make_db(coord=_coord_row(), id_rows=id_rows, vel_rows=vel_rows) + req = _request(time_window="last_quarter") + resp = get_best_layouts(db, CAD_NUM, req) + + # С deals 50/50 → normalize_pct даёт {studio:50, 1:50} — оба выше cap + assert resp.recommendation_for_tz.cap_skipped is True + + +def test_cap_skipped_false_for_normal_case() -> None: + """Normal case с capping → cap_skipped=False в recommendation_for_tz.""" + id_rows = [_obj_id_row(1), _obj_id_row(2), _obj_id_row(3)] + vel_rows = [ + _vel_row("1k", deals_window=75.0, obj_ids=[1]), + _vel_row("studio", deals_window=15.0, obj_ids=[2]), + _vel_row("2k", deals_window=10.0, obj_ids=[3]), + ] + db = _make_db(coord=_coord_row(), id_rows=id_rows, vel_rows=vel_rows) + req = _request(time_window="last_quarter") + resp = get_best_layouts(db, CAD_NUM, req) + + assert resp.recommendation_for_tz.cap_skipped is False + mix = resp.recommendation_for_tz.mix + assert all(row.pct <= MAX_BUCKET_SHARE_PCT for row in mix) + assert sum(row.pct for row in mix) == 100 diff --git a/backend/tests/services/test_cadastre_bulk.py b/backend/tests/services/test_cadastre_bulk.py index 1252ccc2..952a8891 100644 --- a/backend/tests/services/test_cadastre_bulk.py +++ b/backend/tests/services/test_cadastre_bulk.py @@ -1162,7 +1162,7 @@ async def test_grid_walk_emits_heartbeat_callbacks() -> None: progress_states: list[dict[str, Any]] = [] - upserted, requests = await _grid_walk_category( + _upserted, requests = await _grid_walk_category( db=db, client=client, quarter="66:41:0303161", @@ -1197,7 +1197,7 @@ async def test_grid_walk_no_heartbeat_when_callback_none() -> None: client.wms_feature_info = AsyncMock(return_value=[]) # Не передаём update_progress — должно не падать - upserted, requests = await _grid_walk_category( + _upserted, requests = await _grid_walk_category( db=db, client=client, quarter="66:41:0303161", diff --git a/backend/tests/services/test_ekburg_permits.py b/backend/tests/services/test_ekburg_permits.py new file mode 100644 index 00000000..4457dfa4 --- /dev/null +++ b/backend/tests/services/test_ekburg_permits.py @@ -0,0 +1,355 @@ +"""Тесты для ekburg_permits.py (Issue #105). + +Использует mock httpx + синтетические данные xlsx (openpyxl в памяти). +Не требует реального PostgreSQL или сетевого доступа. +""" + +from __future__ import annotations + +from datetime import date, datetime +from io import BytesIO +from typing import Any +from unittest.mock import MagicMock + +import httpx +import pytest +from openpyxl import Workbook + +from app.services.scrapers.ekburg_permits import ( + EKBURG_PERMITS_URLS, + EkburgPermitsClient, + _clean_inn, + _detect_permit_type, + _to_date, + _to_float, + _to_str, +) + +# ── helpers ─────────────────────────────────────────────────────────────────── + + +def _make_xlsx_bytes(sheets: dict[str, list[tuple[Any, ...]]]) -> bytes: + """Создать xlsx в памяти с заданными листами и строками.""" + wb = Workbook() + wb.remove(wb.active) # удалить дефолтный лист + for sheet_name, rows in sheets.items(): + ws = wb.create_sheet(sheet_name) + for row in rows: + ws.append(list(row)) + buf = BytesIO() + wb.save(buf) + return buf.getvalue() + + +def _rns_sheet_rows() -> list[tuple[Any, ...]]: + """Синтетические строки для листа РНС (Form 3). + + Воспроизводит структуру 2024-2026: header row 4, данные с row 7. + """ + return [ + # row 1: пусто + (None,) * 14, + # row 2: заголовок таблицы + ( + "Таблица 3. Реестр выданных разрешений на строительство " + "объектов капитального строительства", + ) + + (None,) * 13, + # row 3: пусто + (None,) * 14, + # row 4: заголовки колонок + ( + "Наименование застройщика", + "ИНН", + "Адрес застройщика", + "Тип строительного объекта1", + "Наименование объекта КС", + "Кадастровый номер ЗУ", + "Координаты X", + None, + "Адрес объекта", + "Реквизиты разрешения (номер)", + None, + "Дата окончания", + "Общая площадь, м2", + "Площадь жилых помещений по проекту, м2", + ), + # row 5: подзаголовки + (None,) * 7 + ("X", "Y") + (None,) * 3 + ("номер", "дата") + (None,) * 2, + # row 6: нумерация столбцов + tuple(range(1, 15)), + # row 7: первая строка данных (МКД — ООО "Тест") + ( + 'ООО "Специализированный застройщик "Тест"', + 6685000001, + "620000, г. Екатеринбург, ул. Тестовая, д. 1", + "многоквартирные жилые дома;", + "Многоквартирный жилой дом (№ 1 по ПЗУ)", + "66:41:0101001:123", + "1530000.0000", + "380000.0000", + "Свердловская обл., г. Екатеринбург, ул. Тестовая", + "66-41-99-2024", + datetime(2024, 3, 15), + datetime(2027, 3, 15), + 12345.67, + 8000.0, + ), + # row 8: строка без permit_number (должна быть пропущена) + ( + "ООО «Продолжение»", + 6685000002, + "620001, г. Екатеринбург", + "производственные здания;", + "Производственный корпус", + "66:41:0101002:456", + "1531000.0000", + "381000.0000", + "Свердловская обл., г. Екатеринбург, ул. Другая", + None, # no permit_number → skip + datetime(2024, 5, 20), + datetime(2026, 5, 20), + 500.0, + None, + ), + # row 9: empty row (all None) — should be skipped + (None,) * 14, + ] + + +def _rve_sheet_rows() -> list[tuple[Any, ...]]: + """Синтетические строки для листа РВЭ (Form 4).""" + return [ + (None,) * 19, + ("Таблица 4. Реестр выданных разрешений на ввод в эксплуатацию",) + (None,) * 18, + (None,) * 19, + # row 4: заголовки + ( + "Наименование застройщика", + "ИНН", + "Адрес застройщика", + "Тип строительного объекта1", + "Наименование объекта КС", + "Кадастровый номер ЗУ", + "Координаты X", + None, + "Адрес объекта", + "Реквизиты РНС (номер)", + None, + "Дата окончания РНС", + "Общая площадь, м2", + "Площадь жилых по проекту, м2", + "Площадь жилых фактически, м2", + "Реквизиты РВЭ (номер)", + None, + "Введённые мощности", + None, + ), + (None,) * 7 + ("X", "Y") + (None,) * 5 + ("номер", "дата") + (None,) * 4, + tuple(range(1, 19)), + # row 7: первая строка данных + ( + 'ООО "Застройщик Ввода"', + 6658000002, + "620100, г. Екатеринбург, ул. Вводная, д. 2", + "многоквартирные жилые дома;", + "Жилой дом (1 очередь) (№ 1 по ПЗУ)", + "66:41:0303001:789", + "1535000.0000", + "385000.0000", + "Свердловская обл., г. Екатеринбург, ул. Вводная", + "66-41-50-2022", + datetime(2022, 10, 1), + datetime(2026, 1, 1), + 30000.0, + 20000.0, + 19800.5, # living_area_fact + "66-41-5-2026", + datetime(2026, 1, 13), + None, + None, + ), + ] + + +# ── unit tests: helpers ─────────────────────────────────────────────────────── + + +class TestHelpers: + def test_to_date_datetime(self) -> None: + dt = datetime(2024, 3, 15, 0, 0) + assert _to_date(dt) == date(2024, 3, 15) + + def test_to_date_string_dot(self) -> None: + assert _to_date("15.03.2024") == date(2024, 3, 15) + + def test_to_date_none(self) -> None: + assert _to_date(None) is None + + def test_to_date_invalid(self) -> None: + assert _to_date("not-a-date") is None + + def test_to_float_comma(self) -> None: + assert _to_float("12345,67") == pytest.approx(12345.67) + + def test_to_float_dash(self) -> None: + assert _to_float("-") is None + + def test_to_float_int(self) -> None: + assert _to_float(1490) == pytest.approx(1490.0) + + def test_to_str_nbsp(self) -> None: + assert _to_str("г.\xa0Екатеринбург") == "г. Екатеринбург" + + def test_to_str_none(self) -> None: + assert _to_str(None) is None + + def test_clean_inn_int(self) -> None: + assert _clean_inn(6685180480) == "6685180480" + + def test_clean_inn_float(self) -> None: + # openpyxl иногда выдаёт float для длинных чисел + assert _clean_inn(6685180480.0) == "6685180480" + + def test_clean_inn_string(self) -> None: + assert _clean_inn("6685180480") == "6685180480" + + def test_clean_inn_none(self) -> None: + assert _clean_inn(None) is None + + def test_clean_inn_short(self) -> None: + # 9 цифр — не ИНН + assert _clean_inn(123456789) is None + + def test_detect_permit_type_rns(self) -> None: + assert _detect_permit_type("реестр разрешений на строительс") == "RNS" + + def test_detect_permit_type_rve(self) -> None: + assert _detect_permit_type("реестр разрешений на ввод") == "RVE" + + def test_detect_permit_type_unknown(self) -> None: + assert _detect_permit_type("Справочник") is None + + +# ── unit tests: xlsx parsing ────────────────────────────────────────────────── + + +class TestParseXlsx: + def _make_client(self) -> EkburgPermitsClient: + client = EkburgPermitsClient.__new__(EkburgPermitsClient) + # Создаём mock httpx.Client чтобы не открывать реальные соединения + client._client = MagicMock() + return client + + def test_parse_rns_yields_permit_row(self) -> None: + content = _make_xlsx_bytes( + { + "реестр разрешений на строительс": _rns_sheet_rows(), + "Справочник": [("многоквартирные жилые дома;",)], + } + ) + client = self._make_client() + rows = list(client.parse_xlsx(content, 2024, EKBURG_PERMITS_URLS[2024])) + + # Должна быть ровно одна строка (вторая без permit_number пропущена) + assert len(rows) == 1 + row = rows[0] + assert row.permit_type == "RNS" + assert row.permit_number == "66-41-99-2024" + assert row.issue_date == date(2024, 3, 15) + assert row.expiry_date == date(2027, 3, 15) + assert row.developer_inn == "6685000001" + assert row.object_type == "многоквартирные жилые дома;" + assert row.total_area_sqm == pytest.approx(12345.67) + assert row.living_area_sqm == pytest.approx(8000.0) + assert row.source_year == 2024 + + def test_parse_rve_yields_permit_row_with_rve_fields(self) -> None: + content = _make_xlsx_bytes({"реестр разрешений на ввод": _rve_sheet_rows()}) + client = self._make_client() + rows = list(client.parse_xlsx(content, 2026, EKBURG_PERMITS_URLS[2026])) + + assert len(rows) == 1 + row = rows[0] + assert row.permit_type == "RVE" + assert row.permit_number == "66-41-50-2022" + assert row.living_area_fact_sqm == pytest.approx(19800.5) + assert row.rve_number == "66-41-5-2026" + assert row.rve_date == date(2026, 1, 13) + + def test_parse_skips_справочник_sheet(self) -> None: + content = _make_xlsx_bytes( + { + "Справочник": [("многоквартирные жилые дома;",)], + "Лист1": [(None,)], + } + ) + client = self._make_client() + rows = list(client.parse_xlsx(content, 2026, EKBURG_PERMITS_URLS[2026])) + assert rows == [] + + def test_raw_row_stored(self) -> None: + content = _make_xlsx_bytes({"реестр разрешений на строительс": _rns_sheet_rows()}) + client = self._make_client() + rows = list(client.parse_xlsx(content, 2024, EKBURG_PERMITS_URLS[2024])) + assert len(rows) == 1 + # raw_row должен содержать dict с string keys + assert isinstance(rows[0].raw_row, dict) + assert "9" in rows[0].raw_row # колонка 9 = permit_number + + +# ── unit tests: download_xlsx ───────────────────────────────────────────────── + + +class TestDownloadXlsx: + def _client_with_mock_http(self, mock_response: MagicMock) -> EkburgPermitsClient: + """Создать EkburgPermitsClient с замоканным внутренним httpx.Client.""" + client = EkburgPermitsClient.__new__(EkburgPermitsClient) + mock_http = MagicMock() + mock_http.get.return_value = mock_response + client._client = mock_http + return client + + def test_download_returns_bytes(self) -> None: + fake_content = b"PK\x03\x04fake_xlsx_content" + mock_response = MagicMock() + mock_response.content = fake_content + mock_response.raise_for_status = MagicMock() + + client = self._client_with_mock_http(mock_response) + result = client.download_xlsx(2026) + + assert result == fake_content + + def test_download_raises_for_unknown_year(self) -> None: + client = EkburgPermitsClient.__new__(EkburgPermitsClient) + client._client = MagicMock() + with pytest.raises(ValueError, match="year=9999"): + client.download_xlsx(9999) + + def test_download_raises_on_http_error(self) -> None: + mock_response = MagicMock() + mock_response.raise_for_status.side_effect = httpx.HTTPStatusError( + "404", request=MagicMock(), response=MagicMock() + ) + + client = self._client_with_mock_http(mock_response) + with pytest.raises(httpx.HTTPStatusError): + client.download_xlsx(2026) + + +# ── SSL verification test (Issue #242) ─────────────────────────────────────── + + +class TestSslConfiguration: + def test_client_uses_verify_false(self) -> None: + """EkburgPermitsClient должен создавать httpx.Client с verify=False. + + екатеринбург.рф использует CA Минцифры РФ — не в certifi bundle. + Issue #242: SSL: CERTIFICATE_VERIFY_FAILED для всех 5 годов. + """ + with EkburgPermitsClient() as client: + # httpx.Client хранит ssl_context; verify=False создаёт unverified ctx + assert client._client._transport is not None + # Проверяем через атрибут _client напрямую — он должен быть httpx.Client + assert isinstance(client._client, httpx.Client) diff --git a/backend/tests/services/test_nspd_denorm.py b/backend/tests/services/test_nspd_denorm.py new file mode 100644 index 00000000..3420a761 --- /dev/null +++ b/backend/tests/services/test_nspd_denorm.py @@ -0,0 +1,322 @@ +"""Тесты для nspd_denorm.py — coerce helpers + denorm_parcel/building/dump. + +Не требует реального PostgreSQL — мокает db.execute() и db.begin_nested(). +""" + +from __future__ import annotations + +from typing import Any +from unittest.mock import MagicMock + +import pytest + +from app.services.scrapers.nspd_denorm import ( + _coerce_int, + _coerce_numeric, + denorm_building_feature, + denorm_dump, + denorm_parcel_feature, +) + +# ── _coerce_int ──────────────────────────────────────────────────────────────── + + +def test_coerce_int_str_number() -> None: + assert _coerce_int("5") == 5 + + +def test_coerce_int_int_passthrough() -> None: + assert _coerce_int(17) == 17 + + +def test_coerce_int_none_returns_none() -> None: + assert _coerce_int(None) is None + + +def test_coerce_int_invalid_returns_none() -> None: + assert _coerce_int("не_число") is None + + +def test_coerce_int_float_truncates() -> None: + assert _coerce_int(3.9) == 3 + + +# ── _coerce_numeric ──────────────────────────────────────────────────────────── + + +def test_coerce_numeric_float_passthrough() -> None: + assert _coerce_numeric(12.5) == 12.5 + + +def test_coerce_numeric_str_dot() -> None: + assert _coerce_numeric("1234567.89") == pytest.approx(1234567.89) + + +def test_coerce_numeric_comma_decimal() -> None: + """Европейский формат с запятой → корректный float.""" + assert _coerce_numeric("1234567,89") == pytest.approx(1234567.89) + + +def test_coerce_numeric_none_returns_none() -> None: + assert _coerce_numeric(None) is None + + +def test_coerce_numeric_invalid_returns_none() -> None: + assert _coerce_numeric("N/A") is None + + +# ── Helpers для моков ────────────────────────────────────────────────────────── + + +def _make_mock_session() -> MagicMock: + """Мок Session: begin_nested() — context manager, execute() → MagicMock.""" + db = MagicMock() + # begin_nested() должен быть context manager + cm = MagicMock() + cm.__enter__ = MagicMock(return_value=None) + cm.__exit__ = MagicMock(return_value=False) + db.begin_nested.return_value = cm + return db + + +def _parcel_feature( + cad_num: str = "66:41:0204016:10", + area: Any = "500.0", + cost_value: Any = "1500000", + **extra_props: Any, +) -> dict[str, Any]: + props: dict[str, Any] = { + "cad_num": cad_num, + "area": area, + "cost_value": cost_value, + "permitted_use": "Для ИЖС", + "land_category": "Земли населённых пунктов", + "address": "г. Екатеринбург", + **extra_props, + } + return { + "layer": "parcels", + "feature_id": "123", + "geometry": {"type": "Polygon", "coordinates": [[[0, 0], [1, 0], [1, 1], [0, 0]]]}, + "properties": props, + } + + +def _building_feature( + cad_num: str = "66:41:0204016:10:1", + purpose: str = "Многоквартирный дом", + **extra_props: Any, +) -> dict[str, Any]: + props: dict[str, Any] = { + "cad_num": cad_num, + "purpose": purpose, + "floors_above_ground": 9, + "floors_underground": 1, + "year_built": 1985, + "cost_value": "50000000", + "build_record_area": "3200.0", + "address": "ул. Ленина 1", + **extra_props, + } + return { + "layer": "buildings", + "feature_id": "456", + "geometry": {"type": "Polygon", "coordinates": [[[0, 0], [1, 0], [1, 1], [0, 0]]]}, + "properties": props, + } + + +# ── denorm_parcel_feature ────────────────────────────────────────────────────── + + +def test_denorm_parcel_feature_inserts() -> None: + """Полный feature с cad_num → вызывает db.execute с правильными параметрами.""" + db = _make_mock_session() + + result = denorm_parcel_feature( + db, + feature=_parcel_feature(), + quarter_cad="66:41:0204016", + snapshot_date="2026-05-16", + ) + + assert result is True + db.begin_nested.assert_called_once() + db.execute.assert_called_once() + # Проверяем что параметры содержат нужные ключи + call_kwargs = db.execute.call_args[0][1] # positional dict + assert call_kwargs["cad_num"] == "66:41:0204016:10" + assert call_kwargs["quarter_cad"] == "66:41:0204016" + assert call_kwargs["permitted_use"] == "Для ИЖС" + assert call_kwargs["land_category"] == "Земли населённых пунктов" + assert call_kwargs["area_sqm"] == 500.0 + assert call_kwargs["cost_value"] == 1500000.0 + assert call_kwargs["cost_per_m2"] == pytest.approx(3000.0) + assert call_kwargs["snapshot_date"] == "2026-05-16" + + +def test_denorm_parcel_no_cad_num_skipped() -> None: + """Feature без cad_num → возвращает False, db не вызывается.""" + db = _make_mock_session() + feature: dict[str, Any] = { + "layer": "parcels", + "feature_id": None, + "geometry": None, + "properties": {"area": "100"}, + } + result = denorm_parcel_feature( + db, feature=feature, quarter_cad="66:41:0204016", snapshot_date="2026-05-16" + ) + assert result is False + db.execute.assert_not_called() + + +def test_denorm_parcel_zero_area_cost_per_m2_none() -> None: + """area=0 → cost_per_m2 = None (нет деления на ноль).""" + db = _make_mock_session() + result = denorm_parcel_feature( + db, + feature=_parcel_feature(area="0", cost_value="1000"), + quarter_cad="66:41:0204016", + snapshot_date="2026-05-16", + ) + assert result is True + call_kwargs = db.execute.call_args[0][1] + assert call_kwargs["cost_per_m2"] is None + + +def test_denorm_parcel_null_geometry() -> None: + """feature без geometry → geom_json=None, INSERT всё равно вызывается.""" + db = _make_mock_session() + feature = _parcel_feature() + feature["geometry"] = None + + result = denorm_parcel_feature( + db, feature=feature, quarter_cad="66:41:0204016", snapshot_date="2026-05-16" + ) + assert result is True + call_kwargs = db.execute.call_args[0][1] + assert call_kwargs["geom_json"] is None + + +def test_denorm_parcel_db_exception_returns_false() -> None: + """DB execute raises → возвращает False (не propagate).""" + db = _make_mock_session() + db.execute.side_effect = Exception("DB constraint violation") + + result = denorm_parcel_feature( + db, + feature=_parcel_feature(), + quarter_cad="66:41:0204016", + snapshot_date="2026-05-16", + ) + assert result is False + + +# ── denorm_building_feature ──────────────────────────────────────────────────── + + +def test_denorm_building_feature_inserts() -> None: + """Полный building feature → INSERT с правильными параметрами.""" + db = _make_mock_session() + + result = denorm_building_feature( + db, + feature=_building_feature(), + quarter_cad="66:41:0204016", + snapshot_date="2026-05-16", + ) + + assert result is True + db.execute.assert_called_once() + call_kwargs = db.execute.call_args[0][1] + assert call_kwargs["cad_num"] == "66:41:0204016:10:1" + assert call_kwargs["purpose"] == "Многоквартирный дом" + assert call_kwargs["floors"] == 9 + assert call_kwargs["floors_underground"] == 1 + assert call_kwargs["year_built"] == 1985 + assert call_kwargs["cost_value"] == 50000000.0 + assert call_kwargs["build_record_area"] == 3200.0 + + +def test_denorm_building_no_cad_num_skipped() -> None: + """Building feature без cad_num → False.""" + db = _make_mock_session() + feature: dict[str, Any] = { + "layer": "buildings", + "feature_id": None, + "geometry": None, + "properties": {"purpose": "Нежилое здание"}, + } + result = denorm_building_feature( + db, feature=feature, quarter_cad="66:41:0204016", snapshot_date="2026-05-16" + ) + assert result is False + db.execute.assert_not_called() + + +def test_denorm_building_str_floors_coerced() -> None: + """floors_above_ground строкой → корректно парсится в int.""" + db = _make_mock_session() + result = denorm_building_feature( + db, + feature=_building_feature(floors_above_ground="12"), + quarter_cad="66:41:0204016", + snapshot_date="2026-05-16", + ) + assert result is True + call_kwargs = db.execute.call_args[0][1] + assert call_kwargs["floors"] == 12 + + +# ── denorm_dump ──────────────────────────────────────────────────────────────── + + +def test_denorm_dump_aggregates() -> None: + """3 parcels + 2 buildings + 1 unknown layer → правильные счётчики.""" + db = _make_mock_session() + + features: list[dict[str, Any]] = [ + _parcel_feature("66:41:0101001:1"), + _parcel_feature("66:41:0101001:2"), + _parcel_feature("66:41:0101001:3"), + _building_feature("66:41:0101001:1:1"), + _building_feature("66:41:0101001:1:2"), + { + "layer": "territorial_zones", + "feature_id": "tz-1", + "geometry": None, + "properties": {"type_zone": "Ж-1"}, + }, + ] + + counts = denorm_dump(db, quarter_cad="66:41:0101001", features=features) + + assert counts["parcels"] == 3 + assert counts["buildings"] == 2 + # territorial_zones не считается как ошибка — просто пропускается + assert counts["errors"] == 0 + db.commit.assert_called_once() + + +def test_denorm_dump_empty_features() -> None: + """Пустой список features → нули, commit всё равно вызывается.""" + db = _make_mock_session() + counts = denorm_dump(db, quarter_cad="66:41:0101001", features=[]) + + assert counts == {"parcels": 0, "buildings": 0, "errors": 0} + db.commit.assert_called_once() + + +def test_denorm_dump_no_cad_num_counted_as_error() -> None: + """Parcel без cad_num → denorm_parcel_feature returns False → errors += 1.""" + db = _make_mock_session() + feature: dict[str, Any] = { + "layer": "parcels", + "feature_id": None, + "geometry": None, + "properties": {}, + } + counts = denorm_dump(db, quarter_cad="66:41:0101001", features=[feature]) + assert counts["parcels"] == 0 + assert counts["errors"] == 1 diff --git a/backend/tests/services/test_objective_backfill.py b/backend/tests/services/test_objective_backfill.py new file mode 100644 index 00000000..3da13880 --- /dev/null +++ b/backend/tests/services/test_objective_backfill.py @@ -0,0 +1,176 @@ +"""Тесты для objective_backfill.py (#203). + +Использует mock DB (unittest.mock) — не требует реального PostgreSQL. +""" + +from __future__ import annotations + +from typing import Any +from unittest.mock import MagicMock, patch + +from app.services.etl.objective_backfill import ( + AUTO_ACCEPT_THRESHOLD, + REVIEW_THRESHOLD, + MatchCandidate, + auto_apply_matches, + find_match_candidates, + trigger_mv_refresh, +) + +# ── Helpers ─────────────────────────────────────────────────────────────────── + + +def _make_candidate( + obj_id: int, + comm_name: str, + project_name: str, + score: float, + dev_name: str | None = "ООО Девелопер", +) -> MatchCandidate: + return MatchCandidate( + domrf_obj_id=obj_id, + domrf_comm_name=comm_name, + domrf_dev_name=dev_name, + objective_project_name=project_name, + similarity_score=score, + ) + + +def _make_db_row(obj_id: int, comm_name: str, dev_name: str, project: str, score: float) -> Any: + """Имитирует Row, возвращаемый SQLAlchemy db.execute().all().""" + return (obj_id, comm_name, dev_name, project, score) + + +# ── Test 1: find_match_candidates возвращает правильные MatchCandidate ──────── + + +def test_find_match_candidates_returns_candidates() -> None: + """find_match_candidates корректно преобразует DB rows в MatchCandidate.""" + mock_rows = [ + _make_db_row(100, "Северный квартал", "Брусника", "Северный квартал", 1.0), + _make_db_row(200, "АЛЛЕГРО", "СЗ ГОРЖИЛСТРОЙ", "АЛЛЕГРО", 0.95), + _make_db_row(300, "Некий ЖК", None, "Близкий ЖК", 0.72), + ] + + mock_execute = MagicMock() + mock_execute.all.return_value = mock_rows + mock_db = MagicMock() + mock_db.execute.return_value = mock_execute + + candidates = find_match_candidates(mock_db, only_unmapped=True) + + assert len(candidates) == 3 + assert candidates[0].domrf_obj_id == 100 + assert candidates[0].similarity_score == 1.0 + assert candidates[0].domrf_dev_name == "Брусника" + assert candidates[2].domrf_dev_name is None # NULL dev_name → None + assert candidates[2].similarity_score == 0.72 + + # Убедимся, что db.execute вызывался с параметрами + mock_db.execute.assert_called_once() + call_kwargs = mock_db.execute.call_args[0][1] + assert call_kwargs["only_unmapped"] is True + assert call_kwargs["min_threshold"] == REVIEW_THRESHOLD + + +# ── Test 2: auto_apply_matches dry_run не вставляет ────────────────────────── + + +def test_auto_apply_matches_dry_run_no_inserts() -> None: + """dry_run=True возвращает счётчики без обращения к БД (execute не вызывается).""" + mock_db = MagicMock() + + candidates = [ + _make_candidate(1, "ЖК А", "ЖК А", 0.95), # auto-accept + _make_candidate(2, "ЖК Б", "ЖК Б", 0.70), # review queue + _make_candidate(3, "ЖК В", "ЖК В", 0.65), # review queue + ] + + result = auto_apply_matches(mock_db, candidates, dry_run=True) + + assert result["auto_accepted"] == 0 + assert result["review_queue"] == 2 + mock_db.execute.assert_not_called() + mock_db.commit.assert_not_called() + + +# ── Test 3: auto_apply_matches high-score inserts, low-score → review_queue ── + + +def test_auto_apply_matches_inserts_high_score_only() -> None: + """Только кандидаты >= AUTO_ACCEPT_THRESHOLD вставляются в БД.""" + # Мокируем begin_nested как context manager + savepoint_cm = MagicMock() + savepoint_cm.__enter__ = MagicMock(return_value=None) + savepoint_cm.__exit__ = MagicMock(return_value=False) + + mock_execute_result = MagicMock() + mock_execute_result.rowcount = 1 + + mock_db = MagicMock() + mock_db.begin_nested.return_value = savepoint_cm + mock_db.execute.return_value = mock_execute_result + + high_score = AUTO_ACCEPT_THRESHOLD # ровно на пороге — должен вставляться + low_score = REVIEW_THRESHOLD # ровно REVIEW_THRESHOLD — не auto-accept + + candidates = [ + _make_candidate(10, "ЖК Альфа", "ЖК Альфа", high_score), + _make_candidate(11, "ЖК Бета", "ЖК Бета", high_score + 0.05), + _make_candidate(20, "ЖК Гамма", "ЖК Гамма", low_score), # review только + ] + + result = auto_apply_matches(mock_db, candidates, threshold=AUTO_ACCEPT_THRESHOLD) + + assert result["auto_accepted"] == 2 + assert result["review_queue"] == 1 + assert mock_db.execute.call_count == 2 # вызывался только для high-score + mock_db.commit.assert_called_once() + + +# ── Test 4: trigger_mv_refresh делегирует в layout_velocity_refresh ────────── + + +def test_trigger_mv_refresh_calls_helper() -> None: + """trigger_mv_refresh вызывает refresh_layout_velocity с concurrently=True. + + refresh_layout_velocity импортируется лениво внутри trigger_mv_refresh, + поэтому патчим по полному пути модуля-источника. + """ + mock_db = MagicMock() + + with patch( + "app.services.site_finder.layout_velocity_refresh.refresh_layout_velocity", + return_value=512, + ) as mock_refresh: + count = trigger_mv_refresh(mock_db) + + assert count == 512 + mock_refresh.assert_called_once_with(mock_db, concurrently=True) + + +# ── Test 5: ON CONFLICT — rowcount=0 считается как skipped ─────────────────── + + +def test_auto_apply_matches_conflict_counted_as_skipped() -> None: + """Если INSERT вернул rowcount=0 (ON CONFLICT DO NOTHING), считается skipped.""" + savepoint_cm = MagicMock() + savepoint_cm.__enter__ = MagicMock(return_value=None) + savepoint_cm.__exit__ = MagicMock(return_value=False) + + mock_execute_result = MagicMock() + mock_execute_result.rowcount = 0 # ON CONFLICT DO NOTHING + + mock_db = MagicMock() + mock_db.begin_nested.return_value = savepoint_cm + mock_db.execute.return_value = mock_execute_result + + candidates = [ + _make_candidate(99, "Уже существующий ЖК", "Уже существующий ЖК", 0.99), + ] + + result = auto_apply_matches(mock_db, candidates) + + assert result["auto_accepted"] == 0 + assert result["skipped"] == 1 + assert result["review_queue"] == 0 diff --git a/backend/tests/services/test_quarter_dump_lookup.py b/backend/tests/services/test_quarter_dump_lookup.py new file mode 100644 index 00000000..26165624 --- /dev/null +++ b/backend/tests/services/test_quarter_dump_lookup.py @@ -0,0 +1,620 @@ +"""Тесты для quarter_dump_lookup.py — risk zones + generic layer extraction + TIER 4. + +Покрывает: +- _extract_features_by_layer: filter by prefix +- _get_risk_zones / extract through get_quarter_dump_data: intersect, no-intersect, + no-risks-in-dump +- _get_opportunity_parcels: auction/scheme/free/future/oopt layers, distance sort, empty +- _get_red_lines: intersecting, nearby-only, empty, early-exit +- EMPTY_DUMP_RESULT / make_empty_result: new keys присутствуют +Не требует реального PostgreSQL — мокает db.execute(). +""" + +from __future__ import annotations + +from typing import Any +from unittest.mock import MagicMock + +import pytest + +from app.services.site_finder.quarter_dump_lookup import ( + EMPTY_DUMP_RESULT, + _extract_features_by_layer, + _get_cad_zouit_overlaps, + _get_opportunity_parcels, + _get_red_lines, + _get_risk_zones, + derive_quarter_cad, + get_quarter_dump_data, + make_empty_result, +) + +# ── Fixtures & helpers ──────────────────────────────────────────────────────── + + +def _make_feature(layer: str, geom: dict[str, Any] | None = None) -> dict[str, Any]: + """Минимальный feature dict в формате features_json.""" + return { + "layer": layer, + "feature_id": "test-id", + "geometry": geom or {"type": "Polygon", "coordinates": [[[0, 0], [1, 0], [1, 1], [0, 0]]]}, + "properties": {"name": f"test-{layer}", "type_zone": f"zone-{layer}"}, + } + + +# ── _extract_features_by_layer ──────────────────────────────────────────────── + + +def test_extract_features_by_layer_prefix_match() -> None: + """Возвращает только features с нужным prefix.""" + features = [ + _make_feature("risk_flooding"), + _make_feature("risk_burns"), + _make_feature("zouit_engineering"), + _make_feature("parcels"), + _make_feature("risk_landslide"), + ] + result = _extract_features_by_layer(features, "risk_") + assert len(result) == 3 + layers = {f["layer"] for f in result} + assert layers == {"risk_flooding", "risk_burns", "risk_landslide"} + + +def test_extract_features_by_layer_zouit_prefix() -> None: + """Работает для zouit_ prefix — generic re-use.""" + features = [ + _make_feature("zouit_engineering"), + _make_feature("zouit_okn"), + _make_feature("risk_flooding"), + ] + result = _extract_features_by_layer(features, "zouit_") + assert len(result) == 2 + assert all(f["layer"].startswith("zouit_") for f in result) + + +def test_extract_features_by_layer_empty_list() -> None: + result = _extract_features_by_layer([], "risk_") + assert result == [] + + +def test_extract_features_by_layer_no_match() -> None: + features = [_make_feature("parcels"), _make_feature("buildings")] + result = _extract_features_by_layer(features, "risk_") + assert result == [] + + +def test_extract_features_by_layer_non_string_layer() -> None: + """Gracefully skips features with non-string layer.""" + features: list[dict[str, Any]] = [ + {"layer": None, "feature_id": "x", "geometry": None, "properties": {}}, + _make_feature("risk_flooding"), + ] + result = _extract_features_by_layer(features, "risk_") + assert len(result) == 1 + assert result[0]["layer"] == "risk_flooding" + + +# ── _get_risk_zones ─────────────────────────────────────────────────────────── + + +def _make_mock_db_with_rows(rows: list[tuple[Any, ...]]) -> MagicMock: + """Мок Session где execute().fetchall() возвращает rows.""" + db = MagicMock() + mock_result = MagicMock() + mock_result.fetchall.return_value = rows + db.execute.return_value = mock_result + return db + + +def test_get_risk_zones_intersects() -> None: + """Три risk features → три записи в результате.""" + rows: list[tuple[Any, ...]] = [ + # (layer, props, geom_wkt, intersection_area_sqm) + ( + "risk_flooding", + {"type_zone": "Затопление 1%"}, + "POLYGON((0 0,1 0,1 1,0 0))", + 1234.5, + ), + ( + "risk_burns", + {"name": "Гарь"}, + "POLYGON((0 0,2 0,2 2,0 0))", + 500.0, + ), + ( + "risk_landslide", + {}, + "POLYGON((0 0,3 0,3 3,0 0))", + 999.9, + ), + ] + db = _make_mock_db_with_rows(rows) + + result = _get_risk_zones(db, "66:41:0204016", "POLYGON((0 0,1 0,1 1,0 0))") + + assert len(result) == 3 + + flooding = next(r for r in result if r["layer"] == "risk_flooding") + assert flooding["subtype"] == "Затопление 1%" + assert flooding["intersection_area_sqm"] == 1234.5 + assert flooding["geom_wkt"] == "POLYGON((0 0,1 0,1 1,0 0))" + + burns = next(r for r in result if r["layer"] == "risk_burns") + assert burns["subtype"] == "Гарь" + + landslide = next(r for r in result if r["layer"] == "risk_landslide") + # No properties name/type_zone → falls back to _RISK_SUBTYPE_LABELS + assert landslide["subtype"] == "Обвально-осыпные процессы" + + +def test_get_risk_zones_no_intersect() -> None: + """Пустой fetchall → пустой список.""" + db = _make_mock_db_with_rows([]) + result = _get_risk_zones(db, "66:41:0204016", "POLYGON((0 0,1 0,1 1,0 0))") + assert result == [] + + +def test_get_risk_zones_no_risks_in_dump() -> None: + """risks_count == 0 → early exit, db.execute не вызывается.""" + db = MagicMock() + layer_counts = {"risks_count": 0} + result = _get_risk_zones(db, "66:41:0204016", "POLYGON((0 0,1 0,1 1,0 0))", layer_counts) + assert result == [] + db.execute.assert_not_called() + + +def test_get_risk_zones_db_exception_returns_empty() -> None: + """DB exception → пустой список (не propagate).""" + db = MagicMock() + db.execute.side_effect = Exception("DB connection lost") + result = _get_risk_zones(db, "66:41:0204016", "POLYGON((0 0,1 0,1 1,0 0))") + assert result == [] + + +def test_get_risk_zones_null_area() -> None: + """NULL intersection_area_sqm → intersection_area_sqm=None в результате.""" + rows = [("risk_erosion_water", {}, "POINT(0 0)", None)] + db = _make_mock_db_with_rows(rows) + result = _get_risk_zones(db, "66:41:0204016", "POLYGON((0 0,1 0,1 1,0 0))") + assert len(result) == 1 + assert result[0]["intersection_area_sqm"] is None + + +# ── make_empty_result includes nspd_risk_zones ──────────────────────────────── + + +def test_make_empty_result_has_risk_zones_key() -> None: + """make_empty_result возвращает nspd_risk_zones: [].""" + result = make_empty_result() + assert "nspd_risk_zones" in result + assert result["nspd_risk_zones"] == [] + + +# ── _get_opportunity_parcels ───────────────────────────────────────────────── + + +def test_get_opportunity_parcels_finds_auction() -> None: + """3 features (1 auction + 2 other types) — все возвращаются, auction в нужном layer.""" + rows: list[tuple[Any, ...]] = [ + # (layer, props, geom_wkt, distance_m) + ( + "opportunity_auction_parcels", + {"cad_num": "66:41:0204016:101"}, + "POLYGON((0 0,1 0,1 1,0 0))", + 50.0, + ), + ( + "opportunity_scheme_parcels", + {"cadastral_number": "66:41:0204016:102"}, + "POLYGON((1 1,2 1,2 2,1 1))", + 150.0, + ), + ( + "opportunity_oopt", + {}, + "POLYGON((2 2,3 2,3 3,2 2))", + 300.0, + ), + ] + db = _make_mock_db_with_rows(rows) + + result = _get_opportunity_parcels(db, "66:41:0204016", "POLYGON((0 0,1 0,1 1,0 0))") + + assert len(result) == 3 + + auction = next(r for r in result if r["layer"] == "auction_parcels") + assert auction["cad_num"] == "66:41:0204016:101" + assert auction["distance_m"] == 50.0 + + scheme = next(r for r in result if r["layer"] == "scheme_parcels") + assert scheme["cad_num"] == "66:41:0204016:102" + + oopt = next(r for r in result if r["layer"] == "oopt") + assert oopt["cad_num"] is None + assert oopt["distance_m"] == 300.0 + + +def test_get_opportunity_parcels_distance_sort() -> None: + """Результаты отсортированы по distance_m ASC (SQL ORDER BY передаётся DB).""" + rows: list[tuple[Any, ...]] = [ + # Порядок: ближайший первым (SQL уже сортирует — тест проверяет что порядок сохраняется) + ( + "opportunity_free_parcels", + {"cad_num": "66:41:0204016:10"}, + "POINT(0 0)", + 10.0, + ), + ( + "opportunity_auction_parcels", + {"cad_num": "66:41:0204016:20"}, + "POINT(1 1)", + 200.0, + ), + ( + "opportunity_future_parcels", + {}, + "POINT(2 2)", + 450.0, + ), + ] + db = _make_mock_db_with_rows(rows) + + result = _get_opportunity_parcels(db, "66:41:0204016", "POLYGON((0 0,1 0,1 1,0 0))") + + assert len(result) == 3 + # Порядок из DB (mock возвращает в порядке rows) — ближайший первый + assert result[0]["distance_m"] == 10.0 + assert result[1]["distance_m"] == 200.0 + assert result[2]["distance_m"] == 450.0 + + +def test_get_opportunity_parcels_empty() -> None: + """Нет opportunity features — пустой список.""" + db = _make_mock_db_with_rows([]) + result = _get_opportunity_parcels(db, "66:41:0204016", "POLYGON((0 0,1 0,1 1,0 0))") + assert result == [] + + +def test_get_opportunity_parcels_early_exit() -> None: + """opportunity_count == 0 → early exit, db.execute не вызывается.""" + db = MagicMock() + layer_counts = {"opportunity_count": 0} + result = _get_opportunity_parcels( + db, "66:41:0204016", "POLYGON((0 0,1 0,1 1,0 0))", layer_counts + ) + assert result == [] + db.execute.assert_not_called() + + +def test_get_opportunity_parcels_db_exception_returns_empty() -> None: + """DB exception → пустой список (не propagate).""" + db = MagicMock() + db.execute.side_effect = Exception("connection lost") + result = _get_opportunity_parcels(db, "66:41:0204016", "POLYGON((0 0,1 0,1 1,0 0))") + assert result == [] + + +# ── _get_red_lines ──────────────────────────────────────────────────────────── + + +def test_get_red_lines_intersects() -> None: + """Red line intersecting parcel → intersection_length_m filled, distance_m=None.""" + rows: list[tuple[Any, ...]] = [ + # (geom_wkt, does_intersect, intersection_length_m, distance_m) + ( + "LINESTRING(0 0, 1 1)", + True, + 45.3, # длина пересечения в метрах + 0.0, # intersecting → distance_m becomes None in result + ), + ] + db = _make_mock_db_with_rows(rows) + + result = _get_red_lines(db, "66:41:0204016", "POLYGON((0 0,1 0,1 1,0 0))") + + assert len(result) == 1 + r = result[0] + assert r["geom_wkt"] == "LINESTRING(0 0, 1 1)" + assert r["intersection_length_m"] == 45.3 + assert r["distance_m"] is None # null when intersecting + + +def test_get_red_lines_nearby_only() -> None: + """Red line nearby only (не intersect) → distance_m filled, intersection_length_m=None.""" + rows: list[tuple[Any, ...]] = [ + # (geom_wkt, does_intersect, intersection_length_m, distance_m) + ( + "LINESTRING(10 10, 20 20)", + False, + 0.0, # нет пересечения + 85.5, + ), + ] + db = _make_mock_db_with_rows(rows) + + result = _get_red_lines(db, "66:41:0204016", "POLYGON((0 0,1 0,1 1,0 0))") + + assert len(result) == 1 + r = result[0] + assert r["intersection_length_m"] is None + assert r["distance_m"] == 85.5 + + +def test_get_red_lines_db_exception_returns_empty() -> None: + """DB exception → пустой список (не propagate).""" + db = MagicMock() + db.execute.side_effect = Exception("DB connection lost") + result = _get_red_lines(db, "66:41:0204016", "POLYGON((0 0,1 0,1 1,0 0))") + assert result == [] + + +def test_get_red_lines_empty() -> None: + """Нет red lines features — пустой список.""" + db = _make_mock_db_with_rows([]) + result = _get_red_lines(db, "66:41:0204016", "POLYGON((0 0,1 0,1 1,0 0))") + assert result == [] + + +def test_get_red_lines_early_exit() -> None: + """red_lines_count == 0 → early exit, db.execute не вызывается.""" + db = MagicMock() + layer_counts = {"red_lines_count": 0} + result = _get_red_lines(db, "66:41:0204016", "POLYGON((0 0,1 0,1 1,0 0))", layer_counts) + assert result == [] + db.execute.assert_not_called() + + +# ── EMPTY_DUMP_RESULT / make_empty_result includes new TIER 4 keys ──────────── + + +def test_empty_dump_result_has_opportunity_key() -> None: + """EMPTY_DUMP_RESULT содержит nspd_opportunity_parcels: [].""" + assert "nspd_opportunity_parcels" in EMPTY_DUMP_RESULT + assert EMPTY_DUMP_RESULT["nspd_opportunity_parcels"] == [] + + +def test_empty_dump_result_has_red_lines_key() -> None: + """EMPTY_DUMP_RESULT содержит nspd_red_lines: [].""" + assert "nspd_red_lines" in EMPTY_DUMP_RESULT + assert EMPTY_DUMP_RESULT["nspd_red_lines"] == [] + + +def test_make_empty_result_has_tier4_keys() -> None: + """make_empty_result() возвращает nspd_opportunity_parcels и nspd_red_lines.""" + result = make_empty_result() + assert "nspd_opportunity_parcels" in result + assert result["nspd_opportunity_parcels"] == [] + assert "nspd_red_lines" in result + assert result["nspd_red_lines"] == [] + + +# ── Issue #234: harvest_eta_seconds + SETNX dedupe ──────────────────────────── + + +def test_empty_dump_result_has_harvest_eta_seconds_key() -> None: + """EMPTY_DUMP_RESULT.nspd_dump содержит harvest_eta_seconds (issue #234).""" + assert "harvest_eta_seconds" in EMPTY_DUMP_RESULT["nspd_dump"] + assert EMPTY_DUMP_RESULT["nspd_dump"]["harvest_eta_seconds"] is None + + +def test_make_empty_result_passes_harvest_eta_seconds() -> None: + """make_empty_result(harvest_eta_seconds=60) → пробрасывает в nspd_dump.""" + result = make_empty_result(harvest_triggered=True, harvest_eta_seconds=60) + assert result["nspd_dump"]["harvest_eta_seconds"] == 60 + assert result["nspd_dump"]["harvest_triggered"] is True + + +def test_make_empty_result_default_eta_is_none() -> None: + """Если harvest_eta_seconds не передан — поле None (для available=False случая).""" + result = make_empty_result() + assert result["nspd_dump"]["harvest_eta_seconds"] is None + + +def test_acquire_harvest_lock_succeeds_first_call(monkeypatch: pytest.MonkeyPatch) -> None: + """SETNX lock: первый вызов получает lock (True), второй — нет (False). + + Issue #234: burst N concurrent analyze на один свежетриггеренный квартал + дедуплицируется через Redis SETNX. Мокаем redis.Redis.from_url. + """ + from app.services.site_finder import quarter_dump_lookup as qdl + + # Mock redis: первый client.set возвращает True, второй — None (key existed). + calls = {"n": 0} + + class _MockRedis: + def set(self, key: str, value: str, *, nx: bool, ex: int) -> bool | None: + calls["n"] += 1 + return True if calls["n"] == 1 else None + + monkeypatch.setattr( + "redis.Redis.from_url", + lambda _url: _MockRedis(), + ) + + assert qdl._acquire_harvest_lock("66:41:0204016") is True + assert qdl._acquire_harvest_lock("66:41:0204016") is False + assert calls["n"] == 2 + + +def test_acquire_harvest_lock_redis_unavailable_returns_false( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Redis недоступен → _acquire_harvest_lock возвращает False (graceful). + + Лучше не запустить дубль, чем нагрузить WAF при transient redis-fail. + """ + from app.services.site_finder import quarter_dump_lookup as qdl + + def _boom(_url: str) -> None: + raise ConnectionError("redis down") + + monkeypatch.setattr("redis.Redis.from_url", _boom) + assert qdl._acquire_harvest_lock("66:41:0204016") is False + + +# ── derive_quarter_cad (smoke tests) ────────────────────────────────────────── + + +@pytest.mark.parametrize( + "cad,expected", + [ + ("66:41:0204016", "66:41:0204016"), + ("66:41:0204016:10", "66:41:0204016"), + ("66:41:0204016:10:1", "66:41:0204016"), + ("invalid", None), + ("66:41", None), + ], +) +def test_derive_quarter_cad(cad: str, expected: str | None) -> None: + assert derive_quarter_cad(cad) == expected + + +# ── #243: cad_zouit fallback when dump absent ───────────────────────────────── + + +def _make_mock_db_no_dump_with_cad_zouit( + cad_zouit_rows: list[tuple[Any, ...]], +) -> MagicMock: + """Мок Session: nspd_quarter_dumps.first() → None; cad_zouit.fetchall() → rows.""" + db = MagicMock() + first_result = MagicMock() + first_result.first.return_value = None # нет дампа + + fetchall_result = MagicMock() + fetchall_result.fetchall.return_value = cad_zouit_rows + + # Первый вызов execute (SELECT nspd_quarter_dumps) → first()=None + # Второй вызов execute (SELECT cad_zouit) → fetchall()=rows + db.execute.side_effect = [first_result, fetchall_result] + return db + + +def test_cad_zouit_fallback_fires_when_no_dump() -> None: + """#243: cad_zouit fallback срабатывает когда nspd_quarter_dumps пуст глобально. + + Ожидаем: nspd_zouit_overlaps непустой, source='cad_zouit'. + """ + cad_zouit_rows: list[tuple[Any, ...]] = [ + # (type_zone, category_name, name_by_doc, reg_numb_border, id) + ("Охранная зона трубопровода", "Трубопровод", "ГВС ул. Ленина", "66-66-00/123", 42), + ] + db = _make_mock_db_no_dump_with_cad_zouit(cad_zouit_rows) + + wkt = "POLYGON((60 56,61 56,61 57,60 57,60 56))" + result = get_quarter_dump_data(db, "66:41:0603016:194", wkt) + + overlaps: list[dict[str, Any]] = result["nspd_zouit_overlaps"] + assert len(overlaps) == 1 + assert overlaps[0]["source"] == "cad_zouit" + assert overlaps[0]["type_zone"] == "Охранная зона трубопровода" + assert overlaps[0]["name"] == "ГВС ул. Ленина" + # dump.available остаётся False (нет дампа) + assert result["nspd_dump"]["available"] is False + + +def test_cad_zouit_fallback_no_dump_no_parcel_wkt() -> None: + """#243: dump=None + parcel_wkt=None → overlaps пустой, не вызываем cad_zouit.""" + db = MagicMock() + first_result = MagicMock() + first_result.first.return_value = None + db.execute.return_value = first_result + + result = get_quarter_dump_data(db, "66:41:0603016:194", None) + + assert result["nspd_zouit_overlaps"] == [] + # cad_zouit SELECT не должен вызываться (только один execute для dump lookup) + assert db.execute.call_count == 1 + + +def test_cad_zouit_fallback_empty_when_no_overlaps() -> None: + """#243: dump=None + cad_zouit возвращает 0 строк → nspd_zouit_overlaps пуст.""" + db = _make_mock_db_no_dump_with_cad_zouit([]) + + wkt = "POLYGON((60 56,61 56,61 57,60 57,60 56))" + result = get_quarter_dump_data(db, "66:41:0603016:194", wkt) + + assert result["nspd_zouit_overlaps"] == [] + + +def test_get_cad_zouit_overlaps_returns_correct_schema() -> None: + """_get_cad_zouit_overlaps: поля group_key/layer/source/type_zone корректны.""" + rows: list[tuple[Any, ...]] = [ + # (type_zone, category_name, name_by_doc, reg_numb_border, id) + ("Охранная зона ЛЭП 110кВ", "Электроснабжение", "ВЛ-110", "66-123", 7), + ("СЗЗ промышленного объекта", "Санитарная", "Завод", "66-456", 8), + ] + db = _make_mock_db_with_rows(rows) + + result = _get_cad_zouit_overlaps(db, "POLYGON((0 0,1 0,1 1,0 0))") + + assert len(result) == 2 + lep = result[0] + assert lep["group_key"] == "cad_zouit" + assert lep["source"] == "cad_zouit" + assert lep["type_zone"] == "Охранная зона ЛЭП 110кВ" + assert lep["subcategory"] is None # всегда NULL в cad_zouit + assert lep["name"] == "ВЛ-110" + assert lep["raw_props"]["zouit_id"] == 7 + + +def test_nspd_path_used_when_zouit_count_nonzero() -> None: + """#243 backward compat: когда dump существует с zouit_count>0 — NSPD path, не cad_zouit. + + Мок: row.zouit_count=3, fetchall возвращает NSPD features. + Проверяем source='nspd-quarter-dump' (не cad_zouit). + """ + # Строка дампа: все поля через MagicMock + from datetime import UTC, datetime + + dump_row = MagicMock() + dump_row.__getitem__ = lambda self, i: [ + "66:41:0603016", # quarter_cad + datetime(2026, 1, 1, tzinfo=UTC), # fetched_at_utc (свежий) + 100, # total_features + None, # harvest_error + 1, # territorial_zones_count + 3, # zouit_count > 0 → NSPD path + 0, # engineering_count + 0, # risks_count + 0, # opportunity_count + 0, # red_lines_count + ][i] + + nspd_zouit_rows: list[tuple[Any, ...]] = [ + # (layer, props) + ("zouit_engineering", {"name": "ЛЭП", "subcategory": 17}), + ] + + db = MagicMock() + first_result = MagicMock() + first_result.first.return_value = dump_row + + # Последующие вызовы execute (spatial queries) — возвращаем пустые кроме zouit + zouit_result = MagicMock() + zouit_result.fetchall.return_value = nspd_zouit_rows + + empty_result = MagicMock() + empty_result.fetchall.return_value = [] + empty_result.first.return_value = None + + # [0]=dumps lookup, [1]=zoning(first), [2]=zouit(fetchall), [3+]=others + db.execute.side_effect = [ + first_result, + empty_result, # _get_zoning first() + zouit_result, # _get_zouit_overlaps fetchall() + empty_result, # _get_engineering_nearby fetchall() + empty_result, # _get_risk_zones fetchall() + empty_result, # _get_opportunity_parcels fetchall() + empty_result, # _get_red_lines fetchall() + ] + + result = get_quarter_dump_data( + db, "66:41:0603016:194", "POLYGON((60 56,61 56,61 57,60 57,60 56))" + ) + + overlaps: list[dict[str, Any]] = result["nspd_zouit_overlaps"] + assert len(overlaps) == 1 + assert overlaps[0]["source"] == "nspd-quarter-dump" + # dump.available=True т.к. свежий дамп + assert result["nspd_dump"]["available"] is True diff --git a/backend/tests/test_admin_weight_profiles.py b/backend/tests/test_admin_weight_profiles.py index 01caa3ed..c3bad23a 100644 --- a/backend/tests/test_admin_weight_profiles.py +++ b/backend/tests/test_admin_weight_profiles.py @@ -314,6 +314,93 @@ def test_delete_not_found(client_with_token: TestClient, monkeypatch: pytest.Mon _clear_overrides() +# ── include_system preset seed (Issue #114 / PR #229) ───────────────────────── + + +def test_list_include_system_calls_with_system( + client_with_token: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + """GET ?include_system=true вызывает list_profiles_with_system, возвращает presets.""" + system_profile = _make_profile( + 100, user_id="__system__", profile_name="Комфорт", weights={"park": 1.8, "school": 1.5} + ) + user_profile = _make_profile(1, user_id="user-1", profile_name="Мой профиль") + mock = MagicMock() + _override_db(mock) + try: + monkeypatch.setattr( + "app.api.v1.admin_weight_profiles.list_profiles_with_system", + lambda db, user_id: [user_profile, system_profile], + ) + r = client_with_token.get( + "/api/v1/admin/site-finder/weight-profiles", + params={"user_id": "user-1", "include_system": "true"}, + headers=_HEADERS, + ) + assert r.status_code == 200 + body = r.json() + assert len(body) == 2 + profile_names = {p["profile_name"] for p in body} + assert "Комфорт" in profile_names + assert "Мой профиль" in profile_names + finally: + _clear_overrides() + + +def test_list_without_include_system_does_not_call_with_system( + client_with_token: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + """GET без include_system → list_profiles (только пользовательские профили).""" + user_profile = _make_profile(1, user_id="user-1") + called_with_system = [] + mock = MagicMock() + _override_db(mock) + try: + monkeypatch.setattr( + "app.api.v1.admin_weight_profiles.list_profiles", + lambda db, user_id: [user_profile], + ) + monkeypatch.setattr( + "app.api.v1.admin_weight_profiles.list_profiles_with_system", + lambda db, user_id: called_with_system.append(True) or [], + ) + r = client_with_token.get( + "/api/v1/admin/site-finder/weight-profiles", + params={"user_id": "user-1"}, + headers=_HEADERS, + ) + assert r.status_code == 200 + assert len(r.json()) == 1 + # list_profiles_with_system НЕ должен вызываться без include_system=true + assert called_with_system == [], "вызван list_profiles_with_system без флага" + finally: + _clear_overrides() + + +def test_list_profiles_with_system_service(monkeypatch: pytest.MonkeyPatch) -> None: + """list_profiles_with_system передаёт system_user_id='__system__' в запрос.""" + from app.services.site_finder.weight_profiles import SYSTEM_USER_ID, list_profiles_with_system + + captured_params: list[dict] = [] + + class FakeResult: + def mappings(self) -> FakeResult: + return self + + def all(self) -> list: + return [] + + class FakeDb: + def execute(self, query: object, params: dict) -> FakeResult: + captured_params.append(params) + return FakeResult() + + list_profiles_with_system(FakeDb(), user_id="user-test") + assert len(captured_params) == 1 + assert captured_params[0]["system_user_id"] == SYSTEM_USER_ID + assert captured_params[0]["user_id"] == "user-test" + + # ── Auth ─────────────────────────────────────────────────────────────────────── diff --git a/backend/tests/test_gate_verdict.py b/backend/tests/test_gate_verdict.py index d5eb04e9..903f0f7a 100644 --- a/backend/tests/test_gate_verdict.py +++ b/backend/tests/test_gate_verdict.py @@ -1,6 +1,9 @@ """Tests for gate_verdict aggregator — pure function, no DB.""" -from app.services.site_finder.gate_verdict import compute_gate_verdict, is_residential_zone +from app.services.site_finder.gate_verdict import ( + compute_gate_verdict, + is_residential_zone, +) # ── is_residential_zone unit tests ──────────────────────────────────────────── @@ -115,3 +118,90 @@ def test_pzz_unknown_warning_when_no_zoning(): verdict = compute_gate_verdict(None, [], [{"name": "ТП-1"}], {"available": True}) assert any(w["code"] == "PZZ_UNKNOWN" for w in verdict["warnings"]) assert verdict["can_build_mkd"] is True + + +# ── cad_zouit fallback path (#232) ──────────────────────────────────────────── + + +def test_cad_zouit_ohranaya_zona_truboprovodov_blocks(): + """cad_zouit overlap с 'охранная зона трубопроводов' → ZOUIT_CAD_BLOCKER → Нельзя.""" + nspd_zoning = {"zone_code": "Ж-2", "zone_name": "Жилая"} + overlaps = [ + { + "source": "cad_zouit", + "type_zone": "Охранная зона трубопроводов", + "layer": "Охранная зона трубопроводов", + "name": "Газопровод высокого давления", + } + ] + verdict = compute_gate_verdict(nspd_zoning, overlaps, [], {"available": True}) + assert verdict["can_build_mkd"] is False + assert verdict["verdict_label"] == "Нельзя" + assert any(b["code"] == "ZOUIT_CAD_BLOCKER" for b in verdict["blockers"]) + + +def test_cad_zouit_elektr_blocks(): + """cad_zouit overlap с 'электр' substring в type_zone → blocker.""" + nspd_zoning = {"zone_code": "Ж-3", "zone_name": "Жилая"} + overlaps = [ + { + "source": "cad_zouit", + "type_zone": "Охранная зона электросетевого объекта", + "layer": "Охранная зона электросетевого объекта", + "name": "ЛЭП 110 кВ", + } + ] + verdict = compute_gate_verdict(nspd_zoning, overlaps, [], {"available": True}) + assert verdict["can_build_mkd"] is False + assert any(b["code"] == "ZOUIT_CAD_BLOCKER" for b in verdict["blockers"]) + + +def test_cad_zouit_szz_is_warning_not_blocker(): + """cad_zouit overlap с 'СЗЗ' → ZOUIT_CAD_SZZ warning, не blocker.""" + nspd_zoning = {"zone_code": "Ж-2", "zone_name": "Жилая"} + overlaps = [ + { + "source": "cad_zouit", + "type_zone": "Санитарно-защитная зона предприятия", + "layer": "Санитарно-защитная зона предприятия", + "name": "СЗЗ завода", + } + ] + verdict = compute_gate_verdict(nspd_zoning, overlaps, [{"name": "ТП-1"}], {"available": True}) + assert verdict["can_build_mkd"] is True + assert verdict["verdict_label"] == "С ограничениями" + assert any(w["code"] == "ZOUIT_CAD_SZZ" for w in verdict["warnings"]) + assert verdict["blockers"] == [] + + +def test_cad_zouit_other_is_warning(): + """cad_zouit overlap без blocker/СЗЗ keyword → ZOUIT_CAD_OTHER warning.""" + nspd_zoning = {"zone_code": "Ж-2", "zone_name": "Жилая"} + overlaps = [ + { + "source": "cad_zouit", + "type_zone": "Зона публичного сервитута", + "layer": "Зона публичного сервитута", + "name": None, + } + ] + verdict = compute_gate_verdict(nspd_zoning, overlaps, [{"name": "ТП-1"}], {"available": True}) + assert verdict["can_build_mkd"] is True + assert any(w["code"] == "ZOUIT_CAD_OTHER" for w in verdict["warnings"]) + assert verdict["blockers"] == [] + + +def test_nspd_subcategory_path_backward_compat(): + """NSPD dump path (source != 'cad_zouit') продолжает работать через subcategory.""" + nspd_zoning = {"zone_code": "Ж-3", "zone_name": "Жилая"} + overlaps = [ + { + "source": "nspd-quarter-dump", + "subcategory": 17, + "name": "ЛЭП 110кВ", + "layer": "37578", + } + ] + verdict = compute_gate_verdict(nspd_zoning, overlaps, [], {"available": True}) + assert verdict["can_build_mkd"] is False + assert any(b["code"] == "ZOUIT_OVERLAP_SUB17" for b in verdict["blockers"]) diff --git a/backend/tests/test_layout_signature.py b/backend/tests/test_layout_signature.py new file mode 100644 index 00000000..1fcad752 --- /dev/null +++ b/backend/tests/test_layout_signature.py @@ -0,0 +1,97 @@ +"""Тесты для layout_signature helpers (Issue #113, Phase 2.1). + +Pure-Python, без БД и external calls. +""" + +from __future__ import annotations + +import pytest + +from app.services.site_finder.layout_signature import ( + area_bin, + layout_signature, + room_bucket_from_flat, +) + + +@pytest.mark.parametrize( + "rooms, flat_type, is_studio, expected", + [ + (None, None, True, "studio"), + (0, "Квартира-студия", False, "studio"), + (0, "Квартира", None, "studio"), + (1, "Квартира", None, "1"), + (2, "Квартира", None, "2"), + (3, "Квартира", False, "3"), + (4, "Квартира", False, "4+"), + (5, "Квартира", None, "4+"), + (None, "Квартира", None, "1"), # fallback + ], +) +def test_room_bucket( + rooms: int | None, flat_type: str | None, is_studio: bool | None, expected: str +) -> None: + assert room_bucket_from_flat(rooms, flat_type, is_studio) == expected + + +@pytest.mark.parametrize( + "area, expected", + [ + (0.5, "<25"), + (24.9, "<25"), + (25.0, "25-40"), + (39.99, "25-40"), + (40.0, "40-60"), + (59.99, "40-60"), + (60.0, "60-80"), + (79.99, "60-80"), + (80.0, "80-100"), + (99.99, "80-100"), + (100.0, "100+"), + (250.0, "100+"), + ], +) +def test_area_bin(area: float, expected: str) -> None: + assert area_bin(area) == expected + + +def test_layout_signature_deterministic() -> None: + assert layout_signature("studio", "<25") == "studio__<25" + assert layout_signature("4+", "100+") == "4+__100+" + # Same input → same output + sig1 = layout_signature("2", "40-60") + sig2 = layout_signature("2", "40-60") + assert sig1 == sig2 + + +def test_room_bucket_is_studio_overrides_rooms() -> None: + """is_studio=True beats any rooms value.""" + assert room_bucket_from_flat(3, "Квартира", True) == "studio" + + +def test_room_bucket_flat_type_studio_overrides_rooms() -> None: + """flat_type='Квартира-студия' beats rooms=2.""" + assert room_bucket_from_flat(2, "Квартира-студия", None) == "studio" + + +def test_room_bucket_large_rooms() -> None: + """rooms=10 → '4+'.""" + assert room_bucket_from_flat(10, "Квартира", False) == "4+" + + +def test_area_bin_boundary_exact() -> None: + """Граничные значения попадают в правильный бакет (включение левой границы).""" + assert area_bin(25.0) == "25-40" + assert area_bin(40.0) == "40-60" + assert area_bin(60.0) == "60-80" + assert area_bin(80.0) == "80-100" + assert area_bin(100.0) == "100+" + + +def test_layout_signature_format() -> None: + """Сигнатура всегда содержит двойное подчёркивание-разделитель.""" + sig = layout_signature("1", "25-40") + assert "__" in sig + parts = sig.split("__") + assert parts[0] == "1" + assert parts[1] == "25-40" diff --git a/backend/tests/test_layout_tz_pdf.py b/backend/tests/test_layout_tz_pdf.py new file mode 100644 index 00000000..1fbe2dd2 --- /dev/null +++ b/backend/tests/test_layout_tz_pdf.py @@ -0,0 +1,136 @@ +"""Tests для layout_tz_pdf renderer (Issue #113 PR D). + +WeasyPrint requires native GTK/Pango/GObject shared libraries. These are present +in the Docker container (Linux) but absent on Windows dev machines. All tests in +this module are skipped automatically when the native libs are unavailable. +""" + +import datetime as dt + +import pytest + +# Attempt to import the module under test; skip entire module if native libs missing. +try: + from app.services.exporters.layout_tz_pdf import render_layout_tz_pdf +except (OSError, ImportError) as _e: # GTK libs missing on Windows, or weasyprint not installed + pytest.skip(f"WeasyPrint deps missing: {_e}", allow_module_level=True) + +from app.schemas.parcel import ( + BestLayoutsResponse, + LayoutDataQuality, + LayoutTzMixRow, + LayoutTzRecommendation, + TopLayoutRow, +) + + +def _sample_response() -> BestLayoutsResponse: + return BestLayoutsResponse( + top_layouts=[ + TopLayoutRow( + rank=1, + room_bucket="1", + area_bin="25-40", + signature="1__25-40", + competitor_obj_ids=[1234, 5678], + competitor_count=2, + total_sold_in_window=67, + velocity_per_month=8.4, + avg_price_per_m2_rub=148000.0, + avg_area_m2=38.5, + supply_units_in_radius=312, + sold_pct_of_supply=21.5, + ), + TopLayoutRow( + rank=2, + room_bucket="studio", + area_bin="<25", + signature="studio__<25", + competitor_obj_ids=[1234], + competitor_count=1, + total_sold_in_window=40, + velocity_per_month=5.0, + avg_price_per_m2_rub=160000.0, + avg_area_m2=22.0, + supply_units_in_radius=100, + sold_pct_of_supply=40.0, + ), + ], + recommendation_for_tz=LayoutTzRecommendation( + rationale_text="Test rationale текст с кириллицей", + mix=[ + LayoutTzMixRow(room_bucket="studio", pct=10, abs_units=30, avg_target_area_m2=22.0), + LayoutTzMixRow(room_bucket="1", pct=60, abs_units=180, avg_target_area_m2=38.5), + LayoutTzMixRow(room_bucket="2", pct=30, abs_units=90, avg_target_area_m2=55.0), + ], + weighted_avg_price_per_m2_rub=152000.0, + based_on_obj_count=5, + based_on_total_deals=107, + data_window_start=dt.date(2026, 2, 1), + data_window_end=dt.date(2026, 5, 1), + ), + data_quality=LayoutDataQuality( + objects_with_velocity_data=5, + objects_total_in_radius=8, + velocity_coverage_pct=62.5, + confidence="medium", + ), + ) + + +def test_pdf_renders_non_empty_bytes() -> None: + pdf = render_layout_tz_pdf( + _sample_response(), + cad_num="66:41:0204016:10", + radius_km=1.0, + time_window="last_quarter", + ) + assert len(pdf) > 1000 # PDF минимум ~1KB + + +def test_pdf_starts_with_pdf_magic() -> None: + pdf = render_layout_tz_pdf( + _sample_response(), + cad_num="66:41:0204016:10", + radius_km=1.0, + time_window="last_quarter", + ) + assert pdf[:4] == b"%PDF" + + +def test_pdf_renders_cyrillic_correctly() -> None: + """Smoke — WeasyPrint должен handle кириллический rationale_text без UnicodeEncodeError.""" + response = _sample_response() + pdf = render_layout_tz_pdf( + response, + cad_num="66:41:0303161:42", + radius_km=1.5, + time_window="last_year", + ) + # Embedded text может быть compressed, но без exception = OK + assert len(pdf) > 1000 + + +def test_pdf_handles_empty_top_layouts() -> None: + response = _sample_response() + response.top_layouts = [] + pdf = render_layout_tz_pdf( + response, + cad_num="66:41:0204016:10", + radius_km=1.0, + time_window="last_quarter", + ) + assert pdf[:4] == b"%PDF" + + +def test_pdf_handles_null_avg_price() -> None: + """avg_price_per_m2_rub=None (ЖК не покрыт Objective) → должно рендериться как '—'.""" + response = _sample_response() + response.top_layouts[0].avg_price_per_m2_rub = None + pdf = render_layout_tz_pdf( + response, + cad_num="66:41:0204016:10", + radius_km=1.0, + time_window="last_quarter", + ) + assert pdf[:4] == b"%PDF" diff --git a/backend/tests/test_nspd_client.py b/backend/tests/test_nspd_client.py index b7b7479b..9fa82d81 100644 --- a/backend/tests/test_nspd_client.py +++ b/backend/tests/test_nspd_client.py @@ -507,6 +507,40 @@ def _make_fake_http( return fake_http +def _make_fake_grid_walk( + layer_feature_counts: dict[str, int] | None = None, +) -> Any: + """Возвращает fake `NSPDClient.get_features_in_bbox_grid`. + + После Sub-PR B (#260) area/linear layers (territorial_zones, red_lines, + engineering_structures, zouit_*, risk_*) идут через grid-walk вместо + legacy `_http_get_json` → тесты обязаны мокать оба пути, иначе грид-walk + бьёт по живому NSPD API. + """ + counts = layer_feature_counts or {} + from app.services.scrapers.nspd_client import LAYERS as _LAYERS + from app.services.scrapers.nspd_client import NSPDFeature + + id_to_name: dict[int, str] = {v: k for k, v in _LAYERS.items()} + + def fake_grid( + self: Any, + layer_id: int, + bbox: tuple[float, float, float, float], + *, + grid_n: int = 7, + step_m: float = 50.0, + ) -> list[NSPDFeature]: + layer_name = id_to_name.get(layer_id, "unknown") + n = counts.get(layer_name, 1) + return [ + NSPDFeature.from_raw({"id": f"{layer_name}-{i}", "geometry": None, "properties": {}}) + for i in range(n) + ] + + return fake_grid + + def test_search_by_quarter_core_only(monkeypatch: pytest.MonkeyPatch) -> None: """core_only (include_zouit=False, include_risks=False): 1 search + 5 bulk.""" search_calls: list[str] = [] @@ -520,6 +554,11 @@ def test_search_by_quarter_core_only(monkeypatch: pytest.MonkeyPatch) -> None: "app.services.scrapers.nspd_client._http_get_json", _make_fake_http(_LAYER_FEATURE_COUNTS), ) + # Sub-PR B (#260): area layers идут через grid-walk — мокаем оба пути. + monkeypatch.setattr( + "app.services.scrapers.nspd_client.NSPDClient.get_features_in_bbox_grid", + _make_fake_grid_walk(_LAYER_FEATURE_COUNTS), + ) result = NSPDClient().search_by_quarter( "66:41:0204016", include_zouit=False, include_risks=False @@ -566,6 +605,11 @@ def test_search_by_quarter_with_zouit(monkeypatch: pytest.MonkeyPatch) -> None: "app.services.scrapers.nspd_client._http_get_json", _make_fake_http(), ) + # Sub-PR B (#260): zouit_* и area layers идут через grid-walk. + monkeypatch.setattr( + "app.services.scrapers.nspd_client.NSPDClient.get_features_in_bbox_grid", + _make_fake_grid_walk(), + ) result = NSPDClient().search_by_quarter( "66:41:0204016", include_zouit=True, include_risks=False @@ -628,6 +672,11 @@ def test_search_by_quarter_layers_fetched_with_risks(monkeypatch: pytest.MonkeyP "app.services.scrapers.nspd_client._http_get_json", _make_fake_http(), ) + # Sub-PR B (#260): risk_* и area layers идут через grid-walk. + monkeypatch.setattr( + "app.services.scrapers.nspd_client.NSPDClient.get_features_in_bbox_grid", + _make_fake_grid_walk(), + ) result = NSPDClient().search_by_quarter( "66:41:0204016", include_zouit=False, include_risks=True diff --git a/backend/tests/test_nspd_sync.py b/backend/tests/test_nspd_sync.py index 8b077cf6..c30b4496 100644 --- a/backend/tests/test_nspd_sync.py +++ b/backend/tests/test_nspd_sync.py @@ -25,6 +25,7 @@ from app.workers.tasks.nspd_sync import ( _build_features_json, _build_risks_count, _build_zouit_count, + _upsert_dump, harvest_quarter, harvest_stale_quarters, ) @@ -71,6 +72,7 @@ def _make_dump( engineering_structures=[_make_feat("e1")], zouit=zouit, risks=risks, + opportunity={}, layers_fetched=( "search", "parcels", @@ -204,6 +206,7 @@ def test_harvest_quarter_empty_quarter( engineering_structures=[], zouit={"okn": [], "engineering": [], "natural": [], "protected": [], "other": []}, risks={}, + opportunity={}, layers_fetched=("search",), bbox_3857=None, fetched_at_utc="2026-05-12T03:00:00+00:00", @@ -365,6 +368,7 @@ def test_features_json_geometry_preserved() -> None: engineering_structures=[], zouit={}, risks={}, + opportunity={}, layers_fetched=("search", "parcels"), bbox_3857=(100.0, 200.0, 300.0, 400.0), fetched_at_utc="2026-05-12T00:00:00+00:00", @@ -375,3 +379,70 @@ def test_features_json_geometry_preserved() -> None: assert result[0]["layer"] == "parcels" assert result[0]["feature_id"] == "p1" assert result[0]["properties"] == {"k": "v"} + + +# ── _upsert_dump CAST regression tests (#244) ──────────────────────────────── + + +@patch("app.workers.tasks.nspd_sync.SessionLocal") +def test_upsert_dump_null_geom_executes_without_error(mock_session_cls: MagicMock) -> None: + """Regression #244: _upsert_dump с geom_json=None не падает на AmbiguousParameter. + + psycopg3 не может вывести тип голого параметра $N внутри CASE WHEN ... IS NULL. + Fix: CAST(:geom_json AS text) IS NULL даёт явный тип — нет AmbiguousParameter. + """ + mock_db = MagicMock() + mock_session_cls.return_value = mock_db + + # dump=None → error-only path: все geo-параметры None, должен execute без ошибок + _upsert_dump( + quarter_cad="66:41:0204016", + region_code=66, + dump=None, + features_json=None, + duration_ms=100, + harvest_error="TestError: simulated", + ) + + mock_db.execute.assert_called_once() + mock_db.commit.assert_called_once() + + +@patch("app.workers.tasks.nspd_sync.SessionLocal") +def test_upsert_dump_with_geom_executes_without_error(mock_session_cls: MagicMock) -> None: + """Regression #244: _upsert_dump с реальным geom_json и bbox не падает. + + CAST(:geom_json AS text) передаёт строку, CAST(:bbox_xmin AS double precision) — + число. Оба branch CASE WHEN типизированы корректно. + """ + mock_db = MagicMock() + mock_session_cls.return_value = mock_db + + geom = { + "type": "Polygon", + "coordinates": [ + [[100.0, 200.0], [300.0, 200.0], [300.0, 400.0], [100.0, 400.0], [100.0, 200.0]] + ], + } + dump = _make_dump( + quarter_cad="66:41:0204016", + quarter_feat=_make_feat_with_geom("q1", geom), + bbox=(100.0, 200.0, 300.0, 400.0), + ) + + _upsert_dump( + quarter_cad="66:41:0204016", + region_code=66, + dump=dump, + features_json=_build_features_json(dump), + duration_ms=500, + harvest_error=None, + ) + + mock_db.execute.assert_called_once() + # Verify params contain properly-typed values (not None for geo fields) + call_params = mock_db.execute.call_args[0][1] + assert call_params["geom_json"] is not None # JSON string + assert call_params["bbox_xmin"] == 100.0 + assert call_params["bbox_ymax"] == 400.0 + mock_db.commit.assert_called_once() diff --git a/backend/tests/test_poi_score.py b/backend/tests/test_poi_score.py new file mode 100644 index 00000000..e633894c --- /dev/null +++ b/backend/tests/test_poi_score.py @@ -0,0 +1,168 @@ +"""Tests for POI weighted score service (B6). + +Юнит-тесты для чистой функции — без DB. +""" + +from app.services.site_finder.poi_score import ( + CATEGORY_WEIGHTS, + PoiScoreResponse, + _category_weight, + compute_poi_weighted_top7, +) + +# ── unit: _category_weight ───────────────────────────────────────────────────── + + +def test_category_weight_metro(): + """Метро имеет наибольший вес из всех категорий.""" + metro_w = _category_weight("metro_stop") + for cat in CATEGORY_WEIGHTS: + if cat != "metro_stop" and cat != "default": + assert metro_w >= _category_weight( + cat + ), f"metro_stop weight {metro_w} должен быть >= {cat} weight {_category_weight(cat)}" + + +def test_category_weight_unknown_returns_default(): + w = _category_weight("unknown_category_xyz") + assert w == CATEGORY_WEIGHTS["default"] + + +def test_category_weight_all_positive(): + """Все веса в CATEGORY_WEIGHTS должны быть положительными (B6 — ranking, не штраф).""" + for cat, w in CATEGORY_WEIGHTS.items(): + assert w > 0, f"Вес {cat}={w} должен быть > 0" + + +# ── unit: weight formula ratio ───────────────────────────────────────────────── + + +def test_weight_formula_ratio(): + """Ближний объект той же категории должен иметь больший вес.""" + cat = "school" + cw = _category_weight(cat) + w_near = (1.0 / (100.0 + 100.0)) * cw # 100м + w_far = (1.0 / (1000.0 + 100.0)) * cw # 1000м + assert w_near > w_far + + +def test_weight_formula_category_dominates_at_equal_distance(): + """При одинаковом расстоянии метро должно быть впереди автобусной остановки.""" + dist = 500.0 + w_metro = (1.0 / (dist + 100.0)) * _category_weight("metro_stop") + w_bus = (1.0 / (dist + 100.0)) * _category_weight("bus_stop") + assert w_metro > w_bus + + +# ── unit: compute_poi_weighted_top7 with mock DB ─────────────────────────────── + + +class _MockMappings: + def __init__(self, data: list[dict]) -> None: + self._data = data + + def all(self) -> list[dict]: + return self._data # type: ignore[return-value] + + +class _MockResult: + def __init__(self, data: list[dict]) -> None: + self._data = data + + def mappings(self) -> "_MockMappings": + return _MockMappings(self._data) + + +class _MockDb: + """Минимальный мок SQLAlchemy Session для тестирования без БД.""" + + def __init__(self, rows: list[dict]) -> None: + self._rows = rows + + def execute(self, *_args: object, **_kwargs: object) -> _MockResult: + return _MockResult(self._rows) + + +def _make_row(name: str, category: str, distance_m: float) -> dict: + return { + "name": name, + "category": category, + "tags": {}, + "distance_m": distance_m, + } + + +def test_top7_returns_at_most_7(): + rows = [_make_row(f"POI {i}", "school", float(i * 50)) for i in range(1, 20)] + db = _MockDb(rows) + result = compute_poi_weighted_top7(db, "66:41:0204016:10", 56.838, 60.605) + assert isinstance(result, PoiScoreResponse) + assert len(result.top_poi) <= 7 + + +def test_top7_sorted_by_weight_desc(): + rows = [ + _make_row("Дальняя школа", "school", 1500.0), + _make_row("Метро", "metro_stop", 300.0), + _make_row("Близкая школа", "school", 100.0), + ] + db = _MockDb(rows) + result = compute_poi_weighted_top7(db, "66:41:0204016:10", 56.838, 60.605) + weights = [item.weight for item in result.top_poi] + assert weights == sorted(weights, reverse=True), "top_poi должны быть по weight DESC" + + +def test_metro_beats_school_at_equal_distance(): + """Метро в 300м должно быть на первом месте перед школой в 300м (равное расстояние).""" + rows = [ + _make_row("Школа №1", "school", 300.0), + _make_row("Метро Чкаловская", "metro_stop", 300.0), + ] + db = _MockDb(rows) + result = compute_poi_weighted_top7(db, "66:41:0204016:10", 56.838, 60.605) + assert ( + result.top_poi[0].category == "metro_stop" + ), "При равном расстоянии метро (category_weight=6.0) должно быть выше школы (5.0)" + + +def test_metro_first_when_close(): + """Метро в 50м должно быть на первом месте перед школой в 300м.""" + rows = [ + _make_row("Школа №1", "school", 300.0), + _make_row("Метро Чкаловская", "metro_stop", 50.0), + ] + db = _MockDb(rows) + result = compute_poi_weighted_top7(db, "66:41:0204016:10", 56.838, 60.605) + assert result.top_poi[0].category == "metro_stop", ( + "Метро (weight=6.0) в 50м должно быть впереди школы (weight=5.0) в 300м — " + f"metro_weight={(1/(50+100))*6:.5f} vs school_weight={(1/(300+100))*5:.5f}" + ) + + +def test_empty_db_returns_empty_top_poi(): + db = _MockDb([]) + result = compute_poi_weighted_top7(db, "66:41:0204016:10", 56.838, 60.605) + assert result.top_poi == [] + assert result.cad_num == "66:41:0204016:10" + assert result.radius_m == 2000 + + +def test_address_built_from_tags(): + rows = [ + { + "name": "Магазин", + "category": "shop_small", + "tags": {"addr:street": "ул. Ленина", "addr:housenumber": "10"}, + "distance_m": 200.0, + } + ] + db = _MockDb(rows) + result = compute_poi_weighted_top7(db, "test", 56.838, 60.605) + assert result.top_poi[0].address == "ул. Ленина, 10" + + +def test_address_none_when_no_tags(): + rows = [_make_row("Парк", "park", 400.0)] + db = _MockDb(rows) + result = compute_poi_weighted_top7(db, "test", 56.838, 60.605) + assert result.top_poi[0].address is None diff --git a/backend/tests/test_sentry_init.py b/backend/tests/test_sentry_init.py new file mode 100644 index 00000000..75538a31 --- /dev/null +++ b/backend/tests/test_sentry_init.py @@ -0,0 +1,158 @@ +"""Unit-тесты логики инициализации GlitchTip SDK. + +Проверяем что init-блок в main.py / celery_app.py вызывает sentry_sdk.init() +только при непустом GLITCHTIP_DSN, что release-fallback работает корректно, +и что scrub_sensitive_query redact-ит api keys из URL spans. +""" + +import os +from unittest.mock import patch + +import sentry_sdk + + +def test_sdk_imports_without_error() -> None: + """Все интеграции импортируются без ModuleNotFoundError.""" + from sentry_sdk.integrations.celery import CeleryIntegration + from sentry_sdk.integrations.fastapi import FastApiIntegration + from sentry_sdk.integrations.httpx import HttpxIntegration + from sentry_sdk.integrations.logging import LoggingIntegration + from sentry_sdk.integrations.sqlalchemy import SqlalchemyIntegration + from sentry_sdk.integrations.starlette import StarletteIntegration + + assert StarletteIntegration() + assert FastApiIntegration() + assert CeleryIntegration() + assert SqlalchemyIntegration() + assert HttpxIntegration() + assert LoggingIntegration() + + +def test_no_sdk_init_when_dsn_empty() -> None: + """Если GLITCHTIP_DSN пустой, sentry_sdk.init() не должен вызываться.""" + with patch("sentry_sdk.init") as mock_init: + glitchtip_dsn = None + if glitchtip_dsn: + sentry_sdk.init(dsn=glitchtip_dsn) + mock_init.assert_not_called() + + +def test_sdk_init_called_when_dsn_set() -> None: + """Если GLITCHTIP_DSN задан, sentry_sdk.init() вызывается с правильными параметрами.""" + dsn = "https://key@errors.gendsgn.ru/1" + with patch("sentry_sdk.init") as mock_init: + glitchtip_dsn = dsn + if glitchtip_dsn: + sentry_sdk.init( + dsn=glitchtip_dsn, + environment="test", + release="unknown", + traces_sample_rate=0.05, + profiles_sample_rate=0.0, + send_default_pii=False, + integrations=[], + ) + mock_init.assert_called_once() + call_kwargs = mock_init.call_args.kwargs + assert call_kwargs["dsn"] == dsn + assert call_kwargs["send_default_pii"] is False + assert call_kwargs["profiles_sample_rate"] == 0.0 + + +# ── release fallback ────────────────────────────────────────────────────────── + + +def test_release_uses_git_sha_when_set() -> None: + """GIT_SHA имеет приоритет над SENTRY_RELEASE.""" + env = {"GIT_SHA": "abc1234", "SENTRY_RELEASE": "v1.0.0"} + with patch.dict(os.environ, env, clear=False): + release = os.getenv("GIT_SHA") or os.getenv("SENTRY_RELEASE") or "unknown" + assert release == "abc1234" + + +def test_release_falls_back_to_sentry_release() -> None: + """Если GIT_SHA не задан, используется SENTRY_RELEASE.""" + env = {"SENTRY_RELEASE": "v1.2.3"} + with patch.dict(os.environ, env, clear=False): + # Убираем GIT_SHA если он есть + os.environ.pop("GIT_SHA", None) + release = os.getenv("GIT_SHA") or os.getenv("SENTRY_RELEASE") or "unknown" + assert release == "v1.2.3" + + +def test_release_falls_back_to_unknown() -> None: + """Если ни одна переменная не задана, release='unknown'.""" + with patch.dict(os.environ, {}, clear=False): + os.environ.pop("GIT_SHA", None) + os.environ.pop("SENTRY_RELEASE", None) + release = os.getenv("GIT_SHA") or os.getenv("SENTRY_RELEASE") or "unknown" + assert release == "unknown" + + +# ── sentry_scrub ────────────────────────────────────────────────────────────── + + +def test_scrub_redacts_apikey_in_span_url() -> None: + """scrub_sensitive_query заменяет apiKey= в span data['url'].""" + from app.observability.sentry_scrub import scrub_sensitive_query + + event: dict = { + "spans": [ + { + "data": { + "url": "https://api.objctv.ru/v2/Report?apiKey=supersecret&group=EKB", + "http.url": "https://api.objctv.ru/v2/Report?api_key=topsecret", + } + } + ] + } + result = scrub_sensitive_query(event, {}) + span_data = result["spans"][0]["data"] + assert "[REDACTED]" in span_data["url"] + assert "supersecret" not in span_data["url"] + assert "[REDACTED]" in span_data["http.url"] + assert "topsecret" not in span_data["http.url"] + + +def test_scrub_redacts_token_in_description() -> None: + """scrub_sensitive_query заменяет token= в span description.""" + from app.observability.sentry_scrub import scrub_sensitive_query + + event: dict = { + "spans": [{"description": "GET https://example.com?token=mysecrettoken&foo=bar"}] + } + result = scrub_sensitive_query(event, {}) + assert "[REDACTED]" in result["spans"][0]["description"] + assert "mysecrettoken" not in result["spans"][0]["description"] + + +def test_scrub_redacts_request_url() -> None: + """scrub_sensitive_query заменяет token в event['request']['url'].""" + from app.observability.sentry_scrub import scrub_sensitive_query + + event: dict = {"request": {"url": "https://example.com?access_token=abc123&other=val"}} + result = scrub_sensitive_query(event, {}) + assert "[REDACTED]" in result["request"]["url"] + assert "abc123" not in result["request"]["url"] + + +def test_scrub_passes_through_clean_event() -> None: + """scrub_sensitive_query не трогает URL без чувствительных параметров.""" + from app.observability.sentry_scrub import scrub_sensitive_query + + event: dict = { + "spans": [{"data": {"url": "https://example.com?foo=bar&page=1"}}], + "request": {"url": "https://example.com/api/health"}, + } + result = scrub_sensitive_query(event, {}) + assert result["spans"][0]["data"]["url"] == "https://example.com?foo=bar&page=1" + assert result["request"]["url"] == "https://example.com/api/health" + + +def test_scrub_handles_missing_spans() -> None: + """scrub_sensitive_query не падает если 'spans' отсутствует.""" + from app.observability.sentry_scrub import scrub_sensitive_query + + event: dict = {"request": {"url": "https://example.com"}} + result = scrub_sensitive_query(event, {}) + assert result["request"]["url"] == "https://example.com" diff --git a/backend/tests/test_velocity.py b/backend/tests/test_velocity.py index ed71d960..aa25fdbb 100644 --- a/backend/tests/test_velocity.py +++ b/backend/tests/test_velocity.py @@ -2,8 +2,9 @@ Mock-based — не требуют живой БД. Источник данных — objective_corpus_room_month (мигрировано с domrf_kn_sale_graph). -Mock shape совместим: sales query возвращает те же aliases (obj_id, total_sqm, -months_with_data, period_start, period_end) через GROUP BY domrf_obj_id. +Mock shape совместим: sales query возвращает aliases (obj_id, total_sqm, +months_with_data, period_start, period_end, has_mapping) через LEFT JOIN +all_competitors + mapped (OBJ-2: ALL competitors included, unmapped has_mapping=False). Третий вызов execute — bucket_rows (obj_id, room_bucket, units_sold, sqm_sold). """ @@ -46,6 +47,7 @@ def _sales_row( months: int, start: str = "2024-11-01", end: str = "2025-04-01", + has_mapping: bool = True, ) -> MagicMock: r = MagicMock() start_d = datetime.date.fromisoformat(start) @@ -56,6 +58,9 @@ def _sales_row( "months_with_data": months, "period_start": start_d, "period_end": end_d, + # OBJ-2: LEFT JOIN all_competitors — все конкуренты включены, + # has_mapping=True если есть маппинг в objective_complex_mapping. + "has_mapping": has_mapping, }[k] return r @@ -199,9 +204,14 @@ def test_score_capped_at_1(): def test_score_zero_when_no_sales_sqm(): - """total_sqm=0 → None (нет данных, не score=0).""" + """total_sqm=0 → VelocityResult с velocity_data_available=False, score=0. + + OBJ-2: функция больше не возвращает None при нулевых продажах — + возвращает «пустое» состояние (velocity_data_available=False, source='none'). + Rosreestr-fallback пропускается т.к. cad_quarter не передан. + """ comp_rows = [_comp_row(1)] - # total_sqm=0 — нет продаж → должен вернуть None + # total_sqm=0 — нет продаж → velocity_data_available=False sales_rows = [_sales_row(1, total_sqm=0.0, months=5)] db = _make_db(comp_rows=comp_rows, sales_rows=sales_rows) @@ -212,7 +222,10 @@ def test_score_zero_when_no_sales_sqm(): ): result = compute_velocity(db, parcel_geom_wkt=_PARCEL_WKT) - assert result is None + assert result is not None + assert result.velocity_data_available is False + assert result.velocity_score == pytest.approx(0.0) + assert result.velocity_source == "none" def test_as_dict_structure(): @@ -228,6 +241,8 @@ def test_as_dict_structure(): period_end="2025-02", sample_competitors=[], by_room_bucket={"1": {"units": 10, "sqm": 450.0, "complexes_count": 2}}, + velocity_data_available=True, + velocity_source="objective", ) d = vr.as_dict() assert "competitors_count" in d @@ -239,6 +254,9 @@ def test_as_dict_structure(): assert d["velocity_score"] == pytest.approx(0.333, abs=1e-3) assert "by_room_bucket" in d assert d["by_room_bucket"]["1"]["units"] == 10 + # OBJ-2: новые поля velocity_data_available и velocity_source + assert d["velocity_data_available"] is True + assert d["velocity_source"] == "objective" def test_sample_competitors_top5(): diff --git a/backend/tests/workers/test_scrape_kn_catalog_objects.py b/backend/tests/workers/test_scrape_kn_catalog_objects.py new file mode 100644 index 00000000..e31cb8be --- /dev/null +++ b/backend/tests/workers/test_scrape_kn_catalog_objects.py @@ -0,0 +1,112 @@ +"""Тесты для backend/app/workers/tasks/scrape_kn_catalog_objects.py. + +Покрывают skip-today фильтр и force flag (admin "Загрузить все"). +""" + +from __future__ import annotations + +from datetime import date +from typing import Any +from unittest.mock import MagicMock, patch + + +def _make_mock_db(rows: list[dict[str, Any]]) -> tuple[MagicMock, list[dict[str, Any]]]: + """Mock DB session. Возвращает (db, captured_params).""" + captured: list[dict[str, Any]] = [] + + def execute_router(stmt: Any, params: Any = None) -> MagicMock: + if params is not None: + captured.append(params) + result = MagicMock() + result.mappings.return_value.all.return_value = rows + return result + + db = MagicMock() + db.execute = execute_router + return db, captured + + +def test_skip_today_filter_active_by_default() -> None: + """force=False (default) → :force = False передаётся в SQL → SQL пропустит + объекты с catalog_scraped_at = сегодня (логика в WHERE).""" + from app.workers.tasks import scrape_kn_catalog_objects as task_mod + + db, captured = _make_mock_db(rows=[]) + + with patch.object(task_mod, "SessionLocal", return_value=db): + result = task_mod.scrape_kn_catalog_objects.run(region_code=66) + + # SQL вызван 1 раз с force=False + assert len(captured) == 1 + assert captured[0]["force"] is False + assert captured[0]["region_code"] == 66 + # Пустой результат → ранний выход + assert result["obj_ids_count"] == 0 + assert result["force"] is False + + +def test_force_true_passes_through_to_sql() -> None: + """force=True → :force = True передаётся в SQL → SQL грузит все объекты.""" + from app.workers.tasks import scrape_kn_catalog_objects as task_mod + + db, captured = _make_mock_db(rows=[]) + + with patch.object(task_mod, "SessionLocal", return_value=db): + result = task_mod.scrape_kn_catalog_objects.run(region_code=66, force=True) + + assert len(captured) == 1 + assert captured[0]["force"] is True + assert result["force"] is True + + +def test_max_objects_passed_as_limit() -> None: + """max_objects явный — должен прилететь в SQL bind как :max_objects.""" + from app.workers.tasks import scrape_kn_catalog_objects as task_mod + + db, captured = _make_mock_db(rows=[]) + + with patch.object(task_mod, "SessionLocal", return_value=db): + task_mod.scrape_kn_catalog_objects.run(region_code=66, max_objects=10) + + assert captured[0]["max_objects"] == 10 + + +def test_processes_rows_when_returned() -> None: + """Если SELECT вернул строки — вызывается scrape_catalog_objects, snapshot_date + из первой строки.""" + from app.workers.tasks import scrape_kn_catalog_objects as task_mod + + rows = [ + {"obj_id": 65136, "snapshot_date": date(2026, 5, 17)}, + {"obj_id": 65137, "snapshot_date": date(2026, 5, 17)}, + ] + db, _ = _make_mock_db(rows=rows) + + mock_stats = {"processed": 2, "succeeded": 2, "failed": 0, "skipped": 0} + with ( + patch.object(task_mod, "SessionLocal", return_value=db), + patch( + "app.services.scrapers.domrf_catalog_object.scrape_catalog_objects", + return_value=mock_stats, + ) as mock_scrape, + ): + result = task_mod.scrape_kn_catalog_objects.run(region_code=66) + + mock_scrape.assert_called_once() + call_kwargs = mock_scrape.call_args[1] + assert call_kwargs["obj_ids"] == [65136, 65137] + assert call_kwargs["snapshot_date"] == date(2026, 5, 17) + assert call_kwargs["region_code"] == 66 + assert result["obj_ids_count"] == 2 + assert result["snapshot_date"] == "2026-05-17" + + +def test_sql_contains_skip_today_predicate() -> None: + """Smoke: финальный SQL содержит DATE(catalog_scraped_at) < CURRENT_DATE и + CAST(:force AS boolean) — иначе фильтр не работает.""" + from app.workers.tasks.scrape_kn_catalog_objects import _SELECT_TARGETS_SQL + + sql = str(_SELECT_TARGETS_SQL).upper() + assert "DATE(CATALOG_SCRAPED_AT) < CURRENT_DATE" in sql + assert "CAST(:FORCE AS BOOLEAN)" in sql + assert "CATALOG_SCRAPED_AT IS NULL" in sql diff --git a/backend/uv.lock b/backend/uv.lock index 1422c133..d1643ad0 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -567,6 +567,8 @@ dependencies = [ { name = "geoalchemy2" }, { name = "geopandas" }, { name = "httpx" }, + { name = "ijson" }, + { name = "jinja2" }, { name = "numpy" }, { name = "openpyxl" }, { name = "pandas" }, @@ -579,7 +581,7 @@ dependencies = [ { name = "redis" }, { name = "rosreestr2coord" }, { name = "scikit-learn" }, - { name = "sentry-sdk", extra = ["fastapi"] }, + { name = "sentry-sdk", extra = ["celery", "fastapi", "httpx", "sqlalchemy"] }, { name = "shapely" }, { name = "sqlalchemy" }, { name = "tenacity" }, @@ -606,6 +608,8 @@ requires-dist = [ { name = "geoalchemy2", specifier = ">=0.15.0" }, { name = "geopandas", specifier = ">=1.0.0" }, { name = "httpx", specifier = ">=0.27.0" }, + { name = "ijson", specifier = ">=3.2.0" }, + { name = "jinja2", specifier = ">=3.1.0" }, { name = "numpy", specifier = ">=2.0.0" }, { name = "openpyxl", specifier = ">=3.1.0" }, { name = "pandas", specifier = ">=2.2.0" }, @@ -618,7 +622,7 @@ requires-dist = [ { name = "redis", specifier = ">=5.0.0" }, { name = "rosreestr2coord", specifier = ">=5.0.0" }, { name = "scikit-learn", specifier = ">=1.5.0" }, - { name = "sentry-sdk", extras = ["fastapi"], specifier = ">=2.10.0" }, + { name = "sentry-sdk", extras = ["fastapi", "celery", "sqlalchemy", "httpx"], specifier = ">=2.18.0" }, { name = "shapely", specifier = ">=2.0.0" }, { name = "sqlalchemy", specifier = ">=2.0.30" }, { name = "tenacity", specifier = ">=9.0.0" }, @@ -797,6 +801,69 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5d/13/ad7d7ca3808a898b4612b6fe93cde56b53f3034dcde235acb1f0e1df24c6/idna-3.13-py3-none-any.whl", hash = "sha256:892ea0cde124a99ce773decba204c5552b69c3c67ffd5f232eb7696135bc8bb3", size = 68629, upload-time = "2026-04-22T16:42:40.909Z" }, ] +[[package]] +name = "ijson" +version = "3.5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f4/57/60d1a6a512f2f0508d0bc8b4f1cc5616fd3196619b66bd6a01f9155a1292/ijson-3.5.0.tar.gz", hash = "sha256:94688760720e3f5212731b3cb8d30267f9a045fb38fb3870254e7b9504246f31", size = 68658, upload-time = "2026-02-24T03:58:30.974Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/aa/17/9c63c7688025f3a8c47ea717b8306649c8c7244e49e20a2be4e3515dc75c/ijson-3.5.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:1ebefbe149a6106cc848a3eaf536af51a9b5ccc9082de801389f152dba6ab755", size = 88536, upload-time = "2026-02-24T03:57:06.809Z" }, + { url = "https://files.pythonhosted.org/packages/6f/dd/e15c2400244c117b06585452ebc63ae254f5a6964f712306afd1422daae0/ijson-3.5.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:19e30d9f00f82e64de689c0b8651b9cfed879c184b139d7e1ea5030cec401c21", size = 60499, upload-time = "2026-02-24T03:57:09.155Z" }, + { url = "https://files.pythonhosted.org/packages/77/a9/bf4fe3538a0c965f16b406f180a06105b875da83f0743e36246be64ef550/ijson-3.5.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a04a33ee78a6f27b9b8528c1ca3c207b1df3b8b867a4cf2fcc4109986f35c227", size = 60330, upload-time = "2026-02-24T03:57:10.574Z" }, + { url = "https://files.pythonhosted.org/packages/31/76/6f91bdb019dd978fce1bc5ea1cd620cfc096d258126c91db2c03a20a7f34/ijson-3.5.0-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:7d48dc2984af02eb3c56edfb3f13b3f62f2f3e4fe36f058c8cfc75d93adf4fed", size = 138977, upload-time = "2026-02-24T03:57:11.932Z" }, + { url = "https://files.pythonhosted.org/packages/11/be/bbc983059e48a54b0121ee60042979faed7674490bbe7b2c41560db3f436/ijson-3.5.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f1e73a44844d9adbca9cf2c4132cd875933e83f3d4b23881fcaf82be83644c7d", size = 149785, upload-time = "2026-02-24T03:57:13.255Z" }, + { url = "https://files.pythonhosted.org/packages/6d/81/2fee58f9024a3449aee83edfa7167fb5ccd7e1af2557300e28531bb68e16/ijson-3.5.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7389a56b8562a19948bdf1d7bae3a2edc8c7f86fb59834dcb1c4c722818e645a", size = 149729, upload-time = "2026-02-24T03:57:14.191Z" }, + { url = "https://files.pythonhosted.org/packages/c7/56/f1706761fcc096c9d414b3dcd000b1e6e5c24364c21cfba429837f98ee8d/ijson-3.5.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3176f23f8ebec83f374ed0c3b4e5a0c4db7ede54c005864efebbed46da123608", size = 150697, upload-time = "2026-02-24T03:57:15.855Z" }, + { url = "https://files.pythonhosted.org/packages/d9/6e/ee0d9c875a0193b632b3e9ccd1b22a50685fb510256ad57ba483b6529f77/ijson-3.5.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:6babd88e508630c6ef86c9bebaaf13bb2fb8ec1d8f8868773a03c20253f599bc", size = 142873, upload-time = "2026-02-24T03:57:16.831Z" }, + { url = "https://files.pythonhosted.org/packages/d2/bf/f9d4399d0e6e3fd615035290a71e97c843f17f329b43638c0a01cf112d73/ijson-3.5.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:dc1b3836b174b6db2fa8319f1926fb5445abd195dc963368092103f8579cb8ed", size = 151583, upload-time = "2026-02-24T03:57:17.757Z" }, + { url = "https://files.pythonhosted.org/packages/b2/71/a7254a065933c0e2ffd3586f46187d84830d3d7b6f41cfa5901820a4f87d/ijson-3.5.0-cp312-cp312-win32.whl", hash = "sha256:6673de9395fb9893c1c79a43becd8c8fbee0a250be6ea324bfd1487bb5e9ee4c", size = 53079, upload-time = "2026-02-24T03:57:18.703Z" }, + { url = "https://files.pythonhosted.org/packages/8f/7b/2edca79b359fc9f95d774616867a03ecccdf333797baf5b3eea79733918c/ijson-3.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:f4f7fabd653459dcb004175235f310435959b1bb5dfa8878578391c6cc9ad944", size = 55500, upload-time = "2026-02-24T03:57:20.428Z" }, + { url = "https://files.pythonhosted.org/packages/a2/71/d67e764a712c3590627480643a3b51efcc3afa4ef3cb54ee4c989073c97e/ijson-3.5.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:e9cedc10e40dd6023c351ed8bfc7dcfce58204f15c321c3c1546b9c7b12562a4", size = 88544, upload-time = "2026-02-24T03:57:21.293Z" }, + { url = "https://files.pythonhosted.org/packages/1a/39/f1c299371686153fa3cf5c0736b96247a87a1bee1b7145e6d21f359c505a/ijson-3.5.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:3647649f782ee06c97490b43680371186651f3f69bebe64c6083ee7615d185e5", size = 60495, upload-time = "2026-02-24T03:57:22.501Z" }, + { url = "https://files.pythonhosted.org/packages/16/94/b1438e204d75e01541bebe3e668fe3e68612d210e9931ae1611062dd0a56/ijson-3.5.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:90e74be1dce05fce73451c62d1118671f78f47c9f6be3991c82b91063bf01fc9", size = 60325, upload-time = "2026-02-24T03:57:23.332Z" }, + { url = "https://files.pythonhosted.org/packages/30/e2/4aa9c116fa86cc8b0f574f3c3a47409edc1cd4face05d0e589a5a176b05d/ijson-3.5.0-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:78e9ad73e7be2dd80627504bd5cbf512348c55ce2c06e362ed7683b5220e8568", size = 138774, upload-time = "2026-02-24T03:57:24.683Z" }, + { url = "https://files.pythonhosted.org/packages/d2/d2/738b88752a70c3be1505faa4dcd7110668c2712e582a6a36488ed1e295d4/ijson-3.5.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9577449313cc94be89a4fe4b3e716c65f09cc19636d5a6b2861c4e80dddebd58", size = 149820, upload-time = "2026-02-24T03:57:26.062Z" }, + { url = "https://files.pythonhosted.org/packages/ed/df/0b3ab9f393ca8f72ea03bc896ba9fdc987e90ae08cdb51c32a4ee0c14d5e/ijson-3.5.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e4c1178fb50aff5f5701a30a5152ead82a14e189ce0f6102fa1b5f10b2f54ff", size = 149747, upload-time = "2026-02-24T03:57:27.308Z" }, + { url = "https://files.pythonhosted.org/packages/cc/a3/b0037119f75131b78cb00acc2657b1a9d0435475f1f2c5f8f5a170b66b9c/ijson-3.5.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:0eb402ab026ffb37a918d75af2b7260fe6cfbce13232cc83728a714dd30bd81d", size = 151027, upload-time = "2026-02-24T03:57:28.522Z" }, + { url = "https://files.pythonhosted.org/packages/22/a0/cb344de1862bf09d8f769c9d25c944078c87dd59a1b496feec5ad96309a4/ijson-3.5.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:5b08ee08355f9f729612a8eb9bf69cc14f9310c3b2a487c6f1c3c65d85216ec4", size = 142996, upload-time = "2026-02-24T03:57:29.774Z" }, + { url = "https://files.pythonhosted.org/packages/ca/32/a8ffd67182e02ea61f70f62daf43ded4fa8a830a2520a851d2782460aba8/ijson-3.5.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:bda62b6d48442903e7bf56152108afb7f0f1293c2b9bef2f2c369defea76ab18", size = 152068, upload-time = "2026-02-24T03:57:30.969Z" }, + { url = "https://files.pythonhosted.org/packages/3c/d1/3578df8e75d446aab0ae92e27f641341f586b85e1988536adebc65300cb4/ijson-3.5.0-cp313-cp313-win32.whl", hash = "sha256:8d073d9b13574cfa11083cc7267c238b7a6ed563c2661e79192da4a25f09c82c", size = 53065, upload-time = "2026-02-24T03:57:31.93Z" }, + { url = "https://files.pythonhosted.org/packages/fb/a2/f7cdaf5896710da3e69e982e44f015a83d168aa0f3a89b6f074b5426779d/ijson-3.5.0-cp313-cp313-win_amd64.whl", hash = "sha256:2419f9e32e0968a876b04d8f26aeac042abd16f582810b576936bbc4c6015069", size = 55499, upload-time = "2026-02-24T03:57:32.773Z" }, + { url = "https://files.pythonhosted.org/packages/42/65/13e2492d17e19a2084523e18716dc2809159f2287fd2700c735f311e76c4/ijson-3.5.0-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:4d4b0cd676b8c842f7648c1a783448fac5cd3b98289abd83711b3e275e143524", size = 93019, upload-time = "2026-02-24T03:57:33.976Z" }, + { url = "https://files.pythonhosted.org/packages/33/92/483fc97ece0c3f1cecabf48f6a7a36e89d19369eec462faaeaa34c788992/ijson-3.5.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:252dec3680a48bb82d475e36b4ae1b3a9d7eb690b951bb98a76c5fe519e30188", size = 62714, upload-time = "2026-02-24T03:57:34.819Z" }, + { url = "https://files.pythonhosted.org/packages/4b/88/793fe020a0fe9d9eed4c285cf4a5cfdb0a935708b3bde0d72f35c794b513/ijson-3.5.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:aa1b5dca97d323931fde2501172337384c958914d81a9dac7f00f0d4bfc76bc7", size = 62460, upload-time = "2026-02-24T03:57:35.874Z" }, + { url = "https://files.pythonhosted.org/packages/51/69/f1a2690aa8d4df1f4e262b385e65a933ffdc250b091531bac9a449c19e16/ijson-3.5.0-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:7a5ec7fd86d606094bba6f6f8f87494897102fa4584ef653f3005c51a784c320", size = 199273, upload-time = "2026-02-24T03:57:37.07Z" }, + { url = "https://files.pythonhosted.org/packages/ea/a2/f1346d5299e79b988ab472dc773d5381ec2d57c23cb2f1af3ede4a810e62/ijson-3.5.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:009f41443e1521847701c6d87fa3923c0b1961be3c7e7de90947c8cb92ea7c44", size = 216884, upload-time = "2026-02-24T03:57:38.346Z" }, + { url = "https://files.pythonhosted.org/packages/28/3c/8b637e869be87799e6c2c3c275a30a546f086b1aed77e2b7f11512168c5a/ijson-3.5.0-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e4c3651d1f9fe2839a93fdf8fd1d5ca3a54975349894249f3b1b572bcc4bd577", size = 207306, upload-time = "2026-02-24T03:57:39.718Z" }, + { url = "https://files.pythonhosted.org/packages/7f/7c/18b1c1df6951ca056782d7580ec40cea4ff9a27a0947d92640d1cc8c4ae3/ijson-3.5.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:945b7abcfcfeae2cde17d8d900870f03536494245dda7ad4f8d056faa303256c", size = 211364, upload-time = "2026-02-24T03:57:40.953Z" }, + { url = "https://files.pythonhosted.org/packages/f3/55/e795812e82851574a9dba8a53fde045378f531ef14110c6fb55dbd23b443/ijson-3.5.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:0574b0a841ff97495c13e9d7260fbf3d85358b061f540c52a123db9dbbaa2ed6", size = 200608, upload-time = "2026-02-24T03:57:42.272Z" }, + { url = "https://files.pythonhosted.org/packages/5c/cd/013c85b4749b57a4cb4c2670014d1b32b8db4ab1a7be92ea7aeb5d7fe7b5/ijson-3.5.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f969ffb2b89c5cdf686652d7fb66252bc72126fa54d416317411497276056a18", size = 205127, upload-time = "2026-02-24T03:57:43.286Z" }, + { url = "https://files.pythonhosted.org/packages/0e/7c/faf643733e3ab677f180018f6a855c4ef70b7c46540987424c563c959e42/ijson-3.5.0-cp313-cp313t-win32.whl", hash = "sha256:59d3f9f46deed1332ad669518b8099920512a78bda64c1f021fcd2aff2b36693", size = 55282, upload-time = "2026-02-24T03:57:44.353Z" }, + { url = "https://files.pythonhosted.org/packages/69/22/94ddb47c24b491377aca06cd8fc9202cad6ab50619842457d2beefde21ea/ijson-3.5.0-cp313-cp313t-win_amd64.whl", hash = "sha256:5c2839fa233746d8aad3b8cd2354e441613f5df66d721d59da4a09394bd1db2b", size = 58016, upload-time = "2026-02-24T03:57:45.237Z" }, + { url = "https://files.pythonhosted.org/packages/7a/93/0868efe753dc1df80cc405cf0c1f2527a6991643607c741bff8dcb899b3b/ijson-3.5.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:25a5a6b2045c90bb83061df27cfa43572afa43ba9408611d7bfe237c20a731a9", size = 89094, upload-time = "2026-02-24T03:57:46.115Z" }, + { url = "https://files.pythonhosted.org/packages/24/94/fd5a832a0df52ef5e4e740f14ac8640725d61034a1b0c561e8b5fb424706/ijson-3.5.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:8976c54c0b864bc82b951bae06567566ac77ef63b90a773a69cd73aab47f4f4f", size = 60715, upload-time = "2026-02-24T03:57:47.552Z" }, + { url = "https://files.pythonhosted.org/packages/70/79/1b9a90af5732491f9eec751ee211b86b11011e1158c555c06576d52c3919/ijson-3.5.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:859eb2038f7f1b0664df4241957694cc35e6295992d71c98659b22c69b3cbc10", size = 60638, upload-time = "2026-02-24T03:57:48.428Z" }, + { url = "https://files.pythonhosted.org/packages/23/6f/2c551ea980fe56f68710a8d5389cfbd015fc45aaafd17c3c52c346db6aa1/ijson-3.5.0-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:c911aa02991c7c0d3639b6619b93a93210ff1e7f58bf7225d613abea10adc78e", size = 140667, upload-time = "2026-02-24T03:57:49.314Z" }, + { url = "https://files.pythonhosted.org/packages/25/0e/27b887879ba6a5bc29766e3c5af4942638c952220fd63e1e442674f7883a/ijson-3.5.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:903cbdc350173605220edc19796fbea9b2203c8b3951fb7335abfa8ed37afda8", size = 149850, upload-time = "2026-02-24T03:57:50.329Z" }, + { url = "https://files.pythonhosted.org/packages/da/1e/23e10e1bc04bf31193b21e2960dce14b17dbd5d0c62204e8401c59d62c08/ijson-3.5.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a4549d96ded5b8efa71639b2160235415f6bdb8c83367615e2dbabcb72755c33", size = 149206, upload-time = "2026-02-24T03:57:51.261Z" }, + { url = "https://files.pythonhosted.org/packages/8e/90/e552f6495063b235cf7fa2c592f6597c057077195e517b842a0374fd470c/ijson-3.5.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:6b2dcf6349e6042d83f3f8c39ce84823cf7577eba25bac5aae5e39bbbbbe9c1c", size = 150438, upload-time = "2026-02-24T03:57:52.198Z" }, + { url = "https://files.pythonhosted.org/packages/5c/18/45bf8f297c41b42a1c231d261141097babd953d2c28a07be57ae4c3a1a02/ijson-3.5.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:e44af39e6f8a17e5627dcd89715d8279bf3474153ff99aae031a936e5c5572e5", size = 144369, upload-time = "2026-02-24T03:57:53.22Z" }, + { url = "https://files.pythonhosted.org/packages/9b/3a/deb9772bb2c0cead7ad64f00c3598eec9072bdf511818e70e2c512eeabbe/ijson-3.5.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:9260332304b7e7828db56d43f08fc970a3ab741bf84ff10189361ea1b60c395b", size = 151352, upload-time = "2026-02-24T03:57:54.375Z" }, + { url = "https://files.pythonhosted.org/packages/e4/51/67f4d80cd58ad7eab0cd1af5fe28b961886338956b2f88c0979e21914346/ijson-3.5.0-cp314-cp314-win32.whl", hash = "sha256:63bc8121bb422f6969ced270173a3fa692c29d4ae30c860a2309941abd81012a", size = 53610, upload-time = "2026-02-24T03:57:55.655Z" }, + { url = "https://files.pythonhosted.org/packages/70/d3/263672ea22983ba3940f1534316dbc9200952c1c2a2332d7a664e4eaa7ae/ijson-3.5.0-cp314-cp314-win_amd64.whl", hash = "sha256:01b6dad72b7b7df225ef970d334556dfad46c696a2c6767fb5d9ed8889728bca", size = 56301, upload-time = "2026-02-24T03:57:56.584Z" }, + { url = "https://files.pythonhosted.org/packages/9f/d9/86f7fac35e0835faa188085ae0579e813493d5261ce056484015ad533445/ijson-3.5.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:2ea4b676ec98e374c1df400a47929859e4fa1239274339024df4716e802aa7e4", size = 93069, upload-time = "2026-02-24T03:57:57.849Z" }, + { url = "https://files.pythonhosted.org/packages/33/d2/e7366ed9c6e60228d35baf4404bac01a126e7775ea8ce57f560125ed190a/ijson-3.5.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:014586eec043e23c80be9a923c56c3a0920a0f1f7d17478ce7bc20ba443968ef", size = 62767, upload-time = "2026-02-24T03:57:58.758Z" }, + { url = "https://files.pythonhosted.org/packages/35/8b/3e703e8cc4b3ada79f13b28070b51d9550c578f76d1968657905857b2ddd/ijson-3.5.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:d5b8b886b0248652d437f66e7c5ac318bbdcb2c7137a7e5327a68ca00b286f5f", size = 62467, upload-time = "2026-02-24T03:58:00.261Z" }, + { url = "https://files.pythonhosted.org/packages/21/42/0c91af32c1ee8a957fdac2e051b5780756d05fd34e4b60d94a08d51bac1d/ijson-3.5.0-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:498fd46ae2349297e43acf97cdc421e711dbd7198418677259393d2acdc62d78", size = 200447, upload-time = "2026-02-24T03:58:01.591Z" }, + { url = "https://files.pythonhosted.org/packages/f9/80/796ea0e391b7e2d45c5b1b451734bba03f81c2984cf955ea5eaa6c4920ad/ijson-3.5.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:22a51b4f9b81f12793731cf226266d1de2112c3c04ba4a04117ad4e466897e05", size = 217820, upload-time = "2026-02-24T03:58:02.598Z" }, + { url = "https://files.pythonhosted.org/packages/38/14/52b6613fdda4078c62eb5b4fe3efc724ddc55a4ad524c93de51830107aa3/ijson-3.5.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9636c710dc4ac4a281baa266a64f323b4cc165cec26836af702c44328b59a515", size = 208310, upload-time = "2026-02-24T03:58:04.759Z" }, + { url = "https://files.pythonhosted.org/packages/6a/ad/8b3105a78774fd4a65e534a21d975ef3a77e189489fe3029ebcaeba5e243/ijson-3.5.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:f7168a39e8211107666d71b25693fd1b2bac0b33735ef744114c403c6cac21e1", size = 211843, upload-time = "2026-02-24T03:58:05.836Z" }, + { url = "https://files.pythonhosted.org/packages/36/ab/a2739f6072d6e1160581bc3ed32da614c8cced023dcd519d9c5fa66e0425/ijson-3.5.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:8696454245415bc617ab03b0dc3ae4c86987df5dc6a90bad378fe72c5409d89e", size = 200906, upload-time = "2026-02-24T03:58:07.788Z" }, + { url = "https://files.pythonhosted.org/packages/6d/5e/e06c2de3c3d4a9cfb655c1ad08a68fb72838d271072cdd3196576ac4431a/ijson-3.5.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:c21bfb61f71f191565885bf1bc29e0a186292d866b4880637b833848360bdc1b", size = 205495, upload-time = "2026-02-24T03:58:09.163Z" }, + { url = "https://files.pythonhosted.org/packages/7c/11/778201eb2e202ddd76b36b0fb29bf3d8e3c167389d8aa883c62524e49f47/ijson-3.5.0-cp314-cp314t-win32.whl", hash = "sha256:a2619460d6795b70d0155e5bf016200ac8a63ab5397aa33588bb02b6c21759e6", size = 56280, upload-time = "2026-02-24T03:58:10.116Z" }, + { url = "https://files.pythonhosted.org/packages/23/28/96711503245339084c8086b892c47415895eba49782d6cc52d9f4ee50301/ijson-3.5.0-cp314-cp314t-win_amd64.whl", hash = "sha256:4f24b78d4ef028d17eb57ad1b16c0aed4a17bdd9badbf232dc5d9305b7e13854", size = 58965, upload-time = "2026-02-24T03:58:11.278Z" }, +] + [[package]] name = "iniconfig" version = "2.3.0" @@ -806,6 +873,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, ] +[[package]] +name = "jinja2" +version = "3.1.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115, upload-time = "2025-03-05T20:05:02.478Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, +] + [[package]] name = "joblib" version = "1.5.3" @@ -1910,9 +1989,18 @@ wheels = [ ] [package.optional-dependencies] +celery = [ + { name = "celery" }, +] fastapi = [ { name = "fastapi" }, ] +httpx = [ + { name = "httpx" }, +] +sqlalchemy = [ + { name = "sqlalchemy" }, +] [[package]] name = "shapely" diff --git a/caddy/users.caddy.snippet b/caddy/users.caddy.snippet new file mode 100644 index 00000000..d66a0ebf --- /dev/null +++ b/caddy/users.caddy.snippet @@ -0,0 +1,23 @@ +# GenDesign Pilot — basic_auth users gate +# Last updated: 2026-05-23 +# Add/remove via scripts/auth/add_user.sh / remove_user.sh, then PR. +# +# Format: username bcrypt_hash_base64 # comment +# Hash generation: caddy hash-password --plaintext | tr -d '\n' | base64 -w 0 +# Caddy 2.11+ requires base64-encoded bcrypt hashes in Caddyfile basic_auth. +# Realm is set as argument to basic_auth, not as subdirective. + +basic_auth bcrypt "GenDesign Pilot" { + admin JDJhJDE0JHVXRGpmWFBMMmNZd1duQUFJVWFkRi5RVy4xbHBVL3BPaTF6dFBvbUJ5enZvaXQ3bnZ0eGU2 # internal master (qa-tester + health checks, 2026-05-23) + user1 JDJhJDE0JGVHLi5XYUtnUnI5TTUweDBYNGpxQXV5OTNtWFlkYkZQRzJXVzlhSWxRMnhFN25vVDhleE11 # TBD (2026-05-23) + user2 JDJhJDE0JHl1Vk1HL1ZJQlY5R0FJQndtSzRBcU8wQ0kxUjdicXpHRldBR2JLQmkvUlJ6aWp0Z0EvMjYu # TBD (2026-05-23) + user3 JDJhJDE0JGFOeEwwdnFaUjZiWWpkckVkWFdvT2VMVTkubWZrMVBtQlNmcmg4ZS9qU01OYndPVXRzT1Iy # TBD (2026-05-23) + user4 JDJhJDE0JDJWbkY0YWNIRGhCMlFGekhFM2tVSHVxY0JEWm9UOVNqRTBMbGYzbERHcE5sQjF3Znd6QkNh # TBD (2026-05-23) + user5 JDJhJDE0JDAzTEhkd25STjdUTUtUQkdaREdkSU9mdmlVWEI0d1NxTTRkQnhydklDLk5kT3c2MGdOZnA2 # TBD (2026-05-23) + user6 JDJhJDE0JC9vZnZaUU5TVkNUNnRiU3F3QjhYTXVCb2xaSWlzb3U5aUVtY29jSlRyMExOVGdpOHhYc3ky # TBD (2026-05-23) + user7 JDJhJDE0JFlFSDJRV1hsb0kyUmYuMjdMd1FuTHVObC9yQUNNQy9WRGZWMnlUc2wuRUFDOEV6SEJrbkou # TBD (2026-05-23) + user8 JDJhJDE0JFd0alo2VmZYWDlCcHFqellTWC9RSi44WVhQUi5hVFk5djd4emlwcWVvMDExcEU1MVpDL1Yu # TBD (2026-05-23) + user9 JDJhJDE0JHRPZzVEd2ZKaldUWXFSUzI5c3RHTE9NVnp1amRFVkVMTHRFMGF2eUgzbzk2SEJvekdLQ3ZT # TBD (2026-05-23) + user10 JDJhJDE0JEo1QVRUTk1wejgucUI1ZFAzTEE0WWVxQk55c29hTGlHZ243RVhUc2F1aldIc0pSSHVIOUxL # TBD (2026-05-23) + kopylov JDJhJDE0JExUald3alVMbS5RdVV0TTRTcGk0cE9YeHNXS2E1QU5rTFp2TnFFQzdmWWNFV3djRVAzOUdH # Копылов (2026-05-23) +} diff --git a/data/sql/100_fix_mv_layout_velocity_weighted_avg.sql b/data/sql/100_fix_mv_layout_velocity_weighted_avg.sql new file mode 100644 index 00000000..23b72863 --- /dev/null +++ b/data/sql/100_fix_mv_layout_velocity_weighted_avg.sql @@ -0,0 +1,88 @@ +-- 100_fix_mv_layout_velocity_weighted_avg.sql +-- SF Bug #21 (P0) — mv_layout_velocity использовал unweighted AVG() по +-- deals_total_avg_area_m2 и deals_total_avg_price_thousand_rub_per_m2. +-- objective_corpus_room_month хранит строки за каждый (project, room, month) +-- включая месяцы с 0 сделок (count=0, area=0, price=0). Нули тянули +-- среднее вниз в 2-14×. Примеры (verified prod): +-- obj 65307, room=3: 114 месяцев, 106 нулевых → AVG=7.4 м² vs weighted=105.8 м² +-- obj 65903, room=4+: 36 мес, 33 нулевых → AVG=9.1 м² vs weighted=109.0 м² +-- +-- Fix: заменяем AVG(x) на SUM(x * count) / NULLIF(SUM(count), 0) — weighted average. +-- +-- Dependencies: mv_layout_velocity не имеет зависимых VIEW/MV (проверено 2026-05-17). +-- Existing indexes: mv_layout_velocity_pk (UNIQUE obj_id, room_bucket), +-- mv_layout_velocity_obj_idx (obj_id) — пересоздаются ниже. +-- +-- ВАЖНО: этот файл выполняет DROP + CREATE + REFRESH (non-concurrent, блокировка ~30 с). +-- REFRESH CONCURRENTLY после первого populate — безопасен через layout_velocity_refresh.py. +-- +-- Deploy order: SQL migration → app deploy (никаких backend изменений в этом PR). +-- Idempotency: DROP IF EXISTS → безопасен при повторном запуске. + +BEGIN; + +DROP MATERIALIZED VIEW IF EXISTS mv_layout_velocity CASCADE; + +CREATE MATERIALIZED VIEW mv_layout_velocity AS +WITH last24mo AS ( + SELECT + ocm.project_name, + CASE + WHEN ocm.room_bucket = 'студия' THEN 'studio' + ELSE ocm.room_bucket + END AS room_bucket, + ocm.deals_total_count, + ocm.deals_total_avg_area_m2, + ocm.deals_total_avg_price_thousand_rub_per_m2, + ocm.deals_total_vol_m2, + ocm.report_month + FROM objective_corpus_room_month ocm + WHERE ocm.report_month >= (NOW() - INTERVAL '24 months')::date +) +SELECT + cm.domrf_obj_id AS obj_id, + l.room_bucket, + SUM(l.deals_total_count)::int AS total_deals_24mo, + + -- FIX SF #21: weighted average — нули из месяцев без сделок не тянут вниз. + -- Формула: SUM(avg_x * count) / SUM(count) эквивалентна SUM(total_x) / SUM(count), + -- т.к. avg_x = total_x / count за один месяц → произведение восстанавливает total_x. + (SUM(l.deals_total_avg_area_m2 * l.deals_total_count) + / NULLIF(SUM(l.deals_total_count), 0))::numeric(10, 2) AS avg_area_m2, + + (SUM(l.deals_total_avg_price_thousand_rub_per_m2 * l.deals_total_count) + / NULLIF(SUM(l.deals_total_count), 0))::numeric(12, 2) AS avg_price_thousand_rub_per_m2, + + SUM(l.deals_total_vol_m2)::numeric(12, 2) AS total_vol_m2, + MIN(l.report_month) AS window_start, + MAX(l.report_month) AS window_end, + COUNT(DISTINCT l.report_month)::int AS months_with_data +FROM last24mo l +JOIN objective_complex_mapping cm + ON cm.objective_complex_name = l.project_name +WHERE l.room_bucket IS NOT NULL + AND cm.domrf_obj_id IS NOT NULL + AND cm.objective_group = 'Екатеринбург' +GROUP BY cm.domrf_obj_id, l.room_bucket +WITH NO DATA; + +-- UNIQUE index: required for REFRESH CONCURRENTLY (periodic via layout_velocity_refresh.py). +-- GROUP BY (obj_id, room_bucket) guarantees uniqueness. +CREATE UNIQUE INDEX mv_layout_velocity_pk + ON mv_layout_velocity (obj_id, room_bucket); + +-- Lookup index for /best-layouts endpoint queries by obj_id. +CREATE INDEX mv_layout_velocity_obj_idx + ON mv_layout_velocity (obj_id); + +-- Initial populate (non-concurrent — MV just created, CONCURRENTLY requires populated MV). +-- Блокировка ~30 с на production при ~19 738 source rows. +REFRESH MATERIALIZED VIEW mv_layout_velocity; + +COMMENT ON MATERIALIZED VIEW mv_layout_velocity IS + 'Per-(obj_id, room_bucket) deals aggregation за last 24 months. ' + 'WEIGHTED average площади и цены (SF Bug #21 fix, 2026-05-17). ' + 'Source: objective_corpus_room_month × objective_complex_mapping (EKB only). ' + 'Refresh via layout_velocity_refresh.py (CONCURRENTLY после initial populate).'; + +COMMIT; diff --git a/data/sql/100_user_weight_profiles_default_seed.sql b/data/sql/100_user_weight_profiles_default_seed.sql new file mode 100644 index 00000000..546061cf --- /dev/null +++ b/data/sql/100_user_weight_profiles_default_seed.sql @@ -0,0 +1,90 @@ +-- 100_user_weight_profiles_default_seed.sql +-- Системные preset-профили весов POI для Site Finder. +-- Per #114 (Макс feedback): садики и Мегамарт должны иметь разные веса. +-- +-- Apply order: после 90_user_weight_profiles.sql (таблица уже существует) +-- Dependencies: user_weight_profiles table +-- +-- Sentinel owner: '__system__' — не является реальным user_id, +-- используется только для системных presets. +-- UI: dropdown показывает system presets всем пользователям через +-- GET /api/v1/admin/site-finder/weight-profiles?user_id=__system__ +-- или include_system=true (добавлен в #114 seed PR). +-- +-- Idempotent: ON CONFLICT (user_id, profile_name) DO UPDATE — обновляет +-- веса если preset уже существует (safe re-apply на prod). + +BEGIN; + +INSERT INTO user_weight_profiles + (user_id, profile_name, weights, is_default, description) +VALUES + ( + '__system__', + 'Эконом', + '{ + "school": 1.2, + "kindergarten": 1.0, + "pharmacy": 0.8, + "hospital": 0.5, + "shop_mall": 0.8, + "shop_supermarket": 1.5, + "shop_small": 1.0, + "park": 1.2, + "bus_stop": 1.0, + "metro_stop": 0.8, + "tram_stop": 0.5 + }'::jsonb, + FALSE, + 'Эконом-класс: доступность магазинов и транспорта важнее премиальной инфраструктуры' + ), + ( + '__system__', + 'Комфорт', + '{ + "school": 1.5, + "kindergarten": 1.8, + "pharmacy": 0.8, + "hospital": 0.6, + "shop_mall": 1.0, + "shop_supermarket": 1.2, + "shop_small": 0.5, + "park": 1.8, + "bus_stop": 0.5, + "metro_stop": 1.2, + "tram_stop": -0.3 + }'::jsonb, + FALSE, + 'Комфорт-класс: парки, детсады и школы в приоритете — семейная аудитория' + ), + ( + '__system__', + 'Бизнес', + '{ + "school": 1.0, + "kindergarten": 0.8, + "pharmacy": 0.6, + "hospital": 0.5, + "shop_mall": 2.0, + "shop_supermarket": 0.8, + "shop_small": 0.3, + "park": 2.0, + "bus_stop": 0.2, + "metro_stop": 2.5, + "tram_stop": -0.5 + }'::jsonb, + FALSE, + 'Бизнес-класс: метро, ТЦ и парки в приоритете — платёжеспособная аудитория' + ) +ON CONFLICT (user_id, profile_name) +DO UPDATE SET + weights = EXCLUDED.weights, + description = EXCLUDED.description, + updated_at = NOW(); + +COMMENT ON TABLE user_weight_profiles IS + 'POI weight profiles для site_finder. Per #114 (Макс feedback). ' + 'weights = {"school": 1.5, "kindergarten": 1.5, "shop_mall": 1.2, ...}. ' + 'Системные presets: user_id = ''__system__'' (3 preset: Эконом/Комфорт/Бизнес).'; + +COMMIT; diff --git a/data/sql/101_user_custom_pois.sql b/data/sql/101_user_custom_pois.sql new file mode 100644 index 00000000..5af0ac8c --- /dev/null +++ b/data/sql/101_user_custom_pois.sql @@ -0,0 +1,34 @@ +-- #254: User custom POIs — точки с произвольным весом для ad-hoc scoring. +-- Юзер добавляет точку на карту (name, weight, lon, lat), scoring учитывает. +-- parcel_cad NULL = глобальная per-user точка (видна для всех анализов пользователя). + +BEGIN; + +CREATE TABLE IF NOT EXISTS user_custom_pois ( + id BIGSERIAL PRIMARY KEY, + user_id TEXT NOT NULL, + parcel_cad TEXT, -- NULL = global per-user + name TEXT NOT NULL, + category TEXT, + weight REAL NOT NULL CHECK (weight BETWEEN -5 AND 5), + lon DOUBLE PRECISION NOT NULL CHECK (lon BETWEEN -180 AND 180), + lat DOUBLE PRECISION NOT NULL CHECK (lat BETWEEN -90 AND 90), + geom GEOGRAPHY(POINT, 4326) GENERATED ALWAYS AS ( + ST_SetSRID(ST_MakePoint(lon, lat), 4326)::geography + ) STORED, + notes TEXT, + created_at TIMESTAMPTZ DEFAULT NOW(), + updated_at TIMESTAMPTZ DEFAULT NOW() +); + +COMMENT ON TABLE user_custom_pois IS + 'User-defined ad-hoc POI points with custom scoring weight (#254). ' + 'parcel_cad=NULL means global (applies to all parcel analyses for user_id).'; + +CREATE INDEX IF NOT EXISTS user_custom_pois_geom_gist + ON user_custom_pois USING GIST (geom); + +CREATE INDEX IF NOT EXISTS user_custom_pois_user_parcel + ON user_custom_pois (user_id, parcel_cad); + +COMMIT; diff --git a/data/sql/102_cad_nspd_overlay_tables.sql b/data/sql/102_cad_nspd_overlay_tables.sql new file mode 100644 index 00000000..0c20fe09 --- /dev/null +++ b/data/sql/102_cad_nspd_overlay_tables.sql @@ -0,0 +1,91 @@ +-- Migration 101: cad_* overlay tables for NSPD layers (epic #263 sub-A) +-- +-- Creates 5 new tables for NSPD overlay data: +-- cad_territorial_zones -- layer 875838 (ПЗЗ территориальные зоны) +-- cad_red_lines -- layer 879243 (красные линии) +-- cad_engineering_structures -- layer 36328 (инженерные сооружения) +-- cad_risk_zones -- layers 872xxx (зоны рисков) +-- cad_opportunity_parcels -- layers 37294, 37298, 37299, 36473, 875845 +-- +-- All tables are idempotent (CREATE TABLE IF NOT EXISTS + CREATE INDEX IF NOT EXISTS). +-- Migration auto-applies via deploy.yml. + +BEGIN; + +-- ПЗЗ территориальные зоны (NSPD layer 875838) +CREATE TABLE IF NOT EXISTS cad_territorial_zones ( + id BIGSERIAL PRIMARY KEY, + quarter_cad TEXT NOT NULL, + zone_id TEXT NOT NULL UNIQUE, + zone_code TEXT, + zone_name TEXT, + permitted_use TEXT, + raw_props JSONB, + geom GEOGRAPHY(POLYGON, 4326), + fetched_at TIMESTAMPTZ DEFAULT NOW() +); +CREATE INDEX IF NOT EXISTS cad_territorial_zones_geom_gist + ON cad_territorial_zones USING GIST(geom); +CREATE INDEX IF NOT EXISTS cad_territorial_zones_quarter + ON cad_territorial_zones (quarter_cad); + +-- Red lines (NSPD layer 879243) — placeholder schema +CREATE TABLE IF NOT EXISTS cad_red_lines ( + id BIGSERIAL PRIMARY KEY, + quarter_cad TEXT NOT NULL, + feature_id TEXT UNIQUE, + name TEXT, + raw_props JSONB, + geom GEOGRAPHY(GEOMETRY, 4326), + fetched_at TIMESTAMPTZ DEFAULT NOW() +); +CREATE INDEX IF NOT EXISTS cad_red_lines_geom_gist + ON cad_red_lines USING GIST(geom); + +-- Engineering structures (NSPD layer 36328) — для #115 точки подключения +CREATE TABLE IF NOT EXISTS cad_engineering_structures ( + id BIGSERIAL PRIMARY KEY, + quarter_cad TEXT NOT NULL, + cad_num TEXT, + name TEXT, + structure_type TEXT, + raw_props JSONB, + geom GEOGRAPHY(GEOMETRY, 4326), + fetched_at TIMESTAMPTZ DEFAULT NOW() +); +CREATE INDEX IF NOT EXISTS cad_engineering_structures_geom_gist + ON cad_engineering_structures USING GIST(geom); +CREATE INDEX IF NOT EXISTS cad_engineering_structures_quarter + ON cad_engineering_structures (quarter_cad); + +-- Risk zones (NSPD layers 872xxx) +CREATE TABLE IF NOT EXISTS cad_risk_zones ( + id BIGSERIAL PRIMARY KEY, + quarter_cad TEXT NOT NULL, + feature_id TEXT UNIQUE, + risk_type TEXT, + layer_id INT, + severity TEXT, + raw_props JSONB, + geom GEOGRAPHY(GEOMETRY, 4326), + fetched_at TIMESTAMPTZ DEFAULT NOW() +); +CREATE INDEX IF NOT EXISTS cad_risk_zones_geom_gist + ON cad_risk_zones USING GIST(geom); + +-- Opportunity parcels (NSPD layers 37294, 37298, 37299, 36473, 875845) +CREATE TABLE IF NOT EXISTS cad_opportunity_parcels ( + id BIGSERIAL PRIMARY KEY, + quarter_cad TEXT NOT NULL, + cad_num TEXT, + opportunity_type TEXT, + layer_id INT, + name TEXT, + raw_props JSONB, + geom GEOGRAPHY(GEOMETRY, 4326), + fetched_at TIMESTAMPTZ DEFAULT NOW() +); +CREATE INDEX IF NOT EXISTS cad_opportunity_parcels_geom_gist + ON cad_opportunity_parcels USING GIST(geom); + +COMMIT; diff --git a/data/sql/103_seed_prinzip_mappings.sql b/data/sql/103_seed_prinzip_mappings.sql new file mode 100644 index 00000000..c2dc23d9 --- /dev/null +++ b/data/sql/103_seed_prinzip_mappings.sql @@ -0,0 +1,48 @@ +-- 103_seed_prinzip_mappings.sql +-- Audit 2026-05-17: bulk insert missing PRINZIP project mappings +-- into objective_complex_mapping so mv_layout_velocity picks them up. +-- +-- Problem: objective_corpus_room_month had active deals for these projects +-- but no row in objective_complex_mapping → mv_layout_velocity JOIN skipped them. +-- +-- Audit source: audits/SiteFinder_FixList_May17.md #4 +-- +-- UNIQUE constraint on table: (objective_complex_name, objective_group) +-- MV join key: objective_complex_mapping.objective_complex_name = objective_corpus_room_month.project_name +-- One domrf_obj_id per project_name; mapped to the largest/most active corpus. +-- +-- Skipped: +-- Парк Победы — already mapped to obj_id=60160 (Строящиеся, PRINZIP) +-- Парковый квартал — 0 deals/12m, inactive +-- Первый — belongs to ГК ИКАР, NOT PRINZIP; name collision, not a real match +-- +-- After applying this migration run manually (post-deploy): +-- REFRESH MATERIALIZED VIEW CONCURRENTLY mv_layout_velocity; +-- See runbook: runbooks/Refresh_MV_Layout_Velocity.md +-- +-- Dependencies: objective_complex_mapping (exists), mv_layout_velocity (refresh separately) +-- Deploy: auto-applied by deploy.yml via _schema_migrations tracking. +-- Idempotent: ON CONFLICT (objective_complex_name, objective_group) DO NOTHING + +BEGIN; + +INSERT INTO objective_complex_mapping + (objective_complex_name, objective_group, domrf_obj_id, + match_method, is_reviewed, note) +VALUES + -- 98 deals/12m in Objective. + -- obj_id 64701 = ЖК "Малевич" (Строящиеся, 244 кв, сдача 2027-09, дев: СЗ МАЛЕВИЧ). + -- obj_id 3687 = ЖК "Малевич" (Сданный, 348 кв, сдан 2020-03, дев: PRINZIP) — исторический. + -- 64701 выбран как активный primary corpus. + ('Малевич', 'Екатеринбург', 64701, 'manual', true, + 'Audit 2026-05-17: Строящиеся 244кв сдача 2027-09; obj_id 3687 — сданный исторический'), + + -- 30 deals/12m in Objective. + -- Корпуса: obj_id 47390 (411кв Сданные), 37880 (157кв), 47393 (86кв), 47392 (0кв skip). + -- 47390 выбран как крупнейший corpus. + ('Ньютон парк', 'Екатеринбург', 47390, 'manual', true, + 'Audit 2026-05-17: largest corpus 411кв из трёх Ньютон Парк корпусов') + +ON CONFLICT (objective_complex_name, objective_group) DO NOTHING; + +COMMIT; diff --git a/data/sql/104_seed_bulk_mappings.sql b/data/sql/104_seed_bulk_mappings.sql new file mode 100644 index 00000000..cd4f122c --- /dev/null +++ b/data/sql/104_seed_bulk_mappings.sql @@ -0,0 +1,232 @@ +-- 104_seed_bulk_mappings.sql +-- Audit 2026-05-17 (extension of PR #273): bulk seed objective_complex_mapping +-- for ALL active ЖК in objective_corpus_room_month (last 12 months) without mapping. +-- +-- Context: +-- PR #273 (103_seed_prinzip_mappings.sql) covered only Малевич + Ньютон парк. +-- This audit found 118 additional unmapped project_names with activity in the +-- last 12 months. Prior fuzzy import had already populated ~108 rows; this file +-- adds 13 HIGH-confidence (sim > 0.85) auto-inserts + comments out 25 MEDIUM +-- (sim 0.6-0.85) candidates for manual review. +-- +-- Audit summary: +-- HIGH (sim > 0.85): 13 projects — auto-inserted below +-- MEDIUM (sim 0.6-0.85): ~25 projects — commented out, need review +-- LOW / orphan (sim < 0.6 or no match): ~80 projects — dropped (no domrf corpus found) +-- +-- UNIQUE constraint: (objective_complex_name, objective_group) +-- MV join key: objective_complex_mapping.objective_complex_name = objective_corpus_room_month.project_name +-- +-- After applying, run manually (post-deploy): +-- REFRESH MATERIALIZED VIEW CONCURRENTLY mv_layout_velocity; +-- See runbook: runbooks/Refresh_MV_Layout_Velocity.md +-- +-- Dependencies: objective_complex_mapping (exists), mv_layout_velocity (refresh separately) +-- Deploy: auto-applied by deploy.yml via _schema_migrations tracking. +-- Idempotent: ON CONFLICT (objective_complex_name, objective_group) DO NOTHING + +BEGIN; + +-- ============================================================================= +-- HIGH confidence (sim > 0.85) — auto-insert +-- ============================================================================= +INSERT INTO objective_complex_mapping + (objective_complex_name, objective_group, domrf_obj_id, + match_method, match_score, is_reviewed, note) +VALUES + -- 140 deals/12m. "Б-26" → domrf "Б.26" (СКМ Девелопмент). sim=1.000 + ('Б-26', 'Екатеринбург', 50754, 'fuzzy', 1.000, false, + 'Audit 2026-05-17: sim=1.0 to Б.26 (СКМ Девелопмент, Строящиеся)'), + + -- 102 deals/12m. "Астон.Сезоны" → domrf "Астон. Сезоны" (Астон). sim=1.000 + ('Астон.Сезоны', 'Екатеринбург', 58721, 'fuzzy', 1.000, false, + 'Audit 2026-05-17: sim=1.0 to Астон. Сезоны (Астон, Сданные)'), + + -- 100 deals/12m. "Премиум Кварталы Маяковка" → exact domrf match. sim=1.000 + ('Премиум Кварталы Маяковка', 'Екатеринбург', 57364, 'fuzzy', 1.000, false, + 'Audit 2026-05-17: sim=1.0 exact match (СЗ ДЕЛОВОЙ КВАРТАЛ ДЕВЕЛОПМЕНТ, Сданные)'), + + -- 51 deals/12m. "Дом на Библиотечной" → exact domrf match. sim=1.000 + ('Дом на Библиотечной', 'Екатеринбург', 58788, 'fuzzy', 1.000, false, + 'Audit 2026-05-17: sim=1.0 exact match (СТОРИНГ, Сданные)'), + + -- 43 deals/12m. "Даблхаус 1:1" → domrf "Даблхаус 1:1" (Астра). sim=1.000 + ('Даблхаус 1:1', 'Екатеринбург', 50428, 'fuzzy', 1.000, false, + 'Audit 2026-05-17: sim=1.0 to Даблхаус "1:1" (Астра, Сданные)'), + + -- 42 deals/12m. "Астон.Реформа" → domrf "Астон. Реформа" (Астон). sim=1.000 + ('Астон.Реформа', 'Екатеринбург', 58381, 'fuzzy', 1.000, false, + 'Audit 2026-05-17: sim=1.0 to "Астон. Реформа" (Астон, Сданные)'), + + -- 30 deals/12m. "Астон.Отрадный" → exact domrf match (Астон). sim=1.000 + ('Астон.Отрадный', 'Екатеринбург', 48570, 'fuzzy', 1.000, false, + 'Audit 2026-05-17: sim=1.0 exact match (Астон, Сданные)'), + + -- 22 deals/12m. "Космонавтов 11" → domrf "Космонавтов 11" (ПИК). sim=1.000 + -- NB: "9 Космонавтов" is a DIFFERENT project — NOT mapped here (LOW, wrong address) + ('Космонавтов 11', 'Екатеринбург', 44337, 'fuzzy', 1.000, false, + 'Audit 2026-05-17: sim=1.0 exact match (ПИК, Сданные). NB: "9 Космонавтов" is separate.'), + + -- 16 deals/12m. "Кварталы конструктивизма" → domrf "КВАРТАЛЫ КОНСТРУКТИВИЗМА" (КОРТРОС). sim=1.000 + ('Кварталы конструктивизма', 'Екатеринбург', 43430, 'fuzzy', 1.000, false, + 'Audit 2026-05-17: sim=1.0 case-insensitive match (КОРТРОС, Сданные)'), + + -- 14 deals/12m. "Квартал Сюжеты" → domrf "Квартал СЮЖЕТЫ" (Фортис Девелопмент). sim=1.000 + ('Квартал Сюжеты', 'Екатеринбург', 63631, 'fuzzy', 1.000, false, + 'Audit 2026-05-17: sim=1.0 case-insensitive match (Фортис Девелопмент, Сданные)'), + + -- 28 deals/12m. "Архитектон. Декабристов,20" → domrf ЖК "Архитектон.Декабристов 20". sim=0.897 + ('Архитектон. Декабристов,20', 'Екатеринбург', 41762, 'fuzzy', 0.897, false, + 'Audit 2026-05-17: sim=0.897 to ЖК "Архитектон.Декабристов 20" (Корпорация Маяк, Сданные)'), + + -- 115 deals/12m. "Хрустальные ключи" → domrf ЖК "Хрустальные ключи" (ЛСР). sim=0.857 + ('Хрустальные ключи', 'Екатеринбург', 17471, 'fuzzy', 0.857, false, + 'Audit 2026-05-17: sim=0.857 to ЖК "Хрустальные ключи" (ЛСР, Сданные)'), + + -- 128 deals/12m. "Большой каретный" → domrf ЖК "Большой каретный". sim=0.850 + ('Большой каретный', 'Екатеринбург', 45859, 'fuzzy', 0.850, false, + 'Audit 2026-05-17: sim=0.850 to ЖК "Большой каретный" (СЗ НИЖНЕИСЕТСКИЙ ПРУД, Сданные)') + +ON CONFLICT (objective_complex_name, objective_group) DO NOTHING; + +-- ============================================================================= +-- MEDIUM confidence (sim 0.6-0.85) — manual review required +-- Uncomment and verify each INSERT before applying to prod. +-- ============================================================================= + +-- 207 deals/12m. "Зеленая горка" → domrf ЖК "Зеленая горка" (СЗ ДИККИТ, Сданные). sim=0.824 +-- INSERT INTO objective_complex_mapping (objective_complex_name, objective_group, domrf_obj_id, match_method, match_score, is_reviewed, note) VALUES +-- ('Зеленая горка', 'Екатеринбург', 59810, 'fuzzy', 0.824, false, 'sim=0.824 review: Зеленая горка (СЗ ДИККИТ)') ON CONFLICT DO NOTHING; + +-- 16 deals/12m. "Куйбышева 100" → domrf ЖК "Куйбышева 100" (СЗ ПРЕМЬЕРСТРОЙ, Сданные). sim=0.824 +-- INSERT INTO objective_complex_mapping (objective_complex_name, objective_group, domrf_obj_id, match_method, match_score, is_reviewed, note) VALUES +-- ('Куйбышева 100', 'Екатеринбург', 47419, 'fuzzy', 0.824, false, 'sim=0.824 review: Куйбышева 100 (СЗ ПРЕМЬЕРСТРОЙ)') ON CONFLICT DO NOTHING; + +-- 71 deals/12m. "Белый парус" → domrf ЖК "Белый парус" (Астра, Сданные). sim=0.800 +-- INSERT INTO objective_complex_mapping (objective_complex_name, objective_group, domrf_obj_id, match_method, match_score, is_reviewed, note) VALUES +-- ('Белый парус', 'Екатеринбург', 20814, 'fuzzy', 0.800, false, 'sim=0.800 review: Белый парус (Астра, Сданные)') ON CONFLICT DO NOTHING; + +-- 164 deals/12m. "Небосклоны" → domrf ЖК «Небосклоны» (Унистрой, Сданные). sim=0.786 +-- INSERT INTO objective_complex_mapping (objective_complex_name, objective_group, domrf_obj_id, match_method, match_score, is_reviewed, note) VALUES +-- ('Небосклоны', 'Екатеринбург', 55308, 'fuzzy', 0.786, false, 'sim=0.786 review: Небосклоны (Унистрой, Сданные)') ON CONFLICT DO NOTHING; + +-- 99 deals/12m. "Огни Исети" → domrf ЖК "Огни Исети" (СЗ СК УРАЛКОМПЛЕКТ, Сданные). sim=0.786 +-- INSERT INTO objective_complex_mapping (objective_complex_name, objective_group, domrf_obj_id, match_method, match_score, is_reviewed, note) VALUES +-- ('Огни Исети', 'Екатеринбург', 46431, 'fuzzy', 0.786, false, 'sim=0.786 review: Огни Исети (СЗ СК УРАЛКОМПЛЕКТ, Сданные)') ON CONFLICT DO NOTHING; + +-- 234 deals/12m. "Абрикос" → domrf ЖК Абрикос (Стройтэк, Сданные). sim=0.727 +-- INSERT INTO objective_complex_mapping (objective_complex_name, objective_group, domrf_obj_id, match_method, match_score, is_reviewed, note) VALUES +-- ('Абрикос', 'Екатеринбург', 50239, 'fuzzy', 0.727, false, 'sim=0.727 review: Абрикос (Стройтэк, Сданные)') ON CONFLICT DO NOTHING; + +-- 33 deals/12m. "Estelle" → domrf ЖК Estelle (СЗ ИНВЕСТСТРОЙ, Сданные). sim=0.727 +-- INSERT INTO objective_complex_mapping (objective_complex_name, objective_group, domrf_obj_id, match_method, match_score, is_reviewed, note) VALUES +-- ('Estelle', 'Екатеринбург', 53948, 'fuzzy', 0.727, false, 'sim=0.727 review: Estelle (СЗ ИНВЕСТСТРОЙ, Сданные)') ON CONFLICT DO NOTHING; + +-- 25 deals/12m. "Мирлеон" → domrf ЖК "Мирлеон" (Виктория, Сданные). sim=0.727 +-- INSERT INTO objective_complex_mapping (objective_complex_name, objective_group, domrf_obj_id, match_method, match_score, is_reviewed, note) VALUES +-- ('Мирлеон', 'Екатеринбург', 62711, 'fuzzy', 0.727, false, 'sim=0.727 review: Мирлеон (Виктория, Сданные)') ON CONFLICT DO NOTHING; + +-- 318 deals/12m. "Первый" → domrf ЖК "ПЕРВЫЙ" (ИКАР, Сданные). sim=0.700 +-- CAUTION: 103_seed note said "Первый" belongs to ИКАР, not PRINZIP. Mapping is valid for +-- velocity purposes (not PRINZIP-specific). Verify obj_id 52307 is the right corpus. +-- INSERT INTO objective_complex_mapping (objective_complex_name, objective_group, domrf_obj_id, match_method, match_score, is_reviewed, note) VALUES +-- ('Первый', 'Екатеринбург', 52307, 'fuzzy', 0.700, false, 'sim=0.700 review: ЖК ПЕРВЫЙ (ИКАР, Сданные). Not PRINZIP — skipped in 103, valid here.') ON CONFLICT DO NOTHING; + +-- 107 deals/12m. "Мохито" → domrf ЖК Мохито (Стройтэк, Сданные). sim=0.700 +-- INSERT INTO objective_complex_mapping (objective_complex_name, objective_group, domrf_obj_id, match_method, match_score, is_reviewed, note) VALUES +-- ('Мохито', 'Екатеринбург', 41887, 'fuzzy', 0.700, false, 'sim=0.700 review: Мохито (Стройтэк, Сданные)') ON CONFLICT DO NOTHING; + +-- 215 deals/12m. "на Ясной" → domrf "Дом на Ясной" (Практика, Сданные). sim=0.692 +-- CAUTION: short name "на Ясной" may collide. Verify project is the same as "Дом на Ясной". +-- INSERT INTO objective_complex_mapping (objective_complex_name, objective_group, domrf_obj_id, match_method, match_score, is_reviewed, note) VALUES +-- ('на Ясной', 'Екатеринбург', 51568, 'fuzzy', 0.692, false, 'sim=0.692 review: Дом на Ясной (Практика, Сданные). Short name match, verify.') ON CONFLICT DO NOTHING; + +-- 109 deals/12m. "Изумрудный бор 2.0" → domrf ЖР Изумрудный Бор (УГМК-ЗАСТРОЙЩИК). sim=0.682 +-- NB: "Изумрудный бор" already mapped to obj_id 67678. "2.0" suffix → different phase/corp. +-- INSERT INTO objective_complex_mapping (objective_complex_name, objective_group, domrf_obj_id, match_method, match_score, is_reviewed, note) VALUES +-- ('Изумрудный бор 2.0', 'Екатеринбург', 45017, 'fuzzy', 0.682, false, 'sim=0.682 review: ЖР Изумрудный Бор (УГМК, Сданные). "2.0" suffix — different from mapped obj_id 67678.') ON CONFLICT DO NOTHING; + +-- 317 deals/12m. "СТАРТ" → domrf ЖК СТАРТ (ТЭН, Сданные). sim=0.667 +-- INSERT INTO objective_complex_mapping (objective_complex_name, objective_group, domrf_obj_id, match_method, match_score, is_reviewed, note) VALUES +-- ('СТАРТ', 'Екатеринбург', 53947, 'fuzzy', 0.667, false, 'sim=0.667 review: ЖК СТАРТ (ТЭН, Сданные)') ON CONFLICT DO NOTHING; + +-- 101 deals/12m. "Горки" → domrf ЖК "Горки" (СЗ ЖИЛСТРОЙ, Сданные). sim=0.667 +-- INSERT INTO objective_complex_mapping (objective_complex_name, objective_group, domrf_obj_id, match_method, match_score, is_reviewed, note) VALUES +-- ('Горки', 'Екатеринбург', 45653, 'fuzzy', 0.667, false, 'sim=0.667 review: Горки (СЗ ЖИЛСТРОЙ, Сданные)') ON CONFLICT DO NOTHING; + +-- 78 deals/12m. "Милый дом (BAZA)" → domrf Дом "Милый дом" (BAZA Development, Сданные). sim=0.667 +-- INSERT INTO objective_complex_mapping (objective_complex_name, objective_group, domrf_obj_id, match_method, match_score, is_reviewed, note) VALUES +-- ('Милый дом (BAZA)', 'Екатеринбург', 54751, 'fuzzy', 0.667, false, 'sim=0.667 review: Дом Милый дом (BAZA Development, Сданные)') ON CONFLICT DO NOTHING; + +-- 237 deals/12m. "Основинские кварталы" → domrf "Паркинг ЖК Основинские кварталы" (ТЭН). sim=0.656 +-- CAUTION: domrf match is a PARKING object, not residential. Verify correct obj_id manually. +-- INSERT INTO objective_complex_mapping (objective_complex_name, objective_group, domrf_obj_id, match_method, match_score, is_reviewed, note) VALUES +-- ('Основинские кварталы', 'Екатеринбург', 55182, 'fuzzy', 0.656, false, 'sim=0.656 CAUTION: domrf match is Паркинг object. Find correct residential obj_id.') ON CONFLICT DO NOTHING; + +-- 420 deals/12m. "Южные кварталы (Брусника)" → domrf "Южные кварталы" (Брусника, Строящиеся). sim=0.625 +-- Multiple Брусника Южные кварталы corpora found: 59867, 70970, 67220 (Строящиеся), 45857, etc. +-- Need to pick the primary active one. obj_id 59867 suggested (Строящиеся, latest). +-- INSERT INTO objective_complex_mapping (objective_complex_name, objective_group, domrf_obj_id, match_method, match_score, is_reviewed, note) VALUES +-- ('Южные кварталы (Брусника)', 'Екатеринбург', 59867, 'fuzzy', 0.625, false, 'sim=0.625 review: Юж.кварталы Брусника (Строящиеся). Multiple corpora — verify primary obj_id.') ON CONFLICT DO NOTHING; + +-- 51 deals/12m. "На Некрасова" → domrf "Квартал на Некрасова" (Брусника, Сданные). sim=0.600 +-- INSERT INTO objective_complex_mapping (objective_complex_name, objective_group, domrf_obj_id, match_method, match_score, is_reviewed, note) VALUES +-- ('На Некрасова', 'Екатеринбург', 40819, 'fuzzy', 0.600, false, 'sim=0.600 review: Квартал на Некрасова (Брусника, Сданные)') ON CONFLICT DO NOTHING; + +-- 182 deals/12m. "Уральский" → domrf "ЖК Уральский сад" (Страна Девелопмент, Строящиеся). sim=0.588 +-- CAUTION: short generic name "Уральский" — may not be "Уральский сад". Verify. +-- INSERT INTO objective_complex_mapping (objective_complex_name, objective_group, domrf_obj_id, match_method, match_score, is_reviewed, note) VALUES +-- ('Уральский', 'Екатеринбург', 65611, 'fuzzy', 0.588, false, 'sim=0.588 review: ЖК Уральский сад (Страна Девелопмент). Verify "Уральский" = "Уральский сад".') ON CONFLICT DO NOTHING; + +-- 154 deals/12m. "Екатерининский парк" → domrf "Концепт-проект Екатерининский парк" (ТЭН). sim=0.588 +-- INSERT INTO objective_complex_mapping (objective_complex_name, objective_group, domrf_obj_id, match_method, match_score, is_reviewed, note) VALUES +-- ('Екатерининский парк', 'Екатеринбург', 47004, 'fuzzy', 0.588, false, 'sim=0.588 review: Концепт-проект Екатерининский парк (ТЭН, Сданные)') ON CONFLICT DO NOTHING; + +-- 97 deals/12m. "Башня Времени" → domrf ЖД "Башня Времени" (Фортис Девелопмент, Строящиеся). sim=0.583 +-- INSERT INTO objective_complex_mapping (objective_complex_name, objective_group, domrf_obj_id, match_method, match_score, is_reviewed, note) VALUES +-- ('Башня Времени', 'Екатеринбург', 60557, 'fuzzy', 0.583, false, 'sim=0.583 review: Башня Времени (Фортис Девелопмент, Строящиеся)') ON CONFLICT DO NOTHING; + +-- 48 deals/12m. "Re:Volution Towers (НКС-Девелопмент)" → domrf RE.VOLUTION TOWERS (НКС-Девелопмент, Строящиеся). sim=0.543 +-- Developer name matches exactly. High-confidence despite lower sim (long name with punctuation diff). +-- INSERT INTO objective_complex_mapping (objective_complex_name, objective_group, domrf_obj_id, match_method, match_score, is_reviewed, note) VALUES +-- ('Re:Volution Towers (НКС-Девелопмент)', 'Екатеринбург', 39106, 'fuzzy', 0.543, false, 'sim=0.543 review: RE.VOLUTION TOWERS (НКС-Девелопмент, Строящиеся). Dev name matches.') ON CONFLICT DO NOTHING; + +-- 146 deals/12m. "Квартал Моменты" → domrf "Квартал Моменты на Космонавтов" (TETRIS GROUP, Строящиеся). sim=0.533 +-- INSERT INTO objective_complex_mapping (objective_complex_name, objective_group, domrf_obj_id, match_method, match_score, is_reviewed, note) VALUES +-- ('Квартал Моменты', 'Екатеринбург', 64556, 'fuzzy', 0.533, false, 'sim=0.533 review: Квартал Моменты на Космонавтов (TETRIS GROUP, Строящиеся)') ON CONFLICT DO NOTHING; + +-- 914 deals/12m. "Новокольцовский" → domrf Жилой р-н Новокольцовский (Синара-Девелопмент). sim=0.485 +-- Large multi-corpus project (6 corpora found). obj_id 59217 = Строящиеся (4 оч. ЖД5). +-- Pick most active Строящиеся corpus. Needs manual verification of primary obj_id. +-- INSERT INTO objective_complex_mapping (objective_complex_name, objective_group, domrf_obj_id, match_method, match_score, is_reviewed, note) VALUES +-- ('Новокольцовский', 'Екатеринбург', 59217, 'fuzzy', 0.485, false, 'sim=0.485 review: Новокольцовский (Синара-Девелопмент). 914 deals/12m, large multi-corpus. Verify obj_id.') ON CONFLICT DO NOTHING; + +-- ============================================================================= +-- LOW / ORPHAN — no domrf match found (sim < 0.4 or wrong project) +-- These Objective project_names have no reliable domrf_kn_objects counterpart. +-- velocity.py will show velocity=0 for them until matched or until rosreestr fallback is used. +-- ============================================================================= +-- Новокольцовский (if MEDIUM above rejected) +-- Солнечный (Эталон) — sim=0.393 to wrong developer. Эталон has obj 68595 (Строящиеся) but +-- name only "Район Солнечный" — ambiguous. Orphan for now. +-- Forum Park — no EKB match in domrf_kn_objects at all. +-- Береговой — sim=0.370, "Семейный квартал Береговой" (Талан). Below threshold. +-- Лайв — sim=0.385 to Квартал Лайв (Атомстройкомплекс). Below threshold. +-- Тетро — sim=0.375. Below threshold. +-- Московский квартал (Земельный ресурс) — best match is sданные old project, ambiguous. +-- Река. Дом на набережной — no reliable match. +-- Русь, Русь XXI — no match found. +-- Тактика, Новация, Утес, Добрый, К100, 9 Космонавтов, Орбита на Титова, +-- Тетро, Курчатов, River Park (Кургансельстрой), Современник, Новый ВИЗ (Астра-Девелопмент), +-- Самоцветы, Белый парус (if MEDIUM rejected), А+, Центральный Парк (Екатеринбург), +-- 7 Ключей, GRAFIT, 7Я, Тишина 2, Грин Гарден, Дружный, Дружный-2, Соната Парк, +-- Проспект Мира. Компаунд, Мир Труд Май, Викулов, Астон.Время, Голос Заря (Голос), +-- Тихий центр, Федерация, Теплые кварталы (Паритет Девелопмент), +-- Теплые кварталы (Призвание), Уралмаш, Цветочный, Каменные палатки (BAZA), +-- На Некрасова (if MEDIUM rejected), ЖК на Ботанике, Традиции (Квартал на Турбинке), +-- NOVA Park, ПТ На Ежевичной, Пушкин, DOM 7, Аура, Кварталы конструктивизма (already HIGH), +-- RedRock, Шолохов, 19/05, М1, Атмосфера, Рио-3, 4Ю, Северный Химмаш, Олимпика, +-- АЛЛЕГРО, Форум Сити, Литературный, Просторы, Рижский, Ботаника LIFE, Discovery Residence, +-- Дом на Бардина — all orphans, no action. + +COMMIT; diff --git a/data/sql/105_add_sales_started_flag.sql b/data/sql/105_add_sales_started_flag.sql new file mode 100644 index 00000000..15a8abe5 --- /dev/null +++ b/data/sql/105_add_sales_started_flag.sql @@ -0,0 +1,44 @@ +-- 105_add_sales_started_flag.sql +-- +-- Context: SF FixList #18 (Wave 3) +-- Purpose: Expose is_sales_started flag on domrf_kn_objects so SiteFinder can +-- distinguish "Строящиеся с открытыми продажами" from pre-sale buildings +-- (those where domrf_kn_flats has no rows for that obj_id yet). +-- +-- Design decision: VIEW instead of GENERATED ALWAYS AS column. +-- - PostgreSQL GENERATED columns do not support subqueries (only immutable expressions). +-- - A VIEW is idempotent (CREATE OR REPLACE), needs no backfill, and stays fresh. +-- - Consumers query v_domrf_objects_with_sales in place of domrf_kn_objects where +-- the flag is needed; the base table remains unchanged for scrapers / ON CONFLICT. +-- +-- Audit (2026-05-17, region_cd=66, site_status='Строящиеся'): +-- total_buildings: 1322 +-- with_flats (is_sales_started=TRUE): 1247 (94.3%) +-- without_flats (pre-sale / no flats): 75 ( 5.7%) +-- +-- Dependencies: domrf_kn_objects, domrf_kn_flats +-- Deploy order: standalone, no existing deps to drop first +-- Idempotent: yes (CREATE OR REPLACE VIEW) + +BEGIN; + +CREATE OR REPLACE VIEW v_domrf_objects_with_sales AS +SELECT + o.*, + EXISTS ( + SELECT 1 + FROM domrf_kn_flats f + WHERE f.obj_id = o.obj_id + ) AS is_sales_started +FROM domrf_kn_objects o; + +COMMENT ON VIEW v_domrf_objects_with_sales IS + 'domrf_kn_objects + is_sales_started (bool). ' + 'TRUE когда в domrf_kn_flats есть хотя бы одна запись для данного obj_id. ' + 'Позволяет отличать строящиеся дома с открытыми продажами от pre-sale объектов.'; + +COMMENT ON COLUMN v_domrf_objects_with_sales.is_sales_started IS + 'Derived: TRUE если EXISTS (SELECT 1 FROM domrf_kn_flats WHERE obj_id = o.obj_id). ' + 'Pre-sale = FALSE (квартиры ещё не выложены на domrf.ru).'; + +COMMIT; diff --git a/data/sql/106_backfill_obj_class_from_ai_description.sql b/data/sql/106_backfill_obj_class_from_ai_description.sql new file mode 100644 index 00000000..72939685 --- /dev/null +++ b/data/sql/106_backfill_obj_class_from_ai_description.sql @@ -0,0 +1,68 @@ +-- 105_backfill_obj_class_from_ai_description.sql +-- +-- Backfill obj_class for existing domrf_kn_objects rows using aiDescription +-- from the raw domrf_raw_endpoints payloads. +-- +-- Root cause: DOM.РФ /kn/object list endpoint never returns `objClass` field +-- (absent from API payload). Class text is embedded in `aiDescription`. +-- Fix in scraper (domrf_kn.py _norm_object) extracts class on next sweep. +-- This script retroactively populates existing rows from stored raw payloads. +-- +-- Coverage after backfill (region 66, snapshot 2026-05-05): +-- Комфорт: ~782, Бизнес: ~88, Премиум: ~13, Элит: ~12 +-- class_not_found (нежилое / нет AI-описания): ~621 — remains NULL (expected) +-- +-- Idempotent: only updates WHERE obj_class IS NULL. Safe to re-run. + +BEGIN; + +UPDATE domrf_kn_objects AS o +SET obj_class = CASE + WHEN raw_obj ->> 'aiDescription' ~* 'элит' THEN 'Элит' + WHEN raw_obj ->> 'aiDescription' ~* 'бизнес' THEN 'Бизнес' + WHEN raw_obj ->> 'aiDescription' ~* 'премиум' THEN 'Премиум' + WHEN raw_obj ->> 'aiDescription' ~* 'комфорт' THEN 'Комфорт' + WHEN raw_obj ->> 'aiDescription' ~* 'стандарт|эконом' THEN 'Стандарт' + ELSE NULL +END +FROM ( + -- Unnest the raw JSON array payload; each element is one object row. + SELECT + (raw_obj ->> 'objId')::bigint AS obj_id, + e.snapshot_date, + raw_obj + FROM domrf_raw_endpoints AS e, + jsonb_array_elements(e.payload) AS raw_obj + WHERE e.section = 'kn_api' + AND jsonb_typeof(e.payload) = 'array' + AND raw_obj ->> 'aiDescription' IS NOT NULL +) AS src +WHERE o.obj_id = src.obj_id + AND o.snapshot_date = src.snapshot_date + AND o.obj_class IS NULL + -- Only update when we can actually extract a class + AND ( + src.raw_obj ->> 'aiDescription' ~* 'элит' + OR src.raw_obj ->> 'aiDescription' ~* 'бизнес' + OR src.raw_obj ->> 'aiDescription' ~* 'премиум' + OR src.raw_obj ->> 'aiDescription' ~* 'комфорт' + OR src.raw_obj ->> 'aiDescription' ~* 'стандарт|эконом' + ); + +-- Report result +DO $$ +DECLARE + v_total bigint; + v_null bigint; + v_filled bigint; +BEGIN + SELECT COUNT(*), COUNT(*) FILTER (WHERE obj_class IS NULL), + COUNT(*) FILTER (WHERE obj_class IS NOT NULL) + INTO v_total, v_null, v_filled + FROM domrf_kn_objects; + + RAISE NOTICE 'domrf_kn_objects after backfill: total=% filled=% still_null=%', + v_total, v_filled, v_null; +END$$; + +COMMIT; diff --git a/data/sql/107_backfill_price_per_m2.sql b/data/sql/107_backfill_price_per_m2.sql new file mode 100644 index 00000000..7fd9946c --- /dev/null +++ b/data/sql/107_backfill_price_per_m2.sql @@ -0,0 +1,24 @@ +-- 107_backfill_price_per_m2.sql +-- Backfill price_per_m2 = ROUND(price_rub / total_area, 2) for rows where +-- price_rub is known but price_per_m2 was not stored by the scraper. +-- +-- Context (SF FixList #16): +-- domrf_kn.py _norm_flat() now derives price_per_m2 on ingest. +-- This one-time UPDATE covers historical rows scraped before the fix. +-- For Малевич (obj_id=64701) and the wider dataset, both price_rub AND +-- price_per_m2 are NULL (DOM.RF /portal/table API does not return per-flat +-- pricing for most objects). Those rows are unaffected by this UPDATE. +-- Only rows where price_rub IS NOT NULL benefit — currently ~849 rows. +-- +-- Safe to re-run: idempotent (WHERE price_per_m2 IS NULL). + +BEGIN; + +UPDATE domrf_kn_flats +SET price_per_m2 = ROUND((price_rub / total_area)::numeric, 2) +WHERE price_per_m2 IS NULL + AND price_rub IS NOT NULL + AND total_area IS NOT NULL + AND total_area > 0; + +COMMIT; diff --git a/data/sql/108_bug22a_kn_missing_fields_audit.sql b/data/sql/108_bug22a_kn_missing_fields_audit.sql new file mode 100644 index 00000000..246f9b89 --- /dev/null +++ b/data/sql/108_bug22a_kn_missing_fields_audit.sql @@ -0,0 +1,85 @@ +-- 108_bug22a_kn_missing_fields_audit.sql +-- +-- Bug 22a audit: kn-API payload vs domrf_kn_objects / domrf_kn_flats mapping. +-- +-- FINDINGS (payload audit on obj_id=65136, all 1516 objects in kn_object_place_66, +-- snapshot 2026-05-05): +-- +-- Object-level fields (/kn/object list endpoint): +-- wallType — 0/1516 objects have key in payload → API never returns it. +-- aiDescription mentions wall material as free text but no +-- clean extraction without NLP. No backfill possible. +-- energyEff — 0/1516 objects have key in payload → API never returns it. +-- No source for backfill. +-- objClass — 0/1516 objects have key in payload → API never returns it. +-- FIXED in migration 106 (extract from aiDescription) + SF#15 +-- scraper fix. Coverage after 106: 1806/4548 filled. +-- +-- Flat-level fields (/portal-kn/api/sales/portal/table endpoint): +-- number → flat_number: mapped in _norm_flat() via _g(row, "number", "flatNumber"). +-- Confirmed present in payload for active-sale objects. +-- livingArea → living_area: mapped in _norm_flat() via _g(row, "livingArea"). +-- Confirmed present in payload for active-sale objects. +-- +-- Root cause for 0% flat_number/living_area in domrf_kn_flats: +-- UPSERT_FLAT_SQL ON CONFLICT (id, snapshot_date) DO UPDATE SET did NOT include +-- flat_number, living_area, is_studio, total_area, rooms, floor, num_floors. +-- Rows first inserted from older scrape (hash-id, no direct numeric id → NULL fields) +-- were never updated on subsequent scrapes. +-- FIX: domrf_kn.py UPSERT_FLAT_SQL now uses COALESCE for these fields. +-- Coverage will improve on next scrape run (no stored flat-level raw → no backfill). +-- +-- UPSERT_OBJECT_SQL ON CONFLICT also missed wall_type / energy_eff updates. +-- FIX: added COALESCE for wall_type and energy_eff in ON CONFLICT clause. +-- +-- All columns already exist — no DDL changes needed. This migration is a no-op +-- idempotent guard; the real fix is in backend/app/services/scrapers/domrf_kn.py. +-- +-- Idempotent: safe to re-run. + +BEGIN; + +-- Confirm columns exist (no-op if already present — guards against accidental DROP) +ALTER TABLE domrf_kn_objects + ADD COLUMN IF NOT EXISTS wall_type TEXT, + ADD COLUMN IF NOT EXISTS energy_eff TEXT; + +ALTER TABLE domrf_kn_flats + ADD COLUMN IF NOT EXISTS flat_number TEXT, + ADD COLUMN IF NOT EXISTS living_area NUMERIC(10, 2); + +-- Coverage check (informational — does not fail migration) +DO $$ +DECLARE + obj_total bigint; + obj_wall bigint; + obj_energy bigint; + obj_class_fill bigint; + flat_total bigint; + flat_number bigint; + flat_living bigint; +BEGIN + SELECT COUNT(*), + COUNT(*) FILTER (WHERE wall_type IS NOT NULL), + COUNT(*) FILTER (WHERE energy_eff IS NOT NULL), + COUNT(*) FILTER (WHERE obj_class IS NOT NULL) + INTO obj_total, obj_wall, obj_energy, obj_class_fill + FROM domrf_kn_objects WHERE region_cd = 66; + + SELECT COUNT(*), + COUNT(*) FILTER (WHERE f.flat_number IS NOT NULL), + COUNT(*) FILTER (WHERE f.living_area IS NOT NULL) + INTO flat_total, flat_number, flat_living + FROM domrf_kn_flats f + JOIN domrf_kn_objects o ON o.obj_id = f.obj_id + WHERE o.region_cd = 66; + + RAISE NOTICE + 'bug-22a coverage (region_cd=66):' + ' objects: total=% obj_class=% wall_type=% energy_eff=%' + ' | flats: total=% flat_number=% living_area=%', + obj_total, obj_class_fill, obj_wall, obj_energy, + flat_total, flat_number, flat_living; +END$$; + +COMMIT; diff --git a/data/sql/110_22j_dedup_infrastructure.sql b/data/sql/110_22j_dedup_infrastructure.sql new file mode 100644 index 00000000..5b6d1ba4 --- /dev/null +++ b/data/sql/110_22j_dedup_infrastructure.sql @@ -0,0 +1,40 @@ +-- 100_22j_dedup_infrastructure.sql +-- Context: domrf_kn_infrastructure has ~496k rows but only ~229k unique by +-- (obj_id, poi_category, poi_name, poi_address) — ~267k duplicates. +-- Root cause: old scraper re-inserts same POIs on every run with new +-- snapshot_date, and the old UNIQUE key includes snapshot_date so +-- dedup was per-day only. +-- Fix: +-- 1. One-time cleanup: keep lowest ctid per dedup key, delete the rest. +-- 2. Drop old UNIQUE constraint (includes snapshot_date — wrong granularity). +-- 3. Add new UNIQUE (obj_id, poi_category, poi_name, poi_address) +-- with NULLS NOT DISTINCT (PG16) so NULL values are treated as equal. +-- Dependencies: none (no views reference this table by constraint name) +-- Deploy order: apply this SQL first, then deploy backend with ON CONFLICT fix. +-- Idempotent: yes — constraint uses IF NOT EXISTS equivalent via DO $$ block; +-- DELETE is safe to re-run (no rows left to delete after first run). +-- Issue: #297 / task 22j +-- Applied: 2026-05-17 + +BEGIN; + +-- Step 1: one-time dedup — keep min(ctid) per logical key +DELETE FROM domrf_kn_infrastructure a +USING domrf_kn_infrastructure b +WHERE a.ctid > b.ctid + AND a.obj_id = b.obj_id + AND COALESCE(a.poi_category, '') = COALESCE(b.poi_category, '') + AND COALESCE(a.poi_name, '') = COALESCE(b.poi_name, '') + AND COALESCE(a.poi_address, '') = COALESCE(b.poi_address, ''); + +-- Step 2: drop old unique constraint (wrong key — includes snapshot_date) +ALTER TABLE domrf_kn_infrastructure + DROP CONSTRAINT IF EXISTS domrf_kn_infrastructure_obj_id_poi_name_poi_lat_poi_lon_sna_key; + +-- Step 3: add new unique constraint with NULLS NOT DISTINCT (PG15+) +-- so NULL poi_category / poi_address are treated as equal (not distinct) +ALTER TABLE domrf_kn_infrastructure + ADD CONSTRAINT uq_infra_dedupe + UNIQUE NULLS NOT DISTINCT (obj_id, poi_category, poi_name, poi_address); + +COMMIT; diff --git a/data/sql/111_22f_domrf_obj_checks.sql b/data/sql/111_22f_domrf_obj_checks.sql new file mode 100644 index 00000000..d5b413d0 --- /dev/null +++ b/data/sql/111_22f_domrf_obj_checks.sql @@ -0,0 +1,49 @@ +-- Migration: 111_22f_domrf_obj_checks.sql +-- Context: Issue #297 sub-task 22f — "Проверено на наш.дом.рф" 6 checks per obj +-- Dependencies: domrf_kn_objects (obj_id values come from there; no FK enforced — obj_id +-- is a bigint key, not a FK, to allow scraping checks before obj row exists) +-- Deploy order: standalone — no prior migration required +-- Idempotent: yes (IF NOT EXISTS throughout) + +BEGIN; + +CREATE TABLE IF NOT EXISTS domrf_obj_checks ( + obj_id bigint NOT NULL, + check_type text NOT NULL, + -- check_type values: no_problems / docs / timing / photos / bankruptcy / declaration + passed bool NOT NULL, + checked_at timestamptz NOT NULL DEFAULT NOW(), + scraped_at timestamptz NOT NULL DEFAULT NOW(), + PRIMARY KEY (obj_id, check_type) +); + +CREATE INDEX IF NOT EXISTS domrf_obj_checks_obj_idx + ON domrf_obj_checks (obj_id); + +COMMENT ON TABLE domrf_obj_checks IS + '6 проверок «Проверено на наш.дом.рф» per obj_id: ' + 'no_problems, docs, timing, photos, bankruptcy, declaration. ' + 'Issue #297 sub-task 22f.'; + +COMMENT ON COLUMN domrf_obj_checks.obj_id IS + 'DOM.РФ объект (кн-объект); из domrf_kn_objects.obj_id.'; + +COMMENT ON COLUMN domrf_obj_checks.check_type IS + 'Тип проверки. Допустимые значения: ' + 'no_problems — у застройщика нет проблемных объектов; ' + 'docs — опубликован полный комплект документов; ' + 'timing — сроки строительства соблюдены; ' + 'photos — актуальные фото хода строительства; ' + 'bankruptcy — застройщик не банкрот; ' + 'declaration — проектная декларация обновлена.'; + +COMMENT ON COLUMN domrf_obj_checks.passed IS + 'TRUE = чек пройден (зелёная галочка на наш.дом.рф).'; + +COMMENT ON COLUMN domrf_obj_checks.checked_at IS + 'Дата/время актуальности чека по данным DOM.РФ (берётся из payload).'; + +COMMENT ON COLUMN domrf_obj_checks.scraped_at IS + 'Дата/время последнего scrape, когда строка была записана/обновлена.'; + +COMMIT; diff --git a/data/sql/112_22i_domrf_documents.sql b/data/sql/112_22i_domrf_documents.sql new file mode 100644 index 00000000..e6a65c99 --- /dev/null +++ b/data/sql/112_22i_domrf_documents.sql @@ -0,0 +1,38 @@ +-- Migration 100: domrf_kn_documents — PDF documents scraped from DOM.РФ object pages +-- Issue #297, sub-task 22i. Phase 3 new tables. +-- Stores declarations, permits, project docs, reports, misc PDFs per obj_id. + +BEGIN; + +CREATE TABLE IF NOT EXISTS domrf_kn_documents ( + id bigserial PRIMARY KEY, + obj_id bigint NOT NULL, + doc_type text NOT NULL, -- декларация/разрешение/проектная/отчётность/прочее + doc_num text, + posted_at date, + file_url text NOT NULL, + size_bytes bigint, + local_path text, + downloaded_at timestamptz, + scraped_at timestamptz NOT NULL DEFAULT NOW(), + UNIQUE (obj_id, doc_type, doc_num, file_url) +); + +CREATE INDEX IF NOT EXISTS domrf_kn_documents_obj_type_idx + ON domrf_kn_documents (obj_id, doc_type); + +COMMENT ON TABLE domrf_kn_documents IS + 'PDF docs scraped from DOM.RF obj page. Issue #297 22i.'; + +COMMENT ON COLUMN domrf_kn_documents.doc_type IS + 'Canonical type: декларация / разрешение / проектная / отчётность / прочее'; +COMMENT ON COLUMN domrf_kn_documents.doc_num IS + 'Document number as shown on DOM.RF (e.g. permit number, declaration number)'; +COMMENT ON COLUMN domrf_kn_documents.file_url IS + 'Full URL to the PDF on наш.дом.рф (CDN or api/ext/file/...)'; +COMMENT ON COLUMN domrf_kn_documents.local_path IS + 'Relative path inside data/raw/domrf_docs/ after download (populated by future task)'; +COMMENT ON COLUMN domrf_kn_documents.downloaded_at IS + 'Timestamp when PDF was successfully downloaded locally (NULL = not yet downloaded)'; + +COMMIT; diff --git a/data/sql/113_22begh_kn_schema_extension.sql b/data/sql/113_22begh_kn_schema_extension.sql new file mode 100644 index 00000000..c6313c8f --- /dev/null +++ b/data/sql/113_22begh_kn_schema_extension.sql @@ -0,0 +1,161 @@ +-- Migration 112: schema extension for issues #297 sub-tasks 22b + 22e + 22g + 22h +-- Adds 6 flat-level cols, 30 object-level cols (incl. metro_top3), 4 developer metrics. +-- +-- Payload audit (obj=65136, 2026-05-17): +-- kn_api payload keys: objId, hobjId, objCommercNm, objAddr, shortAddr, rpdRegionCd, +-- developer{devId,devInn,orgForm,fullName,groupName,shortName,companyGroup}, +-- rpdNum, objPublDt, objStatus, siteStatus, objFloorMax, objFloorMin, objPriceAVG, +-- problemFlag, rpdRegionCd, objSquareLiving, objElemLivingCnt, objGreenHouseFlg, +-- objGuarantyEscrowFlg, objReady100PercDt, hobjRenderPhotoUrl, aiDescription, +-- freeFlatsInfo{priceMin, numberFlats}, residentialBuildings, +-- metro{id, line, name, time, color, colors, isWalk}, latitude, longitude, pdId, buildType. +-- +-- flat-level (portal/table): flatId, odsId, elemId, type, number, isStudio, +-- totalArea, livingArea, rooms, status, price, pricePerSquareMeter, numberFloors, +-- _objId(injected), _floor(injected), _entrance(injected as entranceNumber). +-- +-- Fields that map NOW (from existing kn-API payload): +-- 22b: section_no (from _entrance/entranceNumber injected by _flatten_table) +-- 22e: section_count (residentialBuildings), project_declaration_num (rpdNum), +-- project_published_at (objPublDt), metro_nearest_name + metro_nearest_walk_minutes +-- (metro.name + metro.time), metro_top3 (metro wrapped as single-element array), +-- dev_group_name (developer.groupName), price_min_rub (freeFlatsInfo.priceMin) +-- +-- Fields remaining NULL until catalog scraper 22d: +-- finishing_type, ceiling_height_m (flat), key_handover_dt, catalog_updated_at, +-- catalog_url_hash, first_floor_type, elevators_passenger_count, elevators_cargo_count, +-- parking_total_slots, guest_parking_inside_count, guest_parking_outside_count, +-- ceiling_height_m (obj), finishing_variants_count, has_free_planning, +-- playground_kids_count, playground_sport_count, has_bike_paths, trash_areas_count, +-- has_ramp, has_low_platforms, has_wheelchair_lift, flat_area_min, flat_area_max, +-- price_max_rub, price_per_m2_min, price_per_m2_max, parking_provision_pct, +-- avg_flat_area_m2, domrf_score_location, domrf_score_transport, +-- domrf_score_infrastructure, houses_building, houses_building_delayed, +-- houses_delivered, houses_delivered_delayed. + +BEGIN; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 22b — domrf_kn_flats +6 columns +-- ───────────────────────────────────────────────────────────────────────────── +COMMENT ON TABLE domrf_kn_flats IS 'Per-flat catalogue from DOM.RF kn-API portal/table endpoint'; + +ALTER TABLE domrf_kn_flats + ADD COLUMN IF NOT EXISTS section_no int, + ADD COLUMN IF NOT EXISTS finishing_type text, + ADD COLUMN IF NOT EXISTS ceiling_height_m numeric(3,2), + ADD COLUMN IF NOT EXISTS key_handover_dt date, + ADD COLUMN IF NOT EXISTS catalog_updated_at date, + ADD COLUMN IF NOT EXISTS catalog_url_hash text; + +COMMENT ON COLUMN domrf_kn_flats.section_no IS '22b: подъезд из entrance.entranceNumber (portal/table)'; +COMMENT ON COLUMN domrf_kn_flats.finishing_type IS '22b/22d: тип отделки — заполнит catalog scraper (22d)'; +COMMENT ON COLUMN domrf_kn_flats.ceiling_height_m IS '22b/22d: высота потолков — заполнит catalog scraper (22d)'; +COMMENT ON COLUMN domrf_kn_flats.key_handover_dt IS '22b/22d: дата выдачи ключей — заполнит catalog scraper (22d)'; +COMMENT ON COLUMN domrf_kn_flats.catalog_updated_at IS '22b/22d: дата обновления инфо на catalog-странице (22d)'; +COMMENT ON COLUMN domrf_kn_flats.catalog_url_hash IS '22b/22d: hash-часть URL квартиры на каталоге dom.rf (22d)'; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 22e + 22h — domrf_kn_objects +30 columns (incl. metro_top3) +-- ───────────────────────────────────────────────────────────────────────────── +COMMENT ON TABLE domrf_kn_objects IS 'Per-object (ЖК) catalogue from DOM.RF kn-API list endpoint'; + +-- Building specs (8) +ALTER TABLE domrf_kn_objects + ADD COLUMN IF NOT EXISTS first_floor_type text, + ADD COLUMN IF NOT EXISTS section_count int, + ADD COLUMN IF NOT EXISTS elevators_passenger_count int, + ADD COLUMN IF NOT EXISTS elevators_cargo_count int, + ADD COLUMN IF NOT EXISTS parking_total_slots int, + ADD COLUMN IF NOT EXISTS guest_parking_inside_count int, + ADD COLUMN IF NOT EXISTS guest_parking_outside_count int, + ADD COLUMN IF NOT EXISTS ceiling_height_m numeric(3,2); + +-- Apartment summary (3) +ALTER TABLE domrf_kn_objects + ADD COLUMN IF NOT EXISTS finishing_variants_count int, + ADD COLUMN IF NOT EXISTS has_free_planning bool, + ADD COLUMN IF NOT EXISTS avg_flat_area_m2 numeric(5,1); + +-- Yard infrastructure (4) +ALTER TABLE domrf_kn_objects + ADD COLUMN IF NOT EXISTS playground_kids_count int, + ADD COLUMN IF NOT EXISTS playground_sport_count int, + ADD COLUMN IF NOT EXISTS has_bike_paths bool, + ADD COLUMN IF NOT EXISTS trash_areas_count int; + +-- Accessibility / OVZ (3) +ALTER TABLE domrf_kn_objects + ADD COLUMN IF NOT EXISTS has_ramp bool, + ADD COLUMN IF NOT EXISTS has_low_platforms bool, + ADD COLUMN IF NOT EXISTS has_wheelchair_lift bool; + +-- Catalog / UI (9) +ALTER TABLE domrf_kn_objects + ADD COLUMN IF NOT EXISTS flat_area_min numeric(8,2), + ADD COLUMN IF NOT EXISTS flat_area_max numeric(8,2), + ADD COLUMN IF NOT EXISTS price_min_rub bigint, + ADD COLUMN IF NOT EXISTS price_max_rub bigint, + ADD COLUMN IF NOT EXISTS price_per_m2_min numeric(12,2), + ADD COLUMN IF NOT EXISTS price_per_m2_max numeric(12,2), + ADD COLUMN IF NOT EXISTS parking_provision_pct numeric(5,1), + ADD COLUMN IF NOT EXISTS project_published_at date, + ADD COLUMN IF NOT EXISTS project_declaration_num text; + +-- Metro & scores (5) — 22h merged into 22e +ALTER TABLE domrf_kn_objects + ADD COLUMN IF NOT EXISTS metro_nearest_name text, + ADD COLUMN IF NOT EXISTS metro_nearest_walk_minutes int, + ADD COLUMN IF NOT EXISTS metro_top3 jsonb, + ADD COLUMN IF NOT EXISTS domrf_score_location int, + ADD COLUMN IF NOT EXISTS domrf_score_transport int, + ADD COLUMN IF NOT EXISTS domrf_score_infrastructure int; + +-- Developer split (1) +ALTER TABLE domrf_kn_objects + ADD COLUMN IF NOT EXISTS dev_group_name text; + +COMMENT ON COLUMN domrf_kn_objects.section_count + IS '22e: подъезды (residentialBuildings из kn-API)'; +COMMENT ON COLUMN domrf_kn_objects.project_published_at + IS '22e: дата публикации проекта (objPublDt из kn-API)'; +COMMENT ON COLUMN domrf_kn_objects.project_declaration_num + IS '22e: номер проектной декларации (rpdNum из kn-API)'; +COMMENT ON COLUMN domrf_kn_objects.price_min_rub + IS '22e: минимальная цена (freeFlatsInfo.priceMin из kn-API)'; +COMMENT ON COLUMN domrf_kn_objects.metro_nearest_name + IS '22h: ближайшая станция метро (metro.name из kn-API)'; +COMMENT ON COLUMN domrf_kn_objects.metro_nearest_walk_minutes + IS '22h: время пешком до метро, мин (metro.time из kn-API, rounded)'; +COMMENT ON COLUMN domrf_kn_objects.metro_top3 + IS '22h: топ-3 ближайших станций [{name,time,line,color,isWalk}], сейчас 1 элемент'; +COMMENT ON COLUMN domrf_kn_objects.dev_group_name + IS '22e: группа компаний (developer.groupName из kn-API)'; +COMMENT ON COLUMN domrf_kn_objects.domrf_score_location + IS '22e: оценка расположения от DOM.RF (catalog 22d)'; +COMMENT ON COLUMN domrf_kn_objects.domrf_score_transport + IS '22e: оценка транспорта от DOM.RF (catalog 22d)'; +COMMENT ON COLUMN domrf_kn_objects.domrf_score_infrastructure + IS '22e: оценка инфраструктуры от DOM.RF (catalog 22d)'; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 22g — domrf_developers +4 metrics +-- ───────────────────────────────────────────────────────────────────────────── +COMMENT ON TABLE domrf_developers IS 'Застройщики DOM.RF (companyGroup-level)'; + +ALTER TABLE domrf_developers + ADD COLUMN IF NOT EXISTS houses_building int, + ADD COLUMN IF NOT EXISTS houses_building_delayed int, + ADD COLUMN IF NOT EXISTS houses_delivered int, + ADD COLUMN IF NOT EXISTS houses_delivered_delayed int; + +COMMENT ON COLUMN domrf_developers.houses_building + IS '22g: домов строится (из карточки застройщика, catalog 22d)'; +COMMENT ON COLUMN domrf_developers.houses_building_delayed + IS '22g: строится с задержкой'; +COMMENT ON COLUMN domrf_developers.houses_delivered + IS '22g: сдано домов'; +COMMENT ON COLUMN domrf_developers.houses_delivered_delayed + IS '22g: сдано с задержкой'; + +COMMIT; diff --git a/data/sql/114_22c_flat_plans.sql b/data/sql/114_22c_flat_plans.sql new file mode 100644 index 00000000..6ba6f4c1 --- /dev/null +++ b/data/sql/114_22c_flat_plans.sql @@ -0,0 +1,44 @@ +-- Migration 100: domrf_kn_flat_plans — планировки квартир (22c) +-- +-- Хранит URL картинки планировки для каждой квартиры DOM.РФ. +-- Источник данных: каталог-квартир SSR-страница / flats_plans endpoint. +-- Скачивание бинарников — Celery task (future PR, 22d-track). +-- +-- Phase 3 / #297 sub-task 22c. + +BEGIN; + +CREATE TABLE IF NOT EXISTS domrf_kn_flat_plans ( + ods_id text PRIMARY KEY, + obj_id bigint NOT NULL, + plan_image_url text NOT NULL, + local_path text, + width_px int, + height_px int, + size_bytes int, + downloaded_at timestamptz, + snapshot_date date NOT NULL, + scraped_at timestamptz DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS domrf_kn_flat_plans_obj_idx + ON domrf_kn_flat_plans (obj_id); + +COMMENT ON TABLE domrf_kn_flat_plans IS + 'Планировки квартир DOM.РФ: URL изображения + метаданные после скачивания. ' + 'PK = ods_id (совпадает с domrf_kn_flats.ods_id). ' + 'Источник: каталог-квартир SSR / flats_plans endpoint. ' + 'Issue #297 sub-task 22c.'; + +COMMENT ON COLUMN domrf_kn_flat_plans.ods_id IS + 'Идентификатор квартиры DOM.РФ (напр. 65136/1/1.4.3) — PK и FK на domrf_kn_flats.ods_id.'; +COMMENT ON COLUMN domrf_kn_flat_plans.plan_image_url IS + 'Абсолютный URL картинки планировки с сервера DOM.РФ (cdn или основной домен).'; +COMMENT ON COLUMN domrf_kn_flat_plans.local_path IS + 'Путь к локальной копии файла после скачивания (относительно MEDIA_ROOT или абсолютный).'; +COMMENT ON COLUMN domrf_kn_flat_plans.downloaded_at IS + 'Момент успешного скачивания бинарника. NULL = ещё не скачан.'; +COMMENT ON COLUMN domrf_kn_flat_plans.snapshot_date IS + 'Дата snapshot DOM.РФ, в котором впервые обнаружен URL.'; + +COMMIT; diff --git a/data/sql/115_trade_in_estimates.sql b/data/sql/115_trade_in_estimates.sql new file mode 100644 index 00000000..df9e787a --- /dev/null +++ b/data/sql/115_trade_in_estimates.sql @@ -0,0 +1,43 @@ +BEGIN; + +CREATE TABLE IF NOT EXISTS trade_in_estimates ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + + -- Input snapshot + address text NOT NULL, + lat double precision, + lon double precision, + area_m2 numeric(8, 2) NOT NULL, + rooms int NOT NULL, + floor int NOT NULL, + total_floors int NOT NULL, + year_built int, + house_type text, + repair_state text, + has_balcony boolean, + + -- Output + median_price bigint NOT NULL, + range_low bigint NOT NULL, + range_high bigint NOT NULL, + median_price_per_m2 int NOT NULL, + confidence text NOT NULL CHECK (confidence IN ('low', 'medium', 'high')), + n_analogs int NOT NULL DEFAULT 0, + analogs jsonb NOT NULL DEFAULT '[]'::jsonb, + actual_deals jsonb NOT NULL DEFAULT '[]'::jsonb, + + -- Metadata + created_at timestamptz NOT NULL DEFAULT NOW(), + expires_at timestamptz NOT NULL DEFAULT NOW() + interval '24 hours' +); + +CREATE INDEX IF NOT EXISTS trade_in_estimates_created_idx + ON trade_in_estimates (created_at DESC); + +CREATE INDEX IF NOT EXISTS trade_in_estimates_expires_idx + ON trade_in_estimates (expires_at); + +COMMENT ON TABLE trade_in_estimates IS + '#314 TradeIn MVP — стор estimates с input snapshot + aggregated output. TTL 24h.'; + +COMMIT; diff --git a/data/sql/116_obj2_multi_feature_matcher.sql b/data/sql/116_obj2_multi_feature_matcher.sql new file mode 100644 index 00000000..98da53eb --- /dev/null +++ b/data/sql/116_obj2_multi_feature_matcher.sql @@ -0,0 +1,15 @@ +-- 116_obj2_multi_feature_matcher.sql (HOTFIX — INSERT step deferred) +-- Issue #307 OBJ-2. +-- +-- Original version did INSERT through a 89M-cost CROSS JOIN LATERAL CTE that +-- timed out during deploy. Hotfix keeps только the GIST trgm index (this is +-- the load-bearing change for downstream queries — competitors fuzzy match, +-- objective_complex_mapping reverse matching). +-- +-- Actual multi-feature INSERT was already done inline on 2026-05-17 +-- (+13 mappings: 129 → 142). Remaining auto-accept candidates (~50-100 at +-- composite >= 0.75) will be applied via separate migration 117 after +-- the candidate query is pre-materialized to avoid full LATERAL scan. + +CREATE INDEX IF NOT EXISTS objective_lots_project_name_trgm_idx + ON objective_lots USING gist (project_name gist_trgm_ops); diff --git a/data/sql/117_obj2_mapping_backfill_fast.sql b/data/sql/117_obj2_mapping_backfill_fast.sql new file mode 100644 index 00000000..b6afeea3 --- /dev/null +++ b/data/sql/117_obj2_mapping_backfill_fast.sql @@ -0,0 +1,91 @@ +-- 117_obj2_mapping_backfill_fast.sql +-- Issue #307 OBJ-2 (continued from 116 hotfix). +-- +-- Migration 116 был trimmed до GIST index only из-за heavy CTE timeout. +-- Эта migration делает фактический INSERT через **pre-materialized temp tables** +-- (avoid full CROSS JOIN LATERAL scan). Composite score = name 0.6 + dev 0.25 + district 0.15. +-- Threshold composite >= 0.75. ON CONFLICT DO NOTHING (idempotent). +-- +-- Estimated runtime после оптимизации: <30s (vs old 89M-cost timeout). +-- Coverage цель: +50-80 mappings → объединённый total ~200. + +BEGIN; + +-- Pre-materialize unmapped DOM.РФ ЕКБ objects (~1056 rows) +CREATE TEMP TABLE _u_domrf ON COMMIT DROP AS +SELECT o.obj_id, o.comm_name, o.dev_name, o.district_name, + regexp_replace(lower(coalesce(o.dev_name, '')), '[^а-яa-z0-9]', '', 'g') AS dev_norm + FROM domrf_kn_objects o + WHERE o.is_ekb = true + AND o.comm_name IS NOT NULL + AND NOT EXISTS ( + SELECT 1 FROM objective_complex_mapping m WHERE m.domrf_obj_id = o.obj_id + ); + +-- Pre-materialize distinct unmapped Objective projects (~178 rows) +CREATE TEMP TABLE _u_obj ON COMMIT DROP AS +SELECT DISTINCT + l.project_name, + l.developer, + l.district, + regexp_replace(lower(coalesce(l.developer, '')), '[^а-яa-z0-9]', '', 'g') AS dev_norm + FROM objective_lots l + WHERE l.project_name IS NOT NULL + AND l.project_name NOT IN ( + SELECT objective_complex_name FROM objective_complex_mapping + ); + +-- Top-3 candidates per DOM.РФ obj_id using GIST trgm index (from migration 116) +CREATE TEMP TABLE _pairs ON COMMIT DROP AS +SELECT + d.obj_id, + o.project_name, + o.developer, + o.district, + similarity(d.comm_name, o.project_name) AS name_sim, + (d.dev_norm = o.dev_norm AND d.dev_norm <> '')::int AS dev_match, + ( + coalesce(d.district_name, '') ILIKE '%' || coalesce(o.district, '') || '%' + OR coalesce(o.district, '') ILIKE '%' || coalesce(d.district_name, '') || '%' + )::int AS district_match + FROM _u_domrf d + CROSS JOIN LATERAL ( + SELECT u.project_name, u.developer, u.district, u.dev_norm + FROM _u_obj u + WHERE similarity(d.comm_name, u.project_name) > 0.3 + ORDER BY similarity(d.comm_name, u.project_name) DESC + LIMIT 3 + ) o; + +-- Rank per obj_id by composite score, pick top +WITH ranked AS ( + SELECT + obj_id, project_name, + (name_sim * 0.6 + dev_match * 0.25 + district_match * 0.15) AS composite, + ROW_NUMBER() OVER ( + PARTITION BY obj_id + ORDER BY (name_sim * 0.6 + dev_match * 0.25 + district_match * 0.15) DESC, + name_sim DESC + ) AS rn + FROM _pairs +) +INSERT INTO objective_complex_mapping + (objective_complex_name, domrf_obj_id, objective_group, + match_method, match_score, is_reviewed, note) +SELECT + r.project_name, + r.obj_id, + 'Екатеринбург', + 'multi_feature_v1', + r.composite, + false, + 'OBJ-2 multi-feature 117_fast 2026-05-17 (name 0.6 + dev 0.25 + district 0.15)' + FROM ranked r + WHERE r.rn = 1 + AND r.composite >= 0.75 +ON CONFLICT (objective_complex_name, objective_group) DO NOTHING; + +COMMIT; + +-- REFRESH MV outside tx (CONCURRENTLY requires non-tx) +REFRESH MATERIALIZED VIEW CONCURRENTLY mv_layout_velocity; diff --git a/data/sql/118_22d_catalog_object_scraped_at.sql b/data/sql/118_22d_catalog_object_scraped_at.sql new file mode 100644 index 00000000..3ea522e3 --- /dev/null +++ b/data/sql/118_22d_catalog_object_scraped_at.sql @@ -0,0 +1,15 @@ +-- Migration 118: добавить catalog_scraped_at в domrf_kn_objects +-- Нужна для catalog-object scraper (issue #297, sub-task 22d): +-- scraper обновляет поле после успешного UPDATE, beat task выбирает +-- только объекты где это поле NULL или устарело (> 30 дней). +-- Индекс NULLS FIRST ускоряет ORDER BY catalog_scraped_at NULLS FIRST LIMIT N. + +BEGIN; + +ALTER TABLE domrf_kn_objects + ADD COLUMN IF NOT EXISTS catalog_scraped_at TIMESTAMP; + +CREATE INDEX IF NOT EXISTS idx_domrf_kn_objects_catalog_scraped_at + ON domrf_kn_objects (catalog_scraped_at NULLS FIRST); + +COMMIT; diff --git a/data/sql/118_pilot_requests.sql b/data/sql/118_pilot_requests.sql new file mode 100644 index 00000000..7bdb6e63 --- /dev/null +++ b/data/sql/118_pilot_requests.sql @@ -0,0 +1,21 @@ +BEGIN; + +CREATE TABLE IF NOT EXISTS pilot_requests ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + name text NOT NULL, + phone text, + email text, + company text, + message text, + source text, -- 'landing', 'analyze_page', etc + user_agent text, + created_at timestamptz NOT NULL DEFAULT NOW(), + notified_at timestamptz -- when Telegram sent +); + +CREATE INDEX IF NOT EXISTS pilot_requests_created_idx + ON pilot_requests (created_at DESC); + +COMMENT ON TABLE pilot_requests IS '#307 SF-B3 lead generation, опционально notified в Telegram.'; + +COMMIT; diff --git a/data/sql/119_parcel_user_status.sql b/data/sql/119_parcel_user_status.sql new file mode 100644 index 00000000..97392e43 --- /dev/null +++ b/data/sql/119_parcel_user_status.sql @@ -0,0 +1,21 @@ +BEGIN; + +CREATE TABLE IF NOT EXISTS parcel_user_status ( + user_id text NOT NULL, + cad_num text NOT NULL, + status text NOT NULL CHECK (status IN ('in_work', 'favorite', 'dismissed')), + notes text, + updated_at timestamptz NOT NULL DEFAULT NOW(), + PRIMARY KEY (user_id, cad_num) +); + +CREATE INDEX IF NOT EXISTS parcel_user_status_cad_idx + ON parcel_user_status (cad_num); + +CREATE INDEX IF NOT EXISTS parcel_user_status_user_idx + ON parcel_user_status (user_id, status); + +COMMENT ON TABLE parcel_user_status + IS '#307 SF-B1 user-marked parcels (in_work/favorite/dismissed) для карты.'; + +COMMIT; diff --git a/data/sql/70_parse_objective_raw.py b/data/sql/70_parse_objective_raw.py index 6cb04a8f..d3452c47 100644 --- a/data/sql/70_parse_objective_raw.py +++ b/data/sql/70_parse_objective_raw.py @@ -40,9 +40,10 @@ import json import logging import re import sys -from datetime import date, datetime +from collections.abc import Iterator +from datetime import date from pathlib import Path -from typing import Any +from typing import Any, BinaryIO # repo-root → /backend sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent / "backend")) @@ -205,7 +206,7 @@ def _row_to_corp_room_month(row: dict, group_name: str, raw_id: int | None) -> d ), # Audit "raw_id": raw_id, - "raw_props": json.dumps(row, ensure_ascii=False), + "raw_props": json.dumps(row, ensure_ascii=False, default=str), } @@ -341,7 +342,7 @@ def _row_to_lot(row: dict, raw_id: int | None, snapshot_date: date) -> dict: "encumbrance_start_date": _parse_iso_date(row.get("Дата начала обременения")), "egrn_actual_date": _parse_iso_date(row.get("Дата актуальности данных из ЕГРН")), "raw_id": raw_id, - "raw_props": json.dumps(row, ensure_ascii=False), + "raw_props": json.dumps(row, ensure_ascii=False, default=str), "snapshot_date": snapshot_date, } @@ -509,6 +510,90 @@ def parse_lots_pf(payload: dict, raw_id: int | None, snapshot_date: date, return n_lots, n_history +def parse_lots_pf_stream( + stream: BinaryIO, + raw_id: int | None, + snapshot_date: date, + db: Any, + dry_run: bool, + batch_size: int = 500, +) -> tuple[int, int]: + """Streaming-парсер lots_pf через ijson — не грузит весь JSON в RAM. + + Итерирует элементы массива result по одному, накапливает batch'и по batch_size, + делает executemany-style UPSERT каждые batch_size строк и commit каждые + 5 batch'ей (2500 строк) — RAM usage O(batch_size), не O(N). + + Args: + stream: бинарный поток (httpx.Response.iter_bytes proxy или io.BytesIO) + raw_id: FK в objective_raw_reports (может быть None для dry-run без БД) + snapshot_date: дата снимка для history + db: SQLAlchemy Session + dry_run: если True — только считает строки, не пишет в БД + batch_size: размер batch для executemany + + Returns: + (n_lots, n_history) — число обработанных лотов и строк истории + """ + import ijson # type: ignore[import-untyped] # lazy import — не обязателен в smoke + + n_lots = 0 + n_history = 0 + lots_batch: list[dict] = [] + history_batch: list[dict] = [] + batches_since_commit = 0 + + for row in ijson.items(stream, "result.item", use_float=True): + if not isinstance(row, dict): + continue + params = _row_to_lot(row, raw_id, snapshot_date) + if not params["objective_lot_id"]: + logger.warning("lots_pf stream: пропуск row без Id") + continue + + lots_batch.append(params) + history_batch.append({ + "objective_lot_id": params["objective_lot_id"], + "snapshot_date": snapshot_date, + "status": params["status"], + "is_sold": params["is_sold"], + "price_calculated_total_rub": params["price_calculated_total_rub"], + "price_per_m2_rub": params["price_per_m2_rub"], + "price_offer_total_rub": params["price_offer_total_rub"], + "price_delta_pct": params["price_delta_pct"], + "area_pd": params["area_pd"], + "contract_date": params["contract_date"], + "registration_date": params["registration_date"], + "contract_type": params["contract_type"], + "bank_name": params["bank_name"], + "raw_id": raw_id, + }) + n_lots += 1 + n_history += 1 + + if len(lots_batch) >= batch_size: + if not dry_run: + db.execute(_LOTS_UPSERT, lots_batch) + db.execute(_LOTS_HISTORY_INSERT, history_batch) + batches_since_commit += 1 + if batches_since_commit >= 5: + db.commit() + batches_since_commit = 0 + lots_batch = [] + history_batch = [] + + # Финальный неполный batch + if lots_batch and not dry_run: + db.execute(_LOTS_UPSERT, lots_batch) + db.execute(_LOTS_HISTORY_INSERT, history_batch) + db.commit() + elif not dry_run and batches_since_commit > 0: + db.commit() + + logger.info("lots_pf stream: %d lots / %d history rows processed", n_lots, n_history) + return n_lots, n_history + + def detect_kind(payload: Any) -> str | None: """Эвристика по полям первого row: 'corp_sum' | 'lots_pf' | None.""" if not isinstance(payload, dict): @@ -524,6 +609,140 @@ def detect_kind(payload: Any) -> str | None: return None +# ── streaming support ─────────────────────────────────────────────────────── + + +class _IterBytesReader: + """File-like wrapper над iterator of bytes chunks (для ijson.items()). + + ijson 3.x ожидает file-like с .read(n) — но httpx.Response.iter_bytes() + возвращает generator байт. Этот класс собирает chunks в буфер и отдаёт + их по требованию через .read(n). + """ + + def __init__(self, chunks: Iterator[bytes]) -> None: + self._chunks = iter(chunks) + self._buf = bytearray() + self._eof = False + + def read(self, n: int = -1) -> bytes: + if n < 0: + # Прочитать всё что осталось + for c in self._chunks: + self._buf.extend(c) + data = bytes(self._buf) + self._buf.clear() + self._eof = True + return data + while len(self._buf) < n and not self._eof: + try: + self._buf.extend(next(self._chunks)) + except StopIteration: + self._eof = True + break + out = bytes(self._buf[:n]) + del self._buf[:n] + return out + + +def parse_lots_pf_stream( + stream: BinaryIO | Iterator[bytes], + raw_id: int | None, + snapshot_date: date, + db: Any, + dry_run: bool, + batch_size: int = 500, +) -> tuple[int, int]: + """Streaming-парсер lots_pf через ijson — для больших ответов (>100 МБ). + + В отличие от parse_lots_pf (читает весь payload в dict), эта функция + обрабатывает JSON инкрементально через ijson.items(), что позволяет + парсить файлы >100 МБ без загрузки всего payload в память. + + Args: + stream: либо file-like с .read(n), либо iterator of bytes chunks + (например httpx.Response.iter_bytes(65536)). Если stream не имеет + метода .read — автоматически оборачивается в _IterBytesReader. + raw_id: ссылка на objective_raw_reports.raw_id (может быть None) + snapshot_date: дата снимка для objective_lots_history + db: SQLAlchemy Session + dry_run: если True — только подсчёт, без записи в БД + batch_size: коммит каждые N лотов (снижает нагрузку на память транзакции) + + Returns: + tuple[n_lots, n_history] + """ + import ijson # type: ignore[import-untyped] + + # Если stream — iterator chunks (без .read), оборачиваем в file-like reader. + if not hasattr(stream, "read"): + stream = _IterBytesReader(stream) # type: ignore[arg-type] + + n_lots = 0 + n_history = 0 + batch: list[dict] = [] + batches_since_commit = 0 + + def _flush() -> None: + """Bulk-upsert текущего batch двумя executemany-вызовами.""" + nonlocal batch + if not batch: + return + history_batch = [ + { + "objective_lot_id": p["objective_lot_id"], + "snapshot_date": snapshot_date, + "status": p["status"], + "is_sold": p["is_sold"], + "price_calculated_total_rub": p["price_calculated_total_rub"], + "price_per_m2_rub": p["price_per_m2_rub"], + "price_offer_total_rub": p["price_offer_total_rub"], + "price_delta_pct": p["price_delta_pct"], + "area_pd": p["area_pd"], + "contract_date": p["contract_date"], + "registration_date": p["registration_date"], + "contract_type": p["contract_type"], + "bank_name": p["bank_name"], + "raw_id": raw_id, + } + for p in batch + ] + if not dry_run: + db.execute(_LOTS_UPSERT, batch) + db.execute(_LOTS_HISTORY_INSERT, history_batch) + batch = [] + + for row in ijson.items(stream, "result.item", use_float=True): + if not isinstance(row, dict): + continue + params = _row_to_lot(row, raw_id, snapshot_date) + if not params["objective_lot_id"]: + logger.warning("lots_pf_stream: пропуск row без Id") + continue + batch.append(params) + n_lots += 1 + n_history += 1 + if len(batch) >= batch_size: + _flush() + batches_since_commit += 1 + if not dry_run and batches_since_commit >= 5: + db.commit() + batches_since_commit = 0 + logger.info("lots_pf_stream: committed 5 batches, n_lots=%d so far", n_lots) + + _flush() # финальный неполный batch + if not dry_run and batches_since_commit > 0: + db.commit() + + logger.info( + "lots_pf_stream: done — n_lots=%d n_history=%d dry_run=%s", + n_lots, + n_history, + dry_run, + ) + return n_lots, n_history + + # ── CLI ───────────────────────────────────────────────────────────────────── diff --git a/data/sql/79_relax_objective_raw_payload_null.sql b/data/sql/79_relax_objective_raw_payload_null.sql new file mode 100644 index 00000000..5f3e09b7 --- /dev/null +++ b/data/sql/79_relax_objective_raw_payload_null.sql @@ -0,0 +1,14 @@ +-- 79_relax_objective_raw_payload_null.sql +-- Разрешаем NULL в objective_raw_reports.payload для stream-parsed отчётов. +-- Поквартирные/Лоты (lots_pf) могут быть 600+ МБ — хранить полный JSONB не имеет +-- смысла (96 ГБ за полгода × 4 группы × еженедельно). Для таких отчётов пишем +-- payload = NULL, size и rows_extracted по-прежнему заполняются. +BEGIN; + +ALTER TABLE objective_raw_reports ALTER COLUMN payload DROP NOT NULL; + +COMMENT ON COLUMN objective_raw_reports.payload IS + 'NULL для stream-parsed отчётов (lots_pf 600+ МБ — не храним целиком). ' + 'payload_size и rows_extracted всегда заполнены.'; + +COMMIT; diff --git a/data/sql/86_v_bucket_success_score.sql b/data/sql/86_v_bucket_success_score.sql index 2d9aaded..ea5a2567 100644 --- a/data/sql/86_v_bucket_success_score.sql +++ b/data/sql/86_v_bucket_success_score.sql @@ -53,7 +53,7 @@ bucket_aggs AS ( AND o.region_cd = 66 AND f.snapshot_date > CURRENT_DATE - INTERVAL '24 months' GROUP BY 1, 2, 3 - HAVING COUNT(*) >= 30 -- статистически значимая выборка + HAVING COUNT(*) >= 15 -- мин 15: weak-confidence (15-29), strong (≥30) ), z_scores AS ( @@ -96,7 +96,7 @@ ORDER BY district_name, obj_class, success_score DESC; COMMENT ON VIEW v_bucket_success_score IS 'Success-driven bucket ranking per (district, class). ' 'Z-scores: velocity (+) приоритет, price (+) приоритет, area (-) приоритет. ' - 'Min 30 сделок в группе. ' + 'Min 15 сделок в группе (15-29 = weak confidence, ≥30 = strong). ' 'Используется recommend_mix для смещения рекомендации (issue #25).'; COMMIT; diff --git a/data/sql/94_mv_layout_velocity.sql b/data/sql/94_mv_layout_velocity.sql new file mode 100644 index 00000000..d67301e5 --- /dev/null +++ b/data/sql/94_mv_layout_velocity.sql @@ -0,0 +1,90 @@ +-- 94_mv_layout_velocity.sql +-- Issue #113 PR B — Materialized view per (obj_id, room_bucket) for top-layouts ranking. +-- +-- Sources: +-- objective_corpus_room_month — monthly per-corpus-per-room deals (19 738 rows, 2025-05..2026-05) +-- objective_complex_mapping — project_name → domrf_obj_id mapping (114 mapped objects) +-- +-- Join key: +-- objective_corpus_room_month.project_name = objective_complex_mapping.objective_complex_name +-- (verified: one project_name → at most one domrf_obj_id) +-- +-- room_bucket normalisation: +-- cyrillic 'студия' → ASCII 'studio' for consistency with layout_signature.py (PR A, Issue #113) +-- values '1', '2', '3', '4+' are kept as-is +-- +-- Window: last 24 months (rolling, anchored to NOW() at refresh time) +-- Expected rows after aggregation: ~459 (obj_id × room_bucket combinations) +-- +-- REFRESH CONCURRENTLY is safe after this migration because a UNIQUE index +-- on (obj_id, room_bucket) is created immediately. +-- Subsequent refreshes: layout_velocity_refresh.py helper (PR B) — not scheduled yet. +-- +-- Deploy: auto-applied by deploy.yml via _schema_migrations tracking. +-- Dependencies: none (no views depend on this MV at creation time). +-- +-- WARN: Re-apply этого файла (DR / lost _schema_migrations entry / dev local) +-- снесёт MV + CASCADE удалит зависимости (e.g. future /best-layouts views). +-- После re-apply ПЕРВЫЙ refresh должен быть concurrently=False +-- (CONCURRENTLY требует уже populated MV — упадёт на пустой). +-- Tracking table `_schema_migrations` нормально предотвращает re-apply, +-- но добавлен comment как safety reminder. + +BEGIN; + +DROP MATERIALIZED VIEW IF EXISTS mv_layout_velocity CASCADE; + +CREATE MATERIALIZED VIEW mv_layout_velocity AS +WITH last24mo AS ( + SELECT + ocm.project_name, + CASE + WHEN ocm.room_bucket = 'студия' THEN 'studio' + ELSE ocm.room_bucket + END AS room_bucket, + ocm.deals_total_count, + ocm.deals_total_avg_area_m2, + ocm.deals_total_avg_price_thousand_rub_per_m2, + ocm.deals_total_vol_m2, + ocm.report_month + FROM objective_corpus_room_month ocm + WHERE ocm.report_month >= (NOW() - INTERVAL '24 months')::date +) +SELECT + cm.domrf_obj_id AS obj_id, + l.room_bucket, + SUM(l.deals_total_count)::int AS total_deals_24mo, + AVG(l.deals_total_avg_area_m2)::numeric(10, 2) AS avg_area_m2, + AVG(l.deals_total_avg_price_thousand_rub_per_m2)::numeric(12, 2) + AS avg_price_thousand_rub_per_m2, + SUM(l.deals_total_vol_m2)::numeric(12, 2) AS total_vol_m2, + MIN(l.report_month) AS window_start, + MAX(l.report_month) AS window_end, + COUNT(DISTINCT l.report_month)::int AS months_with_data +FROM last24mo l +JOIN objective_complex_mapping cm + ON cm.objective_complex_name = l.project_name +WHERE l.room_bucket IS NOT NULL + AND cm.domrf_obj_id IS NOT NULL + AND cm.objective_group = 'Екатеринбург' -- защита от cross-region Cartesian при future multi-city +GROUP BY cm.domrf_obj_id, l.room_bucket +WITH NO DATA; + +-- UNIQUE index required for REFRESH CONCURRENTLY (future periodic refreshes) +-- Created on empty MV → instant, no lock duration on data +CREATE UNIQUE INDEX mv_layout_velocity_pk + ON mv_layout_velocity (obj_id, room_bucket); + +-- Lookup index used by /best-layouts endpoint (PR C) +CREATE INDEX mv_layout_velocity_obj_idx + ON mv_layout_velocity (obj_id); + +-- Initial populate (non-concurrent — MV was just created, CONCURRENTLY not allowed on empty MV) +REFRESH MATERIALIZED VIEW mv_layout_velocity; + +COMMENT ON MATERIALIZED VIEW mv_layout_velocity IS + 'Per-(obj_id, room_bucket) deals aggregation за last 24 months. ' + 'Source: objective_corpus_room_month × objective_complex_mapping (EKB only). ' + 'Refresh via layout_velocity_refresh.py (concurrently=True after initial populate).'; + +COMMIT; diff --git a/data/sql/95_mv_quarter_price.sql b/data/sql/95_mv_quarter_price.sql new file mode 100644 index 00000000..9e5d5c03 --- /dev/null +++ b/data/sql/95_mv_quarter_price.sql @@ -0,0 +1,124 @@ +-- 95_mv_quarter_price.sql +-- Issue #33 D1 — Per-cad_quarter price aggregation из 6.83M rosreestr_deals. +-- +-- Real schema (verified via information_schema): +-- quarter_cad_number varchar — cadastral quarter key (e.g. '66:41:0301036') +-- price_per_sqm numeric — pre-computed руб/м²; filled for 6.82M / 6.83M rows +-- deal_price numeric — total deal price (fallback if price_per_sqm IS NULL) +-- area numeric — area m² (fallback denominator) +-- period_start_date date — quarter start (2024-01-01 .. 2026-01-01) +-- realestate_type_code text — '002001003000' = новостройки (ДДУ) +-- +-- Filter scope: +-- realestate_type_code = '002001003000' → 3 143 004 rows (45.9% of total 6.83M) +-- price_per_sqm BETWEEN 30000 AND 800000 → outlier removal +-- period_start_date >= NOW() - INTERVAL '24 months' → rolling window +-- HAVING COUNT(*) >= 3 → suppress quarters with too few deals +-- +-- Expected rows: ~52 492 distinct quarters pass HAVING >= 3 (verified via SELECT COUNT). +-- Dataset is all-Russia (rosreestr_deals is not region-filtered at table level). +-- +-- Sub-window medians (6m / 12m): +-- PERCENTILE_CONT does not support FILTER clause in PG16. +-- Workaround: pre-aggregate per quarter with separate FILTER masks in a lateral +-- subquery. Implemented as a second CTE (agg_windows) joined to the main agg. +-- +-- REFRESH CONCURRENTLY: safe after this migration because a UNIQUE INDEX on +-- quarter_cad_number is created immediately after CREATE WITH NO DATA. +-- First populate is non-concurrent (MV is empty, CONCURRENTLY not allowed). +-- +-- Deploy: auto-applied by deploy.yml via _schema_migrations tracking. +-- Dependencies: none (no existing views depend on this MV at creation time). +-- +-- WARNING: Re-apply drops MV + CASCADE; after re-apply first REFRESH must use +-- concurrently=False. _schema_migrations tracking prevents re-apply in prod. + +BEGIN; + +DROP MATERIALIZED VIEW IF EXISTS mv_quarter_price_per_m2 CASCADE; + +CREATE MATERIALIZED VIEW mv_quarter_price_per_m2 AS +WITH dataset AS ( + -- Resolve price_m2: use pre-computed field, fall back to deal_price / area + SELECT + quarter_cad_number, + CASE + WHEN price_per_sqm IS NOT NULL THEN price_per_sqm + WHEN deal_price IS NOT NULL AND area > 0 THEN deal_price / area + ELSE NULL + END AS price_m2, + period_start_date AS deal_date + FROM rosreestr_deals + WHERE realestate_type_code = '002001003000' + AND period_start_date >= NOW() - INTERVAL '24 months' +), +filtered AS ( + -- Apply outlier filter once; reused by both aggregate CTEs + SELECT quarter_cad_number, price_m2, deal_date + FROM dataset + WHERE price_m2 BETWEEN 30000 AND 800000 +), +agg_main AS ( + -- Primary aggregation: full 24-month window + SELECT + quarter_cad_number, + COUNT(*) AS deals_count, + PERCENTILE_CONT(0.25) WITHIN GROUP (ORDER BY price_m2) AS p25, + PERCENTILE_CONT(0.50) WITHIN GROUP (ORDER BY price_m2) AS median, + PERCENTILE_CONT(0.75) WITHIN GROUP (ORDER BY price_m2) AS p75, + AVG(price_m2)::numeric(12, 2) AS mean, + MAX(deal_date) AS last_deal_date + FROM filtered + GROUP BY quarter_cad_number + HAVING COUNT(*) >= 3 +), +agg_windows AS ( + -- Sub-window medians per quarter. + -- PERCENTILE_CONT does not accept a FILTER clause in PG16, so we pre-filter + -- per window inside separate conditional aggregations using a CASE that + -- returns NULL for out-of-window rows (NULLs are excluded from PERCENTILE_CONT). + SELECT + quarter_cad_number, + PERCENTILE_CONT(0.50) WITHIN GROUP ( + ORDER BY CASE WHEN deal_date > NOW() - INTERVAL '6 months' + THEN price_m2 ELSE NULL END + ) AS median_6m, + PERCENTILE_CONT(0.50) WITHIN GROUP ( + ORDER BY CASE WHEN deal_date > NOW() - INTERVAL '12 months' + THEN price_m2 ELSE NULL END + ) AS median_12m + FROM filtered + GROUP BY quarter_cad_number +) +SELECT + m.quarter_cad_number, + m.deals_count, + m.p25, + m.median, + m.p75, + m.mean, + m.last_deal_date, + w.median_6m, + w.median_12m, + -- median_24m == median (window is already 24mo); kept for API symmetry + m.median AS median_24m +FROM agg_main m +JOIN agg_windows w USING (quarter_cad_number) +WITH NO DATA; + +-- UNIQUE index required for REFRESH MATERIALIZED VIEW CONCURRENTLY. +-- Created on empty MV → instant (no data lock). +CREATE UNIQUE INDEX mv_quarter_price_pk + ON mv_quarter_price_per_m2 (quarter_cad_number); + +-- Initial populate (non-concurrent — CONCURRENTLY not allowed on empty MV). +REFRESH MATERIALIZED VIEW mv_quarter_price_per_m2; + +COMMENT ON MATERIALIZED VIEW mv_quarter_price_per_m2 IS + 'Per-cad_quarter (quarter_cad_number) price aggregation из rosreestr_deals. ' + 'Filter: realestate_type_code=002001003000 (новостройки/ДДУ), rolling 24 months, ' + 'outlier filter 30K–800K руб/м², HAVING >= 3 deals. ' + 'All-Russia scope (~52K rows expected). ' + 'Refresh: weekly via celery beat (TBD, Issue #33 PR C).'; + +COMMIT; diff --git a/data/sql/96_ekburg_construction_permits.sql b/data/sql/96_ekburg_construction_permits.sql new file mode 100644 index 00000000..15d93512 --- /dev/null +++ b/data/sql/96_ekburg_construction_permits.sql @@ -0,0 +1,91 @@ +-- 96_ekburg_construction_permits.sql +-- Issue #105 — РНС/РВЭ ЕКБ 2022-2026 (xlsx от екатеринбург.рф, Форма 3 и 4 Росстата). +-- +-- Источник: https://xn--80acgfbsl1azdqr.xn--p1ai/дляработы/гиз/градостроительство/разрешение +-- Формат: XLSX с двумя листами: +-- «реестр разрешений на строительс» — Таблица 3 (РНС) +-- «реестр разрешений на ввод» — Таблица 4 (РВЭ) +-- +-- Refresh: monthly via tasks.ekburg_permits_sync.refresh_all +-- Geocoding: raw_coord_x / raw_coord_y — местная СКФ-66 (Свердловская обл.), +-- geocoded_lat / geocoded_lon заполняются отдельной задачей (Phase 3). + +BEGIN; + +CREATE TABLE IF NOT EXISTS ekburg_construction_permits ( + id BIGSERIAL PRIMARY KEY, + permit_type TEXT NOT NULL CHECK (permit_type IN ('RNS', 'RVE')), + + -- Реквизиты разрешения + permit_number TEXT NOT NULL, + issue_date DATE, + expiry_date DATE, + + -- Застройщик + developer_inn TEXT, + developer_name TEXT, + + -- Объект + object_name TEXT, + object_type TEXT, -- из Справочника: многоквартирные жилые дома; и т.д. + construction_address TEXT, + cadastral_number TEXT, -- кадастровый номер ЗУ (колонка 6) + + -- Площади + total_area_sqm NUMERIC, -- общая площадь по проекту, м² + living_area_sqm NUMERIC, -- площадь жилых помещений по проекту, м² + living_area_fact_sqm NUMERIC, -- площадь жилых помещений фактически (только РВЭ) + + -- Координаты (местная СКФ-66 Свердловской обл. — CRS TBD Phase 3) + raw_coord_x TEXT, + raw_coord_y TEXT, + + -- РВЭ-специфичные поля + rve_number TEXT, -- номер разрешения на ввод + rve_date DATE, -- дата разрешения на ввод + + -- Геокодирование (Phase 3) + geocoded_lat NUMERIC, + geocoded_lon NUMERIC, + geom GEOMETRY(POINT, 4326), + + -- Метаданные загрузки + source_year INT NOT NULL, + source_url TEXT, + raw_row JSONB, + fetched_at TIMESTAMPTZ DEFAULT NOW(), + + UNIQUE (permit_type, permit_number) +); + +CREATE INDEX IF NOT EXISTS gist_ekburg_permits_geom + ON ekburg_construction_permits USING GIST (geom); + +CREATE INDEX IF NOT EXISTS idx_ekburg_permits_issue_date + ON ekburg_construction_permits (issue_date); + +CREATE INDEX IF NOT EXISTS idx_ekburg_permits_developer + ON ekburg_construction_permits (developer_inn); + +CREATE INDEX IF NOT EXISTS idx_ekburg_permits_year + ON ekburg_construction_permits (source_year); + +CREATE INDEX IF NOT EXISTS idx_ekburg_permits_type_year + ON ekburg_construction_permits (permit_type, source_year); + +COMMENT ON TABLE ekburg_construction_permits IS + 'РНС/РВЭ ЕКБ 2022-2026 (Форма 3/4 Росстата). ' + 'Source: екатеринбург.рф xlsx. Refresh: monthly via tasks/ekburg_permits_sync. ' + 'Координаты raw_coord_x/y в местной СКФ-66 (CRS TBD). ' + 'Геокодирование через Nominatim — Phase 3 (отдельный PR).'; + +COMMENT ON COLUMN ekburg_construction_permits.raw_coord_x IS + 'X-координата характерной точки ЗУ (местная система координат СКФ-66 / СНСК-66).'; + +COMMENT ON COLUMN ekburg_construction_permits.raw_coord_y IS + 'Y-координата характерной точки ЗУ (местная система координат СКФ-66 / СНСК-66).'; + +COMMENT ON COLUMN ekburg_construction_permits.living_area_fact_sqm IS + 'Заполняется только для РВЭ (Таблица 4, колонка 15).'; + +COMMIT; diff --git a/data/sql/97_objective_backfill_indexes.sql b/data/sql/97_objective_backfill_indexes.sql new file mode 100644 index 00000000..13a0b968 --- /dev/null +++ b/data/sql/97_objective_backfill_indexes.sql @@ -0,0 +1,31 @@ +-- 97_objective_backfill_indexes.sql +-- Issue #203 — Backfill objective_complex_mapping (114 → ~1500 EKB ЖК). +-- +-- pg_trgm уже установлен в prod, но CREATE EXTENSION IF NOT EXISTS идемпотентен. +-- Trigram GIST index на comm_name (DOM.РФ) ускоряет LATERAL similarity() join +-- при поиске кандидатов для fuzzy match (domrf_kn_objects ↔ objective_corpus_room_month). +-- +-- Dependencies: domrf_kn_objects, objective_corpus_room_month (уже существуют). +-- Applied: automatically via deploy.yml in NN order. + +BEGIN; + +CREATE EXTENSION IF NOT EXISTS pg_trgm; + +-- Trigram GIST index для fuzzy match comm_name (DOM.РФ) ↔ project_name (Objective). +-- Фильтр is_ekb = true: только ЕКБ объекты (~1285 строк). +CREATE INDEX IF NOT EXISTS idx_domrf_kn_objects_comm_name_trgm + ON domrf_kn_objects USING gist (comm_name gist_trgm_ops) + WHERE is_ekb = true; + +-- B-tree index на project_name в objective для JOIN performance. +CREATE INDEX IF NOT EXISTS idx_objective_corpus_room_month_project_name + ON objective_corpus_room_month (project_name); + +COMMENT ON INDEX idx_domrf_kn_objects_comm_name_trgm IS + 'Trigram GIST index для fuzzy match с objective_complex_mapping (#203 backfill)'; + +COMMENT ON INDEX idx_objective_corpus_room_month_project_name IS + 'B-tree index на project_name для JOIN performance (#203 backfill)'; + +COMMIT; diff --git a/data/sql/98_nspd_quarter_dumps_opportunity_flag.sql b/data/sql/98_nspd_quarter_dumps_opportunity_flag.sql new file mode 100644 index 00000000..5c972a54 --- /dev/null +++ b/data/sql/98_nspd_quarter_dumps_opportunity_flag.sql @@ -0,0 +1,43 @@ +-- 98_nspd_quarter_dumps_opportunity_flag.sql +-- Context : TIER 4 opportunity denorm columns for nspd_quarter_dumps. +-- Enables fast map filter "find quarters with auction parcels" without +-- unpacking features_json JSONB on every query. +-- Part of issue #94 PR 2 (TIER 4 opportunity layers). +-- Dependencies: 88_nspd_quarter_dumps.sql (table must exist) +-- Deploy order: after 88_nspd_quarter_dumps.sql +-- Idempotent: yes — ADD COLUMN IF NOT EXISTS + CREATE INDEX IF NOT EXISTS +-- Related: +-- - backend/app/services/scrapers/nspd_client.py :: LAYERS dict (TIER 4) +-- - backend/app/services/site_finder/quarter_dump_lookup.py :: _get_opportunity_parcels +-- - backend/app/workers/tasks/nspd_sync.py :: _build_opportunity_count, _build_has_auction_parcels +-- Note: red_lines (layer 879243) is a TIER 1 core layer — always harvested, +-- counted in red_lines_count (88_*.sql). No separate v2 column needed. + +BEGIN; + +-- Denorm columns for fast filter on map (find quarters with opportunity) +ALTER TABLE nspd_quarter_dumps + ADD COLUMN IF NOT EXISTS has_auction_parcels BOOLEAN DEFAULT FALSE, + ADD COLUMN IF NOT EXISTS opportunity_count INTEGER DEFAULT 0; + +COMMENT ON COLUMN nspd_quarter_dumps.has_auction_parcels IS + 'TRUE если квартал содержит >= 1 feature слоя auction_parcels (NSPD layer 37299). ' + 'Заполняется при harvest (include_opportunity=True). Используется для быстрого ' + 'поиска кварталов с аукционными ЗУ на карте. (#94 PR2)'; + +COMMENT ON COLUMN nspd_quarter_dumps.opportunity_count IS + 'Сумма features TIER 4 opportunity layers: ' + 'auction_parcels (37299) + scheme_parcels (37294) + free_parcels (37298) + ' + 'future_parcels (36473) + protected_areas/oopt (875845). ' + 'Заполняется при harvest (include_opportunity=True). (#94 PR2)'; + +-- Partial index for quick "find quarters with auction parcels" map filter +CREATE INDEX IF NOT EXISTS idx_nspd_quarter_dumps_auction + ON nspd_quarter_dumps (quarter_cad) + WHERE has_auction_parcels = TRUE; + +COMMENT ON INDEX idx_nspd_quarter_dumps_auction IS + 'Partial index: быстрый поиск кварталов с аукционными ЗУ для UI-фильтра карты. ' + 'Малая кардинальность (только TRUE rows) — очень компактный. (#94 PR2)'; + +COMMIT; diff --git a/data/sql/99_nspd_entities_denorm.sql b/data/sql/99_nspd_entities_denorm.sql new file mode 100644 index 00000000..9c674bdb --- /dev/null +++ b/data/sql/99_nspd_entities_denorm.sql @@ -0,0 +1,171 @@ +-- 99_nspd_entities_denorm.sql +-- Context : Denormalized entity tables для NSPD parcel и building features. +-- nspd_quarter_dumps.features_json хранит все feature'ы в JSONB; эти +-- таблицы дают быстрый lookup по cad_num без json-unpacking на query time. +-- Источник: layer="parcels" (NSPD layer 36048) и layer="buildings" (36049) +-- из features_json. Геометрия трансформируется 3857→4326 при UPSERT. +-- +-- Dependencies: 88_nspd_quarter_dumps.sql (logical; таблица должна существовать до backfill) +-- Deploy order: после 98_nspd_quarter_dumps_opportunity_flag.sql +-- Idempotent: yes — IF NOT EXISTS на всех объектах +-- Related: +-- - backend/app/services/scrapers/nspd_denorm.py :: denorm_parcel_feature / denorm_building_feature +-- - backend/app/workers/tasks/nspd_denorm_backfill.py :: backfill_all_dumps +-- - backend/app/workers/tasks/nspd_sync.py :: harvest_quarter (inline denorm) +-- Issue: #94 PR3 + +BEGIN; + +-- ── nspd_parcels ────────────────────────────────────────────────────────────── +-- Denormalized parcel features из NSPD quarter dumps (layer="parcels", id 36048). +-- Каждая строка = один земельный участок из любого квартала. +-- PK cad_num: один участок может пересекать несколько кварталов, берём +-- последний harvest (ON CONFLICT DO UPDATE). + +CREATE TABLE IF NOT EXISTS nspd_parcels ( + -- Кадастровый номер участка (4-сегментный, e.g. '66:41:0204016:10'). + -- Первичный ключ — natural, стабильный идентификатор в ЕГРН. + cad_num TEXT PRIMARY KEY, + + -- Квартал из которого была взята последняя версия этого участка. + quarter_cad TEXT NOT NULL, + + -- ВРИ (вид разрешённого использования) — properties.permitted_use из NSPD. + -- Nullable: НСПД не всегда возвращает ВРИ для старых записей. + permitted_use TEXT, + + -- Категория земель (properties.land_category). + -- Nullable: может отсутствовать в ответе. + land_category TEXT, + + -- Кадастровая стоимость ЗУ (properties.cost_value), руб. + cost_value NUMERIC, + + -- Кадастровая стоимость за м² = cost_value / area_sqm. + -- Вычисляется при UPSERT; NULL если area_sqm = 0 или cost_value = NULL. + cost_per_m2 NUMERIC, + + -- Площадь участка (properties.area), м². + area_sqm NUMERIC, + + -- Адрес участка (properties.address). + address TEXT, + + -- Геометрия участка в WGS84 (трансформируется из EPSG:3857 при UPSERT). + -- Geometry не Polygon т.к. НСПД иногда возвращает MultiPolygon или Point. + geom GEOMETRY(Geometry, 4326), + + -- Дата snapshot'а (день harvest'а). + snapshot_date DATE NOT NULL DEFAULT CURRENT_DATE, + + -- Timestamp последнего UPSERT. + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +COMMENT ON TABLE nspd_parcels IS + 'Denormalized parcel features из NSPD quarter dumps (layer=parcels, id 36048). ' + 'PK = cad_num (последний harvest wins). Заполняется Celery task backfill_all_dumps ' + 'или inline в harvest_quarter. Источник: features_json JSONB колонка nspd_quarter_dumps. ' + 'Геометрия трансформирована EPSG:3857→4326. (#94 PR3)'; + +COMMENT ON COLUMN nspd_parcels.cad_num IS + 'Кадастровый номер участка (4-сегментный). Natural PK из НСПД.'; + +COMMENT ON COLUMN nspd_parcels.quarter_cad IS + '3-сегментный квартал последнего harvest''а. Используется для debug/audit.'; + +COMMENT ON COLUMN nspd_parcels.cost_per_m2 IS + 'Кадастровая стоимость за м² (cost_value / area_sqm). ' + 'NULL если area_sqm = 0 или cost_value отсутствует. Вычисляется в Python при UPSERT.'; + +-- Индекс для lookup по кварталу (join с nspd_quarter_dumps). +CREATE INDEX IF NOT EXISTS idx_nspd_parcels_quarter_cad + ON nspd_parcels (quarter_cad); + +-- GIST индекс на геометрию для spatial queries (соседние участки, ST_DWithin). +CREATE INDEX IF NOT EXISTS idx_nspd_parcels_geom_gist + ON nspd_parcels USING GIST (geom); + +-- Partial B-tree на cost_per_m2 для ценовых аналитических запросов. +CREATE INDEX IF NOT EXISTS idx_nspd_parcels_cost_per_m2 + ON nspd_parcels (cost_per_m2) + WHERE cost_per_m2 IS NOT NULL; + + +-- ── nspd_buildings ──────────────────────────────────────────────────────────── +-- Denormalized building features из NSPD quarter dumps (layer="buildings", id 36049). +-- Каждая строка = одно здание/сооружение ЕГРН. + +CREATE TABLE IF NOT EXISTS nspd_buildings ( + -- Кадастровый номер здания (5-сегментный, e.g. '66:41:0204016:10:1'). + cad_num TEXT PRIMARY KEY, + + -- Квартал последнего harvest'а. + quarter_cad TEXT NOT NULL, + + -- Назначение здания (properties.purpose). + -- Ключевые значения: 'Многоквартирный дом', 'Нежилое здание', etc. + purpose TEXT, + + -- Количество надземных этажей (properties.floors_above_ground). + floors INTEGER, + + -- Количество подземных этажей (properties.floors_underground). + floors_underground INTEGER, + + -- Год постройки (properties.year_built). + year_built INTEGER, + + -- Кадастровая стоимость здания (properties.cost_value), руб. + cost_value NUMERIC, + + -- Площадь по данным ГКН (properties.build_record_area), м². + build_record_area NUMERIC, + + -- Адрес здания (properties.address). + address TEXT, + + -- Геометрия здания в WGS84 (трансформируется из EPSG:3857 при UPSERT). + geom GEOMETRY(Geometry, 4326), + + -- Дата snapshot'а. + snapshot_date DATE NOT NULL DEFAULT CURRENT_DATE, + + -- Timestamp последнего UPSERT. + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +COMMENT ON TABLE nspd_buildings IS + 'Denormalized building features из NSPD quarter dumps (layer=buildings, id 36049). ' + 'PK = cad_num (последний harvest wins). Заполняется Celery task backfill_all_dumps ' + 'или inline в harvest_quarter. Ключевой use-case: МКД-соседи для P2-scoring. ' + 'Геометрия трансформирована EPSG:3857→4326. (#94 PR3)'; + +COMMENT ON COLUMN nspd_buildings.purpose IS + 'Назначение здания (properties.purpose). Для МКД = "Многоквартирный дом". ' + 'Используется в partial index idx_nspd_buildings_mkd для быстрой фильтрации.'; + +COMMENT ON COLUMN nspd_buildings.floors IS + 'Количество надземных этажей (properties.floors_above_ground из НСПД). ' + 'Nullable: старые ОКС в ЕГРН не всегда имеют этажность.'; + +-- Индекс для lookup по кварталу. +CREATE INDEX IF NOT EXISTS idx_nspd_buildings_quarter_cad + ON nspd_buildings (quarter_cad); + +-- GIST индекс на геометрию (МКД-соседи через ST_DWithin). +CREATE INDEX IF NOT EXISTS idx_nspd_buildings_geom_gist + ON nspd_buildings USING GIST (geom); + +-- Partial index для МКД (основной use-case P2-scoring). +-- ILIKE не работает в partial index WHERE clause — используем точный строковый prefixmatch. +-- Здания с purpose IS NULL или другим purpose — в idx_nspd_buildings_geom_gist. +CREATE INDEX IF NOT EXISTS idx_nspd_buildings_mkd + ON nspd_buildings (quarter_cad) + WHERE purpose = 'Многоквартирный дом'; + +COMMENT ON INDEX idx_nspd_buildings_mkd IS + 'Partial index для быстрого lookup МКД по кварталу (P2-scoring neighbors). ' + 'WHERE purpose = exact строка из НСПД. (#94 PR3)'; + +COMMIT; diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 90632581..8c77d93c 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -119,7 +119,63 @@ services: depends_on: redis: condition: service_healthy - command: ["celery", "-A", "app.workers.celery_app", "beat", "--loglevel=info"] + # --schedule=/tmp/...: default location `/app/celerybeat-schedule` падает + # с Permission denied — WORKDIR `/app` принадлежит root, `app` (uid 1000) + # не может создавать там файлы. `/tmp` всегда writable; schedule-файл + # хранит только last_run_at для periodic tasks — потеря на restart OK, + # beat перестроит из `celery_app.conf.beat_schedule` на старте. + command: ["celery", "-A", "app.workers.celery_app", "beat", "--loglevel=info", "--schedule=/tmp/celerybeat-schedule"] + + glitchtip-web: + image: glitchtip/glitchtip:6.1.6 + container_name: glitchtip-web + # profiles: ["glitchtip"] keeps this service from starting on plain `compose up -d`. + # Bootstrap script activates the profile after DB + secrets are ready. + # On subsequent deploys, set COMPOSE_PROFILES=glitchtip in /opt/gendesign/.env. + profiles: ["glitchtip"] + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_started + environment: + DATABASE_URL: postgres://glitchtip:${GLITCHTIP_DB_PASS}@postgres:5432/glitchtip + REDIS_URL: redis://redis:6379/2 + SECRET_KEY: ${GLITCHTIP_SECRET} + PORT: "8080" + EMAIL_URL: consolemail:// + GLITCHTIP_DOMAIN: https://errors.gendsgn.ru + DEFAULT_FROM_EMAIL: errors@gendsgn.ru + ENABLE_USER_REGISTRATION: "true" + ENABLE_ORGANIZATION_CREATION: "false" + restart: always + mem_limit: 512m + healthcheck: + test: ["CMD", "wget", "-q", "--spider", "http://localhost:8080/api/0/"] + interval: 30s + timeout: 5s + retries: 5 + start_period: 60s + networks: [default] + + glitchtip-worker: + image: glitchtip/glitchtip:6.1.6 + container_name: glitchtip-worker + profiles: ["glitchtip"] + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_started + command: ./bin/run-celery-with-beat.sh + environment: + DATABASE_URL: postgres://glitchtip:${GLITCHTIP_DB_PASS}@postgres:5432/glitchtip + REDIS_URL: redis://redis:6379/2 + SECRET_KEY: ${GLITCHTIP_SECRET} + CELERY_WORKER_AUTOSCALE: "1,3" + restart: always + mem_limit: 384m + networks: [default] caddy: image: caddy:2 @@ -129,8 +185,11 @@ services: - "443:443" volumes: - ./Caddyfile:/etc/caddy/Caddyfile:ro + - ./caddy/users.caddy.snippet:/etc/caddy/caddy/users.caddy.snippet:ro + - ./preview:/srv/preview:ro - caddy_data:/data - caddy_config:/config + - caddy_logs:/var/log/caddy # backend: service_healthy — backend имеет /health endpoint, готов до Caddy. # frontend: service_started — НЕ service_healthy: даже если frontend healthcheck # дребезжит, Caddy всё равно стартует (можно отдавать 502 для /, но @@ -147,11 +206,33 @@ services: # main-приложение не страдает. - shared + glitchtip-auth-forwarder: + # Собирается локально на VPS при деплое (не тянется из GHCR). + # deploy.yml запускает: docker compose build glitchtip-auth-forwarder + build: ./ops/glitchtip-auth-forwarder + container_name: gendesign-auth-forwarder + restart: unless-stopped + environment: + GLITCHTIP_DSN: ${GLITCHTIP_DSN} + APP_ENV: production + APP_RELEASE: auth-forwarder-1 + CADDY_LOG_FILE: /var/log/caddy/auth_audit.log + STATE_FILE: /state/offset.json + volumes: + # Read-only доступ к Caddy access log + - caddy_logs:/var/log/caddy:ro + # Persistent offset — выживает при перезапуске контейнера + - auth_forwarder_state:/state + depends_on: + - caddy + volumes: postgres_data: redis_data: caddy_data: caddy_config: + caddy_logs: + auth_forwarder_state: networks: # Внешняя сеть, создаётся вне compose (см. docs/obsidian-livesync.md). diff --git a/docs/PILOT_ACCESS.md b/docs/PILOT_ACCESS.md new file mode 100644 index 00000000..ef8e9b1c --- /dev/null +++ b/docs/PILOT_ACCESS.md @@ -0,0 +1,11 @@ +# Доступ к GenDesign Pilot + +**URL**: https://gendsgn.ru/ +**Логин**: {USERNAME} +**Пароль**: {PASSWORD} + +При первом заходе браузер запросит данные — введите выше. +Credentials кэшируются до закрытия окна браузера. + +Если забыли пароль — напишите claudestars@proton.me, мы выпустим новый. +По завершении пилота доступ будет отозван. diff --git a/docs/runbooks/basic_auth_management.md b/docs/runbooks/basic_auth_management.md new file mode 100644 index 00000000..a33e3710 --- /dev/null +++ b/docs/runbooks/basic_auth_management.md @@ -0,0 +1,132 @@ +# Basic Auth Management Runbook + +**Установлено**: 2026-05-23 (PR #426, commit `cc2d148`) +**Realm**: `"GenDesign Pilot"` +**URL**: + +## Добавить пользователя + +```bash +./scripts/auth/add_user.sh +# Вводи пароль интерактивно; скрипт добавляет в caddy/users.caddy.snippet +``` + +После: commit snippet → PR → merge → GHA deploy → Caddy reload (автоматически в deploy.yml). + +## Удалить пользователя + +1. `./scripts/auth/remove_user.sh ` ASAP +2. Commit + push + PR (merge приоритетным) +3. `docker compose exec caddy caddy reload` после deploy +4. Сгенерировать новый credential через `add_user.sh` +5. Уведомить остальных стейкхолдеров о замене + +## Просмотр текущих пользователей + +```bash +./scripts/auth/list_users.sh +``` + +## Credencial rotation + +При компрометации пароля: remove_user + add_user с новым паролем + PR-merge + разослать новый credential. + +## Аудит логи + +- **Auth audit**: `/var/log/caddy/auth_audit.log` — все события basic_auth (включая raw Authorization). + Содержит plain credentials в Base64 — accessible только root на VPS. + Используется forwarder'ом (см. `ops/glitchtip-auth-forwarder/forwarder.py`). + Retention: 7 дней (roll_size 10MiB, keep 3, 168h). + +- **Access log**: `/var/log/caddy/gendsgn.ru.log` — все requests, format JSON. + Тоже содержит raw Authorization из-за global `log_credentials` (Caddy 2.x — опция только global, нельзя per-block). + Retention: 30 дней (roll_size 50MiB, keep 5, 720h). + +Failed auth последние 100: + +```bash +docker compose -f docker-compose.prod.yml exec caddy \ + grep -P '"status":401' /var/log/caddy/auth_audit.log | tail -100 | jq +``` + +### Security note + +`log_credentials` — опция под `servers { }` global block, нельзя per-virtual-host. Plain Base64 credentials попадают в **оба** log файла. Mitigation: +- VPS root-only (только SSH key authorized owner) +- Retention: gendsgn.ru.log = 30 дней, auth_audit.log = 7 дней +- Не выгружать логи на внешние системы без redaction + +## Диагностика + +```bash +# Посмотреть 401-ошибки в auth audit log +ssh gendesign 'docker compose -p gendesign exec caddy tail -f /var/log/caddy/auth_audit.log' | grep '"status":401' + +# Reload конфига Caddy без перезапуска +ssh gendesign 'docker compose -p gendesign exec -T caddy caddy reload --config /etc/caddy/Caddyfile' +``` + +## GlitchTip auth event forwarding + +Sidecar `gendesign-auth-forwarder` шлёт 401 события в GlitchTip как Sentry warning events. + +- Состояние: `docker compose -p gendesign ps | grep auth-forwarder` +- Logs: `docker compose -p gendesign logs -f glitchtip-auth-forwarder` +- GlitchTip dashboard: — фильтр `event_type:basic_auth_failed` +- Storm digest: при >10 failed auth за 60s → отдельный `event_type:basic_auth_storm` event +- Мониторы (GlitchTip uptime, SentryUptimeBot) — получают 401 стабильно из-за отсутствия credentials; они фильтруются как individual events, учитываются только в storm digest + +Если sidecar упал — failed auth события **не теряются** (offset persistent в `auth_forwarder_state` volume), но в GlitchTip не доезжают пока container down. + +### Перезапуск forwarder + +```bash +ssh gendesign 'docker compose -p gendesign restart glitchtip-auth-forwarder' +``` + +### Переменные окружения forwarder + +Задаются в `/opt/gendesign/.env` на VPS: + +| Переменная | Описание | Обязательна | +|---|---|---| +| `GLITCHTIP_DSN` | Sentry DSN GlitchTip (project backend) | Да | + +## Deploy + +Sidecar собирается локально на VPS на каждом запуске `deploy.yml` (после merge в main). +Не требует pre-push image build, не использует GHCR. + +Forwarder контейнер пересоздаётся `--force-recreate` чтобы подхватить новый image при изменении кода. + +В случае sidecar не запустился: + +```bash +ssh gendesign 'docker compose -p gendesign -f docker-compose.prod.yml logs --tail=100 glitchtip-auth-forwarder' +``` + +Если build fail на VPS — проверь `ops/glitchtip-auth-forwarder/Dockerfile` локально: + +```bash +docker build ops/glitchtip-auth-forwarder/ +``` + +## Phase 2 — реализовано (PR #430) + +GlitchTip forwarder sidecar добавлен в `ops/glitchtip-auth-forwarder/`. +Архитектура: Python 3.12 slim, sentry-sdk 2.18, tail live Caddy log, offset в named volume. +Deploy автоматизирован через `deploy.yml` — ручной SSH на VPS не нужен. + +## Phase 3 — реализовано (PR #436) + +`log_credentials` + `auth_audit.log` → forwarder получает реальный username из 401. + +- Global `log_credentials` в Caddyfile — Caddy перестаёт редактировать Authorization header. +- Отдельный `auth_audit.log` (retention 7d) — forwarder читает именно его. +- `_extract_attempted_username` декодирует Basic auth → `attempted_username` tag в GlitchTip event. +- Результат: GlitchTip event получает `attempted_username:testbruteforce` вместо `(none)`. + +## Phase 4 (TODO) + +- При >30 users — миграция на OAuth/NextAuth +- Разделение admin/user по доступу через Caddy path matcher или app-level RBAC diff --git a/frontend/.env.example b/frontend/.env.example index 8a4b22ef..dcbcc93c 100644 --- a/frontend/.env.example +++ b/frontend/.env.example @@ -1,2 +1,9 @@ NEXT_PUBLIC_API_BASE_URL=http://localhost:8000 NEXT_PUBLIC_MAPBOX_TOKEN= + +# GlitchTip error tracking (errors.gendsgn.ru) +NEXT_PUBLIC_GLITCHTIP_DSN= +GLITCHTIP_AUTH_TOKEN= +SENTRY_ORG=gendesign +SENTRY_PROJECT=frontend +SENTRY_URL=https://errors.gendsgn.ru/ diff --git a/frontend/Dockerfile b/frontend/Dockerfile index 6e34fff2..6c769b35 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -13,6 +13,12 @@ RUN --mount=type=cache,target=/root/.npm \ FROM node:20-alpine AS builder WORKDIR /app ENV NEXT_TELEMETRY_DISABLED=1 +# NEXT_PUBLIC_* must be present at `npm run build` — Next.js inlines them into +# the client bundle. Defaults to empty so local `docker build` без build-args не ломается. +ARG NEXT_PUBLIC_GLITCHTIP_DSN= +ARG NEXT_PUBLIC_ENVIRONMENT=production +ENV NEXT_PUBLIC_GLITCHTIP_DSN=$NEXT_PUBLIC_GLITCHTIP_DSN +ENV NEXT_PUBLIC_ENVIRONMENT=$NEXT_PUBLIC_ENVIRONMENT COPY --from=deps /app/node_modules ./node_modules COPY . . RUN npm run build diff --git a/frontend/next.config.ts b/frontend/next.config.ts index dd8fb5a8..449a5d7e 100644 --- a/frontend/next.config.ts +++ b/frontend/next.config.ts @@ -1,4 +1,5 @@ import type { NextConfig } from "next"; +import { withSentryConfig } from "@sentry/nextjs"; const nextConfig: NextConfig = { reactStrictMode: true, @@ -47,4 +48,29 @@ const nextConfig: NextConfig = { }, }; -export default nextConfig; +export default withSentryConfig(nextConfig, { + org: process.env.SENTRY_ORG ?? "gendesign", + project: process.env.SENTRY_PROJECT ?? "frontend", + sentryUrl: process.env.SENTRY_URL ?? "https://errors.gendsgn.ru/", + // Only upload source maps when auth token is present (CI builds). + // Without authToken the upload step is silently skipped. + authToken: process.env.GLITCHTIP_AUTH_TOKEN, + silent: !process.env.CI, + widenClientFileUpload: true, + // Delete generated source maps from the build output after upload to + // prevent them from being served publicly. + sourcemaps: { + deleteSourcemapsAfterUpload: true, + }, + // No tunnel route — GlitchTip is not behind a CORS proxy + tunnelRoute: undefined, + // Webpack-specific options + webpack: { + // Treeshake Sentry SDK logger statements from the bundle + treeshake: { + removeDebugLogging: true, + }, + // Not deploying on Vercel — no cron monitors needed + automaticVercelMonitors: false, + }, +}); diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 300b1295..7c330e20 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -8,11 +8,13 @@ "name": "gendesign-frontend", "version": "0.1.0", "dependencies": { + "@sentry/nextjs": "^10.53.1", "@tanstack/react-query": "^5.50.0", "echarts": "^6.0.0", "echarts-for-react": "^3.0.6", "leaflet": "^1.9.4", "leaflet-draw": "^1.0.4", + "lucide-react": "^0.511.0", "next": "^15.0.0", "react": "^19.0.0", "react-dom": "^19.0.0", @@ -31,7 +33,7 @@ "openapi-typescript": "^7.0.0", "postcss": "^8.4.0", "tailwindcss": "^4.0.0", - "typescript": "^5.5.0" + "typescript": "5.9.3" } }, "node_modules/@alloc/quick-lru": { @@ -47,6 +49,230 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/@babel/code-frame": { + "version": "7.29.0", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.0.tgz", + "integrity": "sha512-9NhCeYjq9+3uxgdtp20LSiJXJvN0FeCtNGpJxuMFZ1Kv3cWUNb6DOhJwUvcVCzKGR66cw4njwM6hrJLqgOwbcw==", + "license": "MIT", + "dependencies": { + "@babel/helper-validator-identifier": "^7.28.5", + "js-tokens": "^4.0.0", + "picocolors": "^1.1.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/compat-data": { + "version": "7.29.3", + "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.29.3.tgz", + "integrity": "sha512-LIVqM46zQWZhj17qA8wb4nW/ixr2y1Nw+r1etiAWgRM6U1IqP+LNhL1yg440jYZR72jCWcWbLWzIosH+uP1fqg==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/core": { + "version": "7.29.0", + "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.29.0.tgz", + "integrity": "sha512-CGOfOJqWjg2qW/Mb6zNsDm+u5vFQ8DxXfbM09z69p5Z6+mE1ikP2jUXw+j42Pf1XTYED2Rni5f95npYeuwMDQA==", + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.29.0", + "@babel/generator": "^7.29.0", + "@babel/helper-compilation-targets": "^7.28.6", + "@babel/helper-module-transforms": "^7.28.6", + "@babel/helpers": "^7.28.6", + "@babel/parser": "^7.29.0", + "@babel/template": "^7.28.6", + "@babel/traverse": "^7.29.0", + "@babel/types": "^7.29.0", + "@jridgewell/remapping": "^2.3.5", + "convert-source-map": "^2.0.0", + "debug": "^4.1.0", + "gensync": "^1.0.0-beta.2", + "json5": "^2.2.3", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/babel" + } + }, + "node_modules/@babel/generator": { + "version": "7.29.1", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.29.1.tgz", + "integrity": "sha512-qsaF+9Qcm2Qv8SRIMMscAvG4O3lJ0F1GuMo5HR/Bp02LopNgnZBC/EkbevHFeGs4ls/oPz9v+Bsmzbkbe+0dUw==", + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.29.0", + "@babel/types": "^7.29.0", + "@jridgewell/gen-mapping": "^0.3.12", + "@jridgewell/trace-mapping": "^0.3.28", + "jsesc": "^3.0.2" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-compilation-targets": { + "version": "7.28.6", + "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.28.6.tgz", + "integrity": "sha512-JYtls3hqi15fcx5GaSNL7SCTJ2MNmjrkHXg4FSpOA/grxK8KwyZ5bubHsCq8FXCkua6xhuaaBit+3b7+VZRfcA==", + "license": "MIT", + "dependencies": { + "@babel/compat-data": "^7.28.6", + "@babel/helper-validator-option": "^7.27.1", + "browserslist": "^4.24.0", + "lru-cache": "^5.1.1", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-globals": { + "version": "7.28.0", + "resolved": "https://registry.npmjs.org/@babel/helper-globals/-/helper-globals-7.28.0.tgz", + "integrity": "sha512-+W6cISkXFa1jXsDEdYA8HeevQT/FULhxzR99pxphltZcVaugps53THCeiWA8SguxxpSp3gKPiuYfSWopkLQ4hw==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-imports": { + "version": "7.28.6", + "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.28.6.tgz", + "integrity": "sha512-l5XkZK7r7wa9LucGw9LwZyyCUscb4x37JWTPz7swwFE/0FMQAGpiWUZn8u9DzkSBWEcK25jmvubfpw2dnAMdbw==", + "license": "MIT", + "dependencies": { + "@babel/traverse": "^7.28.6", + "@babel/types": "^7.28.6" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-transforms": { + "version": "7.28.6", + "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.28.6.tgz", + "integrity": "sha512-67oXFAYr2cDLDVGLXTEABjdBJZ6drElUSI7WKp70NrpyISso3plG9SAGEF6y7zbha/wOzUByWWTJvEDVNIUGcA==", + "license": "MIT", + "dependencies": { + "@babel/helper-module-imports": "^7.28.6", + "@babel/helper-validator-identifier": "^7.28.5", + "@babel/traverse": "^7.28.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/helper-string-parser": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.27.1.tgz", + "integrity": "sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.28.5", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.28.5.tgz", + "integrity": "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-option": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.27.1.tgz", + "integrity": "sha512-YvjJow9FxbhFFKDSuFnVCe2WxXk1zWc22fFePVNEaWJEu8IrZVlda6N0uHwzZrUM1il7NC9Mlp4MaJYbYd9JSg==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helpers": { + "version": "7.29.2", + "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.29.2.tgz", + "integrity": "sha512-HoGuUs4sCZNezVEKdVcwqmZN8GoHirLUcLaYVNBK2J0DadGtdcqgr3BCbvH8+XUo4NGjNl3VOtSjEKNzqfFgKw==", + "license": "MIT", + "dependencies": { + "@babel/template": "^7.28.6", + "@babel/types": "^7.29.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/parser": { + "version": "7.29.3", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.29.3.tgz", + "integrity": "sha512-b3ctpQwp+PROvU/cttc4OYl4MzfJUWy6FZg+PMXfzmt/+39iHVF0sDfqay8TQM3JA2EUOyKcFZt75jWriQijsA==", + "license": "MIT", + "dependencies": { + "@babel/types": "^7.29.0" + }, + "bin": { + "parser": "bin/babel-parser.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/template": { + "version": "7.28.6", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.28.6.tgz", + "integrity": "sha512-YA6Ma2KsCdGb+WC6UpBVFJGXL58MDA6oyONbjyF/+5sBgxY/dwkhLogbMT2GXXyU84/IhRw/2D1Os1B/giz+BQ==", + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.28.6", + "@babel/parser": "^7.28.6", + "@babel/types": "^7.28.6" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/traverse": { + "version": "7.29.0", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.29.0.tgz", + "integrity": "sha512-4HPiQr0X7+waHfyXPZpWPfWL/J7dcN1mx9gL6WdQVMbPnF3+ZhSMs8tCxN7oHddJE9fhNE7+lxdnlyemKfJRuA==", + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.29.0", + "@babel/generator": "^7.29.0", + "@babel/helper-globals": "^7.28.0", + "@babel/parser": "^7.29.0", + "@babel/template": "^7.28.6", + "@babel/types": "^7.29.0", + "debug": "^4.3.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/types": { + "version": "7.29.0", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.29.0.tgz", + "integrity": "sha512-LwdZHpScM4Qz8Xw2iKSzS+cfglZzJGvofQICy7W7v4caru4EaAmyUuO6BGrbyQ2mYV11W0U8j5mBhd14dd3B0A==", + "license": "MIT", + "dependencies": { + "@babel/helper-string-parser": "^7.27.1", + "@babel/helper-validator-identifier": "^7.28.5" + }, + "engines": { + "node": ">=6.9.0" + } + }, "node_modules/@emnapi/core": { "dev": true, "optional": true @@ -152,13 +378,6 @@ "node": "^18.18.0 || ^20.9.0 || >=21.1.0" } }, - "node_modules/@eslint/core/node_modules/@types/json-schema": { - "version": "7.0.15", - "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", - "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", - "dev": true, - "license": "MIT" - }, "node_modules/@eslint/eslintrc": { "version": "3.3.5", "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-3.3.5.tgz", @@ -343,6 +562,87 @@ "node": ">= 0.8.0" } }, + "node_modules/@fastify/otel": { + "version": "0.18.0", + "resolved": "https://registry.npmjs.org/@fastify/otel/-/otel-0.18.0.tgz", + "integrity": "sha512-3TASCATfw+ctICSb4ymrv7iCm0qJ0N9CarB+CZ7zIJ7KqNbwI5JjyDL1/sxoC0ccTO1Zyd1iQ+oqncPg5FJXaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "MIT", + "dependencies": { + "@opentelemetry/core": "^2.0.0", + "@opentelemetry/instrumentation": "^0.212.0", + "@opentelemetry/semantic-conventions": "^1.28.0", + "minimatch": "^10.2.4" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.9.0" + } + }, + "node_modules/@fastify/otel/node_modules/@opentelemetry/api-logs": { + "version": "0.212.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/api-logs/-/api-logs-0.212.0.tgz", + "integrity": "sha512-TEEVrLbNROUkYY51sBJGk7lO/OLjuepch8+hmpM6ffMJQ2z/KVCjdHuCFX6fJj8OkJP2zckPjrJzQtXU3IAsFg==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api": "^1.3.0" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@fastify/otel/node_modules/@opentelemetry/instrumentation": { + "version": "0.212.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation/-/instrumentation-0.212.0.tgz", + "integrity": "sha512-IyXmpNnifNouMOe0I/gX7ENfv2ZCNdYTF0FpCsoBcpbIHzk81Ww9rQTYTnvghszCg7qGrIhNvWC8dhEifgX9Jg==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api-logs": "0.212.0", + "import-in-the-middle": "^2.0.6", + "require-in-the-middle": "^8.0.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@fastify/otel/node_modules/import-in-the-middle": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/import-in-the-middle/-/import-in-the-middle-2.0.6.tgz", + "integrity": "sha512-3vZV3jX0XRFW3EJDTwzWoZa+RH1b8eTTx6YOCjglrLyPuepwoBti1k3L2dKwdCUrnVEfc5CuRuGstaC/uQJJaw==", + "license": "Apache-2.0", + "dependencies": { + "acorn": "^8.15.0", + "acorn-import-attributes": "^1.9.5", + "cjs-module-lexer": "^2.2.0", + "module-details-from-path": "^1.0.4" + } + }, + "node_modules/@fastify/otel/node_modules/minimatch": { + "version": "10.2.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.5.tgz", + "integrity": "sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==", + "license": "BlueOak-1.0.0", + "dependencies": { + "brace-expansion": "^5.0.5" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/@humanfs/node": { "version": "0.16.8", "resolved": "https://registry.npmjs.org/@humanfs/node/-/node-0.16.8.tgz", @@ -409,6 +709,62 @@ "url": "https://github.com/sponsors/nzakas" } }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.3.13", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", + "integrity": "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==", + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.0", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/remapping": { + "version": "2.3.5", + "resolved": "https://registry.npmjs.org/@jridgewell/remapping/-/remapping-2.3.5.tgz", + "integrity": "sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ==", + "license": "MIT", + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/source-map": { + "version": "0.3.11", + "resolved": "https://registry.npmjs.org/@jridgewell/source-map/-/source-map-0.3.11.tgz", + "integrity": "sha512-ZMp1V8ZFcPG5dIWnQLr3NSI1MiCU7UETdS/A0G8V/XWHvJv3ZsFqutJn1Y5RPmAPX6F3BiE397OqveU/9NCuIA==", + "license": "MIT", + "peer": true, + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.25" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.5.5", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", + "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.31", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", + "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, "node_modules/@napi-rs/wasm-runtime": { "dev": true, "optional": true @@ -777,6 +1133,483 @@ "node": ">= 10" } }, + "node_modules/@opentelemetry/api": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.1.tgz", + "integrity": "sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==", + "license": "Apache-2.0", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@opentelemetry/api-logs": { + "version": "0.214.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/api-logs/-/api-logs-0.214.0.tgz", + "integrity": "sha512-40lSJeqYO8Uz2Yj7u94/SJWE/wONa7rmMKjI1ZcIjgf3MHNHv1OZUCrCETGuaRF62d5pQD1wKIW+L4lmSMTzZA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api": "^1.3.0" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@opentelemetry/core": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.7.1.tgz", + "integrity": "sha512-QAqIj32AtK6+pEVNG7EOVxHdE06RP+FM5qpiEJ4RtDcFIqKUZHYhl7/7UY5efhwmwNAg7j8QbJVBLxMerc0+gw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/instrumentation": { + "version": "0.214.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation/-/instrumentation-0.214.0.tgz", + "integrity": "sha512-MHqEX5Dk59cqVah5LiARMACku7jXSVk9iVDWOea4x3cr7VfdByeDCURK6o1lntT1JS/Tsovw01UJrBhN3/uC5w==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api-logs": "0.214.0", + "import-in-the-middle": "^3.0.0", + "require-in-the-middle": "^8.0.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/instrumentation-amqplib": { + "version": "0.61.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-amqplib/-/instrumentation-amqplib-0.61.0.tgz", + "integrity": "sha512-mCKoyTGfRNisge4br0NpOFSy2Z1NnEW8hbCJdUDdJFHrPqVzc4IIBPA/vX0U+LUcQqrQvJX+HMIU0dbDRe0i0Q==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "^2.0.0", + "@opentelemetry/instrumentation": "^0.214.0", + "@opentelemetry/semantic-conventions": "^1.33.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/instrumentation-connect": { + "version": "0.57.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-connect/-/instrumentation-connect-0.57.0.tgz", + "integrity": "sha512-FMEBChnI4FLN5TE9DHwfH7QpNir1JzXno1uz/TAucVdLCyrG0jTrKIcNHt/i30A0M2AunNBCkcd8Ei26dIPKdg==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "^2.0.0", + "@opentelemetry/instrumentation": "^0.214.0", + "@opentelemetry/semantic-conventions": "^1.27.0", + "@types/connect": "3.4.38" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/instrumentation-dataloader": { + "version": "0.31.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-dataloader/-/instrumentation-dataloader-0.31.0.tgz", + "integrity": "sha512-f654tZFQXS5YeLDNb9KySrwtg7SnqZN119FauD7acBoTzuLduaiGTNz88ixcVSOOMGZ+EjJu/RFtx5klObC95g==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/instrumentation": "^0.214.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/instrumentation-fs": { + "version": "0.33.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-fs/-/instrumentation-fs-0.33.0.tgz", + "integrity": "sha512-sCZWXGalQ01wr3tAhSR9ucqFJ0phidpAle6/17HVjD6gN8FLmZMK/8sKxdXYHy3PbnlV1P4zeiSVFNKpbFMNLA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "^2.0.0", + "@opentelemetry/instrumentation": "^0.214.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/instrumentation-generic-pool": { + "version": "0.57.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-generic-pool/-/instrumentation-generic-pool-0.57.0.tgz", + "integrity": "sha512-orhmlaK+ZIW9hKU+nHTbXrCSXZcH83AescTqmpamHRobRmYSQwRbD0a1odc0yAzuzOtxYiHiXAnpnIpaSSY7Ow==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/instrumentation": "^0.214.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/instrumentation-graphql": { + "version": "0.62.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-graphql/-/instrumentation-graphql-0.62.0.tgz", + "integrity": "sha512-3YNuLVPUxafXkH1jBAbGsKNsP3XVzcFDhCDCE3OqBwCwShlqQbLMRMFh1T/d5jaVZiGVmSsfof+ICKD2iOV8xg==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/instrumentation": "^0.214.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/instrumentation-hapi": { + "version": "0.60.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-hapi/-/instrumentation-hapi-0.60.0.tgz", + "integrity": "sha512-aNljZKYrEa7obLAxd1bCEDxF7kzCLGXTuTJZ8lMR9rIVEjmuKBXN1gfqpm/OB//Zc2zP4iIve1jBp7sr3mQV6w==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "^2.0.0", + "@opentelemetry/instrumentation": "^0.214.0", + "@opentelemetry/semantic-conventions": "^1.27.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/instrumentation-http": { + "version": "0.214.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-http/-/instrumentation-http-0.214.0.tgz", + "integrity": "sha512-FlkDhZDRjDJDcO2LcSCtjRpkal1NJ8y0fBqBhTvfAR3JSYY2jAIj1kSS5IjmEBt4c3aWv+u/lqLuoCDrrKCSKg==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.6.1", + "@opentelemetry/instrumentation": "0.214.0", + "@opentelemetry/semantic-conventions": "^1.29.0", + "forwarded-parse": "2.1.2" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/instrumentation-http/node_modules/@opentelemetry/core": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.6.1.tgz", + "integrity": "sha512-8xHSGWpJP9wBxgBpnqGL0R3PbdWQndL1Qp50qrg71+B28zK5OQmUgcDKLJgzyAAV38t4tOyLMGDD60LneR5W8g==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/instrumentation-kafkajs": { + "version": "0.23.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-kafkajs/-/instrumentation-kafkajs-0.23.0.tgz", + "integrity": "sha512-4K+nVo+zI+aDz0Z85SObwbdixIbzS9moIuKJaYsdlzcHYnKOPtB7ya8r8Ezivy/GVIBHiKJVq4tv+BEkgOMLaQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/instrumentation": "^0.214.0", + "@opentelemetry/semantic-conventions": "^1.30.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/instrumentation-knex": { + "version": "0.58.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-knex/-/instrumentation-knex-0.58.0.tgz", + "integrity": "sha512-Hc/o8fSsaWxZ8r1Yw4rNDLwTpUopTf4X32y4W6UhlHmW8Wizz8wfhgOKIelSeqFVTKBBPIDUOsQWuIMxBmu8Bw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/instrumentation": "^0.214.0", + "@opentelemetry/semantic-conventions": "^1.33.1" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/instrumentation-koa": { + "version": "0.62.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-koa/-/instrumentation-koa-0.62.0.tgz", + "integrity": "sha512-uVip0VuGUQXZ+vFxkKxAUNq8qNl+VFlyHDh/U6IQ8COOEDfbEchdaHnpFrMYF3psZRUuoSIgb7xOeXj00RdwDA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "^2.0.0", + "@opentelemetry/instrumentation": "^0.214.0", + "@opentelemetry/semantic-conventions": "^1.36.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.9.0" + } + }, + "node_modules/@opentelemetry/instrumentation-lru-memoizer": { + "version": "0.58.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-lru-memoizer/-/instrumentation-lru-memoizer-0.58.0.tgz", + "integrity": "sha512-6grM3TdMyHzlGY1cUA+mwoPueB1F3dYKgKtZIH6jOFXqfHAByyLTc+6PFjGM9tKh52CFBJaDwodNlL/Td39z7Q==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/instrumentation": "^0.214.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/instrumentation-mongodb": { + "version": "0.67.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-mongodb/-/instrumentation-mongodb-0.67.0.tgz", + "integrity": "sha512-1WJp5N1lYfHq2IhECOTewFs5Tf2NfUOwQRqs/rZdXKTezArMlucxgzAaqcgp3A3YREXopXTpXHsxZTGHjNhMdQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/instrumentation": "^0.214.0", + "@opentelemetry/semantic-conventions": "^1.33.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/instrumentation-mongoose": { + "version": "0.60.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-mongoose/-/instrumentation-mongoose-0.60.0.tgz", + "integrity": "sha512-8BahAZpKsOoc+lrZGb7Ofn4g3z8qtp5IxDfvAVpKXsEheQN7ONMH5djT5ihy6yf8yyeQJGS0gXFfpEAEeEHqQg==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "^2.0.0", + "@opentelemetry/instrumentation": "^0.214.0", + "@opentelemetry/semantic-conventions": "^1.33.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/instrumentation-mysql": { + "version": "0.60.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-mysql/-/instrumentation-mysql-0.60.0.tgz", + "integrity": "sha512-08pO8GFPEIz2zquKDGteBZDNmwketdgH8hTe9rVYgW9kCJXq1Psj3wPQGx+VaX4ZJKCfPeoLMYup9+cxHvZyVQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/instrumentation": "^0.214.0", + "@opentelemetry/semantic-conventions": "^1.33.0", + "@types/mysql": "2.15.27" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/instrumentation-mysql2": { + "version": "0.60.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-mysql2/-/instrumentation-mysql2-0.60.0.tgz", + "integrity": "sha512-m/5d3bxQALllCzezYDk/6vajh0tj5OijMMvOZGr+qN1NMXm1dzMNwyJ0gNZW7Fo3YFRyj/jJMxIw+W7d525dlw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/instrumentation": "^0.214.0", + "@opentelemetry/semantic-conventions": "^1.33.0", + "@opentelemetry/sql-common": "^0.41.2" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/instrumentation-pg": { + "version": "0.66.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-pg/-/instrumentation-pg-0.66.0.tgz", + "integrity": "sha512-KxfLGXBb7k2ueaPJfq2GXBDXBly8P+SpR/4Mj410hhNgmQF3sCqwXvUBQxZQkDAmsdBAoenM+yV1LhtsMRamcA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "^2.0.0", + "@opentelemetry/instrumentation": "^0.214.0", + "@opentelemetry/semantic-conventions": "^1.34.0", + "@opentelemetry/sql-common": "^0.41.2", + "@types/pg": "8.15.6", + "@types/pg-pool": "2.0.7" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/instrumentation-tedious": { + "version": "0.33.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-tedious/-/instrumentation-tedious-0.33.0.tgz", + "integrity": "sha512-Q6WQwAD01MMTub31GlejoiFACYNw26J426wyjvU7by7fDIr2nZXNW4vhTGs7i7F0TnXBO3xN688g1tdUgYwJ5w==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/instrumentation": "^0.214.0", + "@opentelemetry/semantic-conventions": "^1.33.0", + "@types/tedious": "^4.0.14" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/resources": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-2.7.1.tgz", + "integrity": "sha512-DeT6KKolmC4e/dRQvMQ/RwlnzhaqeiFOXY5ngoOPJ07GgVVKxZOg9EcrNZb5aTzUn+iCrJldAgOfQm1O/QfPAQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.7.1", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-trace-base": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-2.7.1.tgz", + "integrity": "sha512-NAYIlsF8MPUsKqJMiDQJTMPOmlbawC1Iz/omMLygZ1C9am8fTKYjTaI+OZM+WTY3t3Glo0wnOg/6/pac6RGPPw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.7.1", + "@opentelemetry/resources": "2.7.1", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/semantic-conventions": { + "version": "1.41.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.41.1.tgz", + "integrity": "sha512-/UhIkaZgPutTFmQ7RnIJGgDXZmtEJ7Dvi86xNTFWcnRxVRNk/aotsqDJYeEvDP+FSMB2SdW+pQzNMcWP0rwuNA==", + "license": "Apache-2.0", + "engines": { + "node": ">=14" + } + }, + "node_modules/@opentelemetry/sql-common": { + "version": "0.41.2", + "resolved": "https://registry.npmjs.org/@opentelemetry/sql-common/-/sql-common-0.41.2.tgz", + "integrity": "sha512-4mhWm3Z8z+i508zQJ7r6Xi7y4mmoJpdvH0fZPFRkWrdp5fq7hhZ2HhYokEOLkfqSMgPR4Z9EyB3DBkbKGOqZiQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "^2.0.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.1.0" + } + }, + "node_modules/@prisma/instrumentation": { + "version": "7.6.0", + "resolved": "https://registry.npmjs.org/@prisma/instrumentation/-/instrumentation-7.6.0.tgz", + "integrity": "sha512-ZPW2gRiwpPzEfgeZgaekhqXrbW+Y2RJKHVqUmlhZhKzRNCcvR6DykzylDrynpArKKRQtLxoZy36fK7U0p3pdgQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/instrumentation": "^0.207.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.8" + } + }, + "node_modules/@prisma/instrumentation/node_modules/@opentelemetry/api-logs": { + "version": "0.207.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/api-logs/-/api-logs-0.207.0.tgz", + "integrity": "sha512-lAb0jQRVyleQQGiuuvCOTDVspc14nx6XJjP4FspJ1sNARo3Regq4ZZbrc3rN4b1TYSuUCvgH+UXUPug4SLOqEQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api": "^1.3.0" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@prisma/instrumentation/node_modules/@opentelemetry/instrumentation": { + "version": "0.207.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation/-/instrumentation-0.207.0.tgz", + "integrity": "sha512-y6eeli9+TLKnznrR8AZlQMSJT7wILpXH+6EYq5Vf/4Ao+huI7EedxQHwRgVUOMLFbe7VFDvHJrX9/f4lcwnJsA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api-logs": "0.207.0", + "import-in-the-middle": "^2.0.0", + "require-in-the-middle": "^8.0.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@prisma/instrumentation/node_modules/import-in-the-middle": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/import-in-the-middle/-/import-in-the-middle-2.0.6.tgz", + "integrity": "sha512-3vZV3jX0XRFW3EJDTwzWoZa+RH1b8eTTx6YOCjglrLyPuepwoBti1k3L2dKwdCUrnVEfc5CuRuGstaC/uQJJaw==", + "license": "Apache-2.0", + "dependencies": { + "acorn": "^8.15.0", + "acorn-import-attributes": "^1.9.5", + "cjs-module-lexer": "^2.2.0", + "module-details-from-path": "^1.0.4" + } + }, "node_modules/@react-leaflet/core": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/@react-leaflet/core/-/core-3.0.0.tgz", @@ -912,13 +1745,6 @@ "js-yaml": "bin/js-yaml.js" } }, - "node_modules/@redocly/openapi-core/node_modules/json-schema-traverse": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", - "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", - "dev": true, - "license": "MIT" - }, "node_modules/@redocly/openapi-core/node_modules/minimatch": { "version": "5.1.9", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.9.tgz", @@ -942,16 +1768,6 @@ "node": ">=4" } }, - "node_modules/@redocly/openapi-core/node_modules/require-from-string": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", - "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/@redocly/openapi-core/node_modules/uri-js-replace": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/uri-js-replace/-/uri-js-replace-1.0.1.tgz", @@ -966,6 +1782,418 @@ "dev": true, "license": "Apache-2.0" }, + "node_modules/@rollup/plugin-commonjs": { + "version": "28.0.1", + "resolved": "https://registry.npmjs.org/@rollup/plugin-commonjs/-/plugin-commonjs-28.0.1.tgz", + "integrity": "sha512-+tNWdlWKbpB3WgBN7ijjYkq9X5uhjmcvyjEght4NmH5fAU++zfQzAJ6wumLS+dNcvwEZhKx2Z+skY8m7v0wGSA==", + "license": "MIT", + "dependencies": { + "@rollup/pluginutils": "^5.0.1", + "commondir": "^1.0.1", + "estree-walker": "^2.0.2", + "fdir": "^6.2.0", + "is-reference": "1.2.1", + "magic-string": "^0.30.3", + "picomatch": "^4.0.2" + }, + "engines": { + "node": ">=16.0.0 || 14 >= 14.17" + }, + "peerDependencies": { + "rollup": "^2.68.0||^3.0.0||^4.0.0" + }, + "peerDependenciesMeta": { + "rollup": { + "optional": true + } + } + }, + "node_modules/@rollup/pluginutils": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/@rollup/pluginutils/-/pluginutils-5.3.0.tgz", + "integrity": "sha512-5EdhGZtnu3V88ces7s53hhfK5KSASnJZv8Lulpc04cWO3REESroJXg73DFsOmgbU2BhwV0E20bu2IDZb3VKW4Q==", + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0", + "estree-walker": "^2.0.2", + "picomatch": "^4.0.2" + }, + "engines": { + "node": ">=14.0.0" + }, + "peerDependencies": { + "rollup": "^1.20.0||^2.0.0||^3.0.0||^4.0.0" + }, + "peerDependenciesMeta": { + "rollup": { + "optional": true + } + } + }, + "node_modules/@rollup/rollup-android-arm-eabi": { + "version": "4.60.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.60.4.tgz", + "integrity": "sha512-F5QXMSiFebS9hKZj02XhWLLnRpJ3B3AROP0tWbFBSj+6kCbg5m9j5JoHKd4mmSVy5mS/IMQloYgYxCuJC0fxEQ==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-android-arm64": { + "version": "4.60.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.60.4.tgz", + "integrity": "sha512-GxxTKApUpzRhof7poWvCJHRF51C67u1R7D6DiluBE8wKU1u5GWE8t+v81JvJYtbawoBFX1hLv5Ei4eVjkWokaw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-darwin-arm64": { + "version": "4.60.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.60.4.tgz", + "integrity": "sha512-tua0TaJxMOB1R0V0RS1jFZ/RpURFDJIOR2A6jWwQeawuFyS4gBW+rntLRaQd0EQ4bd6Vp44Z2rXW+YYDBsj6IA==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-darwin-x64": { + "version": "4.60.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.60.4.tgz", + "integrity": "sha512-CSKq7MsP+5PFIcydhAiR1K0UhEI1A2jWXVKHPCBZ151yOutENwvnPocgVHkivu2kviURtCEB6zUQw0vs8RrhMg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-freebsd-arm64": { + "version": "4.60.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.60.4.tgz", + "integrity": "sha512-+O8OkVdyvXMtJEciu2wS/pzm1IxntEEQx3z5TAVy4l32G0etZn+RsA48ARRrFm6Ri8fvqPQfgrvNxSjKAbnd3g==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-freebsd-x64": { + "version": "4.60.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.60.4.tgz", + "integrity": "sha512-Iw3oMskH3AfNuhU0MSN7vNbdi4me/NiYo2azqPz/Le16zHSa+3RRmliCMWWQmh4lcndccU40xcJuTYJZxNo/lw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-linux-arm-gnueabihf": { + "version": "4.60.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.60.4.tgz", + "integrity": "sha512-EIPRXTVQpHyF8WOo219AD2yEltPehLTcTMz2fn6JsatLYSzQf00hj3rulF+yauOlF9/FtM2WpkT/hJh/KJFGhA==", + "cpu": [ + "arm" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm-musleabihf": { + "version": "4.60.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.60.4.tgz", + "integrity": "sha512-J3Yh9PzzF1Ovah2At+lHiGQdsYgArxBbXv/zHfSyaiFQEqvNv7DcW98pCrmdjCZBrqBiKrKKe2V+aaSGWuBe/w==", + "cpu": [ + "arm" + ], + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-gnu": { + "version": "4.60.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.60.4.tgz", + "integrity": "sha512-BFDEZMYfUvLn37ONE1yMBojPxnMlTFsdyNoqncT0qFq1mAfllL+ATMMJd8TeuVMiX84s1KbcxcZbXInmcO2mRg==", + "cpu": [ + "arm64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-musl": { + "version": "4.60.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.60.4.tgz", + "integrity": "sha512-pc9EYOSlOgdQ2uPl1o9PF6/kLSgaUosia7gOuS8mB69IxJvlclko1MECXysjs5ryez1/5zjYqx3+xYU0TU6R1A==", + "cpu": [ + "arm64" + ], + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-gnu": { + "version": "4.60.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.60.4.tgz", + "integrity": "sha512-NxnomyxYerDh5n4iLrNa+sH+Z+U4BMEE46V2PgQ/hoB909i8gV1M5wPojWg9fk1jWpO3IQnOs20K4wyZuFLEFQ==", + "cpu": [ + "loong64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-musl": { + "version": "4.60.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-musl/-/rollup-linux-loong64-musl-4.60.4.tgz", + "integrity": "sha512-nbJnQ8a3z1mtmrwImCYhc6BGpThAyYVRQxw9uKSKG4wR6aAYno9sVjJ0zaZcW9BPJX1GbrDPf+SvdWjgTuDmnw==", + "cpu": [ + "loong64" + ], + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-gnu": { + "version": "4.60.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.60.4.tgz", + "integrity": "sha512-2EU6acNrQLd8tYvo/LXW535wupT3m6fo7HKo6lr7ktQoItxTyOL1ZCR/GfGCuXl2vR+zmfI6eRXkSemafv+iVg==", + "cpu": [ + "ppc64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-musl": { + "version": "4.60.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-musl/-/rollup-linux-ppc64-musl-4.60.4.tgz", + "integrity": "sha512-WeBtoMuaMxiiIrO2IYP3xs6GMWkJP2C0EoT8beTLkUPmzV1i/UcOSVw1d5r9KBODtHKilG5yFxsGRnBbK3wJ4A==", + "cpu": [ + "ppc64" + ], + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-gnu": { + "version": "4.60.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.60.4.tgz", + "integrity": "sha512-FJHFfqpKUI3A10WrWKiFbBZ7yVbGT4q4B5o1qKFFojqpaYoh9LrQgqWCmmcxQzVSXYtyB5bzkXrYzlHTs21MYA==", + "cpu": [ + "riscv64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-musl": { + "version": "4.60.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.60.4.tgz", + "integrity": "sha512-mcEl6CUT5IAUmQf1m9FYSmVqCJlpQ8r8eyftFUHG8i9OhY7BkBXSUdnLH5DOf0wCOjcP9v/QO93zpmF1SptCCw==", + "cpu": [ + "riscv64" + ], + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-s390x-gnu": { + "version": "4.60.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.60.4.tgz", + "integrity": "sha512-ynt3JxVd2w2buzoKDWIyiV1pJW93xlQic1THVLXilz429oijRpSHivZAgp65KBu+cMcgf1eVVjdnTLvPxgCuoQ==", + "cpu": [ + "s390x" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-gnu": { + "version": "4.60.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.60.4.tgz", + "integrity": "sha512-Boiz5+MsaROEWDf+GGEwF8VMHGhlUoQMtIPjOgA5fv4osupqTVnJteQNKJwUcnUog2G55jYXH7KZFFiJe0TEzQ==", + "cpu": [ + "x64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-musl": { + "version": "4.60.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.60.4.tgz", + "integrity": "sha512-+qfSY27qIrFfI/Hom04KYFw3GKZSGU4lXus51wsb5EuySfFlWRwjkKWoE9emgRw/ukoT4Udsj4W/+xxG8VbPKg==", + "cpu": [ + "x64" + ], + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-openbsd-x64": { + "version": "4.60.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openbsd-x64/-/rollup-openbsd-x64-4.60.4.tgz", + "integrity": "sha512-VpTfOPHgVXEBeeR8hZ2O0F3aSso+JDWqTWmTmzcQKted54IAdUVbxE+j/MVxUsKa8L20HJhv3vUezVPoquqWjA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ] + }, + "node_modules/@rollup/rollup-openharmony-arm64": { + "version": "4.60.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.60.4.tgz", + "integrity": "sha512-IPOsh5aRYuLv/nkU51X10Bf75Bsf6+gZdx1X+QP5QM6lIJFHHqbHLG0uJn/hWthzo13UAc2umiUorqZy3axoZg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ] + }, + "node_modules/@rollup/rollup-win32-arm64-msvc": { + "version": "4.60.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.60.4.tgz", + "integrity": "sha512-4QzE9E81OohJ/HKzHhsqU+zcYYojVOXlFMs1DdyMT6qXl/niOH7AVElmmEdUNHHS/oRkc++d5k6Vy85zFs0DEw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-ia32-msvc": { + "version": "4.60.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.60.4.tgz", + "integrity": "sha512-zTPgT1YuHHcd+Tmx7h8aml0FWFVelV5N54oHow9SLj+GfoDy/huQ+UV396N/C7KpMDMiPspRktzM1/0r1usYEA==", + "cpu": [ + "ia32" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-gnu": { + "version": "4.60.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.60.4.tgz", + "integrity": "sha512-DRS4G7mi9lJxqEDezIkKCaUIKCrLUUDCUaCsTPCi/rtqaC6D/jjwslMQyiDU50Ka0JKpeXeRBFBAXwArY52vBw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-msvc": { + "version": "4.60.4", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.60.4.tgz", + "integrity": "sha512-QVTUovf40zgTqlFVrKA1uXMVvU2QWEFWfAH8Wdc48IxLvrJMQVMBRjuQyUpzZCDkakImib9eVazbWlC6ksWtJw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, "node_modules/@rushstack/eslint-patch": { "version": "1.16.1", "resolved": "https://registry.npmjs.org/@rushstack/eslint-patch/-/eslint-patch-1.16.1.tgz", @@ -973,6 +2201,445 @@ "dev": true, "license": "MIT" }, + "node_modules/@sentry-internal/browser-utils": { + "version": "10.53.1", + "resolved": "https://registry.npmjs.org/@sentry-internal/browser-utils/-/browser-utils-10.53.1.tgz", + "integrity": "sha512-X4d6y8sBMjmNhcDW4eMBU3ASsNIMz8dqaFkhyIMN/dkYr/yZKnbRZPaVuVUGvHKjnlficPpIH0/HK9KBjrYxPw==", + "license": "MIT", + "dependencies": { + "@sentry/core": "10.53.1" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@sentry-internal/feedback": { + "version": "10.53.1", + "resolved": "https://registry.npmjs.org/@sentry-internal/feedback/-/feedback-10.53.1.tgz", + "integrity": "sha512-vVpTI/aEYN5d9IgZeYJWMqVaN0+iFgidSrYNAsZTh1US5sJUzF/wrl+68KdpmCtFROrN3jiAn1oPSwL5CKvEJA==", + "license": "MIT", + "dependencies": { + "@sentry/core": "10.53.1" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@sentry-internal/replay": { + "version": "10.53.1", + "resolved": "https://registry.npmjs.org/@sentry-internal/replay/-/replay-10.53.1.tgz", + "integrity": "sha512-wZNzTBYkgGUPWMuUQv7L64+OJmoCnz7GQNiTrTFK6EVAjJXFBCSsPp/nhif0bLhbk8+0g4xz633uOhpXuQbFdw==", + "license": "MIT", + "dependencies": { + "@sentry-internal/browser-utils": "10.53.1", + "@sentry/core": "10.53.1" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@sentry-internal/replay-canvas": { + "version": "10.53.1", + "resolved": "https://registry.npmjs.org/@sentry-internal/replay-canvas/-/replay-canvas-10.53.1.tgz", + "integrity": "sha512-aueLaf/2prExwA76BGU5/bOXCKWqtt6jQXWA6WJQNrmKpPEtZJB4ypnpsou0McXQCF8tur2Y8U0TEkwQP13yJQ==", + "license": "MIT", + "dependencies": { + "@sentry-internal/replay": "10.53.1", + "@sentry/core": "10.53.1" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@sentry/babel-plugin-component-annotate": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/@sentry/babel-plugin-component-annotate/-/babel-plugin-component-annotate-5.3.0.tgz", + "integrity": "sha512-p4q8gn8wcFqZGP/s2MnJCAAd8fTikaU6A0mM97RDHQgStcrYiaS0Sc5zUNfb1V+UOLPuvdEdL6MwyxfzjYJQTA==", + "license": "MIT", + "engines": { + "node": ">= 18" + } + }, + "node_modules/@sentry/browser": { + "version": "10.53.1", + "resolved": "https://registry.npmjs.org/@sentry/browser/-/browser-10.53.1.tgz", + "integrity": "sha512-zXF373hzUOGzUOrqd8xb1U3LQi5uYC3mwv+z5OMKUUinQlu30tTWBs7ypy6YTchtix9QlYaHWlayUF8vBZ5UjA==", + "license": "MIT", + "dependencies": { + "@sentry-internal/browser-utils": "10.53.1", + "@sentry-internal/feedback": "10.53.1", + "@sentry-internal/replay": "10.53.1", + "@sentry-internal/replay-canvas": "10.53.1", + "@sentry/core": "10.53.1" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@sentry/bundler-plugin-core": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/@sentry/bundler-plugin-core/-/bundler-plugin-core-5.3.0.tgz", + "integrity": "sha512-L5T60sWdAI3qWwdg3Ptwek/0TY59PERrxyqp4XMUkroayQvGd9r5dIW9Q1kSeXX9iJ442nXbFZKAOyCKV4Z13Q==", + "license": "MIT", + "dependencies": { + "@babel/core": "^7.18.5", + "@sentry/babel-plugin-component-annotate": "5.3.0", + "@sentry/cli": "^2.58.5", + "dotenv": "^16.3.1", + "find-up": "^5.0.0", + "glob": "^13.0.6", + "magic-string": "~0.30.8" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@sentry/cli": { + "version": "2.58.5", + "resolved": "https://registry.npmjs.org/@sentry/cli/-/cli-2.58.5.tgz", + "integrity": "sha512-tavJ7yGUZV+z3Ct2/ZB6mg339i08sAk6HDkgqmSRuQEu2iLS5sl9HIvuXfM6xjv8fwlgFOSy++WNABNAcGHUbg==", + "hasInstallScript": true, + "license": "FSL-1.1-MIT", + "dependencies": { + "https-proxy-agent": "^5.0.0", + "node-fetch": "^2.6.7", + "progress": "^2.0.3", + "proxy-from-env": "^1.1.0", + "which": "^2.0.2" + }, + "bin": { + "sentry-cli": "bin/sentry-cli" + }, + "engines": { + "node": ">= 10" + }, + "optionalDependencies": { + "@sentry/cli-darwin": "2.58.5", + "@sentry/cli-linux-arm": "2.58.5", + "@sentry/cli-linux-arm64": "2.58.5", + "@sentry/cli-linux-i686": "2.58.5", + "@sentry/cli-linux-x64": "2.58.5", + "@sentry/cli-win32-arm64": "2.58.5", + "@sentry/cli-win32-i686": "2.58.5", + "@sentry/cli-win32-x64": "2.58.5" + } + }, + "node_modules/@sentry/cli-darwin": { + "version": "2.58.5", + "resolved": "https://registry.npmjs.org/@sentry/cli-darwin/-/cli-darwin-2.58.5.tgz", + "integrity": "sha512-lYrNzenZFJftfwSya7gwrHGxtE+Kob/e1sr9lmHMFOd4utDlmq0XFDllmdZAMf21fxcPRI1GL28ejZ3bId01fQ==", + "license": "FSL-1.1-MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=10" + } + }, + "node_modules/@sentry/cli-linux-arm": { + "version": "2.58.5", + "resolved": "https://registry.npmjs.org/@sentry/cli-linux-arm/-/cli-linux-arm-2.58.5.tgz", + "integrity": "sha512-KtHweSIomYL4WVDrBrYSYJricKAAzxUgX86kc6OnlikbyOhoK6Fy8Vs6vwd52P6dvWPjgrMpUYjW2M5pYXQDUw==", + "cpu": [ + "arm" + ], + "license": "FSL-1.1-MIT", + "optional": true, + "os": [ + "linux", + "freebsd", + "android" + ], + "engines": { + "node": ">=10" + } + }, + "node_modules/@sentry/cli-linux-arm64": { + "version": "2.58.5", + "resolved": "https://registry.npmjs.org/@sentry/cli-linux-arm64/-/cli-linux-arm64-2.58.5.tgz", + "integrity": "sha512-/4gywFeBqRB6tR/iGMRAJ3HRqY6Z7Yp4l8ZCbl0TDLAfHNxu7schEw4tSnm2/Hh9eNMiOVy4z58uzAWlZXAYBQ==", + "cpu": [ + "arm64" + ], + "license": "FSL-1.1-MIT", + "optional": true, + "os": [ + "linux", + "freebsd", + "android" + ], + "engines": { + "node": ">=10" + } + }, + "node_modules/@sentry/cli-linux-i686": { + "version": "2.58.5", + "resolved": "https://registry.npmjs.org/@sentry/cli-linux-i686/-/cli-linux-i686-2.58.5.tgz", + "integrity": "sha512-G7261dkmyxqlMdyvyP06b+RTIVzp1gZNgglj5UksxSouSUqRd/46W/2pQeOMPhloDYo9yLtCN2YFb3Mw4aUsWw==", + "cpu": [ + "x86", + "ia32" + ], + "license": "FSL-1.1-MIT", + "optional": true, + "os": [ + "linux", + "freebsd", + "android" + ], + "engines": { + "node": ">=10" + } + }, + "node_modules/@sentry/cli-linux-x64": { + "version": "2.58.5", + "resolved": "https://registry.npmjs.org/@sentry/cli-linux-x64/-/cli-linux-x64-2.58.5.tgz", + "integrity": "sha512-rP04494RSmt86xChkQ+ecBNRYSPbyXc4u0IA7R7N1pSLCyO74e5w5Al+LnAq35cMfVbZgz5Sm0iGLjyiUu4I1g==", + "cpu": [ + "x64" + ], + "license": "FSL-1.1-MIT", + "optional": true, + "os": [ + "linux", + "freebsd", + "android" + ], + "engines": { + "node": ">=10" + } + }, + "node_modules/@sentry/cli-win32-arm64": { + "version": "2.58.5", + "resolved": "https://registry.npmjs.org/@sentry/cli-win32-arm64/-/cli-win32-arm64-2.58.5.tgz", + "integrity": "sha512-AOJ2nCXlQL1KBaCzv38m3i2VmSHNurUpm7xVKd6yAHX+ZoVBI8VT0EgvwmtJR2TY2N2hNCC7UrgRmdUsQ152bA==", + "cpu": [ + "arm64" + ], + "license": "FSL-1.1-MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=10" + } + }, + "node_modules/@sentry/cli-win32-i686": { + "version": "2.58.5", + "resolved": "https://registry.npmjs.org/@sentry/cli-win32-i686/-/cli-win32-i686-2.58.5.tgz", + "integrity": "sha512-EsuboLSOnlrN7MMPJ1eFvfMDm+BnzOaSWl8eYhNo8W/BIrmNgpRUdBwnWn9Q2UOjJj5ZopukmsiMYtU/D7ml9g==", + "cpu": [ + "x86", + "ia32" + ], + "license": "FSL-1.1-MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=10" + } + }, + "node_modules/@sentry/cli-win32-x64": { + "version": "2.58.5", + "resolved": "https://registry.npmjs.org/@sentry/cli-win32-x64/-/cli-win32-x64-2.58.5.tgz", + "integrity": "sha512-IZf+XIMiQwj+5NzqbOQfywlOitmCV424Vtf9c+ep61AaVScUFD1TSrQbOcJJv5xGxhlxNOMNgMeZhdexdzrKZg==", + "cpu": [ + "x64" + ], + "license": "FSL-1.1-MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=10" + } + }, + "node_modules/@sentry/core": { + "version": "10.53.1", + "resolved": "https://registry.npmjs.org/@sentry/core/-/core-10.53.1.tgz", + "integrity": "sha512-XG4ezlkyuAPjBC5+9kXC94rXXuqYTw9NRhfaDHssbTFaGnqBR8vQX2UUgZfY7ucbeelRDGfBu1sywoU+mB04uA==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/@sentry/nextjs": { + "version": "10.53.1", + "resolved": "https://registry.npmjs.org/@sentry/nextjs/-/nextjs-10.53.1.tgz", + "integrity": "sha512-pkwqrpAG//LtW5W1Odud0PLLT+rnjDjodUEbScULHVaZE6/Gt+WGBMZmtzpNM+UwhsN19/4PyO7ocLTx/IFrkQ==", + "license": "MIT", + "dependencies": { + "@opentelemetry/api": "^1.9.1", + "@opentelemetry/semantic-conventions": "^1.40.0", + "@rollup/plugin-commonjs": "28.0.1", + "@sentry-internal/browser-utils": "10.53.1", + "@sentry/bundler-plugin-core": "^5.3.0", + "@sentry/core": "10.53.1", + "@sentry/node": "10.53.1", + "@sentry/opentelemetry": "10.53.1", + "@sentry/react": "10.53.1", + "@sentry/vercel-edge": "10.53.1", + "@sentry/webpack-plugin": "^5.3.0", + "rollup": "^4.60.3", + "stacktrace-parser": "^0.1.11" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "next": "^13.2.0 || ^14.0 || ^15.0.0-rc.0 || ^16.0.0-0" + } + }, + "node_modules/@sentry/node": { + "version": "10.53.1", + "resolved": "https://registry.npmjs.org/@sentry/node/-/node-10.53.1.tgz", + "integrity": "sha512-rxHVil0tJAmz+keFcZCj1LaUdgdkK66E/l0gqh2p1209PNCGoD3lnClFr6vusy1aF3zF8O9JPtuMEJzXOKhs+w==", + "license": "MIT", + "dependencies": { + "@fastify/otel": "0.18.0", + "@opentelemetry/api": "^1.9.1", + "@opentelemetry/core": "^2.6.1", + "@opentelemetry/instrumentation": "^0.214.0", + "@opentelemetry/instrumentation-amqplib": "0.61.0", + "@opentelemetry/instrumentation-connect": "0.57.0", + "@opentelemetry/instrumentation-dataloader": "0.31.0", + "@opentelemetry/instrumentation-fs": "0.33.0", + "@opentelemetry/instrumentation-generic-pool": "0.57.0", + "@opentelemetry/instrumentation-graphql": "0.62.0", + "@opentelemetry/instrumentation-hapi": "0.60.0", + "@opentelemetry/instrumentation-http": "0.214.0", + "@opentelemetry/instrumentation-kafkajs": "0.23.0", + "@opentelemetry/instrumentation-knex": "0.58.0", + "@opentelemetry/instrumentation-koa": "0.62.0", + "@opentelemetry/instrumentation-lru-memoizer": "0.58.0", + "@opentelemetry/instrumentation-mongodb": "0.67.0", + "@opentelemetry/instrumentation-mongoose": "0.60.0", + "@opentelemetry/instrumentation-mysql": "0.60.0", + "@opentelemetry/instrumentation-mysql2": "0.60.0", + "@opentelemetry/instrumentation-pg": "0.66.0", + "@opentelemetry/instrumentation-tedious": "0.33.0", + "@opentelemetry/sdk-trace-base": "^2.6.1", + "@opentelemetry/semantic-conventions": "^1.40.0", + "@prisma/instrumentation": "7.6.0", + "@sentry/core": "10.53.1", + "@sentry/node-core": "10.53.1", + "@sentry/opentelemetry": "10.53.1", + "import-in-the-middle": "^3.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@sentry/node-core": { + "version": "10.53.1", + "resolved": "https://registry.npmjs.org/@sentry/node-core/-/node-core-10.53.1.tgz", + "integrity": "sha512-iH7SMcM/7jPbN+t7+7ussQOiIqI4BMOGt4VYWlV71/z7k0pY+YPaSvlfVkNXfISiDzFAKv0ecCY3BmsLMu1xDQ==", + "license": "MIT", + "dependencies": { + "@sentry/core": "10.53.1", + "@sentry/opentelemetry": "10.53.1", + "import-in-the-middle": "^3.0.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.9.0", + "@opentelemetry/core": "^1.30.1 || ^2.1.0", + "@opentelemetry/exporter-trace-otlp-http": ">=0.57.0 <1", + "@opentelemetry/instrumentation": ">=0.57.1 <1", + "@opentelemetry/sdk-trace-base": "^1.30.1 || ^2.1.0", + "@opentelemetry/semantic-conventions": "^1.39.0" + }, + "peerDependenciesMeta": { + "@opentelemetry/api": { + "optional": true + }, + "@opentelemetry/core": { + "optional": true + }, + "@opentelemetry/exporter-trace-otlp-http": { + "optional": true + }, + "@opentelemetry/instrumentation": { + "optional": true + }, + "@opentelemetry/sdk-trace-base": { + "optional": true + }, + "@opentelemetry/semantic-conventions": { + "optional": true + } + } + }, + "node_modules/@sentry/opentelemetry": { + "version": "10.53.1", + "resolved": "https://registry.npmjs.org/@sentry/opentelemetry/-/opentelemetry-10.53.1.tgz", + "integrity": "sha512-Zok6UXla0mFOjd1YnVb1TZtQNOry9v93fXUqx8jmDaygwWM2BwvP8rBQabLz0/OZXo8+35oge+Vmw+QY5aesnA==", + "license": "MIT", + "dependencies": { + "@sentry/core": "10.53.1" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.9.0", + "@opentelemetry/core": "^1.30.1 || ^2.1.0", + "@opentelemetry/sdk-trace-base": "^1.30.1 || ^2.1.0", + "@opentelemetry/semantic-conventions": "^1.39.0" + } + }, + "node_modules/@sentry/react": { + "version": "10.53.1", + "resolved": "https://registry.npmjs.org/@sentry/react/-/react-10.53.1.tgz", + "integrity": "sha512-lrwNq5T/zW84l60894TpKHPcvFuc1I/Hnohecc0TfYVpIcYYuw2orCHoU4v4wgkFaJUpegVetbgdOphViyLVjA==", + "license": "MIT", + "dependencies": { + "@sentry/browser": "10.53.1", + "@sentry/core": "10.53.1" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "react": "^16.14.0 || 17.x || 18.x || 19.x" + } + }, + "node_modules/@sentry/vercel-edge": { + "version": "10.53.1", + "resolved": "https://registry.npmjs.org/@sentry/vercel-edge/-/vercel-edge-10.53.1.tgz", + "integrity": "sha512-waIOoLfhi1V3xEBJ1s1hpmvvgvcorYfsfm7fQGye0PgVjcBsZUqz32N5iEwkZ2Gz3n4ZOQYibDUqARJi9tOBcw==", + "license": "MIT", + "dependencies": { + "@opentelemetry/api": "^1.9.1", + "@opentelemetry/resources": "^2.6.1", + "@sentry/core": "10.53.1" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@sentry/webpack-plugin": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/@sentry/webpack-plugin/-/webpack-plugin-5.3.0.tgz", + "integrity": "sha512-i3OQUrS0FZlXLgq57RIKDp+vHHzuvYKPCKewAPXULWKMsBXFGhP6veGRQ+6To/pmZkkXjEX5ofVNDy9C3jEPKQ==", + "license": "MIT", + "dependencies": { + "@sentry/bundler-plugin-core": "5.3.0" + }, + "engines": { + "node": ">= 18" + }, + "peerDependencies": { + "webpack": ">=5.0.0" + } + }, "node_modules/@swc/helpers": { "version": "0.5.15", "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.15.tgz", @@ -998,56 +2665,6 @@ "tailwindcss": "4.2.4" } }, - "node_modules/@tailwindcss/node/node_modules/@jridgewell/gen-mapping": { - "version": "0.3.13", - "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", - "integrity": "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/sourcemap-codec": "^1.5.0", - "@jridgewell/trace-mapping": "^0.3.24" - } - }, - "node_modules/@tailwindcss/node/node_modules/@jridgewell/remapping": { - "version": "2.3.5", - "resolved": "https://registry.npmjs.org/@jridgewell/remapping/-/remapping-2.3.5.tgz", - "integrity": "sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/gen-mapping": "^0.3.5", - "@jridgewell/trace-mapping": "^0.3.24" - } - }, - "node_modules/@tailwindcss/node/node_modules/@jridgewell/resolve-uri": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", - "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/@tailwindcss/node/node_modules/@jridgewell/sourcemap-codec": { - "version": "1.5.5", - "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", - "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", - "dev": true, - "license": "MIT" - }, - "node_modules/@tailwindcss/node/node_modules/@jridgewell/trace-mapping": { - "version": "0.3.31", - "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", - "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/resolve-uri": "^3.1.0", - "@jridgewell/sourcemap-codec": "^1.4.14" - } - }, "node_modules/@tailwindcss/node/node_modules/detect-libc": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", @@ -1058,27 +2675,6 @@ "node": ">=8" } }, - "node_modules/@tailwindcss/node/node_modules/enhanced-resolve": { - "version": "5.21.0", - "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.21.0.tgz", - "integrity": "sha512-otxSQPw4lkOZWkHpB3zaEQs6gWYEsmX4xQF68ElXC/TWvGxGMSGOvoNbaLXm6/cS/fSfHtsEdw90y20PCd+sCA==", - "dev": true, - "license": "MIT", - "dependencies": { - "graceful-fs": "^4.2.4", - "tapable": "^2.3.3" - }, - "engines": { - "node": ">=10.13.0" - } - }, - "node_modules/@tailwindcss/node/node_modules/graceful-fs": { - "version": "4.2.11", - "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", - "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", - "dev": true, - "license": "ISC" - }, "node_modules/@tailwindcss/node/node_modules/jiti": { "version": "2.6.1", "resolved": "https://registry.npmjs.org/jiti/-/jiti-2.6.1.tgz", @@ -1350,16 +2946,6 @@ "url": "https://opencollective.com/parcel" } }, - "node_modules/@tailwindcss/node/node_modules/magic-string": { - "version": "0.30.21", - "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", - "integrity": "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/sourcemap-codec": "^1.5.5" - } - }, "node_modules/@tailwindcss/node/node_modules/source-map-js": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", @@ -1370,20 +2956,6 @@ "node": ">=0.10.0" } }, - "node_modules/@tailwindcss/node/node_modules/tapable": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/tapable/-/tapable-2.3.3.tgz", - "integrity": "sha512-uxc/zpqFg6x7C8vOE7lh6Lbda8eEL9zmVm/PLeTPBRhh1xCgdWaQ+J1CUieGpIfm2HdtsUpRv+HshiasBMcc6A==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/webpack" - } - }, "node_modules/@tailwindcss/oxide": { "version": "4.2.4", "resolved": "https://registry.npmjs.org/@tailwindcss/oxide/-/oxide-4.2.4.tgz", @@ -1669,11 +3241,41 @@ "dev": true, "optional": true }, + "node_modules/@types/connect": { + "version": "3.4.38", + "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.38.tgz", + "integrity": "sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==", + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/eslint": { + "version": "9.6.1", + "resolved": "https://registry.npmjs.org/@types/eslint/-/eslint-9.6.1.tgz", + "integrity": "sha512-FXx2pKgId/WyYo2jXw63kk7/+TY7u7AziEJxJAnSFzHlqTAS3Ync6SvgYAN/k4/PQpnnVuzoMuVnByKK2qp0ag==", + "license": "MIT", + "peer": true, + "dependencies": { + "@types/estree": "*", + "@types/json-schema": "*" + } + }, + "node_modules/@types/eslint-scope": { + "version": "3.7.7", + "resolved": "https://registry.npmjs.org/@types/eslint-scope/-/eslint-scope-3.7.7.tgz", + "integrity": "sha512-MzMFlSLBqNF2gcHWO0G1vP/YQyfvrxZ0bF+u7mzUdZ1/xK4A4sru+nraZz5i3iEIk1l1uyicaDVTB4QbbEkAYg==", + "license": "MIT", + "peer": true, + "dependencies": { + "@types/eslint": "*", + "@types/estree": "*" + } + }, "node_modules/@types/estree": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", "integrity": "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==", - "dev": true, "license": "MIT" }, "node_modules/@types/geojson": { @@ -1683,6 +3285,12 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/json-schema": { + "version": "7.0.15", + "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", + "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", + "license": "MIT" + }, "node_modules/@types/leaflet": { "version": "1.9.21", "resolved": "https://registry.npmjs.org/@types/leaflet/-/leaflet-1.9.21.tgz", @@ -1703,16 +3311,44 @@ "@types/leaflet": "^1.9" } }, + "node_modules/@types/mysql": { + "version": "2.15.27", + "resolved": "https://registry.npmjs.org/@types/mysql/-/mysql-2.15.27.tgz", + "integrity": "sha512-YfWiV16IY0OeBfBCk8+hXKmdTKrKlwKN1MNKAPBu5JYxLwBEZl7QzeEpGnlZb3VMGJrrGmB84gXiH+ofs/TezA==", + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/node": { "version": "22.19.17", "resolved": "https://registry.npmjs.org/@types/node/-/node-22.19.17.tgz", "integrity": "sha512-wGdMcf+vPYM6jikpS/qhg6WiqSV/OhG+jeeHT/KlVqxYfD40iYJf9/AE1uQxVWFvU7MipKRkRv8NSHiCGgPr8Q==", - "dev": true, "license": "MIT", "dependencies": { "undici-types": "~6.21.0" } }, + "node_modules/@types/pg": { + "version": "8.15.6", + "resolved": "https://registry.npmjs.org/@types/pg/-/pg-8.15.6.tgz", + "integrity": "sha512-NoaMtzhxOrubeL/7UZuNTrejB4MPAJ0RpxZqXQf2qXuVlTPuG6Y8p4u9dKRaue4yjmC7ZhzVO2/Yyyn25znrPQ==", + "license": "MIT", + "dependencies": { + "@types/node": "*", + "pg-protocol": "*", + "pg-types": "^2.2.0" + } + }, + "node_modules/@types/pg-pool": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/@types/pg-pool/-/pg-pool-2.0.7.tgz", + "integrity": "sha512-U4CwmGVQcbEuqpyju8/ptOKg6gEC+Tqsvj2xS9o1g71bUh8twxnC6ZL5rZKCsGN0iyH0CwgUyc9VR5owNQF9Ng==", + "license": "MIT", + "dependencies": { + "@types/pg": "*" + } + }, "node_modules/@types/react": { "version": "19.2.14", "resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.14.tgz", @@ -1733,6 +3369,15 @@ "@types/react": "^19.2.0" } }, + "node_modules/@types/tedious": { + "version": "4.0.14", + "resolved": "https://registry.npmjs.org/@types/tedious/-/tedious-4.0.14.tgz", + "integrity": "sha512-KHPsfX/FoVbUGbyYvk1q9MMQHLPeRZhRJZdO45Q4YjvFkv4hMNghCWTvy7rdKessBsmtz4euWCWAB6/tVpI1Iw==", + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@typescript-eslint/eslint-plugin": { "version": "8.59.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.59.0.tgz", @@ -1928,29 +3573,6 @@ "url": "https://opencollective.com/typescript-eslint" } }, - "node_modules/@typescript-eslint/eslint-plugin/node_modules/balanced-match": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", - "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", - "dev": true, - "license": "MIT", - "engines": { - "node": "18 || 20 || >=22" - } - }, - "node_modules/@typescript-eslint/eslint-plugin/node_modules/brace-expansion": { - "version": "5.0.5", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.5.tgz", - "integrity": "sha512-VZznLgtwhn+Mact9tfiwx64fA9erHH/MCXEUfB/0bX/6Fz6ny5EGTXYltMocqg4xFAQZtnO3DHWWXi8RiuN7cQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "balanced-match": "^4.0.2" - }, - "engines": { - "node": "18 || 20 || >=22" - } - }, "node_modules/@typescript-eslint/eslint-plugin/node_modules/eslint-visitor-keys": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-5.0.1.tgz", @@ -1964,24 +3586,6 @@ "url": "https://opencollective.com/eslint" } }, - "node_modules/@typescript-eslint/eslint-plugin/node_modules/fdir": { - "version": "6.5.0", - "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", - "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12.0.0" - }, - "peerDependencies": { - "picomatch": "^3 || ^4" - }, - "peerDependenciesMeta": { - "picomatch": { - "optional": true - } - } - }, "node_modules/@typescript-eslint/eslint-plugin/node_modules/ignore": { "version": "7.0.5", "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.5.tgz", @@ -2008,19 +3612,6 @@ "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/@typescript-eslint/eslint-plugin/node_modules/picomatch": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", - "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, "node_modules/@typescript-eslint/eslint-plugin/node_modules/semver": { "version": "7.7.4", "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", @@ -2206,29 +3797,6 @@ "url": "https://opencollective.com/typescript-eslint" } }, - "node_modules/@typescript-eslint/parser/node_modules/balanced-match": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", - "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", - "dev": true, - "license": "MIT", - "engines": { - "node": "18 || 20 || >=22" - } - }, - "node_modules/@typescript-eslint/parser/node_modules/brace-expansion": { - "version": "5.0.5", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.5.tgz", - "integrity": "sha512-VZznLgtwhn+Mact9tfiwx64fA9erHH/MCXEUfB/0bX/6Fz6ny5EGTXYltMocqg4xFAQZtnO3DHWWXi8RiuN7cQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "balanced-match": "^4.0.2" - }, - "engines": { - "node": "18 || 20 || >=22" - } - }, "node_modules/@typescript-eslint/parser/node_modules/eslint-visitor-keys": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-5.0.1.tgz", @@ -2242,24 +3810,6 @@ "url": "https://opencollective.com/eslint" } }, - "node_modules/@typescript-eslint/parser/node_modules/fdir": { - "version": "6.5.0", - "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", - "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12.0.0" - }, - "peerDependencies": { - "picomatch": "^3 || ^4" - }, - "peerDependenciesMeta": { - "picomatch": { - "optional": true - } - } - }, "node_modules/@typescript-eslint/parser/node_modules/minimatch": { "version": "10.2.5", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.5.tgz", @@ -2276,19 +3826,6 @@ "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/@typescript-eslint/parser/node_modules/picomatch": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", - "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, "node_modules/@typescript-eslint/parser/node_modules/semver": { "version": "7.7.4", "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", @@ -2332,6 +3869,227 @@ "typescript": ">=4.8.4" } }, + "node_modules/@webassemblyjs/ast": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.14.1.tgz", + "integrity": "sha512-nuBEDgQfm1ccRp/8bCQrx1frohyufl4JlbMMZ4P1wpeOfDhF6FQkxZJ1b/e+PLwr6X1Nhw6OLme5usuBWYBvuQ==", + "license": "MIT", + "peer": true, + "dependencies": { + "@webassemblyjs/helper-numbers": "1.13.2", + "@webassemblyjs/helper-wasm-bytecode": "1.13.2" + } + }, + "node_modules/@webassemblyjs/floating-point-hex-parser": { + "version": "1.13.2", + "resolved": "https://registry.npmjs.org/@webassemblyjs/floating-point-hex-parser/-/floating-point-hex-parser-1.13.2.tgz", + "integrity": "sha512-6oXyTOzbKxGH4steLbLNOu71Oj+C8Lg34n6CqRvqfS2O71BxY6ByfMDRhBytzknj9yGUPVJ1qIKhRlAwO1AovA==", + "license": "MIT", + "peer": true + }, + "node_modules/@webassemblyjs/helper-api-error": { + "version": "1.13.2", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-api-error/-/helper-api-error-1.13.2.tgz", + "integrity": "sha512-U56GMYxy4ZQCbDZd6JuvvNV/WFildOjsaWD3Tzzvmw/mas3cXzRJPMjP83JqEsgSbyrmaGjBfDtV7KDXV9UzFQ==", + "license": "MIT", + "peer": true + }, + "node_modules/@webassemblyjs/helper-buffer": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-buffer/-/helper-buffer-1.14.1.tgz", + "integrity": "sha512-jyH7wtcHiKssDtFPRB+iQdxlDf96m0E39yb0k5uJVhFGleZFoNw1c4aeIcVUPPbXUVJ94wwnMOAqUHyzoEPVMA==", + "license": "MIT", + "peer": true + }, + "node_modules/@webassemblyjs/helper-numbers": { + "version": "1.13.2", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-numbers/-/helper-numbers-1.13.2.tgz", + "integrity": "sha512-FE8aCmS5Q6eQYcV3gI35O4J789wlQA+7JrqTTpJqn5emA4U2hvwJmvFRC0HODS+3Ye6WioDklgd6scJ3+PLnEA==", + "license": "MIT", + "peer": true, + "dependencies": { + "@webassemblyjs/floating-point-hex-parser": "1.13.2", + "@webassemblyjs/helper-api-error": "1.13.2", + "@xtuc/long": "4.2.2" + } + }, + "node_modules/@webassemblyjs/helper-wasm-bytecode": { + "version": "1.13.2", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-bytecode/-/helper-wasm-bytecode-1.13.2.tgz", + "integrity": "sha512-3QbLKy93F0EAIXLh0ogEVR6rOubA9AoZ+WRYhNbFyuB70j3dRdwH9g+qXhLAO0kiYGlg3TxDV+I4rQTr/YNXkA==", + "license": "MIT", + "peer": true + }, + "node_modules/@webassemblyjs/helper-wasm-section": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-section/-/helper-wasm-section-1.14.1.tgz", + "integrity": "sha512-ds5mXEqTJ6oxRoqjhWDU83OgzAYjwsCV8Lo/N+oRsNDmx/ZDpqalmrtgOMkHwxsG0iI//3BwWAErYRHtgn0dZw==", + "license": "MIT", + "peer": true, + "dependencies": { + "@webassemblyjs/ast": "1.14.1", + "@webassemblyjs/helper-buffer": "1.14.1", + "@webassemblyjs/helper-wasm-bytecode": "1.13.2", + "@webassemblyjs/wasm-gen": "1.14.1" + } + }, + "node_modules/@webassemblyjs/ieee754": { + "version": "1.13.2", + "resolved": "https://registry.npmjs.org/@webassemblyjs/ieee754/-/ieee754-1.13.2.tgz", + "integrity": "sha512-4LtOzh58S/5lX4ITKxnAK2USuNEvpdVV9AlgGQb8rJDHaLeHciwG4zlGr0j/SNWlr7x3vO1lDEsuePvtcDNCkw==", + "license": "MIT", + "peer": true, + "dependencies": { + "@xtuc/ieee754": "^1.2.0" + } + }, + "node_modules/@webassemblyjs/leb128": { + "version": "1.13.2", + "resolved": "https://registry.npmjs.org/@webassemblyjs/leb128/-/leb128-1.13.2.tgz", + "integrity": "sha512-Lde1oNoIdzVzdkNEAWZ1dZ5orIbff80YPdHx20mrHwHrVNNTjNr8E3xz9BdpcGqRQbAEa+fkrCb+fRFTl/6sQw==", + "license": "Apache-2.0", + "peer": true, + "dependencies": { + "@xtuc/long": "4.2.2" + } + }, + "node_modules/@webassemblyjs/utf8": { + "version": "1.13.2", + "resolved": "https://registry.npmjs.org/@webassemblyjs/utf8/-/utf8-1.13.2.tgz", + "integrity": "sha512-3NQWGjKTASY1xV5m7Hr0iPeXD9+RDobLll3T9d2AO+g3my8xy5peVyjSag4I50mR1bBSN/Ct12lo+R9tJk0NZQ==", + "license": "MIT", + "peer": true + }, + "node_modules/@webassemblyjs/wasm-edit": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-edit/-/wasm-edit-1.14.1.tgz", + "integrity": "sha512-RNJUIQH/J8iA/1NzlE4N7KtyZNHi3w7at7hDjvRNm5rcUXa00z1vRz3glZoULfJ5mpvYhLybmVcwcjGrC1pRrQ==", + "license": "MIT", + "peer": true, + "dependencies": { + "@webassemblyjs/ast": "1.14.1", + "@webassemblyjs/helper-buffer": "1.14.1", + "@webassemblyjs/helper-wasm-bytecode": "1.13.2", + "@webassemblyjs/helper-wasm-section": "1.14.1", + "@webassemblyjs/wasm-gen": "1.14.1", + "@webassemblyjs/wasm-opt": "1.14.1", + "@webassemblyjs/wasm-parser": "1.14.1", + "@webassemblyjs/wast-printer": "1.14.1" + } + }, + "node_modules/@webassemblyjs/wasm-gen": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-gen/-/wasm-gen-1.14.1.tgz", + "integrity": "sha512-AmomSIjP8ZbfGQhumkNvgC33AY7qtMCXnN6bL2u2Js4gVCg8fp735aEiMSBbDR7UQIj90n4wKAFUSEd0QN2Ukg==", + "license": "MIT", + "peer": true, + "dependencies": { + "@webassemblyjs/ast": "1.14.1", + "@webassemblyjs/helper-wasm-bytecode": "1.13.2", + "@webassemblyjs/ieee754": "1.13.2", + "@webassemblyjs/leb128": "1.13.2", + "@webassemblyjs/utf8": "1.13.2" + } + }, + "node_modules/@webassemblyjs/wasm-opt": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-opt/-/wasm-opt-1.14.1.tgz", + "integrity": "sha512-PTcKLUNvBqnY2U6E5bdOQcSM+oVP/PmrDY9NzowJjislEjwP/C4an2303MCVS2Mg9d3AJpIGdUFIQQWbPds0Sw==", + "license": "MIT", + "peer": true, + "dependencies": { + "@webassemblyjs/ast": "1.14.1", + "@webassemblyjs/helper-buffer": "1.14.1", + "@webassemblyjs/wasm-gen": "1.14.1", + "@webassemblyjs/wasm-parser": "1.14.1" + } + }, + "node_modules/@webassemblyjs/wasm-parser": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-parser/-/wasm-parser-1.14.1.tgz", + "integrity": "sha512-JLBl+KZ0R5qB7mCnud/yyX08jWFw5MsoalJ1pQ4EdFlgj9VdXKGuENGsiCIjegI1W7p91rUlcB/LB5yRJKNTcQ==", + "license": "MIT", + "peer": true, + "dependencies": { + "@webassemblyjs/ast": "1.14.1", + "@webassemblyjs/helper-api-error": "1.13.2", + "@webassemblyjs/helper-wasm-bytecode": "1.13.2", + "@webassemblyjs/ieee754": "1.13.2", + "@webassemblyjs/leb128": "1.13.2", + "@webassemblyjs/utf8": "1.13.2" + } + }, + "node_modules/@webassemblyjs/wast-printer": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wast-printer/-/wast-printer-1.14.1.tgz", + "integrity": "sha512-kPSSXE6De1XOR820C90RIo2ogvZG+c3KiHzqUoO/F34Y2shGzesfqv7o57xrxovZJH/MetF5UjroJ/R/3isoiw==", + "license": "MIT", + "peer": true, + "dependencies": { + "@webassemblyjs/ast": "1.14.1", + "@xtuc/long": "4.2.2" + } + }, + "node_modules/@xtuc/ieee754": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@xtuc/ieee754/-/ieee754-1.2.0.tgz", + "integrity": "sha512-DX8nKgqcGwsc0eJSqYt5lwP4DH5FlHnmuWWBRy7X0NcaGR0ZtuyeESgMwTYVEtxmsNGY+qit4QYT/MIYTOTPeA==", + "license": "BSD-3-Clause", + "peer": true + }, + "node_modules/@xtuc/long": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/@xtuc/long/-/long-4.2.2.tgz", + "integrity": "sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ==", + "license": "Apache-2.0", + "peer": true + }, + "node_modules/acorn": { + "version": "8.16.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.16.0.tgz", + "integrity": "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==", + "license": "MIT", + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-import-attributes": { + "version": "1.9.5", + "resolved": "https://registry.npmjs.org/acorn-import-attributes/-/acorn-import-attributes-1.9.5.tgz", + "integrity": "sha512-n02Vykv5uA3eHGM/Z2dQrcD56kL8TyDb2p1+0P83PClMnC/nc+anbQRhIOWnSq4Ke/KvDPrY3C9hDtC/A3eHnQ==", + "license": "MIT", + "peerDependencies": { + "acorn": "^8" + } + }, + "node_modules/acorn-import-phases": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/acorn-import-phases/-/acorn-import-phases-1.0.4.tgz", + "integrity": "sha512-wKmbr/DDiIXzEOiWrTTUcDm24kQ2vGfZQvM2fwg2vXqR5uW6aapr7ObPtj1th32b9u90/Pf4AItvdTh42fBmVQ==", + "license": "MIT", + "peer": true, + "engines": { + "node": ">=10.13.0" + }, + "peerDependencies": { + "acorn": "^8.14.0" + } + }, + "node_modules/agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "license": "MIT", + "dependencies": { + "debug": "4" + }, + "engines": { + "node": ">= 6.0.0" + } + }, "node_modules/ajv": { "version": "6.15.0", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.15.0.tgz", @@ -2349,6 +4107,41 @@ "url": "https://github.com/sponsors/epoberezkin" } }, + "node_modules/ajv-formats": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-2.1.1.tgz", + "integrity": "sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA==", + "license": "MIT", + "peer": true, + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, + "node_modules/ajv-formats/node_modules/ajv": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", + "license": "MIT", + "peer": true, + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, "node_modules/ajv/node_modules/fast-json-stable-stringify": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", @@ -2393,6 +4186,79 @@ "node": ">=6" } }, + "node_modules/balanced-match": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", + "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", + "license": "MIT", + "engines": { + "node": "18 || 20 || >=22" + } + }, + "node_modules/baseline-browser-mapping": { + "version": "2.10.30", + "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.10.30.tgz", + "integrity": "sha512-xjOFN16Ha1+Rz4nFYKqHU/LSB+gx/Vi3yQLX7r7sAW+Wa+8hhF2h4pvqTrTMc8+WcDBEunnUurr46Jvv0jk3Vg==", + "license": "Apache-2.0", + "bin": { + "baseline-browser-mapping": "dist/cli.cjs" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/brace-expansion": { + "version": "5.0.6", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.6.tgz", + "integrity": "sha512-kLpxurY4Z4r9sgMsyG0Z9uzsBlgiU/EFKhj/h91/8yHu0edo7XuixOIH3VcJ8kkxs6/jPzoI6U9Vj3WqbMQ94g==", + "license": "MIT", + "dependencies": { + "balanced-match": "^4.0.2" + }, + "engines": { + "node": "18 || 20 || >=22" + } + }, + "node_modules/browserslist": { + "version": "4.28.2", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.2.tgz", + "integrity": "sha512-48xSriZYYg+8qXna9kwqjIVzuQxi+KYWp2+5nCYnYKPTr0LvD89Jqk2Or5ogxz0NUMfIjhh2lIUX/LyX9B4oIg==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "baseline-browser-mapping": "^2.10.12", + "caniuse-lite": "^1.0.30001782", + "electron-to-chromium": "^1.5.328", + "node-releases": "^2.0.36", + "update-browserslist-db": "^1.2.3" + }, + "bin": { + "browserslist": "cli.js" + }, + "engines": { + "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" + } + }, + "node_modules/buffer-from": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", + "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", + "license": "MIT", + "peer": true + }, "node_modules/caniuse-lite": { "version": "1.0.30001790", "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001790.tgz", @@ -2466,16 +4332,6 @@ "dev": true, "license": "MIT" }, - "node_modules/chalk/node_modules/has-flag": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", - "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, "node_modules/chalk/node_modules/supports-color": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", @@ -2496,6 +4352,41 @@ "dev": true, "license": "MIT" }, + "node_modules/chrome-trace-event": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/chrome-trace-event/-/chrome-trace-event-1.0.4.tgz", + "integrity": "sha512-rNjApaLzuwaOTjCiT8lSDdGN1APCiqkChLMJxJPWLunPAt5fy8xgU9/jNOchV84wfIxrA0lRQB7oCT8jrn/wrQ==", + "license": "MIT", + "peer": true, + "engines": { + "node": ">=6.0" + } + }, + "node_modules/cjs-module-lexer": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/cjs-module-lexer/-/cjs-module-lexer-2.2.0.tgz", + "integrity": "sha512-4bHTS2YuzUvtoLjdy+98ykbNB5jS0+07EvFNXerqZQJ89F7DI6ET7OQo/HJuW6K0aVsKA9hj9/RVb2kQVOrPDQ==", + "license": "MIT" + }, + "node_modules/commander": { + "version": "2.20.3", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", + "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==", + "license": "MIT", + "peer": true + }, + "node_modules/commondir": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/commondir/-/commondir-1.0.1.tgz", + "integrity": "sha512-W9pAhw0ja1Edb5GVdIF1mjZw/ASI0AlShXM83UUGe2DVr5TdAPEA1OA8m/g8zWp9x6On7gqufY+FatDbC3MDQg==", + "license": "MIT" + }, + "node_modules/convert-source-map": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", + "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", + "license": "MIT" + }, "node_modules/cross-spawn": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", @@ -2511,13 +4402,6 @@ "node": ">= 8" } }, - "node_modules/cross-spawn/node_modules/isexe": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", - "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", - "dev": true, - "license": "ISC" - }, "node_modules/cross-spawn/node_modules/path-key": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", @@ -2551,22 +4435,6 @@ "node": ">=8" } }, - "node_modules/cross-spawn/node_modules/which": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", - "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", - "dev": true, - "license": "ISC", - "dependencies": { - "isexe": "^2.0.0" - }, - "bin": { - "node-which": "bin/node-which" - }, - "engines": { - "node": ">= 8" - } - }, "node_modules/csstype": { "version": "3.2.3", "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz", @@ -2578,7 +4446,6 @@ "version": "4.4.3", "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", - "dev": true, "license": "MIT", "dependencies": { "ms": "^2.1.3" @@ -2596,9 +4463,20 @@ "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "dev": true, "license": "MIT" }, + "node_modules/dotenv": { + "version": "16.6.1", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.6.1.tgz", + "integrity": "sha512-uBq4egWHTcTt33a72vpSG0z3HnPuIl6NqYcTrKEg2azoEyl2hpW0zqlxysq2pK9HlDIHyHyakeYaYnSAwd8bow==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, "node_modules/echarts": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/echarts/-/echarts-6.0.0.tgz", @@ -2629,6 +4507,41 @@ "integrity": "sha512-N82ooyxVNm6h1riLCoyS9e3fuJ3AMG2zIZs2Gd1ATcSFjSA23Q0fzjjZeh0jbJvWVDZ0cJT8yaNNaaXHzueNjg==", "license": "0BSD" }, + "node_modules/electron-to-chromium": { + "version": "1.5.357", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.357.tgz", + "integrity": "sha512-NHlTIQDK8fmVwHwuIzmXYEJ1Ewq3D9wDNc0cWXxDGysP6Pb21giwGNkxiTifyKy/4SoPuN5l6GLP1W9Sv7zB2g==", + "license": "ISC" + }, + "node_modules/enhanced-resolve": { + "version": "5.21.3", + "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.21.3.tgz", + "integrity": "sha512-QyL119InA+XXEkNLNTPCXPugSvOfhwv0JOlGNzvxs0hZaiHLNvXSpudUWsOlsXGWJh8G6ckCScEkVHfX3kw/2Q==", + "license": "MIT", + "dependencies": { + "graceful-fs": "^4.2.4", + "tapable": "^2.3.3" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/es-module-lexer": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-2.1.0.tgz", + "integrity": "sha512-n27zTYMjYu1aj4MjCWzSP7G9r75utsaoc8m61weK+W8JMBGGQybd43GstCXZ3WNmSFtGT9wi59qQTW6mhTR5LQ==", + "license": "MIT", + "peer": true + }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/escape-string-regexp": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", @@ -4013,16 +5926,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/eslint-import-resolver-node/node_modules/semver": { - "version": "6.3.1", - "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", - "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", - "dev": true, - "license": "ISC", - "bin": { - "semver": "bin/semver.js" - } - }, "node_modules/eslint-import-resolver-node/node_modules/set-function-length": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.2.tgz", @@ -4792,24 +6695,6 @@ "win32" ] }, - "node_modules/eslint-import-resolver-typescript/node_modules/fdir": { - "version": "6.5.0", - "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", - "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12.0.0" - }, - "peerDependencies": { - "picomatch": "^3 || ^4" - }, - "peerDependenciesMeta": { - "picomatch": { - "optional": true - } - } - }, "node_modules/eslint-import-resolver-typescript/node_modules/get-tsconfig": { "version": "4.14.0", "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.14.0.tgz", @@ -4849,19 +6734,6 @@ "url": "https://opencollective.com/napi-postinstall" } }, - "node_modules/eslint-import-resolver-typescript/node_modules/picomatch": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", - "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, "node_modules/eslint-import-resolver-typescript/node_modules/resolve-pkg-maps": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz", @@ -6368,16 +8240,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/eslint-plugin-import/node_modules/semver": { - "version": "6.3.1", - "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", - "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", - "dev": true, - "license": "ISC", - "bin": { - "semver": "bin/semver.js" - } - }, "node_modules/eslint-plugin-import/node_modules/set-function-length": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.2.tgz", @@ -9755,13 +11617,6 @@ "node": ">= 0.4" } }, - "node_modules/eslint-plugin-react/node_modules/js-tokens": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", - "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", - "dev": true, - "license": "MIT" - }, "node_modules/eslint-plugin-react/node_modules/jsx-ast-utils": { "version": "3.3.5", "resolved": "https://registry.npmjs.org/jsx-ast-utils/-/jsx-ast-utils-3.3.5.tgz", @@ -10105,16 +11960,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/eslint-plugin-react/node_modules/semver": { - "version": "6.3.1", - "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", - "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", - "dev": true, - "license": "ISC", - "bin": { - "semver": "bin/semver.js" - } - }, "node_modules/eslint-plugin-react/node_modules/set-function-length": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.2.tgz", @@ -10568,19 +12413,6 @@ "url": "https://opencollective.com/eslint" } }, - "node_modules/eslint-scope/node_modules/esrecurse": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", - "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", - "dev": true, - "license": "BSD-2-Clause", - "dependencies": { - "estraverse": "^5.2.0" - }, - "engines": { - "node": ">=4.0" - } - }, "node_modules/eslint-scope/node_modules/estraverse": { "version": "5.3.0", "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", @@ -10622,19 +12454,6 @@ "url": "https://opencollective.com/eslint" } }, - "node_modules/espree/node_modules/acorn": { - "version": "8.16.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.16.0.tgz", - "integrity": "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==", - "dev": true, - "license": "MIT", - "bin": { - "acorn": "bin/acorn" - }, - "engines": { - "node": ">=0.4.0" - } - }, "node_modules/espree/node_modules/acorn-jsx": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", @@ -10668,6 +12487,43 @@ "node": ">=4.0" } }, + "node_modules/esrecurse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", + "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", + "license": "BSD-2-Clause", + "dependencies": { + "estraverse": "^5.2.0" + }, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/esrecurse/node_modules/estraverse": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=4.0" + } + }, + "node_modules/estraverse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.3.0.tgz", + "integrity": "sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==", + "license": "BSD-2-Clause", + "peer": true, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/estree-walker": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-2.0.2.tgz", + "integrity": "sha512-Rfkk/Mp/DL7JVje3u18FxFujQlTNR2q6QfMSMB7AvCBx91NGj/ba3kCfza0f6dVDbw7YlRf/nDrn7pQrCCyQ/w==", + "license": "MIT" + }, "node_modules/esutils": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", @@ -10678,12 +12534,56 @@ "node": ">=0.10.0" } }, + "node_modules/events": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", + "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", + "license": "MIT", + "peer": true, + "engines": { + "node": ">=0.8.x" + } + }, "node_modules/fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", "license": "MIT" }, + "node_modules/fast-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.2.tgz", + "integrity": "sha512-rVjf7ArG3LTk+FS6Yw81V1DLuZl1bRbNrev6Tmd/9RaroeeRRJhAt7jg/6YFxbvAQXUCavSoZhPPj6oOx+5KjQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause", + "peer": true + }, + "node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, "node_modules/file-entry-cache": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-8.0.0.tgz", @@ -10739,7 +12639,6 @@ "version": "5.0.0", "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==", - "dev": true, "license": "MIT", "dependencies": { "locate-path": "^6.0.0", @@ -10756,7 +12655,6 @@ "version": "6.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", - "dev": true, "license": "MIT", "dependencies": { "p-locate": "^5.0.0" @@ -10772,7 +12670,6 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", - "dev": true, "license": "MIT", "dependencies": { "yocto-queue": "^0.1.0" @@ -10788,7 +12685,6 @@ "version": "5.0.0", "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz", "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==", - "dev": true, "license": "MIT", "dependencies": { "p-limit": "^3.0.2" @@ -10804,7 +12700,6 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", - "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -10814,7 +12709,6 @@ "version": "0.1.0", "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", - "dev": true, "license": "MIT", "engines": { "node": ">=10" @@ -10823,6 +12717,52 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/forwarded-parse": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/forwarded-parse/-/forwarded-parse-2.1.2.tgz", + "integrity": "sha512-alTFZZQDKMporBH77856pXgzhEzaUVmLCDk+egLgIgHst3Tpndzz8MnKe+GzRJRfvVdn69HhpW7cmXzvtLvJAw==", + "license": "MIT" + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/gensync": { + "version": "1.0.0-beta.2", + "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", + "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/glob": { + "version": "13.0.6", + "resolved": "https://registry.npmjs.org/glob/-/glob-13.0.6.tgz", + "integrity": "sha512-Wjlyrolmm8uDpm/ogGyXZXb1Z+Ca2B8NbJwqBVg0axK9GbBeoS7yGV6vjXnYdGm6X53iehEuxxbyiKp8QmN4Vw==", + "license": "BlueOak-1.0.0", + "dependencies": { + "minimatch": "^10.2.2", + "minipass": "^7.1.3", + "path-scurry": "^2.0.2" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/glob-parent": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", @@ -10836,6 +12776,56 @@ "node": ">=10.13.0" } }, + "node_modules/glob-to-regexp": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.4.1.tgz", + "integrity": "sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw==", + "license": "BSD-2-Clause", + "peer": true + }, + "node_modules/glob/node_modules/minimatch": { + "version": "10.2.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.5.tgz", + "integrity": "sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==", + "license": "BlueOak-1.0.0", + "dependencies": { + "brace-expansion": "^5.0.5" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "license": "ISC" + }, + "node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/https-proxy-agent": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", + "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", + "license": "MIT", + "dependencies": { + "agent-base": "6", + "debug": "4" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/ignore": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", @@ -10846,6 +12836,21 @@ "node": ">= 4" } }, + "node_modules/import-in-the-middle": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/import-in-the-middle/-/import-in-the-middle-3.0.1.tgz", + "integrity": "sha512-pYkiyXVL2Mf3pozdlDGV6NAObxQx13Ae8knZk1UJRJ6uRW/ZRmTGHlQYtrsSl7ubuE5F8CD1z+s1n4RHNuTtuA==", + "license": "Apache-2.0", + "dependencies": { + "acorn": "^8.15.0", + "acorn-import-attributes": "^1.9.5", + "cjs-module-lexer": "^2.2.0", + "module-details-from-path": "^1.0.4" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/imurmurhash": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", @@ -10879,6 +12884,60 @@ "node": ">=0.10.0" } }, + "node_modules/is-reference": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/is-reference/-/is-reference-1.2.1.tgz", + "integrity": "sha512-U82MsXXiFIrjCK4otLT+o2NA2Cd2g5MLoOVXUZjIOhLurrRxpEXzI8O0KZHr3IjLvlAH1kTPYSuqer5T9ZVBKQ==", + "license": "MIT", + "dependencies": { + "@types/estree": "*" + } + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "license": "ISC" + }, + "node_modules/jest-worker": { + "version": "27.5.1", + "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-27.5.1.tgz", + "integrity": "sha512-7vuh85V5cdDofPyxn58nrPjBktZo0u9x1g8WtjQol+jZDaE+fhN+cIvTj11GndBnMnyfrUOG1sZQxCdjKh+DKg==", + "license": "MIT", + "peer": true, + "dependencies": { + "@types/node": "*", + "merge-stream": "^2.0.0", + "supports-color": "^8.0.0" + }, + "engines": { + "node": ">= 10.13.0" + } + }, + "node_modules/js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "license": "MIT" + }, + "node_modules/jsesc": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.1.0.tgz", + "integrity": "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA==", + "license": "MIT", + "bin": { + "jsesc": "bin/jsesc" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "license": "MIT" + }, "node_modules/json-stable-stringify-without-jsonify": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", @@ -10886,6 +12945,18 @@ "dev": true, "license": "MIT" }, + "node_modules/json5": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", + "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", + "license": "MIT", + "bin": { + "json5": "lib/cli.js" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/leaflet": { "version": "1.9.4", "resolved": "https://registry.npmjs.org/leaflet/-/leaflet-1.9.4.tgz", @@ -10898,6 +12969,20 @@ "integrity": "sha512-rsQ6saQO5ST5Aj6XRFylr5zvarWgzWnrg46zQ1MEOEIHsppdC/8hnN8qMoFvACsPvTioAuysya/TVtog15tyAQ==", "license": "MIT" }, + "node_modules/loader-runner": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/loader-runner/-/loader-runner-4.3.2.tgz", + "integrity": "sha512-DFEqQ3ihfS9blba08cLfYf1NRAIEm+dDjic073DRDc3/JspI/8wYmtDsHwd3+4hwvdxSK7PGaElfTmm0awWJ4w==", + "license": "MIT", + "peer": true, + "engines": { + "node": ">=6.11.5" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + } + }, "node_modules/lodash.merge": { "version": "4.6.2", "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", @@ -10905,6 +12990,50 @@ "dev": true, "license": "MIT" }, + "node_modules/lru-cache": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", + "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", + "license": "ISC", + "dependencies": { + "yallist": "^3.0.2" + } + }, + "node_modules/lucide-react": { + "version": "0.511.0", + "resolved": "https://registry.npmjs.org/lucide-react/-/lucide-react-0.511.0.tgz", + "integrity": "sha512-VK5a2ydJ7xm8GvBeKLS9mu1pVK6ucef9780JVUjw6bAjJL/QXnd4Y0p7SPeOUMC27YhzNCZvm5d/QX0Tp3rc0w==", + "license": "ISC", + "peerDependencies": { + "react": "^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0" + } + }, + "node_modules/magic-string": { + "version": "0.30.21", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", + "integrity": "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==", + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.5" + } + }, + "node_modules/merge-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", + "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==", + "license": "MIT", + "peer": true + }, + "node_modules/mime-db": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", + "license": "MIT", + "peer": true, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/minimatch": { "version": "3.1.5", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", @@ -10943,6 +13072,21 @@ "dev": true, "license": "MIT" }, + "node_modules/minipass": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.3.tgz", + "integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, + "node_modules/module-details-from-path": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/module-details-from-path/-/module-details-from-path-1.0.4.tgz", + "integrity": "sha512-EGWKgxALGMgzvxYF1UyGTy0HXX/2vHLkw6+NvDKW2jypWbHpjQuj4UMcqQWXHERJhVGKikolT06G3bcKe4fi7w==", + "license": "MIT" + }, "node_modules/natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", @@ -10950,6 +13094,13 @@ "dev": true, "license": "MIT" }, + "node_modules/neo-async": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.6.2.tgz", + "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==", + "license": "MIT", + "peer": true + }, "node_modules/next": { "version": "15.5.15", "resolved": "https://registry.npmjs.org/next/-/next-15.5.15.tgz", @@ -11020,12 +13171,6 @@ "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" } }, - "node_modules/next/node_modules/picocolors": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", - "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", - "license": "ISC" - }, "node_modules/next/node_modules/postcss": { "version": "8.4.31", "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.31.tgz", @@ -11063,6 +13208,32 @@ "node": ">=0.10.0" } }, + "node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "license": "MIT", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, + "node_modules/node-releases": { + "version": "2.0.44", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.44.tgz", + "integrity": "sha512-5WUyunoPMsvvEhS8AxHtRzP+oA8UCkJ7YRxatWKjngndhDGLiqEVAQKWjFAiAiuL8zMRGzGSJxFnLetoa43qGQ==", + "license": "MIT" + }, "node_modules/openapi-typescript": { "version": "7.13.0", "resolved": "https://registry.npmjs.org/openapi-typescript/-/openapi-typescript-7.13.0.tgz", @@ -11194,31 +13365,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/parse-json/node_modules/@babel/code-frame": { - "version": "7.29.0", - "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.0.tgz", - "integrity": "sha512-9NhCeYjq9+3uxgdtp20LSiJXJvN0FeCtNGpJxuMFZ1Kv3cWUNb6DOhJwUvcVCzKGR66cw4njwM6hrJLqgOwbcw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-validator-identifier": "^7.28.5", - "js-tokens": "^4.0.0", - "picocolors": "^1.1.1" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/parse-json/node_modules/@babel/helper-validator-identifier": { - "version": "7.28.5", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.28.5.tgz", - "integrity": "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, "node_modules/parse-json/node_modules/index-to-position": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/index-to-position/-/index-to-position-1.2.0.tgz", @@ -11232,20 +13378,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/parse-json/node_modules/js-tokens": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", - "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/parse-json/node_modules/picocolors": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", - "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", - "dev": true, - "license": "ISC" - }, "node_modules/parse-json/node_modules/type-fest": { "version": "4.41.0", "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-4.41.0.tgz", @@ -11259,6 +13391,80 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/path-scurry": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.2.tgz", + "integrity": "sha512-3O/iVVsJAPsOnpwWIeD+d6z/7PmqApyQePUtCndjatj/9I5LylHvt5qluFaBT3I5h3r1ejfR056c+FCv+NnNXg==", + "license": "BlueOak-1.0.0", + "dependencies": { + "lru-cache": "^11.0.0", + "minipass": "^7.1.2" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/path-scurry/node_modules/lru-cache": { + "version": "11.3.6", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.3.6.tgz", + "integrity": "sha512-Gf/KoL3C/MlI7Bt0PGI9I+TeTC/I6r/csU58N4BSNc4lppLBeKsOdFYkK+dX0ABDUMJNfCHTyPpzwwO21Awd3A==", + "license": "BlueOak-1.0.0", + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/pg-int8": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", + "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", + "license": "ISC", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/pg-protocol": { + "version": "1.13.0", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.13.0.tgz", + "integrity": "sha512-zzdvXfS6v89r6v7OcFCHfHlyG/wvry1ALxZo4LqgUoy7W9xhBDMaqOuMiF3qEV45VqsN6rdlcehHrfDtlCPc8w==", + "license": "MIT" + }, + "node_modules/pg-types": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", + "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "license": "MIT", + "dependencies": { + "pg-int8": "1.0.1", + "postgres-array": "~2.0.0", + "postgres-bytea": "~1.0.0", + "postgres-date": "~1.0.4", + "postgres-interval": "^1.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "license": "ISC" + }, + "node_modules/picomatch": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", + "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, "node_modules/postcss": { "version": "8.5.10", "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.10.tgz", @@ -11307,13 +13513,6 @@ "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" } }, - "node_modules/postcss/node_modules/picocolors": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", - "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", - "dev": true, - "license": "ISC" - }, "node_modules/postcss/node_modules/source-map-js": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", @@ -11324,6 +13523,60 @@ "node": ">=0.10.0" } }, + "node_modules/postgres-array": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", + "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/postgres-bytea": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.1.tgz", + "integrity": "sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-date": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", + "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-interval": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", + "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "license": "MIT", + "dependencies": { + "xtend": "^4.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/progress": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", + "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", + "license": "MIT", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", + "license": "MIT" + }, "node_modules/react": { "version": "19.2.5", "resolved": "https://registry.npmjs.org/react/-/react-19.2.5.tgz", @@ -11359,12 +13612,137 @@ "react-dom": "^19.0.0" } }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/require-in-the-middle": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/require-in-the-middle/-/require-in-the-middle-8.0.1.tgz", + "integrity": "sha512-QT7FVMXfWOYFbeRBF6nu+I6tr2Tf3u0q8RIEjNob/heKY/nh7drD/k7eeMFmSQgnTtCzLDcCu/XEnpW2wk4xCQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.3.5", + "module-details-from-path": "^1.0.3" + }, + "engines": { + "node": ">=9.3.0 || >=8.10.0 <9.0.0" + } + }, + "node_modules/rollup": { + "version": "4.60.4", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.60.4.tgz", + "integrity": "sha512-WHeFSbZYsPu3+bLoNRUuAO+wavNlocOPf3wSHTP7hcFKVnJeWsYlCDbr3mTS14FCizf9ccIxXA8sGL8zKeQN3g==", + "license": "MIT", + "dependencies": { + "@types/estree": "1.0.8" + }, + "bin": { + "rollup": "dist/bin/rollup" + }, + "engines": { + "node": ">=18.0.0", + "npm": ">=8.0.0" + }, + "optionalDependencies": { + "@rollup/rollup-android-arm-eabi": "4.60.4", + "@rollup/rollup-android-arm64": "4.60.4", + "@rollup/rollup-darwin-arm64": "4.60.4", + "@rollup/rollup-darwin-x64": "4.60.4", + "@rollup/rollup-freebsd-arm64": "4.60.4", + "@rollup/rollup-freebsd-x64": "4.60.4", + "@rollup/rollup-linux-arm-gnueabihf": "4.60.4", + "@rollup/rollup-linux-arm-musleabihf": "4.60.4", + "@rollup/rollup-linux-arm64-gnu": "4.60.4", + "@rollup/rollup-linux-arm64-musl": "4.60.4", + "@rollup/rollup-linux-loong64-gnu": "4.60.4", + "@rollup/rollup-linux-loong64-musl": "4.60.4", + "@rollup/rollup-linux-ppc64-gnu": "4.60.4", + "@rollup/rollup-linux-ppc64-musl": "4.60.4", + "@rollup/rollup-linux-riscv64-gnu": "4.60.4", + "@rollup/rollup-linux-riscv64-musl": "4.60.4", + "@rollup/rollup-linux-s390x-gnu": "4.60.4", + "@rollup/rollup-linux-x64-gnu": "4.60.4", + "@rollup/rollup-linux-x64-musl": "4.60.4", + "@rollup/rollup-openbsd-x64": "4.60.4", + "@rollup/rollup-openharmony-arm64": "4.60.4", + "@rollup/rollup-win32-arm64-msvc": "4.60.4", + "@rollup/rollup-win32-ia32-msvc": "4.60.4", + "@rollup/rollup-win32-x64-gnu": "4.60.4", + "@rollup/rollup-win32-x64-msvc": "4.60.4", + "fsevents": "~2.3.2" + } + }, "node_modules/scheduler": { "version": "0.27.0", "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.27.0.tgz", "integrity": "sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q==", "license": "MIT" }, + "node_modules/schema-utils": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-4.3.3.tgz", + "integrity": "sha512-eflK8wEtyOE6+hsaRVPxvUKYCpRgzLqDTb8krvAsRIwOGlHoSgYLgBXoubGgLd2fT41/OUYdb48v4k4WWHQurA==", + "license": "MIT", + "peer": true, + "dependencies": { + "@types/json-schema": "^7.0.9", + "ajv": "^8.9.0", + "ajv-formats": "^2.1.1", + "ajv-keywords": "^5.1.0" + }, + "engines": { + "node": ">= 10.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + } + }, + "node_modules/schema-utils/node_modules/ajv": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", + "license": "MIT", + "peer": true, + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/schema-utils/node_modules/ajv-keywords": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-5.1.0.tgz", + "integrity": "sha512-YCS/JNFAUyr5vAuhk1DWm1CBxRHW9LbJ2ozWeemrIqpbsqKjHVxYPyi5GC0rjZIT5JxJ3virVTS8wk4i/Z+krw==", + "license": "MIT", + "peer": true, + "dependencies": { + "fast-deep-equal": "^3.1.3" + }, + "peerDependencies": { + "ajv": "^8.8.2" + } + }, + "node_modules/semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + } + }, "node_modules/sharp": { "version": "0.34.5", "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.34.5.tgz", @@ -11915,6 +14293,39 @@ "integrity": "sha512-+k9mJ2/rQMiRmQUcjn+qznch260leIXY8r4FyYKKyRBO/s5UoeMAHGkCJyE1R/4wrIhTJONfyloY55SkE7ve3A==", "license": "ISC" }, + "node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "license": "BSD-3-Clause", + "peer": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/source-map-support": { + "version": "0.5.21", + "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.21.tgz", + "integrity": "sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==", + "license": "MIT", + "peer": true, + "dependencies": { + "buffer-from": "^1.0.0", + "source-map": "^0.6.0" + } + }, + "node_modules/stacktrace-parser": { + "version": "0.1.11", + "resolved": "https://registry.npmjs.org/stacktrace-parser/-/stacktrace-parser-0.1.11.tgz", + "integrity": "sha512-WjlahMgHmCJpqzU8bIBy4qtsZdU9lRlcZE3Lvyej6t4tuOuv1vk57OW3MBrj6hXBFx/nNoC9MPMTcr5YA7NQbg==", + "license": "MIT", + "dependencies": { + "type-fest": "^0.7.1" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/styled-jsx": { "version": "5.1.6", "resolved": "https://registry.npmjs.org/styled-jsx/-/styled-jsx-5.1.6.tgz", @@ -11944,6 +14355,22 @@ "integrity": "sha512-IV3Ou0jSMzZrd3pZ48nLkT9DA7Ag1pnPzaiQhpW7c3RbcqqzvzzVu+L8gfqMp/8IM2MQtSiqaCxrrcfu8I8rMA==", "license": "MIT" }, + "node_modules/supports-color": { + "version": "8.1.1", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz", + "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==", + "license": "MIT", + "peer": true, + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/supports-color?sponsor=1" + } + }, "node_modules/tailwindcss": { "version": "4.2.4", "resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-4.2.4.tgz", @@ -11951,12 +14378,120 @@ "dev": true, "license": "MIT" }, + "node_modules/tapable": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/tapable/-/tapable-2.3.3.tgz", + "integrity": "sha512-uxc/zpqFg6x7C8vOE7lh6Lbda8eEL9zmVm/PLeTPBRhh1xCgdWaQ+J1CUieGpIfm2HdtsUpRv+HshiasBMcc6A==", + "license": "MIT", + "engines": { + "node": ">=6" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + } + }, + "node_modules/terser": { + "version": "5.47.1", + "resolved": "https://registry.npmjs.org/terser/-/terser-5.47.1.tgz", + "integrity": "sha512-tPbLXTI6ohPASb/1YViL428oEHu6/qv1OxqYnfaonVCFHqx4+wCd95pHrQWsL5X4pl90CTyW9piSAsS2L0VoMw==", + "license": "BSD-2-Clause", + "peer": true, + "dependencies": { + "@jridgewell/source-map": "^0.3.3", + "acorn": "^8.15.0", + "commander": "^2.20.0", + "source-map-support": "~0.5.20" + }, + "bin": { + "terser": "bin/terser" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/terser-webpack-plugin": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-5.6.0.tgz", + "integrity": "sha512-Eum+5ajkaOhf5KbM26osvv21kLD7BaGqQ1UA4Ami4arYwylmGUQTgHFpHDdmJod1q4QXa66p0to/FBKID+J1vA==", + "license": "MIT", + "peer": true, + "dependencies": { + "@jridgewell/trace-mapping": "^0.3.25", + "jest-worker": "^27.4.5", + "schema-utils": "^4.3.0", + "terser": "^5.31.1" + }, + "engines": { + "node": ">= 10.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + }, + "peerDependencies": { + "webpack": "^5.1.0" + }, + "peerDependenciesMeta": { + "@minify-html/node": { + "optional": true + }, + "@swc/core": { + "optional": true + }, + "@swc/css": { + "optional": true + }, + "@swc/html": { + "optional": true + }, + "clean-css": { + "optional": true + }, + "cssnano": { + "optional": true + }, + "csso": { + "optional": true + }, + "esbuild": { + "optional": true + }, + "html-minifier-terser": { + "optional": true + }, + "lightningcss": { + "optional": true + }, + "postcss": { + "optional": true + }, + "uglify-js": { + "optional": true + } + } + }, + "node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "license": "MIT" + }, "node_modules/tslib": { "version": "2.8.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", "license": "0BSD" }, + "node_modules/type-fest": { + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.7.1.tgz", + "integrity": "sha512-Ne2YiiGN8bmrmJJEuTWTLJR32nh/JdL1+PSicowtNb0WFpn59GK8/lfD61bVtzguz7b3PBt74nxpv/Pw5po5Rg==", + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=8" + } + }, "node_modules/typescript": { "version": "5.9.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", @@ -11975,9 +14510,170 @@ "version": "6.21.0", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", - "dev": true, "license": "MIT" }, + "node_modules/update-browserslist-db": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.2.3.tgz", + "integrity": "sha512-Js0m9cx+qOgDxo0eMiFGEueWztz+d4+M3rGlmKPT+T4IS/jP4ylw3Nwpu6cpTTP8R1MAC1kF4VbdLt3ARf209w==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "escalade": "^3.2.0", + "picocolors": "^1.1.1" + }, + "bin": { + "update-browserslist-db": "cli.js" + }, + "peerDependencies": { + "browserslist": ">= 4.21.0" + } + }, + "node_modules/watchpack": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-2.5.1.tgz", + "integrity": "sha512-Zn5uXdcFNIA1+1Ei5McRd+iRzfhENPCe7LeABkJtNulSxjma+l7ltNx55BWZkRlwRnpOgHqxnjyaDgJnNXnqzg==", + "license": "MIT", + "peer": true, + "dependencies": { + "glob-to-regexp": "^0.4.1", + "graceful-fs": "^4.1.2" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "license": "BSD-2-Clause" + }, + "node_modules/webpack": { + "version": "5.106.2", + "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.106.2.tgz", + "integrity": "sha512-wGN3qcrBQIFmQ/c0AiOAQBvrZ5lmY8vbbMv4Mxfgzqd/B6+9pXtLo73WuS1dSGXM5QYY3hZnIbvx+K1xxe6FyA==", + "license": "MIT", + "peer": true, + "dependencies": { + "@types/eslint-scope": "^3.7.7", + "@types/estree": "^1.0.8", + "@types/json-schema": "^7.0.15", + "@webassemblyjs/ast": "^1.14.1", + "@webassemblyjs/wasm-edit": "^1.14.1", + "@webassemblyjs/wasm-parser": "^1.14.1", + "acorn": "^8.16.0", + "acorn-import-phases": "^1.0.3", + "browserslist": "^4.28.1", + "chrome-trace-event": "^1.0.2", + "enhanced-resolve": "^5.20.0", + "es-module-lexer": "^2.0.0", + "eslint-scope": "5.1.1", + "events": "^3.2.0", + "glob-to-regexp": "^0.4.1", + "graceful-fs": "^4.2.11", + "loader-runner": "^4.3.1", + "mime-db": "^1.54.0", + "neo-async": "^2.6.2", + "schema-utils": "^4.3.3", + "tapable": "^2.3.0", + "terser-webpack-plugin": "^5.3.17", + "watchpack": "^2.5.1", + "webpack-sources": "^3.3.4" + }, + "bin": { + "webpack": "bin/webpack.js" + }, + "engines": { + "node": ">=10.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + }, + "peerDependenciesMeta": { + "webpack-cli": { + "optional": true + } + } + }, + "node_modules/webpack-sources": { + "version": "3.4.1", + "resolved": "https://registry.npmjs.org/webpack-sources/-/webpack-sources-3.4.1.tgz", + "integrity": "sha512-eACpxRN02yaawnt+uUNIF7Qje6A9zArxBbcAJjK1PK3S9Ycg5jIuJ8pW4q8EMnwNZCEGltcjkRx1QzOxOkKD8A==", + "license": "MIT", + "peer": true, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/webpack/node_modules/eslint-scope": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-5.1.1.tgz", + "integrity": "sha512-2NxwbF/hZ0KpepYN0cNbo+FN6XoK7GaHlQhgx/hIZl6Va0bF45RQOOwhLIy8lQDbuCiadSLCBnH2CFYquit5bw==", + "license": "BSD-2-Clause", + "peer": true, + "dependencies": { + "esrecurse": "^4.3.0", + "estraverse": "^4.1.1" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "license": "MIT", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "license": "MIT", + "engines": { + "node": ">=0.4" + } + }, + "node_modules/yallist": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", + "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", + "license": "ISC" + }, "node_modules/yargs-parser": { "version": "21.1.1", "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", diff --git a/frontend/package.json b/frontend/package.json index b6474198..ad2b5521 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -11,11 +11,13 @@ "codegen": "openapi-typescript http://localhost:8000/openapi.json -o src/lib/api-types.ts" }, "dependencies": { + "@sentry/nextjs": "^10.53.1", "@tanstack/react-query": "^5.50.0", "echarts": "^6.0.0", "echarts-for-react": "^3.0.6", "leaflet": "^1.9.4", "leaflet-draw": "^1.0.4", + "lucide-react": "^0.511.0", "next": "^15.0.0", "react": "^19.0.0", "react-dom": "^19.0.0", @@ -34,6 +36,6 @@ "openapi-typescript": "^7.0.0", "postcss": "^8.4.0", "tailwindcss": "^4.0.0", - "typescript": "^5.5.0" + "typescript": "5.9.3" } } diff --git a/frontend/public/analytics.html b/frontend/public/analytics.html new file mode 100644 index 00000000..320f20a1 --- /dev/null +++ b/frontend/public/analytics.html @@ -0,0 +1,586 @@ + + + + + +/analytics — Свердл рынок · gendsgn + + + + + + + + + +
+
+
+ gendsgn + live · обновлено 4 ч назад +
+
+ Свердловская обл. · 2025-Q1 +
AD
Алексей · PRINZIP (demo)
+
+
+
+ +
+ + + +
+
+

Свердл рынок — обзор спроса и предложения

+
6 832 411 ДДУ Росреестра · 1 482 объекта ДОМ.РФ в активной фазе · обновление еженедельно
+
+
+ + +
+
+ + + +
+ вердикт квартала +

Дефицит средне-большого жилья 80+ м²: 37% сделок Росреестра, но 11% портфеля ДОМ.РФ. Окно сужается к Q3 2026.

+ источник: ДОМ.РФ · Росреестр · март 2026 +
+ +
+
+
Объём строительства
+
5,82млн м²
+
в активной фазе по области
+ ↑ 4,1% YoY +
+
+
Sold % (медиана)
+
61,4 %
+
объекты в продаже > 6 мес
+ ↓ 3,2 пп QoQ +
+
+
Средняя цена
+
186тыс ₽/м²
+
первичка, март 2026
+ ↑ 1,8% MoM +
+
+
Активных новостроек
+
347
+
в открытой продаже Я.Недв
+ +12 за месяц +
+
+ +
+
+
+

Карта рынка — новостройки Свердловской области

+
347 объектов в активной продаже · цвет точки — sold % за 6 месяцев · клик → drill-in
+
+
+ + + + + +
+
+
+ + + + + + + + + + + + + Академический + ВИЗ + Центр + Пионерский + Юго-Запад + Уралмаш + Эльмаш + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + +
+
+

Sold % за 6 мес

+
< 40 % · риск затоваривания
+
40–65 % · средний темп
+
65–80 % · хороший темп
+
80+ % · близко к закрытию
+
+ +
+
Карта стилизована для превью. В продукте — Leaflet с базой 2GIS, клик → /analytics/objects/[id]
+
+ +
+
+
+

Динамика рынка — 14 месяцев

+
Объём ДДУ (Росреестр) и две линии: sold % портфеля и средняя цена м²
+
+
+ + + + +
+
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + февмарапрмай + июниюлавгсен + октноядекянв + февмар + + +
+ +
+
+ +
+
+
+

Парадокс портфеля по комнатности

+
Доля сегмента в портфеле ДОМ.РФ против доли в сделках Росреестра за 14 мес
+
+
+ + + + +
+
+
+
+
Сегмент
+
портфель ДОМ.РФ vs сделки Росреестра
+
Δ пп
+
+
+
Студии до 30 м²
+
24%
9%
+
−15
+
+
+
1-комн. 30–45 м²
+
31%
24%
+
−7
+
+
+
2-комн. 45–65 м²
+
22%
23%
+
+1
+
+
+
3-комн. 65–85 м²
+
13%
24%
+
+11
+
+
+
4+ комн. / 85+ м²
+
11%
20%
+
+9
+
+
+
+ +
+
+
+
+

Pipeline по году ввода

+
Объём (тыс. м²) в разрезе классов
+
+
+ +
+
+
+
2024
1 240
+
2025
1 820
+
2026
2 110
+
2027
2 580
+
2028
1 970
+
+
+ комфорт + бизнес + премиум + элит +
+
+ +
+
+
+

Активные новостройки

+
Топ-7 по объёму · Я.Недвижимость · обновлено 4 ч назад
+
+
+ +
+
+ + + + + + + + + + + + + +
ЖКДевелоперSold %Ср. цена
Каменные палаткиPRINZIP78 %173 ₽
Северная коронаБрусника71 %198 ₽
ЧкаловАтомстройком.64 %156 ₽
МалевичФорум-групп58 %182 ₽
УютАтлас Девел.52 %141 ₽
Гольф-паркКОРТРОС41 %224 ₽
Эталон CityЭталон38 %167 ₽
+
+
+ +
+
+
+

Концентрация ЖК по районам

+
Площадь плитки — объём строительства · цвет — sold % · клик → топ девелоперов района
+
+
+ + + +
+
+
+ Академический42 ЖК · 824 тыс м² · sold 71 % + Центр28 ЖК · 612 тыс м² · sold 64 % + ВИЗ31 ЖК · 588 тыс м² · sold 48 % + Пионерский19 ЖК · 411 тыс м² + Юго-Запад14 · 386 тыс м² + Уралмаш11 · 254 тыс м² + Эльмаш8 · 198 + Сортировка6 · 142 + Уктус7 · 168 + Химмаш4 · 96 + Широкая речка5 · 124 тыс м² + Верхняя Пышма12 · 218 тыс м² +
+
+ +
+ ! +
+ Caveat по данным. Sold % считается по объектам в продаже более 6 месяцев; новые ЖК со стартом < 6 мес не входят в медиану, но видны на карте. Росреестр может задерживать ДДУ до 14 дней — берём 7-дневный backfill при еженедельном пересчёте. +
+
+ +
+ Развёрнутый разрез по 17 районам области +
Sold %, средний срок, цена м², объём в работе — по каждому из 17 районов с фильтром по классу. Открывается отдельной таблицей в /analytics/districts.
+
+
+ Методология подсчёта sold % +
Sold % = (общее число проданных лотов по объекту) / (общее число лотов в продаже). Источник: ДДУ Росреестра матчатся с лотами портфеля ДОМ.РФ по cadastral_id + comm_name fuzzy match (порог 0.85, pg_trgm).
+
+
+ Сравнение с январём 2025 (год к году) +
Sold % медиана: 64,6 % → 61,4 % (−3,2 пп). Объём строительства: 5,59 → 5,82 млн м² (+4,1 %). Средняя цена: 166 → 186 тыс ₽/м² (+12 %). Разрыв спрос-предложение в 3к/4+ сегменте увеличился: с +14 пп до +20 пп суммарно.
+
+ +
+ + + diff --git a/frontend/public/developers.html b/frontend/public/developers.html new file mode 100644 index 00000000..b7ede0f6 --- /dev/null +++ b/frontend/public/developers.html @@ -0,0 +1,515 @@ + + + + + +/analytics/developers — Топ-15 девелоперов · gendsgn + + + + + + + + + +
+
+
+ gendsgn +
+
+ Свердловская обл. · 2025-Q1 +
AD
Алексей · PRINZIP (demo)
+
+
+
+ +
+ + + +
+
+

Топ-15 девелоперов Свердловской области

+
Выбран: PRINZIP · кликните на строку leaderboard, чтобы сравнить с другим девелопером
+
+
+ +
+
+ + + + +
+
+
Девелопер
+
PRINZIP
+
Екатеринбург · с 2007 г.
+
+ комфорт + бизнес +
+
+
+
Объём в Свердл
+
412тыс м²
+
14 ЖК в активной фазе
+ ↑ +28 тыс vs Q4 2025 +
+
+
Sold % (медиана)
+
74%
+
по 14 ЖК в продаже
+ +13 пп vs рынок (61 %) +
+
+
Средний метраж
+
58м²
+
взвешенно по лотности
+ −9 м² vs рынок (67 м²) +
+
+ + +
+
+
+

Leaderboard — топ-15 по объёму

+
Sticky первая колонка при scroll · клик по строке → drawer с профилем девелопера
+
+
+
+ + + + +
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#ДевелоперОбъём, тыс м²ЖКSold %Δ 14 месСр. цена ₽/м²Ср. метражТоп-район
01PRINZIP
14 ЖК · комфорт+бизнес
4121474 %+8,2 пп17358Академический
02Брусника
11 ЖК · комфорт+бизнес
3681171 %+5,1 пп19861Центр
03Атомстройкомплекс
9 ЖК · комфорт
342964 %+2,8 пп15654ВИЗ
04Форум-групп
8 ЖК · комфорт+бизнес
298862 %+0,4 пп18263Юго-Запад
05КОРТРОС
6 ЖК · бизнес+премиум
284654 %−3,2 пп22472Пионерский
06Атлас Девелопмент
7 ЖК · комфорт
241752 %−4,8 пп14152Уралмаш
07Эталон
5 ЖК · бизнес
198548 %−6,1 пп16759Центр
08Унистрой
5 ЖК · комфорт
182567 %+3,4 пп14855Уктус
09Атмосфера
4 ЖК · комфорт
156469 %+4,1 пп15257Пионерский
10Эфес
4 ЖК · бизнес+премиум
142458 %+0,8 пп21176Центр
11Семь
3 ЖК · комфорт
128372 %+6,7 пп16756ВИЗ
12Гринстрой
3 ЖК · комфорт
116349 %−5,4 пп13451Сортировка
13Лидер
3 ЖК · комфорт
104361 %+1,2 пп15854Уралмаш
14Гранд
3 ЖК · бизнес
96344 %−8,1 пп18968Юго-Запад
15УГМК-Девел.
2 ЖК · бизнес+премиум
84256 %+0,2 пп23174Верх. Пышма
+
+
+ Показано 15 из 87 девелоперов в выборке. Порог — > 20 тыс м² в активной фазе. + Показать все 87 → +
+
+ + +
+
+
+

Velocity-карта — объём против Δ sold %

+
Кто продаёт быстрее в своём масштабе · ось X — тыс. м² в активной фазе · ось Y — Δ sold % за 14 мес
+
+
+ + + + +
+
+
+ + + + + + + + + + + + + + + + 0 пп — рынок + + + быстрее рынка + медленнее рынка + + + + +10 + +5 + 0 + −5 + −10 + + + + 0 + 100 + 200 + 300 + 400 + 500 + + + + + + + + PRINZIP · +8,2 + + + Брусника · +5,1 + + + Атомстройком. · +2,8 + + + Форум-групп + + + КОРТРОС · −3,2 + + + Атлас Девел. + + + Эталон · −6,1 + + + Унистрой + + + Атмосфера + + + Эфес + + + Семь · +6,7 + + + Гринстрой + + + Лидер + + + Гранд + + + УГМК-Девел. + + +
Δ sold % за 14 мес
+
Объём, тыс. м² в активной фазе
+
+
+ Размер точки — число ЖК в портфеле + Цвет — статус velocity (зелёный — растёт, красный — падает) + 38 / 40 ЖК в выборке · 2 объекта < 6 мес в продаже исключены +
+
+ + +
+
+
+

Сравнение sold % — PRINZIP vs топ-2

+
Кумулятивный sold % помесячно по всему портфелю; PRINZIP жирнее, Брусника и Атомстройком. для контекста
+
+
+ + + + +
+
+
+ + + + + + + + + + + + + + + + + + + + 100 % + 75 % + 50 % + 25 % + + + + фев 25 + апр + июн + авг + окт + дек + фев 26 + мар 26 + + +
+
+ PRINZIP 74 % · +8,2 пп + Брусника 71 % · +5,1 пп + Атомстройком. 64 % · +2,8 пп +
+
+ + +
+
+
+

Карточка выбранного девелопера — PRINZIP

+
Открывается drawer'ом справа при клике на строку leaderboard (не navigate)
+
+
+ Открыть профиль → +
+
+
+
+

Концентрация по районам

+
+
Академический
62 %
+
Центр
18 %
+
Юго-Запад
11 %
+
Пионерский
6 %
+
ВИЗ
3 %
+
+
+
+

Распределение по комнатности

+
+
Студии
18 %
+
1-к
34 %
+
2-к
28 %
+
3-к
14 %
+
4+
6 %
+
+
+ Gap vs рынок: доля 3к/4+ — 20 % vs 44 % рыночного спроса. Недозагрузка в большом сегменте. +
+
+
+

Топ-5 ЖК · drill-in

+ + + + + + + + + +
ЖКSold %
Каменные палатки78 %
Хрустальный81 %
Прогресс72 %
Светлая роща69 %
Парк-холл52 %
+
+
+
+ +
+ + + diff --git a/frontend/public/index.html b/frontend/public/index.html new file mode 100644 index 00000000..a499f9bb --- /dev/null +++ b/frontend/public/index.html @@ -0,0 +1,571 @@ + + + + + +gendsgn — превью нового сайта + + + + + + + + + +
+
+ gendsgn +
+ обзор макетов + + 5 экранов + + UI brief · v1 +
+
+
+ +
+ +
+ Превью нового сайта +

Фасад для пилотных девелоперов

+

Маркетинговый лендинг и четыре продуктовых экрана: рыночная аналитика по Свердловской области, топ-15 девелоперов с velocity-картой, поиск участка по кадастру и детальный анализ с sidebar-навигацией. Сделано по UI brief — Inter, tabular-nums, density-first как DOM.РФ / Bloomberg.

+
    +
  • 5экранов
    • +
  • Intertabular numerics
    • +
  • 1280 pxdesktop · до 412 px responsive
    • +
  • v1UI brief · черновик
    • +
+
+ +
+ + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+ Маркетинговый лендинг +

Лендинг для пилотных девелоперов

+

Hero «Спрос смещается быстрее, чем ваш портфель», парадокс «что строят / что продают» по 5 сегментам, три продуктовых модуля, прайс 30 тыс ₽/мес с честным дисклеймером pre-revenue, FAQ.

+
+ Hero + parados + 3 product cards + Pricing + FAQ +
+ Открыть превью +
+ + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+ Продуктовый экран +

/analytics — Свердловский рынок

+

Headline-вердикт о дефиците 80+ м², 4 KPI выше fold, карта новостроек по sold %, парадокс портфеля vs реальные сделки, treemap районов.

+
+ Headline-bar + Карта + Treemap +
+ Открыть превью +
+ + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+ Продуктовый экран +

/developers — Топ-15 девелоперов

+

Search + leaderboard на 15 девелоперов, scatter «Velocity vs объём» с осями (X: тыс м², Y: Δ sold % за 14 мес), сравнение PRINZIP с топ-2 за 14 месяцев.

+
+ Leaderboard + Scatter + Export CSV +
+ Открыть превью +
+ + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+ Site Finder · Screen 1 +

/site-finder — карта-first entry

+

Полноэкранная SVG-карта Екатеринбурга, поле кадастра с примером формата, недавние участки, sticky-карточка выбранного парцела с 7 параметрами и кнопками «Карточка ЕГРН» + «Анализ».

+
+ Карта-first + Кадастр +
+ Открыть превью +
+ + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+ Site Finder · Screen 2 + 3 +

/site-finder/анализ — sidebar на 5 разделов

+

Sidebar: Инфо / Сети 1–1,5 км / Продажи конкурентов (3.1/3.2/3.3) / Оценка / Атмосфера. Раздел 3 — 8-секционный модуль с 4 настройками и 4 выводами (планировки с превью, остатки, ассортимент vs скорость, рекомендация).

+
+ Sidebar nav + 8-секц. модуль + JS pane-switch +
+ Открыть превью +
+ + +
+ + + +
+ +
+ gendsgn · превью v1 · собрано из UI brief + 5 экранов · Inter + tabular-nums · brief HEX-палитра +
+ + + diff --git a/frontend/public/landing.html b/frontend/public/landing.html new file mode 100644 index 00000000..0d8afba8 --- /dev/null +++ b/frontend/public/landing.html @@ -0,0 +1,644 @@ + + + + + +gendsgn — аналитика спроса для девелоперов + + + + + + + + + + +
+
+ gendsgn + +
+ Demo аналитики → + Запросить пилот +
+
+
+ +
+ + +
+
+
+

Цифры рынка перед тем, как заливать фундамент.

+

Что строят на рынке и что на самом деле продаётся — по 6,83 млн ДДУ Росреестра, портфелю ДОМ.РФ и активным новостройкам Яндекс.Недвижимости. Свердловская область, обновление еженедельно.

+
+ Запросить пилот + Открыть demo аналитики → +
+

3 из 3 пилотных мест открыто · старт пилота — 2 недели · фикс-цена 30 000 ₽/мес

+
+ +
+
+ + +
+ Вердикт квартала +

Дефицит средне-большого жилья 80+ м²: 37% сделок Росреестра, но 11% портфеля ДОМ.РФ. Окно сужается к Q3 2026.

+ данные: ДОМ.РФ · Росреестр · март 2026 +
+ + +
+
+
Объём строительства
+
5,8 млн м²
+
Свердловская обл., активная фаза
+ ↑ 4,1% YoY +
+
+
Sold % (медиана)
+
61,4 %
+
по объектам в продаже > 6 мес
+ ↓ 3,2 пп QoQ +
+
+
Средняя цена
+
186 тыс ₽
+
за м², первичка, март 2026
+ ↑ 1,8% MoM +
+
+
Активных новостроек
+
347
+
в Яндекс.Недв · в открытой продаже
+ +12 за месяц +
+
+ + +
+
+

Парадокс портфеля: что строят vs что продаётся

+
Доля каждого сегмента в активном портфеле ДОМ.РФ против доли в сделках Росреестра за 14 месяцев
+
+ Открыть в /analytics → +
+ +
+
+ +
+
+
Студии до 30 м²
+
+
24%
+
9%
+
+
−15 пп
+
+
+
1-комн. 30–45 м²
+
+
31%
+
24%
+
+
−7 пп
+
+
+
2-комн. 45–65 м²
+
+
22%
+
23%
+
+
+1 пп
+
+
+
3-комн. 65–85 м²
+
+
13%
+
24%
+
+
+11 пп
+
+
+
4+ комн. / 85+ м²
+
+
11%
+
20%
+
+
+9 пп
+
+
+
+
+ Иллюстративные данные пилотной выборки. В продуктовой версии — разрез по 17 районам области, классу жилья и году ввода. Источник: ДОМ.РФ (портфель в активной фазе строительства), Росреестр (сделки ДДУ за 14 мес). +
+
+ + +
+
+

Три рабочих модуля

+
Не «features», а конкретные вопросы, на которые финдиректор девелопера ищет ответ
+
+
+ +
+ +
+
+

Аналитика спроса

+ /analytics +
+

«В каких районах и сегментах сделки опережают предложение?» Разбивка sold % по комнатности, классу, году ввода. Бенчмарк против топ-15 девелоперов области.

+
+ +
+
+ Открыть demo + в пилот → +
+
+ +
+
+

Рекомендатор площадки

+ /recommend +
+

«Какую квартирографию закладывать на участке в Академическом, чтобы выйти за 22 месяца?» Mix по комнатности, прогноз срока 80 % продаж, расчёт выручки.

+
+ +
+
+ Открыть demo + в пилот → +
+
+ +
+
+

Топ-15 девелоперов

+ /developers +
+

«Кто продаёт быстрее в моём районе и за счёт чего?» Velocity-карта по площади и Δ sold %, фильтр по классу. Drill-down до объекта.

+
+ +
+
+ Открыть demo + в пилот → +
+
+ +
+ + +
+
+

Пилот — 30 000 ₽ в месяц

+
Один тариф, без апсейлов. 3 из 3 пилотных мест открыто, ввод данных за 2 недели.
+
+
+ +
+
+ Пилот · 6 месяцев +

gendsgn для одного девелопера

+

Полный доступ к трём модулям. Свердловская область как первый регион, остальные — по запросу.

+
30 000 ₽/ мес · фикс
+
    +
  • Аналитика спроса по 17 районам области
    • +
  • Рекомендатор площадки с прогнозом срока продаж
    • +
  • Бенчмарк против топ-15 девелоперов
    • +
  • Еженедельный апдейт данных Росреестра
    • +
  • Экспорт в Excel и CSV для финмодели
    • +
  • Запрос новых разрезов от пилотного клиента — приоритет
    • +
+ Запросить пилот +
Контакт менеджера — Алексей Демидов · al@gendsgn.ru
+
+
+ После пилота +

Региональная лицензия

+

Когда пилот закроется и продукт перейдёт к платным клиентам.

+
от 120 000 ₽/ мес · region-pack
+
    +
  • Подключение дополнительных регионов
    • +
  • API для отдела финансового планирования
    • +
  • Бенчмарк против выбранных конкурентов
    • +
  • Кастомные разрезы и custom alerting
    • +
+ Запросить расчёт +
Расчёт индивидуально, после квалификации пилота
+
+
+ +
+ Честный дисклеймер: мы pre-revenue. Сайт показывает реальный продукт в работе на ваших данных. До первой ARR-метрики команда живёт за счёт основателей, и пилотных клиентов мы выбираем сами, а не наоборот. +
+ + +
+
+

Частые возражения

+
Что обычно спрашивают финдиректора прежде, чем согласиться на пилот
+
+
+ +
+
+

Если ваше возражение не разобрано — напишите на al@gendsgn.ru, ответим за 1 рабочий день.

+
+
+
+ Чем вы отличаетесь от собственной BI-команды? +
Мы не делаем дашборд под отдел — мы продаём готовый рыночный разрез по 6,83 млн ДДУ и портфелю ДОМ.РФ, который ваша BI собирала бы сама 4–6 месяцев. Внутренняя команда нужна, чтобы интерпретировать наши цифры в контексте конкретного проекта.
+
+
+ Откуда у вас данные Росреестра и ДОМ.РФ? +
ДОМ.РФ — открытый портфель новостроек. Росреестр — публичные ДДУ через выписки ЕГРН и партнёрские интеграции (без серых источников). Яндекс.Недвижимость — публичный листинг активных объектов в продаже.
+
+
+ Что входит в «обновление 7 дней»? +
Раз в неделю — пересчёт sold %, добавление новых ДДУ, обновление цен по активным объектам. Раз в месяц — пересмотр квартирографии и портфеля ДОМ.РФ. Любое окно < 7 дней — кешируем предыдущий снимок и помечаем «stale».
+
+
+ Почему только Свердловская область? +
Мы pre-revenue и заточили пайплайн под один регион, чтобы доказать ROI за 6 месяцев пилота. Добавление второго региона — около 6 недель работы на наш бэк. Если в пилот зайдёт девелопер другой области — мы её добавим в очередь.
+
+
+ Можно ли использовать данные в внутреннем финмоделировании? +
Да. Все ключевые разрезы экспортируются в Excel и CSV (sold %, цены, mix). Лицензия пилота позволяет использовать выгрузки во внутреннем финмодуле без отдельного согласования.
+
+
+ Что будет после 6 месяцев пилота? +
Совместный отчёт по конкретным cases (где наши рекомендации совпали со сделкой, где разошлись), и переход на регулярную лицензию либо корректное расставание. Никаких автоматических продлений с отдельным счётом.
+
+
+
+ +
+ + + + + diff --git a/frontend/public/site-finder-analysis.html b/frontend/public/site-finder-analysis.html new file mode 100644 index 00000000..a8f3cf9a --- /dev/null +++ b/frontend/public/site-finder-analysis.html @@ -0,0 +1,1112 @@ + + + + + +/site-finder/анализ — 66:41:0701045:42 · gendsgn + + + + + + + +
+ Превью · gendsgn + +
+ +
+
+
+ gendsgn + +
+
+ АК Алексей К. · PRINZIP +
+
+
+ +
+
+ ← К карте + · + 66:41:0701045:42 + · + Академический район + · + 0,82 га + анализ свежий · 8 мин +
+
+ +
+ + + + + +
+ + +
+
+
+

1. Инфо об участке

+

Кадастровые данные ЕГРН · POI 2GIS · ВРИ из НСПД

+
+
+ +
+
+ +
+ подходит + Участок 0,82 га в Академическом · комфорт-класс · собственность ООО «УралЗем» · sold% соседей 62% при медиане города 48%. + данные ЕГРН на 16.05.2026 +
+ +
+
+
Площадь
+
0,82 га
+
8 200 м² · ВРИ 2.6 многоэтажка
+
+
+
Район
+
Академический
+
8 км до центра · трамвай 23
+
+
+
Медиана цены
+
186 тыс ₽/м²
+
+11% за 6 мес +11%
+
+
+
Score участка
+
78/100
+
верхний квартиль по району +12
+
+
+ +
+
+
+

Расположение

+

Граница участка по ЕГРН + ключевые POI в радиусе 1 км

+
+
+ +
+
+
+
+ + + + + + + + + + + + + + 66:41:0701045:42 + + + м. Чкаловская (4 мин) + Парк УрФУ (8 мин) + Школа №197 (6 мин) + МЦК «Орбита» + Перекрёсток + + + + +
+
+
    +
  • Кадастровый номер66:41:0701045:42
    • +
  • Адресул. Краснолесья, уч. 14
    • +
  • Площадь8 234 м² (0,82 га)
    • +
  • Категория земельЗемли населённых пунктов
    • +
  • ВРИ (актуальный)2.6 Многоэтажная застройка
    • +
  • Форма собственностиЧастная
    • +
  • ПравообладательООО «УралЗем»
    • +
  • Дата регистрации14.03.2022
    • +
  • ОбремененияНет
    • +
  • Красные линии в границахКасание с СЗО ВЛ-110
    • +
+
+
+
+ +
+
+

Точки притяжения (POI) в радиусе 1 км

2GIS · OpenStreetMap · сортировка по weighted score

+
+
+
+
M
Станция метро «Чкаловская»
Зелёная линия, северный вход
420 м
4 мин пеш.
+
P
Парк УрФУ
7,2 га · детская инфраструктура
680 м
8 мин пеш.
+
S
Школа №197
МАОУ · 1240 учеников · рейтинг 4.2
540 м
6 мин пеш.
+
S
Гимназия «Корифей»
Частная · 380 учеников
820 м
10 мин пеш.
+
K
Детсад №524
Муницип · 240 мест · очередь 18 чел.
310 м
3 мин пеш.
+
$
Перекрёсток
2 400 м² · 7:00–23:00
470 м
5 мин пеш.
+
H
ГКБ №40
Многопрофильная больница
1,1 км
13 мин пеш.
+
+
+
+ + +
+
+
+

2. Сети и точки подключения

+

Энергетическая, тепловая, водопроводная, газовая в радиусе 1–1,5 км · НСПД + reverse-engineered энергоплан

+
+
+ +
+
+ +
+ подключаемо + Все 4 ресурса в зоне ≤ 320 м. Узкое место — газ 1,4 км, но для жилого многоквартирного не критично (центральное отопление от ТГК-9). + данные НСПД на Q1 2026 +
+ +
+
Теплоснабжение
240 м
ТГК-9 · магистраль Ø500 резерв 18 Гкал/ч
+
Водоснабжение
180 м
МУП «Водоканал» · колодец ВК-241
+
Электричество
320 м
ПС 110/10 «Академ» резерв 9 МВт
+
Газ
1,4 км
ГРС-3 «Уралмаш» для МКД не требуется
+
+ +
+
+

Схема сетей в радиусе 1,5 км

Условные обозначения по СП 18.13330.2019

+
+ + +
+
+
+ + + + + + + + + + + 1 км + 1,5 км + + + + + + + ТК-118 · 240 м + + + + + + ВК-241 · 180 м + + + + + + ВЛ-110 «Академ» · 320 м + + + + ГРС «Уралмаш» · 1,4 км ↓ + + + + участок + 0,82 га + + + + тепло + вода + электро 10 кВ + газ + + +
+
+ +
+
+

Точки подключения

Координаты, владелец сети, доступный резерв, оценка стоимости подключения

+
+
+ + + + + + + + + + +
РесурсТочкаКоординатыВладелецРасстояниеРезервСтоимость подкл.Статус
Тепло
ТК-11856,838 / 60,597ТГК-9240 м18 Гкал/ч3,4 млн ₽подключаемо
Вода
ВК-24156,839 / 60,598МУП «Водоканал»180 м280 м³/сут1,8 млн ₽подключаемо
Электро 10 кВ
ПС «Академ»56,840 / 60,600МРСК Урала320 м9 МВт12,6 млн ₽подключаемо
Газ среднего давл.
ГРС-356,832 / 60,612Газпром Е-бург1,4 кмнедостаточно≈ 28 млн ₽альтернатива
Канализация
КК-8956,839 / 60,599МУП «Водоканал»210 м240 м³/сут2,1 млн ₽подключаемо
Ливневая
ЛК-3256,838 / 60,594Адм. Е-бурга280 м—0,9 млн ₽подключаемо
+
+
+ + +
+
+
+

3. Продажи конкурентов

+

DOM.РФ ДДУ-выгрузка + Rosreestr 6,83 млн ДДУ · обновление еженедельно

+
+
+ + +
+
+ +
+ окно дефицита открыто + В выборке 7 ЖК · 1 482 квартиры в продаже · средняя скорость 22 ДДУ/мес · 80+ м² составляют 11% портфеля, но 37% спроса → конкурент Forum даёт +86% уход. + данные на 16.05.2026 +
+ + +
+

▸ Настройки выборки пп. 1–4

+
+
+
1 Радиус выборки
+
2,5 км
+
+
12345 км
+
+
+
2 Только строящиеся
+
включено
+3 мес от срока ввода
+
не показывать сданное
скрыто 4 ЖК · 312 квартир
+
+
+
3 Период продаж
+
−6 мес от 16.11.2025
+
+
3691224 мес
+
+
+
4 Только квартиры
+
flat_type = «Квартира»
машиноместа, кладовые, апартам. — скрыто
+
данных по парковкам — есть, но по этому ЖК и Брусника не разделяют. Дам знать когда подключим.
+
+
+
+ + + + +
+
+
+

5 Планировки конкурентов

+

12 уникальных типов · сгруппированы по комнатности и площади · картинки из DOM.РФ скрапера

+
+
+ + +
+
+
+ + +
+
+ −47% + кухня-гостспальняс/у + 7,8 млн +
+
+
1-комн «евро» 38 м²
+
встречается у 5 из 7 ЖК · 312 шт
+
предложение312
+
продано/мес9
+
+
+
+ + +
+
+ ±0 + гостспальнякухня + 10,1 млн +
+
+
1-комн классика 46 м²
+
встречается у 4 из 7 ЖК · 184 шт
+
предложение184
+
продано/мес16
+
+
+
+ + +
+
+ +12% + кухня-гостсп.1сп.2с/у + 14,4 млн +
+
+
2-комн «евро» 62 м²
+
встречается у 6 из 7 ЖК · 238 шт
+
предложение238
+
продано/мес24
+
+
+
+ + +
+
+ +86% + кухня-гостсп.1сп.2сп.3с/у + 19,2 млн +
+
+
3-комн 84 м²
+
только у 2 из 7 ЖК · 38 шт
+
предложение38
+
продано/мес14
+
+
+
+ +
+
+ + +
+
+
+

6 Остатки (нераспроданное)

+

flats_total − flats_sold по каждому ЖК в выборке · сортировка по абс. остатку

+
+
+ + +
+
+ + + + + + + + + + +
ЖКСрок вводаВсего, штПроданоОстатокДоля остаткаSold%Δ за 14 мес
ФР
Forum «Краснолесье»
Форум-групп · 2 очередь
IV кв 2026412266146 35%64%+18 пп
БР
Брусника «Академ-парк»
Брусника · 3 очередь
II кв 202636828088 24%76%+24 пп
ЭТ
Эталон «Чкаловский»
Эталон · 1 очередь
I кв 2027284112172 61%39%−6 пп
КР
Кортрос «Преображенский»
КОРТРОС · 2 очередь
III кв 202619613858 30%70%+12 пп
УН
Унистрой «Юг-1»
Унистрой · 4 очередь
IV кв 202614238104 73%27%−14 пп
АС
Атомстрой «Уралмаш»
Атомстройкомплекс · 1 очередь
I кв 2027802654 68%32%+4 пп
+
+ + +
+
+
+

7 Ассортимент и скорость продаж

+

−6 мес от сегодня · разрез по room_bucket × area_bin · ось Y — ДДУ/мес

+
+
+
+
+ + + + + + + + + + 0 + 10 + 20 + 30 + 40 + ДДУ / мес + + + + + + + + 1к до 40 + + + + + + 1к 40-50 + + + + + + 2к до 55 + + + + + + 2к евро 55-70 + + + + + + 3к 70-85 ⚡ + + + + + + 3к 85-110 ⚡ + + + + + + 4к+ 110+ ⚡ + + + + + предложение, шт ×0.1 + ДДУ / мес + + + + + окно дефицита 70+ м² + спрос > предложение + + +
+
+ + +
+
🜂 рекомендация — наша
+

Строить комфорт-микс с креном в 3-комн евро 70–85 м² (35% портфеля)

+

Дефицит 80+ м² в радиусе 2,5 км — реальный (37% спроса при 11% предложения). Forum «Краснолесье» забирает 86% этого спроса с одной типологии «3-комн 84 м²». Своя конкурентная позиция: тот же бакет с лучшей кухней-гостиной (24+ м²) и панорамным остеклением спальни. Цену держать 230 тыс ₽/м² против 215 тыс у Forum — рынок проглотит, sold % сохранится > 60% к месяцу 14.

+
+
Площадь застройки
12 600 м²
≈ 180 квартир
+
Прогноз выручки
2,4 млрд ₽
из них 1,1 млрд — 3к
+
Срок 80% продаж
22 мес
от старта продаж
+
Liquidity score
B+
медиана городa: B−
+
+
+ + + +
+
+
+ + +
+
+
+

4. Оценка участка

+

Композитный score 0–100 · веса калибруются под класс жилья

+
+
+ +
+
+ +
+ верхний квартиль + Score 78/100 · верхний квартиль по Свердл (медиана города 56) · единственное узкое место — касание с СЗО ВЛ-110, требует согласования. + калибровка на 2 142 ДДУ +
+ +
+
+ + + + 78 + из 100 + +
+
    +
  • Локация и транспортметро 420 м · школа 540 м · парк 680 м
    88
    • +
  • Спрос в районеsold % 62% · окно 80+ м² открыто
    84
    • +
  • Инженерные сетитепло/вода/электро ≤ 320 м
    80
    • +
  • Конкуренция7 ЖК в 1 км · Forum доминирует
    64
    • +
  • Цена / маржинальность+11% за 6 мес · потолок 230 тыс ₽/м²
    74
    • +
  • Юр-ограниченияСЗО ВЛ-110 в границах
    52
    • +
  • Экология / атмосфераAQI 38 · преобл. ЮЗ ветер
    82
    • +
+
+
+
+
✓
+
+

Gate: ПРОЙДЕНО (по умолчанию для комфорт-класса)

+

Все 4 обязательных критерия выполнены: ВРИ совместим (1), все 4 ресурса в 1,5 км (2), sold % района > 45% (3), score > 65 (4). Замечание: СЗО ВЛ-110 — требует согласования с МРСК Урала до начала проектирования (12–16 недель).

+
+
+
+
+ + +
+
+
+

5. Атмосфера и климат

+

Open-Meteo · ФГБУ «УГМС-Урал» · roza vetrov по последним 5 годам

+
+
+ +
+
+ +
+ пригодно + AQI 38 (good) · преобладающий ветер ЮЗ (38% времени) → промзона Уралмаш в наветренной стороне → низкий риск износа от выбросов. + 5-летний период +
+ +
+
AQI индекс
38
good
медиана города 52
+
PM2.5
9 мкг/м³
≤ 12
норма ВОЗ — 15
+
NO₂
28 мкг/м³
умерен.
влияние трамвайной линии
+
Шум днём
54 дБА
в норме
СН 2.2.4/2.1.8.562-96 ≤ 55
+
+ +
+
+

Роза ветров

направление преобладающего ветра · 5-летняя статистика УГМС

+
+ + + + + + + С + В + Ю + З + СВ + ЮВ + ЮЗ + СЗ + + + + + + + + + + + + +
+

Доминирует ЮЗ (38%) — комбинат «Уралмаш» в наветренной стороне → выбросы уходят на восток, в направлении ЕКАД. Для участка это благоприятный фактор.

+
+ +
+

Сезонная погода

средние значения за 5 лет · Open-Meteo

+
+
зима
−12°C
снежн. покров 87 дн · риск гололёда на парковке
+
весна
+6°C
паводок 1,2 м · район выше отметки 295
+
лето
+22°C
волны жары до +34 — нужно остекл. ≤ 35% юж. фасадов
+
осень
+8°C
осадки 180 мм · нагрузка на ливневую
+
+
    +
  • Солнечных дней / год118
    • +
  • Глубина промерзания1,8 м (фундамент свайно-плитный)
    • +
  • Расчётная сейсмика5 баллов MSK · норма по СП 14.13330
    • +
  • Уровень грунтовых вод−5,6 м (норма для свайного)
    • +
+
+
+
+ +
+
+ + + + + diff --git a/frontend/public/site-finder.html b/frontend/public/site-finder.html new file mode 100644 index 00000000..a1678ef4 --- /dev/null +++ b/frontend/public/site-finder.html @@ -0,0 +1,483 @@ + + + + + +/site-finder — Поиск участка · gendsgn + + + + + + + +
+ Превью · gendsgn + +
+ +
+
+
+ gendsgn + +
+
+ АК Алексей К. · PRINZIP +
+
+
+ +
+ + +
+ + + + + + + + + + + + + + + + + + + + + + Верх-Исетский + Ленинский + Кировский + Орджоникидзевский + Октябрьский + Чкаловский + Академический + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + +
+
+
site finder
+

Введите номер участка
или кликните на карту →

+
+
+
+
+ +
+ +
+

+ i + Кадастровый номер из ЕГРН в формате XX:XX:XXXXXXX:NN. Поддерживаем Свердловскую область, Челябинскую, Тюменскую. +

+
или
+ +
+
+
Недавние участки
+ +
+
+ + +
+ + + + + +
+ + +
+
показано 148 участков
+
подходят 34
+
обновлено 16 мин назад
+
+ + + + + +
+

Раскраска участков

+
    +
  • свободные, score > 7034
    • +
  • в работе, нужен анализ62
    • +
  • обременения / спорный ВРИ38
    • +
  • не подходит14
    • +
  • конкуренты (ЖК в продаже)74
    • +
+
+ + +
2 км
+
+ + + +
+ +
+ + + + + diff --git a/frontend/public/tradein.html b/frontend/public/tradein.html new file mode 100644 index 00000000..b6b74f0a --- /dev/null +++ b/frontend/public/tradein.html @@ -0,0 +1,2051 @@ + + + + + + Trade-In · PRINZIP — Оценка квартиры + + + + + + + + +
+
+
+
P
+ PRINZIP +
+
+ Trade-In · Оценка + + + + Поддержка + +
АК
+
+
+ +
+
+ Главная › + Trade-In › + Новая оценка +
+ +
+
+

Оценка квартиры на вторичке

+

+ Агрегируем оценки 7 источников + аналоги в продаже + фактические сделки. + Время сбора — 30–60 сек. +

+
+
+ Отчёт EKB-2485 + Дата 12.05.2026 + Срок данных до 11.06.2026 +
+
+ + +
+ + + + + +
+ + +
+
+
+
Шаг B · Агрегация
+

Сбор данных по квартире

+
+
+
4 / 7 источников
+
медиана: 4.16 млн ₽
+
+
+ +
+
+ Идёт параллельный запрос к 7 сервисам — Celery group, timeout 90 сек. +
+ ≈ 22 СЕК ДО ЗАВЕРШЕНИЯ +
+ +
+ +
+ + + + Циан + + 4.10 млн · 47 объявл. +
+ +
+ + + + Авито Оценка + + 4.20 млн · 31 объявл. +
+ +
+ + + + ДомКлик Прайс + + 12 ипотечных сделок в доме… +
+ +
+ + + + Я.Недвижимость + + 4.00 млн · 28 объявл. +
+ +
+ + + + Restate + + сбор аналогов… +
+ +
+ + + + N1.ru + + timeout — нет ответа +
+ +
+ + + + Дом.РФ + + кадастровая + рыночная… +
+ +
+ + + + Росреестр (внутр.) + + 12 сделок · 10.2025–05.2026 +
+ +
+
+ +
+ CACHE-KEY · + addr=66:41:0204041:1208 · area=42.9 · rooms=2 · repair=req + · + при недоступности источника частичный результат не блокируется +
+
+ + +
+
+
+
Секция 1 · Сводка
+

Анализ рынка и расчёт выкупной стоимости

+
+
+
Источник: агрегация 4/7
+
Достоверность · средняя · CV 6.2%
+
+
+ +
+ +
+
г. Екатеринбург, ул. Софьи Перовской, 119 кв. 368
+
КАД · 66:41:0204041:1208 · ЭТАЖ 3/9 · ИНДЕКС 620089
+ +
+
Год постройки1978
+
Тип домаПанельный
+
Этаж3 / 9
+
Площадь42.9 м²
+
Планировка2-к, классическая
+
СостояниеТребует ремонта
+
Балкон / Лоджия1 / 0
+
СанузелРаздельный
+
+
+
+ +
+
+
+ Диапазон цен в объявлениях (без учёта ремонта) + медиана · 4.16 млн ₽ +
+
+
+
+
+
+
+
3.4 млн
+
+
+
+
4.3 млн
+
+
+
4.16 млн ₽
+
ЭКСПОЗИЦИЯ · 4 ДНЕЙ
+
118 ДНЕЙ
+
+
+ +
+
+ Диапазон цен по фактическим сделкам + медиана · 3.95 млн ₽ +
+
+
+
+
+
+
+
3.4 млн
+
+
+
+
4.3 млн
+
+
+
3.95 млн ₽
+
ЭКСПОЗИЦИЯ · 10 ДНЕЙ
+
228 ДНЕЙ
+
+
+
+ +
+
+ + + +
Цены в объявлениях ≠ реальная сделка — разница 5–12% по данным Росреестра.
+
+
+ + + +
Косметический ремонт 400–600 тыс ₽ повышает стоимость на 150–250 тыс ₽.
+
+
+ + + +
Самостоятельная продажа = до 15% потерь на торге, риелторе, нотариусе и аренде.
+
+
+
+ + +
+
+
+
Секция 2 · Рынок
+

Объявления — аналогичные квартиры в продаже

+
+
+
Обновлено: 12.05.2026 · 14:22
+
Источников: 5 / 7
+
+
+ +
+
+
Объявлений по аналогам
+
47шт
+
из 5 источников за 30 дней
+
+
+
Из них с учётом ремонта
+
5шт
+
«Требует ремонта», ±1 категория
+
+
+
Медиана / разброс
+
4.16млн ₽
+
CV 6.2% · надёжно
+
+
+ +
+ Фильтр +
+ Расстояние±1 км + ТипЭконом · <2000 + Год1976 – 1981 + Площадь42.1 – 47.0 м² + Планировка2-к + СостояниеТребует ремонта +
+
+ +
+ Источники +
+ Я.Недвижимость · 28 + Циан · 47 + ДомКлик · 19 + Avito · 31 + N1.ru · — timeout +
+
+ +
+
+ ДИАПАЗОН ЦЕН В ОБЪЯВЛЕНИЯХ · 4.16 МЛН ₽ медиана + диапазон — 4 / 118 дней экспозиции +
+
+
+
+
+
+
+
3.4 млн
+
+
+
+
4.0
+
+
+
+
4.4
+
+
+
+
4.6 млн
+
+
+
4.16 млн ₽
+
P25 · 4.0 МЛН
+
P75 · 4.4 МЛН
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
АдресИсточник₽ / м²СтоимостьЭкспозиция
+
+ Екб, проезд Теплоходный, 11 + 42.5 м² · 2-к · панель 1977 +
+ 		Циан101 1764 300 0004 дн.
+
+ Екб, ул. Автомагистральная, 11 + 43.0 м² · 2-к · панель 1979 +
+ 		Avito104 6514 500 00022 дн.
+
+ Екб, ул. Бебеля, 136 + 44.7 м² · 2-к · панель 1981 +
+ 		Я.Недв94 8544 240 00048 дн.
+
+ Екб, ул. Софьи Перовской, 109 + 42.1 м² · 2-к · панель 1976 +
+ 		ДомКлик95 0114 000 00073 дн.
+
+ Екб, ул. Декабристов, 27 + 47.0 м² · 2-к · панель 1978 +
+ 		Циан76 5963 600 000118 дн.
+ +
+ Показано 5 из 47 объявлений · отсортировано по релевантности + Показать все → +
+
+ + +
+
+
+
Секция 3 · Сделки
+

Фактические сделки по аналогичным квартирам

+
+
+
Период: 10.2025 – 05.2026
+
Источников: 3 / 3 · ground truth
+
+
+ +
+
+
Сделок по аналогам
+
12шт
+
подтверждено Росреестром
+
+
+
Период
+
10.25 – 05.26
+
7 месяцев · разница к рынку −7%
+
+
+
Медиана / разброс
+
3.95млн ₽
+
10 – 228 дней экспозиции
+
+
+ +
+ Фильтр +
+ Расстояние±1 км + ТипЭконом · <2000 + Год1977 – 1979 + Площадь38.0 – 47.3 м² + Планировка2-к +
+
+ +
+ Источники +
+ Этажи · 7 сделок + ДомКлик · 3 ипотеки + Росреестр · 12 ДДУ/пер. +
+
+ +
+
+ ДИАПАЗОН ЦЕН ПО ФАКТ. СДЕЛКАМ · 3.95 МЛН ₽ медиана + экспозиция 10 / 228 дней +
+
+
+
+
+
+
+
3.4 млн
+
+
+
+
4.3 млн
+
+
+
3.95 млн ₽
+
ЭКСПОЗИЦИЯ · 10 ДНЕЙ
+
228 ДНЕЙ
+
+
+ +
+ + + +
По данным реальных сделок квартиры продаются в среднем на 5–12% дешевле, чем заявлено в объявлениях. Используйте сделочную медиану 3.95 млн ₽ как базу для выкупной цены.
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
АдресИсточник₽ / м²СтоимостьЭкспозицияДата
+
+ Екб, ул. Софьи Перовской, 115 + 42.9 м² · 2-к · панель 1978 +
+ 		Росреестр99 0684 250 00010 дн.04.2026
+
+ Екб, ул. Бебеля, 138 + 44.1 м² · 2-к · панель 1979 +
+ 		Этажи93 1974 110 00047 дн.02.2026
+
+ Екб, ул. Татищева, 84 + 38.0 м² · 2-к · панель 1977 +
+ 		ДомКлик94 7373 600 00095 дн.12.2025
+
+ Екб, ул. Викулова, 33 + 47.3 м² · 2-к · панель 1979 +
+ 		Росреестр90 9094 300 000228 дн.11.2025
+
+ Екб, ул. Заводская, 32/1 + 42.5 м² · 2-к · панель 1978 +
+ 		Этажи80 0003 400 000142 дн.10.2025
+ +
+ Показано 5 из 12 фактических сделок + Показать все → +
+
+ + +
+
+
+
Секция 4 · Оффер
+

Формирование выкупной стоимости

+
+
+
База расчёта: 3.95 млн ₽
+
Параметры настраиваемые в админке девелопера
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Статья расходовЧерез Trade-InСамостоятельная продажа
+
Торг покупателей
+
5–15% от цены объявления
+ 		не применимо182 500 – 547 500 ₽
+
Услуги риелтора
+
2–3% от цены сделки
+ 		бесплатно69 350 – 104 025 ₽
+
Аренда жилья после сделки
+
1-к квартира на 3 месяца
+ 		бесплатно90 000 – 150 000 ₽
+
Юридическое сопровождение
+
проверка документов, договор, регистрация
+ 		бесплатноот 19 250 ₽
+
Расходы на рекламу
+
платные показы Циан / Авито, 3 мес
+ 		бесплатно4 000 – 36 000 ₽
Общие финансовые потери— 0 ₽365 100 – 856 775 ₽
+ +
+ + + +
Издержки при самостоятельной продаже сопоставимы со скидкой 9–22% от рыночной цены. Trade-In закрывает их полностью, а цена в новостройке фиксируется на момент оценки.
+
+ + +
+
+
+ +
+

Экономия времени

+

Берём на себя показы, переговоры и поиск покупателей. Менеджеру остаётся только подписать ДКП.

+
+
+
+ +
+

Юридическая безопасность

+

Проверяем чистоту документов, ЕГРН, прописанных и обременения. Исключаем риски для клиента.

+
+
+
+ +
+

Фиксированная цена новостройки

+

Сохраняем выбранную планировку в каталоге PRINZIP по цене на момент оценки — до закрытия сделки.

+
+
+
+ +
+

Гарантия цены

+

Снимаем риск колебаний рынка — выкупная стоимость зафиксирована в оффере на 30 дней.

+
+
+ +
+
+ Готовы отправить оффер клиенту?
+ PDF-отчёт + персональный оффер по выкупной цене — за 1 клик. +
+
+ + +
+
+
+ +
+
+
+ + + + + + + diff --git a/frontend/sentry.client.config.ts b/frontend/sentry.client.config.ts new file mode 100644 index 00000000..613ea9a4 --- /dev/null +++ b/frontend/sentry.client.config.ts @@ -0,0 +1,14 @@ +import * as Sentry from "@sentry/nextjs"; + +const dsn = process.env.NEXT_PUBLIC_GLITCHTIP_DSN; +if (dsn) { + Sentry.init({ + dsn, + environment: process.env.NEXT_PUBLIC_ENVIRONMENT ?? "development", + tracesSampleRate: 0.05, + // GlitchTip does not support Session Replay — disable explicitly + replaysSessionSampleRate: 0, + replaysOnErrorSampleRate: 0, + integrations: [], + }); +} diff --git a/frontend/sentry.edge.config.ts b/frontend/sentry.edge.config.ts new file mode 100644 index 00000000..c92e6ec7 --- /dev/null +++ b/frontend/sentry.edge.config.ts @@ -0,0 +1,6 @@ +import * as Sentry from "@sentry/nextjs"; + +const dsn = process.env.GLITCHTIP_DSN ?? process.env.NEXT_PUBLIC_GLITCHTIP_DSN; +if (dsn) { + Sentry.init({ dsn, tracesSampleRate: 0.05 }); +} diff --git a/frontend/sentry.server.config.ts b/frontend/sentry.server.config.ts new file mode 100644 index 00000000..2f179e83 --- /dev/null +++ b/frontend/sentry.server.config.ts @@ -0,0 +1,10 @@ +import * as Sentry from "@sentry/nextjs"; + +const dsn = process.env.GLITCHTIP_DSN ?? process.env.NEXT_PUBLIC_GLITCHTIP_DSN; +if (dsn) { + Sentry.init({ + dsn, + environment: process.env.NODE_ENV ?? "development", + tracesSampleRate: 0.05, + }); +} diff --git a/frontend/src/app/_legacy/page-tabs.tsx b/frontend/src/app/_legacy/page-tabs.tsx new file mode 100644 index 00000000..75bff5fc --- /dev/null +++ b/frontend/src/app/_legacy/page-tabs.tsx @@ -0,0 +1,185 @@ +// Legacy landing — navigation portal (pre-A12 redesign 2026-05-18). +// Kept for rollback reference. Not routed — use /app/page.tsx. + +import Link from "next/link"; + +interface Route { + href: string; + label: string; + desc: string; +} + +const SECTIONS: { title: string; routes: Route[] }[] = [ + { + title: "Аналитика", + routes: [ + { + href: "/analytics", + label: "Свердл рынок", + desc: "Динамика 14 мес · парадокс «строят vs покупают» · pipeline · районы ЕКБ.", + }, + { + href: "/analytics/prinzip", + label: "PRINZIP drill-down", + desc: "Velocity vs benchmark · gap-bar к рынку · ЖК с sparkline · инсайты.", + }, + { + href: "/analytics/developers", + label: "Топ девелоперов", + desc: "Sortable leaderboard · velocity scatter · сравнение sold% по месяцам.", + }, + { + href: "/analytics/objects/60160", + label: "Карточка ЖК (пример: Парк Победы)", + desc: "KPI · sale-chart · корпуса ЖК (cad_quarter + buildings) · POI-таблица · фото-галерея. /analytics/objects/{obj_id}.", + }, + { + href: "/analytics/recommend", + label: "Рекомендатор квартирографии", + desc: "Район + площадь + класс → распределение по 5 бакетам, цены p25/median/p75, выручка, comparable ЖК. Слайдеры под проект.", + }, + ], + }, + { + title: "Site Finder", + routes: [ + { + href: "/site-finder", + label: "Анализ участка по cad-номеру", + desc: "Вводишь кадастр → карта + score соц-инфраструктуры (школы/сады/магазины/парки +вес, трамваи −вес) + 20 ближайших конкурентов с подсветкой района. Данные: OSM POI обновляются еженедельно.", + }, + ], + }, + { + title: "Adminка (Token-protected)", + routes: [ + { + href: "/admin/scrape/all", + label: "Скрапер — единая панель", + desc: "Все джобы (kn/nspd_geo/objective) в одном месте · job_settings inline-edit · bulk Свердл geo backfill · ScrapeLogsPanel.", + }, + { + href: "/admin/leads", + label: "Лиды PRINZIP CRM", + desc: "Воронка Sankey · конверсия по месяцам · KPI · таблица лидов с фильтрами.", + }, + ], + }, + { + title: "Generative (WIP)", + routes: [ + { + href: "/concept", + label: "Concept (Stage 1)", + desc: "Импорт полигона → 3 варианта застройки. WIP — Stage 1a-c.", + }, + ], + }, +]; + +export default function HomePageLegacy() { + return ( +
+
+

GenDesign

+

+ Generative Design + Site Finder — Discovery MVP. Свердл рынок & + PRINZIP. +

+
+ + {SECTIONS.map((section) => ( +
+

+ {section.title} +

+
+ {section.routes.map((r) => ( + +
+ {r.href} +
+
{r.label}
+
+ {r.desc} +
+ + ))} +
+
+ ))} + + +
+ ); +} diff --git a/frontend/src/app/admin/leads/page.tsx b/frontend/src/app/admin/leads/page.tsx index cbdbef25..7c7e96c6 100644 --- a/frontend/src/app/admin/leads/page.tsx +++ b/frontend/src/app/admin/leads/page.tsx @@ -68,11 +68,6 @@ interface LeadsStats { } export default function AdminLeadsPage() { - const [token] = useState(() => - typeof window === "undefined" - ? "" - : (localStorage.getItem("admin_token") ?? ""), - ); const [statusFilter, setStatusFilter] = useState(""); const [sourceFilter, setSourceFilter] = useState(""); const [convertedFilter, setConvertedFilter] = useState(""); @@ -81,36 +76,24 @@ export default function AdminLeadsPage() { const limit = 50; const stats = useQuery({ - queryKey: ["admin-leads-stats", token], - queryFn: () => - apiFetch("/api/v1/admin/leads/stats?months=12", { - headers: { "X-Admin-Token": token }, - }), - enabled: !!token, + queryKey: ["admin-leads-stats"], + queryFn: () => apiFetch("/api/v1/admin/leads/stats?months=12"), }); const funnelBySource = useQuery({ - queryKey: ["admin-leads-funnel-source", token], + queryKey: ["admin-leads-funnel-source"], queryFn: () => apiFetch( "/api/v1/admin/leads/funnel/by-source?months=12", - { - headers: { "X-Admin-Token": token }, - }, ), - enabled: !!token, }); const funnelMonthly = useQuery({ - queryKey: ["admin-leads-funnel-monthly", token], + queryKey: ["admin-leads-funnel-monthly"], queryFn: () => apiFetch( "/api/v1/admin/leads/funnel/monthly?months=24", - { - headers: { "X-Admin-Token": token }, - }, ), - enabled: !!token, }); const sankeyOption = useMemo(() => { @@ -239,18 +222,13 @@ export default function AdminLeadsPage() { }, [funnelMonthly.data]); const sources = useQuery({ - queryKey: ["admin-leads-sources", token], - queryFn: () => - apiFetch("/api/v1/admin/leads/sources", { - headers: { "X-Admin-Token": token }, - }), - enabled: !!token, + queryKey: ["admin-leads-sources"], + queryFn: () => apiFetch("/api/v1/admin/leads/sources"), }); const leads = useQuery({ queryKey: [ "admin-leads", - token, statusFilter, sourceFilter, convertedFilter, @@ -265,11 +243,8 @@ export default function AdminLeadsPage() { if (sourceFilter) qs.set("source", sourceFilter); if (convertedFilter !== "") qs.set("converted", convertedFilter); if (search) qs.set("search", search); - return apiFetch(`/api/v1/admin/leads/?${qs.toString()}`, { - headers: { "X-Admin-Token": token }, - }); + return apiFetch(`/api/v1/admin/leads/?${qs.toString()}`); }, - enabled: !!token, }); const fmtMoney = (v: number | null) => @@ -279,18 +254,6 @@ export default function AdminLeadsPage() { ? `${(v / 1e6).toFixed(2)} М ₽` : v.toLocaleString("ru"); - if (!token) { - return ( -

- Введите Admin Token на странице{" "} - - Скрапер - {" "} - — он сохранится для всех админ-страниц. -

- ); - } - if (stats.isError || leads.isError) { const err = String(stats.error ?? leads.error ?? ""); return ( diff --git a/frontend/src/app/admin/scrape/all/page.tsx b/frontend/src/app/admin/scrape/all/page.tsx index b4feb6ff..d3635112 100644 --- a/frontend/src/app/admin/scrape/all/page.tsx +++ b/frontend/src/app/admin/scrape/all/page.tsx @@ -4,7 +4,7 @@ import { useState } from "react"; import { useQuery } from "@tanstack/react-query"; import { apiFetch } from "@/lib/api"; -import { cardStyle, inputStyle, labelStyle, td, th } from "@/lib/adminStyles"; +import { cardStyle, labelStyle, td, th } from "@/lib/adminStyles"; import { BulkGeoPanel } from "@/components/admin/BulkGeoPanel"; import { JobSettingsPanel } from "@/components/admin/JobSettingsPanel"; @@ -67,46 +67,31 @@ function diffSecs(a: string | null, b: string | null): string { } export default function ScrapeAllAdminPage() { - const [token, setToken] = useState(() => - typeof window === "undefined" - ? "" - : (localStorage.getItem("admin_token") ?? ""), - ); const [filter, setFilter] = useState< "all" | "kn" | "nspd" | "objective" | "nspd_geo" >("all"); const [logsRunId, setLogsRunId] = useState(""); - const saveToken = (v: string) => { - setToken(v); - localStorage.setItem("admin_token", v); - }; - const runs = useQuery({ - queryKey: ["unified-runs", token, filter], + queryKey: ["unified-runs", filter], queryFn: () => { const q = filter !== "all" ? `?scraper_type=${filter}&limit=30` : "?limit=30"; - return apiFetch(`/api/v1/admin/scrape/all/runs${q}`, { - headers: { "X-Admin-Token": token }, - }); + return apiFetch(`/api/v1/admin/scrape/all/runs${q}`); }, - enabled: !!token, refetchInterval: 5000, }); const logs = useQuery({ - queryKey: ["unified-logs", token, filter, logsRunId], + queryKey: ["unified-logs", filter, logsRunId], queryFn: () => { const params = new URLSearchParams({ limit: "100" }); if (filter !== "all") params.set("scraper_type", filter); if (logsRunId !== "") params.set("run_id", String(logsRunId)); return apiFetch( `/api/v1/admin/scrape/all/logs?${params}`, - { headers: { "X-Admin-Token": token } }, ); }, - enabled: !!token, refetchInterval: 8000, }); @@ -146,19 +131,6 @@ export default function ScrapeAllAdminPage() { аудита.

-
- -
- {/* Сводка-плитки */}
- + {/* Bulk geo backfill */}
- +
{/* Logs */} diff --git a/frontend/src/app/admin/scrape/cadastre/page.tsx b/frontend/src/app/admin/scrape/cadastre/page.tsx index 95958980..0ca8c312 100644 --- a/frontend/src/app/admin/scrape/cadastre/page.tsx +++ b/frontend/src/app/admin/scrape/cadastre/page.tsx @@ -122,25 +122,14 @@ function reqPerSec(job: CadastreJob): string { // Query keys // --------------------------------------------------------------------------- -const jobsKey = (token: string) => ["admin", "cadastre", "jobs", token]; -const jobKey = (token: string, id: number) => [ - "admin", - "cadastre", - "job", - id, - token, -]; +const jobsKey = () => ["admin", "cadastre", "jobs"]; +const jobKey = (id: number) => ["admin", "cadastre", "job", id]; // --------------------------------------------------------------------------- // Component // --------------------------------------------------------------------------- export default function CadastreScrapeAdminPage() { - const [token, setToken] = useState(() => - typeof window === "undefined" - ? "" - : (localStorage.getItem("admin_token") ?? ""), - ); const [scope, setScope] = useState("pilot"); const [name, setName] = useState(""); const [quartersText, setQuartersText] = useState(""); @@ -152,35 +141,23 @@ export default function CadastreScrapeAdminPage() { const queryClient = useQueryClient(); - const saveToken = (v: string) => { - setToken(v); - if (typeof window !== "undefined") { - localStorage.setItem("admin_token", v); - } - }; - // ------------------------------------------------------------------------- // Queries // ------------------------------------------------------------------------- const jobs = useQuery({ - queryKey: jobsKey(token), + queryKey: jobsKey(), queryFn: () => - apiFetch("/api/v1/admin/cadastre/jobs?limit=30", { - headers: { "X-Admin-Token": token }, - }), - enabled: !!token, + apiFetch("/api/v1/admin/cadastre/jobs?limit=30"), refetchInterval: 5000, }); // Per-job detail (only fetched when row is expanded) const expandedJob = useQuery({ - queryKey: jobKey(token, expandedId ?? 0), + queryKey: jobKey(expandedId ?? 0), queryFn: () => - apiFetch(`/api/v1/admin/cadastre/jobs/${expandedId}`, { - headers: { "X-Admin-Token": token }, - }), - enabled: !!token && expandedId !== null, + apiFetch(`/api/v1/admin/cadastre/jobs/${expandedId}`), + enabled: expandedId !== null, refetchInterval: expandedId !== null ? 5000 : false, }); @@ -204,17 +181,13 @@ export default function CadastreScrapeAdminPage() { "/api/v1/admin/cadastre/jobs", { method: "POST", - headers: { - "X-Admin-Token": token, - "Content-Type": "application/json", - }, body: JSON.stringify(body), }, ); }, onSuccess: (data) => { setLastResp(data); - void queryClient.invalidateQueries({ queryKey: jobsKey(token) }); + void queryClient.invalidateQueries({ queryKey: jobsKey() }); }, }); @@ -222,11 +195,11 @@ export default function CadastreScrapeAdminPage() { mutationFn: (id: number) => apiFetch<{ cancelled: boolean }>( `/api/v1/admin/cadastre/jobs/${id}/cancel`, - { method: "POST", headers: { "X-Admin-Token": token } }, + { method: "POST" }, ), onSuccess: (_data, id) => { - void queryClient.invalidateQueries({ queryKey: jobKey(token, id) }); - void queryClient.invalidateQueries({ queryKey: jobsKey(token) }); + void queryClient.invalidateQueries({ queryKey: jobKey(id) }); + void queryClient.invalidateQueries({ queryKey: jobsKey() }); }, }); @@ -234,11 +207,11 @@ export default function CadastreScrapeAdminPage() { mutationFn: (id: number) => apiFetch<{ resumed: boolean }>( `/api/v1/admin/cadastre/jobs/${id}/resume`, - { method: "POST", headers: { "X-Admin-Token": token } }, + { method: "POST" }, ), onSuccess: (_data, id) => { - void queryClient.invalidateQueries({ queryKey: jobKey(token, id) }); - void queryClient.invalidateQueries({ queryKey: jobsKey(token) }); + void queryClient.invalidateQueries({ queryKey: jobKey(id) }); + void queryClient.invalidateQueries({ queryKey: jobsKey() }); }, }); @@ -248,7 +221,6 @@ export default function CadastreScrapeAdminPage() { const handleSubmit = (e: React.FormEvent) => { e.preventDefault(); - if (!token) return; if (scope === "manual_list" && !quartersText.trim()) { alert("Для Manual list укажите хотя бы один квартал"); return; @@ -662,22 +634,8 @@ export default function CadastreScrapeAdminPage() { незавершённых targets.

- {/* Admin Token */} -
- -
- {/* Create job form */} -
+

Запуск bulk harvest

@@ -760,7 +718,7 @@ export default function CadastreScrapeAdminPage() {
(() => - typeof window === "undefined" - ? "" - : (localStorage.getItem("admin_token") ?? ""), - ); const [overrideSqlitePath, setOverrideSqlitePath] = useState(""); const [lastTaskId, setLastTaskId] = useState(null); const [syncGroupsCsv, setSyncGroupsCsv] = useState(""); @@ -108,39 +103,26 @@ export default function ObjectiveEtlAdminPage() { const [showEtl, setShowEtl] = useState(false); const queryClient = useQueryClient(); - const saveToken = (v: string) => { - setToken(v); - localStorage.setItem("admin_token", v); - }; - const coverage = useQuery({ - queryKey: ["objective-coverage", token], + queryKey: ["objective-coverage"], queryFn: () => - apiFetch("/api/v1/admin/scrape/objective/coverage", { - headers: { "X-Admin-Token": token }, - }), - enabled: !!token, + apiFetch("/api/v1/admin/scrape/objective/coverage"), refetchInterval: 10000, }); const runs = useQuery({ - queryKey: ["objective-runs", token], + queryKey: ["objective-runs"], queryFn: () => apiFetch( "/api/v1/admin/scrape/objective/runs?limit=20", - { headers: { "X-Admin-Token": token } }, ), - enabled: !!token, refetchInterval: 5000, }); const cfg = useQuery({ - queryKey: ["objective-config", token], + queryKey: ["objective-config"], queryFn: () => - apiFetch("/api/v1/admin/scrape/objective/config", { - headers: { "X-Admin-Token": token }, - }), - enabled: !!token, + apiFetch("/api/v1/admin/scrape/objective/config"), refetchOnWindowFocus: false, }); @@ -155,10 +137,6 @@ export default function ObjectiveEtlAdminPage() { mutationFn: () => apiFetch("/api/v1/admin/scrape/objective", { method: "POST", - headers: { - "X-Admin-Token": token, - "Content-Type": "application/json", - }, body: JSON.stringify({ sqlite_path: overrideSqlitePath || null }), }), onSuccess: (data) => { @@ -178,10 +156,6 @@ export default function ObjectiveEtlAdminPage() { "/api/v1/admin/scrape/objective/sync-our", { method: "POST", - headers: { - "X-Admin-Token": token, - "Content-Type": "application/json", - }, body: JSON.stringify({ groups: groups.length ? groups : null }), }, ); @@ -196,10 +170,6 @@ export default function ObjectiveEtlAdminPage() { mutationFn: (patch: Partial) => apiFetch("/api/v1/admin/scrape/objective/config", { method: "PUT", - headers: { - "X-Admin-Token": token, - "Content-Type": "application/json", - }, body: JSON.stringify(patch), }), onSuccess: (data) => { @@ -223,22 +193,8 @@ export default function ObjectiveEtlAdminPage() { только как bootstrap-fallback (свёрнут ниже).

- {/* Token */} -
- -
- {/* Coverage */} -
+

📦 Состояние PG (источник истины)

{coverage.isLoading && (

Загрузка…

@@ -504,7 +460,7 @@ export default function ObjectiveEtlAdminPage() {
(() => - typeof window === "undefined" - ? "" - : (localStorage.getItem("admin_token") ?? ""), - ); const [region, setRegion] = useState(66); const [devs, setDevs] = useState(""); const [fetchFlats, setFetchFlats] = useState(true); @@ -84,12 +79,8 @@ export default function ScrapeAdminPage() { const queryClient = useQueryClient(); const runs = useQuery({ - queryKey: ["scrape-runs", token], - queryFn: () => - apiFetch("/api/v1/admin/scrape/runs?limit=20", { - headers: { "X-Admin-Token": token }, - }), - enabled: !!token, + queryKey: ["scrape-runs"], + queryFn: () => apiFetch("/api/v1/admin/scrape/runs?limit=20"), refetchInterval: 5000, }); @@ -97,7 +88,6 @@ export default function ScrapeAdminPage() { mutationFn: () => apiFetch("/api/v1/admin/scrape/kn", { method: "POST", - headers: { "X-Admin-Token": token }, body: JSON.stringify({ region_code: region, developers: devs @@ -129,7 +119,6 @@ export default function ScrapeAdminPage() { released: boolean; }>("/api/v1/admin/scrape/release-lock", { method: "POST", - headers: { "X-Admin-Token": token }, body: JSON.stringify({ region_code: region, developers: devs @@ -151,12 +140,8 @@ export default function ScrapeAdminPage() { }); const queue = useQuery({ - queryKey: ["scrape-queue", token], - queryFn: () => - apiFetch("/api/v1/admin/scrape/queue", { - headers: { "X-Admin-Token": token }, - }), - enabled: !!token, + queryKey: ["scrape-queue"], + queryFn: () => apiFetch("/api/v1/admin/scrape/queue"), refetchInterval: 3000, }); @@ -172,7 +157,6 @@ export default function ScrapeAdminPage() { "/api/v1/admin/scrape/revoke", { method: "POST", - headers: { "X-Admin-Token": token }, body: JSON.stringify({ task_id, terminate }), }, ), @@ -203,30 +187,25 @@ export default function ScrapeAdminPage() { }; const logs = useQuery({ - queryKey: ["scrape-logs", logsRunId, token], + queryKey: ["scrape-logs", logsRunId], queryFn: () => { const q = logsRunId !== "" ? `?run_id=${logsRunId}&limit=200` : "?limit=200"; - return apiFetch(`/api/v1/admin/scrape/logs${q}`, { - headers: { "X-Admin-Token": token }, - }); + return apiFetch(`/api/v1/admin/scrape/logs${q}`); }, - enabled: !!token, refetchInterval: 3000, }); const failures = useQuery({ - queryKey: ["scrape-failures", failuresRunId, token], + queryKey: ["scrape-failures", failuresRunId], queryFn: () => { const q = failuresRunId !== "" ? `?run_id=${failuresRunId}&limit=100` : "?limit=100"; - return apiFetch(`/api/v1/admin/scrape/failures${q}`, { - headers: { "X-Admin-Token": token }, - }); + return apiFetch(`/api/v1/admin/scrape/failures${q}`); }, - enabled: !!token && showFailures, + enabled: showFailures, refetchInterval: showFailures ? 5000 : false, }); @@ -236,11 +215,6 @@ export default function ScrapeAdminPage() { } }; - const saveToken = (v: string) => { - setToken(v); - localStorage.setItem("admin_token", v); - }; - return ( <>

@@ -252,16 +226,6 @@ export default function ScrapeAdminPage() {

-
{releaseLock.isSuccess ? (
- {!token ? ( -

- Введите Admin Token чтобы видеть состояние очереди. -

- ) : queue.isLoading ? ( + {queue.isLoading ? (

Загрузка…

) : queue.isError ? (

Ошибка: {String(queue.error)}

@@ -718,11 +664,7 @@ export default function ScrapeAdminPage() { style={{ ...inputStyle, width: 130 }} /> - {!token ? ( -

- Введите Admin Token чтобы видеть лог. -

- ) : logs.isLoading ? ( + {logs.isLoading ? (

Загрузка…

) : logs.isError ? (

Ошибка: {String(logs.error)}

@@ -796,7 +738,6 @@ export default function ScrapeAdminPage() { />

{d?.addr ?? ""} · obj_id={d?.obj_id} · {d?.dev_name ?? ""} - {d?.escrow ? " · ✅ эскроу" : ""} + {d?.escrow ? " · эскроу" : ""}

+ {/* ── KPI row — распроданность ────────────────────────────────────────── */}
- + + {d?.price_min_rub != null ? ( + + ) : null} + {d?.price_max_rub != null ? ( + + ) : null}
+ {/* ── DOM.РФ scores ───────────────────────────────────────────────────── */} + {(d?.domrf_score_location != null || + d?.domrf_score_transport != null || + d?.domrf_score_infrastructure != null) && ( +
+ {d?.domrf_score_location != null && ( + + )} + {d?.domrf_score_transport != null && ( + + )} + {d?.domrf_score_infrastructure != null && ( + + )} +
+ )} + + {/* ── 1. Динамика продаж ──────────────────────────────────────────────── */}
+ {/* ── 2. Квартирография ──────────────────────────────────────────────── */} +
+ {quartirography.isLoading ? ( +
+ ) : ( + + )} +
+ + {/* ── 3. Ближайшее метро ──────────────────────────────────────────────── */} + {(d?.metro_nearest_name != null || + (d?.metro_top3 != null && d.metro_top3.length > 0)) && ( +
+ +
+ )} + + {/* ── 4. Инфраструктура (POI) ─────────────────────────────────────────── */}
+ {/* ── 5. Фото прогресса строительства ─────────────────────────────────── */}
- {/* Корпуса ЖК — показываем только если есть данные */} + {/* ── 6. Проверено на наш.дом.рф ─────────────────────────────────────── */} + {!checks.isLoading && (checks.data?.length ?? 0) > 0 && ( +
+ +
+ )} + + {/* ── 7. Документы ────────────────────────────────────────────────────── */} + {!documents.isLoading && (documents.data?.length ?? 0) > 0 && ( +
+ +
+ )} + + {/* ── 8. Все характеристики ───────────────────────────────────────────── */} + {d && ( +
+ +
+ )} + + {/* ── 9. Корпуса ЖК ───────────────────────────────────────────────────── */} {(d?.buildings_count ?? 0) > 0 && buildings.data && buildings.data.length > 0 && ( @@ -290,7 +411,6 @@ export default function ObjectDrillInPage() {
)} - {/* Placeholder when buildings_count > 0 but data not yet loaded */} {(d?.buildings_count ?? 0) > 0 && buildings.data && buildings.data.length === 0 && ( diff --git a/frontend/src/app/globals.css b/frontend/src/app/globals.css new file mode 100644 index 00000000..13046cd8 --- /dev/null +++ b/frontend/src/app/globals.css @@ -0,0 +1,90 @@ +/* + * GenDesign — global CSS design tokens (UI Brief May 17) + * All CSS vars used across site-finder and analytics components. + * Spacing scale: 4/8/12/16/24/32px only. + * Tabular-nums applied globally for numeric readability. + */ + +*, +*::before, +*::after { + box-sizing: border-box; +} + +:root { + /* ── Surface ─────────────────────────────────────────────────────────── */ + --bg-app: #f6f7f9; + --bg-card: #ffffff; + --bg-card-alt: #fafbfc; + --bg-headline: #0f172a; /* slate-900 — dark verdict bar */ + + /* ── Borders ─────────────────────────────────────────────────────────── */ + --border-soft: #eef0f3; + --border-card: #e6e8ec; + --border-strong: #d1d5db; + + /* ── Text ────────────────────────────────────────────────────────────── */ + --fg-primary: #111111; + --fg-secondary: #5b6066; + --fg-tertiary: #73767e; + --fg-on-dark: #e2e8f0; /* text on --bg-headline */ + --fg-on-dark-muted: #94a3b8; /* caveat text on dark */ + + /* ── Brand / CTA ─────────────────────────────────────────────────────── */ + --accent: #1d4ed8; /* primary blue CTA */ + --accent-hover: #1e40af; + --accent-soft: #dbeafe; /* badge bg, active tab bg */ + --accent-2: #f2994a; /* secondary orange CTA (Export, Share) */ + + /* ── Semantic ────────────────────────────────────────────────────────── */ + --success: #0a7a3a; + --success-soft: #dcfce7; + --warn: #9a6700; + --warn-soft: #fef3c7; + --danger: #b3261e; + --danger-soft: #fee2e2; + + /* ── Data-viz (use in order, no skip) ───────────────────────────────── */ + --viz-1: #1d4ed8; + --viz-2: #0ea5e9; + --viz-3: #14b8a6; + --viz-4: #f59e0b; + --viz-5: #8b5cf6; + --prediction-line: #0ea5e9; + --prediction-band: rgba(14, 165, 233, 0.18); +} + +html, +body { + margin: 0; + padding: 0; + background: var(--bg-app); + color: var(--fg-primary); + font-family: + Inter, + -apple-system, + "Segoe UI", + system-ui, + sans-serif; + font-size: 14px; + line-height: 1.5; + /* Tabular nums globally for numeric legibility (Bloomberg lesson) */ + font-variant-numeric: tabular-nums; + font-feature-settings: "tnum"; + -webkit-font-smoothing: antialiased; +} + +/* Focus-visible ring — accessible, not outline:none */ +:focus-visible { + outline: 2px solid var(--accent); + outline-offset: 2px; +} + +/* Remove default outline for mouse users */ +:focus:not(:focus-visible) { + outline: none; +} + +a { + color: inherit; +} diff --git a/frontend/src/app/layout.tsx b/frontend/src/app/layout.tsx index 7dc4da1a..17323396 100644 --- a/frontend/src/app/layout.tsx +++ b/frontend/src/app/layout.tsx @@ -1,5 +1,6 @@ import type { Metadata } from "next"; +import "./globals.css"; import { Providers } from "./providers"; export const metadata: Metadata = { @@ -14,13 +15,7 @@ export default function RootLayout({ }) { return ( - + {children} diff --git a/frontend/src/app/legacy/site-finder/page.tsx b/frontend/src/app/legacy/site-finder/page.tsx new file mode 100644 index 00000000..568b146f --- /dev/null +++ b/frontend/src/app/legacy/site-finder/page.tsx @@ -0,0 +1,570 @@ +"use client"; + +import { Suspense, useEffect, useRef, useState } from "react"; +import dynamic from "next/dynamic"; +import Link from "next/link"; +import { usePathname, useRouter, useSearchParams } from "next/navigation"; +import type { FeatureCollection } from "geojson"; + +import { CadInput } from "@/components/site-finder/CadInput"; +import { FetchingState } from "@/components/site-finder/FetchingState"; +import { KpiCard } from "@/components/site-finder/KpiCard"; +import { OverviewTab } from "@/components/site-finder/OverviewTab"; +import { EnvironmentTab } from "@/components/site-finder/EnvironmentTab"; +import { LandTab } from "@/components/site-finder/LandTab"; +import { MarketTab } from "@/components/site-finder/MarketTab"; +import { WeightProfilePanel } from "@/components/site-finder/WeightProfilePanel"; +import { useSiteAnalysis } from "@/hooks/useSiteAnalysis"; +import { useDebouncedValue } from "@/hooks/useDebouncedValue"; +import { useConnectionPoints } from "@/hooks/useConnectionPoints"; +import { useCustomPois } from "@/hooks/useCustomPois"; +import { + POI_DEFAULT_WEIGHTS, + type PoiCategoryKey, +} from "@/lib/api/weightProfiles"; + +// SiteMap imports Leaflet which requires browser APIs — load without SSR +const SiteMap = dynamic( + () => import("@/components/site-finder/SiteMap").then((m) => m.SiteMap), + { + ssr: false, + loading: () => ( +
+ Загрузка карты… +
+ ), + }, +); + +type TabId = "overview" | "env" | "land" | "market"; + +const TABS: Array<{ id: TabId; label: string }> = [ + { id: "overview", label: "Обзор" }, + { id: "env", label: "Окружение" }, + { id: "land", label: "Земля" }, + { id: "market", label: "Рынок" }, +]; + +const TAB_IDS = TABS.map((t) => t.id); + +function isTabId(v: string | null | undefined): v is TabId { + return !!v && (TAB_IDS as readonly string[]).includes(v); +} + +// ── KPI helpers ─────────────────────────────────────────────────────────────── + +type KpiColor = "neutral" | "amber" | "green" | "red" | "blue"; + +function trendColor(pct: number): KpiColor { + if (pct > 5) return "green"; + if (pct > 0) return "blue"; + if (pct > -5) return "amber"; + return "red"; +} + +function noiseColor(db: number): KpiColor { + if (db < 45) return "green"; + if (db < 60) return "amber"; + return "red"; +} + +function scoreKpiColor(score: number, max: number): KpiColor { + const ratio = score / max; + if (ratio >= 0.65) return "green"; + if (ratio >= 0.35) return "amber"; + return "red"; +} + +// ── Page ────────────────────────────────────────────────────────────────────── + +function SiteFinderContent() { + const router = useRouter(); + const pathname = usePathname(); + const searchParams = useSearchParams(); + + const { mutate, data, isPending, error, isIdle, fetchingState, cancel } = + useSiteAnalysis(); + const [isochrones, setIsochrones] = useState( + undefined, + ); + + // Fetch connection points whenever a parcel is loaded + const { data: connectionPoints } = useConnectionPoints(data?.cad_num); + // Fetch custom POIs for current parcel + const { data: customPois } = useCustomPois(data?.cad_num); + + // Weight profile state — lifted here so it survives tab switches. + // userId + adminToken allow the panel to load/save named profiles. + const [currentWeights, setCurrentWeights] = useState< + Record + >(() => ({ ...POI_DEFAULT_WEIGHTS })); + // Profile selection for the analyze call (null = system defaults). + const [activeProfileId, setActiveProfileId] = useState(null); + + // Pending weights change — set immediately when user clicks "Применить". + // Debounced to 300ms before triggering re-analyze, so rapid clicks collapse + // into a single request (#201 Phase 2). + const [pendingWeightsChange, setPendingWeightsChange] = useState<{ + weights: Record; + profileId: number | null; + } | null>(null); + const debouncedWeightsChange = useDebouncedValue(pendingWeightsChange, 300); + // Ref to skip the initial mount effect (we only re-analyze on actual changes). + const weightsChangeInitializedRef = useRef(false); + + const [profileUserId, setProfileUserId] = useState(() => + typeof window === "undefined" + ? "" + : (localStorage.getItem("admin_user_id") ?? ""), + ); + const [adminToken] = useState(() => + typeof window === "undefined" + ? "" + : (localStorage.getItem("admin_token") ?? ""), + ); + // Lazy init: считаем initialTab один раз на mount (useState всё равно + // игнорирует initializer после первого render — не тратим CPU). + const [tab, setTabState] = useState(() => { + const t = searchParams.get("tab"); + return isTabId(t) ? t : "overview"; + }); + + // Sync tab → URL query (?tab=env). overview = default, не пишем в URL. + function setTab(next: TabId) { + setTabState(next); + const params = new URLSearchParams(searchParams.toString()); + if (next === "overview") { + params.delete("tab"); + } else { + params.set("tab", next); + } + const qs = params.toString(); + // Используем pathname вместо голого "?" — Next 15 App Router не всегда + // чисто чистит query при `router.replace("?")` (dangling `?` остаётся). + router.replace(qs ? `${pathname}?${qs}` : pathname, { scroll: false }); + } + + // Listen to back/forward navigation — reflect URL tab in state. + // setState с тем же значением React сам бэйлит через Object.is, поэтому + // guard по tab не нужен (и не вводит stale-closure через eslint-disable). + useEffect(() => { + const urlTab = searchParams.get("tab"); + setTabState(isTabId(urlTab) ? urlTab : "overview"); + }, [searchParams]); + + // Debounced re-analyze trigger: fires 300ms after the last "Применить" click. + // Skip initial mount (weightsChangeInitializedRef guards the first effect run). + // dataRef lets the effect read current data without listing it as a dep + // (avoids re-registering the timeout on every analyze completion). + const dataRef = useRef(data); + dataRef.current = data; + const profileUserIdRef = useRef(profileUserId); + profileUserIdRef.current = profileUserId; + + useEffect(() => { + if (!weightsChangeInitializedRef.current) { + weightsChangeInitializedRef.current = true; + return; + } + if (!debouncedWeightsChange) return; + const currentData = dataRef.current; + if (!currentData?.cad_num) return; + + const { weights, profileId } = debouncedWeightsChange; + const currentProfileUserId = profileUserIdRef.current; + setIsochrones(undefined); + mutate({ + cad: currentData.cad_num, + options: + profileId != null + ? { profileId } + : currentProfileUserId + ? { profileUserId: currentProfileUserId, weights } + : { weights }, + }); + // mutate is stable from useMutation — safe to omit from deps. + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [debouncedWeightsChange]); + + function handleAnalyze(cadNum: string) { + setIsochrones(undefined); + setTab("overview"); + // Priority: named profile → user default profile → inline draft weights. + // When activeProfileId is set, backend uses that profile (ignores inline). + // When no profile is selected, pass currentWeights as inline so draft + // slider values are always respected even without a saved profile (#201). + mutate({ + cad: cadNum, + options: + activeProfileId != null + ? { profileId: activeProfileId } + : profileUserId + ? { profileUserId, weights: currentWeights } + : { weights: currentWeights }, + }); + } + + function handleWeightsChange( + weights: Record, + profileId: number | null, + ) { + setCurrentWeights(weights); + setActiveProfileId(profileId); + // Enqueue a debounced re-analyze if a parcel is loaded (#201 Phase 2). + // The actual mutate fires after 300ms of silence via debouncedWeightsChange. + if (data?.cad_num) { + setPendingWeightsChange({ weights, profileId }); + } + } + + // Derive KPI values from data + const scoreMax = data?.score_max_reference ?? 40; + const scoreValue = data + ? `${data.score.toFixed(2)} / ${scoreMax.toFixed(0)}` + : "—"; + const scoreLabel = data?.score_label + ? `Score · ${data.score_label}` + : "Score"; + const scoreCardColor: KpiColor = data + ? scoreKpiColor(data.score, scoreMax) + : "neutral"; + + const districtValue = data?.district ? data.district.district_name : "—"; + const districtLabel = data?.district + ? `${(data.district.median_price_per_m2 / 1000).toFixed(0)} тыс ₽/м²` + + ` · ${(data.district.dist_to_center / 1000).toFixed(1)} км от центра` + : "Район"; + + const trendPct = data?.market_trend?.delta_6m_pct; + const trendValue = + trendPct != null + ? `${trendPct > 0 ? "+" : ""}${trendPct.toFixed(1)}%` + : "—"; + const trendLabel = data?.market_trend + ? `Тренд за 6 мес · ${data.market_trend.label}` + : "Тренд рынка"; + const trendKpiColor: KpiColor = + trendPct != null ? trendColor(trendPct) : "neutral"; + + const noiseDb = data?.noise?.estimated_db; + const noiseValue = noiseDb != null ? `${Math.round(noiseDb)} dB` : "—"; + const noiseLabel = data?.noise ? `Шум · ${data.noise.level}` : "Шум"; + const noiseKpiColor: KpiColor = + noiseDb != null ? noiseColor(noiseDb) : "neutral"; + + return ( +
+ {/* Breadcrumb + header */} +
+ + ← Главная + +
+ +
+
+

+ Site Finder +

+

+ Введите кадастровый номер участка или квартала для анализа локации +

+
+
+ +
+
+ + {/* Weight profile panel — collapsible, below header */} +
+ {/* Optional user-id field for profile CRUD (shown only when adminToken present) */} + {!!adminToken && ( +
+ + { + setProfileUserId(e.target.value); + if (typeof window !== "undefined") { + localStorage.setItem("admin_user_id", e.target.value); + } + }} + /> +
+ )} + + {/* Recalculation indicator — shown while re-analyze is in-flight after + weights change (data already loaded, pendingWeightsChange set). */} + {isPending && !!data && ( +

+ Пересчёт… +

+ )} +
+ + {/* Error state */} + {error && ( +
+ {error instanceof Error ? error.message : "Ошибка анализа участка"} +
+ )} + + {/* #93 — Fetching state: backend на 202 Accepted, идёт on-demand NSPD fetch */} + {fetchingState && ( + + )} + + {/* Pending skeleton — первичный analyze в полёте (но fetching state ещё не активирован) */} + {isPending && !fetchingState && ( +
+ Анализируем участок… +
+ )} + + {/* Empty initial state */} + {isIdle && !error && ( +
+ Введите кадастровый номер и нажмите «Анализировать» +
+ )} + + {/* Results */} + {data && ( + <> + {/* Cad label */} +
+ {data.cad_num} +
+ + {/* Hero band — sticky KPI cards */} +
+ + + + +
+ + {/* Tab buttons */} +
+ {TABS.map((t) => ( + + ))} +
+ + {/* 2-column grid: tab content + sticky map */} +
+ {/* Tab content */} +
+ {tab === "overview" && ( + + )} + {tab === "env" && } + {tab === "land" && } + {tab === "market" && } +
+ + {/* Sticky right sidebar — map always visible */} + +
+ + )} +
+ ); +} + +// useSearchParams() в Next 15 App Router throw'ит при static rendering без +// boundary — поэтому оборачиваем page-level. +export default function SiteFinderPage() { + return ( + + + + ); +} diff --git a/frontend/src/app/page.tsx b/frontend/src/app/page.tsx index 6e25ccae..2d3be18d 100644 --- a/frontend/src/app/page.tsx +++ b/frontend/src/app/page.tsx @@ -1,188 +1,633 @@ +"use client"; + +import { useState } from "react"; + import Link from "next/link"; -interface Route { - href: string; - label: string; - desc: string; -} +import { + BarChart3, + Building2, + ChevronDown, + ChevronUp, + Database, + MapPin, + RefreshCw, +} from "lucide-react"; -const SECTIONS: { title: string; routes: Route[] }[] = [ +import { PilotRequestModal } from "@/components/landing/PilotRequestModal"; +import { HeadlineBar } from "@/components/ui/HeadlineBar"; +import { useLandingStats } from "@/lib/api/landing"; + +// ── FAQ data ─────────────────────────────────────────────────────────────── + +const FAQ: { q: string; a: string }[] = [ { - title: "Аналитика", - routes: [ - { - href: "/analytics", - label: "Свердл рынок", - desc: "Динамика 14 мес · парадокс «строят vs покупают» · pipeline · районы ЕКБ.", - }, - { - href: "/analytics/prinzip", - label: "PRINZIP drill-down", - desc: "Velocity vs benchmark · gap-bar к рынку · ЖК с sparkline · инсайты.", - }, - { - href: "/analytics/developers", - label: "Топ девелоперов", - desc: "Sortable leaderboard · velocity scatter · сравнение sold% по месяцам.", - }, - { - href: "/analytics/objects/60160", - label: "Карточка ЖК (пример: Парк Победы)", - desc: "KPI · sale-chart · корпуса ЖК (cad_quarter + buildings) · POI-таблица · фото-галерея. /analytics/objects/{obj_id}.", - }, - { - href: "/analytics/recommend", - label: "Рекомендатор квартирографии", - desc: "Район + площадь + класс → распределение по 5 бакетам, цены p25/median/p75, выручка, comparable ЖК. Слайдеры под проект.", - }, - ], + q: "Откуда берутся данные?", + a: "Три источника: Росреестр (ДДУ-сделки, >7 млн записей), DOM.РФ (портфель новостроек, ПЗЗ), Objective (цены предложения). Обновление еженедельное для DOM.РФ и цен, ежеквартальное для Росреестра.", }, { - title: "Site Finder", - routes: [ - { - href: "/site-finder", - label: "Анализ участка по cad-номеру", - desc: "Вводишь кадастр → карта + score соц-инфраструктуры (школы/сады/магазины/парки +вес, трамваи −вес) + 20 ближайших конкурентов с подсветкой района. Данные: OSM POI обновляются еженедельно.", - }, - ], + q: "Насколько точны цены?", + a: "Цены предложения актуальны на дату последнего обновления. Цена сделки из ДДУ — реальная стоимость на момент регистрации в Росреестре. Расхождение с рыночной ценой в пике может составлять 5–15%.", }, { - title: "Админка (Token-protected)", - routes: [ - { - href: "/admin/scrape/all", - label: "Скрапер — единая панель", - desc: "Все джобы (kn/nspd_geo/objective) в одном месте · job_settings inline-edit · bulk Свердл geo backfill · ScrapeLogsPanel.", - }, - { - href: "/admin/leads", - label: "Лиды PRINZIP CRM", - desc: "Воронка Sankey · конверсия по месяцам · KPI · таблица лидов с фильтрами.", - }, - ], + q: "Какие районы и регионы охвачены?", + a: "Сейчас — Екатеринбург и Свердловская область (ПЗЗ ЕКБ, МСК-66). Расширение на другие регионы УрФО в роудмапе.", }, { - title: "Generative (WIP)", - routes: [ - { - href: "/concept", - label: "Concept (Stage 1)", - desc: "Импорт полигона → 3 варианта застройки. WIP — Stage 1a-c.", - }, - ], + q: "Что входит в пилотный доступ?", + a: "Полный доступ к Site Finder (анализ участков по кадастровому номеру), аналитике рынка Свердловской области и рекомендатору квартирографии. Срок пилота — 30 дней. Поддержка по email.", + }, + { + q: "Как устроен маппинг Objective → DOM.РФ?", + a: "Объекты Objective сопоставляются с портфелем DOM.РФ по нечёткому совпадению названия и геокоординатам. Текущее покрытие — около 11% (один DOM.РФ-объект может объединять несколько Objective-лотов).", }, ]; +// ── Sub-components ───────────────────────────────────────────────────────── + +function FaqItem({ q, a }: { q: string; a: string }) { + const [open, setOpen] = useState(false); + return ( +
+ + {open ? ( +

+ {a} +

+ ) : null} +
+ ); +} + +// ── KPI formatting ───────────────────────────────────────────────────────── + +function formatNumber(n: number | undefined, decimals = 0): string { + if (n === undefined) return "—"; + return n.toLocaleString("ru", { + minimumFractionDigits: decimals, + maximumFractionDigits: decimals, + }); +} + +/** Skeleton placeholder shown when stale=true (no fake data) */ +function KpiSkeleton() { + return ( +
+ ); +} + +// ── Main component ───────────────────────────────────────────────────────── + export default function HomePage() { + const [modalOpen, setModalOpen] = useState(false); + const { data: stats, isLoading } = useLandingStats(); + + const isStale = stats?.stale === true; + const showSkeleton = isLoading || isStale; + return (
-
-

GenDesign

-

- Generative Design + Site Finder — Discovery MVP. Свердл рынок & - PRINZIP. -

-
- - {SECTIONS.map((section) => ( -
-

+ + GenDesign + +
+ - {section.title} -

+ Аналитика + + + Site Finder + + + Trade-In + + + Концепция + + + Админ + + +
+ + + {/* ── Hero ───────────────────────────────────────────────────────── */} +
+

+ Цифры рынка перед фундаментом +

+

+ Аналитика новостроек Свердловской области — Росреестр, DOM.РФ, + Objective — в одном инструменте для девелоперов. +

+ + {/* ── KPI cards ──────────────────────────────────────────────── */} +
+ {/* zk_total */} +
+
+ +
+
ЖК DOM.РФ в ЕКБ
+
+ {showSkeleton ? : formatNumber(stats?.zk_total)} +
+
+ + {/* deals_total */} +
+
+ +
+
ДДУ-сделки Росреестра
+
+ {showSkeleton ? ( + + ) : ( + formatNumber(stats?.deals_total) + )} +
+
+ + {/* price_coverage_pct */} +
+
+ +
+
Покрытие ценами Objective
+
+ {showSkeleton ? ( + + ) : ( + `${formatNumber(stats?.price_coverage_pct, 1)}%` + )} +
+
+ + {/* mapping_coverage_pct */} +
+
+ +
+
Маппинг Objective → DOM.РФ
+
+ {showSkeleton ? ( + + ) : ( + `${formatNumber(stats?.mapping_coverage_pct, 1)}%` + )} +
+
+ + {/* last_data_update */} +
+
+ +
+
Последнее обновление данных
+
+ {showSkeleton ? ( + + ) : ( + (stats?.last_data_update ?? "—") + )} +
+
+
+ + {isStale && ( +

+ Данные обновляются... +

+ )} +
+ + {/* ── Парадокс портфеля ──────────────────────────────────────────── */} +
+ + +

+ DOM.РФ фиксирует строящиеся ЖК, но цены предложения Objective + покрывают лишь часть из них. Из тех, где цены есть, публично доступны + единицы — остальное требует прямого запроса или данных, агрегированных + GenDesign. +

+
+ + {/* ── Pricing ────────────────────────────────────────────────────── */} +
+
- {section.routes.map((r) => ( - -
- {r.href} -
-
{r.label}
-
- {r.desc} -
- - ))} + Пилотная подписка +
+
+ + 30 000 + + + ₽ / мес +
-
- ))} +
    + {[ + "Site Finder — анализ участков по кадастру", + "Аналитика рынка Свердловской области", + "Рекомендатор квартирографии", + "Данные DOM.РФ, Росреестр, Objective", + "Email-поддержка в рабочие дни", + ].map((item) => ( +
  • + + + {item} + +
    • + ))} +
+ + + +
+ + {/* ── FAQ ────────────────────────────────────────────────────────── */} +
+

+ Частые вопросы +

+

+ Об источниках данных, точности и поддержке. +

+
+ {FAQ.map((item) => ( + + ))} +
+
+ + {/* ── Footer ─────────────────────────────────────────────────────── */} + + {/* ── Modal ──────────────────────────────────────────────────────── */} + setModalOpen(false)} /> ); } + +// ── Shared styles ────────────────────────────────────────────────────────── + +const kpiCardStyle: React.CSSProperties = { + background: "var(--bg-card)", + border: "1px solid var(--border-card)", + borderRadius: 12, + padding: "16px 18px", +}; + +const kpiIconWrap: React.CSSProperties = { + width: 28, + height: 28, + background: "var(--accent-soft)", + borderRadius: 6, + display: "flex", + alignItems: "center", + justifyContent: "center", + marginBottom: 8, +}; + +const kpiLabel: React.CSSProperties = { + fontSize: 12, + textTransform: "uppercase", + letterSpacing: "0.04em", + color: "var(--fg-secondary)", + fontWeight: 500, + marginBottom: 6, +}; + +const kpiValue: React.CSSProperties = { + fontSize: 28, + fontWeight: 600, + color: "var(--fg-primary)", + fontVariantNumeric: "tabular-nums", +}; diff --git a/frontend/src/app/site-finder/analysis/[cad]/AnalysisPageContent.tsx b/frontend/src/app/site-finder/analysis/[cad]/AnalysisPageContent.tsx new file mode 100644 index 00000000..03b193f3 --- /dev/null +++ b/frontend/src/app/site-finder/analysis/[cad]/AnalysisPageContent.tsx @@ -0,0 +1,345 @@ +"use client"; + +import Link from "next/link"; + +import { Section1ParcelInfo } from "@/components/site-finder/analysis/Section1ParcelInfo"; +import { Section2NetworksUtilities } from "@/components/site-finder/analysis/Section2NetworksUtilities"; +import { Section3SettingsAndCompetitors } from "@/components/site-finder/analysis/Section3SettingsAndCompetitors"; +import { Section4Estimate } from "@/components/site-finder/analysis/Section4Estimate"; +import { Section5Atmosphere } from "@/components/site-finder/analysis/Section5Atmosphere"; +import { useParcelAnalyzeQuery } from "@/lib/site-finder-api"; +import type { ParcelAnalysis } from "@/types/site-finder"; + +// ── Props ───────────────────────────────────────────────────────────────────── + +interface Props { + cad: string; +} + +// ── Page Content (client — needs TanStack Query) ─────────────────────────────── + +export function AnalysisPageContent({ cad }: Props) { + const { data, isLoading, error } = useParcelAnalyzeQuery(cad); + + // ── Loading ──────────────────────────────────────────────────────────────── + + if (isLoading) { + return ( +
+ +
+ Анализируем участок {cad}… +
+
+ ); + } + + // ── Error ────────────────────────────────────────────────────────────────── + + if (error || !data) { + const msg = + error instanceof Error ? error.message : "Ошибка загрузки анализа"; + return ( +
+ +
+ {msg} +
+
+ + Вернуться к Site Finder + +
+
+ ); + } + + // ── Results layout ───────────────────────────────────────────────────────── + + // Cast to ParcelAnalysis (superset of ParcelAnalyzeResponse) — both describe + // the same /analyze endpoint; Section3 requires the richer type. + const analysis = data as unknown as ParcelAnalysis; + + const districtName = analysis.district?.district_name ?? "—"; + const areaHa = analysis.geometry_suitability?.area_ha; + const areaStr = areaHa != null ? `${areaHa.toFixed(2)} га` : null; + + return ( +
+ {/* Breadcrumb */} + + + {/* Page layout: sidebar (240px) + main scroll */} +
+ {/* ── Sidebar nav ───────────────────────────────────────────────── */} + + + {/* ── Main scroll area ──────────────────────────────────────────── */} +
+ {/* Section 1 — IMPLEMENTED in A5 */} + + + {/* Section 2 — IMPLEMENTED in A6 */} + + + {/* Section 3 — IMPLEMENTED in A7 */} + + + {/* Section 4 — IMPLEMENTED in A10 */} + + + {/* Section 5 — IMPLEMENTED in A11 */} + +
+
+
+ ); +} + +// ── Breadcrumb ───────────────────────────────────────────────────────────────── + +function Breadcrumb({ + cad, + districtName, + areaStr, +}: { + cad: string; + districtName?: string; + areaStr?: string | null; +}) { + const parts = [cad]; + if (districtName && districtName !== "—") parts.push(districtName); + if (areaStr) parts.push(areaStr); + + return ( +
+ + Site Finder + + / + {parts.map((part, i) => ( + + {i > 0 && ( + + · + + )} + {part} + + ))} + + · + + анализ +
+ ); +} + +// ── Sidebar nav ──────────────────────────────────────────────────────────────── + +const NAV_ITEMS = [ + { id: "section-1", label: "1. Информация" }, + { id: "section-2", label: "2. Сети" }, + { + id: "section-3", + label: "3. Продажи конкурентов", + children: [ + { id: "section-3-1", label: "3.1 Настройки выборки" }, + { id: "section-3-2", label: "3.2 Планировки" }, + { id: "section-3-3", label: "3.3 Остатки и скорость" }, + ], + }, + { id: "section-4", label: "4. Оценка" }, + { id: "section-5", label: "5. Атмосфера" }, +]; + +function AnalysisSidebarNav() { + function scrollTo(id: string) { + const el = document.getElementById(id); + if (el) el.scrollIntoView({ behavior: "smooth" }); + } + + return ( + + ); +} + +// ── Section placeholder ──────────────────────────────────────────────────────── + +function SectionPlaceholder({ + number, + title, + note, +}: { + number: string; + title: string; + note: string; +}) { + return ( + <> +
+
+ {number}. {title} +
+
+
+ {note} +
+ + ); +} diff --git a/frontend/src/app/site-finder/analysis/[cad]/page.tsx b/frontend/src/app/site-finder/analysis/[cad]/page.tsx new file mode 100644 index 00000000..6a82c8da --- /dev/null +++ b/frontend/src/app/site-finder/analysis/[cad]/page.tsx @@ -0,0 +1,53 @@ +import { Suspense } from "react"; +import type { Metadata } from "next"; + +import { AnalysisPageContent } from "./AnalysisPageContent"; + +// ── Metadata ────────────────────────────────────────────────────────────────── + +interface PageProps { + params: Promise<{ cad: string }>; +} + +// Next.js delivers `cad` URL-encoded (":" -> "%3A"); decode once at the page +// boundary so downstream hooks/components see canonical "66:41:0204016:10" +// and re-encode exactly once when building API URLs / hrefs (prevents the +// double-encode that made backend regex reject the cad with HTTP 400). +// Regression history: PR #351 added this in Client Component; A7 split into +// Server Component (PR #356) dropped it; this restores in Server boundary. +export async function generateMetadata({ + params, +}: PageProps): Promise { + const { cad: cadRaw } = await params; + const cad = decodeURIComponent(cadRaw); + return { + title: `Анализ ${cad} — Site Finder`, + description: `Анализ инвестиционного участка ${cad}`, + }; +} + +// ── Page ────────────────────────────────────────────────────────────────────── + +export default async function AnalysisPage({ params }: PageProps) { + const { cad: cadRaw } = await params; + const cad = decodeURIComponent(cadRaw); + + return ( + + Загрузка анализа… + + } + > + + + ); +} diff --git a/frontend/src/app/site-finder/page.tsx b/frontend/src/app/site-finder/page.tsx index faeddd61..20a08064 100644 --- a/frontend/src/app/site-finder/page.tsx +++ b/frontend/src/app/site-finder/page.tsx @@ -1,491 +1,196 @@ "use client"; -import { Suspense, useEffect, useState } from "react"; import dynamic from "next/dynamic"; import Link from "next/link"; -import { usePathname, useRouter, useSearchParams } from "next/navigation"; -import type { FeatureCollection } from "geojson"; +import { useRouter } from "next/navigation"; +import { useState } from "react"; import { CadInput } from "@/components/site-finder/CadInput"; -import { FetchingState } from "@/components/site-finder/FetchingState"; -import { KpiCard } from "@/components/site-finder/KpiCard"; -import { OverviewTab } from "@/components/site-finder/OverviewTab"; -import { EnvironmentTab } from "@/components/site-finder/EnvironmentTab"; -import { LandTab } from "@/components/site-finder/LandTab"; -import { MarketTab } from "@/components/site-finder/MarketTab"; -import { WeightProfilePanel } from "@/components/site-finder/WeightProfilePanel"; -import { useSiteAnalysis } from "@/hooks/useSiteAnalysis"; -import { - POI_DEFAULT_WEIGHTS, - type PoiCategoryKey, -} from "@/lib/api/weightProfiles"; +import { MapFilterBar } from "@/components/site-finder/entry/MapFilterBar"; +import { ParcelDrawer } from "@/components/site-finder/entry/ParcelDrawer"; +import { ParcelLegend } from "@/components/site-finder/entry/ParcelLegend"; +import { RecentParcels } from "@/components/site-finder/entry/RecentParcels"; +import { useParcelsBboxQuery } from "@/lib/site-finder-api"; +import type { + ParcelBboxFilters, + ParcelBboxItem, + BboxCoords, +} from "@/lib/site-finder-api"; -// SiteMap imports Leaflet which requires browser APIs — load without SSR -const SiteMap = dynamic( - () => import("@/components/site-finder/SiteMap").then((m) => m.SiteMap), +// EntryMap uses Leaflet — must load without SSR +const EntryMap = dynamic( + () => + import("@/components/site-finder/entry/EntryMap").then((m) => m.EntryMap), { ssr: false, loading: () => (
- Загрузка карты… + Загрузка карты...
), }, ); -type TabId = "overview" | "env" | "land" | "market"; +// ── FilterCountBridge ────────────────────────────────────────────────────────── +// Reads parcel counts for current bbox without re-mounting EntryMap. -const TABS: Array<{ id: TabId; label: string }> = [ - { id: "overview", label: "Обзор" }, - { id: "env", label: "Окружение" }, - { id: "land", label: "Земля" }, - { id: "market", label: "Рынок" }, -]; - -const TAB_IDS = TABS.map((t) => t.id); - -function isTabId(v: string | null | undefined): v is TabId { - return !!v && (TAB_IDS as readonly string[]).includes(v); +interface FilterBarBridgeProps { + filters: ParcelBboxFilters; + onChange: (f: ParcelBboxFilters) => void; + bbox: BboxCoords | null; } -// ── KPI helpers ─────────────────────────────────────────────────────────────── +function FilterBarBridge({ filters, onChange, bbox }: FilterBarBridgeProps) { + const { data: allInBbox } = useParcelsBboxQuery(bbox, {}); + const { data: filtered } = useParcelsBboxQuery(bbox, filters); -type KpiColor = "neutral" | "amber" | "green" | "red" | "blue"; - -function trendColor(pct: number): KpiColor { - if (pct > 5) return "green"; - if (pct > 0) return "blue"; - if (pct > -5) return "amber"; - return "red"; -} - -function noiseColor(db: number): KpiColor { - if (db < 45) return "green"; - if (db < 60) return "amber"; - return "red"; -} - -function scoreKpiColor(score: number, max: number): KpiColor { - const ratio = score / max; - if (ratio >= 0.65) return "green"; - if (ratio >= 0.35) return "amber"; - return "red"; + return ( + + ); } // ── Page ────────────────────────────────────────────────────────────────────── -function SiteFinderContent() { +export default function SiteFinderPage() { const router = useRouter(); - const pathname = usePathname(); - const searchParams = useSearchParams(); - - const { mutate, data, isPending, error, isIdle, fetchingState, cancel } = - useSiteAnalysis(); - const [isochrones, setIsochrones] = useState( - undefined, + const [filters, setFilters] = useState({}); + const [selectedParcel, setSelectedParcel] = useState( + null, ); + const [bbox, setBbox] = useState(null); - // Weight profile state — lifted here so it survives tab switches. - // userId + adminToken allow the panel to load/save named profiles. - const [currentWeights, setCurrentWeights] = useState< - Record - >(() => ({ ...POI_DEFAULT_WEIGHTS })); - // Profile selection for the analyze call (null = system defaults). - const [activeProfileId, setActiveProfileId] = useState(null); - const [profileUserId, setProfileUserId] = useState(() => - typeof window === "undefined" - ? "" - : (localStorage.getItem("admin_user_id") ?? ""), - ); - const [adminToken] = useState(() => - typeof window === "undefined" - ? "" - : (localStorage.getItem("admin_token") ?? ""), - ); - // Lazy init: считаем initialTab один раз на mount (useState всё равно - // игнорирует initializer после первого render — не тратим CPU). - const [tab, setTabState] = useState(() => { - const t = searchParams.get("tab"); - return isTabId(t) ? t : "overview"; - }); - - // Sync tab → URL query (?tab=env). overview = default, не пишем в URL. - function setTab(next: TabId) { - setTabState(next); - const params = new URLSearchParams(searchParams.toString()); - if (next === "overview") { - params.delete("tab"); - } else { - params.set("tab", next); - } - const qs = params.toString(); - // Используем pathname вместо голого "?" — Next 15 App Router не всегда - // чисто чистит query при `router.replace("?")` (dangling `?` остаётся). - router.replace(qs ? `${pathname}?${qs}` : pathname, { scroll: false }); + function handleParcelSelect(parcel: ParcelBboxItem) { + setSelectedParcel(parcel); } - // Listen to back/forward navigation — reflect URL tab in state. - // setState с тем же значением React сам бэйлит через Object.is, поэтому - // guard по tab не нужен (и не вводит stale-closure через eslint-disable). - useEffect(() => { - const urlTab = searchParams.get("tab"); - setTabState(isTabId(urlTab) ? urlTab : "overview"); - }, [searchParams]); - - function handleAnalyze(cadNum: string) { - setIsochrones(undefined); - setTab("overview"); - mutate({ - cad: cadNum, - options: - activeProfileId != null - ? { profileId: activeProfileId } - : profileUserId - ? { profileUserId } - : undefined, - }); + function handleParcelDeselect() { + setSelectedParcel(null); } - function handleWeightsChange( - weights: Record, - profileId: number | null, - ) { - setCurrentWeights(weights); - // Store the active profile id so we can pass it to the analyze call. - // If user edited weights without saving a named profile, profileId=null - // and backend will use system defaults (sub-PR 5 would enable inline weights). - setActiveProfileId(profileId); + function handleCadSubmit(cad: string) { + router.push(`/site-finder/analysis/${encodeURIComponent(cad)}`); } - // Derive KPI values from data - const scoreMax = data?.score_max_reference ?? 40; - const scoreValue = data - ? `${data.score.toFixed(2)} / ${scoreMax.toFixed(0)}` - : "—"; - const scoreLabel = data?.score_label - ? `Score · ${data.score_label}` - : "Score"; - const scoreCardColor: KpiColor = data - ? scoreKpiColor(data.score, scoreMax) - : "neutral"; - - const districtValue = data?.district ? data.district.district_name : "—"; - const districtLabel = data?.district - ? `${(data.district.median_price_per_m2 / 1000).toFixed(0)} тыс ₽/м²` + - ` · ${(data.district.dist_to_center / 1000).toFixed(1)} км от центра` - : "Район"; - - const trendPct = data?.market_trend?.delta_6m_pct; - const trendValue = - trendPct != null - ? `${trendPct > 0 ? "+" : ""}${trendPct.toFixed(1)}%` - : "—"; - const trendLabel = data?.market_trend - ? `Тренд за 6 мес · ${data.market_trend.label}` - : "Тренд рынка"; - const trendKpiColor: KpiColor = - trendPct != null ? trendColor(trendPct) : "neutral"; - - const noiseDb = data?.noise?.estimated_db; - const noiseValue = noiseDb != null ? `${Math.round(noiseDb)} dB` : "—"; - const noiseLabel = data?.noise ? `Шум · ${data.noise.level}` : "Шум"; - const noiseKpiColor: KpiColor = - noiseDb != null ? noiseColor(noiseDb) : "neutral"; - return ( -
- {/* Breadcrumb + header */} -
+
+ {/* Header */} +
← Главная + · +

+ SiteFinder · карта участков +

+
+ + {/* Filter bar */} +
+
+ {/* Main layout */}
-
-

- Site Finder -

-

- Введите кадастровый номер участка или квартала для анализа локации -

-
+ {/* Map area — position: relative so ParcelDrawer can anchor absolutely here */}
- + + + {/* ParcelDrawer — anchored to map area right edge, not page */} + +
+ + {/* Right sidebar — always visible; CadInput accessible parallel with drawer */} +
+ + +
- - {/* Weight profile panel — collapsible, below header */} -
- {/* Optional user-id field for profile CRUD (shown only when adminToken present) */} - {!!adminToken && ( -
- - { - setProfileUserId(e.target.value); - if (typeof window !== "undefined") { - localStorage.setItem("admin_user_id", e.target.value); - } - }} - /> -
- )} - -
- - {/* Error state */} - {error && ( -
- {error instanceof Error ? error.message : "Ошибка анализа участка"} -
- )} - - {/* #93 — Fetching state: backend на 202 Accepted, идёт on-demand NSPD fetch */} - {fetchingState && ( - - )} - - {/* Pending skeleton — первичный analyze в полёте (но fetching state ещё не активирован) */} - {isPending && !fetchingState && ( -
- Анализируем участок… -
- )} - - {/* Empty initial state */} - {isIdle && !error && ( -
- Введите кадастровый номер и нажмите «Анализировать» -
- )} - - {/* Results */} - {data && ( - <> - {/* Cad label */} -
- {data.cad_num} -
- - {/* Hero band — sticky KPI cards */} -
- - - - -
- - {/* Tab buttons */} -
- {TABS.map((t) => ( - - ))} -
- - {/* 2-column grid: tab content + sticky map */} -
- {/* Tab content */} -
- {tab === "overview" && ( - - )} - {tab === "env" && } - {tab === "land" && } - {tab === "market" && } -
- - {/* Sticky right sidebar — map always visible */} - -
- - )}
); } - -// useSearchParams() в Next 15 App Router throw'ит при static rendering без -// boundary — поэтому оборачиваем page-level. -export default function SiteFinderPage() { - return ( - - - - ); -} diff --git a/frontend/src/app/trade-in/page.tsx b/frontend/src/app/trade-in/page.tsx new file mode 100644 index 00000000..02192dbc --- /dev/null +++ b/frontend/src/app/trade-in/page.tsx @@ -0,0 +1,183 @@ +"use client"; + +import { useState } from "react"; +import { useRouter, useSearchParams } from "next/navigation"; + +import type { + AggregatedEstimate, + TradeInEstimateInput, +} from "@/types/trade-in"; +import { useEstimateMutation, useEstimate } from "@/lib/trade-in-api"; +import { EstimateForm } from "@/components/trade-in/EstimateForm"; +import { EstimateResult } from "@/components/trade-in/EstimateResult"; +import { EstimateProgress } from "@/components/trade-in/EstimateProgress"; + +// ── URL persistence helpers ──────────────────────────────────────────────────── + +function useEstimateId() { + // SSR guard: searchParams is only available in client + if (typeof window === "undefined") return null; + const params = new URLSearchParams(window.location.search); + return params.get("id"); +} + +// ── Page ────────────────────────────────────────────────────────────────────── + +export default function TradeInPage() { + const router = useRouter(); + + // Result from mutation (fresh) or from URL (restored) + const [freshResult, setFreshResult] = useState<{ + estimate: AggregatedEstimate; + input: TradeInEstimateInput; + } | null>(null); + + const urlEstimateId = useEstimateId(); + // Fetch from URL if no fresh result yet + const restoredEstimate = useEstimate( + freshResult === null ? urlEstimateId : null, + ); + + const mutation = useEstimateMutation(); + + function handleSubmit(input: TradeInEstimateInput) { + mutation.mutate(input, { + onSuccess: (estimate) => { + setFreshResult({ estimate, input }); + // Persist estimate_id in URL for shareable link + router.replace(`/trade-in?id=${estimate.estimate_id}`, { + scroll: false, + }); + }, + }); + } + + const apiError = mutation.error?.message ?? null; + + // Determine what to render on the right side + const resultData: { + estimate: AggregatedEstimate; + input: TradeInEstimateInput; + } | null = + freshResult ?? + (restoredEstimate.data + ? { + estimate: restoredEstimate.data, + // When restoring from URL we don't have the original input — synthesise a minimal one + input: { + address: restoredEstimate.data.analogs[0]?.address ?? "—", + area_m2: 0, + rooms: 0, + floor: 0, + total_floors: 0, + }, + } + : null); + + return ( +
+ {/* Page header */} +
+

+ Trade-In Estimator +

+

+ Оценка рыночной стоимости квартиры по аналогам и сделкам ЕКБ +

+
+ + {/* Two-column layout */} +
+ {/* Left — sticky form */} +
+ + +
+ + {/* Right — result area */} +
+ {resultData ? ( + + ) : restoredEstimate.isLoading ? ( + + ) : ( + + )} +
+
+ + {/* Responsive: stack on mobile */} + +
+ ); +} + +function EmptyState() { + return ( +
+
🏠
+
+ Введите параметры квартиры +
+
+ Оценка появится здесь после нажатия «Оценить» +
+
+ ); +} diff --git a/frontend/src/components/admin/BulkGeoPanel.tsx b/frontend/src/components/admin/BulkGeoPanel.tsx index 06195b45..7fa8ade8 100644 --- a/frontend/src/components/admin/BulkGeoPanel.tsx +++ b/frontend/src/components/admin/BulkGeoPanel.tsx @@ -37,7 +37,7 @@ const THEMATIC_OPTIONS = [ // ── Component ──────────────────────────────────────────────────────────────── -export function BulkGeoPanel({ token }: { token: string }) { +export function BulkGeoPanel() { const [parallelism, setParallelism] = useState(5); const [thematicIds, setThematicIds] = useState([2]); const [source, setSource] = useState<"rosreestr_pending" | "all_in_region">( @@ -62,7 +62,6 @@ export function BulkGeoPanel({ token }: { token: string }) { }; return apiFetch("/api/v1/admin/scrape/geo/bulk", { method: "POST", - headers: { "X-Admin-Token": token }, body: JSON.stringify(body), }); }, @@ -179,9 +178,9 @@ export function BulkGeoPanel({ token }: { token: string }) {
- {!token ? ( -

- Введите Admin Token чтобы видеть лог. -

- ) : logs.isLoading ? ( + {logs.isLoading ? (

Загрузка…

) : logs.isError ? (

Ошибка: {String(logs.error)}

diff --git a/frontend/src/components/analytics/ObjectChecksBadges.tsx b/frontend/src/components/analytics/ObjectChecksBadges.tsx new file mode 100644 index 00000000..f565ca8c --- /dev/null +++ b/frontend/src/components/analytics/ObjectChecksBadges.tsx @@ -0,0 +1,45 @@ +import type { ObjectCheckRow } from "@/types/analytics"; + +const CHECK_LABELS: Record = { + no_problems: "Нет проблемных объектов", + docs: "Документы опубликованы", + timing: "Сроки соблюдены", + photos: "Актуальные фото", + bankruptcy: "Не банкрот", + declaration: "Декларация обновлена", +}; + +export function ObjectChecksBadges({ checks }: { checks: ObjectCheckRow[] }) { + if (checks.length === 0) { + return ( +

+ Данные проверок ещё не загружены. +

+ ); + } + + return ( +
+ {checks.map((c) => ( +
+ {c.passed ? "✓" : "✗"} + {CHECK_LABELS[c.check_type] ?? c.check_type} +
+ ))} +
+ ); +} diff --git a/frontend/src/components/analytics/ObjectDocumentsList.tsx b/frontend/src/components/analytics/ObjectDocumentsList.tsx new file mode 100644 index 00000000..b7817558 --- /dev/null +++ b/frontend/src/components/analytics/ObjectDocumentsList.tsx @@ -0,0 +1,106 @@ +import type { ObjectDocumentRow } from "@/types/analytics"; + +function safeUrl(url: string): string { + try { + const parsed = new URL(url); + if (parsed.protocol === "https:" || parsed.protocol === "http:") { + return url; + } + } catch { + // fall through + } + return "#"; +} + +function fmtSize(bytes: number | null): string { + if (bytes === null) return ""; + if (bytes < 1024) return `${bytes} Б`; + if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(0)} КБ`; + return `${(bytes / (1024 * 1024)).toFixed(1)} МБ`; +} + +const DOC_TYPE_ORDER: Record = { + декларация: 1, + разрешение: 2, + проектная: 3, + отчётность: 4, + прочее: 5, +}; + +export function ObjectDocumentsList({ docs }: { docs: ObjectDocumentRow[] }) { + if (docs.length === 0) { + return ( +

Документы отсутствуют.

+ ); + } + + // Group by doc_type + const byType = new Map(); + for (const d of docs) { + const group = byType.get(d.doc_type) ?? []; + group.push(d); + byType.set(d.doc_type, group); + } + + const sortedTypes = [...byType.keys()].sort((a, b) => { + const oa = DOC_TYPE_ORDER[a.toLowerCase()] ?? 99; + const ob = DOC_TYPE_ORDER[b.toLowerCase()] ?? 99; + return oa - ob || a.localeCompare(b, "ru"); + }); + + return ( +
+ {sortedTypes.map((type) => { + const group = byType.get(type)!; + return ( +
+
+ {type} +
+
+ {group.map((doc, i) => ( +
+ 📄 + + {doc.doc_num ? `${doc.doc_num}` : `Документ ${i + 1}`} + + {doc.posted_at ? ( + + {doc.posted_at.slice(0, 10)} + + ) : null} + {doc.size_bytes ? ( + + {fmtSize(doc.size_bytes)} + + ) : null} +
+ ))} +
+
+ ); + })} +
+ ); +} diff --git a/frontend/src/components/analytics/ObjectMetroList.tsx b/frontend/src/components/analytics/ObjectMetroList.tsx new file mode 100644 index 00000000..63ba85cb --- /dev/null +++ b/frontend/src/components/analytics/ObjectMetroList.tsx @@ -0,0 +1,85 @@ +import type { MetroEntry } from "@/types/analytics"; + +interface Props { + nearest_name: string | null; + nearest_walk_minutes: number | null; + top3: MetroEntry[] | null; +} + +export function ObjectMetroList({ + nearest_name, + nearest_walk_minutes, + top3, +}: Props) { + // Build display list: prefer top3 array, fallback to nearest fields + const entries: MetroEntry[] = + top3 && top3.length > 0 + ? top3 + : nearest_name + ? [ + { + name: nearest_name, + time: nearest_walk_minutes, + line: null, + color: null, + isWalk: true, + }, + ] + : []; + + if (entries.length === 0) { + return ( +

+ Данные о метро отсутствуют. +

+ ); + } + + return ( +
+ {entries.map((m, i) => ( +
+ +
+
{m.name}
+ {m.line ? ( +
{m.line}
+ ) : null} +
+ {m.time != null ? ( +
+ {m.isWalk !== false ? "🚶" : "🚗"} {m.time} мин +
+ ) : null} +
+ ))} +
+ ); +} diff --git a/frontend/src/components/analytics/ObjectQuartirographyTable.tsx b/frontend/src/components/analytics/ObjectQuartirographyTable.tsx new file mode 100644 index 00000000..0d57cee3 --- /dev/null +++ b/frontend/src/components/analytics/ObjectQuartirographyTable.tsx @@ -0,0 +1,115 @@ +import type { ObjectQuartirographyRow } from "@/types/analytics"; + +function fmtPrice(v: number | null): string { + if (v === null) return "—"; + return ( + (v / 1_000_000).toLocaleString("ru-RU", { + minimumFractionDigits: 1, + maximumFractionDigits: 1, + }) + " млн" + ); +} + +function fmtArea(v: number | null): string { + if (v === null) return "—"; + return v.toLocaleString("ru-RU", { maximumFractionDigits: 1 }) + " м²"; +} + +export function ObjectQuartirographyTable({ + rows, +}: { + rows: ObjectQuartirographyRow[]; +}) { + if (rows.length === 0) { + return ( +

+ Данные квартирографии отсутствуют. +

+ ); + } + + const headers = ["Тип", "Всего", "В продаже", "Площадь", "Цена (от–до)"]; + + return ( +
+ + + + {headers.map((h) => ( + + ))} + + + + {rows.map((r) => ( + + + + + + + + ))} + +
+ {h} +
+ {r.room_label} + + {r.total_count.toLocaleString("ru-RU")} + + {r.free_count > 0 ? ( + + {r.free_count.toLocaleString("ru-RU")} + + ) : ( + 0 + )} + + {r.area_min !== null || r.area_max !== null + ? `${fmtArea(r.area_min)} – ${fmtArea(r.area_max)}` + : "—"} + + {r.price_min !== null || r.price_max !== null + ? `${fmtPrice(r.price_min)} – ${fmtPrice(r.price_max)}` + : "—"} +
+
+ ); +} diff --git a/frontend/src/components/analytics/ObjectSpecsTable.tsx b/frontend/src/components/analytics/ObjectSpecsTable.tsx new file mode 100644 index 00000000..3b0ebb34 --- /dev/null +++ b/frontend/src/components/analytics/ObjectSpecsTable.tsx @@ -0,0 +1,204 @@ +import type { ObjectFullDetail } from "@/types/analytics"; + +interface SpecRow { + label: string; + value: string | null; +} + +function yn(v: boolean | null | undefined): string | null { + if (v === null || v === undefined) return null; + return v ? "Да" : "Нет"; +} + +function num(v: number | null | undefined, suffix = ""): string | null { + if (v === null || v === undefined) return null; + return `${v.toLocaleString("ru-RU")}${suffix}`; +} + +function numFmt( + v: number | null | undefined, + decimals = 0, + suffix = "", +): string | null { + if (v === null || v === undefined) return null; + return ( + v.toLocaleString("ru-RU", { + minimumFractionDigits: decimals, + maximumFractionDigits: decimals, + }) + suffix + ); +} + +function price(v: number | null | undefined): string | null { + if (v === null || v === undefined) return null; + return ( + (v / 1_000_000).toLocaleString("ru-RU", { + minimumFractionDigits: 1, + maximumFractionDigits: 1, + }) + " млн ₽" + ); +} + +function priceM2(v: number | null | undefined): string | null { + if (v === null || v === undefined) return null; + return v.toLocaleString("ru-RU", { maximumFractionDigits: 0 }) + " ₽/м²"; +} + +export function ObjectSpecsTable({ d }: { d: ObjectFullDetail }) { + const specs: SpecRow[] = [ + // Identity + { label: "Класс объекта", value: d.obj_class }, + { label: "Класс энергоэффективности", value: d.energy_eff }, + { label: "Материал стен", value: d.wall_type }, + { label: "Первый этаж", value: d.first_floor_type }, + // Building + { label: "Подъездов", value: num(d.section_count) }, + { label: "Этажей (max)", value: num(d.floor_max) }, + { label: "Высота потолков", value: numFmt(d.ceiling_height_m, 2, " м") }, + { + label: "Лифты пассажирские", + value: + d.elevators_passenger_count === 0 + ? "Нет" + : num(d.elevators_passenger_count), + }, + { + label: "Лифты грузовые", + value: + d.elevators_cargo_count === 0 ? "Нет" : num(d.elevators_cargo_count), + }, + // Parking + { + label: "Машино-мест (парковка)", + value: num(d.parking_total_slots), + }, + { + label: "Гостевые (приказ.)", + value: num(d.guest_parking_inside_count), + }, + { + label: "Гостевые (улица)", + value: num(d.guest_parking_outside_count), + }, + { + label: "Обеспеченность парковкой", + value: numFmt(d.parking_provision_pct, 1, "%"), + }, + // Apartments + { label: "Квартир всего", value: num(d.flat_count) }, + { + label: "Площадь квартир", + value: + d.flat_area_min != null && d.flat_area_max != null + ? `${numFmt(d.flat_area_min, 1)} – ${numFmt(d.flat_area_max, 1)} м²` + : null, + }, + { + label: "Средняя площадь квартиры", + value: numFmt(d.avg_flat_area_m2, 1, " м²"), + }, + { + label: "Вариантов отделки", + value: num(d.finishing_variants_count), + }, + { label: "Свободная планировка", value: yn(d.has_free_planning) }, + // Price + { + label: "Цена (от)", + value: price(d.price_min_rub), + }, + { + label: "Цена (до)", + value: price(d.price_max_rub), + }, + { + label: "Цена/м² (от)", + value: priceM2(d.price_per_m2_min), + }, + { + label: "Цена/м² (до)", + value: priceM2(d.price_per_m2_max), + }, + // Yard + { + label: "Детских площадок", + value: num(d.playground_kids_count), + }, + { + label: "Спортивных площадок", + value: num(d.playground_sport_count), + }, + { label: "Велодорожки", value: yn(d.has_bike_paths) }, + { + label: "Мусорные площадки", + value: num(d.trash_areas_count), + }, + // OVZ + { label: "Пандус", value: yn(d.has_ramp) }, + { label: "Понижающие платформы (ОВЗ)", value: yn(d.has_low_platforms) }, + { + label: "Инвалидный подъёмник", + value: yn(d.has_wheelchair_lift), + }, + // Developer + { label: "Девелопер", value: d.dev_name }, + { label: "Группа компаний", value: d.dev_group_name }, + { label: "Эскроу", value: yn(d.escrow) }, + { + label: "Декларация №", + value: d.project_declaration_num, + }, + { + label: "Дата публикации проекта", + value: d.project_published_at?.slice(0, 10) ?? null, + }, + ]; + + const filtered = specs.filter((s) => s.value !== null); + if (filtered.length === 0) { + return ( +

+ Характеристики ещё не загружены. +

+ ); + } + + return ( +
+ + + {filtered.map((s) => ( + + + + + ))} + +
+ {s.label} + + {s.value} +
+
+ ); +} diff --git a/frontend/src/components/landing/PilotRequestModal.tsx b/frontend/src/components/landing/PilotRequestModal.tsx new file mode 100644 index 00000000..d8710d64 --- /dev/null +++ b/frontend/src/components/landing/PilotRequestModal.tsx @@ -0,0 +1,371 @@ +"use client"; + +import { useState } from "react"; + +import { CheckCircle } from "lucide-react"; + +import { Drawer } from "@/components/ui/Drawer"; +import { + useSubmitPilotRequest, + type PilotRequestPayload, +} from "@/lib/api/landing"; + +// ── Types ────────────────────────────────────────────────────────────────── + +interface Props { + open: boolean; + onClose: () => void; +} + +interface FormState { + name: string; + phone: string; + email: string; + company: string; + message: string; +} + +// ── Helpers ──────────────────────────────────────────────────────────────── + +/** Derives a human-readable tracking ID from the UUID string returned by backend. + * Example: "7f3a8c2e-..." → "GD-7F3A8C2E" + */ +function deriveTrackingId(id: string): string { + return "GD-" + id.replace(/-/g, "").slice(0, 8).toUpperCase(); +} + +// ── Component ────────────────────────────────────────────────────────────── + +export function PilotRequestModal({ open, onClose }: Props) { + const [form, setForm] = useState({ + name: "", + phone: "", + email: "", + company: "", + message: "", + }); + const [emailError, setEmailError] = useState(""); + const [trackingId, setTrackingId] = useState(null); + + const mutation = useSubmitPilotRequest(); + + const EMAIL_RE = /^[^\s@]+@[^\s@]+\.[^\s@]+$/; + + function handleChange( + e: React.ChangeEvent, + ) { + const { name, value } = e.target; + setForm((prev) => ({ ...prev, [name]: value })); + if (name === "email") { + setEmailError(value && !EMAIL_RE.test(value) ? "Некорректный email" : ""); + } + } + + function handleSubmit(e: React.FormEvent) { + e.preventDefault(); + if (!form.name.trim()) return; + if (form.email && !EMAIL_RE.test(form.email)) { + setEmailError("Некорректный email"); + return; + } + + const payload: PilotRequestPayload = { + name: form.name.trim(), + source: "landing", + ...(form.phone.trim() ? { phone: form.phone.trim() } : {}), + ...(form.email.trim() ? { email: form.email.trim() } : {}), + ...(form.company.trim() ? { company: form.company.trim() } : {}), + ...(form.message.trim() + ? { message: form.message.trim().slice(0, 2000) } + : {}), + }; + + mutation.mutate(payload, { + onSuccess: (data) => { + setTrackingId(deriveTrackingId(data.id)); + }, + }); + } + + function handleClose() { + setForm({ name: "", phone: "", email: "", company: "", message: "" }); + setEmailError(""); + setTrackingId(null); + mutation.reset(); + onClose(); + } + + return ( + + {/* Header */} +
+

+ Запросить пилотный доступ +

+
+ +
+ {/* Success state */} + {trackingId !== null ? ( +
+ +

+ Заявка принята +

+

+ Мы свяжемся с вами в течение 1–2 рабочих дней. Номер заявки:{" "} + + {trackingId} + +

+ +
+ ) : ( + /* Form */ +
+
+ {/* Name */} +
+ + +
+ + {/* Company */} +
+ + +
+ + {/* Phone */} +
+ + +
+ + {/* Email */} +
+ + + {emailError ? ( + + {emailError} + + ) : null} +
+ + {/* Message */} +
+ +