lekss361/gendesign
1
0
Fork 0
Code Issues 57 Pull requests Projects Releases Packages Wiki Activity Actions

Merge remote-tracking branch 'forgejo/main' into feat/tradein-cian-cookies-ui

Browse source
This commit is contained in:
lekss361 2026-05-23 18:14:06 +03:00
commit 6c4e5a6809
447 changed files with 80740 additions and 2124 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
.env.example
View file

@ -10,3 +10,11 @@ POSTGRES_PASSWORD=changeme
# YC_REGISTRY_ID=
# IMAGE_TAG=latest
# NEXT_PUBLIC_API_BASE_URL=https://your-domain.example.ru
# GlitchTip (errors.gendsgn.ru)
GLITCHTIP_DB_PASS=
GLITCHTIP_SECRET=
# DSN из GlitchTip UI → Project Settings → Client Keys.
# Используется backend Sentry SDK + glitchtip-auth-forwarder sidecar.
GLITCHTIP_DSN=
NEXT_PUBLIC_GLITCHTIP_DSN=

193
.forgejo/workflows/deploy-tradein.yml Normal file
View file

@ -0,0 +1,193 @@
name: Deploy Trade-In
# Forgejo Actions — отдельный pipeline для подпроекта tradein-mvp/.
# Триггерится только на изменения внутри tradein-mvp/ (или этого workflow),
# не пересекается с основным deploy.yml.
on:
push:
branches: [main]
paths:
- "tradein-mvp/**"
- ".forgejo/workflows/deploy-tradein.yml"
workflow_dispatch:
concurrency:
group: deploy-tradein-prod
cancel-in-progress: false
env:
IMAGE_BACKEND: ghcr.io/lekss361/gendesign-tradein-backend
IMAGE_FRONTEND: ghcr.io/lekss361/gendesign-tradein-frontend
jobs:
changes:
runs-on: ubuntu-latest
outputs:
backend: ${{ steps.filter.outputs.backend }}
frontend: ${{ steps.filter.outputs.frontend }}
infra: ${{ steps.filter.outputs.infra }}
steps:
- uses: actions/checkout@v4
- uses: dorny/paths-filter@v3
id: filter
with:
filters: |
backend:
- 'tradein-mvp/backend/**'
frontend:
- 'tradein-mvp/frontend/**'
infra:
- 'tradein-mvp/docker-compose.prod.yml'
- 'tradein-mvp/deploy/**'
- '.forgejo/workflows/deploy-tradein.yml'
build-backend:
runs-on: ubuntu-latest
needs: changes
if: |
needs.changes.outputs.backend == 'true' ||
needs.changes.outputs.infra == 'true' ||
github.event_name == 'workflow_dispatch'
steps:
- uses: actions/checkout@v4
- name: Login to GHCR (shell-based — docker/login-action@v3 unreliable под Forgejo Actions)
env:
GHCR_PAT: ${{ secrets.GHCR_PAT }}
run: |
echo "$GHCR_PAT" | docker login ghcr.io -u lekss361 --password-stdin
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Build & push tradein-backend
uses: docker/build-push-action@v6
with:
context: ./tradein-mvp/backend
push: true
cache-from: type=registry,ref=${{ env.IMAGE_BACKEND }}:buildcache
cache-to: type=registry,ref=${{ env.IMAGE_BACKEND }}:buildcache,mode=max
tags: |
${{ env.IMAGE_BACKEND }}:latest
${{ env.IMAGE_BACKEND }}:${{ github.sha }}
build-frontend:
runs-on: ubuntu-latest
needs: changes
if: |
needs.changes.outputs.frontend == 'true' ||
needs.changes.outputs.infra == 'true' ||
github.event_name == 'workflow_dispatch'
steps:
- uses: actions/checkout@v4
- name: Login to GHCR (shell-based — docker/login-action@v3 unreliable под Forgejo Actions)
env:
GHCR_PAT: ${{ secrets.GHCR_PAT }}
run: |
echo "$GHCR_PAT" | docker login ghcr.io -u lekss361 --password-stdin
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Build & push tradein-frontend
uses: docker/build-push-action@v6
with:
context: ./tradein-mvp/frontend
push: true
# basePath=/trade-in baked-in во время build (Next.js)
build-args: |
NEXT_PUBLIC_BASE_PATH=/trade-in
NEXT_PUBLIC_API_BASE_URL=/trade-in
cache-from: type=registry,ref=${{ env.IMAGE_FRONTEND }}:buildcache
cache-to: type=registry,ref=${{ env.IMAGE_FRONTEND }}:buildcache,mode=max
tags: |
${{ env.IMAGE_FRONTEND }}:latest
${{ env.IMAGE_FRONTEND }}:${{ github.sha }}
deploy:
runs-on: ubuntu-latest
needs: [changes, build-backend, build-frontend]
if: |
always() &&
!cancelled() &&
needs.build-backend.result != 'failure' &&
needs.build-frontend.result != 'failure'
steps:
- name: Deploy via SSH
uses: appleboy/ssh-action@v1.0.3
env:
IMAGE_TAG: latest
GHCR_PAT: ${{ secrets.GHCR_PAT }}
with:
host: ${{ secrets.DEPLOY_HOST }}
username: ${{ secrets.DEPLOY_USER }}
key: ${{ secrets.DEPLOY_SSH_KEY }}
port: ${{ secrets.DEPLOY_PORT }}
envs: IMAGE_TAG,GHCR_PAT
script: |
set -euo pipefail
cd /opt/gendesign
# repo уже clone'ен — origin = Forgejo. Подтягиваем последний main.
git fetch origin main
git reset --hard origin/main
cd tradein-mvp
# .env.runtime создаётся вручную при первом запуске (см. README-АДМИНУ.md).
# Здесь только подгружаем переменные для docker compose (POSTGRES_PASSWORD).
if [ ! -f .env.runtime ]; then
echo "ERROR: /opt/gendesign/tradein-mvp/.env.runtime отсутствует."
echo "Создай его вручную (см. tradein-mvp/README.md или DEPLOY.md)."
exit 1
fi
chmod 600 .env.runtime
set -a; source .env.runtime; set +a
# External network для Caddy (он в основном gendesign-стеке)
docker network inspect gendesign_shared >/dev/null 2>&1 \
|| docker network create gendesign_shared
# Re-login to GHCR (PAT может быть rotated)
echo "$GHCR_PAT" | docker login ghcr.io -u lekss361 --password-stdin
export IMAGE_TAG="$IMAGE_TAG"
docker compose -p gendesign-tradein -f docker-compose.prod.yml pull
docker compose -p gendesign-tradein -f docker-compose.prod.yml up -d
# Применяем SQL миграции (если есть data/sql/*.sql)
# Postgres init load *.sql из /docker-entrypoint-initdb.d ТОЛЬКО при первом
# старте volume. Здесь — для повторных миграций после первого запуска.
sleep 5
for sql_file in $(ls -1 backend/data/sql/*.sql 2>/dev/null | sort); do
fname=$(basename "$sql_file")
echo "→ Migration: $fname"
docker compose -p gendesign-tradein -f docker-compose.prod.yml exec -T postgres \
psql -U "${TRADEIN_POSTGRES_USER:-tradein}" -d tradein \
-v ON_ERROR_STOP=on < "$sql_file" \
|| echo " (skipped — already applied or error: $fname)"
done
# Caddy reload — основной Caddyfile содержит inline tradein routes
# (см. Caddyfile в корне репы). Reload, чтобы Caddy перечитал DNS
# tradein-backend / tradein-frontend (они в gendesign_shared network).
cd /opt/gendesign
docker compose -p gendesign -f docker-compose.prod.yml exec -T caddy \
caddy reload --config /etc/caddy/Caddyfile || true
# Health check
for i in $(seq 1 30); do
docker compose -p gendesign-tradein -f /opt/gendesign/tradein-mvp/docker-compose.prod.yml \
exec -T backend curl -fsS http://localhost:8000/health && break
sleep 1
done
# Cleanup старых образов
for repo in ghcr.io/lekss361/gendesign-tradein-backend \
ghcr.io/lekss361/gendesign-tradein-frontend; do
docker images "$repo" --format '{{.Repository}}:{{.Tag}}' \
| grep -v ':latest$' | tail -n +3 \
| xargs -r docker rmi 2>/dev/null || true
done
docker image prune -af || true

176
.github/workflows/deploy.yml → .forgejo/workflows/deploy.yml
View file

@ -1,7 +1,9 @@
name: Deploy
# Деплоится только при изменениях основного стека.
# Obsidian-стек (CouchDB) — отдельный workflow `deploy-obsidian.yml`.
# Forgejo Actions equivalent of .github/workflows/deploy.yml
# Migration 2026-05-16: GitHub → Forgejo (git.gendsgn.ru)
# Builds images on Forgejo runner, pushes to ghcr.io (GitHub Container Registry),
# SSH-deploys to Beget VPS (46.173.16.127).
on:
push:
branches: [main]
@ -10,10 +12,10 @@ on:
- "frontend/**"
- "docker-compose.prod.yml"
- "Caddyfile"
- ".github/workflows/deploy.yml"
- "data/sql/*.sql"
# NB: shared compose-fragments (network) — тоже триггерят main
# потому что Caddy шарит сеть с obsidian-stack
- "caddy/**"
- ".forgejo/workflows/deploy.yml"
- "data/sql/**"
- "ops/glitchtip-auth-forwarder/**"
workflow_dispatch:
concurrency:
@ -28,9 +30,6 @@ env:
jobs:
changes:
runs-on: ubuntu-latest
permissions:
contents: read
pull-requests: read
outputs:
backend: ${{ steps.filter.outputs.backend }}
frontend: ${{ steps.filter.outputs.frontend }}
@ -43,19 +42,18 @@ jobs:
filters: |
backend:
- 'backend/**'
- 'data/sql/**'
frontend:
- 'frontend/**'
infra:
- 'docker-compose.prod.yml'
- 'Caddyfile'
- '.github/workflows/deploy.yml'
- 'caddy/**'
- '.forgejo/workflows/deploy.yml'
build-backend:
runs-on: ubuntu-latest
needs: changes
permissions:
contents: read
packages: write
if: |
needs.changes.outputs.backend == 'true' ||
needs.changes.outputs.infra == 'true' ||
@ -63,12 +61,11 @@ jobs:
steps:
- uses: actions/checkout@v4
- name: Login to GHCR
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Login to GHCR (shell-based — docker/login-action@v3 unreliable под Forgejo Actions)
env:
GHCR_PAT: ${{ secrets.GHCR_PAT }}
run: |
echo "$GHCR_PAT" | docker login ghcr.io -u lekss361 --password-stdin
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
@ -79,8 +76,8 @@ jobs:
context: ./backend
target: runner
push: true
cache-from: type=gha,scope=backend-lean
cache-to: type=gha,mode=max,scope=backend-lean
cache-from: type=registry,ref=${{ env.IMAGE_BACKEND }}:buildcache
cache-to: type=registry,ref=${{ env.IMAGE_BACKEND }}:buildcache,mode=max
tags: |
${{ env.IMAGE_BACKEND }}:latest
${{ env.IMAGE_BACKEND }}:${{ github.sha }}
@ -88,9 +85,6 @@ jobs:
build-worker:
runs-on: ubuntu-latest
needs: changes
permissions:
contents: read
packages: write
if: |
needs.changes.outputs.backend == 'true' ||
needs.changes.outputs.infra == 'true' ||
@ -98,12 +92,11 @@ jobs:
steps:
- uses: actions/checkout@v4
- name: Login to GHCR
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Login to GHCR (shell-based — docker/login-action@v3 unreliable под Forgejo Actions)
env:
GHCR_PAT: ${{ secrets.GHCR_PAT }}
run: |
echo "$GHCR_PAT" | docker login ghcr.io -u lekss361 --password-stdin
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
@ -114,8 +107,8 @@ jobs:
context: ./backend
target: runner-with-chromium
push: true
cache-from: type=gha,scope=worker-chromium
cache-to: type=gha,mode=max,scope=worker-chromium
cache-from: type=registry,ref=${{ env.IMAGE_WORKER }}:buildcache
cache-to: type=registry,ref=${{ env.IMAGE_WORKER }}:buildcache,mode=max
tags: |
${{ env.IMAGE_WORKER }}:latest
${{ env.IMAGE_WORKER }}:${{ github.sha }}
@ -123,9 +116,6 @@ jobs:
build-frontend:
runs-on: ubuntu-latest
needs: changes
permissions:
contents: read
packages: write
if: |
needs.changes.outputs.frontend == 'true' ||
needs.changes.outputs.infra == 'true' ||
@ -133,12 +123,11 @@ jobs:
steps:
- uses: actions/checkout@v4
- name: Login to GHCR
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Login to GHCR (shell-based — docker/login-action@v3 unreliable под Forgejo Actions)
env:
GHCR_PAT: ${{ secrets.GHCR_PAT }}
run: |
echo "$GHCR_PAT" | docker login ghcr.io -u lekss361 --password-stdin
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
@ -148,8 +137,11 @@ jobs:
with:
context: ./frontend
push: true
cache-from: type=gha,scope=frontend
cache-to: type=gha,mode=max,scope=frontend
build-args: |
NEXT_PUBLIC_GLITCHTIP_DSN=${{ secrets.GLITCHTIP_FRONTEND_DSN }}
NEXT_PUBLIC_ENVIRONMENT=production
cache-from: type=registry,ref=${{ env.IMAGE_FRONTEND }}:buildcache
cache-to: type=registry,ref=${{ env.IMAGE_FRONTEND }}:buildcache,mode=max
tags: |
${{ env.IMAGE_FRONTEND }}:latest
${{ env.IMAGE_FRONTEND }}:${{ github.sha }}
@ -157,10 +149,6 @@ jobs:
deploy:
runs-on: ubuntu-latest
needs: [changes, build-backend, build-worker, build-frontend]
permissions:
contents: read
# Запускается если ни один build не завершился failure.
# `always()` позволяет выполнить job даже когда зависимые jobs были skipped.
if: |
always() &&
!cancelled() &&
@ -173,26 +161,28 @@ jobs:
env:
IMAGE_TAG: latest
SENTRY_RELEASE_VAL: ${{ github.sha }}
GHCR_PAT: ${{ secrets.GHCR_PAT }}
GLITCHTIP_BACKEND_DSN: ${{ secrets.GLITCHTIP_BACKEND_DSN }}
OBJECTIVE_API_KEY: ${{ secrets.OBJECTIVE_API_KEY }}
with:
host: ${{ secrets.DEPLOY_HOST }}
username: ${{ secrets.DEPLOY_USER }}
key: ${{ secrets.DEPLOY_SSH_KEY }}
port: ${{ secrets.DEPLOY_PORT || 22 }}
envs: IMAGE_TAG,SENTRY_RELEASE_VAL
port: ${{ secrets.DEPLOY_PORT }}
envs: IMAGE_TAG,SENTRY_RELEASE_VAL,GHCR_PAT,GLITCHTIP_BACKEND_DSN,OBJECTIVE_API_KEY
script: |
set -euo pipefail
cd /opt/gendesign
# Sync compose / Caddyfile / init scripts from the repo.
# --ff-only refuses if there are local commits on the VM (we don't expect any).
# Origin теперь Forgejo (после migration 2026-05-16) — HTTPS basic auth
# через PAT в URL не нужен т.к. repo читается через deploy key (SSH)
# ИЛИ public read-only mode. Forgejo gendesign — private, нужен auth:
# `git remote get-url origin` должен указывать на Forgejo с auth.
git fetch origin main
git reset --hard origin/main
# Sentry release tracking — записываем git-sha в .env.runtime.
# ВАЖНО: НЕ перезаписываем файл целиком (там могут быть
# COUCHDB_PASSWORD и др. user-managed secrets). Заменяем только
# строку SENTRY_RELEASE или добавляем если её нет.
# IMAGE_TAG=latest для docker pull; SENTRY_RELEASE_VAL = git sha для трекинга.
# Sentry release tracking
mkdir -p backend
touch backend/.env.runtime
if grep -q '^SENTRY_RELEASE=' backend/.env.runtime; then
@ -200,29 +190,39 @@ jobs:
else
printf 'SENTRY_RELEASE=%s\n' "$SENTRY_RELEASE_VAL" >> backend/.env.runtime
fi
# GlitchTip wiring: убираем legacy SENTRY_DSN (auto-promote-логика в
# backend/app/core/config.py:_promote_legacy_sentry_dsn раньше брала
# его и слала события в чужой sentry.io). Устанавливаем GLITCHTIP_DSN
# из Forgejo secret. Пустой secret = no-op (SDK не инициализируется).
sed -i '/^SENTRY_DSN=/d' backend/.env.runtime
if grep -q '^GLITCHTIP_DSN=' backend/.env.runtime; then
sed -i "s|^GLITCHTIP_DSN=.*|GLITCHTIP_DSN=$GLITCHTIP_BACKEND_DSN|" backend/.env.runtime
else
printf 'GLITCHTIP_DSN=%s\n' "$GLITCHTIP_BACKEND_DSN" >> backend/.env.runtime
fi
# Objective API key — для live scraper sync_all_groups (issue #307 OBJ-1).
# Пустой secret = no-op (sync_objective_group skipped с reason=no_api_key).
if grep -q '^OBJECTIVE_API_KEY=' backend/.env.runtime; then
sed -i "s|^OBJECTIVE_API_KEY=.*|OBJECTIVE_API_KEY=$OBJECTIVE_API_KEY|" backend/.env.runtime
else
printf 'OBJECTIVE_API_KEY=%s\n' "$OBJECTIVE_API_KEY" >> backend/.env.runtime
fi
chmod 600 backend/.env.runtime
# Создать external network если её нет (нужна Caddy для маршрута
# obsidian.gendsgn.ru → couchdb из отдельного obsidian-stack).
# External network для Caddy + obsidian-stack share
docker network inspect gendesign_shared >/dev/null 2>&1 \
|| docker network create gendesign_shared
# Re-login to GHCR (PAT может быть rotated после initial setup)
echo "$GHCR_PAT" | docker login ghcr.io -u lekss361 --password-stdin
export IMAGE_TAG="$IMAGE_TAG"
# Project name явно — `gendesign` (по имени папки auto), но
# фиксируем для consistency с obsidian-stack (-p gendesign-obsidian).
docker compose -p gendesign -f docker-compose.prod.yml pull
# ── Apply pending SQL migrations ──────────────────────────────────
# Tracking table: _schema_migrations (filename TEXT PRIMARY KEY).
# Each NN_*.sql file is applied exactly once, in sort order.
# ON_ERROR_STOP=on ensures a failing migration aborts the deploy.
# POSTGRES_USER/PASSWORD/DB живут в /opt/gendesign/.env (root —
# их читает compose через env_file для postgres service).
# backend/.env содержит ТОЛЬКО backend-app vars (SCRAPE_ADMIN_TOKEN,
# OPENROUTESERVICE_API_KEY, etc), без POSTGRES_*.
# Apply pending SQL migrations
set -a; source .env; set +a
# Ensure tracking table exists (idempotent).
docker compose -p gendesign -f docker-compose.prod.yml exec -T postgres \
psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -v ON_ERROR_STOP=on -c "
CREATE TABLE IF NOT EXISTS _schema_migrations (
@ -231,7 +231,6 @@ jobs:
);
"
# Apply each pending .sql file in NN order.
for sql_file in $(ls -1 data/sql/*.sql 2>/dev/null | sort); do
fname=$(basename "$sql_file")
applied=$(docker compose -p gendesign -f docker-compose.prod.yml exec -T postgres \
@ -251,18 +250,35 @@ jobs:
fi
done
echo "All migrations applied."
# ─────────────────────────────────────────────────────────────────
# Build local-only sidecar images (glitchtip-auth-forwarder).
# Эти services не в GHCR — сборка происходит на VPS на каждом deploy.
# Cache-friendly: первый build ~30s, последующие 1-3s если файлы не менялись.
docker compose -p gendesign -f docker-compose.prod.yml build glitchtip-auth-forwarder
docker compose -p gendesign -f docker-compose.prod.yml up -d
# Caddyfile is bind-mounted; up -d won't re-read it. Reload explicitly.
# `|| true` so a temporary Caddy hiccup doesn't fail the whole deploy.
docker compose -p gendesign -f docker-compose.prod.yml exec -T caddy \
caddy reload --config /etc/caddy/Caddyfile || true
# backend/.env.runtime изменения (SENTRY_RELEASE, GLITCHTIP_DSN)
# требуют --force-recreate — обычный `up -d` не перечитывает env_file
# если только image не сменился. На deploy где меняется только runtime
# без backend image change — без этого backend остаётся со старым DSN.
docker compose -p gendesign -f docker-compose.prod.yml up -d \
--force-recreate --no-deps backend worker beat
# Clean up disk — aggressive чтобы избежать накопления.
# 1. Для каждого gendesign-* repo оставляем 2 самых свежих SHA-тега
# + :latest. Docker images сортирует по Created DESC.
# Caddy: force-recreate чтобы подхватить изменения в Caddyfile
# И в особенности новые volume mounts из docker-compose.prod.yml
# (`reload` не пересоздаёт container, поэтому новые binds не появляются —
# был случай 2026-05-17 с PR #268 preview/ — потребовался manual SSH fix).
docker compose -p gendesign -f docker-compose.prod.yml up -d \
--force-recreate --no-deps caddy
# Forwarder: force-recreate чтобы новый image / новые env подхватывались.
# Без --force-recreate обычный `up -d` НЕ recreate'ит при image rebuild
# (т.к. image:latest tag не сменился — docker не видит разницы).
docker compose -p gendesign -f docker-compose.prod.yml up -d \
--force-recreate --no-deps glitchtip-auth-forwarder
# Cleanup old images
for repo in ghcr.io/lekss361/gendesign-backend \
ghcr.io/lekss361/gendesign-worker \
ghcr.io/lekss361/gendesign-frontend; do
@ -271,14 +287,10 @@ jobs:
| tail -n +3 \
| xargs -r docker rmi 2>/dev/null || true
done
# 2. ВСЕ unused images (даже свежие). Running контейнеры безопасны
# — их images не unused. Раньше был filter until=72h, но при
# частых деплоях оставались "fresh dangling" layers, накапливая
# до 50GB за неделю.
docker image prune -af || true
docker builder prune -af || true
# Wait for backend to be healthy (up to 30 s).
# Health check
for i in $(seq 1 30); do
curl -fsS http://localhost:8000/health && break
sleep 1

15
.github/workflows/ci.yml vendored
View file

@ -2,10 +2,23 @@ name: CI
on:
push:
branches: [main]
branches:
- main
- 'feat/**'
- 'fix/**'
- 'refactor/**'
- 'chore/**'
- 'docs/**'
- 'perf/**'
- 'test/**'
- 'hotfix/**'
pull_request:
branches: [main]
concurrency:
group: ci-${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
backend:
runs-on: ubuntu-latest

6
.gitignore vendored
View file

@ -13,6 +13,10 @@ node_modules/
.next/
frontend/.next/
out/
# Sentry CLI config (auto-generated, contains auth tokens)
.sentryclirc
frontend/.sentryclirc
# TypeScript incremental build info — must NOT be committed; locally-generated
# and changes on every build, which busts Docker build context cache.
*.tsbuildinfo
@ -23,6 +27,8 @@ out/
.env.*.local
.env.runtime
.mcp.json
# креды-заметки — НЕ коммитить (см. issue #391)
sshkey.txt
# IDE
.vscode/

142
Caddyfile
View file

@ -2,24 +2,97 @@
#
# - gendsgn.ru — main production site, auto-TLS via Let's Encrypt.
# - www.gendsgn.ru — 301 redirect to apex (canonical URL).
# - :80 — fallback for raw IP access. Will be removed once everyone uses the domain.
# - :80 — fallback for raw IP access. Closed by same auth gate (pilot phase).
#
# `handle /api/*` (NOT `handle_path`) — we keep the /api prefix because
# the FastAPI router is mounted at /api/v1/*.
#
# Auth gate: basic_auth for pilot phase (2026-05-23).
# Users managed via caddy/users.caddy.snippet (git history = audit trail).
# Public exclusions: /health (liveness probe), /preview/* (static mockups).
#
# IMPORTANT: route { } block is required to preserve directive order.
# Without route { }, Caddy executes directives in hard-coded default order
# (basic_auth runs before handle), making /health and /preview/* exclusions
# ineffective. With route { }, handlers are matched top-to-bottom as written.
{
servers {
# ВКЛЮЧАЕТ raw Authorization/Cookie headers в access log
# (по default Caddy 2.x редактирует их как "REDACTED").
# Plain password будет в gendsgn.ru.log + auth_audit.log — оба root-only на VPS.
# См. forwarder.py _extract_attempted_username для использования.
log_credentials
}
}
gendsgn.ru {
encode zstd gzip
handle /api/* {
reverse_proxy backend:8000
log {
output file /var/log/caddy/gendsgn.ru.log {
roll_size 50MiB
roll_keep 5
roll_keep_for 720h
}
format json
}
handle /health {
reverse_proxy backend:8000
# Отдельный лог только для auth-событий.
# Forwarder (ops/glitchtip-auth-forwarder) читает именно этот файл.
# Retention 7 дней (меньше чем main log) — содержит plain Base64 credentials.
log auth_audit {
output file /var/log/caddy/auth_audit.log {
roll_size 10MiB
roll_keep 3
roll_keep_for 168h
}
format json
}
handle {
reverse_proxy frontend:3000
route {
# /health и /preview/* — public, без auth, short-circuit.
handle /health {
reverse_proxy backend:8000
}
# Static HTML mockups для review (audit alternatives).
# Public access — без auth (по запросу 2026-05-17).
handle_path /preview/* {
root * /srv/preview
file_server browse
}
# Auth gate (applies to all routes below within this route block).
import caddy/users.caddy.snippet
# Trade-In MVP subproject (tradein-mvp/) — gendesign-tradein docker stack,
# подключен через gendesign_shared network. Routes ДО универсального handle
# потому что Caddy матчит handle-блоки сверху вниз.
handle /trade-in/api/* {
# `handle_path /trade-in/api/*` стрипал бы целиком /trade-in/api;
# FastAPI router замаунтен на /api/v1/trade-in/* — нужен strip только
# префикса basePath /trade-in (Next.js basePath leak).
uri strip_prefix /trade-in
reverse_proxy tradein-backend:8000
}
# Matcher `path /trade-in /trade-in/*` ловит И /trade-in (без слеша),
# И /trade-in/ + /trade-in/anything. Без обоих случаев `handle /trade-in/*`
# пропускал /trade-in без слеша → попадал в общий frontend → пустой ответ.
@tradein path /trade-in /trade-in/*
handle @tradein {
# Next.js basePath=/trade-in — фронт сам ждёт префикса в URL
reverse_proxy tradein-frontend:3000
}
handle /api/* {
reverse_proxy backend:8000
}
handle {
reverse_proxy frontend:3000
}
}
}
@ -44,20 +117,55 @@ obsidian.gendsgn.ru {
}
}
# Plain HTTP by IP — kept for ssh-tunnel / debugging.
# Caddy issues no TLS here (no hostname).
# GlitchTip — self-hosted error tracking (Sentry-compatible).
# DNS: A-record errors.gendsgn.ru → IP VPS.
errors.gendsgn.ru {
encode zstd gzip
reverse_proxy glitchtip-web:8080
log {
output file /var/log/caddy/errors.gendsgn.ru.log
}
}
# Forgejo — self-hosted git (migration 2026-05-16).
# DNS: A-record git.gendsgn.ru → IP VPS.
# Forgejo container из forgejo-migration/docker-compose.yml на shared
# gendesign_default network. HTTP port 3000 (default Forgejo).
# Был добавлен вручную при migration, потерян при первом auto-deploy после
# изменения Caddyfile (deploy.yml делает git reset --hard). См. fix issue.
git.gendsgn.ru {
encode zstd gzip
reverse_proxy forgejo:3000
log {
output file /var/log/caddy/git.gendsgn.ru.log
}
}
# Plain HTTP by IP — closed by same auth gate (prevent bypass via direct IP / SSH tunnel).
# Caddy issues no TLS here (no hostname). /health remains public.
:80 {
encode zstd gzip
handle /api/* {
reverse_proxy backend:8000
}
route {
# /health — public, без auth (GHA deploy smoke check, liveness probe).
handle /health {
reverse_proxy backend:8000
}
handle /health {
reverse_proxy backend:8000
}
# Auth gate (same snippet as gendsgn.ru).
import caddy/users.caddy.snippet
handle {
reverse_proxy frontend:3000
handle /api/* {
reverse_proxy backend:8000
}
handle {
reverse_proxy frontend:3000
}
}
}
# Test deploy flow 2026-05-15T21:43:32Z

13
backend/.env.example
View file

@ -6,8 +6,12 @@ REDIS_URL=redis://redis:6379/0
CORS_ORIGINS=["http://localhost:3000"]
ENVIRONMENT=dev
# Sentry: create a project at https://sentry.io/, paste the DSN here.
# Empty string = Sentry disabled. main.py initializes only if non-empty.
# DSN для GlitchTip (https://errors.gendsgn.ru). Пустое = SDK no-op.
# Формат: https://<key>@errors.gendsgn.ru/<project_id>
GLITCHTIP_DSN=
GLITCHTIP_TRACES_SAMPLE_RATE=0.05
# DEPRECATED — будет удалён после 1-2 deploy-циклов. Использовать GLITCHTIP_DSN.
# Если задан, а GLITCHTIP_DSN пуст — автоматически продвигается с DeprecationWarning.
SENTRY_DSN=
# External APIs (Stage 2)
@ -23,6 +27,7 @@ SCRAPE_KN_JITTER_SECONDS=1800
SCRAPE_KN_DEFAULT_REGIONS=66
# Путь к Playwright storage_state.json (commited в git, обновляется --save-state).
SCRAPE_KN_STATE_PATH=data/playwright_state.json
# Токен для POST /api/v1/admin/scrape/kn (заголовок X-Admin-Token).
# Пустая строка = endpoint disabled (503). Сгенерировать: openssl rand -hex 32
# DEPRECATED 2026-05-23: app-level admin auth removed (PR #436, Caddy basic_auth достаточен).
# Reinstate: revert changes in admin_*.py чтобы вернуть AdminTokenAuth dep.
# Переменная сохранена в core/deps.py для быстрого rollback.
SCRAPE_ADMIN_TOKEN=

9
backend/app/api/v1/admin_cadastre.py
View file

@ -11,6 +11,9 @@ Scope variants:
pilot — первые 50 кварталов ЕКБ (66:41:%)
ekb_full — все ~2408 кварталов ЕКБ
manual_list — явный список quarters в body.quarters
Auth: gendsgn.ru-wide Caddy basic_auth gate (PR #426). App-level X-Admin-Token
header removed 2026-05-23 — двойная auth избыточна для pilot.
"""
from __future__ import annotations
@ -25,7 +28,6 @@ from sqlalchemy import text
from sqlalchemy.orm import Session
from app.core.db import get_db
from app.core.deps import AdminTokenAuth
logger = logging.getLogger(__name__)
@ -185,7 +187,6 @@ def _serialize_job(row: Any) -> dict[str, Any]:
def create_cadastre_job(
body: CreateCadastreJobRequest,
db: Annotated[Session, Depends(get_db)],
_: AdminTokenAuth,
) -> dict[str, Any]:
"""Создать bulk cadastre harvest job и поставить в очередь.
@ -234,7 +235,6 @@ def create_cadastre_job(
@router.get("/jobs")
def list_cadastre_jobs(
db: Annotated[Session, Depends(get_db)],
_: AdminTokenAuth,
limit: int = 30,
) -> list[dict[str, Any]]:
"""Список последних cadastre harvest jobs."""
@ -262,7 +262,6 @@ def list_cadastre_jobs(
def get_cadastre_job(
job_id: int,
db: Annotated[Session, Depends(get_db)],
_: AdminTokenAuth,
) -> dict[str, Any]:
"""Детали одного cadastre harvest job."""
row = (
@ -297,7 +296,6 @@ def get_cadastre_job(
def cancel_cadastre_job(
job_id: int,
db: Annotated[Session, Depends(get_db)],
_: AdminTokenAuth,
) -> dict[str, Any]:
"""Отменить job. Воркеры увидят status='cancelled' при следующей итерации и skip."""
result = db.execute(
@ -327,7 +325,6 @@ def cancel_cadastre_job(
def resume_cadastre_job(
job_id: int,
db: Annotated[Session, Depends(get_db)],
_: AdminTokenAuth,
) -> dict[str, Any]:
"""Возобновить paused/failed job. Re-enqueue enqueue_cadastre_harvest."""
from app.workers.tasks.scrape_cadastre import enqueue_cadastre_harvest

126
backend/app/api/v1/admin_etl.py Normal file
View file

@ -0,0 +1,126 @@
"""Admin endpoints для ETL operations (#203, #44).
POST /api/v1/admin/etl/objective-backfill
Запустить fuzzy-match backfill objective_complex_mapping.
Поддерживает два режима:
- v1 (default): threshold=0.85, match_method='fuzzy_trgm'
- v2: threshold=0.80, match_method='fuzzy_v2' — для coverage 8.5%→17%
POST /api/v1/admin/etl/nspd-denorm-backfill
Запустить backfill nspd_parcels/nspd_buildings из всех nspd_quarter_dumps.
Auth: gendsgn.ru-wide Caddy basic_auth gate (PR #426). App-level X-Admin-Token
header removed 2026-05-23 — двойная auth избыточна для pilot.
"""
from __future__ import annotations
import logging
from typing import Annotated
from fastapi import APIRouter, Depends, Query
from sqlalchemy.orm import Session
from app.core.db import get_db
from app.services.etl.objective_backfill import (
AUTO_ACCEPT_THRESHOLD,
AUTO_ACCEPT_THRESHOLD_V2,
REVIEW_THRESHOLD,
auto_apply_matches,
find_match_candidates,
trigger_mv_refresh,
)
logger = logging.getLogger(__name__)
router = APIRouter()
@router.post("/objective-backfill")
def run_objective_backfill(
db: Annotated[Session, Depends(get_db)],
dry_run: Annotated[bool, Query(description="Preview без insertions")] = False,
refresh_mv: Annotated[
bool, Query(description="REFRESH mv_layout_velocity после backfill")
] = True,
v2: Annotated[
bool,
Query(
description=(
"v2 mode: threshold=0.80, match_method='fuzzy_v2'. "
"Запускать после v1 run — только для unmapped объектов."
)
),
] = False,
) -> dict[str, object]:
"""Запустить backfill objective_complex_mapping + опционально REFRESH MV.
Ищет DOM.РФ комплексы (is_ekb=true) без mapping и применяет fuzzy match
к project_name из objective_corpus_room_month через pg_trgm similarity.
Режим v1 (default, ?v2=false):
- score >= 0.85 (AUTO_ACCEPT_THRESHOLD): auto-insert, match_method='fuzzy_trgm'
- score >= 0.6 (REVIEW_THRESHOLD) и < 0.85: только счётчик review_queue
Режим v2 (?v2=true, task #44 coverage expansion):
- score >= 0.80 (AUTO_ACCEPT_THRESHOLD_V2): auto-insert, match_method='fuzzy_v2'
- is_reviewed=false — требует ручной проверки (false positives вероятны)
- Целевой прирост: +~47-80 строк, coverage 8.5% → ~17%
Returns dict:
auto_accepted: сколько строк вставлено
review_queue: сколько кандидатов ниже порога auto-accept
skipped: ON CONFLICT + ошибки
mv_rows_after_refresh: строк в MV после REFRESH (0 если refresh_mv=False)
threshold_used: фактический порог (float)
match_method_used: match_method в БД (str)
"""
threshold = AUTO_ACCEPT_THRESHOLD_V2 if v2 else AUTO_ACCEPT_THRESHOLD
method = "fuzzy_v2" if v2 else "fuzzy_trgm"
# v2 ищет кандидатов начиная с threshold (не от REVIEW_THRESHOLD)
search_min = threshold if v2 else REVIEW_THRESHOLD
candidates = find_match_candidates(db, only_unmapped=True, min_threshold=search_min)
logger.info(
"Backfill candidates found: %d (score >= %.2f, method=%s)",
len(candidates),
search_min,
method,
)
result: dict[str, object] = dict(
auto_apply_matches(
db, candidates, threshold=threshold, match_method=method, dry_run=dry_run
)
)
mv_rows = 0
if refresh_mv and not dry_run and result.get("auto_accepted", 0):
mv_rows = trigger_mv_refresh(db)
logger.info("mv_layout_velocity refreshed after backfill: %d rows", mv_rows)
result["mv_rows_after_refresh"] = mv_rows
result["threshold_used"] = threshold
result["match_method_used"] = method
return result
@router.post("/nspd-denorm-backfill")
def run_nspd_denorm_backfill(
limit: Annotated[
int | None,
Query(description="Максимум кварталов для обработки (None = все)"),
] = None,
) -> dict[str, object]:
"""Запустить Celery backfill: denormalize nspd_quarter_dumps → nspd_parcels/nspd_buildings.
Задача идемпотентна (ON CONFLICT DO UPDATE). Безопасно запускать повторно.
Возвращает Celery task_id — статус через /flower или celery inspect.
Args:
limit: если задан — обработать только первые N кварталов ORDER BY quarter_cad.
"""
from app.workers.tasks.nspd_denorm_backfill import backfill_all_dumps
task = backfill_all_dumps.apply_async(kwargs={"limit": limit})
logger.info("nspd-denorm-backfill enqueued: task_id=%s limit=%s", task.id, limit)
return {"task_id": task.id, "status": "enqueued", "limit": limit}

7
backend/app/api/v1/admin_jobs.py
View file

@ -4,7 +4,8 @@ GET /api/v1/admin/jobs/settings — список всех job_settings
GET /api/v1/admin/jobs/settings/{type} — одна строка по job_type
PUT /api/v1/admin/jobs/settings/{type} — PATCH-style update
Auth: X-Admin-Token header (тот же токен что у admin_scrape endpoints).
Auth: gendsgn.ru-wide Caddy basic_auth gate (PR #426). App-level X-Admin-Token
header removed 2026-05-23 — двойная auth избыточна для pilot.
"""
from __future__ import annotations
@ -15,7 +16,6 @@ from fastapi import APIRouter, Depends
from sqlalchemy.orm import Session
from app.core.db import get_db
from app.core.deps import AdminTokenAuth
from app.schemas.job_settings import JobSettingRead, JobSettingUpdate
router = APIRouter()
@ -24,7 +24,6 @@ router = APIRouter()
@router.get("/settings", response_model=list[JobSettingRead])
def list_job_settings(
db: Annotated[Session, Depends(get_db)],
_: AdminTokenAuth,
) -> Any:
"""Список всех job_settings (все job_type)."""
from app.services.job_settings import get_all
@ -36,7 +35,6 @@ def list_job_settings(
def get_job_setting(
job_type: str,
db: Annotated[Session, Depends(get_db)],
_: AdminTokenAuth,
) -> Any:
"""Настройки одного job_type. Возвращает fallback defaults если строки нет в БД."""
from app.services.job_settings import get_one
@ -49,7 +47,6 @@ def update_job_setting(
job_type: str,
payload: JobSettingUpdate,
db: Annotated[Session, Depends(get_db)],
_: AdminTokenAuth,
) -> Any:
"""PATCH-style update: передавать только изменяемые поля.

10
backend/app/api/v1/admin_leads.py
View file

@ -3,7 +3,8 @@
GET /api/v1/admin/leads — list with filters + pagination
GET /api/v1/admin/leads/stats — KPI summary (total/conversion/revenue)
Auth: same X-Admin-Token as /admin/scrape (settings.scrape_admin_token).
Auth: gendsgn.ru-wide Caddy basic_auth gate (PR #426). App-level X-Admin-Token
header removed 2026-05-23 — двойная auth избыточна для pilot.
PII compliance: only phone_last4 and email_hash exposed; raw phone/email
were never persisted (см. data/sql/52_import_prinzip_crm.py).
"""
@ -17,7 +18,6 @@ from sqlalchemy import text
from sqlalchemy.orm import Session
from app.core.db import get_db
from app.core.deps import AdminTokenAuth
from app.services import analytics_queries as q
router = APIRouter()
@ -26,7 +26,6 @@ router = APIRouter()
@router.get("/")
def list_leads(
db: Annotated[Session, Depends(get_db)],
_: AdminTokenAuth,
status: Annotated[str | None, Query()] = None,
source: Annotated[str | None, Query()] = None,
converted: Annotated[bool | None, Query()] = None,
@ -129,7 +128,6 @@ def list_leads(
@router.get("/stats")
def leads_stats(
db: Annotated[Session, Depends(get_db)],
_: AdminTokenAuth,
months: Annotated[int, Query(ge=1, le=120)] = 12,
) -> dict[str, Any]:
"""KPI summary за последние N месяцев."""
@ -190,7 +188,6 @@ def leads_stats(
@router.get("/funnel/monthly")
def funnel_monthly(
db: Annotated[Session, Depends(get_db)],
_: AdminTokenAuth,
months: Annotated[int, Query(ge=1, le=120)] = 24,
) -> list[dict[str, Any]]:
"""Воронка по месяцам: leads → engaged → converted (по source)."""
@ -200,7 +197,6 @@ def funnel_monthly(
@router.get("/funnel/by-source")
def funnel_by_source(
db: Annotated[Session, Depends(get_db)],
_: AdminTokenAuth,
months: Annotated[int, Query(ge=1, le=120)] = 12,
) -> list[dict[str, Any]]:
"""Splitting по source: кто конвертит лучше за последние N месяцев."""
@ -210,7 +206,6 @@ def funnel_by_source(
@router.get("/funnel/by-object")
def funnel_by_object(
db: Annotated[Session, Depends(get_db)],
_: AdminTokenAuth,
) -> list[dict[str, Any]]:
"""Воронка для каждого ЖК PRINZIP: leads / deals / revenue."""
return q.prinzip_funnel_by_object(db)
@ -219,7 +214,6 @@ def funnel_by_object(
@router.get("/sources")
def list_sources(
db: Annotated[Session, Depends(get_db)],
_: AdminTokenAuth,
) -> list[str]:
"""Distinct list of source values for filter dropdown."""
rows = db.execute(

125
backend/app/api/v1/admin_scrape.py
View file

@ -2,9 +2,10 @@
POST /api/v1/admin/scrape/kn
Body: {"region_code": 66, "developers": ["6208_0"]?, "fetch_flats": true?}
Header: X-Admin-Token: <SCRAPE_ADMIN_TOKEN env var>
Disabled if SCRAPE_ADMIN_TOKEN is empty (default).
Auth: gendsgn.ru-wide Caddy basic_auth gate (PR #426). App-level X-Admin-Token
header removed 2026-05-23 — двойная auth избыточна для pilot.
Returns the Celery task id; poll /api/v1/admin/scrape/runs/{run_id} to track DB-side progress.
"""
@ -19,7 +20,6 @@ from sqlalchemy.orm import Session
from app.core.config import settings
from app.core.db import get_db
from app.core.deps import AdminTokenAuth
router = APIRouter()
@ -38,7 +38,6 @@ class TriggerKnRequest(BaseModel):
@router.post("/kn")
def trigger_kn_sweep(
payload: TriggerKnRequest,
_: AdminTokenAuth,
) -> dict[str, Any]:
# Defer Celery import so the API works without a broker in dev.
from app.workers.tasks.scrape_kn import (
@ -72,7 +71,6 @@ def trigger_kn_sweep(
@router.get("/queue")
def queue_status(
db: Annotated[Session, Depends(get_db)],
_: AdminTokenAuth,
) -> dict[str, Any]:
"""Snapshot of Celery state, optimised for UI poll latency.
@ -217,7 +215,6 @@ class ReleaseLockRequest(BaseModel):
@router.post("/release-lock")
def release_lock(
payload: ReleaseLockRequest,
_: AdminTokenAuth,
) -> dict[str, Any]:
"""Принудительно удалить Redis-lock для (region, devs). Использовать когда
зомби-worker оставил lock и новые задачи скипаются с reason='lock_held'."""
@ -239,7 +236,6 @@ class RevokeRequest(BaseModel):
@router.post("/revoke")
def revoke_task(
payload: RevokeRequest,
_: AdminTokenAuth,
) -> dict[str, Any]:
"""Cancel a queued or running task by id."""
from app.workers.celery_app import celery_app
@ -251,7 +247,6 @@ def revoke_task(
@router.get("/failures")
def list_failures(
db: Annotated[Session, Depends(get_db)],
_: AdminTokenAuth,
run_id: int | None = None,
limit: int = 50,
) -> list[dict[str, Any]]:
@ -297,7 +292,6 @@ def list_failures(
@router.get("/logs")
def list_logs(
db: Annotated[Session, Depends(get_db)],
_: AdminTokenAuth,
run_id: int | None = None,
since_id: int | None = None,
limit: int = 200,
@ -353,9 +347,7 @@ def list_logs(
@router.post("/noise-sync")
def trigger_noise_sync(
_: AdminTokenAuth,
) -> dict[str, Any]:
def trigger_noise_sync() -> dict[str, Any]:
"""Manual trigger для синхронизации OSM шумовых источников ЕКБ из Overpass API.
Обычно запускается еженедельно через beat (понед 3:30 МСК).
@ -385,7 +377,6 @@ class HarvestQuarterRequest(BaseModel):
@router.post("/nspd/harvest-quarter")
def trigger_harvest_quarter(
payload: HarvestQuarterRequest,
_: AdminTokenAuth,
) -> dict[str, Any]:
"""Manual trigger одного quarter harvest (для testing / конкретного квартала).
@ -412,9 +403,7 @@ def trigger_harvest_quarter(
@router.post("/pzz-sync")
def trigger_pzz_sync(
_: AdminTokenAuth,
) -> dict[str, Any]:
def trigger_pzz_sync() -> dict[str, Any]:
"""Manual trigger для импорта ПЗЗ территориальных зон ЕКБ из Росреестр PKK6.
Запускать после деплоя миграции 85_pzz_zones_ekb.sql и при необходимости
@ -427,9 +416,7 @@ def trigger_pzz_sync(
@router.post("/poi-sync")
def trigger_poi_sync(
_: AdminTokenAuth,
) -> dict[str, Any]:
def trigger_poi_sync() -> dict[str, Any]:
"""Manual trigger для синхронизации OSM POI ЕКБ из Overpass API.
Обычно запускается еженедельно через beat (понед 3:00 МСК).
@ -452,7 +439,6 @@ class TriggerObjectiveEtlRequest(BaseModel):
@router.post("/objective")
def trigger_objective_etl(
payload: TriggerObjectiveEtlRequest,
_: AdminTokenAuth,
) -> dict[str, Any]:
"""Запустить ETL Антоновского SQLite → наша PG.
@ -477,7 +463,6 @@ def trigger_objective_etl(
@router.get("/objective/runs")
def list_objective_runs(
db: Annotated[Session, Depends(get_db)],
_: AdminTokenAuth,
limit: int = 20,
) -> list[dict[str, Any]]:
rows = (
@ -535,7 +520,6 @@ class TriggerObjectiveSyncRequest(BaseModel):
@router.post("/objective/sync-our")
def trigger_objective_sync_our(
payload: TriggerObjectiveSyncRequest,
_: AdminTokenAuth,
) -> dict[str, Any]:
"""Запустить НАШ sync_all_groups (тратит limits Объектива).
@ -584,7 +568,6 @@ class ObjectiveConfigUpdateRequest(BaseModel):
@router.get("/objective/config")
def get_objective_config(
db: Annotated[Session, Depends(get_db)],
_: AdminTokenAuth,
) -> dict[str, Any]:
"""Текущая динамическая конфигурация Objective sync (single row)."""
from app.services.objective_sync_config import get_config
@ -596,7 +579,6 @@ def get_objective_config(
def update_objective_config(
payload: ObjectiveConfigUpdateRequest,
db: Annotated[Session, Depends(get_db)],
_: AdminTokenAuth,
) -> dict[str, Any]:
"""PATCH-style update. После изменения cron_schedule нужен restart beat,
остальные поля применяются на следующем sync-вызове автоматически."""
@ -626,7 +608,6 @@ def update_objective_config(
@router.get("/objective/coverage")
def objective_coverage(
db: Annotated[Session, Depends(get_db)],
_: AdminTokenAuth,
) -> dict[str, Any]:
"""Что у нас в БД сейчас + что лежит в SQLite Антона (size, last modified).
@ -693,7 +674,6 @@ class EnqueueGeoJobRequest(BaseModel):
def enqueue_geo_job(
payload: EnqueueGeoJobRequest,
db: Annotated[Session, Depends(get_db)],
_: AdminTokenAuth,
) -> dict[str, Any]:
"""Создать bulk geo-job и поставить в очередь.
@ -874,7 +854,6 @@ def _collect_all_in_region(
def bulk_enqueue_geo(
payload: BulkGeoEnqueueRequest,
db: Annotated[Session, Depends(get_db)],
_: AdminTokenAuth,
) -> dict[str, Any]:
"""Разбить pending cad-номера на N чанков и запустить N×len(thematic_ids) geo-jobs.
@ -966,7 +945,6 @@ def bulk_enqueue_geo(
@router.get("/geo/jobs")
def list_geo_jobs(
db: Annotated[Session, Depends(get_db)],
_: AdminTokenAuth,
limit: int = 30,
) -> list[dict[str, Any]]:
"""Список последних geo-jobs (для UI dashboard)."""
@ -1021,7 +999,6 @@ def list_geo_jobs(
def cancel_geo_job(
job_id: int,
db: Annotated[Session, Depends(get_db)],
_: AdminTokenAuth,
) -> dict[str, Any]:
"""Пометить job как cancelled. Worker увидит при следующей итерации."""
db.execute(
@ -1042,7 +1019,6 @@ def cancel_geo_job(
def resume_geo_job(
job_id: int,
db: Annotated[Session, Depends(get_db)],
_: AdminTokenAuth,
) -> dict[str, Any]:
"""Re-enqueue paused/failed job. Resume idempotent через pending targets."""
from app.services.job_settings import get_setting_value
@ -1064,7 +1040,6 @@ def resume_geo_job(
@router.get("/all/runs")
def list_all_runs(
db: Annotated[Session, Depends(get_db)],
_: AdminTokenAuth,
scraper_type: str | None = None,
limit: int = 30,
) -> list[dict[str, Any]]:
@ -1122,7 +1097,6 @@ def list_all_runs(
@router.get("/all/logs")
def list_all_logs(
db: Annotated[Session, Depends(get_db)],
_: AdminTokenAuth,
scraper_type: str | None = None,
run_id: int | None = None,
limit: int = 200,
@ -1174,7 +1148,6 @@ def list_all_logs(
@router.get("/runs")
def list_runs(
db: Annotated[Session, Depends(get_db)],
_: AdminTokenAuth,
limit: int = 20,
) -> list[dict[str, Any]]:
rows = (
@ -1209,3 +1182,89 @@ def list_runs(
}
for r in rows
]
# ── Ekburg construction permits (РНС/РВЭ ЕКБ) ────────────────────────────────
class TriggerEkburgPermitsRequest(BaseModel):
year: int | None = Field(
default=None,
ge=2022,
le=2030,
description=(
"Если None — refresh_all (все 5 лет 2022-2026). "
"Иначе refresh_year(year) для конкретного года."
),
)
@router.post("/ekburg-permits")
def trigger_ekburg_permits(
payload: TriggerEkburgPermitsRequest,
) -> dict[str, Any]:
"""Manual trigger для синхронизации РНС/РВЭ ЕКБ из екатеринбург.рф (xlsx).
Обычно запускается раз в месяц через beat (1-го числа 02:00 UTC).
Этот endpoint — для ad-hoc запуска (после первого деплоя или для smoke-теста).
- year=None → refresh_all: скачивает 5 xlsx (2022-2026), ~3-8 мин.
- year=2026 → refresh_year(2026): один файл, ~1 мин.
"""
from app.workers.tasks.ekburg_permits_sync import refresh_all, refresh_year
if payload.year is None:
result = refresh_all.apply_async()
scope = "all_years_2022_2026"
else:
result = refresh_year.apply_async(args=[payload.year])
scope = f"year_{payload.year}"
return {"task_id": result.id, "scope": scope, "queued_at": "now"}
class TriggerKnCatalogObjectsRequest(BaseModel):
region_code: int = Field(default=66, ge=1, le=99)
max_objects: int | None = Field(default=None, ge=1, le=2000)
force: bool = Field(
default=False,
description=(
"True — игнорировать фильтр 'уже сегодня обновлён' и грузить ВСЕ объекты "
"последнего snapshot ('Загрузить все'). По умолчанию пропускает то, "
"что уже скраплено сегодня."
),
)
@router.post("/kn-catalog-objects")
def trigger_kn_catalog_objects(
payload: TriggerKnCatalogObjectsRequest,
) -> dict[str, Any]:
"""Manual trigger для catalog-OBJECT scraper (заполняет wall_type, energy_eff,
ceiling_height_m, parking_*, playground_*, scores из SSR __NEXT_DATA__).
Beat schedule: Tuesday 04:00 UTC, batch 300/run. Этот endpoint — для ad-hoc
запуска (smoke-тест после деплоя или повторный pass для свежесозданных
объектов до next beat fire).
- max_objects=None → дефолтный лимит таска (300).
- max_objects=3 → smoke-тест.
- force=True → "Загрузить все": игнорирует skip-today, грузит всё подряд.
"""
from app.workers.tasks.scrape_kn_catalog_objects import scrape_kn_catalog_objects
kwargs: dict[str, Any] = {
"region_code": payload.region_code,
"force": payload.force,
}
if payload.max_objects is not None:
kwargs["max_objects"] = payload.max_objects
result = scrape_kn_catalog_objects.apply_async(kwargs=kwargs)
return {
"task_id": result.id,
"region_code": payload.region_code,
"max_objects": payload.max_objects,
"force": payload.force,
"queued_at": "now",
}

24
backend/app/api/v1/admin_weight_profiles.py
View file

@ -1,6 +1,9 @@
"""Admin endpoints для управления weight profiles.
Per #114 (Макс feedback: садики ≠ Мегамарт). Endpoints под X-Admin-Token.
Per #114 (Макс feedback: садики ≠ Мегамарт).
Auth: gendsgn.ru-wide Caddy basic_auth gate (PR #426). App-level X-Admin-Token
header removed 2026-05-23 — двойная auth избыточна для pilot.
GET /api/v1/admin/site-finder/weight-profiles?user_id=... → list
POST /api/v1/admin/site-finder/weight-profiles → create
@ -17,7 +20,6 @@ from fastapi import APIRouter, Depends, HTTPException, Query, status
from sqlalchemy.orm import Session
from app.core.db import get_db
from app.core.deps import AdminTokenAuth
from app.services.site_finder.weight_profiles import (
WeightProfile,
WeightProfileCreate,
@ -26,6 +28,7 @@ from app.services.site_finder.weight_profiles import (
delete_profile,
get_profile,
list_profiles,
list_profiles_with_system,
update_profile,
)
@ -36,9 +39,18 @@ router = APIRouter()
def list_user_profiles(
user_id: Annotated[str, Query(min_length=1, description="user_id для фильтра профилей")],
db: Annotated[Session, Depends(get_db)],
_: AdminTokenAuth,
include_system: Annotated[
bool,
Query(description="Включить системные preset-профили (Эконом/Комфорт/Бизнес)"),
] = False,
) -> Any:
"""Список всех weight profiles для заданного user_id. Default-профиль первым."""
"""Список всех weight profiles для заданного user_id. Default-профиль первым.
При include_system=true добавляются системные presets (user_id='__system__'):
Эконом, Комфорт, Бизнес — готовые шаблоны весов POI.
"""
if include_system:
return list_profiles_with_system(db, user_id)
return list_profiles(db, user_id)
@ -46,7 +58,6 @@ def list_user_profiles(
def create_user_profile(
payload: WeightProfileCreate,
db: Annotated[Session, Depends(get_db)],
_: AdminTokenAuth,
) -> Any:
"""Создать новый weight profile.
@ -61,7 +72,6 @@ def get_user_profile(
profile_id: int,
user_id: Annotated[str, Query(min_length=1, description="Владелец профиля")],
db: Annotated[Session, Depends(get_db)],
_: AdminTokenAuth,
) -> Any:
"""Получить один weight profile по id (scoped к user_id)."""
profile = get_profile(db, user_id, profile_id)
@ -76,7 +86,6 @@ def update_user_profile(
user_id: Annotated[str, Query(min_length=1, description="Владелец профиля")],
payload: WeightProfileUpdate,
db: Annotated[Session, Depends(get_db)],
_: AdminTokenAuth,
) -> Any:
"""PATCH-style обновление weight profile.
@ -94,7 +103,6 @@ def delete_user_profile(
profile_id: int,
user_id: Annotated[str, Query(min_length=1, description="Владелец профиля")],
db: Annotated[Session, Depends(get_db)],
_: AdminTokenAuth,
) -> None:
"""Удалить weight profile. 404 если не найден."""
deleted = delete_profile(db, user_id, profile_id)

39
backend/app/api/v1/analytics.py
View file

@ -160,6 +160,45 @@ def object_detail(
return result
@router.get("/object/{obj_id}/full")
def object_full_detail(
db: Annotated[Session, Depends(get_db)],
obj_id: int,
) -> dict[str, Any]:
"""Extended object info — все 30+ новых полей (22begh: specs/accessibility/metro/scores)."""
result = q.object_full_detail(db, obj_id=obj_id)
if result is None:
raise HTTPException(status_code=404, detail="object not found")
return result
@router.get("/object/{obj_id}/quartirography")
def object_quartirography(
db: Annotated[Session, Depends(get_db)],
obj_id: int,
) -> list[dict[str, Any]]:
"""Квартирография ЖК: группировка flats по комнатности с count/area/price."""
return q.object_flats_quartirography(db, obj_id=obj_id)
@router.get("/object/{obj_id}/checks")
def object_checks(
db: Annotated[Session, Depends(get_db)],
obj_id: int,
) -> list[dict[str, Any]]:
"""6 проверок «Проверено на наш.дом.рф» (22f)."""
return q.object_obj_checks(db, obj_id=obj_id)
@router.get("/object/{obj_id}/documents")
def object_documents(
db: Annotated[Session, Depends(get_db)],
obj_id: int,
) -> list[dict[str, Any]]:
"""PDF документы из domrf_kn_documents (22i): декларации, разрешения, отчётность."""
return q.object_documents(db, obj_id=obj_id)
@router.get("/object/{obj_id}/buildings", response_model=list[ComplexBuilding])
def object_buildings(
db: Annotated[Session, Depends(get_db)],

119
backend/app/api/v1/custom_pois.py Normal file
View file

@ -0,0 +1,119 @@
"""CRUD API для user_custom_pois (#254).
POST /api/v1/custom-pois → 201 CustomPoiOut
GET /api/v1/custom-pois → list[CustomPoiOut]
PATCH /api/v1/custom-pois/{id} → CustomPoiOut
DELETE /api/v1/custom-pois/{id} → 204
Auth: anonymous user_id из header X-Session-Id (workaround до #67 NextAuth).
Если заголовок отсутствует — генерируем UUID и возвращаем в X-Session-Id response header.
Frontend сохраняет в localStorage.
"""
from __future__ import annotations
import logging
import uuid
from typing import Annotated, Any
from fastapi import APIRouter, Depends, Header, HTTPException, Query, Response, status
from sqlalchemy.orm import Session
from app.core.db import get_db
from app.schemas.custom_poi import CustomPoiCreate, CustomPoiOut, CustomPoiUpdate
from app.services.site_finder.custom_pois import (
create_custom_poi,
delete_custom_poi,
list_custom_pois,
update_custom_poi,
)
logger = logging.getLogger(__name__)
router = APIRouter()
def _resolve_session_id(
x_session_id: str | None,
response: Response,
) -> str:
"""Вернуть user_id из X-Session-Id header или сгенерировать UUID.
При генерации устанавливаем X-Session-Id в response header чтобы
frontend мог сохранить значение в localStorage.
"""
if x_session_id and x_session_id.strip():
return x_session_id.strip()
generated = str(uuid.uuid4())
response.headers["X-Session-Id"] = generated
logger.debug("generated session_id=%s (no X-Session-Id header)", generated)
return generated
@router.post("", response_model=CustomPoiOut, status_code=status.HTTP_201_CREATED)
def create_poi(
payload: CustomPoiCreate,
db: Annotated[Session, Depends(get_db)],
response: Response,
x_session_id: Annotated[str | None, Header()] = None,
) -> Any:
"""Создать кастомную POI точку.
Возвращает 201 с созданным объектом. X-Session-Id response header содержит
актуальный session_id (пригодится frontend для localStorage).
"""
user_id = _resolve_session_id(x_session_id, response)
poi = create_custom_poi(db, user_id, payload)
# Echo back session_id чтобы frontend всегда имел его в response
response.headers["X-Session-Id"] = user_id
return poi
@router.get("", response_model=list[CustomPoiOut])
def list_pois(
db: Annotated[Session, Depends(get_db)],
response: Response,
x_session_id: Annotated[str | None, Header()] = None,
parcel_cad: Annotated[
str | None,
Query(description="Фильтр по кадастровому номеру участка (возвращает global + parcel POI)"),
] = None,
) -> Any:
"""Список кастомных POI пользователя.
Если parcel_cad задан — возвращает global (parcel_cad IS NULL) + parcel-specific POI.
"""
user_id = _resolve_session_id(x_session_id, response)
response.headers["X-Session-Id"] = user_id
return list_custom_pois(db, user_id, parcel_cad)
@router.patch("/{poi_id}", response_model=CustomPoiOut)
def patch_poi(
poi_id: int,
payload: CustomPoiUpdate,
db: Annotated[Session, Depends(get_db)],
response: Response,
x_session_id: Annotated[str | None, Header()] = None,
) -> Any:
"""Обновить поля кастомной POI (PATCH-style). 404 если не найдена."""
user_id = _resolve_session_id(x_session_id, response)
response.headers["X-Session-Id"] = user_id
poi = update_custom_poi(db, poi_id, user_id, payload)
if poi is None:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Custom POI not found")
return poi
@router.delete("/{poi_id}", status_code=status.HTTP_204_NO_CONTENT)
def delete_poi(
poi_id: int,
db: Annotated[Session, Depends(get_db)],
response: Response,
x_session_id: Annotated[str | None, Header()] = None,
) -> None:
"""Удалить кастомную POI. 404 если не найдена."""
user_id = _resolve_session_id(x_session_id, response)
deleted = delete_custom_poi(db, poi_id, user_id)
if not deleted:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Custom POI not found")

148
backend/app/api/v1/landing.py Normal file
View file

@ -0,0 +1,148 @@
"""Landing page stats endpoint — 5 KPI для hero-секции."""
from __future__ import annotations
import logging
from datetime import date
from typing import Annotated, Any
from fastapi import APIRouter, Depends
from pydantic import BaseModel
from sqlalchemy import text
from sqlalchemy.orm import Session
from app.core.db import get_db
logger = logging.getLogger(__name__)
router = APIRouter()
# ---------------------------------------------------------------------------
# Response schema
# ---------------------------------------------------------------------------
class LandingStatsOut(BaseModel):
zk_total: int
deals_total: int
price_coverage_pct: float
mapping_coverage_pct: float
last_data_update: str
paradox: str
stale: bool = False
# ---------------------------------------------------------------------------
# Fallback defaults (на случай пустой dev БД или ошибки запроса)
# ---------------------------------------------------------------------------
_FALLBACK_DATA: dict[str, Any] = {
"zk_total": 0,
"deals_total": 0,
"price_coverage_pct": 0.0,
"mapping_coverage_pct": 0.0,
"last_data_update": "",
"paradox": "",
"stale": True,
}
def _query_stats(db: Session) -> dict[str, Any] | None:
"""Выполняет агрегирующие запросы к БД. Возвращает None при любой ошибке."""
try:
# KPI 1: COUNT ЖК DOM.РФ (ЕКБ)
row_zk = db.execute(
text("SELECT COUNT(*) FROM domrf_kn_objects WHERE is_ekb = TRUE")
).scalar()
zk_total = int(row_zk or 0)
# KPI 2: COUNT ДДУ-сделок (через pg_class для партицированной таблицы)
row_deals = db.execute(
text(
"SELECT COALESCE(SUM(CAST(reltuples AS bigint)), 0)"
" FROM pg_class"
" WHERE relname LIKE 'rosreestr_deals_%'"
" AND relkind = 'r'"
)
).scalar()
deals_total = int(row_deals or 0)
# KPI 3: % objective_lots с ценой
row_price = db.execute(
text(
"SELECT"
" COUNT(*) FILTER (WHERE price_per_m2_rub IS NOT NULL) * 100.0"
" / NULLIF(COUNT(*), 0)"
" FROM objective_lots"
)
).scalar()
price_coverage_pct = round(float(row_price or 0.0), 1)
# KPI 4: % mapping (objective_complex_mapping / domrf_kn_objects ekb)
row_mapping = db.execute(
text(
"SELECT"
" (SELECT COUNT(*) FROM objective_complex_mapping) * 100.0"
" / NULLIF("
" (SELECT COUNT(*) FROM domrf_kn_objects WHERE is_ekb = TRUE),"
" 0"
" )"
)
).scalar()
mapping_coverage_pct = round(float(row_mapping or 0.0), 1)
# KPI 5: дата последнего обновления (MAX snapshot_date из objective_lots + domrf)
row_date_obj = db.execute(text("SELECT MAX(snapshot_date) FROM objective_lots")).scalar()
row_date_domrf = db.execute(
text("SELECT MAX(snapshot_date) FROM domrf_kn_objects")
).scalar()
dates = [d for d in (row_date_obj, row_date_domrf) if d is not None]
last_data_update: str
if dates:
last_data_update = str(max(dates))
else:
last_data_update = str(date.today())
# Если все KPI нули — скорее всего пустая БД → stale
if zk_total == 0 and deals_total == 0:
logger.warning("landing/stats: all KPIs are zero — returning stale=True")
return None
return {
"zk_total": zk_total,
"deals_total": deals_total,
"price_coverage_pct": price_coverage_pct,
"mapping_coverage_pct": mapping_coverage_pct,
"last_data_update": last_data_update,
"paradox": (
f"из {zk_total} ЖК у {price_coverage_pct}% есть цены"
" — но в публичном доступе только 0.3%"
),
"stale": False,
}
except Exception:
logger.exception("landing/stats: DB query failed, returning stale=True")
return None
@router.get("/landing/stats", response_model=LandingStatsOut)
def landing_stats(
db: Annotated[Session, Depends(get_db)],
) -> LandingStatsOut:
"""5 KPI для hero-секции лендинга.
Поля ответа:
- zk_total: количество ЖК DOM.РФ в ЕКБ
- deals_total: суммарное кол-во ДДУ-сделок в индексе Росреестра
- price_coverage_pct: % объектов objective_lots с ценой
- mapping_coverage_pct: % маппинга objective → domrf_kn_objects (ЕКБ)
- last_data_update: дата последнего обновления данных (YYYY-MM-DD)
- paradox: строка-парадокс портфеля для hero CTA
- stale: true если данные недоступны (DB ошибка или пустая БД)
"""
result = _query_stats(db)
if result is None:
return LandingStatsOut(**_FALLBACK_DATA)
return LandingStatsOut(**result)

1132
backend/app/api/v1/parcels.py
View file

File diff suppressed because it is too large Load diff

77
backend/app/api/v1/pilot.py Normal file
View file

@ -0,0 +1,77 @@
"""Pilot request lead-gen endpoint.
POST /api/v1/pilot/request — принимает заявку на пилот (лид с лендинга или страницы анализа),
сохраняет в таблицу pilot_requests.
Telegram-уведомление — TODO (creds не настроены, см. #307 SF-B3).
"""
from __future__ import annotations
import logging
from typing import Annotated, Any, Literal
from fastapi import APIRouter, Depends, Request
from pydantic import BaseModel, Field
from sqlalchemy import text
from sqlalchemy.orm import Session
from app.core.db import get_db
logger = logging.getLogger(__name__)
router = APIRouter()
class PilotRequestInput(BaseModel):
name: str = Field(min_length=2, max_length=200)
phone: str | None = Field(default=None, max_length=50)
# Note: was EmailStr but `email-validator` package not in deps → ImportError
# on backend startup → container unhealthy → whole deploy fails. Using plain
# str + minimal regex; full validation can happen on frontend/CRM side.
email: str | None = Field(default=None, max_length=200, pattern=r"^[^@\s]+@[^@\s]+\.[^@\s]+$")
company: str | None = Field(default=None, max_length=200)
message: str | None = Field(default=None, max_length=2000)
source: Literal["landing", "analyze_page", "other"] = "landing"
@router.post("/request")
async def create_pilot_request(
payload: PilotRequestInput,
request: Request,
db: Annotated[Session, Depends(get_db)],
) -> dict[str, Any]:
"""Сохраняет заявку на пилот в pilot_requests."""
user_agent = request.headers.get("user-agent")
row = (
db.execute(
text(
"""
INSERT INTO pilot_requests (name, phone, email, company, message, source, user_agent)
VALUES (:name, :phone, :email, :company, :message, :source, :user_agent)
RETURNING CAST(id AS text), created_at
"""
),
{
"name": payload.name,
"phone": payload.phone,
"email": str(payload.email) if payload.email else None,
"company": payload.company,
"message": payload.message,
"source": payload.source,
"user_agent": user_agent,
},
)
.mappings()
.one()
)
db.commit()
logger.info("pilot_request saved id=%s source=%s", row["id"], payload.source)
return {
"id": row["id"],
"created_at": row["created_at"].isoformat(),
"status": "received",
}

385
backend/app/api/v1/trade_in.py Normal file
View file

@ -0,0 +1,385 @@
"""Trade-In Estimator — endpoints (TI-1 mock + TI-2 PDF).
MOCK implementation: returns realistic ЕКБ price bands by rooms/floor/repair.
TODO TI-1b: заменить _mock_estimate() на реальный SQL aggregation из
objective_lots + rosreestr_deals после OBJ-1/2 merge.
"""
from __future__ import annotations
import json
import logging
import random
from datetime import UTC, datetime, timedelta
from typing import Annotated
from uuid import UUID, uuid4
from fastapi import APIRouter, Depends, HTTPException, Response
from sqlalchemy import text
from sqlalchemy.orm import Session
from app.core.db import get_db
from app.schemas.trade_in import AggregatedEstimate, AnalogLot, TradeInEstimateInput
from app.services.exporters.trade_in_pdf import generate_trade_in_pdf
logger = logging.getLogger(__name__)
router = APIRouter()
# ЕКБ-адреса для фейковых аналогов (реальные улицы центра)
_EKB_STREETS = [
"ул. Малышева",
"ул. Куйбышева",
"ул. 8 Марта",
"ул. Белинского",
"пр. Ленина",
"ул. Толмачёва",
"ул. Радищева",
"ул. Мамина-Сибиряка",
"ул. Луначарского",
"ул. Первомайская",
]
# Базовые ценовые диапазоны по комнатности (ЕКБ, 2026)
_PRICE_BANDS: dict[int, dict[str, int | float | str]] = {
0: { # студия ~25 м²
"median": 6_500_000,
"low": 5_800_000,
"high": 7_500_000,
"ppm2": 260_000,
"ref_area": 25.0,
},
1: { # 1к ~40 м²
"median": 9_000_000,
"low": 8_000_000,
"high": 10_500_000,
"ppm2": 225_000,
"ref_area": 40.0,
},
2: { # 2к ~60 м²
"median": 12_500_000,
"low": 11_000_000,
"high": 14_000_000,
"ppm2": 208_000,
"ref_area": 60.0,
},
3: { # 3к ~80 м²
"median": 17_000_000,
"low": 15_000_000,
"high": 19_000_000,
"ppm2": 213_000,
"ref_area": 80.0,
},
}
def _floor_factor(floor: int, total_floors: int) -> float:
"""±5% поправка за этаж: 1й и последний этаж снижают цену."""
if floor == 1:
return 0.95
if floor == total_floors:
return 0.97
return 1.0
def _repair_factor(repair_state: str | None) -> float:
"""±10% поправка за состояние отделки."""
factors = {
"needs_repair": 0.90,
"standard": 1.00,
"good": 1.05,
"excellent": 1.10,
}
return factors.get(repair_state or "standard", 1.0)
def _confidence(rooms: int) -> str:
if 1 <= rooms <= 3:
return "high"
return "medium"
def _gen_analogs(
rooms: int,
area_m2: float,
base_ppm2: int,
n: int,
*,
is_listing: bool,
) -> list[AnalogLot]:
"""Генерирует список фейковых аналогов (объявления или сделки)."""
rng = random.Random(42 + rooms + n)
result: list[AnalogLot] = []
today = datetime.now(tz=UTC).date()
for i in range(n):
street = _EKB_STREETS[i % len(_EKB_STREETS)]
building_no = rng.randint(1, 120)
apt_no = rng.randint(1, 300)
addr = f"г. Екатеринбург, {street}, {building_no}, кв. {apt_no}"
area_jitter = area_m2 * rng.uniform(0.85, 1.15)
ppm2_jitter = int(base_ppm2 * rng.uniform(0.90, 1.10))
price = int(area_jitter * ppm2_jitter)
floor_val = rng.randint(2, 16)
total_fl = rng.randint(floor_val, 20)
if is_listing:
dom = rng.randint(5, 120)
listing_dt = today - timedelta(days=dom)
else:
dom = rng.randint(10, 60)
listing_dt = today - timedelta(days=rng.randint(30, 365))
result.append(
AnalogLot(
address=addr,
area_m2=round(area_jitter, 1),
rooms=rooms if rooms > 0 else 0,
floor=floor_val,
total_floors=total_fl,
price_rub=price,
price_per_m2=ppm2_jitter,
listing_date=listing_dt,
days_on_market=dom,
photo_url=None,
)
)
return result
def _mock_estimate(payload: TradeInEstimateInput) -> AggregatedEstimate:
"""Возвращает mock-оценку на основе диапазонов ЕКБ 2026.
Логика:
- Берём базовый band по rooms (0-3+).
- Масштабируем на фактическую площадь относительно референсной.
- Применяем поправку за этаж (±5%) и отделку (±10%).
- Генерируем 7-10 аналогов (листинги) и 3-5 actual_deals.
"""
rooms_key = min(payload.rooms, 3) # 4к+ → диапазон 3к
band = _PRICE_BANDS[rooms_key]
# Масштаб по площади
ref_area: float = band["ref_area"] # type: ignore[assignment]
area_scale = payload.area_m2 / ref_area
ff = _floor_factor(payload.floor, payload.total_floors)
rf = _repair_factor(payload.repair_state)
combined = area_scale * ff * rf
median = int(band["median"] * combined) # type: ignore[operator]
low = int(band["low"] * combined) # type: ignore[operator]
high = int(band["high"] * combined) # type: ignore[operator]
ppm2 = int(band["ppm2"] * ff * rf) # type: ignore[operator]
n_analogs = random.randint(7, 10)
n_deals = random.randint(3, 5)
analogs = _gen_analogs(rooms_key, payload.area_m2, ppm2, n_analogs, is_listing=True)
actual_deals = _gen_analogs(
rooms_key, payload.area_m2, int(ppm2 * 0.93), n_deals, is_listing=False
)
now = datetime.now(tz=UTC)
return AggregatedEstimate(
estimate_id=uuid4(),
median_price_rub=median,
range_low_rub=low,
range_high_rub=high,
median_price_per_m2=ppm2,
confidence=_confidence(payload.rooms),
n_analogs=n_analogs + n_deals,
period_months=24,
analogs=analogs,
actual_deals=actual_deals,
expires_at=now + timedelta(hours=24),
)
@router.post("/estimate", response_model=AggregatedEstimate)
def estimate(
payload: TradeInEstimateInput,
db: Annotated[Session, Depends(get_db)],
) -> AggregatedEstimate:
"""MOCK реализация оценки квартиры для Trade-In.
TODO TI-1b: заменить на реальный SQL aggregation из objective_lots
после OBJ-1/2 merge (issue #314).
"""
result = _mock_estimate(payload)
analogs_json = json.dumps(
[a.model_dump(mode="json") for a in result.analogs],
ensure_ascii=False,
)
deals_json = json.dumps(
[a.model_dump(mode="json") for a in result.actual_deals],
ensure_ascii=False,
)
db.execute(
text(
"""
INSERT INTO trade_in_estimates (
id, address, area_m2, rooms, floor, total_floors,
year_built, house_type, repair_state, has_balcony,
median_price, range_low, range_high, median_price_per_m2,
confidence, n_analogs, analogs, actual_deals, expires_at
) VALUES (
CAST(:id AS uuid),
:address, :area_m2, :rooms, :floor, :total_floors,
:year_built, :house_type, :repair_state, :has_balcony,
:median_price, :range_low, :range_high, :median_price_per_m2,
:confidence, :n_analogs,
CAST(:analogs AS jsonb),
CAST(:actual_deals AS jsonb),
:expires_at
)
"""
),
{
"id": str(result.estimate_id),
"address": payload.address,
"area_m2": payload.area_m2,
"rooms": payload.rooms,
"floor": payload.floor,
"total_floors": payload.total_floors,
"year_built": payload.year_built,
"house_type": payload.house_type,
"repair_state": payload.repair_state,
"has_balcony": payload.has_balcony,
"median_price": result.median_price_rub,
"range_low": result.range_low_rub,
"range_high": result.range_high_rub,
"median_price_per_m2": result.median_price_per_m2,
"confidence": result.confidence,
"n_analogs": result.n_analogs,
"analogs": analogs_json,
"actual_deals": deals_json,
"expires_at": result.expires_at,
},
)
db.commit()
logger.info(
"trade_in estimate saved id=%s rooms=%d area=%.1f confidence=%s",
result.estimate_id,
payload.rooms,
payload.area_m2,
result.confidence,
)
return result
@router.get("/estimate/{estimate_id}", response_model=AggregatedEstimate)
def get_estimate(
estimate_id: UUID,
db: Annotated[Session, Depends(get_db)],
) -> AggregatedEstimate:
"""Получить сохранённую оценку по UUID (для генерации PDF).
Возвращает 404 если оценка не найдена или TTL истёк.
"""
row = db.execute(
text(
"""
SELECT id, median_price, range_low, range_high, median_price_per_m2,
confidence, n_analogs, analogs, actual_deals, expires_at,
address, area_m2, rooms
FROM trade_in_estimates
WHERE id = CAST(:id AS uuid)
AND expires_at > NOW()
"""
),
{"id": str(estimate_id)},
).fetchone()
if row is None:
raise HTTPException(status_code=404, detail="estimate not found or expired")
analogs = [AnalogLot(**a) for a in (row.analogs or [])]
actual_deals = [AnalogLot(**a) for a in (row.actual_deals or [])]
return AggregatedEstimate(
estimate_id=row.id,
median_price_rub=row.median_price,
range_low_rub=row.range_low,
range_high_rub=row.range_high,
median_price_per_m2=row.median_price_per_m2,
confidence=row.confidence,
n_analogs=row.n_analogs,
period_months=24,
analogs=analogs,
actual_deals=actual_deals,
expires_at=row.expires_at,
)
@router.get("/estimate/{estimate_id}/pdf")
def estimate_pdf(
estimate_id: UUID,
db: Annotated[Session, Depends(get_db)],
) -> Response:
"""Скачать 4-страничный PDF-отчёт для оценки trade-in.
Возвращает application/pdf attachment.
404 — оценка не найдена.
410 — оценка просрочена (TTL 24ч).
"""
row = db.execute(
text(
"""
SELECT id, median_price, range_low, range_high, median_price_per_m2,
confidence, n_analogs, analogs, actual_deals, expires_at,
address, area_m2, rooms, floor, total_floors,
year_built, house_type, repair_state, has_balcony
FROM trade_in_estimates
WHERE id = CAST(:id AS uuid)
"""
),
{"id": str(estimate_id)},
).fetchone()
if row is None:
raise HTTPException(status_code=404, detail="estimate not found")
if row.expires_at.replace(tzinfo=UTC) < datetime.now(tz=UTC):
raise HTTPException(status_code=410, detail="estimate expired (24h TTL)")
analogs = [AnalogLot(**a) for a in (row.analogs or [])]
actual_deals = [AnalogLot(**a) for a in (row.actual_deals or [])]
estimate = AggregatedEstimate(
estimate_id=row.id,
median_price_rub=row.median_price,
range_low_rub=row.range_low,
range_high_rub=row.range_high,
median_price_per_m2=row.median_price_per_m2,
confidence=row.confidence,
n_analogs=row.n_analogs,
period_months=24,
analogs=analogs,
actual_deals=actual_deals,
expires_at=row.expires_at,
)
input_snapshot = {
"address": row.address,
"area_m2": row.area_m2,
"rooms": row.rooms,
"floor": row.floor,
"total_floors": row.total_floors,
"year_built": row.year_built,
"house_type": row.house_type,
"repair_state": row.repair_state,
"has_balcony": row.has_balcony,
}
pdf_bytes = generate_trade_in_pdf(estimate, input_snapshot)
filename = f"trade-in-{estimate_id}.pdf"
logger.info("PDF generated estimate_id=%s size=%d", estimate_id, len(pdf_bytes))
return Response(
content=pdf_bytes,
media_type="application/pdf",
headers={"Content-Disposition": f'attachment; filename="{filename}"'},
)

30
backend/app/api/v1/users.py Normal file
View file

@ -0,0 +1,30 @@
"""Users endpoints.
GET /api/v1/users/me/recent-parcels — история недавно просмотренных участков.
STUB: возвращает пустой список до реализации NextAuth (Wave 3).
"""
from __future__ import annotations
import logging
from typing import Annotated
from fastapi import APIRouter, Query
logger = logging.getLogger(__name__)
router = APIRouter()
@router.get("/users/me/recent-parcels")
def get_recent_parcels(
limit: Annotated[int, Query(ge=1, le=50)] = 10,
) -> dict:
"""STUB. Real impl после NextAuth (Wave 3) — будет читать parcel_analysis_history table."""
logger.info("recent-parcels stub called, limit=%d", limit)
return {
"items": [],
"total": 0,
"stub": True,
"real_impl_after": "NextAuth",
}

43
backend/app/core/config.py
View file

@ -1,3 +1,7 @@
import os
import warnings
from pydantic import model_validator
from pydantic_settings import BaseSettings, SettingsConfigDict
@ -7,13 +11,42 @@ class Settings(BaseSettings):
database_url: str = "postgresql+psycopg://gendesign:gendesign@localhost:5432/gendesign"
redis_url: str = "redis://localhost:6379/0"
cors_origins: list[str] = ["http://localhost:3000"]
sentry_dsn: str | None = None
# Release tag для Sentry — обычно git short sha, проставляется
# deploy.yml в backend/.env.runtime (см. workflow). Локально оставляем
# пустым — Sentry припишет 'unknown'.
sentry_release: str | None = None
# GlitchTip error tracking (Sentry-compatible self-hosted).
# Формат DSN: https://<key>@errors.gendsgn.ru/<project_id>
# Пустая строка / None = SDK не инициализируется (no-op).
glitchtip_dsn: str | None = None
glitchtip_traces_sample_rate: float = 0.05
environment: str = "dev"
@model_validator(mode="after")
def _promote_legacy_sentry_dsn(self) -> "Settings":
"""Backward-compat: legacy SENTRY_DSN → glitchtip_dsn ТОЛЬКО для self-hosted.
Старый VPS .env.runtime мог содержать SENTRY_DSN=https://...@sentry.io/...
(legacy SaaS Sentry). Промоутить его НЕЛЬЗЯ — события пойдут в чужой проект.
Принимаем только URLs указывающие на наш errors.gendsgn.ru host.
"""
if not self.glitchtip_dsn:
legacy = os.getenv("SENTRY_DSN")
if legacy:
if "errors.gendsgn.ru" in legacy:
warnings.warn(
"SENTRY_DSN is set but ignored by pydantic; rename to GLITCHTIP_DSN. "
"Auto-promoting for backward compat.",
DeprecationWarning,
stacklevel=2,
)
self.glitchtip_dsn = legacy
else:
warnings.warn(
f"SENTRY_DSN points to non-GlitchTip host "
f"({legacy.split('@', 1)[-1][:40]}...) — ignoring. "
"Set GLITCHTIP_DSN explicitly.",
UserWarning,
stacklevel=2,
)
return self
# External APIs (Stage 2)
rosreestr_pkk_base_url: str = "https://pkk.rosreestr.ru/api/features/1"
overpass_url: str = "https://overpass-api.de/api/interpreter"

61
backend/app/main.py
View file

@ -1,33 +1,70 @@
import logging
import os
from collections.abc import AsyncIterator
from contextlib import asynccontextmanager
import sentry_sdk
from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
from sentry_sdk.integrations.celery import CeleryIntegration
from sentry_sdk.integrations.fastapi import FastApiIntegration
from sentry_sdk.integrations.httpx import HttpxIntegration
from sentry_sdk.integrations.logging import LoggingIntegration
from sentry_sdk.integrations.sqlalchemy import SqlalchemyIntegration
from sentry_sdk.integrations.starlette import StarletteIntegration
from app.api.v1 import (
admin_cadastre,
admin_etl,
admin_jobs,
admin_leads,
admin_scrape,
admin_weight_profiles,
analytics,
concepts,
custom_pois,
landing,
parcels,
photos,
pilot,
trade_in,
users,
)
from app.core.config import settings
from app.observability.sentry_scrub import scrub_sensitive_query
logger = logging.getLogger(__name__)
# Инициализируем SDK до создания FastAPI app, чтобы все инструменты
# (middleware, маршруты) видели активный client с самого старта процесса.
# GlitchTip не поддерживает profiling — profiles_sample_rate=0.0.
if settings.glitchtip_dsn:
sentry_sdk.init(
dsn=settings.glitchtip_dsn,
environment=settings.environment,
release=os.getenv("GIT_SHA") or os.getenv("SENTRY_RELEASE") or "unknown",
traces_sample_rate=settings.glitchtip_traces_sample_rate,
profiles_sample_rate=0.0,
send_default_pii=False,
before_send_transaction=scrub_sensitive_query,
integrations=[
StarletteIntegration(),
FastApiIntegration(),
CeleryIntegration(monitor_beat_tasks=True),
SqlalchemyIntegration(),
HttpxIntegration(),
LoggingIntegration(level=logging.INFO, event_level=logging.ERROR),
],
)
logger.info(
"GlitchTip SDK initialised (env=%s, traces=%.2f)",
settings.environment,
settings.glitchtip_traces_sample_rate,
)
@asynccontextmanager
async def lifespan(app: FastAPI) -> AsyncIterator[None]:
if settings.sentry_dsn:
sentry_sdk.init(
dsn=settings.sentry_dsn,
environment=settings.environment,
release=settings.sentry_release,
traces_sample_rate=0.1,
)
yield
@ -53,11 +90,21 @@ app.include_router(
tags=["admin", "site-finder"],
)
app.include_router(photos.router, prefix="/api/v1/photos", tags=["photos"])
app.include_router(custom_pois.router, prefix="/api/v1/custom-pois", tags=["custom-pois"])
app.include_router(
admin_cadastre.router,
prefix="/api/v1/admin/cadastre",
tags=["admin", "cadastre"],
)
app.include_router(
admin_etl.router,
prefix="/api/v1/admin/etl",
tags=["admin", "etl"],
)
app.include_router(trade_in.router, prefix="/api/v1/trade-in", tags=["trade-in"])
app.include_router(landing.router, prefix="/api/v1", tags=["landing"])
app.include_router(pilot.router, prefix="/api/v1/pilot", tags=["pilot"])
app.include_router(users.router, prefix="/api/v1", tags=["users"])
@app.get("/health")

0
backend/app/observability/__init__.py Normal file
View file

49
backend/app/observability/sentry_scrub.py Normal file
View file

@ -0,0 +1,49 @@
"""Хук before_send_transaction для GlitchTip/Sentry SDK.
Redact-ит api keys / tokens из URL-spans перед отправкой — чтобы
секреты (apiKey=..., api_key=..., token=...) не утекали в GlitchTip
через HttpxIntegration performance-spans.
"""
from __future__ import annotations
import re
from typing import Any
from sentry_sdk.types import Event
_SENSITIVE_PARAM_RE = re.compile(
r"((?:api[_-]?[Kk]ey|token|access[_-]?token|secret)=)([^&\s]+)",
re.IGNORECASE,
)
def _redact(value: Any) -> Any:
"""Заменить значения чувствительных query-параметров на [REDACTED]."""
if isinstance(value, str):
return _SENSITIVE_PARAM_RE.sub(r"\1[REDACTED]", value)
return value
def scrub_sensitive_query(event: Event, _hint: dict[str, Any]) -> Event | None:
"""Redact api keys / tokens из URL spans перед отправкой в GlitchTip.
Обрабатывает:
- span.data["url"], span.data["http.url"], span.data["http.target"]
- span["description"]
- event["request"]["url"]
"""
for span in event.get("spans") or []:
data = span.get("data")
if isinstance(data, dict):
for key in ("url", "http.url", "http.target"):
if key in data:
data[key] = _redact(data[key])
if "description" in span:
span["description"] = _redact(span["description"])
request = event.get("request")
if isinstance(request, dict) and "url" in request:
request["url"] = _redact(request["url"])
return event

60
backend/app/schemas/custom_poi.py Normal file
View file

@ -0,0 +1,60 @@
"""Pydantic schemas для user_custom_pois (#254).
CustomPoiCreate — тело POST-запроса.
CustomPoiUpdate — тело PATCH-запроса (все поля опциональны).
CustomPoiOut — ответ API (включает id, created_at, updated_at).
"""
from __future__ import annotations
from datetime import datetime
from pydantic import BaseModel, Field
# Диапазон весов кастомных POI шире системных ([-5, 5]),
# чтобы пользователь мог сильно усилить или исключить POI.
_CUSTOM_WEIGHT_MIN: float = -5.0
_CUSTOM_WEIGHT_MAX: float = 5.0
class CustomPoiCreate(BaseModel):
name: str = Field(..., min_length=1, max_length=200, description="Отображаемое название точки")
category: str | None = Field(None, max_length=100, description="Произвольная категория")
weight: float = Field(
...,
ge=_CUSTOM_WEIGHT_MIN,
le=_CUSTOM_WEIGHT_MAX,
description="Вес точки в scoring [-5, 5]",
)
lon: float = Field(..., ge=-180.0, le=180.0, description="Долгота WGS-84")
lat: float = Field(..., ge=-90.0, le=90.0, description="Широта WGS-84")
parcel_cad: str | None = Field(
None,
max_length=100,
description="Кадастровый номер участка; NULL = глобальная для пользователя",
)
notes: str | None = Field(None, max_length=2000, description="Примечания")
class CustomPoiUpdate(BaseModel):
name: str | None = Field(None, min_length=1, max_length=200)
category: str | None = None
weight: float | None = Field(None, ge=_CUSTOM_WEIGHT_MIN, le=_CUSTOM_WEIGHT_MAX)
lon: float | None = Field(None, ge=-180.0, le=180.0)
lat: float | None = Field(None, ge=-90.0, le=90.0)
parcel_cad: str | None = Field(None, max_length=100)
notes: str | None = None
class CustomPoiOut(BaseModel):
id: int
user_id: str
parcel_cad: str | None
name: str
category: str | None
weight: float
lon: float
lat: float
notes: str | None
created_at: datetime
updated_at: datetime

338
backend/app/schemas/parcel.py
View file

@ -1,7 +1,127 @@
from typing import Any
import datetime as dt
from typing import Any, Literal
from pydantic import BaseModel, Field
# ── #105 Phase 5: Recent permits schemas ──────────────────────────────────────
class RecentPermit(BaseModel):
"""Одно строительное разрешение (РНС или РВЭ) из ekburg_construction_permits."""
permit_type: str
permit_number: str
issue_date: str | None
developer_name: str | None
developer_inn: str | None
object_name: str | None
object_type: str | None
construction_address: str | None
total_area_sqm: float | None
class PermitsSummary(BaseModel):
"""Агрегированная сводка по разрешениям в квартале."""
rns_count: int
rve_count: int
rns_total_area_sqm: float
by_developer: list[dict[str, Any]]
# ── Connection points schemas (issue #115) ────────────────────────────────────
class EngineeringStructure(BaseModel):
name: str | None
type: str | None
cad_num: str | None
distance_to_boundary_m: float
geometry_geojson: dict[str, Any]
readable_address: str | None
raw_props: dict[str, Any]
source: str
class ZouitOverlap(BaseModel):
reg_numb_border: str | None
type_zone: str | None
subcategory: int | None
intersects_parcel: bool
geometry_geojson: dict[str, Any]
raw_props: dict[str, Any]
source: str
class ConnectionPointsSummary(BaseModel):
nearest_structure_distance_m: float | None
in_protection_zone: bool
protection_zones_intersecting: int
total_structures_in_radius: int
class ConnectionPointsResponse(BaseModel):
engineering_structures: list[EngineeringStructure]
zouit_engineering_overlaps: list[ZouitOverlap]
summary: ConnectionPointsSummary
dump_available: bool
dump_fetched_at: str | None
# ── NSPD Risk Zones schemas (issue #94 TIER 3) ───────────────────────────────
class RiskZone(BaseModel):
"""Одна риск-зона НСПД (TIER 3), пересекающая участок."""
layer: str # e.g. "risk_flooding", "risk_landslide"
subtype: str | None # человекочитаемое название из NSPD properties
geom_wkt: str | None # WKT геометрии risk-зоны в EPSG:4326
intersection_area_sqm: float | None # площадь пересечения с участком, м²
# ── NSPD Opportunity + Red Lines schemas (issue #94 TIER 4) ──────────────────
class OpportunityParcel(BaseModel):
"""TIER 4: opportunity ЗУ вблизи участка (auction/scheme/free/future/oopt).
Поля:
layer: тип opportunity — "auction_parcels" | "scheme_parcels" |
"free_parcels" | "future_parcels" | "oopt"
cad_num: кадастровый номер ЗУ (если доступен в NSPD properties)
distance_m: расстояние от centroid анализируемого участка до ЗУ, м
geom_wkt: WKT геометрии ЗУ в EPSG:4326
"""
layer: str
cad_num: str | None
distance_m: float | None
geom_wkt: str | None
class RedLine(BaseModel):
"""TIER 4: красная линия застройки (NSPD layer 879243).
Поля:
geom_wkt: WKT геометрии линии в EPSG:4326
intersection_length_m: длина пересечения линии с участком, м
(null если только nearby, не intersect)
distance_m: расстояние от ближайшей точки линии до участка, м
(null если линия пересекает участок)
UI-семантика:
intersection_length_m != null → красная линия ПЕРЕСЕКАЕТ участок (ALERT)
distance_m != null → красная линия рядом (warning)
"""
geom_wkt: str | None
intersection_length_m: float | None # null если только nearby, не intersect
distance_m: float | None # null если intersect
# ── Parcel search/detail schemas ──────────────────────────────────────────────
class ParcelFilter(BaseModel):
vri: list[str] | None = None
@ -48,3 +168,219 @@ class ParcelSearchResponse(BaseModel):
class ParcelDetail(ParcelSummary):
geometry_geojson: dict[str, Any]
enrichment: dict[str, Any]
# ── Competitors endpoint schemas ──────────────────────────────────────────────
TimeWindow = Literal["last_month", "last_quarter", "last_year"]
ObjClassFilter = Literal["economy", "comfort", "business"]
class CompetitorsRequest(BaseModel):
radius_km: float = Field(default=1.0, ge=0.1, le=1.5)
time_window: TimeWindow = "last_quarter"
obj_class_filter: ObjClassFilter | None = None
exclude_obj_ids: list[int] = Field(default_factory=list)
class Competitor(BaseModel):
obj_id: int
comm_name: str | None
dev_name: str | None
obj_class: str | None
distance_m: float
lat: float
lng: float
stage: str | None
site_status: str | None = None
ready_dt: dt.date | None = None
flats_total: int | None
flats_sold: int | None
sold_pct: float | None
velocity_per_month: float
avg_price_per_m2: float | None
is_active: bool
class CompetitorsSummary(BaseModel):
total_competitors: int
active_count: int
weighted_avg_velocity: float
radius_km: float
time_window: str
class CompetitorsResponse(BaseModel):
competitors: list[Competitor]
summary: CompetitorsSummary
# ── Layout analysis (Issue #113) ───────────────────────────────────────────
class LayoutSignature(BaseModel):
"""Минимальная сигнатура планировки = (room_bucket, area_bin).
Phase 2.1: layout_type/balcony_count в БД нет, ждут B2B Объектив (#52).
"""
room_bucket: Literal["studio", "1", "2", "3", "4+"]
area_bin: Literal["<25", "25-40", "40-60", "60-80", "80-100", "100+"]
class BestLayoutsRequest(BaseModel):
"""Параметры запроса top-планировок в радиусе вокруг участка."""
radius_km: float = Field(default=1.0, ge=0.1, le=1.5)
time_window: Literal["last_month", "last_quarter", "last_year"] = "last_quarter"
filter_competitor_obj_ids: list[int] | None = None
exclude_competitor_obj_ids: list[int] = Field(default_factory=list)
min_velocity_per_month: float = Field(default=0.5, ge=0.0, le=100.0)
obj_class_filter: Literal["economy", "comfort", "business"] | None = None
target_total_flats: int | None = Field(default=None, ge=1, le=10000)
class TopLayoutRow(BaseModel):
"""Одна строка top-planirovok ranking'а."""
rank: int
room_bucket: str
area_bin: str
signature: str
competitor_obj_ids: list[int]
competitor_count: int
total_sold_in_window: int
velocity_per_month: float
avg_price_per_m2_rub: float | None # NULL если objective не покрывает obj
avg_area_m2: float
supply_units_in_radius: int
sold_pct_of_supply: float | None # NULL если supply=0; clamped at 100.0
is_oversold: bool # True когда raw sum_deals/supply > 100% (несопоставимые окна)
class LayoutTzMixRow(BaseModel):
"""Строка рекомендации unit-mix для ТЗ."""
room_bucket: str
pct: int # 0..100, sum total = 100
abs_units: int | None # NULL если target_total_flats не задан
avg_target_area_m2: float | None
class LayoutTzRecommendation(BaseModel):
"""Рекомендация для ТЗ на проектирование."""
rationale_text: str
mix: list[LayoutTzMixRow]
weighted_avg_price_per_m2_rub: float | None
based_on_obj_count: int
based_on_total_deals: int
data_window_start: dt.date
data_window_end: dt.date
# Fix SF-09 review: True если pathological case — все bucket'ы выше cap,
# redistribute невозможен. Frontend использует для отображения warning banner.
cap_skipped: bool = False
class LayoutDataQuality(BaseModel):
"""Метаданные качества данных (coverage)."""
objects_with_velocity_data: int
objects_total_in_radius: int
velocity_coverage_pct: float
confidence: Literal["high", "medium", "low"]
class BestLayoutsResponse(BaseModel):
"""Ответ /best-layouts endpoint'а."""
top_layouts: list[TopLayoutRow]
recommendation_for_tz: LayoutTzRecommendation
data_quality: LayoutDataQuality
# ── Analyze endpoint market price (#33) ──────────────────────────────────────
class MarketPrice(BaseModel):
"""Ценовая статистика квартала из mv_quarter_price_per_m2 (Issue #33).
Источник: rosreestr_deals, фильтр realestate_type_code='002001003000' (ДДУ),
скользящее окно 24 мес., 30K–800K руб/м², HAVING >= 3 сделок.
"""
p25: float | None = None
median: float | None = None
p75: float | None = None
mean: float | None = None
deals_count: int = 0
median_6m: float | None = None
median_12m: float | None = None
median_24m: float | None = None
last_deal_date: str | None = None # ISO date
source: Literal["quarter_mv", "no_data"] = "no_data"
# ── Analyze endpoint parcel meta (#29 G2) ────────────────────────────────────
class ParcelMeta(BaseModel):
"""Кадастровые метаданные участка из cad_parcels (#29 G2).
Поля берутся из NSPD bulk-ingest (issue #168). Строки сырые, без нормализации.
"""
permitted_use: str | None = None # permitted_use_established_by_document
land_category: str | None = None # land_record_category_type
land_subtype: str | None = None # land_record_subtype
cad_cost: float | None = None # cost_value (кадастровая стоимость, руб.)
source: str = "cad_parcels"
# ── Analyze endpoint inline weights (#201) ────────────────────────────────────
class AnalyzeRequest(BaseModel):
"""Опциональное тело запроса POST /analyze.
Позволяет передать inline POI-веса напрямую в запросе без сохранения
профиля. Если задан weights — применяется с наивысшим приоритетом
(выше profile_id и user default).
"""
weights: dict[str, float] | None = Field(
default=None,
description=(
"Inline POI weights override (категория → weight). "
"Если задан — применяется к scoring, без обязательного profile save. "
"Validated против ALLOWED_CATEGORIES + MIN_WEIGHT/MAX_WEIGHT."
),
)
# ── SF-B1: by-bbox map entry schemas (#307) ──────────────────────────────────
class ParcelMapMarker(BaseModel):
"""Один маркер участка на карте — облегчённый ответ для bbox-запроса.
status берётся из parcel_user_status если передан user_id, иначе null.
last_analysis_date — placeholder до реализации B2 auth.
"""
cad_num: str
centroid_lat: float
centroid_lon: float
area_m2: float | None
land_category: str | None
status: str | None # 'in_work' | 'favorite' | 'dismissed' | null
last_analysis_date: str | None # placeholder, real after B2 auth
class ParcelBboxResponse(BaseModel):
"""Ответ GET /parcels/by-bbox."""
parcels: list[ParcelMapMarker]
count: int
limit: int
bbox_area_km2: float

51
backend/app/schemas/trade_in.py Normal file
View file

@ -0,0 +1,51 @@
"""Pydantic schemas for Trade-In Estimator.
POST /api/v1/trade-in/estimate → AggregatedEstimate
"""
from __future__ import annotations
from datetime import date, datetime
from typing import Literal
from uuid import UUID
from pydantic import BaseModel, Field
class TradeInEstimateInput(BaseModel):
address: str = Field(min_length=3, max_length=500)
area_m2: float = Field(gt=10, lt=500)
rooms: int = Field(ge=0, le=10) # 0 = студия
floor: int = Field(ge=1, le=100)
total_floors: int = Field(ge=1, le=100)
year_built: int | None = Field(default=None, ge=1800, le=2100)
house_type: Literal["panel", "brick", "monolith", "monolith_brick", "other"] | None = None
repair_state: Literal["needs_repair", "standard", "good", "excellent"] | None = None
has_balcony: bool | None = None
class AnalogLot(BaseModel):
address: str
area_m2: float
rooms: int
floor: int | None
total_floors: int | None
price_rub: int
price_per_m2: int
listing_date: date | None
days_on_market: int | None
photo_url: str | None = None
class AggregatedEstimate(BaseModel):
estimate_id: UUID
median_price_rub: int
range_low_rub: int
range_high_rub: int
median_price_per_m2: int
confidence: Literal["low", "medium", "high"]
n_analogs: int
period_months: int # 24
analogs: list[AnalogLot] # top 5-10 listings
actual_deals: list[AnalogLot] # реальные продажи last 12 mo
expires_at: datetime

120
backend/app/scrapers/nspd_bulk_client.py
View file

@ -57,8 +57,12 @@ _CAT_TO_THEMATIC_SEARCH_ID: dict[int, int] = {
39663: 15, # ЕНК
}
# Лимит одновременных запросов (глобальный, shared между всеми instance'ами)
_SEMAPHORE = asyncio.Semaphore(3)
# Concurrency cap per HTTP fetch loop. Issue #260 (Sub-PR B re-review):
# module-level asyncio.Semaphore() bind'ится к event loop первого asyncio.run()
# вызова, на втором — RuntimeError: bound to a different event loop.
# search_by_quarter делает N sequential asyncio.run() per layer → semaphore надо
# создавать per-instance в __aenter__ (под текущим running loop).
_SEMAPHORE_LIMIT = 3
# Таймаут подключения и чтения
DEFAULT_TIMEOUT = httpx.Timeout(30.0, connect=10.0)
@ -131,6 +135,10 @@ class NSPDBulkClient:
self._headers = headers or DEFAULT_HEADERS
self._max_retries = max_retries
self._client: httpx.AsyncClient | None = None
# Issue #260: semaphore создаётся в __aenter__ под текущий event loop.
# Module-level не работал — sequential asyncio.run() bind'ил его к
# закрытому loop'у первого вызова.
self._sem: asyncio.Semaphore | None = None
async def __aenter__(self) -> NSPDBulkClient:
# NSPD prod chain contains a self-signed/internal CA on Beget VPS →
@ -144,6 +152,8 @@ class NSPDBulkClient:
follow_redirects=True,
verify=False,
)
# Создаём semaphore под running loop — safe для sequential asyncio.run().
self._sem = asyncio.Semaphore(_SEMAPHORE_LIMIT)
return self
async def __aexit__(self, *_: Any) -> None:
@ -161,12 +171,12 @@ class NSPDBulkClient:
NspdBulkRateLimitError при 429 после исчерпания retries.
NspdBulkError при прочих 4xx/5xx.
"""
if self._client is None:
if self._client is None or self._sem is None:
raise RuntimeError("NSPDBulkClient не инициализирован — используй async with")
attempt = 0
while True:
async with _SEMAPHORE:
async with self._sem:
# Небольшой jitter чтобы N одновременных запросов не начинались в 0мс
await asyncio.sleep(0.05)
try:
@ -374,7 +384,107 @@ class NSPDBulkClient:
raw_features: list[dict[str, Any]] = (data or {}).get("features") or []
return [NSPDBulkFeature.model_validate(f) for f in raw_features]
# ── 3. list_objects_in_building ───────────────────────────────────────────
# ── 3. get_features_in_bbox_grid ─────────────────────────────────────────
async def get_features_in_bbox_grid(
self,
layer_id: int,
bbox: tuple[float, float, float, float],
*,
grid_n: int = 7,
tile_size: int = 512,
) -> list[dict]:
"""Grid-walk WMS GetFeatureInfo по layer_id в bbox.
Разбивает bbox на grid_n × grid_n ячеек, кликает центр каждой.
Дедуплицирует результаты по feature id.
Args:
layer_id: NSPD layer ID (например 875838 для ПЗЗ).
bbox: (xmin, ymin, xmax, ymax) в EPSG:3857 (метры).
grid_n: количество ячеек по каждой оси (7 → 49 запросов).
tile_size: размер виртуального WMS тайла в пикселях.
Returns:
Список raw feature dict'ов с полями id, geometry, properties.
Дедуплицированы по feature id.
"""
xmin, ymin, xmax, ymax = bbox
x_step = (xmax - xmin) / grid_n
y_step = (ymax - ymin) / grid_n
click_xy = (tile_size // 2, tile_size // 2)
tasks = []
for i in range(grid_n):
for j in range(grid_n):
cell_bbox = (
xmin + i * x_step,
ymin + j * y_step,
xmin + (i + 1) * x_step,
ymin + (j + 1) * y_step,
)
tasks.append(
self.wms_feature_info(
layer_id=layer_id,
bbox=cell_bbox,
click_xy=click_xy,
width=tile_size,
height=tile_size,
)
)
# Запускаем все ячейки конкурентно (semaphore ограничивает до 3 concurrent)
cell_results = await asyncio.gather(*tasks, return_exceptions=True)
seen_ids: set[str] = set()
results: list[dict] = []
for idx, cell_result in enumerate(cell_results):
if isinstance(cell_result, Exception):
logger.debug(
"get_features_in_bbox_grid: layer=%d cell=%d error: %s",
layer_id,
idx,
cell_result,
)
continue
for feature in cell_result:
fid = str(feature.id) if feature.id is not None else ""
if fid and fid in seen_ids:
continue
if fid:
seen_ids.add(fid)
results.append(
{
"id": feature.id,
"geometry": feature.geometry,
"properties": feature.properties,
}
)
logger.info(
"get_features_in_bbox_grid: layer=%d grid=%dx%d unique_features=%d",
layer_id,
grid_n,
grid_n,
len(results),
)
return results
async def get_territorial_zones_in_bbox(
self,
bbox: tuple[float, float, float, float],
*,
grid_n: int = 7,
) -> list[dict]:
"""Grid-walk WMS GetFeatureInfo для layer 875838 (ПЗЗ территориальные зоны).
Returns: list of feature dicts с полями id, geometry, properties.
Дедуплицирует по feature id.
"""
return await self.get_features_in_bbox_grid(875838, bbox, grid_n=grid_n)
# ── 4. list_objects_in_building ───────────────────────────────────────────
# Q3 deferred — метод реализован, но не вызывается в bulk_harvest_quarter MVP.
# Готов для per-building помещения/парковка фазы.

252
backend/app/services/analytics_queries.py
View file

@ -842,6 +842,251 @@ def object_photos(db: Session, obj_id: int, limit: int = 100) -> list[dict[str,
]
def object_full_detail(db: Session, obj_id: int) -> dict[str, Any] | None:
"""Extended object detail — adds all Wave A+B columns (22begh).
Returns the same base fields as object_detail() PLUS the 30 new columns
from 113_22begh_kn_schema_extension.sql. Falls back gracefully: columns
that haven't been scraped yet return NULL.
"""
row = (
db.execute(
text(
"""
SELECT o.obj_id, o.hobj_id, o.comm_name, o.addr, o.short_addr, o.region_cd,
o.dev_id, o.dev_name, o.dev_group_name,
o.floor_min, o.floor_max, o.flat_count,
o.square_living, o.ready_dt, o.site_status, o.escrow, o.obj_class,
o.latitude, o.longitude, o.obj_status, o.snapshot_date,
o.energy_eff, o.wall_type,
-- Building specs (22e)
o.first_floor_type, o.section_count,
o.elevators_passenger_count, o.elevators_cargo_count,
o.parking_total_slots, o.guest_parking_inside_count,
o.guest_parking_outside_count, o.ceiling_height_m,
-- Apartment summary (22e)
o.finishing_variants_count, o.has_free_planning, o.avg_flat_area_m2,
-- Yard (22e)
o.playground_kids_count, o.playground_sport_count,
o.has_bike_paths, o.trash_areas_count,
-- OVZ (22e)
o.has_ramp, o.has_low_platforms, o.has_wheelchair_lift,
-- Catalog/UI (22e)
o.flat_area_min, o.flat_area_max,
o.price_min_rub, o.price_max_rub,
o.price_per_m2_min, o.price_per_m2_max,
o.parking_provision_pct, o.project_published_at,
o.project_declaration_num,
-- Metro & scores (22e/22h)
o.metro_nearest_name, o.metro_nearest_walk_minutes, o.metro_top3,
o.domrf_score_location, o.domrf_score_transport,
o.domrf_score_infrastructure,
COALESCE(cb.buildings_count, 0) AS buildings_count
FROM domrf_kn_objects o
LEFT JOIN v_complex_buildings cb ON cb.complex_id = o.obj_id
WHERE o.obj_id = :obj
ORDER BY o.snapshot_date DESC
LIMIT 1
"""
),
{"obj": obj_id},
)
.mappings()
.first()
)
if not row:
return None
metro_top3 = row["metro_top3"]
return {
"obj_id": row["obj_id"],
"hobj_id": row["hobj_id"],
"comm_name": row["comm_name"],
"addr": row["addr"],
"short_addr": row["short_addr"],
"region_cd": row["region_cd"],
"dev_id": row["dev_id"],
"dev_name": row["dev_name"],
"dev_group_name": row["dev_group_name"],
"floor_min": row["floor_min"],
"floor_max": row["floor_max"],
"flat_count": row["flat_count"],
"square_living": _f(row["square_living"]),
"ready_dt": row["ready_dt"].isoformat() if row["ready_dt"] else None,
"site_status": row["site_status"],
"escrow": row["escrow"],
"obj_class": row["obj_class"],
"latitude": _f(row["latitude"]),
"longitude": _f(row["longitude"]),
"obj_status": row["obj_status"],
"snapshot_date": row["snapshot_date"].isoformat() if row["snapshot_date"] else None,
"energy_eff": row["energy_eff"],
"wall_type": row["wall_type"],
# Building specs
"first_floor_type": row["first_floor_type"],
"section_count": row["section_count"],
"elevators_passenger_count": row["elevators_passenger_count"],
"elevators_cargo_count": row["elevators_cargo_count"],
"parking_total_slots": row["parking_total_slots"],
"guest_parking_inside_count": row["guest_parking_inside_count"],
"guest_parking_outside_count": row["guest_parking_outside_count"],
"ceiling_height_m": _f(row["ceiling_height_m"]),
# Apartment summary
"finishing_variants_count": row["finishing_variants_count"],
"has_free_planning": row["has_free_planning"],
"avg_flat_area_m2": _f(row["avg_flat_area_m2"]),
# Yard
"playground_kids_count": row["playground_kids_count"],
"playground_sport_count": row["playground_sport_count"],
"has_bike_paths": row["has_bike_paths"],
"trash_areas_count": row["trash_areas_count"],
# OVZ
"has_ramp": row["has_ramp"],
"has_low_platforms": row["has_low_platforms"],
"has_wheelchair_lift": row["has_wheelchair_lift"],
# Catalog/UI
"flat_area_min": _f(row["flat_area_min"]),
"flat_area_max": _f(row["flat_area_max"]),
"price_min_rub": row["price_min_rub"],
"price_max_rub": row["price_max_rub"],
"price_per_m2_min": _f(row["price_per_m2_min"]),
"price_per_m2_max": _f(row["price_per_m2_max"]),
"parking_provision_pct": _f(row["parking_provision_pct"]),
"project_published_at": (
row["project_published_at"].isoformat() if row["project_published_at"] else None
),
"project_declaration_num": row["project_declaration_num"],
# Metro & scores
"metro_nearest_name": row["metro_nearest_name"],
"metro_nearest_walk_minutes": row["metro_nearest_walk_minutes"],
"metro_top3": metro_top3, # already jsonb → dict/list from psycopg3
"domrf_score_location": row["domrf_score_location"],
"domrf_score_transport": row["domrf_score_transport"],
"domrf_score_infrastructure": row["domrf_score_infrastructure"],
"buildings_count": int(row["buildings_count"]),
}
def object_flats_quartirography(db: Session, obj_id: int) -> list[dict[str, Any]]:
"""Per-rooms aggregation из domrf_kn_flats для объекта.
Группирует по rooms: 1/2/3/Нежилые (rooms IS NULL).
Возвращает count total, count 'free', min/max area, min/max price.
"""
rows = (
db.execute(
text(
"""
WITH latest AS (
SELECT MAX(snapshot_date) AS snap
FROM domrf_kn_flats
WHERE obj_id = :obj
)
SELECT
CASE
WHEN f.rooms IS NULL OR LOWER(f.flat_type) LIKE '%нежил%'
OR LOWER(f.flat_type) LIKE '%nonliv%' THEN 'Нежилые'
WHEN f.rooms = 0 THEN 'Студия'
WHEN f.rooms = 1 THEN '1-комн.'
WHEN f.rooms = 2 THEN '2-комн.'
WHEN f.rooms = 3 THEN '3-комн.'
ELSE (f.rooms::text || '-комн.')
END AS room_label,
COALESCE(f.rooms, -1) AS sort_key,
COUNT(*) AS total_count,
COUNT(*) FILTER (WHERE LOWER(f.status) = 'free'
OR LOWER(f.status) LIKE '%свобод%')
AS free_count,
MIN(f.total_area) AS area_min,
MAX(f.total_area) AS area_max,
MIN(f.price_rub) FILTER (WHERE f.price_rub > 0)
AS price_min,
MAX(f.price_rub) FILTER (WHERE f.price_rub > 0)
AS price_max
FROM domrf_kn_flats f
CROSS JOIN latest l
WHERE f.obj_id = :obj
AND f.snapshot_date = l.snap
GROUP BY room_label, sort_key
ORDER BY sort_key
"""
),
{"obj": obj_id},
)
.mappings()
.all()
)
return [
{
"room_label": r["room_label"],
"total_count": r["total_count"],
"free_count": r["free_count"],
"area_min": _f(r["area_min"]),
"area_max": _f(r["area_max"]),
"price_min": r["price_min"],
"price_max": r["price_max"],
}
for r in rows
]
def object_obj_checks(db: Session, obj_id: int) -> list[dict[str, Any]]:
"""6 «Проверено на наш.дом.рф» checks из domrf_obj_checks (22f)."""
rows = (
db.execute(
text(
"""
SELECT check_type, passed, checked_at
FROM domrf_obj_checks
WHERE obj_id = :obj
ORDER BY check_type
"""
),
{"obj": obj_id},
)
.mappings()
.all()
)
return [
{
"check_type": r["check_type"],
"passed": r["passed"],
"checked_at": r["checked_at"].isoformat() if r["checked_at"] else None,
}
for r in rows
]
def object_documents(db: Session, obj_id: int) -> list[dict[str, Any]]:
"""PDF documents из domrf_kn_documents (22i), сортировка по doc_type + posted_at."""
rows = (
db.execute(
text(
"""
SELECT doc_type, doc_num, posted_at, file_url, size_bytes
FROM domrf_kn_documents
WHERE obj_id = :obj
ORDER BY doc_type, posted_at DESC NULLS LAST
"""
),
{"obj": obj_id},
)
.mappings()
.all()
)
return [
{
"doc_type": r["doc_type"],
"doc_num": r["doc_num"],
"posted_at": r["posted_at"].isoformat() if r["posted_at"] else None,
"file_url": r["file_url"],
"size_bytes": r["size_bytes"],
}
for r in rows
]
def prinzip_funnel_monthly(db: Session, months: int = 24) -> list[dict[str, Any]]:
"""Воронка по месяцам из materialized view."""
rows = (
@ -1349,10 +1594,9 @@ def _district_cadastre_baseline(db: Session, *, district_name: str) -> dict[str,
WHERE cb.cost_value IS NOT NULL
AND cb.area IS NOT NULL
AND cb.area >= 100
-- floors хранится как TEXT (встречаются '1-2', '2-3') —
-- считаем только чистые числа ≥3, либо purpose-fallback.
AND ((cb.floors ~ '^[0-9]+$' AND cb.floors::int >= 3)
OR cb.purpose ILIKE '%многокв%')
-- floors INTEGER (Rosreestr ETL приводит к int); NULL = unknown.
-- Считаем МКД если floors ≥3 или purpose содержит «многокв».
AND (cb.floors >= 3 OR cb.purpose ILIKE '%многокв%')
AND (cb.cost_value / NULLIF(cb.area, 0))
BETWEEN 5000 AND 500000
)

106
backend/app/services/cadastre/bulk_harvest.py
View file

@ -19,6 +19,7 @@ Resumable: phase_state в cadastre_jobs показывает прогресс.
from __future__ import annotations
import hashlib
import json
import logging
from collections.abc import Callable
@ -236,6 +237,20 @@ async def harvest_quarter(
}
)
# ── Phase 2.5: grid-walk для territorial_zones (ПЗЗ, layer 875838) ────────
# Выполняем после основного grid-walk (Phase 2-3). Требует bbox квартала.
quarter_bbox = quarter_bbox_3857(db, quarter)
if quarter_bbox is not None:
update_progress({"phase": "territorial_zones_started", "quarter": quarter})
try:
tz_features = await client.get_territorial_zones_in_bbox(quarter_bbox)
tz_count = _save_territorial_zones(db, quarter, tz_features)
logger.info(
"harvest_quarter: territorial_zones quarter=%s upserted=%d", quarter, tz_count
)
except Exception as e:
logger.warning("harvest_quarter: territorial_zones failed quarter=%s: %s", quarter, e)
# ── Phase 4: quarter stats + auto-heal geom из snapshot ─────────────────
stats_features = [f for f in snapshot.features if f.category_id == CAT_QUARTER_STATS]
if stats_features:
@ -1176,6 +1191,97 @@ def upsert_quarter_stats(
)
# ── cad_territorial_zones upsert ────────────────────────────────────────────
def _save_territorial_zones(db: Session, quarter_cad: str, features: list[dict]) -> int:
"""UPSERT territorial_zones features в cad_territorial_zones по zone_id.
Args:
db: SQLAlchemy session.
quarter_cad: кадастровый номер квартала (3 сегмента).
features: list of raw feature dicts от get_features_in_bbox_grid.
Returns:
Количество успешно upserted строк.
"""
inserted = 0
for f in features:
props: dict = f.get("properties") or {}
geom = f.get("geometry")
geom_geojson: str | None = json.dumps(geom) if geom else None
# zone_id — NSPD feature id или стабильный fallback на основе md5 от properties.
# md5 гарантирует идемпотентность между runs (счётчик inserted сбрасывается).
_raw_id = props.get("id") or props.get("zone_id") or f.get("id")
if _raw_id:
zone_id = str(_raw_id)
else:
_props_hash = hashlib.md5(
json.dumps(props, sort_keys=True).encode("utf-8")
).hexdigest()[:12]
zone_id = f"{quarter_cad}_{_props_hash}"
zone_code = (
props.get("zone_code") or props.get("zone_index") or props.get("reg_numb_border")
)
zone_name = props.get("zone_name") or props.get("zone_type_name") or props.get("type_zone")
permitted_use = props.get("permitted_use") or props.get("vri")
try:
# begin_nested() требует активной outer-транзакции для SAVEPOINT.
# SQLAlchemy Session (autobegin=True) автоматически начинает tx при первом
# db.execute() в этом loop — outer tx гарантирована.
with db.begin_nested():
db.execute(
text("""
INSERT INTO cad_territorial_zones
(quarter_cad, zone_id, zone_code, zone_name,
permitted_use, raw_props, geom)
VALUES (
CAST(:quarter_cad AS text),
CAST(:zone_id AS text),
CAST(:zone_code AS text),
CAST(:zone_name AS text),
CAST(:permitted_use AS text),
CAST(:raw_props AS jsonb),
CASE WHEN CAST(:geom AS text) IS NOT NULL
THEN ST_Transform(
ST_SetSRID(ST_GeomFromGeoJSON(CAST(:geom AS text)), 3857),
4326
)::geography
ELSE NULL
END
)
ON CONFLICT (zone_id) DO UPDATE SET
zone_code = EXCLUDED.zone_code,
zone_name = EXCLUDED.zone_name,
permitted_use = EXCLUDED.permitted_use,
raw_props = EXCLUDED.raw_props,
geom = EXCLUDED.geom,
fetched_at = NOW()
"""),
{
"quarter_cad": quarter_cad,
"zone_id": zone_id,
"zone_code": zone_code,
"zone_name": zone_name,
"permitted_use": permitted_use,
"raw_props": json.dumps(props, ensure_ascii=False),
"geom": geom_geojson,
},
)
inserted += 1
except Exception as e:
logger.warning(
"_save_territorial_zones: upsert failed zone_id=%s quarter=%s: %s",
zone_id,
quarter_cad,
e,
)
db.commit()
return inserted
# ── Утилиты ──────────────────────────────────────────────────────────────────

0
backend/app/services/etl/__init__.py Normal file
View file

235
backend/app/services/etl/objective_backfill.py Normal file
View file

@ -0,0 +1,235 @@
"""Backfill objective_complex_mapping (#203).
Auto-match DOM.РФ комплексов к Objective dataset через fuzzy name matching.
Goal: coverage 3% → 40%+ для mv_layout_velocity.
Schema facts (confirmed via pg MCP):
- domrf_kn_objects: фильтр is_ekb = true (~1285 ЕКБ объектов).
district_name ILIKE '%екатеринбург%' возвращает 0 строк — не использовать.
- objective_corpus_room_month: group_name = 'Екатеринбург' (единственное значение),
263 distinct project_name.
- objective_complex_mapping: UNIQUE (objective_complex_name, objective_group),
колонки match_method + match_score + is_reviewed поддерживают audit trail.
match_method history:
'fuzzy' — legacy Anton SQLite import (avg score 0.98, 127 rows)
'fuzzy_trgm' — pg_trgm backfill, auto-accept threshold=0.85 (первый запуск)
'fuzzy_v2' — pg_trgm backfill, pruned threshold=0.80 (второй запуск, #44)
'manual' — ручная корректура
"""
from __future__ import annotations
import logging
from dataclasses import dataclass
from sqlalchemy import text
from sqlalchemy.orm import Session
logger = logging.getLogger(__name__)
# Порог для auto-insert (высокая уверенность)
AUTO_ACCEPT_THRESHOLD = 0.85
# Pruned threshold для v2 run (ниже уверенность, is_reviewed=false → review queue)
AUTO_ACCEPT_THRESHOLD_V2 = 0.80
# Порог для review queue (средняя уверенность — Phase 2)
REVIEW_THRESHOLD = 0.6
@dataclass
class MatchCandidate:
"""Один candidate match DOM.РФ ↔ Objective."""
domrf_obj_id: int
domrf_comm_name: str
domrf_dev_name: str | None
objective_project_name: str
similarity_score: float # 0.0..1.0
def find_match_candidates(
db: Session,
*,
only_unmapped: bool = True,
min_threshold: float | None = None,
limit: int | None = None,
) -> list[MatchCandidate]:
"""Поиск candidates через pg_trgm similarity.
Использует CROSS JOIN LATERAL + similarity() для fuzzy match
comm_name (DOM.РФ) ↔ project_name (Objective).
Args:
db: SQLAlchemy sync Session.
only_unmapped: Если True — пропускает уже-mapped obj_id.
min_threshold: Нижняя граница similarity для фильтрации кандидатов.
По умолчанию REVIEW_THRESHOLD (0.6).
limit: Максимальное число строк результата (для тестирования).
Returns:
Список MatchCandidate, отсортированных по убыванию similarity_score.
"""
effective_min = min_threshold if min_threshold is not None else REVIEW_THRESHOLD
# Формируем SQL. LIMIT добавляем через int() — SQL injection safe (только число).
limit_clause = f"LIMIT {int(limit)}" if limit is not None else ""
sql = text(
f"""
WITH domrf_unmapped AS (
SELECT o.obj_id, o.comm_name, o.dev_name
FROM domrf_kn_objects o
WHERE o.is_ekb = true
AND o.comm_name IS NOT NULL
AND (
CAST(:only_unmapped AS boolean) = FALSE
OR NOT EXISTS (
SELECT 1 FROM objective_complex_mapping cm
WHERE cm.domrf_obj_id = o.obj_id
)
)
),
objective_distinct AS (
SELECT DISTINCT project_name
FROM objective_corpus_room_month
)
SELECT
d.obj_id,
d.comm_name,
d.dev_name,
obj.project_name,
similarity(d.comm_name, obj.project_name) AS sim_score
FROM domrf_unmapped d
CROSS JOIN LATERAL (
SELECT project_name
FROM objective_distinct
WHERE similarity(d.comm_name, project_name) > 0.4
ORDER BY similarity(d.comm_name, project_name) DESC
LIMIT 1
) obj
WHERE similarity(d.comm_name, obj.project_name) >= CAST(:min_threshold AS float)
ORDER BY sim_score DESC
{limit_clause}
"""
)
rows = db.execute(
sql,
{"only_unmapped": only_unmapped, "min_threshold": effective_min},
).all()
return [
MatchCandidate(
domrf_obj_id=int(r[0]),
domrf_comm_name=str(r[1]),
domrf_dev_name=str(r[2]) if r[2] is not None else None,
objective_project_name=str(r[3]),
similarity_score=float(r[4]),
)
for r in rows
]
def auto_apply_matches(
db: Session,
candidates: list[MatchCandidate],
*,
threshold: float = AUTO_ACCEPT_THRESHOLD,
match_method: str = "fuzzy_trgm",
dry_run: bool = False,
) -> dict[str, int]:
"""Apply candidates с score >= threshold в objective_complex_mapping.
Кандидаты ниже threshold, но >= REVIEW_THRESHOLD попадают в review_queue
(Phase 2 — UI для ручного review, в этом PR только считаются).
ON CONFLICT DO NOTHING — если пара (objective_complex_name, objective_group)
уже существует, строка пропускается без ошибки.
Args:
db: SQLAlchemy sync Session.
candidates: Список из find_match_candidates().
threshold: Минимальный score для auto-insert (default 0.85).
match_method: Значение колонки match_method в БД. По умолчанию 'fuzzy_trgm'.
Для re-run с пониженным порогом передавать 'fuzzy_v2'.
dry_run: Если True — только логирует, не пишет в БД.
Returns:
dict с ключами auto_accepted, review_queue, skipped.
"""
auto = [c for c in candidates if c.similarity_score >= threshold]
review = [c for c in candidates if REVIEW_THRESHOLD <= c.similarity_score < threshold]
if dry_run:
logger.info(
"DRY RUN: would auto-accept %d, review queue %d",
len(auto),
len(review),
)
return {"auto_accepted": 0, "review_queue": len(review), "skipped": 0}
inserted = 0
skipped = 0
for c in auto:
try:
with db.begin_nested():
result = db.execute(
text(
"""
INSERT INTO objective_complex_mapping
(objective_complex_name, domrf_obj_id, objective_group,
match_method, match_score, is_reviewed)
VALUES (
CAST(:name AS text),
CAST(:obj_id AS bigint),
CAST(:group AS text),
CAST(:method AS text),
CAST(:score AS numeric),
CAST(:reviewed AS boolean)
)
ON CONFLICT (objective_complex_name, objective_group) DO NOTHING
"""
),
{
"name": c.objective_project_name,
"obj_id": c.domrf_obj_id,
"group": "Екатеринбург",
"method": match_method,
"score": c.similarity_score,
"reviewed": False,
},
)
if result.rowcount > 0:
inserted += 1
else:
skipped += 1
except Exception as e:
logger.warning(
"Insert failed для %s ↔ %s: %s",
c.domrf_comm_name,
c.objective_project_name,
e,
)
skipped += 1
db.commit()
logger.info(
"Backfill complete: auto_accepted=%d skipped=%d review_queue=%d",
inserted,
skipped,
len(review),
)
return {"auto_accepted": inserted, "review_queue": len(review), "skipped": skipped}
def trigger_mv_refresh(db: Session) -> int:
"""REFRESH mv_layout_velocity после backfill.
Вызывает существующий helper из layout_velocity_refresh.
Передаём concurrently=True (MV уже заполнен).
"""
from app.services.site_finder.layout_velocity_refresh import refresh_layout_velocity
return refresh_layout_velocity(db, concurrently=True)

161
backend/app/services/exporters/layout_tz_pdf.py Normal file
View file

@ -0,0 +1,161 @@
"""PDF render для ТЗ (Layout Analysis #113 PR D).
Pattern reference: backend/app/services/exporters/pdf.py (existing WeasyPrint).
"""
from __future__ import annotations
import datetime as dt
import html as _html
import logging
from weasyprint import HTML
from app.schemas.parcel import BestLayoutsResponse
logger = logging.getLogger(__name__)
def render_layout_tz_pdf(
response: BestLayoutsResponse,
*,
cad_num: str,
parcel_address: str | None = None,
radius_km: float,
time_window: str,
) -> bytes:
"""Render ТЗ PDF от best-layouts response.
Args:
response: BestLayoutsResponse от /best-layouts endpoint
cad_num: кадастровый номер участка
parcel_address: optional human address (если known через geocoder)
radius_km: радиус анализа конкурентов
time_window: окно анализа (last_month/quarter/year)
Returns:
PDF bytes (готово для StreamingResponse)
"""
today = dt.date.today().strftime("%d.%m.%Y")
safe_cad = _html.escape(cad_num)
safe_addr = _html.escape(parcel_address) if parcel_address else None
safe_time_window = _html.escape(time_window)
addr_line = f"<p>Адрес: {safe_addr}</p>" if safe_addr else ""
def _price_cell(val: float | None) -> str:
if val is None:
return "<td>—</td>"
return f"<td>{val:,.0f}".replace(",", " ") + " ₽</td>"
# Top layouts table rows
top_rows = "".join(
"<tr>"
f"<td>{r.rank}</td>"
f"<td>{_html.escape(r.room_bucket)}</td>"
f"<td>{_html.escape(r.area_bin)}</td>"
f"<td>{r.velocity_per_month:.1f}</td>"
f"<td>{r.avg_area_m2:.1f}</td>"
f"{_price_cell(r.avg_price_per_m2_rub)}"
f"<td>{r.total_sold_in_window}</td>"
"</tr>"
for r in response.top_layouts
)
# Recommendation mix table rows
mix_rows = "".join(
"<tr>"
f"<td>{_html.escape(m.room_bucket)}</td>"
f"<td>{m.pct}%</td>"
f"<td>{m.abs_units if m.abs_units is not None else '—'}</td>"
f"<td>{f'{m.avg_target_area_m2:.1f}' if m.avg_target_area_m2 is not None else '—'}</td>"
"</tr>"
for m in response.recommendation_for_tz.mix
)
rec = response.recommendation_for_tz
safe_rationale = _html.escape(rec.rationale_text)
weighted_price = (
f"{rec.weighted_avg_price_per_m2_rub:,.0f}".replace(",", " ") + " ₽/м²"
if rec.weighted_avg_price_per_m2_rub is not None
else "нет данных"
)
dq = response.data_quality
html = f"""<!DOCTYPE html>
<html lang="ru">
<head>
<meta charset="UTF-8">
<title>ТЗ на проектирование — {safe_cad}</title>
<style>
body {{ font-family: 'Helvetica', sans-serif; font-size: 11pt; color: #222; }}
h1 {{ font-size: 18pt; margin-bottom: 0.2em; }}
h2 {{ font-size: 14pt; margin-top: 1.2em; border-bottom: 1px solid #ccc; }}
.meta {{ color: #666; font-size: 10pt; margin-bottom: 1em; }}
table {{ width: 100%; border-collapse: collapse; margin: 0.5em 0; }}
th, td {{ padding: 6px 10px; border: 1px solid #ddd; text-align: left; }}
th {{ background: #f5f5f5; font-weight: bold; }}
.rationale {{ background: #f8f8f8; padding: 10px; border-left: 3px solid #4a90e2;
margin: 1em 0; }}
.footer {{ margin-top: 2em; padding-top: 1em; border-top: 1px solid #ddd;
color: #888; font-size: 9pt; }}
.confidence-high {{ color: #2a8c2a; }}
.confidence-medium {{ color: #c9a132; }}
.confidence-low {{ color: #b03434; }}
</style>
</head>
<body>
<h1>Техническое задание на проектирование (data-driven)</h1>
<div class="meta">
<p>Кадастровый номер: <strong>{safe_cad}</strong></p>
{addr_line}
<p>Радиус анализа: {radius_km} км · Окно: {safe_time_window}</p>
<p>Дата формирования: {today}</p>
</div>
<h2>Рекомендуемая структура квартирографии (unit-mix)</h2>
<div class="rationale">{safe_rationale}</div>
<table>
<thead><tr>
<th>Комнатность</th><th>Доля</th><th>Кол-во (от target)</th><th>Целевая площадь, м²</th>
</tr></thead>
<tbody>{mix_rows}</tbody>
</table>
<p>Средневзвешенная цена benchmark: <strong>{weighted_price}</strong></p>
<p>Основано на {rec.based_on_obj_count} ЖК / {rec.based_on_total_deals} сделок</p>
<p>Период данных:
{rec.data_window_start.strftime("%d.%m.%Y")} – {rec.data_window_end.strftime("%d.%m.%Y")}
</p>
<h2>Топ планировок конкурентов по продажам</h2>
<table>
<thead><tr>
<th>#</th><th>Комнаты</th><th>Площадь</th><th>Продажи/мес</th>
<th>Ср. площадь, м²</th><th>Ср. цена, ₽/м²</th><th>Продано (окно)</th>
</tr></thead>
<tbody>{top_rows}</tbody>
</table>
<h2>Качество данных</h2>
<p>
Покрытие: {dq.objects_with_velocity_data} из
{dq.objects_total_in_radius} ЖК с данными velocity
({dq.velocity_coverage_pct:.1f}%)
</p>
<p>
Уверенность:
<span class="confidence-{dq.confidence}">
{dq.confidence.upper()}
</span>
</p>
<div class="footer">
<p>GenDesign Site Finder · сгенерировано из данных DOM.РФ + Objective + Росреестр</p>
<p>Phase 2.1: без layout_type (евро/классика/панорама) и balcony_count.</p>
</div>
</body>
</html>"""
pdf_bytes = HTML(string=html).write_pdf()
logger.info("Generated layout TZ PDF for cad %s: %d bytes", cad_num, len(pdf_bytes))
return pdf_bytes

204
backend/app/services/exporters/snapshot_pdf.py Normal file
View file

@ -0,0 +1,204 @@
"""Генерация одностраничного PDF-снимка кадастрового участка.
Использует WeasyPrint + Jinja2. Шрифты — DejaVu Sans из системы (Dockerfile)
или из пакета weasyprint (font fallback). Шаблон: app/templates/parcel_snapshot.html.
"""
import datetime
import logging
import pathlib
from typing import Any
from jinja2 import Environment, FileSystemLoader, select_autoescape
logger = logging.getLogger(__name__)
# Путь к директории шаблонов (относительно этого файла — 2 уровня вверх, затем templates)
_TEMPLATE_DIR = pathlib.Path(__file__).parent.parent / "templates"
# Системные пути DejaVu Sans (Ubuntu/Debian Docker-образ + Alpine резерв)
_DEJAVU_CANDIDATES: list[str] = [
"/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf",
"/usr/share/fonts/dejavu/DejaVuSans.ttf",
"/usr/share/fonts/TTF/DejaVuSans.ttf",
]
_CATEGORY_RU: dict[str, str] = {
"school": "Школа",
"kindergarten": "Детский сад",
"pharmacy": "Аптека",
"hospital": "Больница",
"shop_mall": "ТЦ",
"shop_supermarket": "Супермаркет",
"shop_small": "Магазин",
"park": "Парк",
"bus_stop": "Автобус",
"metro_stop": "Метро",
"tram_stop": "Трамвай",
}
# Веса POI-категорий — должны совпадать с _POI_WEIGHTS в parcels.py.
# Дублированы здесь чтобы exporter не импортировал из api-слоя.
_POI_WEIGHTS: dict[str, float] = {
"school": 1.5,
"kindergarten": 1.5,
"pharmacy": 0.8,
"hospital": 0.6,
"shop_mall": 1.2,
"shop_supermarket": 1.0,
"shop_small": 0.5,
"park": 1.8,
"bus_stop": 0.3,
"metro_stop": 1.5,
"tram_stop": -0.5,
}
_WALK_SPEED_M_PER_MIN: float = 80.0 # ~5 км/ч
def _find_font_url() -> str:
"""Вернуть file:// URL для DejaVu Sans или пустую строку (system fallback).
WeasyPrint умеет сам находить системные шрифты через fonttools/fontconfig,
поэтому пустая строка допустима — шрифт тогда подбирается CSS generic.
"""
for path in _DEJAVU_CANDIDATES:
if pathlib.Path(path).exists():
return f"file://{path}"
logger.warning(
"snapshot_pdf: DejaVu Sans не найден в стандартных путях — используем system fallback"
)
return ""
def _format_cost(value: float | None) -> str:
"""Форматировать кадастровую стоимость в читаемый вид (млн/тыс ₽)."""
if value is None:
return "—"
if value >= 1_000_000:
return f"{value / 1_000_000:.1f} млн ₽"
if value >= 1_000:
return f"{value / 1_000:.0f} тыс ₽"
return f"{value:.0f} ₽"
def _build_poi_items(poi_rows: list[dict[str, Any]], limit: int = 7) -> list[dict[str, Any]]:
"""Вычислить weighted_score для каждого POI и вернуть топ-N отсортированных.
Формула: weighted_score = weight * max(0, 1 - distance_m / 1000)
Отрицательные вклады (трамвай) — не включаем в топ-список.
"""
items: list[dict[str, Any]] = []
for p in poi_rows:
cat: str = p.get("category", "")
w = _POI_WEIGHTS.get(cat, 0.0)
distance_m = float(p.get("distance_m") or 0)
decay = max(0.0, 1.0 - distance_m / 1000.0)
score = round(w * decay, 2)
if score <= 0:
continue
walk_min = max(1, round(distance_m / _WALK_SPEED_M_PER_MIN))
items.append(
{
"category_ru": _CATEGORY_RU.get(cat, cat),
"name": p.get("name") or "",
"distance_m": round(distance_m),
"walk_min": walk_min,
"weighted_score": score,
}
)
items.sort(key=lambda x: x["weighted_score"], reverse=True)
return items[:limit]
def generate_snapshot_pdf(
*,
cad_num: str,
address: str | None,
district: str | None,
area_m2: float | None,
cadastral_cost_rub: float | None,
land_category: str | None,
vri: str | None,
last_update: str | None,
poi_rows: list[dict[str, Any]],
competitor_rows: list[dict[str, Any]],
competitors_limit: int = 5,
) -> bytes:
"""Сгенерировать PDF-снимок участка (1 страница A4).
Аргументы:
cad_num: кадастровый номер.
address: адрес из cad_parcels.
district: район города.
area_m2: площадь в кв. м (конвертируем в га для отображения).
cadastral_cost_rub: кадастровая стоимость в рублях.
land_category: категория земель.
vri: вид разрешённого использования.
last_update: строка даты последнего обновления данных.
poi_rows: сырые строки из osm_poi_ekb (category, name, distance_m).
competitor_rows: строки конкурентов из domrf_kn_objects.
competitors_limit: сколько конкурентов выводить (3-5 по ТЗ).
Возвращает: bytes PDF-документа.
"""
# WeasyPrint импортируем локально — тяжёлый; не нужен при импорте модуля
try:
from weasyprint import HTML
except ImportError as exc:
raise RuntimeError(
"WeasyPrint не установлен. Добавь 'weasyprint>=62.0' в pyproject.toml."
) from exc
env = Environment(
loader=FileSystemLoader(str(_TEMPLATE_DIR)),
autoescape=select_autoescape(["html"]),
)
template = env.get_template("parcel_snapshot.html")
area_ha = f"{area_m2 / 10_000:.2f}" if area_m2 else "—"
poi_items = _build_poi_items(poi_rows, limit=7)
# Конкуренты — берём топ N ближайших (уже отсортированы по flat_count DESC;
# переупорядочиваем по distance_m для удобства чтения)
competitors_display = sorted(
competitor_rows[:competitors_limit],
key=lambda r: float(r.get("distance_m") or 0),
)
competitors_ctx: list[dict[str, Any]] = [
{
"comm_name": r.get("comm_name"),
"dev_name": r.get("dev_name"),
"obj_class": r.get("obj_class"),
"flat_count": r.get("flat_count"),
"distance_m": round(float(r.get("distance_m") or 0)),
}
for r in competitors_display
]
generated_at = datetime.datetime.now(tz=datetime.UTC).strftime("%d.%m.%Y %H:%M UTC")
html_str = template.render(
cad_num=cad_num,
address=address,
district=district,
area_ha=area_ha,
cadastral_cost=_format_cost(cadastral_cost_rub),
land_category=land_category,
vri=vri,
last_update=last_update or "—",
poi_items=poi_items,
competitors=competitors_ctx,
generated_at=generated_at,
font_url=_find_font_url(),
)
logger.info(
"snapshot_pdf: rendering PDF for %s (%d POI, %d competitors)",
cad_num,
len(poi_items),
len(competitors_ctx),
)
pdf_bytes: bytes = HTML(string=html_str, base_url=str(_TEMPLATE_DIR)).write_pdf()
return pdf_bytes

424
backend/app/services/exporters/trade_in_pdf.py Normal file
View file

@ -0,0 +1,424 @@
"""PDF генератор для отчёта Trade-In Estimator (TI-2).
Структура отчёта — 4 страницы (как у Брусника.Обмен):
1. Cover — шапка + адрес + параметры квартиры + hero-цена
2. Listings — таблица объявлений-аналогов (top 10)
3. Deals — таблица фактических сделок (last 12 мес.)
4. Offer — выкупная стоимость trade-in (placeholder Phase 2)
Pattern: backend/app/services/exporters/layout_tz_pdf.py (WeasyPrint, уже работает).
"""
from __future__ import annotations
import datetime as dt
import html as _html
import logging
from weasyprint import CSS, HTML
from app.schemas.trade_in import AggregatedEstimate, AnalogLot
logger = logging.getLogger(__name__)
# ── helpers ──────────────────────────────────────────────────────────────────
def _fmt_rub(value: int) -> str:
"""Форматирование рублей с пробелами-разделителями: 12 500 000 ₽"""
return f"{value:,}".replace(",", " ") + " ₽"
def _fmt_rub_m(value: int) -> str:
"""Форматирование в миллионах: 12,50 млн ₽"""
m = value / 1_000_000
return f"{m:.2f}".replace(".", ",") + " млн ₽"
def _fmt_ppm2(value: int) -> str:
return f"{value:,}".replace(",", " ") + " ₽/м²"
def _conf_color(confidence: str) -> tuple[str, str, str]:
"""(bg, fg, border) для badge уверенности."""
mapping = {
"high": ("#dcfce7", "#15803d", "#86efac"),
"medium": ("#fef9c3", "#a16207", "#fde68a"),
"low": ("#fee2e2", "#b91c1c", "#fca5a5"),
}
return mapping.get(confidence, ("#f3f4f6", "#374151", "#d1d5db"))
def _conf_label(confidence: str) -> str:
return {"high": "Высокая", "medium": "Средняя", "low": "Низкая"}.get(confidence, confidence)
def _analog_rows(lots: list[AnalogLot], *, is_deal: bool) -> str:
if not lots:
return "<tr><td colspan='6' class='empty'>Нет данных</td></tr>"
rows = []
for lot in lots:
date_val = lot.listing_date.strftime("%d.%m.%Y") if lot.listing_date else "—"
dom_val = str(lot.days_on_market) if lot.days_on_market is not None else "—"
floor_val = f"{lot.floor}/{lot.total_floors}" if lot.floor and lot.total_floors else "—"
label = "Дата сделки" if is_deal else "В продаже"
_ = label # used for header only
rows.append(
"<tr>"
f"<td>{_html.escape(lot.address)}</td>"
f"<td class='num'>{lot.area_m2:.1f}</td>"
f"<td class='num'>{lot.rooms if lot.rooms else 'С'}</td>"
f"<td class='num'>{floor_val}</td>"
f"<td class='num'>{_fmt_rub(lot.price_rub)}</td>"
f"<td class='num'>{date_val} ({dom_val}д.)</td>"
"</tr>"
)
return "".join(rows)
# ── CSS ──────────────────────────────────────────────────────────────────────
def _build_css() -> str:
return """
@page {
size: A4;
margin: 20mm 18mm 20mm 18mm;
}
* { box-sizing: border-box; }
body {
font-family: 'Helvetica', 'Arial', sans-serif;
font-size: 10pt;
color: #1a1d23;
margin: 0;
padding: 0;
}
h1 { font-size: 18pt; margin: 0 0 4pt 0; line-height: 1.2; }
h2 { font-size: 13pt; margin: 0 0 8pt 0; border-bottom: 1.5px solid #e6e8ec; padding-bottom: 4pt; }
h3 { font-size: 11pt; margin: 0 0 6pt 0; color: #374151; }
.page { page-break-after: always; }
.page:last-child { page-break-after: avoid; }
/* Cover */
.cover-header {
background: #1d4ed8; color: #fff;
padding: 20pt; border-radius: 6pt; margin-bottom: 16pt;
}
.cover-header h1 { color: #fff; }
.cover-meta { font-size: 9pt; color: rgba(255,255,255,0.8); margin-top: 4pt; }
.hero-price {
font-size: 28pt; font-weight: 800;
font-variant-numeric: tabular-nums; margin: 12pt 0 4pt;
}
.hero-ppm2 { font-size: 12pt; color: #5b6066; font-variant-numeric: tabular-nums; }
.range-bar-wrap { margin: 10pt 0; }
.range-label { font-size: 9pt; color: #9ca3af; }
.badge {
display: inline-block; padding: 4pt 10pt;
border-radius: 6pt; font-size: 9pt; font-weight: 700; margin-top: 8pt;
}
.params-grid {
display: grid; grid-template-columns: 1fr 1fr 1fr;
gap: 8pt; margin-top: 10pt;
}
.param-item .param-label {
font-size: 8pt; color: #9ca3af;
text-transform: uppercase; letter-spacing: 0.03em;
}
.param-item .param-value { font-size: 10pt; font-weight: 600; }
/* Tables */
table { width: 100%; border-collapse: collapse; margin-top: 8pt; font-size: 9pt; }
thead tr { background: #f3f4f6; }
th {
padding: 5pt 7pt; text-align: left; font-weight: 700;
color: #374151; border-bottom: 1.5px solid #d1d5db;
}
td { padding: 5pt 7pt; border-bottom: 1px solid #e6e8ec; vertical-align: top; }
td.num { text-align: right; font-variant-numeric: tabular-nums; white-space: nowrap; }
tr:nth-child(even) td { background: #f9fafb; }
td.empty { text-align: center; color: #9ca3af; padding: 14pt; }
/* Offer page */
.offer-table { margin-top: 14pt; }
.offer-table td { font-size: 10pt; }
.offer-total td { font-weight: 700; background: #eff6ff !important; }
.advantages { display: grid; grid-template-columns: 1fr 1fr; gap: 10pt; margin-top: 16pt; }
.advantage-item { border: 1px solid #e6e8ec; border-radius: 6pt; padding: 10pt; }
.advantage-item .adv-title { font-weight: 700; font-size: 10pt; margin-bottom: 4pt; }
.advantage-item .adv-desc { font-size: 9pt; color: #5b6066; }
/* Footer */
.footer {
margin-top: 20pt; padding-top: 8pt;
border-top: 1px solid #e6e8ec; font-size: 8pt; color: #9ca3af;
}
.source-logos { color: #6b7280; font-size: 8pt; margin-top: 4pt; }
"""
# ── HTML builder ─────────────────────────────────────────────────────────────
def _build_html(estimate: AggregatedEstimate, input_snapshot: dict) -> str: # type: ignore[type-arg]
today = dt.date.today().strftime("%d.%m.%Y")
address = _html.escape(input_snapshot.get("address", "—"))
area_m2: float = input_snapshot.get("area_m2", 0)
rooms: int = input_snapshot.get("rooms", 0)
floor: int = input_snapshot.get("floor", 0)
total_floors: int = input_snapshot.get("total_floors", 0)
year_built = input_snapshot.get("year_built")
house_type = input_snapshot.get("house_type")
repair_state = input_snapshot.get("repair_state")
has_balcony = input_snapshot.get("has_balcony")
conf_bg, conf_fg, conf_border = _conf_color(estimate.confidence)
conf_label = _conf_label(estimate.confidence)
rooms_label = "Студия" if rooms == 0 else f"{rooms}-комн."
house_labels = {
"panel": "Панель",
"brick": "Кирпич",
"monolith": "Монолит",
"monolith_brick": "Монолит-кирпич",
"other": "Другое",
}
repair_labels = {
"needs_repair": "Требует ремонта",
"standard": "Стандартный",
"good": "Хороший",
"excellent": "Евроремонт",
}
# Cover params grid items
params = [
("Площадь", f"{area_m2} м²"),
("Комнат", rooms_label),
("Этаж", f"{floor} из {total_floors}"),
]
if year_built:
params.append(("Год постройки", str(year_built)))
if house_type:
params.append(("Тип дома", house_labels.get(house_type, house_type)))
if repair_state:
params.append(("Ремонт", repair_labels.get(repair_state, repair_state)))
if has_balcony is not None:
params.append(("Балкон", "Есть" if has_balcony else "Нет"))
params_html = "".join(
f"""<div class="param-item">
<div class="param-label">{k}</div>
<div class="param-value">{_html.escape(str(v))}</div>
</div>"""
for k, v in params
)
listing_rows = _analog_rows(estimate.analogs, is_deal=False)
deal_rows = _analog_rows(estimate.actual_deals, is_deal=True)
# Trade-in cost table (stub: -7% buyout)
median = estimate.median_price_rub
torg_pct = 7
buyout_pct = 5 # trade-in discount on top
torg = int(median * torg_pct / 100)
buyout_discount = int(median * buyout_pct / 100)
tradein_price = median - torg - buyout_discount
return f"""<!DOCTYPE html>
<html lang="ru">
<head>
<meta charset="UTF-8">
<title>Trade-In Оценка — {address}</title>
</head>
<body>
<!-- ═══════════════════════════ PAGE 1 — COVER ═══════════════════════════ -->
<div class="page">
<div class="cover-header">
<h1>Оценка квартиры — Trade-In</h1>
<div class="cover-meta">{address}</div>
<div class="cover-meta">Сформировано: {today} · Действительно 24 часа</div>
</div>
<div style="display:flex; align-items:flex-start; justify-content:space-between; gap:16pt;">
<div>
<div style="font-size:9pt; color:#5b6066; text-transform:uppercase; letter-spacing:0.05em;">
Оценочная стоимость
</div>
<div class="hero-price">{_fmt_rub_m(estimate.median_price_rub)}</div>
<div class="hero-ppm2">{_fmt_ppm2(estimate.median_price_per_m2)}</div>
<div class="range-label" style="margin-top:8pt;">
Диапазон: {_fmt_rub_m(estimate.range_low_rub)} — {_fmt_rub_m(estimate.range_high_rub)}
· данные за {estimate.period_months} мес.
</div>
</div>
<div class="badge"
style="background:{conf_bg}; color:{conf_fg}; border:1px solid {conf_border};">
Достоверность: {conf_label}<br>
<span style="font-size:8pt; font-weight:400;">{estimate.n_analogs} аналогов</span>
</div>
</div>
<h3 style="margin-top:16pt;">Параметры объекта</h3>
<div class="params-grid">
{params_html}
</div>
<div class="footer">
<div>GenDesign · Trade-In Estimator · mock-данные (Phase 1 MVP)</div>
<div class="source-logos">Источники данных: Циан · Авито · ДомКлик · Росреестр</div>
</div>
</div>
<!-- ═══════════════════════════ PAGE 2 — LISTINGS ════════════════════════ -->
<div class="page">
<h2>Объявления-аналоги ({len(estimate.analogs)})</h2>
<p style="font-size:9pt; color:#6b7280; margin-top:0;">
Аналоги из открытых источников (Циан, Авито, ДомКлик) — схожая комнатность,
площадь ±15%, тот же район. Используются как ориентир рыночной цены.
</p>
<table>
<thead>
<tr>
<th>Адрес</th>
<th>Пл., м²</th>
<th>Комн.</th>
<th>Этаж</th>
<th>Цена</th>
<th>Дата / Экспозиция</th>
</tr>
</thead>
<tbody>
{listing_rows}
</tbody>
</table>
<div class="footer">
<div>GenDesign · Trade-In Estimator · стр. 2</div>
</div>
</div>
<!-- ═══════════════════════════ PAGE 3 — DEALS ═══════════════════════════ -->
<div class="page">
<h2>Фактические сделки ({len(estimate.actual_deals)})</h2>
<p style="font-size:9pt; color:#6b7280; margin-top:0;">
Сделки из Росреестра (ДДУ + переуступка) за последние 12 месяцев — отражают
реальные цены покупки, в отличие от цен предложения.
</p>
<table>
<thead>
<tr>
<th>Адрес</th>
<th>Пл., м²</th>
<th>Комн.</th>
<th>Этаж</th>
<th>Цена сделки</th>
<th>Дата / Срок</th>
</tr>
</thead>
<tbody>
{deal_rows}
</tbody>
</table>
<div class="footer">
<div>GenDesign · Trade-In Estimator · стр. 3</div>
</div>
</div>
<!-- ═══════════════════════════ PAGE 4 — TRADE-IN COST ══════════════════ -->
<div class="page">
<h2>Выкупная стоимость (trade-in)</h2>
<p style="font-size:9pt; color:#6b7280; margin-top:0;">
Ориентировочный расчёт. Финальная цена выкупа согласовывается с менеджером
и зависит от конкретного объекта и условий сделки.
</p>
<table class="offer-table">
<tbody>
<tr>
<td>Рыночная стоимость (медиана)</td>
<td class="num">{_fmt_rub(median)}</td>
</tr>
<tr>
<td>— Торговый дисконт ({torg_pct}%)</td>
<td class="num">−{_fmt_rub(torg)}</td>
</tr>
<tr>
<td>— Дисконт за срочность выкупа ({buyout_pct}%)</td>
<td class="num">−{_fmt_rub(buyout_discount)}</td>
</tr>
<tr class="offer-total">
<td>Выкупная цена (ориентир)</td>
<td class="num">{_fmt_rub(tradein_price)}</td>
</tr>
</tbody>
</table>
<div style="margin-top:14pt; padding:10pt; background:#eff6ff; border-left:3px solid #1d4ed8;
border-radius:0 4pt 4pt 0; font-size:9pt; color:#1e40af;">
<strong>Важно:</strong> данные получены из mock-источников (MVP Phase 1).
В Phase 2 расчёт будет основан на реальных данных Циан/Авито/Росреестр
для конкретного адреса и подтверждён менеджером.
</div>
<h3 style="margin-top:18pt;">4 преимущества trade-in</h3>
<div class="advantages">
<div class="advantage-item">
<div class="adv-title">Скорость</div>
<div class="adv-desc">Сделка за 2–4 недели вместо 3–6 месяцев самостоятельной продажи</div>
</div>
<div class="advantage-item">
<div class="adv-title">Без хлопот</div>
<div class="adv-desc">Показы, торг, документы — всё берёт на себя девелопер</div>
</div>
<div class="advantage-item">
<div class="adv-title">Зачёт в счёт новостройки</div>
<div class="adv-desc">Стоимость квартиры напрямую идёт в оплату нового жилья</div>
</div>
<div class="advantage-item">
<div class="adv-title">Фиксация цены</div>
<div class="adv-desc">Цена новостройки фиксируется на момент подачи заявки</div>
</div>
</div>
<div class="footer">
<div>GenDesign · Trade-In Estimator · стр. 4</div>
<div class="source-logos">
Расчёт носит ориентировочный характер и не является офертой.
</div>
</div>
</div>
</body>
</html>"""
# ── public API ────────────────────────────────────────────────────────────────
def generate_trade_in_pdf(estimate: AggregatedEstimate, input_snapshot: dict) -> bytes: # type: ignore[type-arg]
"""Генерирует 4-страничный WeasyPrint PDF для Trade-In оценки.
Pages:
1. Cover — шапка + адрес + параметры + hero-цена
2. Listings — таблица объявлений-аналогов (top 10)
3. Deals — таблица фактических сделок (last 12 мес.)
4. Offer — выкупная стоимость trade-in + 4 преимущества
Args:
estimate: AggregatedEstimate из БД
input_snapshot: словарь с полями ввода пользователя (address, area_m2, ...)
Returns:
PDF bytes готовые для Response(media_type="application/pdf")
"""
html_str = _build_html(estimate, input_snapshot)
css_str = _build_css()
pdf_bytes = HTML(string=html_str).write_pdf(stylesheets=[CSS(string=css_str)])
logger.info(
"Generated trade-in PDF estimate_id=%s pages=4 size=%d bytes",
estimate.estimate_id,
len(pdf_bytes),
)
return pdf_bytes

193
backend/app/services/scrapers/documents.py Normal file
View file

@ -0,0 +1,193 @@
"""Parser и upsert для PDF-документов DOM.РФ (декларации, разрешения, проектная,
отчётность, прочее).
Issue #297, sub-task 22i.
Источник данных: /сервисы/api/object/{obj_id}/documents (per-object endpoint,
аналогичный /infrastructure и /photos — не входит в bulk kn/object list).
Ключи payload (наблюдаемые через chrome-devtools, структура DOM.РФ kn-API v1):
[
{
"docTypeId": 1, # int — тип по справочнику DOM.РФ
"docTypeName": "Декларация", # text
"docNum": "66-001686-Д", # text или null
"postedDate": "2024-03-15", # date string "YYYY-MM-DD" или null
"fileUrl": "https://xn--80az8a.../api/ext/file/...",
"fileSize": 1234567, # bytes, может быть null
},
...
]
Поле doc_type нормализуется к одному из канонических значений:
декларация / разрешение / проектная / отчётность / прочее
NOTE: download PDF-файлов — отдельная Celery-задача (future PR).
Этот модуль только парсит и upsert-ит метаданные в domrf_kn_documents.
"""
from __future__ import annotations
import logging
from datetime import date, datetime
from typing import Any
from sqlalchemy import text
from sqlalchemy.orm import Session
logger = logging.getLogger(__name__)
# ── Canonical doc_type mapping ────────────────────────────────────────────────
_DOC_TYPE_MAP: dict[str, str] = {
# Exact substrings → canonical type (case-insensitive, checked in order).
"деклар": "декларация",
"разреш": "разрешение",
"проект": "проектная",
"отчёт": "отчётность",
"отчет": "отчётность", # without ё
}
_DOC_TYPE_FALLBACK = "прочее"
_UPSERT_DOC_SQL = text(
"""
INSERT INTO domrf_kn_documents
(obj_id, doc_type, doc_num, posted_at, file_url, size_bytes)
VALUES
(:obj_id, :doc_type, :doc_num, :posted_at, :file_url, :size_bytes)
ON CONFLICT (obj_id, doc_type, doc_num, file_url) DO UPDATE
SET size_bytes = EXCLUDED.size_bytes,
local_path = COALESCE(domrf_kn_documents.local_path, EXCLUDED.local_path),
downloaded_at = COALESCE(domrf_kn_documents.downloaded_at, EXCLUDED.downloaded_at),
scraped_at = NOW()
"""
)
# ── helpers ───────────────────────────────────────────────────────────────────
def _canonical_doc_type(raw_type: str | None) -> str:
"""Нормализовать docTypeName DOM.РФ к одному из канонических значений."""
if not raw_type:
return _DOC_TYPE_FALLBACK
lower = raw_type.strip().lower()
for substr, canonical in _DOC_TYPE_MAP.items():
if substr in lower:
return canonical
return _DOC_TYPE_FALLBACK
def _parse_date(v: Any) -> date | None:
"""Coerce date-like value to date. Accepts 'YYYY-MM-DD' or datetime."""
if v is None or v == "":
return None
if isinstance(v, date) and not isinstance(v, datetime):
return v
if isinstance(v, datetime):
return v.date()
if isinstance(v, str):
s = v.strip()
for fmt in ("%Y-%m-%d", "%d.%m.%Y", "%d-%m-%Y"):
try:
return datetime.strptime(s, fmt).date()
except ValueError:
pass
return None
# ── Public API ────────────────────────────────────────────────────────────────
def extract_documents(raw_payload: list[dict[str, Any]]) -> list[dict[str, Any]]:
"""Извлечь список документов из payload endpoint /object/{obj_id}/documents.
Возвращает список dict, готовых для передачи в upsert_documents:
obj_id — передаётся отдельно в upsert_documents, здесь NOT SET
doc_type — канонический тип (декларация/разрешение/…)
doc_num — номер документа или None
posted_at — дата публикации или None
file_url — URL PDF (обязательно, строки без URL пропускаются)
size_bytes — размер файла в байтах или None
Записи без file_url пропускаются с warning.
"""
result: list[dict[str, Any]] = []
for item in raw_payload:
if not isinstance(item, dict):
continue
file_url: str | None = item.get("fileUrl") or item.get("file_url") or None
if not file_url or not isinstance(file_url, str) or not file_url.startswith("http"):
logger.warning("domrf documents: skipping item without valid fileUrl: %s", item)
continue
raw_type = item.get("docTypeName") or item.get("doc_type_name") or None
doc_num_raw = item.get("docNum") or item.get("doc_num") or None
doc_num = str(doc_num_raw).strip() if doc_num_raw is not None else None
size_raw = item.get("fileSize") or item.get("file_size") or None
try:
size_bytes: int | None = int(size_raw) if size_raw is not None else None
except (TypeError, ValueError):
size_bytes = None
result.append(
{
"doc_type": _canonical_doc_type(raw_type),
"doc_num": doc_num or None,
"posted_at": _parse_date(item.get("postedDate") or item.get("posted_date")),
"file_url": file_url.strip(),
"size_bytes": size_bytes,
}
)
return result
def upsert_documents(db: Session, obj_id: int, docs: list[dict[str, Any]]) -> tuple[int, int]:
"""INSERT/UPDATE документов объекта в domrf_kn_documents.
Использует SAVEPOINT (begin_nested) per-row чтобы один битый URL
не откатывал всю транзакцию.
Возвращает (inserted_or_updated, skipped).
"""
if not docs:
return 0, 0
ok = 0
skip = 0
for doc in docs:
params: dict[str, Any] = {
"obj_id": obj_id,
"doc_type": doc["doc_type"],
"doc_num": doc.get("doc_num"),
"posted_at": doc.get("posted_at"),
"file_url": doc["file_url"],
"size_bytes": doc.get("size_bytes"),
}
try:
with db.begin_nested():
db.execute(_UPSERT_DOC_SQL, params)
ok += 1
except Exception as exc:
logger.warning(
"upsert document obj=%s url=%s failed: %s",
obj_id,
doc.get("file_url"),
exc,
)
skip += 1
db.commit()
logger.info("domrf documents obj=%s: upserted=%d skipped=%d", obj_id, ok, skip)
return ok, skip
# ── Stub for future Celery download task ──────────────────────────────────────
def download_document_stub(obj_id: int, doc_id: int, file_url: str) -> None:
"""Placeholder для будущей Celery-задачи download_domrf_documents.
Скачивает PDF, сохраняет в data/raw/domrf_docs/{obj_id}/{filename},
обновляет domrf_kn_documents.local_path + downloaded_at.
Реализация — отдельный PR (future task).
"""
raise NotImplementedError("PDF download not implemented in 22i. See issue #297 future PR.")

563
backend/app/services/scrapers/domrf_catalog.py Normal file
View file

@ -0,0 +1,563 @@
"""DOM.РФ catalog-квартир HTML scraper (issue #297 22d).
kn-API не возвращает цену для большинства квартир (91.5% NULL). Цены живут на
отдельной странице каталога:
https://наш.дом.рф/сервисы/каталог-квартир/квартира/{catalog_url_hash}
Этот модуль:
1. Строит URL каталога по `catalog_url_hash` (колонка появляется после миграции 22b).
2. Получает SSR-HTML через BrowserSession (Playwright, anti-bot — тот же паттерн
что и get_json, но возвращает HTML text вместо JSON).
3. Извлекает price_rub, status, finishing_type, ceiling_height_m, section_no,
catalog_updated_at из HTML с помощью stdlib `html.parser` + regex.
4. Пишет только catalog-derived поля через UPDATE ... WHERE ods_id = :ods_id —
НЕ перетирает kn-API метаданные (total_area, rooms и т.д.).
Зависимости: нет новых. Использует `html.parser` из stdlib + `re`.
NOTE: beautifulsoup4 НЕ установлен (нет в pyproject.toml). Если потребуется
структурированный парсинг — добавить `beautifulsoup4>=4.12` в pyproject.toml
и заменить _HtmlTextExtractor на `BeautifulSoup(html, "html.parser")`.
Контекст Roadmap:
- Phase 5 (22d) — catalog scraper для цен
- Согласно update 2026-05-17 (Objective goldmine): Objective уже содержит 81.4%
цен. Для `domrf_kn_flats` этот scraper остаётся полезен для полей:
finishing_type, ceiling_height_m, section_no, catalog_updated_at, catalog_url_hash.
Price coverage через Objective (OBJ-3) — приоритетнее.
Wiring (отдельный PR):
- Celery task: `backend/app/workers/tasks/scrape_catalog.py`
- Beat schedule: кварть + `catalog_updated_at < NOW() - INTERVAL '30 days'`
"""
from __future__ import annotations
import asyncio
import logging
import re
from datetime import date
from html.parser import HTMLParser
from typing import Any
from sqlalchemy import text
from sqlalchemy.orm import Session
from app.services.scrapers.stealth import BASE_URL, BrowserSession, jitter_sleep
logger = logging.getLogger(__name__)
# Per-flat catalog page URL template (IDN encoded — same as what browsers send).
# Человекочитаемый вид: https://наш.дом.рф/сервисы/каталог-квартир/квартира/{hash}
CATALOG_FLAT_PATH = "/сервисы/каталог-квартир/квартира/{catalog_url_hash}"
# JS snippet: выполняется внутри живой Playwright-страницы.
# Возвращает HTML текст страницы (text/html).
# Это аналог _FETCH_JS из stealth.py, но для text/html вместо application/json.
_FETCH_HTML_JS = """
async ({url}) => {
try {
const r = await fetch(url, {credentials: 'include'});
const ctype = r.headers.get('content-type') || '';
const body = await r.text();
return {ok: r.ok, status: r.status, body, contentType: ctype};
} catch (e) {
return {ok: false, status: 0, body: String(e), contentType: ''};
}
}
"""
# Нормализованные значения статуса продажи.
STATUS_FREE = "free"
STATUS_SOLD = "sold"
STATUS_RESERVED = "reserved"
# ── HTML fetching ─────────────────────────────────────────────────────────────
async def fetch_catalog_html(session: BrowserSession, catalog_url_hash: str) -> str:
"""Получить SSR-HTML страницы квартиры в каталоге DOM.РФ.
Использует тот же паттерн что get_json(): fetch() внутри живой Playwright-страницы.
Так WAF-fingerprint идентичен браузеру, cookies проброшены автоматически.
Raises:
RuntimeError: при транзиентной ошибке после 5 попыток.
WafBlockedError: (из stealth) если вернулся JS-challenge вместо HTML.
"""
if session._page is None:
raise RuntimeError("BrowserSession not bootstrapped")
url = BASE_URL + CATALOG_FLAT_PATH.format(catalog_url_hash=catalog_url_hash)
last_err: Exception | None = None
for attempt in range(5):
async with session._sem:
await jitter_sleep()
try:
session._request_count += 1
result = await session._page.evaluate(_FETCH_HTML_JS, {"url": url})
except Exception as exc:
last_err = exc
logger.warning(
"catalog html evaluate err attempt=%d hash=%s: %r",
attempt,
catalog_url_hash,
exc,
)
await asyncio.sleep(2**attempt)
continue
status: int = result.get("status", 0)
body: str = result.get("body", "")
ctype: str = result.get("contentType", "")
if status in (429,) or status >= 500 or status == 0:
last_err = RuntimeError(f"transient status={status}")
logger.warning(
"catalog html transient status=%d attempt=%d hash=%s, backing off",
status,
attempt,
catalog_url_hash,
)
await asyncio.sleep(2**attempt)
continue
if status == 404:
raise RuntimeError(f"catalog 404 for hash={catalog_url_hash}")
if status != 200:
raise RuntimeError(f"catalog http {status}: {body[:200]} hash={catalog_url_hash}")
# Успех — но нужно проверить что не пришёл WAF JS-challenge (нет text/html)
# Страница каталога — SSR, всегда text/html. Если что-то другое — WAF.
if body and "text/html" not in ctype and "<!doctype" not in body[:100].lower():
logger.warning(
"catalog unexpected content-type=%r for hash=%s, treating as HTML anyway",
ctype,
catalog_url_hash,
)
if not body:
raise RuntimeError(f"catalog empty body for hash={catalog_url_hash}")
return body
raise RuntimeError(f"catalog html max retries exhausted hash={catalog_url_hash}: {last_err!r}")
# ── HTML parsing ──────────────────────────────────────────────────────────────
class _TextCollector(HTMLParser):
"""Минимальный HTMLParser: собирает текстовые блоки с сохранением атрибутов class/data-*.
Вместо полного DOM строим плоский список (tag, attrs_dict, text) для regex-поиска.
Это сознательный trade-off: не нужен beautifulsoup4 — stdlib достаточно для
extraction известных структур страницы каталога.
"""
def __init__(self) -> None:
super().__init__()
self._stack: list[tuple[str, dict[str, str]]] = []
# Список записей: (class_hint, full_text)
self.blocks: list[tuple[str, str]] = []
self._buf: list[str] = []
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
attr_dict = {k: (v or "") for k, v in attrs}
self._stack.append((tag, attr_dict))
self._buf.append("") # начало нового буфера для этого тега
def handle_endtag(self, _tag: str) -> None:
if not self._stack:
return
tag, attr_dict = self._stack.pop()
text = (self._buf.pop() if self._buf else "").strip()
cls = attr_dict.get("class", "")
if text and (cls or tag in ("h1", "h2", "h3", "p", "span", "div", "li")):
self.blocks.append((cls, text))
# Propagate accumulated text up to parent buffer
if self._buf:
self._buf[-1] += " " + text
def handle_data(self, data: str) -> None:
if self._buf:
self._buf[-1] += data
def _find_text_near(
blocks: list[tuple[str, str]], label_pattern: str, value_pattern: str | None = None
) -> str | None:
"""Найти текстовое значение рядом с блоком, matching label_pattern.
Стратегия: ищем блок где text матчит label_pattern — берём следующий блок
как значение (value_pattern если задан).
"""
label_re = re.compile(label_pattern, re.IGNORECASE)
for i, (_cls, block_text) in enumerate(blocks):
if label_re.search(block_text):
# Попробовать next block
for j in range(i + 1, min(i + 4, len(blocks))):
candidate = blocks[j][1].strip()
if candidate:
if value_pattern is None:
return candidate
if re.search(value_pattern, candidate, re.IGNORECASE):
return candidate
return None
def parse_catalog_flat(html: str) -> dict[str, Any]:
"""Извлечь поля из SSR-HTML страницы квартиры DOM.РФ.
Возвращаемые поля (None если не найдено):
- price_rub (int) Цена квартиры в рублях
- price_per_m2 (float) Цена за м² (если указана отдельно)
- status (str) 'free' | 'sold' | 'reserved'
- finishing_type (str) Тип отделки (Предчистовая, Чистовая, Без отделки, ...)
- ceiling_height_m (float) Высота потолков в метрах
- section_no (int) Номер подъезда / секции
- catalog_updated_at (date) Дата обновления информации на странице
Парсинг хрупкий по природе (SSR HTML DOM.РФ меняется без уведомлений).
Все extraction best-effort — KeyError/AttributeError обёрнуты внутри.
"""
result: dict[str, Any] = {}
# ── Шаг 1: собрать все текстовые блоки через HTMLParser ──────────────────
collector = _TextCollector()
try:
collector.feed(html)
except Exception as exc:
logger.warning("html parse error (non-fatal): %s", exc)
blocks = collector.blocks
# ── Шаг 2: regex-extraction из полного HTML текста ────────────────────────
# Страница DOM.РФ SSR встраивает данные и в мета-тегах и в JSON-LD.
# Ищем в сыром HTML — надёжнее чем DOM-обход для хрупкой структуры.
# Price: "7 890 000 ₽" или "7 890 000 руб"
price_match = re.search(
r"([\d][\d\s]{3,12}[\d])\s*(?:₽|руб)",
html,
re.UNICODE,
)
if price_match:
raw_price = re.sub(r"\s+", "", price_match.group(1))
try:
price_val = int(raw_price)
# Санity: цена квартиры в ЕКБ от 1 до 500 млн
if 1_000_000 <= price_val <= 500_000_000:
result["price_rub"] = price_val
except ValueError:
pass
# Price per m²: "217 835 ₽/м²"
ppm2_match = re.search(
r"([\d][\d\s]{2,9}[\d])\s*(?:₽|руб)[/⁄](?:м²|кв\.?\s*м)",
html,
re.UNICODE,
)
if ppm2_match:
raw_ppm2 = re.sub(r"\s+", "", ppm2_match.group(1))
try:
result["price_per_m2"] = float(raw_ppm2)
except ValueError:
pass
# Status: ищем характерные слова рядом с "статус" или в badge
status_match = re.search(
r"(в\s*продаже|свободна|free|продано|sold|забронирована|бронь|reserved)",
html,
re.IGNORECASE | re.UNICODE,
)
if status_match:
s = status_match.group(1).lower()
if any(kw in s for kw in ("продаже", "свободна", "free")):
result["status"] = STATUS_FREE
elif any(kw in s for kw in ("продано", "sold")):
result["status"] = STATUS_SOLD
elif any(kw in s for kw in ("бронь", "забронирована", "reserved")):
result["status"] = STATUS_RESERVED
# Finishing type: "Предчистовая", "Чистовая", "Без отделки", "Под ключ"
finishing_match = re.search(
r"(предчистовая|чистовая|без\s+отделки|под\s+ключ|white\s+box|whitebox)",
html,
re.IGNORECASE | re.UNICODE,
)
if finishing_match:
result["finishing_type"] = finishing_match.group(1).strip().capitalize()
# Ceiling height: "2,7 м" или "2.7 м" или "высота потолков 2,7"
ceiling_match = re.search(
r"(?:высота\s*потолков?|потолки?)\D{0,20}?([\d][,.][\d])\s*м",
html,
re.IGNORECASE | re.UNICODE,
)
if not ceiling_match:
# Fallback: просто "2,7 м" в характеристиках квартиры (диапазон 2.0–4.5 м)
ceiling_match = re.search(
r"\b([2-4][,.][\d])\s*м\b",
html,
re.UNICODE,
)
if ceiling_match:
raw_ceil = ceiling_match.group(1).replace(",", ".")
try:
ceil_val = float(raw_ceil)
if 2.0 <= ceil_val <= 6.0:
result["ceiling_height_m"] = ceil_val
except ValueError:
pass
# Section (подъезд): "Подъезд 1", "Секция 3", "Подъезд №2"
section_match = re.search(
r"(?:подъезд|секция)\s*[№#]?\s*(\d+)",
html,
re.IGNORECASE | re.UNICODE,
)
if section_match:
try:
result["section_no"] = int(section_match.group(1))
except ValueError:
pass
# catalog_updated_at: "Информация обновлена 17.04.2026" или "Обновлено 17.04.2026"
updated_match = re.search(
r"(?:информация\s+обновлена|обновлено|обновлён?а?)\D{0,10}?(\d{1,2}[./]\d{1,2}[./]\d{4})",
html,
re.IGNORECASE | re.UNICODE,
)
if updated_match:
raw_dt = updated_match.group(1).replace("/", ".")
try:
parts = raw_dt.split(".")
if len(parts) == 3:
result["catalog_updated_at"] = date(int(parts[2]), int(parts[1]), int(parts[0]))
except (ValueError, IndexError):
pass
# ── Шаг 3: блочный fallback для ceiling / section (если regex не нашёл) ──
if "ceiling_height_m" not in result:
candidate = _find_text_near(blocks, r"потолк|высота", r"[23][,.][\d]")
if candidate:
m = re.search(r"([23][,.][\d])", candidate)
if m:
try:
v = float(m.group(1).replace(",", "."))
if 2.0 <= v <= 6.0:
result["ceiling_height_m"] = v
except ValueError:
pass
if "section_no" not in result:
candidate = _find_text_near(blocks, r"подъезд|секция", r"^\d+$")
if candidate:
try:
result["section_no"] = int(candidate.strip())
except ValueError:
pass
logger.debug(
"parse_catalog_flat: extracted fields=%s",
list(result.keys()),
)
return result
# ── DB writes ─────────────────────────────────────────────────────────────────
def upsert_catalog_data(
db: Session, ods_id: str, catalog_url_hash: str, data: dict[str, Any]
) -> bool:
"""UPDATE catalog-derived поля в domrf_kn_flats для конкретной квартиры.
Обновляет ТОЛЬКО catalog-only колонки:
price_rub, price_per_m2, status, finishing_type, ceiling_height_m,
section_no, catalog_updated_at, catalog_url_hash.
НЕ трогает: total_area, rooms, floor, num_floors, flat_type, obj_id и
другие kn-API метаданные.
Использует COALESCE: если новое значение NULL — старое сохраняется.
Это позволяет повторно запускать scraper не затирая частично заполненные поля.
ВАЖНО: колонки section_no, finishing_type, ceiling_height_m,
catalog_updated_at, catalog_url_hash должны существовать в таблице.
Они появляются после миграции 22b. Если таблица старая — UPDATE упадёт
с 'column does not exist'. Решение: сначала выполнить data/sql/NN_22b_flats_cols.sql.
Возвращает True если строка найдена и обновлена, False если ods_id не найден.
"""
params: dict[str, Any] = {
"ods_id": ods_id,
"catalog_url_hash": catalog_url_hash,
"price_rub": data.get("price_rub"),
"price_per_m2": data.get("price_per_m2"),
"status": data.get("status"),
"finishing_type": data.get("finishing_type"),
"ceiling_height_m": data.get("ceiling_height_m"),
"section_no": data.get("section_no"),
"catalog_updated_at": data.get("catalog_updated_at"),
}
try:
with db.begin_nested():
result = db.execute(
text(
"""
UPDATE domrf_kn_flats SET
catalog_url_hash = :catalog_url_hash,
price_rub = COALESCE(:price_rub, price_rub),
price_per_m2 = COALESCE(:price_per_m2, price_per_m2),
status = COALESCE(:status, status),
finishing_type = COALESCE(:finishing_type, finishing_type),
ceiling_height_m = COALESCE(:ceiling_height_m, ceiling_height_m),
section_no = COALESCE(:section_no, section_no),
catalog_updated_at = COALESCE(:catalog_updated_at, catalog_updated_at)
WHERE ods_id = :ods_id
"""
),
params,
)
except Exception as exc:
logger.warning("upsert_catalog_data ods_id=%s failed: %s", ods_id, exc)
return False
rows_affected: int = result.rowcount or 0
if rows_affected == 0:
logger.warning("upsert_catalog_data: ods_id=%s not found in domrf_kn_flats", ods_id)
return rows_affected > 0
# ── Per-flat scrape orchestration ─────────────────────────────────────────────
async def scrape_one_flat(
session: BrowserSession,
db: Session,
ods_id: str,
catalog_url_hash: str,
) -> dict[str, Any]:
"""Scrape одной квартиры: fetch HTML → parse → upsert.
Возвращает dict с результатом: {ods_id, success, fields_extracted, updated}.
Ошибки fetch/parse логируются, не бросаются — caller обрабатывает результат.
"""
outcome: dict[str, Any] = {
"ods_id": ods_id,
"catalog_url_hash": catalog_url_hash,
"success": False,
"fields_extracted": 0,
"updated": False,
"error": None,
}
try:
html = await fetch_catalog_html(session, catalog_url_hash)
except Exception as exc:
logger.warning(
"catalog fetch failed ods_id=%s hash=%s: %s",
ods_id,
catalog_url_hash,
exc,
)
outcome["error"] = str(exc)[:500]
return outcome
try:
data = parse_catalog_flat(html)
except Exception as exc:
logger.warning(
"catalog parse failed ods_id=%s hash=%s: %s",
ods_id,
catalog_url_hash,
exc,
)
outcome["error"] = f"parse: {exc!s}"[:500]
return outcome
outcome["fields_extracted"] = len([v for v in data.values() if v is not None])
outcome["updated"] = upsert_catalog_data(db, ods_id, catalog_url_hash, data)
outcome["success"] = True
logger.info(
"catalog scrape ods_id=%s: fields=%d updated=%s",
ods_id,
outcome["fields_extracted"],
outcome["updated"],
)
return outcome
async def scrape_catalog_batch(
db: Session,
flats: list[dict[str, Any]],
region_code: int = 66,
headed: bool = False,
load_state: str | None = None,
) -> dict[str, Any]:
"""Scrape пачки квартир каталога DOM.РФ.
`flats` — список dict'ов с ключами {ods_id, catalog_url_hash}.
Типовой источник: SELECT ods_id, catalog_url_hash FROM domrf_kn_flats
WHERE catalog_url_hash IS NOT NULL
AND (catalog_updated_at IS NULL OR catalog_updated_at < NOW() - INTERVAL '30 days')
LIMIT :batch_size.
Использует один BrowserSession на весь пакет (bootstrapped 1 раз).
jitter_sleep между запросами встроен в fetch_catalog_html (через BrowserSession._sem).
Returns:
{total, success, failed, fields_total}
"""
stats: dict[str, Any] = {
"total": len(flats),
"success": 0,
"failed": 0,
"fields_total": 0,
}
if not flats:
logger.info("scrape_catalog_batch: empty batch, nothing to do")
return stats
logger.info(
"scrape_catalog_batch: starting %d flats region=%d",
len(flats),
region_code,
)
async with BrowserSession(
region_code=region_code,
headed=headed,
load_state=load_state,
# auth=None — страницы каталога публичные, Basic auth не нужен
auth=None,
) as session:
for flat in flats:
ods_id = flat.get("ods_id", "")
catalog_url_hash = flat.get("catalog_url_hash", "")
if not ods_id or not catalog_url_hash:
logger.warning("scrape_catalog_batch: skip flat with missing ods_id/hash: %r", flat)
stats["failed"] += 1
continue
outcome = await scrape_one_flat(session, db, ods_id, catalog_url_hash)
if outcome["success"]:
stats["success"] += 1
stats["fields_total"] += outcome["fields_extracted"]
else:
stats["failed"] += 1
logger.info(
"scrape_catalog_batch done: total=%d success=%d failed=%d fields_total=%d",
stats["total"],
stats["success"],
stats["failed"],
stats["fields_total"],
)
return stats

468
backend/app/services/scrapers/domrf_catalog_object.py Normal file
View file

@ -0,0 +1,468 @@
"""DOM.РФ catalog-OBJECT scraper (issue #297 sub-task 22d).
Fills ~25 NULL columns in domrf_kn_objects from public SSR catalog page:
https://наш.дом.рф/сервисы/каталог-новостроек/объект/{obj_id}
kn-API не возвращает: wall_type, energy_eff, ceiling_height_m, parking_*,
playground_*, finishing_variants_count, etc. — все эти поля есть в
__NEXT_DATA__ JSON блоке на странице каталога (Next.js SSR).
Uses BrowserSession from app.services.scrapers.stealth (Playwright + WAF bypass).
Fetches HTML, extracts __NEXT_DATA__ JSON, maps to DB columns,
UPDATE domrf_kn_objects WHERE obj_id = :id (не перетирает kn-API данные).
"""
from __future__ import annotations
import asyncio
import json
import logging
import re
from datetime import date
from typing import Any
from sqlalchemy import text
from sqlalchemy.orm import Session
from app.services.scrapers.stealth import BASE_URL, BrowserSession, WafBlockedError, jitter_sleep
logger = logging.getLogger(__name__)
# URL шаблон страницы объекта в каталоге DOM.РФ.
# Человекочитаемый вид: https://наш.дом.рф/сервисы/каталог-новостроек/объект/{obj_id}
CATALOG_OBJECT_PATH = "/сервисы/каталог-новостроек/объект/{obj_id}"
# JS snippet — аналог _FETCH_HTML_JS из domrf_catalog.py.
# Выполняется внутри живой Playwright-страницы, возвращает HTML текст.
_FETCH_HTML_JS = """
async ({url}) => {
try {
const r = await fetch(url, {credentials: 'include'});
const ctype = r.headers.get('content-type') || '';
const body = await r.text();
return {ok: r.ok, status: r.status, body, contentType: ctype};
} catch (e) {
return {ok: false, status: 0, body: String(e), contentType: ''};
}
}
"""
# UPDATE SQL — обновляет только catalog-derived поля.
# COALESCE гарантирует что NULL-значение не перетирает уже заполненное поле.
UPDATE_OBJECT_CATALOG_SQL = text(
"""
UPDATE domrf_kn_objects SET
obj_class = COALESCE(:obj_class, obj_class),
wall_type = COALESCE(:wall_type, wall_type),
energy_eff = COALESCE(:energy_eff, energy_eff),
section_count = COALESCE(:section_count, section_count),
parking_total_slots = COALESCE(:parking_total_slots, parking_total_slots),
guest_parking_inside_count = COALESCE(
:guest_parking_inside_count, guest_parking_inside_count
),
guest_parking_outside_count = COALESCE(
:guest_parking_outside_count, guest_parking_outside_count
),
ceiling_height_m = COALESCE(:ceiling_height_m, ceiling_height_m),
finishing_variants_count = COALESCE(:finishing_variants_count, finishing_variants_count),
has_free_planning = COALESCE(:has_free_planning, has_free_planning),
avg_flat_area_m2 = COALESCE(:avg_flat_area_m2, avg_flat_area_m2),
elevators_passenger_count = COALESCE(
:elevators_passenger_count, elevators_passenger_count
),
elevators_cargo_count = COALESCE(:elevators_cargo_count, elevators_cargo_count),
playground_kids_count = COALESCE(:playground_kids_count, playground_kids_count),
playground_sport_count = COALESCE(:playground_sport_count, playground_sport_count),
has_bike_paths = COALESCE(:has_bike_paths, has_bike_paths),
trash_areas_count = COALESCE(:trash_areas_count, trash_areas_count),
has_ramp = COALESCE(:has_ramp, has_ramp),
has_low_platforms = COALESCE(:has_low_platforms, has_low_platforms),
has_wheelchair_lift = COALESCE(:has_wheelchair_lift, has_wheelchair_lift),
first_floor_type = COALESCE(:first_floor_type, first_floor_type),
parking_provision_pct = COALESCE(:parking_provision_pct, parking_provision_pct),
project_published_at = COALESCE(:project_published_at, project_published_at),
project_declaration_num = COALESCE(:project_declaration_num, project_declaration_num),
domrf_score_infrastructure = COALESCE(
:domrf_score_infrastructure, domrf_score_infrastructure
),
domrf_score_transport = COALESCE(:domrf_score_transport, domrf_score_transport),
catalog_scraped_at = NOW()
WHERE obj_id = :obj_id
AND snapshot_date = :snapshot_date
"""
)
# ── Value helpers ─────────────────────────────────────────────────────────────
def _to_numeric_comma(s: Any) -> float | None:
"""Конвертировать строку с запятой-десятичным разделителем в float.
Примеры: "2,7" → 2.7; "2.7" → 2.7; "" → None; None → None.
"""
if s is None:
return None
raw = str(s).strip().replace(",", ".")
if not raw:
return None
try:
return float(raw)
except ValueError:
return None
def _to_date_ddmmyyyy(s: Any) -> date | None:
"""Конвертировать строку "DD.MM.YYYY" в date.
Примеры: "31.03.2025" → date(2025, 3, 31); "" → None; invalid → None.
"""
if not s:
return None
raw = str(s).strip()
if not raw:
return None
try:
parts = raw.split(".")
if len(parts) == 3:
return date(int(parts[2]), int(parts[1]), int(parts[0]))
except (ValueError, IndexError):
pass
return None
def _to_bool_int(v: Any) -> bool | None:
"""Конвертировать 0/1 (или любое int-like) в bool.
Примеры: 1 → True; 0 → False; None → None; 3 → True (>0).
"""
if v is None:
return None
try:
return int(v) > 0
except (ValueError, TypeError):
return None
def _to_bool_da_net(s: Any) -> bool | None:
"""Конвертировать "Да"/"Нет" строку в bool.
Примеры: "Да" → True; "Нет" → False; "" → None; None → None.
"""
if s is None:
return None
raw = str(s).strip().lower()
if raw == "да":
return True
if raw == "нет":
return False
return None
def _safe_int(v: Any) -> int | None:
"""Безопасная конвертация в int, None при ошибке."""
if v is None:
return None
try:
return int(v)
except (ValueError, TypeError):
return None
# ── HTML fetching ─────────────────────────────────────────────────────────────
async def fetch_catalog_object_html(session: BrowserSession, obj_id: int) -> str:
"""Получить SSR-HTML страницы объекта в каталоге DOM.РФ.
Использует тот же паттерн что fetch_catalog_html из domrf_catalog.py:
fetch() внутри живой Playwright-страницы — WAF-fingerprint идентичен браузеру.
Raises:
WafBlockedError: если вернулся не-HTML (JS-challenge или JSON).
RuntimeError: при 404 или исчерпании попыток.
"""
if session._page is None:
raise RuntimeError("BrowserSession not bootstrapped")
url = BASE_URL + CATALOG_OBJECT_PATH.format(obj_id=obj_id)
last_err: Exception | None = None
for attempt in range(5):
async with session._sem:
await jitter_sleep(300, 700)
try:
session._request_count += 1
result = await session._page.evaluate(_FETCH_HTML_JS, {"url": url})
except Exception as exc:
last_err = exc
logger.warning(
"catalog_object html evaluate err attempt=%d obj_id=%d: %r",
attempt,
obj_id,
exc,
)
await asyncio.sleep(2**attempt)
continue
status: int = result.get("status", 0)
body: str = result.get("body", "")
ctype: str = result.get("contentType", "")
if status in (429,) or status >= 500 or status == 0:
last_err = RuntimeError(f"transient status={status}")
logger.warning(
"catalog_object transient status=%d attempt=%d obj_id=%d, backing off",
status,
attempt,
obj_id,
)
await asyncio.sleep(2**attempt)
continue
if status == 404:
raise RuntimeError(f"catalog_object 404 for obj_id={obj_id}")
if status != 200:
raise RuntimeError(f"catalog_object http {status}: {body[:200]} obj_id={obj_id}")
# Проверяем что вернулся HTML, а не WAF JS-challenge.
is_html = "text/html" in ctype or "<!doctype" in body[:100].lower()
if body and not is_html:
raise WafBlockedError(
f"non-HTML response for obj_id={obj_id}: status={status} ctype={ctype!r}"
f" body[:120]={body[:120]!r}"
)
if not body:
raise RuntimeError(f"catalog_object empty body for obj_id={obj_id}")
return body
raise RuntimeError(f"catalog_object html max retries exhausted obj_id={obj_id}: {last_err!r}")
# ── __NEXT_DATA__ extraction ──────────────────────────────────────────────────
def extract_next_data(html: str) -> dict[str, Any]:
"""Извлечь JSON из тега <script id="__NEXT_DATA__"> в SSR HTML.
Raises:
ValueError: если тег не найден или JSON не парсится.
"""
match = re.search(
r'<script\s+id=["\']__NEXT_DATA__["\'][^>]*>(.+?)</script>',
html,
re.DOTALL,
)
if not match:
raise ValueError("__NEXT_DATA__ script tag not found in HTML")
raw_json = match.group(1).strip()
try:
return json.loads(raw_json) # type: ignore[no-any-return]
except json.JSONDecodeError as exc:
raise ValueError(f"__NEXT_DATA__ JSON parse error: {exc}") from exc
# ── Field mapping ─────────────────────────────────────────────────────────────
def parse_catalog_object(next_data: dict[str, Any]) -> dict[str, Any]:
"""Извлечь поля объекта из __NEXT_DATA__ и вернуть dict для UPDATE.
Все .get() безопасны — partial responses OK, отсутствующие поля = None.
Возвращает dict с bind-параметрами для UPDATE_OBJECT_CATALOG_SQL.
"""
pp: dict[str, Any] = next_data.get("props", {}).get("pageProps", {})
ai: dict[str, Any] = pp.get("additionalInfo") or {}
quart: dict[str, Any] = pp.get("quartography") or {}
indexes: dict[str, Any] = pp.get("indexes") or {}
decl: dict[str, Any] = pp.get("projectDeclaration") or {}
# first_floor_type: 1 = нежилой, 0 = жилой
first_floor_raw = quart.get("nonLivFirstFloor")
first_floor_type: str | None = None
if first_floor_raw is not None:
try:
first_floor_type = "нежилой" if int(first_floor_raw) == 1 else "жилой"
except (ValueError, TypeError):
pass
# elevators_cargo_count = cargoElevatorsCount + cargoPassengerElevatorCount
cargo = _safe_int(ai.get("cargoElevatorsCount"))
cargo_pass = _safe_int(ai.get("cargoPassengerElevatorCount"))
if cargo is not None or cargo_pass is not None:
elevators_cargo_count: int | None = (cargo or 0) + (cargo_pass or 0)
else:
elevators_cargo_count = None
return {
"obj_class": pp.get("buildingClass"),
"wall_type": pp.get("wallMaterial"),
"energy_eff": pp.get("objEnergyEfficiency"),
"section_count": _safe_int(quart.get("objLivElemEntrCnt")),
"parking_total_slots": _safe_int(pp.get("parkingCount")),
"guest_parking_inside_count": _safe_int(ai.get("objectParkingPlaces")),
"guest_parking_outside_count": _safe_int(ai.get("nearbyParkingPlaces")),
"ceiling_height_m": _to_numeric_comma(ai.get("ceilingHeight")),
"finishing_variants_count": _safe_int(pp.get("finishTypeCount")),
"has_free_planning": _to_bool_da_net(pp.get("freePlan")),
"avg_flat_area_m2": _to_numeric_comma(quart.get("objLivElemSqAvg")),
"elevators_passenger_count": _safe_int(ai.get("passengerElevatorsCount")),
"elevators_cargo_count": elevators_cargo_count,
"playground_kids_count": _safe_int(ai.get("playgroundsCount")),
"playground_sport_count": _safe_int(ai.get("sportsgroundCount")),
"has_bike_paths": _to_bool_int(ai.get("bicycleLane")),
"trash_areas_count": _safe_int(ai.get("trashAreaCount")),
"has_ramp": _to_bool_int(ai.get("ramp")),
"has_low_platforms": _to_bool_int(ai.get("curbLowering")),
"has_wheelchair_lift": _to_bool_int(ai.get("wheelchairElevatorsCount")),
"first_floor_type": first_floor_type,
"parking_provision_pct": _to_numeric_comma(ai.get("parkingAvailabilityPerc")),
"project_published_at": _to_date_ddmmyyyy(pp.get("publicationDate")),
"project_declaration_num": decl.get("number"),
"domrf_score_infrastructure": _safe_int(indexes.get("infrastructure")),
"domrf_score_transport": _safe_int(indexes.get("transport")),
# TODO: obj_checks (6 detailed checks) — separate investigation (task #21).
# pageProps.isChecked (bool), verificationId, verificationFlg available here
# but detailed per-check breakdown requires separate API investigation.
}
# ── DB write ──────────────────────────────────────────────────────────────────
async def scrape_catalog_object(
db: Session,
session: BrowserSession,
obj_id: int,
snapshot_date: date,
) -> bool:
"""Scrape одного объекта: fetch HTML → extract __NEXT_DATA__ → parse → UPDATE.
Использует SAVEPOINT (begin_nested) для изоляции per-row ошибок.
Логирует результат через logger.info.
Returns:
True если UPDATE затронул строку, False при ошибке или 0 rows.
"""
logger.info("catalog_object scrape start obj_id=%d snapshot_date=%s", obj_id, snapshot_date)
try:
html = await fetch_catalog_object_html(session, obj_id)
except WafBlockedError as exc:
logger.warning("catalog_object WAF blocked obj_id=%d: %s", obj_id, exc)
return False
except Exception as exc:
logger.warning("catalog_object fetch failed obj_id=%d: %s", obj_id, exc)
return False
try:
next_data = extract_next_data(html)
except ValueError as exc:
logger.warning("catalog_object extract_next_data failed obj_id=%d: %s", obj_id, exc)
return False
try:
data = parse_catalog_object(next_data)
except Exception as exc:
logger.warning("catalog_object parse failed obj_id=%d: %s", obj_id, exc)
return False
fields_extracted = len([v for v in data.values() if v is not None])
params: dict[str, Any] = {
"obj_id": obj_id,
"snapshot_date": snapshot_date,
**data,
}
try:
with db.begin_nested():
result = db.execute(UPDATE_OBJECT_CATALOG_SQL, params)
rows_affected: int = result.rowcount or 0
except Exception as exc:
logger.warning("catalog_object UPDATE failed obj_id=%d: %s", obj_id, exc)
return False
if rows_affected == 0:
logger.warning(
"catalog_object UPDATE 0 rows obj_id=%d snapshot_date=%s — not in DB?",
obj_id,
snapshot_date,
)
return False
logger.info(
"catalog_object scraped obj_id=%d fields=%d rows_updated=%d",
obj_id,
fields_extracted,
rows_affected,
)
return True
async def scrape_catalog_objects(
db: Session,
obj_ids: list[int],
snapshot_date: date,
region_code: int = 66,
) -> dict[str, int]:
"""Scrape списка объектов через один BrowserSession.
Запускает один BrowserSession на весь batch; jitter_sleep (300–700 мс)
встроен в fetch_catalog_object_html для защиты от rate-limit.
Returns:
{"processed": N, "succeeded": N, "failed": N, "skipped": N}
"""
stats: dict[str, int] = {
"processed": 0,
"succeeded": 0,
"failed": 0,
"skipped": 0,
}
if not obj_ids:
logger.info("scrape_catalog_objects: empty list, nothing to do")
return stats
logger.info(
"scrape_catalog_objects: starting %d objects region=%d snapshot_date=%s",
len(obj_ids),
region_code,
snapshot_date,
)
async with BrowserSession(
region_code=region_code,
# Страницы каталога публичные — Basic auth не нужен
auth=None,
) as session:
for obj_id in obj_ids:
stats["processed"] += 1
ok = await scrape_catalog_object(db, session, obj_id, snapshot_date)
if ok:
stats["succeeded"] += 1
else:
stats["failed"] += 1
# Commit outer transaction: SAVEPOINT (`begin_nested`) releases внутри loop,
# но outer tx остаётся autobegin'd — без commit() все UPDATE'ы откатятся
# при db.close() в Celery task.
try:
db.commit()
except Exception:
db.rollback()
raise
logger.info(
"scrape_catalog_objects done: processed=%d succeeded=%d failed=%d skipped=%d",
stats["processed"],
stats["succeeded"],
stats["failed"],
stats["skipped"],
)
return stats

707
backend/app/services/scrapers/domrf_kn.py
View file

@ -16,6 +16,7 @@ auth shipped in their frontend bundle).
from __future__ import annotations
import asyncio
import json
import logging
from datetime import date, datetime
@ -26,6 +27,11 @@ from sqlalchemy import text
from sqlalchemy.orm import Session
from app.core.db import SessionLocal
from app.services.scrapers.documents import extract_documents, upsert_documents
# obj_checks import temporarily disabled — endpoint /checks returns 404 (run #19).
# Re-enable with _fetch_obj_checks_safe when endpoint is found (see TODO in Phase B/C).
# from app.services.scrapers.obj_checks import extract_obj_checks, upsert_obj_checks
from app.services.scrapers.stealth import BASE_URL, BrowserSession
logger = logging.getLogger(__name__)
@ -36,6 +42,16 @@ PATH_SALE_GRAPH = "/сервисы/api/object/{obj_id}/sale_graph" # ?type=apar
PATH_SALES_AGG = "/сервисы/api/object/{obj_id}/sales_agg"
PATH_INFRA = "/сервисы/api/object/{obj_id}/infrastructure"
PATH_PHOTOS = "/сервисы/api/object/construction/progress/photo/{obj_id}"
# Five separate document endpoints discovered via Playwright on obj=65136 (2026-05-17).
# Former single PATH_DOCUMENTS (/documents) returned 404 — replaced by these 5 real paths.
PATH_DOC_RPD = "/сервисы/api/object/{obj_id}/document/rpd"
PATH_DOC_DEVELOPER_REPORT = "/сервисы/api/object/{obj_id}/developer/report"
PATH_DOC_PROJECT_DOCUMENTATION = "/сервисы/api/object/{obj_id}/project/documentation"
PATH_DOC_DOCUMENTATION_OTHER = "/сервисы/api/object/{obj_id}/documentation/other"
PATH_DOC_PERMITS = "/сервисы/api/object/{obj_id}/document/permits"
# NOTE: PATH_CHECKS URL not verified via devtools — likely pattern analogous to /infrastructure.
# If endpoint returns 404/error the failure is logged to kn_scrape_failures and scrape continues.
PATH_CHECKS = "/сервисы/api/object/{obj_id}/checks"
RAW_SECTION = "kn_api"
SALE_GRAPH_TYPES = ("apartments", "parking")
@ -115,6 +131,36 @@ def _to_date(v: Any) -> date | None:
return None
_OBJ_CLASS_PATTERNS: list[tuple[str, str]] = [
# Ordered from most to least specific to avoid 'бизнес' matching inside longer phrases.
# Each tuple: (regex pattern, canonical class name)
(r"элит", "Элит"),
(r"бизнес", "Бизнес"),
(r"премиум", "Премиум"),
(r"комфорт", "Комфорт"),
(r"стандарт|эконом", "Стандарт"),
]
def _extract_obj_class_from_ai(ai_description: str | None) -> str | None:
"""Извлечь класс жилья из AI-описания DOM.РФ.
DOM.РФ не отдаёт `objClass` в /kn/object list — поле всегда NULL.
Однако `aiDescription` содержит текстовое упоминание класса ('комфорт-класса',
'«Комфорт»', 'класс «Бизнес»', 'Класс объекта — комфорт' и т.д.).
Возвращает каноническое название или None если описание пустое / класс не найден.
"""
import re
if not ai_description:
return None
for pattern, class_name in _OBJ_CLASS_PATTERNS:
if re.search(pattern, ai_description, re.IGNORECASE):
return class_name
return None
def _problem_text(v: Any) -> str | None:
"""Coerce problem flag (sometimes int 0/1, sometimes text) to text or None."""
if v is None:
@ -156,17 +202,68 @@ def _extract_list(payload: dict[str, Any]) -> list[dict[str, Any]]:
def _norm_object(row: dict[str, Any], region_cd: int | None = None) -> dict[str, Any]:
"""Map naш.дом.рф /kn/object row → domrf_kn_objects column dict.
Real API field names:
Real API field names (confirmed via payload audit 2026-05-17, obj=65136):
objId, hobjId, developer{devId, shortName, fullName, groupName, companyGroup, devInn},
rpdRegionCd, objAddr, shortAddr, objCommercNm, objFloorMin, objFloorMax,
objElemLivingCnt, objSquareLiving, objReady100PercDt, objClass, latitude, longitude,
objProblemFlg, problemFlag, siteStatus, objGreenHouseFlg, objGuarantyEscrowFlg,
objStatus, freeFlatsInfo{priceMin, numberFlats}.
objStatus, aiDescription, freeFlatsInfo{priceMin, numberFlats},
residentialBuildings (section_count), rpdNum (project_declaration_num),
objPublDt (project_published_at), metro{name, time, line, color, colors, isWalk, id}.
Note: DOM.РФ /kn/object list endpoint never populates `objClass` in API responses
(field absent from payload). Class is extracted from `aiDescription` text as fallback.
22e fields NOT in kn-API list payload (remain NULL until catalog scraper 22d):
first_floor_type, elevators_passenger/cargo_count, parking_total_slots,
guest_parking_inside/outside_count, ceiling_height_m, finishing_variants_count,
has_free_planning, playground_kids/sport_count, has_bike_paths, trash_areas_count,
has_ramp, has_low_platforms, has_wheelchair_lift, flat_area_min/max,
price_max_rub, price_per_m2_min/max, parking_provision_pct, avg_flat_area_m2,
domrf_score_location/transport/infrastructure.
"""
dev = row.get("developer") if isinstance(row.get("developer"), dict) else {}
company_group = _g(dev, "companyGroup") if dev else None
# Our DB convention: dev_id="<companyGroup>_0" matches v_developer_full_metrics.
dev_id = f"{company_group}_0" if company_group else None
# objClass is absent from the list endpoint — extract from aiDescription instead.
obj_class = _g(row, "objClass") or _extract_obj_class_from_ai(_g(row, "aiDescription"))
if obj_class is None:
obj_id = _g(row, "objId", "obj_id", "id")
logger.debug(
"obj_class not found for obj_id=%s (no objClass field, no aiDescription class match)",
obj_id,
)
# Metro: kn-API returns a single metro object (nearest station).
# Stored as metro_top3 JSON array (single element) for future multi-station extension.
metro_raw = row.get("metro")
metro_nearest_name: str | None = None
metro_nearest_walk_minutes: int | None = None
metro_top3: list[dict[str, Any]] | None = None
if isinstance(metro_raw, dict) and metro_raw.get("name"):
metro_nearest_name = metro_raw.get("name")
raw_time = metro_raw.get("time")
if raw_time is not None:
try:
metro_nearest_walk_minutes = round(float(raw_time))
except (TypeError, ValueError):
pass
metro_top3 = [
{
"name": metro_raw.get("name"),
"time": raw_time,
"line": metro_raw.get("line"),
"color": metro_raw.get("color"),
"isWalk": metro_raw.get("isWalk"),
}
]
# freeFlatsInfo.priceMin — price of cheapest available flat in the object
free_info = row.get("freeFlatsInfo") if isinstance(row.get("freeFlatsInfo"), dict) else {}
price_min_rub = _g(free_info, "priceMin") if free_info else None
return {
"obj_id": _g(row, "objId", "obj_id", "id"),
"hobj_id": _g(row, "hobjId", "hobj_id"),
@ -186,12 +283,49 @@ def _norm_object(row: dict[str, Any], region_cd: int | None = None) -> dict[str,
"site_status": _g(row, "siteStatus"),
"green_house": _to_bool(_g(row, "objGreenHouseFlg")),
"escrow": _to_bool(_g(row, "objGuarantyEscrowFlg")),
"obj_class": _g(row, "objClass"),
"obj_class": obj_class,
"wall_type": _g(row, "wallType"),
"energy_eff": _g(row, "energyEff"),
"latitude": _g(row, "latitude"),
"longitude": _g(row, "longitude"),
"obj_status": _g(row, "objStatus"),
# 22e: fields mappable from kn-API list payload
"section_count": _g(row, "residentialBuildings"),
"project_declaration_num": _g(row, "rpdNum"),
"project_published_at": _to_date(_g(row, "objPublDt")),
"price_min_rub": price_min_rub,
"dev_group_name": _g(dev, "groupName"),
# 22e + 22h: metro (nearest station from kn-API, single object)
"metro_nearest_name": metro_nearest_name,
"metro_nearest_walk_minutes": metro_nearest_walk_minutes,
"metro_top3": json.dumps(metro_top3, ensure_ascii=False) if metro_top3 else None,
# 22e: fields NOT in kn-API list payload — filled by catalog scraper (22d)
"first_floor_type": None,
"elevators_passenger_count": None,
"elevators_cargo_count": None,
"parking_total_slots": None,
"guest_parking_inside_count": None,
"guest_parking_outside_count": None,
"ceiling_height_m": None,
"finishing_variants_count": None,
"has_free_planning": None,
"avg_flat_area_m2": None,
"playground_kids_count": None,
"playground_sport_count": None,
"has_bike_paths": None,
"trash_areas_count": None,
"has_ramp": None,
"has_low_platforms": None,
"has_wheelchair_lift": None,
"flat_area_min": None,
"flat_area_max": None,
"price_max_rub": None,
"price_per_m2_min": None,
"price_per_m2_max": None,
"parking_provision_pct": None,
"domrf_score_location": None,
"domrf_score_transport": None,
"domrf_score_infrastructure": None,
}
@ -219,6 +353,11 @@ def _norm_flat(row: dict[str, Any], region_cd: int | None) -> dict[str, Any]:
Real fields: flatId|id (numeric, may be null), odsId, elemId (uuid hash),
type, number, isStudio, totalArea, livingArea, rooms, status (free|booked|sold),
price, pricePerSquareMeter, numberFloors. Plus injected _objId and _floor.
Note: /portal/table returns price=null for most objects (sold/booked flats and
objects that don't expose per-flat pricing via this endpoint). price_per_m2 is
derived from price_rub / total_area when the API returns price_rub but omits
pricePerSquareMeter — defensive fallback for partial API responses.
"""
flat_id = _g(row, "flatId", "id")
if flat_id is None:
@ -226,24 +365,47 @@ def _norm_flat(row: dict[str, Any], region_cd: int | None) -> dict[str, Any]:
elem = _g(row, "elemId")
if elem:
flat_id = abs(hash(elem)) % (2**63 - 1)
price_rub = _g(row, "price")
price_per_m2 = _g(row, "pricePerSquareMeter")
total_area = _g(row, "totalArea")
# Derive price_per_m2 when API returns price_rub but omits pricePerSquareMeter.
# Covers cases where the table endpoint has the flat price but no pre-computed m² rate.
if price_per_m2 is None and price_rub is not None and total_area and total_area > 0:
price_per_m2 = round(price_rub / total_area, 2)
logger.info(
"derive price_per_m2=%.2f for flat ods_id=%s obj_id=%s",
price_per_m2,
_g(row, "odsId"),
_g(row, "_objId", "objId"),
)
return {
"id": flat_id,
"ods_id": _g(row, "odsId"),
"flat_type": _g(row, "type", "flatType"),
"flat_number": _g(row, "number", "flatNumber"),
"is_studio": _to_bool(_g(row, "isStudio")),
"total_area": _g(row, "totalArea"),
"total_area": total_area,
"living_area": _g(row, "livingArea"),
"rooms": _g(row, "rooms"),
"status": _g(row, "status"),
"price_rub": _g(row, "price"),
"price_per_m2": _g(row, "pricePerSquareMeter"),
"price_rub": price_rub,
"price_per_m2": price_per_m2,
"floor": _g(row, "_floor", "floor"),
"num_floors": _g(row, "numberFloors"),
"obj_id": _g(row, "_objId", "objId"),
"city": _g(row, "city"),
"region_cd": region_cd,
"obj_name": _g(row, "objName") or _g(row.get("objInfo") or {}, "objCommercNm"),
# 22b: новые поля квартиры
"section_no": _g(row, "_entrance"), # entranceNumber injected by _flatten_table
"finishing_type": None, # заполнит catalog scraper 22d
"ceiling_height_m": None, # заполнит catalog scraper 22d
"key_handover_dt": None, # заполнит catalog scraper 22d
"catalog_updated_at": None, # заполнит catalog scraper 22d
"catalog_url_hash": None, # заполнит catalog scraper 22d
}
@ -257,14 +419,38 @@ UPSERT_OBJECT_SQL = text(
floor_min, floor_max, flat_count, square_living, ready_dt,
problem_flag, site_status, green_house, escrow,
obj_class, wall_type, energy_eff,
latitude, longitude, obj_status, snapshot_date
latitude, longitude, obj_status,
section_count, project_declaration_num, project_published_at,
price_min_rub, dev_group_name,
metro_nearest_name, metro_nearest_walk_minutes, metro_top3,
first_floor_type, elevators_passenger_count, elevators_cargo_count,
parking_total_slots, guest_parking_inside_count, guest_parking_outside_count,
ceiling_height_m, finishing_variants_count, has_free_planning, avg_flat_area_m2,
playground_kids_count, playground_sport_count, has_bike_paths, trash_areas_count,
has_ramp, has_low_platforms, has_wheelchair_lift,
flat_area_min, flat_area_max, price_max_rub, price_per_m2_min, price_per_m2_max,
parking_provision_pct, domrf_score_location, domrf_score_transport,
domrf_score_infrastructure,
snapshot_date
) VALUES (
:obj_id, :hobj_id, :comm_name, :addr, :short_addr, :region_cd,
:dev_id, :dev_name, :dev_inn,
:floor_min, :floor_max, :flat_count, :square_living, :ready_dt,
:problem_flag, :site_status, :green_house, :escrow,
:obj_class, :wall_type, :energy_eff,
:latitude, :longitude, :obj_status, :snapshot_date
:latitude, :longitude, :obj_status,
:section_count, :project_declaration_num, :project_published_at,
:price_min_rub, :dev_group_name,
:metro_nearest_name, :metro_nearest_walk_minutes, CAST(:metro_top3 AS jsonb),
:first_floor_type, :elevators_passenger_count, :elevators_cargo_count,
:parking_total_slots, :guest_parking_inside_count, :guest_parking_outside_count,
:ceiling_height_m, :finishing_variants_count, :has_free_planning, :avg_flat_area_m2,
:playground_kids_count, :playground_sport_count, :has_bike_paths, :trash_areas_count,
:has_ramp, :has_low_platforms, :has_wheelchair_lift,
:flat_area_min, :flat_area_max, :price_max_rub, :price_per_m2_min, :price_per_m2_max,
:parking_provision_pct, :domrf_score_location, :domrf_score_transport,
:domrf_score_infrastructure,
:snapshot_date
)
ON CONFLICT (obj_id, snapshot_date) DO UPDATE SET
comm_name = EXCLUDED.comm_name,
@ -276,10 +462,96 @@ UPSERT_OBJECT_SQL = text(
ready_dt = EXCLUDED.ready_dt,
site_status = EXCLUDED.site_status,
escrow = EXCLUDED.escrow,
obj_class = EXCLUDED.obj_class,
obj_class = COALESCE(EXCLUDED.obj_class, domrf_kn_objects.obj_class),
wall_type = COALESCE(EXCLUDED.wall_type, domrf_kn_objects.wall_type),
energy_eff = COALESCE(EXCLUDED.energy_eff, domrf_kn_objects.energy_eff),
latitude = EXCLUDED.latitude,
longitude = EXCLUDED.longitude,
obj_status = EXCLUDED.obj_status
obj_status = EXCLUDED.obj_status,
section_count = COALESCE(EXCLUDED.section_count, domrf_kn_objects.section_count),
project_declaration_num = COALESCE(
EXCLUDED.project_declaration_num, domrf_kn_objects.project_declaration_num
),
project_published_at = COALESCE(
EXCLUDED.project_published_at, domrf_kn_objects.project_published_at
),
price_min_rub = EXCLUDED.price_min_rub,
dev_group_name = COALESCE(EXCLUDED.dev_group_name, domrf_kn_objects.dev_group_name),
metro_nearest_name = COALESCE(
EXCLUDED.metro_nearest_name, domrf_kn_objects.metro_nearest_name
),
metro_nearest_walk_minutes = COALESCE(
EXCLUDED.metro_nearest_walk_minutes, domrf_kn_objects.metro_nearest_walk_minutes
),
metro_top3 = COALESCE(EXCLUDED.metro_top3, domrf_kn_objects.metro_top3),
first_floor_type = COALESCE(
EXCLUDED.first_floor_type, domrf_kn_objects.first_floor_type
),
elevators_passenger_count = COALESCE(
EXCLUDED.elevators_passenger_count, domrf_kn_objects.elevators_passenger_count
),
elevators_cargo_count = COALESCE(
EXCLUDED.elevators_cargo_count, domrf_kn_objects.elevators_cargo_count
),
parking_total_slots = COALESCE(
EXCLUDED.parking_total_slots, domrf_kn_objects.parking_total_slots
),
guest_parking_inside_count = COALESCE(
EXCLUDED.guest_parking_inside_count, domrf_kn_objects.guest_parking_inside_count
),
guest_parking_outside_count = COALESCE(
EXCLUDED.guest_parking_outside_count, domrf_kn_objects.guest_parking_outside_count
),
ceiling_height_m = COALESCE(
EXCLUDED.ceiling_height_m, domrf_kn_objects.ceiling_height_m
),
finishing_variants_count = COALESCE(
EXCLUDED.finishing_variants_count, domrf_kn_objects.finishing_variants_count
),
has_free_planning = COALESCE(
EXCLUDED.has_free_planning, domrf_kn_objects.has_free_planning
),
avg_flat_area_m2 = COALESCE(
EXCLUDED.avg_flat_area_m2, domrf_kn_objects.avg_flat_area_m2
),
playground_kids_count = COALESCE(
EXCLUDED.playground_kids_count, domrf_kn_objects.playground_kids_count
),
playground_sport_count = COALESCE(
EXCLUDED.playground_sport_count, domrf_kn_objects.playground_sport_count
),
has_bike_paths = COALESCE(EXCLUDED.has_bike_paths, domrf_kn_objects.has_bike_paths),
trash_areas_count = COALESCE(
EXCLUDED.trash_areas_count, domrf_kn_objects.trash_areas_count
),
has_ramp = COALESCE(EXCLUDED.has_ramp, domrf_kn_objects.has_ramp),
has_low_platforms = COALESCE(
EXCLUDED.has_low_platforms, domrf_kn_objects.has_low_platforms
),
has_wheelchair_lift = COALESCE(
EXCLUDED.has_wheelchair_lift, domrf_kn_objects.has_wheelchair_lift
),
flat_area_min = COALESCE(EXCLUDED.flat_area_min, domrf_kn_objects.flat_area_min),
flat_area_max = COALESCE(EXCLUDED.flat_area_max, domrf_kn_objects.flat_area_max),
price_max_rub = COALESCE(EXCLUDED.price_max_rub, domrf_kn_objects.price_max_rub),
price_per_m2_min = COALESCE(
EXCLUDED.price_per_m2_min, domrf_kn_objects.price_per_m2_min
),
price_per_m2_max = COALESCE(
EXCLUDED.price_per_m2_max, domrf_kn_objects.price_per_m2_max
),
parking_provision_pct = COALESCE(
EXCLUDED.parking_provision_pct, domrf_kn_objects.parking_provision_pct
),
domrf_score_location = COALESCE(
EXCLUDED.domrf_score_location, domrf_kn_objects.domrf_score_location
),
domrf_score_transport = COALESCE(
EXCLUDED.domrf_score_transport, domrf_kn_objects.domrf_score_transport
),
domrf_score_infrastructure = COALESCE(
EXCLUDED.domrf_score_infrastructure, domrf_kn_objects.domrf_score_infrastructure
)
"""
)
@ -288,11 +560,17 @@ UPSERT_FLAT_SQL = text(
INSERT INTO domrf_kn_flats (
id, ods_id, flat_type, flat_number, is_studio, total_area, living_area,
rooms, status, price_rub, price_per_m2, floor, num_floors, obj_id,
city, region_cd, obj_name, snapshot_date
city, region_cd, obj_name,
section_no, finishing_type, ceiling_height_m,
key_handover_dt, catalog_updated_at, catalog_url_hash,
snapshot_date
) VALUES (
:id, :ods_id, :flat_type, :flat_number, :is_studio, :total_area, :living_area,
:rooms, :status, :price_rub, :price_per_m2, :floor, :num_floors, :obj_id,
:city, :region_cd, :obj_name, :snapshot_date
:city, :region_cd, :obj_name,
:section_no, :finishing_type, :ceiling_height_m,
:key_handover_dt, :catalog_updated_at, :catalog_url_hash,
:snapshot_date
)
ON CONFLICT (id, snapshot_date) DO UPDATE SET
status = EXCLUDED.status,
@ -300,7 +578,26 @@ UPSERT_FLAT_SQL = text(
price_per_m2 = EXCLUDED.price_per_m2,
obj_id = EXCLUDED.obj_id,
region_cd = EXCLUDED.region_cd,
obj_name = EXCLUDED.obj_name
obj_name = EXCLUDED.obj_name,
flat_number = COALESCE(EXCLUDED.flat_number, domrf_kn_flats.flat_number),
living_area = COALESCE(EXCLUDED.living_area, domrf_kn_flats.living_area),
is_studio = COALESCE(EXCLUDED.is_studio, domrf_kn_flats.is_studio),
total_area = COALESCE(EXCLUDED.total_area, domrf_kn_flats.total_area),
rooms = COALESCE(EXCLUDED.rooms, domrf_kn_flats.rooms),
floor = COALESCE(EXCLUDED.floor, domrf_kn_flats.floor),
num_floors = COALESCE(EXCLUDED.num_floors, domrf_kn_flats.num_floors),
section_no = COALESCE(EXCLUDED.section_no, domrf_kn_flats.section_no),
finishing_type = COALESCE(EXCLUDED.finishing_type, domrf_kn_flats.finishing_type),
ceiling_height_m = COALESCE(
EXCLUDED.ceiling_height_m, domrf_kn_flats.ceiling_height_m
),
key_handover_dt = COALESCE(EXCLUDED.key_handover_dt, domrf_kn_flats.key_handover_dt),
catalog_updated_at = COALESCE(
EXCLUDED.catalog_updated_at, domrf_kn_flats.catalog_updated_at
),
catalog_url_hash = COALESCE(
EXCLUDED.catalog_url_hash, domrf_kn_flats.catalog_url_hash
)
"""
)
@ -520,11 +817,9 @@ UPSERT_INFRA_SQL = text(
:obj_id, :poi_name, :poi_subtitle, :poi_category, :poi_address,
:poi_lat, :poi_lon, :distance_m, :snapshot_date
)
ON CONFLICT (obj_id, poi_name, poi_lat, poi_lon, snapshot_date) DO UPDATE SET
poi_subtitle = EXCLUDED.poi_subtitle,
poi_category = EXCLUDED.poi_category,
poi_address = EXCLUDED.poi_address,
distance_m = EXCLUDED.distance_m
-- Issue #297 22j: новый UNIQUE (obj_id, poi_category, poi_name, poi_address) — без
-- snapshot_date, чтобы каждый scrape run не накапливал ×N дубликаты POI.
ON CONFLICT (obj_id, poi_category, poi_name, poi_address) DO NOTHING
"""
)
@ -711,6 +1006,191 @@ async def fetch_photos(sess: BrowserSession, obj_id: int) -> tuple[list[dict[str
return (payload.get("data") or []), full_url
async def _fetch_doc_endpoint(
sess: BrowserSession, path_template: str, obj_id: int
) -> tuple[list[dict[str, Any]], str]:
"""Generic helper: fetch one document endpoint, return (items, full_url)."""
path = path_template.format(obj_id=obj_id)
payload = await sess.get_json(path, {})
full_url = f"{BASE_URL}{path}"
if isinstance(payload, list):
return payload, full_url
data = payload.get("data")
if isinstance(data, list):
return data, full_url
return [], full_url
async def fetch_doc_rpd(sess: BrowserSession, obj_id: int) -> tuple[list[dict[str, Any]], str]:
"""214-ФЗ отчёт (РПД)."""
return await _fetch_doc_endpoint(sess, PATH_DOC_RPD, obj_id)
async def fetch_doc_developer_report(
sess: BrowserSession, obj_id: int
) -> tuple[list[dict[str, Any]], str]:
"""Отчёт застройщика."""
return await _fetch_doc_endpoint(sess, PATH_DOC_DEVELOPER_REPORT, obj_id)
async def fetch_doc_project_documentation(
sess: BrowserSession, obj_id: int
) -> tuple[list[dict[str, Any]], str]:
"""Проектная декларация."""
return await _fetch_doc_endpoint(sess, PATH_DOC_PROJECT_DOCUMENTATION, obj_id)
async def fetch_doc_documentation_other(
sess: BrowserSession, obj_id: int
) -> tuple[list[dict[str, Any]], str]:
"""Прочие документы."""
return await _fetch_doc_endpoint(sess, PATH_DOC_DOCUMENTATION_OTHER, obj_id)
async def fetch_doc_permits(sess: BrowserSession, obj_id: int) -> tuple[list[dict[str, Any]], str]:
"""Разрешения на строительство."""
return await _fetch_doc_endpoint(sess, PATH_DOC_PERMITS, obj_id)
async def fetch_obj_checks(sess: BrowserSession, obj_id: int) -> tuple[Any, str]:
"""Fetch 6 «Проверено на наш.дом.рф» checks for one object. Returns (payload, full_url).
Endpoint URL (/checks) не верифицирован через devtools — выведен по паттерну
/infrastructure, /documents. При HTTP-ошибке вызывающий код запишет failure в
kn_scrape_failures; scrape не прерывается.
"""
path = PATH_CHECKS.format(obj_id=obj_id)
payload = await sess.get_json(path, {})
return payload, f"{BASE_URL}{path}"
# ── _fetch_*_safe wrappers for asyncio.gather in Phase B/C ───────────────────
# Каждый wrapper возвращает (kind, full_url, result_or_exception).
# Exceptions НЕ raise — помещаются в возвращаемый tuple.
# BrowserSession._sem (Semaphore(3)) bounds concurrency per-request автоматически.
async def _fetch_flats_safe(
sess: BrowserSession, obj_id: int
) -> tuple[str, str, list[dict[str, Any]] | Exception]:
full_url = f"{BASE_URL}{PATH_FLATS_TABLE}?externalId={obj_id}"
try:
flats = await fetch_flats_for_object(sess, obj_id)
return ("flats", full_url, flats)
except Exception as e:
return ("flats", full_url, e)
async def _fetch_sale_graph_safe(
sess: BrowserSession, obj_id: int, type_: str
) -> tuple[str, str, tuple[list[dict[str, Any]], str] | Exception]:
full_url = f"{BASE_URL}{PATH_SALE_GRAPH.format(obj_id=obj_id)}?type={type_}"
try:
rows, url = await fetch_sale_graph(sess, obj_id, type_)
return (f"sale_graph_{type_}", url, (rows, url))
except Exception as e:
return (f"sale_graph_{type_}", full_url, e)
async def _fetch_sales_agg_safe(
sess: BrowserSession, obj_id: int
) -> tuple[str, str, tuple[dict[str, Any], str] | Exception]:
full_url = f"{BASE_URL}{PATH_SALES_AGG.format(obj_id=obj_id)}"
try:
agg, url = await fetch_sales_agg(sess, obj_id)
return ("sales_agg", url, (agg, url))
except Exception as e:
return ("sales_agg", full_url, e)
async def _fetch_infrastructure_safe(
sess: BrowserSession, obj_id: int
) -> tuple[str, str, tuple[list[dict[str, Any]], str] | Exception]:
full_url = f"{BASE_URL}{PATH_INFRA.format(obj_id=obj_id)}"
try:
pois, url = await fetch_infrastructure(sess, obj_id)
return ("infrastructure", url, (pois, url))
except Exception as e:
return ("infrastructure", full_url, e)
async def _fetch_photos_safe(
sess: BrowserSession, obj_id: int
) -> tuple[str, str, tuple[list[dict[str, Any]], str] | Exception]:
full_url = f"{BASE_URL}{PATH_PHOTOS.format(obj_id=obj_id)}"
try:
photos, url = await fetch_photos(sess, obj_id)
return ("photos", url, (photos, url))
except Exception as e:
return ("photos", full_url, e)
async def _fetch_doc_rpd_safe(
sess: BrowserSession, obj_id: int
) -> tuple[str, str, tuple[list[dict[str, Any]], str] | Exception]:
full_url = f"{BASE_URL}{PATH_DOC_RPD.format(obj_id=obj_id)}"
try:
items, url = await fetch_doc_rpd(sess, obj_id)
return ("doc_rpd", url, (items, url))
except Exception as e:
return ("doc_rpd", full_url, e)
async def _fetch_doc_developer_report_safe(
sess: BrowserSession, obj_id: int
) -> tuple[str, str, tuple[list[dict[str, Any]], str] | Exception]:
full_url = f"{BASE_URL}{PATH_DOC_DEVELOPER_REPORT.format(obj_id=obj_id)}"
try:
items, url = await fetch_doc_developer_report(sess, obj_id)
return ("doc_developer_report", url, (items, url))
except Exception as e:
return ("doc_developer_report", full_url, e)
async def _fetch_doc_project_documentation_safe(
sess: BrowserSession, obj_id: int
) -> tuple[str, str, tuple[list[dict[str, Any]], str] | Exception]:
full_url = f"{BASE_URL}{PATH_DOC_PROJECT_DOCUMENTATION.format(obj_id=obj_id)}"
try:
items, url = await fetch_doc_project_documentation(sess, obj_id)
return ("doc_project_documentation", url, (items, url))
except Exception as e:
return ("doc_project_documentation", full_url, e)
async def _fetch_doc_documentation_other_safe(
sess: BrowserSession, obj_id: int
) -> tuple[str, str, tuple[list[dict[str, Any]], str] | Exception]:
full_url = f"{BASE_URL}{PATH_DOC_DOCUMENTATION_OTHER.format(obj_id=obj_id)}"
try:
items, url = await fetch_doc_documentation_other(sess, obj_id)
return ("doc_documentation_other", url, (items, url))
except Exception as e:
return ("doc_documentation_other", full_url, e)
async def _fetch_doc_permits_safe(
sess: BrowserSession, obj_id: int
) -> tuple[str, str, tuple[list[dict[str, Any]], str] | Exception]:
full_url = f"{BASE_URL}{PATH_DOC_PERMITS.format(obj_id=obj_id)}"
try:
items, url = await fetch_doc_permits(sess, obj_id)
return ("doc_permits", url, (items, url))
except Exception as e:
return ("doc_permits", full_url, e)
async def _fetch_obj_checks_safe(
sess: BrowserSession, obj_id: int
) -> tuple[str, str, tuple[Any, str] | Exception]:
full_url = f"{BASE_URL}{PATH_CHECKS.format(obj_id=obj_id)}"
try:
payload, url = await fetch_obj_checks(sess, obj_id)
return ("obj_checks", url, (payload, url))
except Exception as e:
return ("obj_checks", full_url, e)
def upsert_sale_graph(
db: Session, obj_id: int, type_: str, rows: list[dict[str, Any]], snapshot_date: date
) -> int:
@ -1191,6 +1671,8 @@ async def run_region_sweep(
"infra_rows": 0,
"photos_rows": 0,
"photos_downloaded": 0,
"documents_rows": 0,
"checks_rows": 0,
}
total_flats = 0
request_count = 0
@ -1273,7 +1755,11 @@ async def run_region_sweep(
stage="phase_a",
)
# ── Phase B/C — per-object processing (resumable) ───────────────
# ── Phase B/C — per-object processing (resumable, parallel per-object) ─
# Все endpoint'ы одного obj_id запускаются параллельно через asyncio.gather.
# BrowserSession._sem (Semaphore(3)) ограничивает одновременные запросы.
# DB upserts выполняются последовательно после gather — один db Session
# не thread-safe для параллельной записи.
pdir = Path(photos_dir) if photos_dir else PHOTOS_DIR_DEFAULT
total = len(all_objects)
for i in range(start_index, total):
@ -1282,75 +1768,97 @@ async def run_region_sweep(
if not obj_id:
continue
# Flats per obj — committed immediately
# Собираем корутины для параллельного запуска
coros: list[Any] = []
if fetch_flats:
try:
flats = await fetch_flats_for_object(sess, obj_id)
if flats:
total_flats += upsert_flats(db, flats, snapshot_date, region_code)
except Exception as e:
log_progress(
db,
run_id,
f"flats failed obj={obj_id}: {type(e).__name__}: {str(e)[:120]}",
level="warn",
stage="fetch_flats",
obj_id=obj_id,
)
coros.append(_fetch_flats_safe(sess, obj_id))
if extras:
# sale_graph (apartments + parking)
for type_ in SALE_GRAPH_TYPES:
try:
rows, full_url = await fetch_sale_graph(sess, obj_id, type_)
extras_counts["sale_graph_rows"] += upsert_sale_graph(
db, obj_id, type_, rows, snapshot_date
)
except Exception as e:
full_url = (
f"{BASE_URL}"
f"{PATH_SALE_GRAPH.format(obj_id=obj_id)}?type={type_}"
)
_classify_and_log(
db, run_id, obj_id, f"sale_graph_{type_}", full_url, e
)
coros.append(_fetch_sale_graph_safe(sess, obj_id, "apartments"))
coros.append(_fetch_sale_graph_safe(sess, obj_id, "parking"))
coros.append(_fetch_sales_agg_safe(sess, obj_id))
coros.append(_fetch_infrastructure_safe(sess, obj_id))
coros.append(_fetch_photos_safe(sess, obj_id))
coros.append(_fetch_doc_rpd_safe(sess, obj_id))
coros.append(_fetch_doc_developer_report_safe(sess, obj_id))
coros.append(_fetch_doc_project_documentation_safe(sess, obj_id))
coros.append(_fetch_doc_documentation_other_safe(sess, obj_id))
coros.append(_fetch_doc_permits_safe(sess, obj_id))
# TODO: obj_checks endpoint not found at /api/object/{id}/checks (404).
# 6 чек-боксов "Проверено на наш.дом.рф" вероятно inline в kn/object payload.
# Re-enable после investigation структуры объекта (separate PR).
# coros.append(_fetch_obj_checks_safe(sess, obj_id))
# sales_agg
try:
agg, full_url = await fetch_sales_agg(sess, obj_id)
if not coros:
continue
# Параллельный fetch всех endpoint'ов одного объекта
results = await asyncio.gather(*coros, return_exceptions=False)
# Sequential upsert — DB session не thread-safe
all_docs: list[dict[str, Any]] = []
_doc_kinds = frozenset(
(
"doc_rpd",
"doc_developer_report",
"doc_project_documentation",
"doc_documentation_other",
"doc_permits",
)
)
for kind_tag, full_url, result in results:
if isinstance(result, Exception):
_classify_and_log(db, run_id, obj_id, kind_tag, full_url, result)
continue
if kind_tag == "flats":
flats_list: list[dict[str, Any]] = result # type: ignore[assignment]
if flats_list:
total_flats += upsert_flats(db, flats_list, snapshot_date, region_code)
elif kind_tag in ("sale_graph_apartments", "sale_graph_parking"):
sg_type = kind_tag.replace("sale_graph_", "")
rows_sg, _ = result # type: ignore[misc]
extras_counts["sale_graph_rows"] += upsert_sale_graph(
db, obj_id, sg_type, rows_sg, snapshot_date
)
elif kind_tag == "sales_agg":
agg_data, _ = result # type: ignore[misc]
extras_counts["sales_agg_rows"] += upsert_sales_agg(
db, obj_id, agg, snapshot_date
db, obj_id, agg_data, snapshot_date
)
except Exception as e:
full_url = f"{BASE_URL}{PATH_SALES_AGG.format(obj_id=obj_id)}"
_classify_and_log(db, run_id, obj_id, "sales_agg", full_url, e)
# infrastructure
try:
pois, full_url = await fetch_infrastructure(sess, obj_id)
elif kind_tag == "infrastructure":
pois_data, _ = result # type: ignore[misc]
extras_counts["infra_rows"] += upsert_infrastructure(
db, obj_id, pois, snapshot_date
db, obj_id, pois_data, snapshot_date
)
except Exception as e:
full_url = f"{BASE_URL}{PATH_INFRA.format(obj_id=obj_id)}"
_classify_and_log(db, run_id, obj_id, "infrastructure", full_url, e)
# photos
try:
photos, full_url = await fetch_photos(sess, obj_id)
elif kind_tag == "photos":
photos_data, _ = result # type: ignore[misc]
local_paths: dict[str, str] = {}
thumb_paths: dict[str, str] = {}
if download_photos_binary and photos:
if download_photos_binary and photos_data:
local_paths, thumb_paths = await download_photos(
sess, obj_id, photos, pdir
sess, obj_id, photos_data, pdir
)
extras_counts["photos_downloaded"] += len(local_paths)
extras_counts["photos_rows"] += upsert_photos(
db, obj_id, photos, local_paths, thumb_paths
db, obj_id, photos_data, local_paths, thumb_paths
)
except Exception as e:
full_url = f"{BASE_URL}{PATH_PHOTOS.format(obj_id=obj_id)}"
_classify_and_log(db, run_id, obj_id, "photos", full_url, e)
elif kind_tag in _doc_kinds:
# Каждый из 5 doc-endpoint'ов отдаёт свой список документов.
# Накапливаем в all_docs — единый upsert после цикла.
doc_items, _ = result # type: ignore[misc]
all_docs.extend(extract_documents(doc_items or []))
# Единый upsert всех документов объекта после обработки 5 endpoint'ов.
if all_docs:
ins, _skip = upsert_documents(db, obj_id, all_docs)
extras_counts["documents_rows"] += ins
# Checkpoint раз в 10 объектов: запись прогресса + heartbeat.
if (i + 1) % 10 == 0:
@ -1363,7 +1871,9 @@ async def run_region_sweep(
f" agg={extras_counts['sales_agg_rows']}"
f" infra={extras_counts['infra_rows']}"
f" photos={extras_counts['photos_rows']}"
f" downloaded={extras_counts['photos_downloaded']}",
f" downloaded={extras_counts['photos_downloaded']}"
f" docs={extras_counts['documents_rows']}"
f" checks={extras_counts['checks_rows']}",
stage="extras" if extras else "fetch_flats",
)
@ -1389,7 +1899,58 @@ async def run_region_sweep(
log_progress(
db,
run_id,
f"Готово ✅ objects={total} flats={total_flats} requests={request_count}",
f"Phase D done: objects={total} flats={total_flats} requests={request_count}",
stage="phase_d",
)
# ── Phase E — derive is_ekb for this snapshot ──────────────────────
# Проставляем is_ekb=TRUE для объектов Екатеринбурга/Свердловской обл.
# по district_name (заполнен PostGIS join в Phase A).
# Только новые/изменившиеся строки: COALESCE(is_ekb, FALSE) = FALSE.
try:
result_e = db.execute(
text(
"""
UPDATE domrf_kn_objects
SET is_ekb = TRUE
WHERE snapshot_date = :snap
AND (
district_name ILIKE '%екатеринбург%'
OR district_name ILIKE '%свердловск%'
)
AND COALESCE(is_ekb, FALSE) = FALSE
"""
),
{"snap": snapshot_date},
)
db.commit()
ekb_updated = result_e.rowcount if result_e.rowcount >= 0 else -1
log_progress(
db,
run_id,
f"Phase E done: is_ekb derived, updated={ekb_updated} rows"
f" for snap {snapshot_date}",
stage="phase_e",
)
except Exception as e:
logger.warning("Phase E is_ekb derive failed: %s", e)
try:
db.rollback()
except Exception:
pass
log_progress(
db,
run_id,
f"Phase E is_ekb derive FAILED: {type(e).__name__}: {str(e)[:200]}",
level="warn",
stage="phase_e",
)
log_progress(
db,
run_id,
f"Готово objects={total} flats={total_flats} requests={request_count}"
f" docs={extras_counts['documents_rows']} checks={extras_counts['checks_rows']}",
stage="done",
)
return {

349
backend/app/services/scrapers/ekburg_permits.py Normal file
View file

@ -0,0 +1,349 @@
"""ЕКБ РНС/РВЭ xlsx parser (Issue #105).
Source: https://xn--80acgfbsl1azdqr.xn--p1ai/дляработы/гиз/градостроительство/разрешение
Format: Form 3 (РНС) + Form 4 (РВЭ) — стандартные Росстат шаблоны.
Структура xlsx (проверена на всех файлах 2022-2026):
Лист «реестр разрешений на строительс» → РНС (Таблица 3)
Лист «реестр разрешений на ввод» → РВЭ (Таблица 4)
Расположение заголовков:
2022: строки 5-7 — заголовок/подзаголовок/номера; данные с row 8
2023+: строки 4-6 — заголовок/подзаголовок/номера; данные с row 7
Колонки (0-based) одинаковы во всех годах:
0 developer_name — Наименование застройщика
1 developer_inn — ИНН (int или str)
2 developer_address — Адрес застройщика (не сохраняем)
3 object_type — Тип строительного объекта
4 object_name — Наименование объекта КС
5 cadastral_number — Кадастровый номер ЗУ
6 raw_coord_x — X (СКФ-66, местная СК Свердловской обл.)
7 raw_coord_y — Y (СКФ-66)
8 construction_address — Адрес объекта
9 permit_number — Номер разрешения на строительство
10 issue_date — Дата разрешения на строительство
11 expiry_date — Дата окончания разрешения
12 total_area_sqm — Общая площадь, м²
13 living_area_sqm — Площадь жилых помещений по проекту, м²
-- дополнительно только для РВЭ:
14 living_area_fact_sqm — Площадь жилых помещений фактически, м²
15 rve_number — Номер разрешения на ввод
16 rve_date — Дата разрешения на ввод
Координаты raw_coord_x / raw_coord_y хранятся как строки — CRS неизвестна (предположительно
СНСК-66 / СКФ-66 Свердловская обл., local offset system). Геокодирование через construction_address
планируется в Phase 3 (отдельный PR).
"""
from __future__ import annotations
import logging
import re
from collections.abc import Iterator
from dataclasses import dataclass, field
from datetime import date, datetime
from io import BytesIO
from typing import Any
import httpx
from openpyxl import load_workbook
logger = logging.getLogger(__name__)
EKBURG_PERMITS_URLS: dict[int, str] = {
2026: "https://xn--80acgfbsl1azdqr.xn--p1ai/file/70bf01bf31538ee9dd82dadfc47192a0",
2025: "https://xn--80acgfbsl1azdqr.xn--p1ai/file/6a0a18c9ee327e6e4f76c32a5385a6bd",
2024: "https://xn--80acgfbsl1azdqr.xn--p1ai/file/907bfa0cf78d5a93c6ccafed1af51fc5",
2023: "https://xn--80acgfbsl1azdqr.xn--p1ai/file/3b2ef86bf5673adaa65263672f0c623f",
2022: "https://xn--80acgfbsl1azdqr.xn--p1ai/file/51a7e5654d2fa018bc6db402ab4d5775",
}
# Листы которые нужно пропустить (справочник, вспомогательные)
_SKIP_SHEETS = {"справочник", "лист1", "sheet1"}
# Паттерн для проверки что строка — строка данных (первая ячейка непустая и не число-нумерация)
_INN_RE = re.compile(r"\d{10,12}")
@dataclass
class PermitRow:
"""Parsed row from РНС/РВЭ xlsx."""
permit_type: str # "RNS" | "RVE"
permit_number: str
issue_date: date | None
expiry_date: date | None
developer_inn: str | None
developer_name: str | None
object_name: str | None
object_type: str | None
construction_address: str | None
cadastral_number: str | None
total_area_sqm: float | None
living_area_sqm: float | None
living_area_fact_sqm: float | None # только РВЭ
rve_number: str | None # только РВЭ
rve_date: date | None # только РВЭ
raw_coord_x: str | None
raw_coord_y: str | None
source_year: int
source_url: str
raw_row: dict[str, Any] = field(default_factory=dict)
# ── helpers ──────────────────────────────────────────────────────────────────
def _to_str(v: Any) -> str | None:
"""Привести значение ячейки к строке, убрать лишние пробелы. None если пусто."""
if v is None:
return None
s = str(v).strip().replace("\xa0", " ")
return s if s and s != "-" else None
def _to_date(v: Any) -> date | None:
"""Привести значение ячейки к date.
Принимает: datetime (openpyxl), date, строки 'DD.MM.YYYY', 'YYYY-MM-DD'.
"""
if v is None:
return None
if isinstance(v, datetime):
return v.date()
if isinstance(v, date):
return v
if isinstance(v, str):
s = v.strip()
for fmt in ("%d.%m.%Y", "%Y-%m-%d", "%d-%m-%Y"):
try:
return datetime.strptime(s, fmt).date()
except ValueError:
continue
return None
def _to_float(v: Any) -> float | None:
"""Привести значение ячейки к float. Обрабатывает запятую как разделитель."""
if v is None:
return None
if isinstance(v, int | float):
return float(v) if not isinstance(v, bool) else None
s = str(v).strip().replace(",", ".").replace("\xa0", "").replace(" ", "")
if not s or s == "-":
return None
try:
return float(s)
except ValueError:
return None
def _clean_inn(v: Any) -> str | None:
"""Извлечь ИНН из значения ячейки. ИНН — 10 или 12 цифр."""
if v is None:
return None
s = str(v).strip().replace("\xa0", "")
# Если ячейка — число (openpyxl выдаёт int/float)
if isinstance(v, int | float) and not isinstance(v, bool):
digits = str(int(v))
if 10 <= len(digits) <= 12:
return digits
return None
m = _INN_RE.search(s)
return m.group(0) if m else None
def _detect_permit_type(sheet_name: str) -> str | None:
"""Определить тип разрешения по имени листа."""
name = sheet_name.lower()
if "строит" in name:
return "RNS"
if "ввод" in name:
return "RVE"
return None
def _detect_header_row(sheet: Any) -> int:
"""Найти строку с заголовком 'Наименование застройщика' (1-based).
Возвращает номер строки-данных (header_row + 3, так как за заголовком идут
две строки подзаголовков и нумерация).
"""
for row_idx, row in enumerate(
sheet.iter_rows(min_row=1, max_row=10, values_only=True), start=1
):
if row and row[0] and "застройщик" in str(row[0]).lower():
# +3: подзаголовок (X/Y), нумерация (1,2,3...), первая данных
return row_idx + 3
# Fallback: стандартные позиции
# 2022: header=5, данные с 8; 2023+: header=4, данные с 7
return 7
def _is_data_row(row: tuple[Any, ...]) -> bool:
"""Вернуть True если строка содержит реальные данные (не пустая, не заголовок)."""
if not row or all(v is None for v in row):
return False
first = row[0]
if first is None:
# Продолжение предыдущей записи (merged cells) — пропускаем
return False
# Проверяем, что первая ячейка — не номер (нумерация столбцов в header)
if isinstance(first, int | float) and not isinstance(first, bool):
val = int(first)
if 1 <= val <= 30:
return False
return True
def _parse_row(
row: tuple[Any, ...],
permit_type: str,
year: int,
source_url: str,
) -> PermitRow | None:
"""Распарсить одну строку данных в PermitRow.
Возвращает None если строка не содержит номера разрешения (обязательное поле).
"""
# permit_number — колонка 9 (0-based)
permit_number = _to_str(row[9]) if len(row) > 9 else None
if not permit_number:
return None
raw: dict[str, Any] = {str(i): str(v) for i, v in enumerate(row) if v is not None}
return PermitRow(
permit_type=permit_type,
permit_number=permit_number,
issue_date=_to_date(row[10]) if len(row) > 10 else None,
expiry_date=_to_date(row[11]) if len(row) > 11 else None,
developer_inn=_clean_inn(row[1]) if len(row) > 1 else None,
developer_name=_to_str(row[0]) if len(row) > 0 else None,
object_name=_to_str(row[4]) if len(row) > 4 else None,
object_type=_to_str(row[3]) if len(row) > 3 else None,
construction_address=_to_str(row[8]) if len(row) > 8 else None,
cadastral_number=_to_str(row[5]) if len(row) > 5 else None,
total_area_sqm=_to_float(row[12]) if len(row) > 12 else None,
living_area_sqm=_to_float(row[13]) if len(row) > 13 else None,
living_area_fact_sqm=_to_float(row[14]) if permit_type == "RVE" and len(row) > 14 else None,
rve_number=_to_str(row[15]) if permit_type == "RVE" and len(row) > 15 else None,
rve_date=_to_date(row[16]) if permit_type == "RVE" and len(row) > 16 else None,
raw_coord_x=_to_str(row[6]) if len(row) > 6 else None,
raw_coord_y=_to_str(row[7]) if len(row) > 7 else None,
source_year=year,
source_url=source_url,
raw_row=raw,
)
# ── client ───────────────────────────────────────────────────────────────────
class EkburgPermitsClient:
"""Client для загрузки + парсинга РНС/РВЭ xlsx с екатеринбург.рф."""
DEFAULT_TIMEOUT = 60.0
USER_AGENT = "GenDesign/1.0 (+https://gendsgn.ru) Site Finder permits scraper"
def __init__(self, *, timeout: float = DEFAULT_TIMEOUT) -> None:
# verify=False: екатеринбург.рф подписан CA Минцифры РФ (нет в certifi).
# Данные публичные open-data — SSL pinning здесь не требуется. Issue #242.
self._client = httpx.Client(
timeout=timeout,
follow_redirects=True,
headers={"User-Agent": self.USER_AGENT},
verify=False,
)
def __enter__(self) -> EkburgPermitsClient:
return self
def __exit__(self, *_: Any) -> None:
self._client.close()
def download_xlsx(self, year: int) -> bytes:
"""GET xlsx для заданного года. Поднимает ValueError для неизвестного года."""
url = EKBURG_PERMITS_URLS.get(year)
if not url:
raise ValueError(f"No URL configured for year={year}")
response = self._client.get(url)
response.raise_for_status()
logger.info("Downloaded ekburg permits xlsx %d: %d bytes", year, len(response.content))
return response.content
def parse_xlsx(self, content: bytes, year: int, source_url: str) -> Iterator[PermitRow]:
"""Parse xlsx bytes → yield PermitRow.
Автоматически определяет тип листа (РНС/РВЭ) по названию.
Пропускает листы «Справочник», «Лист1» и неизвестные.
"""
wb = load_workbook(BytesIO(content), read_only=True, data_only=True)
for sheet_name in wb.sheetnames:
if sheet_name.lower() in _SKIP_SHEETS:
continue
permit_type = _detect_permit_type(sheet_name)
if permit_type is None:
logger.debug("Skipping unknown sheet %r in year %d", sheet_name, year)
continue
sheet = wb[sheet_name]
data_start = _detect_header_row(sheet)
logger.info(
"Parsing sheet %r (%s) year=%d, data starts at row %d",
sheet_name,
permit_type,
year,
data_start,
)
yield from self._parse_sheet(sheet, permit_type, year, source_url, data_start)
def _parse_sheet(
self,
sheet: Any,
permit_type: str,
year: int,
source_url: str,
data_start: int,
) -> Iterator[PermitRow]:
"""Parse один лист → yield PermitRow per data row."""
row_count = 0
skip_count = 0
for row_idx, row in enumerate(
sheet.iter_rows(min_row=data_start, values_only=True), start=data_start
):
if not _is_data_row(row):
skip_count += 1
continue
try:
permit = _parse_row(row, permit_type, year, source_url)
except Exception as exc:
logger.warning(
"Failed to parse row %d sheet %s year %d: %s",
row_idx,
permit_type,
year,
exc,
)
skip_count += 1
continue
if permit is None:
skip_count += 1
continue
row_count += 1
yield permit
logger.info(
"Sheet %s year=%d: parsed=%d skipped=%d",
permit_type,
year,
row_count,
skip_count,
)

241
backend/app/services/scrapers/flat_plans.py Normal file
View file

@ -0,0 +1,241 @@
"""Парсер и DB-writer для планировок квартир DOM.РФ (domrf_kn_flat_plans).
Источник данных
---------------
План квартиры отображается на странице
/сервисы/каталог-квартир/квартира/{catalog_hash}
в блоке «Планировка». URL картинки встроен в SSR-HTML страницы и недоступен
через kn-API list-endpoint (/сервисы/api/kn/object) или flat-table endpoint
(/portal-kn/api/sales/portal/table).
Audit-результат (2026-05-17, obj_id=65136)
------------------------------------------
- kn_object_place_66 payload: plan_image / planImage / planUrl — NOT FOUND.
- portal/table flat items: поля planImageUrl / layoutUrl — NOT FOUND.
- Вывод: plan image URL находится ТОЛЬКО в SSR-HTML каталога.
Структура модуля
----------------
- `extract_flat_plans(raw_payload)` — парсит plan_image_url из flat-table payload
(заглушка — возвращает пустой список до реализации SSR-scraper или endpoint).
- `upsert_flat_plans(db, obj_id, plans, snapshot_date)` — UPSERT в
domrf_kn_flat_plans с COALESCE-защитой скачанных файлов.
- `download_plan_image_stub()` — NotImplementedError placeholder.
Реальная загрузка — отдельный Celery task в рамках 22d-track / #299.
Связанные файлы
---------------
data/sql/100_22c_flat_plans.sql — DDL таблицы.
backend/app/services/scrapers/domrf_kn.py — основной kn-scraper (НЕ трогаем
в этом PR; wiring через отдельный PR после реализации SSR-scraper).
Issue #297 sub-task 22c.
"""
from __future__ import annotations
import logging
from datetime import date
from typing import Any
from sqlalchemy import text
from sqlalchemy.orm import Session
logger = logging.getLogger(__name__)
# ── payload parsing ───────────────────────────────────────────────────────────
def extract_flat_plans(raw_payload: dict[str, Any]) -> list[dict[str, Any]]:
"""Извлечь plan_image_url из flat-table payload.
Текущий статус: plan image URL в kn-API flat-table payload ОТСУТСТВУЕТ
(audit 2026-05-17). Функция зарезервирована для будущей интеграции:
- если DOM.РФ добавит поле в portal/table,
- или при подключении SSR-scraper каталога (22d-track).
Формат входных данных (portal/table response):
{
"externalId": 65136,
"entrances": [
{
"entranceNumber": 1,
"floors": [
{
"floorNumber": 4,
"flats": [
{
"odsId": "65136/1/1.4.3",
"elemId": "d8c7a8103f26c52e427ace5a996706ba",
"totalArea": 36.22,
...
# планировка НЕ содержится в этом payload
}
]
}
]
}
]
}
Возвращает список dict с ключами:
ods_id str — идентификатор квартиры
plan_image_url str — URL картинки планировки
obj_id int — внешний ID объекта
При отсутствии нужного поля возвращает пустой список.
"""
obj_id = raw_payload.get("externalId")
plans: list[dict[str, Any]] = []
for entrance in raw_payload.get("entrances") or []:
for floor_data in entrance.get("floors") or []:
for flat in floor_data.get("flats") or []:
ods_id = flat.get("odsId")
if not ods_id:
continue
# Пробуем несколько возможных имён поля — на случай если API
# в будущем добавит это поле под одним из вариантов.
plan_url = (
flat.get("planImageUrl")
or flat.get("layoutImageUrl")
or flat.get("planUrl")
or flat.get("layoutUrl")
or flat.get("imageUrl")
)
if not plan_url:
continue
plans.append(
{
"ods_id": ods_id,
"obj_id": obj_id,
"plan_image_url": plan_url,
}
)
if not plans:
logger.debug(
"extract_flat_plans: obj_id=%s — plan_image_url не найден в payload "
"(ожидаемо: API не содержит это поле, нужен SSR-scraper каталога)",
obj_id,
)
return plans
# ── DB write ──────────────────────────────────────────────────────────────────
_UPSERT_FLAT_PLAN_SQL = text(
"""
INSERT INTO domrf_kn_flat_plans (
ods_id, obj_id, plan_image_url, local_path,
width_px, height_px, size_bytes, downloaded_at,
snapshot_date, scraped_at
) VALUES (
:ods_id, :obj_id, :plan_image_url, :local_path,
:width_px, :height_px, :size_bytes, :downloaded_at,
:snapshot_date, NOW()
)
ON CONFLICT (ods_id) DO UPDATE SET
plan_image_url = EXCLUDED.plan_image_url,
obj_id = EXCLUDED.obj_id,
snapshot_date = EXCLUDED.snapshot_date,
scraped_at = NOW(),
local_path = COALESCE(domrf_kn_flat_plans.local_path, EXCLUDED.local_path),
downloaded_at = COALESCE(domrf_kn_flat_plans.downloaded_at, EXCLUDED.downloaded_at),
width_px = COALESCE(domrf_kn_flat_plans.width_px, EXCLUDED.width_px),
height_px = COALESCE(domrf_kn_flat_plans.height_px, EXCLUDED.height_px),
size_bytes = COALESCE(domrf_kn_flat_plans.size_bytes, EXCLUDED.size_bytes)
"""
)
def upsert_flat_plans(
db: Session,
obj_id: int,
plans: list[dict[str, Any]],
snapshot_date: date,
) -> int:
"""UPSERT планировок квартир в domrf_kn_flat_plans.
Для каждой строки из `plans` (список dict от `extract_flat_plans` или
SSR-scraper) выполняет INSERT ON CONFLICT UPDATE.
COALESCE-логика: уже скачанные файлы (local_path, downloaded_at,
width_px, height_px, size_bytes) НЕ перезаписываются — только
обновляются plan_image_url и snapshot_date.
Args:
db: SQLAlchemy Session.
obj_id: ID объекта DOM.РФ (для логирования).
plans: Список dict с ключами ods_id, plan_image_url [, obj_id].
snapshot_date: Дата snapshot, в котором найден URL.
Returns:
Количество успешно обработанных строк.
"""
inserted = 0
for plan in plans:
ods_id = plan.get("ods_id")
plan_url = plan.get("plan_image_url")
if not ods_id or not plan_url:
logger.warning(
"upsert_flat_plans obj=%s: пропущена запись без ods_id/plan_image_url: %s",
obj_id,
plan,
)
continue
try:
with db.begin_nested():
db.execute(
_UPSERT_FLAT_PLAN_SQL,
{
"ods_id": ods_id,
"obj_id": plan.get("obj_id") or obj_id,
"plan_image_url": plan_url,
"local_path": plan.get("local_path"),
"width_px": plan.get("width_px"),
"height_px": plan.get("height_px"),
"size_bytes": plan.get("size_bytes"),
"downloaded_at": plan.get("downloaded_at"),
"snapshot_date": snapshot_date,
},
)
inserted += 1
except Exception as e:
logger.warning("upsert_flat_plans obj=%s ods_id=%s failed: %s", obj_id, ods_id, e)
if inserted:
logger.info("upsert_flat_plans obj=%s: %d планировок записано", obj_id, inserted)
return inserted
# ── download stub ─────────────────────────────────────────────────────────────
def download_plan_image_stub(
plan_image_url: str,
ods_id: str,
dest_dir: str | None = None,
) -> str:
"""Заглушка для скачивания бинарника планировки.
Реальная реализация — отдельный Celery task (22d-track, issue #299).
До реализации бросает NotImplementedError, чтобы случайный вызов
не прошёл незаметно.
Args:
plan_image_url: URL картинки планировки.
ods_id: Идентификатор квартиры (для имени файла).
dest_dir: Директория для сохранения (None = MEDIA_ROOT/flat_plans/).
Raises:
NotImplementedError: всегда — до реализации.
"""
raise NotImplementedError(
"download_plan_image_stub: скачивание планировок не реализовано. "
"Реальный downloader — отдельный Celery task (issue #299, 22d-track). "
f"URL={plan_image_url!r} ods_id={ods_id!r} dest_dir={dest_dir!r}"
)

264
backend/app/services/scrapers/nspd_client.py
View file

@ -31,6 +31,7 @@ WMS endpoints (per #94 issue body, TIER 1-6 каталог слоёв):
from __future__ import annotations
import asyncio
import datetime as _dt
import json
import logging
@ -42,6 +43,7 @@ import urllib.request
from dataclasses import dataclass
from typing import Any
from app.services.scrapers.nspd_denorm import classify_engineering_kind
from app.services.scrapers.nspd_lite import (
_SSL_CTX,
HEADERS,
@ -109,6 +111,37 @@ LAYERS: dict[str, int] = {
}
# Layers where grid-walk (get_features_in_bbox_grid) must be used instead of
# the legacy single-pixel WMS probe (get_features_in_bbox). These are area/
# linear layers (territorial zones, red lines, engineering structures, ЗОУИТ,
# risks) that span large areas and are under-returned by a single GetFeatureInfo
# call. Point/polygon EGRN layers (parcels, buildings) stay on legacy for now.
_GRID_WALK_LAYERS: frozenset[str] = frozenset(
{
"territorial_zones",
"red_lines",
"engineering_structures",
# ЗОУИТ (TIER 2)
"zouit_okn",
"zouit_engineering",
"zouit_natural",
"zouit_protected",
"zouit_other",
# Risks (TIER 3)
"risk_flooding_underground",
"risk_flooding",
"risk_swampification",
"risk_landslide",
"risk_abrasion",
"risk_erosion_water",
"risk_erosion_linear",
"risk_erosion_wind",
"risk_desertification",
"risk_clutter",
"risk_burns",
}
)
# Default rate limit (мс между запросами) — баланс между скоростью и WAF
DEFAULT_RATE_MS = 600
@ -199,6 +232,9 @@ class QuarterDump:
engineering_structures: list[NSPDFeature]
zouit: dict[str, list[NSPDFeature]] # {"okn": [...], "engineering": [...], ...}
risks: dict[str, list[NSPDFeature]] # {"flooding": [...], "landslide": [...], ...}
# TIER 4 opportunity layers (issue #94 PR2).
# {"auction_parcels": [...], "scheme_parcels": [...], "free_parcels": [...], ...}
opportunity: dict[str, list[NSPDFeature]]
# tuple, не list — frozen dataclass + immutable contents (audit/debug snapshot)
layers_fetched: tuple[str, ...]
bbox_3857: tuple[float, float, float, float] | None # bbox квартала
@ -215,6 +251,7 @@ class QuarterDump:
+ len(self.engineering_structures)
+ sum(len(v) for v in self.zouit.values())
+ sum(len(v) for v in self.risks.values())
+ sum(len(v) for v in self.opportunity.values())
)
@ -380,24 +417,22 @@ class NSPDClient:
width: int = 4096,
height: int = 4096,
) -> list[NSPDFeature]:
"""Bulk fetch features в bbox через GetFeatureInfo с большим bbox.
"""WMS GetFeatureInfo на одном центральном пикселе bbox.
Workaround: WFS GetCapabilities → 404 на nspd.gov.ru, нет WFS
GetFeature endpoint. Решение: использовать GetFeatureInfo с large
bbox и точкой в центре (I=W/2, J=H/2) — возвращает все features
пересекающиеся с bbox.
DEPRECATED: возвращает 0-3 features под одним пикселем (I=W/2, J=H/2).
НЕ является bulk fetch несмотря на исходный docstring — WMS GetFeatureInfo
по стандарту OGC возвращает объекты строго под одной pixel-точкой, а не
во всём bbox. Для получения всех объектов в bbox используй
`get_features_in_bbox_grid`.
See: fixes/Bug_NSPD_WMS_NotBulk_2026_May14.md
Args:
bbox_3857: (xmin, ymin, xmax, ymax) в EPSG:3857 метрах.
width/height: размер виртуального tile. Большой → большой bbox.
width/height: размер виртуального tile.
Returns:
list[NSPDFeature]; пусто если ничего не найдено.
Use cases (per #94 acceptance):
- sync_territorial_zones_bbox → закрывает G1 #28 ПЗЗ
- sync_zouit_*_bbox → G3 #30
- sync_risk_zones_bbox → новый risk overlay
"""
xmin, ymin, xmax, ymax = bbox_3857
params = {
@ -422,6 +457,149 @@ class NSPDClient:
feats = (data or {}).get("features") or []
return [NSPDFeature.from_raw(f) for f in feats]
# ── 3b. get_features_in_bbox_grid ───────────────────────────────────────
def get_features_in_bbox_grid(
self,
layer_id: int,
bbox: tuple[float, float, float, float],
*,
grid_n: int = 7,
step_m: float = 50.0,
tile_size: int = 512,
) -> list[NSPDFeature]:
"""Bulk-аппроксимация bbox через grid-walk WMS GetFeatureInfo.
Разбивает bbox на grid_n × grid_n равных ячеек. В каждой ячейке
вызывает WMS GetFeatureInfo в центральном пикселе. Дедуплицирует
результаты по feature_id / cad_num / reg_numb_border — возвращает
список уникальных NSPDFeature.
Делегирует HTTP через NSPDBulkClient.wms_feature_info (async httpx
с semaphore и retry), запуская asyncio event loop синхронно через
asyncio.run(). Предназначен для вызова из синхронного кода (Celery
task, FastAPI sync handler).
Concurrency: NSPDBulkClient._sem (per-instance, capacity=3) ограничивает параллельные
запросы. При grid_n=7 (49 ячеек) — все 49 ячеек запускаются одним
gather; семафор пропускает не более 3 одновременно. Thread-safety:
каждый вызов get_features_in_bbox_grid создаёт новый event loop
через asyncio.run() — безопасно из разных Celery workers (process-
уровень изоляции).
Args:
layer_id: NSPD layer ID (например 36328 сооружения, 37578 ЗОУИТ).
bbox: (xmin, ymin, xmax, ymax) в EPSG:3857 (метры).
grid_n: размер сетки по каждой оси. 7 → 49 запросов (~coarse),
15 → 225 запросов (~fine). По умолчанию 7 для первичного scan.
step_m: минимальный шаг ячейки в метрах. Если bbox меньше
grid_n*step_m — grid_n уменьшается автоматически чтобы
ячейки не становились меньше step_m.
tile_size: размер виртуального WMS тайла (пиксели).
Returns:
Дедуплицированный list[NSPDFeature]. Может быть пуст если в bbox
нет объектов данного layer'а.
Note:
Не делает live HTTP вызовы если вызван с mock NSPDBulkClient.
Rate-limit управляется семафором NSPDBulkClient._sem (per-instance, capacity=3) +
asyncio.sleep(0.05) jitter — не через self.rate_ms.
"""
# Импортируем здесь чтобы избежать circular import:
# nspd_client ← nspd_bulk_client (оба top-level scrapers, не cross-domain)
from app.scrapers.nspd_bulk_client import NSPDBulkClient
xmin, ymin, xmax, ymax = bbox
width_m = xmax - xmin
height_m = ymax - ymin
# Авто-коррекция grid_n если bbox мал для шага step_m
effective_n = min(
grid_n,
max(1, int(width_m / step_m)),
max(1, int(height_m / step_m)),
)
if effective_n < grid_n:
logger.info(
"get_features_in_bbox_grid layer=%d: bbox %.0fx%.0fm < grid_n=%d×step_m=%.0f"
" — уменьшаем grid до %d×%d",
layer_id,
width_m,
height_m,
grid_n,
step_m,
effective_n,
effective_n,
)
x_step = width_m / effective_n
y_step = height_m / effective_n
# Генерируем список (sub_bbox, click_xy) ячеек
cells: list[tuple[tuple[float, float, float, float], tuple[int, int]]] = []
click_px = tile_size // 2
for i in range(effective_n):
for j in range(effective_n):
cell_xmin = xmin + i * x_step
cell_ymin = ymin + j * y_step
cell_xmax = cell_xmin + x_step
cell_ymax = cell_ymin + y_step
cells.append(((cell_xmin, cell_ymin, cell_xmax, cell_ymax), (click_px, click_px)))
async def _run_grid() -> list[NSPDFeature]:
async with NSPDBulkClient() as client:
tasks = [
client.wms_feature_info(layer_id, sub_bbox, click_xy, tile_size, tile_size)
for sub_bbox, click_xy in cells
]
results = await asyncio.gather(*tasks, return_exceptions=True)
features: list[NSPDFeature] = []
for r in results:
if isinstance(r, Exception):
logger.warning("get_features_in_bbox_grid layer=%d cell error: %s", layer_id, r)
continue
for bulk_feat in r:
raw = {
"id": bulk_feat.id,
"geometry": bulk_feat.geometry,
"properties": bulk_feat.properties,
}
features.append(NSPDFeature.from_raw(raw))
return features
raw_features = asyncio.run(_run_grid())
# Дедупликация — приоритет ключей: feature_id > cad_num > reg_numb_border
seen: set[str] = set()
deduped: list[NSPDFeature] = []
for feat in raw_features:
props = feat.properties
dedup_key = (
feat.feature_id
or props.get("cad_num")
or props.get("cad_number")
or props.get("reg_numb_border")
or props.get("label")
)
if dedup_key is not None:
if dedup_key in seen:
continue
seen.add(dedup_key)
deduped.append(feat)
logger.info(
"get_features_in_bbox_grid layer=%d grid=%dx%d cells=%d raw=%d deduped=%d",
layer_id,
effective_n,
effective_n,
len(cells),
len(raw_features),
len(deduped),
)
return deduped
# ── 4. list_layers ──────────────────────────────────────────────────────
def list_layers(self, theme_id: int = THEME_PKK) -> list[NSPDLayer]:
@ -489,6 +667,16 @@ class NSPDClient:
"clutter": "risk_clutter",
"burns": "risk_burns",
}
# TIER 4 — Opportunity layers (issue #94 PR2).
# short_name → LAYERS dict key (for get_features_in_bbox lookup).
# Features stored in features_json с layer = "opportunity_<short_name>".
QUARTER_OPPORTUNITY_LAYERS: dict[str, str] = { # noqa: RUF012
"auction_parcels": "auction_parcels", # 37299 — аукционные ЗУ
"scheme_parcels": "scheme_parcels", # 37294 — схемы расположения ЗУ
"free_parcels": "free_parcels", # 37298 — свободные ЗУ
"future_parcels": "future_parcels", # 36473 — планируемые ЗУ
"oopt": "protected_areas", # 875845 — ООПТ
}
def search_by_quarter(
self,
@ -496,6 +684,7 @@ class NSPDClient:
*,
include_zouit: bool = True,
include_risks: bool = False,
include_opportunity: bool = False,
) -> QuarterDump:
"""Harvest всех NSPD-данных для квартала: 1 vacuum, N layers.
@ -517,13 +706,17 @@ class NSPDClient:
include_zouit: Включать TIER 2 ЗОУИТ layers (G3). Default True.
include_risks: Включать TIER 3 risk zones. Default False (rate-limit
budget; для отдельного D-N risk score можно включить).
include_opportunity: Включать TIER 4 opportunity layers (auction_parcels,
scheme_parcels, free_parcels, future_parcels, oopt). Default False.
+5 HTTP запросов при включении.
Returns:
QuarterDump с per-layer feature lists. Если NSPD пуст / quarter
не найден — `quarter=None`, `bbox_3857=None`, все feature lists
пустые (no bulk-fetch без bounds — нет смысла). При этом dict-
поля `zouit` / `risks` всё равно populated с пустыми lists для
каждого включённого short_name (структура контракта стабильна).
поля `zouit` / `risks` / `opportunity` всё равно populated с пустыми
lists для каждого включённого short_name
(структура контракта стабильна).
`layers_fetched` в этом случае содержит только `('search',)`.
Raises:
@ -532,7 +725,7 @@ class NSPDClient:
операция атомарна (failure → exception).
Закрывает: foundation для G1 #28 ПЗЗ, G3 #30 ЗОУИТ, P2 #46 neighbors,
E1 #51 parcels backfill, #96 ЕГРН помещения.
E1 #51 parcels backfill, #96 ЕГРН помещения, #94 PR2 opportunity.
"""
# 1. Quarter geometry через REST search
quarter_search = self.search_by_cad(quarter_cad, thematic_id=2)
@ -548,7 +741,17 @@ class NSPDClient:
layers_fetched: list[str] = ["search"]
def _fetch_layer(name_in_dump: str, layer_key: str) -> list[NSPDFeature]:
"""Helper: безопасно получить features для одного layer."""
"""Helper: безопасно получить features для одного layer.
Dispatch:
- area/linear layers (_GRID_WALK_LAYERS) → grid-walk
(get_features_in_bbox_grid, grid_n=7, step_m=50)
- point/polygon EGRN layers (parcels, buildings, ons, enk)
→ legacy single-pixel WMS (get_features_in_bbox)
Engineering structures features дополнительно обогащаются
properties["classified_kind"] через classify_engineering_kind.
"""
if bbox is None:
return []
layer_id = LAYERS.get(layer_key)
@ -556,7 +759,29 @@ class NSPDClient:
logger.warning("search_by_quarter: unknown layer key %s", layer_key)
return []
layers_fetched.append(name_in_dump)
return self.get_features_in_bbox(layer_id, bbox)
if layer_key in _GRID_WALK_LAYERS:
features = self.get_features_in_bbox_grid(layer_id, bbox, grid_n=7, step_m=50.0)
# Обогатить engineering_structures classified_kind
if layer_key == "engineering_structures":
for feat in features:
feat.properties["classified_kind"] = classify_engineering_kind(
feat.properties
)
logger.info(
"search_by_quarter layer=%s method=grid_walk count=%d quarter=%s",
name_in_dump,
len(features),
quarter_cad,
)
return features
features_legacy = self.get_features_in_bbox(layer_id, bbox)
logger.info(
"search_by_quarter layer=%s method=legacy count=%d quarter=%s",
name_in_dump,
len(features_legacy),
quarter_cad,
)
return features_legacy
# 3. Core layers
parcels = _fetch_layer("parcels", "parcels")
@ -577,6 +802,12 @@ class NSPDClient:
for short_name, layer_key in self.QUARTER_RISK_LAYERS.items():
risks[short_name] = _fetch_layer(f"risk_{short_name}", layer_key)
# 6. Opportunity layers (TIER 4, issue #94 PR2)
opportunity: dict[str, list[NSPDFeature]] = {}
if include_opportunity:
for short_name, layer_key in self.QUARTER_OPPORTUNITY_LAYERS.items():
opportunity[short_name] = _fetch_layer(f"opportunity_{short_name}", layer_key)
return QuarterDump(
quarter_cad=quarter_cad,
quarter=quarter_feat,
@ -587,6 +818,7 @@ class NSPDClient:
engineering_structures=engineering_structures,
zouit=zouit,
risks=risks,
opportunity=opportunity,
layers_fetched=tuple(layers_fetched),
bbox_3857=bbox,
fetched_at_utc=_dt.datetime.now(_dt.UTC).isoformat(),

369
backend/app/services/scrapers/nspd_denorm.py Normal file
View file

@ -0,0 +1,369 @@
"""Denormalize nspd_quarter_dumps.features_json → nspd_parcels / nspd_buildings.
Используется:
- Inline в harvest_quarter task после UPSERT dump'а
- Backfill task для existing dumps (backfill_all_dumps)
Property mapping (NSPD WMS GetFeatureInfo responses):
Parcels (layer="parcels", layer id 36048):
cad_num — кадастровый номер ЗУ
permitted_use — ВРИ
land_category — категория земель
cost_value — кадастровая стоимость, руб.
area — площадь, м²
address — адрес
Buildings (layer="buildings", layer id 36049):
cad_num — кадастровый номер ОКС
purpose — назначение ("Многоквартирный дом" / "Нежилое здание" / ...)
floors_above_ground — надземных этажей
floors_underground — подземных этажей
year_built — год постройки
cost_value — кадастровая стоимость, руб.
build_record_area — площадь по ГКН, м²
address — адрес
Геометрия в features_json хранится в EPSG:3857 (как возвращает NSPD WMS) —
трансформация 3857→4326 делается в SQL через ST_Transform.
"""
from __future__ import annotations
import datetime
import json
import logging
import re
from typing import Any
from sqlalchemy import text
from sqlalchemy.orm import Session
logger = logging.getLogger(__name__)
# ── Engineering classifier ─────────────────────────────────────────────────────
# Паттерны для классификации инженерных сооружений (layer 36328) по текстовым
# свойствам. Источник: bug-post-mortem fixes/Bug_NSPD_WMS_NotBulk_2026_May14.md
# + live test данные 7×7 grid на 1км² центра ЕКБ.
# Порядок проверки важен: более специфичные паттерны идут первыми.
_ENGINEERING_PATTERNS: list[tuple[re.Pattern[str], str]] = [
# gas — газопроводы, ГРП, ГК (газовые колодцы)
(re.compile(r"газопровод|газоснабж|ГРП\b|ГК\b", re.IGNORECASE), "gas"),
# sewage — канализация, сток, ливневые сети
(re.compile(r"канализ|сточ|ливнев|самотёч|самотеч", re.IGNORECASE), "sewage"),
# heat — тепловые сети, котельные, ТЭЦ
(
re.compile(r"теплов(ая|ой|ые)|теплосеть|теплоснабж|котельн|ТЭЦ\b", re.IGNORECASE),
"heat",
),
# electric — электросети, ВЛ, КЛ, ЛЭП, подстанции, трансформаторы, ТП
(
re.compile(
r"электроэнерг|ВЛ[\s\-]|ВЛ-?\d|КЛ[\s\-]|КЛ-?\d|ЛЭП\b"
r"|подстанц|трансформ|ТП[\s\-]?\d",
re.IGNORECASE,
),
"electric",
),
# water — водопровод, водоснабжение, хозбытовые водосети
(re.compile(r"водопровод|водоснабж|хозбытов|водовод", re.IGNORECASE), "water"),
]
# Поля из NSPD properties в которых ищем паттерны (по приоритету)
_ENGINEERING_TEXT_FIELDS = ("params_name", "name", "params_purpose", "purpose", "label")
def classify_engineering_kind(properties: dict[str, Any]) -> str:
"""Классифицировать инженерное сооружение (layer 36328) по его properties.
Проверяет поля `params_name`, `name`, `params_purpose`, `purpose`, `label`
против regex-паттернов. Возвращает первое совпадение.
Args:
properties: dict свойств NSPDFeature.properties из WMS GetFeatureInfo
или search/geoportal response.
Returns:
Одно из: ``"water"`` | ``"sewage"`` | ``"gas"`` | ``"heat"``
| ``"electric"`` | ``"other"``.
Examples:
>>> classify_engineering_kind({"params_name": "Газопровод высокого давления"})
'gas'
>>> classify_engineering_kind({"name": "КЛ 10 кВ ТП 64102"})
'electric'
>>> classify_engineering_kind({"params_purpose": "Водопровод хозбытовой"})
'water'
>>> classify_engineering_kind({"params_name": "Тепловая сеть"})
'heat'
>>> classify_engineering_kind({"name": "Канализация"})
'sewage'
"""
# Собираем текст для проверки из всех релевантных полей
text_parts: list[str] = []
for field in _ENGINEERING_TEXT_FIELDS:
val = properties.get(field)
if val and isinstance(val, str):
text_parts.append(val)
combined = " ".join(text_parts)
if not combined:
return "other"
for pattern, kind in _ENGINEERING_PATTERNS:
if pattern.search(combined):
return kind
return "other"
# ── Type coercions ─────────────────────────────────────────────────────────────
def _coerce_int(v: Any) -> int | None:
"""NSPD properties могут быть str / int / None."""
if v is None:
return None
try:
return int(v)
except (ValueError, TypeError):
return None
def _coerce_numeric(v: Any) -> float | None:
"""NSPD properties могут быть str с запятой (европейский формат) или None."""
if v is None:
return None
try:
return float(str(v).replace(",", ".").strip())
except (ValueError, TypeError):
return None
# ── Parcel UPSERT ──────────────────────────────────────────────────────────────
_PARCEL_UPSERT_SQL = text(
"""
INSERT INTO nspd_parcels (
cad_num, quarter_cad, permitted_use, land_category,
cost_value, cost_per_m2, area_sqm, address, geom, snapshot_date
) VALUES (
CAST(:cad_num AS text),
CAST(:quarter_cad AS text),
CAST(:permitted_use AS text),
CAST(:land_category AS text),
CAST(:cost_value AS numeric),
CAST(:cost_per_m2 AS numeric),
CAST(:area_sqm AS numeric),
CAST(:address AS text),
CASE WHEN :geom_json IS NULL THEN NULL
ELSE ST_Transform(ST_SetSRID(ST_GeomFromGeoJSON(:geom_json), 3857), 4326)
END,
CAST(:snapshot_date AS date)
)
ON CONFLICT (cad_num) DO UPDATE SET
quarter_cad = EXCLUDED.quarter_cad,
permitted_use = EXCLUDED.permitted_use,
land_category = EXCLUDED.land_category,
cost_value = EXCLUDED.cost_value,
cost_per_m2 = EXCLUDED.cost_per_m2,
area_sqm = EXCLUDED.area_sqm,
address = EXCLUDED.address,
geom = EXCLUDED.geom,
snapshot_date = EXCLUDED.snapshot_date,
updated_at = NOW()
"""
)
def denorm_parcel_feature(
db: Session,
*,
feature: dict[str, Any],
quarter_cad: str,
snapshot_date: str,
) -> bool:
"""UPSERT one parcel feature → nspd_parcels.
Returns True если строка вставлена/обновлена, False при пропуске (нет cad_num).
"""
props = feature.get("properties") or {}
cad_num = props.get("cad_num") or props.get("cadastral_number")
if not cad_num:
return False
area = _coerce_numeric(props.get("area"))
cost_value = _coerce_numeric(props.get("cost_value"))
cost_per_m2: float | None = None
if cost_value is not None and area is not None and area > 0:
cost_per_m2 = cost_value / area
geom = feature.get("geometry")
geom_json: str | None = json.dumps(geom, ensure_ascii=False) if geom else None
try:
with db.begin_nested():
db.execute(
_PARCEL_UPSERT_SQL,
{
"cad_num": cad_num,
"quarter_cad": quarter_cad,
"permitted_use": props.get("permitted_use"),
"land_category": props.get("land_category"),
"cost_value": cost_value,
"cost_per_m2": cost_per_m2,
"area_sqm": area,
"address": props.get("address"),
"geom_json": geom_json,
"snapshot_date": snapshot_date,
},
)
return True
except Exception as e:
logger.warning("denorm parcel %s failed: %s", cad_num, e)
return False
# ── Building UPSERT ────────────────────────────────────────────────────────────
_BUILDING_UPSERT_SQL = text(
"""
INSERT INTO nspd_buildings (
cad_num, quarter_cad, purpose, floors, floors_underground,
year_built, cost_value, build_record_area, address, geom, snapshot_date
) VALUES (
CAST(:cad_num AS text),
CAST(:quarter_cad AS text),
CAST(:purpose AS text),
CAST(:floors AS integer),
CAST(:floors_underground AS integer),
CAST(:year_built AS integer),
CAST(:cost_value AS numeric),
CAST(:build_record_area AS numeric),
CAST(:address AS text),
CASE WHEN :geom_json IS NULL THEN NULL
ELSE ST_Transform(ST_SetSRID(ST_GeomFromGeoJSON(:geom_json), 3857), 4326)
END,
CAST(:snapshot_date AS date)
)
ON CONFLICT (cad_num) DO UPDATE SET
quarter_cad = EXCLUDED.quarter_cad,
purpose = EXCLUDED.purpose,
floors = EXCLUDED.floors,
floors_underground = EXCLUDED.floors_underground,
year_built = EXCLUDED.year_built,
cost_value = EXCLUDED.cost_value,
build_record_area = EXCLUDED.build_record_area,
address = EXCLUDED.address,
geom = EXCLUDED.geom,
snapshot_date = EXCLUDED.snapshot_date,
updated_at = NOW()
"""
)
def denorm_building_feature(
db: Session,
*,
feature: dict[str, Any],
quarter_cad: str,
snapshot_date: str,
) -> bool:
"""UPSERT one building feature → nspd_buildings.
Returns True если строка вставлена/обновлена, False при пропуске (нет cad_num).
"""
props = feature.get("properties") or {}
cad_num = props.get("cad_num") or props.get("cadastral_number")
if not cad_num:
return False
geom = feature.get("geometry")
geom_json: str | None = json.dumps(geom, ensure_ascii=False) if geom else None
try:
with db.begin_nested():
db.execute(
_BUILDING_UPSERT_SQL,
{
"cad_num": cad_num,
"quarter_cad": quarter_cad,
"purpose": props.get("purpose"),
"floors": _coerce_int(props.get("floors_above_ground")),
"floors_underground": _coerce_int(props.get("floors_underground")),
"year_built": _coerce_int(props.get("year_built")),
"cost_value": _coerce_numeric(props.get("cost_value")),
"build_record_area": _coerce_numeric(props.get("build_record_area")),
"address": props.get("address"),
"geom_json": geom_json,
"snapshot_date": snapshot_date,
},
)
return True
except Exception as e:
logger.warning("denorm building %s failed: %s", cad_num, e)
return False
# ── Batch helper ───────────────────────────────────────────────────────────────
def denorm_dump(
db: Session,
*,
quarter_cad: str,
features: list[dict[str, Any]],
) -> dict[str, int]:
"""Denorm all features из одного quarter dump → nspd_parcels / nspd_buildings.
Итерирует features_json плоский список. Записи с layer="parcels" идут в
nspd_parcels, layer="buildings" → nspd_buildings. Остальные layers пропускаются.
Каждая строка UPSERT'ится в отдельном SAVEPOINT (begin_nested) — ошибка
одной строки не откатывает весь batch.
Args:
db: SQLAlchemy Session. Caller отвечает за commit/close после вызова.
quarter_cad: 3-сегментный кадастровый квартал.
features: плоский list из features_json JSONB (уже декодированный Python list).
Returns:
dict {"parcels": N, "buildings": M, "errors": K} — количество обработанных строк.
"""
snapshot_date = datetime.date.today().isoformat()
parcels_n = 0
buildings_n = 0
errors_n = 0
for feat in features:
layer = feat.get("layer", "")
try:
if layer == "parcels":
if denorm_parcel_feature(
db, feature=feat, quarter_cad=quarter_cad, snapshot_date=snapshot_date
):
parcels_n += 1
else:
errors_n += 1
elif layer == "buildings":
if denorm_building_feature(
db, feature=feat, quarter_cad=quarter_cad, snapshot_date=snapshot_date
):
buildings_n += 1
else:
errors_n += 1
# Остальные layers (territorial_zones, zouit_*, risk_*, ...) — пропускаем
except Exception as e:
logger.warning("denorm feature (layer=%s quarter=%s) failed: %s", layer, quarter_cad, e)
errors_n += 1
db.commit()
logger.info(
"denorm_dump quarter=%s parcels=%d buildings=%d errors=%d",
quarter_cad,
parcels_n,
buildings_n,
errors_n,
)
return {"parcels": parcels_n, "buildings": buildings_n, "errors": errors_n}

175
backend/app/services/scrapers/obj_checks.py Normal file
View file

@ -0,0 +1,175 @@
"""Extractor for 6 «Проверено на наш.дом.рф» checks per object.
Issue #297, sub-task 22f. Table created in PR #303 (data/sql/111_22f_domrf_obj_checks.sql).
Check types (canonical):
no_problems / docs / timing / photos / bankruptcy / declaration
Source endpoint: /сервисы/api/object/{obj_id}/checks
(pattern analogичен /infrastructure и /photos — те же сервисы/api/object/{obj_id}/ prefix)
URL not verified via devtools — структура payload выведена из аудита страницы ЖК.
Если endpoint не существует — scraper получит HTTP-ошибку, которую _classify_and_log
запишет в kn_scrape_failures, данные в domrf_obj_checks не поступят, scrape не упадёт.
Expected payload shape (предположительно):
{
"data": {
"noProblemObjects": true, # no_problems
"hasDocuments": true, # docs
"meetsDeadlines": true, # timing
"hasPhotos": true, # photos
"notBankrupt": true, # bankruptcy
"hasDeclaration": true # declaration
}
}
OR possibly an array:
[{"checkType": "no_problems", "passed": true}, ...]
Поддерживаются оба варианта: dict-payload (поля маппятся через CHECK_FIELD_MAP)
и list-payload (поля check_type + passed/value).
"""
from __future__ import annotations
import logging
from typing import Any
from sqlalchemy import text
from sqlalchemy.orm import Session
logger = logging.getLogger(__name__)
CHECK_TYPES = ["no_problems", "docs", "timing", "photos", "bankruptcy", "declaration"]
# Mapping of possible API field names → canonical check_type.
# Best-guess from DOM.РФ API field naming conventions (кейс camelCase).
_CHECK_FIELD_MAP: dict[str, str] = {
"noProblemObjects": "no_problems",
"noProblemFlg": "no_problems",
"hasDocuments": "docs",
"documentsFlg": "docs",
"meetsDeadlines": "timing",
"deadlinesFlg": "timing",
"hasPhotos": "photos",
"photosFlg": "photos",
"notBankrupt": "bankruptcy",
"bankruptcyFlg": "bankruptcy",
"hasDeclaration": "declaration",
"declarationFlg": "declaration",
}
# Canonical check_type name → possible API field name aliases
_CHECK_TYPE_ALIASES: dict[str, list[str]] = {
"no_problems": ["no_problems", "noProblemObjects", "noProblemFlg"],
"docs": ["docs", "hasDocuments", "documentsFlg"],
"timing": ["timing", "meetsDeadlines", "deadlinesFlg"],
"photos": ["photos", "hasPhotos", "photosFlg"],
"bankruptcy": ["bankruptcy", "notBankrupt", "bankruptcyFlg"],
"declaration": ["declaration", "hasDeclaration", "declarationFlg"],
}
_UPSERT_CHECKS_SQL = text(
"""
INSERT INTO domrf_obj_checks (obj_id, check_type, passed, checked_at, scraped_at)
VALUES (:obj_id, :check_type, :passed, NOW(), NOW())
ON CONFLICT (obj_id, check_type) DO UPDATE SET
passed = EXCLUDED.passed,
checked_at = NOW(),
scraped_at = NOW()
"""
)
def extract_obj_checks(raw_payload: Any) -> list[dict[str, Any]]:
"""Извлечь 6 чекбоксов из payload endpoint /object/{obj_id}/checks.
Поддерживает два варианта payload:
1. dict с полями (ожидаемый API-формат): {"data": {"noProblemObjects": true, ...}}
2. list объектов: [{"checkType": "no_problems", "passed": true}, ...]
Для неизвестных полей и неподдерживаемых форматов — WARNING + пустой список.
"""
if not raw_payload:
return []
# Вариант 1: dict с data-обёрткой
data: Any = raw_payload
if isinstance(raw_payload, dict):
data = raw_payload.get("data") or raw_payload
results: list[dict[str, Any]] = []
if isinstance(data, dict):
# Map известных полей к canonical check_type
found: dict[str, bool] = {}
for field, value in data.items():
ct = _CHECK_FIELD_MAP.get(field)
if ct and ct not in found:
found[ct] = bool(value)
# Также проверить canonical names напрямую
for ct in CHECK_TYPES:
if ct not in found and ct in data:
found[ct] = bool(data[ct])
if found:
for ct in CHECK_TYPES:
results.append({"check_type": ct, "passed": found.get(ct, False)})
return results
# dict не содержит известных полей — попробуем как list-формат ниже
logger.warning(
"domrf obj_checks: dict payload has no known check fields: %s", list(data)[:10]
)
return []
if isinstance(data, list):
found_list: dict[str, bool] = {}
for item in data:
if not isinstance(item, dict):
continue
ct_raw = item.get("checkType") or item.get("check_type") or item.get("type")
if ct_raw and str(ct_raw) in CHECK_TYPES:
passed_raw = item.get("passed") or item.get("value") or item.get("status")
found_list[str(ct_raw)] = bool(passed_raw)
if found_list:
for ct in CHECK_TYPES:
results.append({"check_type": ct, "passed": found_list.get(ct, False)})
return results
logger.warning(
"domrf obj_checks: list payload has no recognisable check items: %s", data[:3]
)
return []
logger.warning("domrf obj_checks: unexpected payload type %s", type(raw_payload))
return []
def upsert_obj_checks(db: Session, obj_id: int, checks: list[dict[str, Any]]) -> int:
"""UPSERT 6 чек-строк в domrf_obj_checks. Returns count of inserted/updated rows.
Использует SAVEPOINT (begin_nested) per-row — одна битая строка не откатывает
всю транзакцию.
"""
if not checks:
return 0
ok = 0
for c in checks:
try:
with db.begin_nested():
db.execute(
_UPSERT_CHECKS_SQL,
{
"obj_id": obj_id,
"check_type": c["check_type"],
"passed": c["passed"],
},
)
ok += 1
except Exception as exc:
logger.warning(
"upsert obj_checks obj=%s check_type=%s failed: %s",
obj_id,
c.get("check_type"),
exc,
)
db.commit()
return ok

144
backend/app/services/scrapers/objective.py
View file

@ -26,6 +26,8 @@ from __future__ import annotations
import json
import logging
import time
from collections.abc import Iterator
from contextlib import contextmanager
from datetime import date, datetime
from typing import Any
@ -238,6 +240,148 @@ class ObjectiveClient:
f"Невалидный JSON: {e}; первые 200 символов: {r.text[:200]}"
) from e
def _open_stream(
self,
path: str,
params: dict[str, Any],
*,
attempt: int = 0,
force_refresh_token: bool = False,
) -> httpx.Response:
"""Открывает streaming GET, возвращает httpx.Response (не читает тело).
Включает retry/auth логику идентичную _request_authed().
Caller обязан держать контекст через `with self._client.stream(...)` —
вызов делается внутри stream_report() который гарантирует закрытие.
"""
token = self._get_token(force_refresh=force_refresh_token)
url = f"{self.base_url}{path}"
self._wait_rate_limit()
try:
r = self._client.send(
self._client.build_request(
"GET",
url,
params=params,
headers={"Authorization": f"Bearer {token}"},
),
stream=True,
)
except httpx.HTTPError as e:
if attempt < self.retries:
wait = min(2**attempt, 30)
logger.warning(
"Objective stream net-error %s (attempt %d/%d), retry in %ds",
e,
attempt + 1,
self.retries,
wait,
)
time.sleep(wait)
return self._open_stream(path, params, attempt=attempt + 1)
raise ObjectiveAPIError(
f"Сетевая ошибка stream после {self.retries} попыток: {e}"
) from e
if r.status_code == 401:
r.close()
if not force_refresh_token:
logger.info("Objective stream: 401, рефрешим токен и повторяем")
return self._open_stream(path, params, attempt=attempt, force_refresh_token=True)
raise ObjectiveAuthError("401 даже после refresh токена — apiKey невалиден")
if r.status_code in (429, 502, 503, 504):
retry_after_hdr = r.headers.get("Retry-After")
r.close()
if attempt < self.retries:
if retry_after_hdr and retry_after_hdr.isdigit():
wait = min(int(retry_after_hdr), 300)
else:
wait = min(30 * (2**attempt), 300)
logger.warning(
"Objective stream HTTP %s (attempt %d/%d), retry in %ds",
r.status_code,
attempt + 1,
self.retries,
wait,
)
time.sleep(wait)
return self._open_stream(path, params, attempt=attempt + 1)
raise ObjectiveAPIError(f"Stream HTTP {r.status_code} после ретраев")
if r.status_code != 200:
body_preview = r.read()[:200]
r.close()
raise ObjectiveAPIError(f"Stream HTTP {r.status_code}: {body_preview!r}")
return r
@contextmanager
def stream_report(
self,
*,
report_section: str = "Объединенные данные",
report_type: str = "Поквартирные",
report_name: str = "Лоты",
group_name: str | None = None,
complex_name: str | None = None,
start_date: date | str | None = None,
end_date: date | str | None = None,
use_ddu: bool | None = True,
use_dkp: bool | None = None,
page: str = "Отчеты",
v2: bool = True,
) -> Iterator[httpx.Response]:
"""Контекст-менеджер для streaming GetReport.
Usage:
with client.stream_report(report_type='Поквартирные', report_name='Лоты',
group_name='Свердловская область') as resp:
for chunk in resp.iter_bytes(65536):
...
Caller итерирует тело через resp.iter_bytes() или resp.iter_raw().
Метод НЕ вызывает .json() — всё чтение на стороне caller'а.
"""
path = "/v2/Report/GetReport" if v2 else "/Report/GetReport"
params: dict[str, Any] = {
"Page": page,
"ReportSection": report_section,
"ReportType": report_type,
"ReportName": report_name,
"GroupName": group_name or settings.objective_default_group,
}
if complex_name:
params["ComplexName"] = complex_name
if start_date is not None:
params["StartDate"] = self._fmt_date(start_date)
if end_date is not None:
params["EndDate"] = self._fmt_date(end_date)
if v2:
if use_ddu is not None:
params["UseDdu"] = "true" if use_ddu else "false"
if use_dkp is not None:
params["UseDkp"] = "true" if use_dkp else "false"
logger.info(
"Objective.stream_report: %s/%s/%s gr=%s cn=%s ddu=%s dkp=%s",
report_section,
report_type,
report_name,
params["GroupName"],
complex_name,
use_ddu,
use_dkp,
)
resp = self._open_stream(path, params)
try:
yield resp
finally:
try:
resp.close()
except Exception:
pass
# ── публичные методы отчётов ────────────────────────────────────────────
@staticmethod

2
backend/app/services/scrapers/stealth.py
View file

@ -104,7 +104,7 @@ class BrowserSession:
self._browser: Browser | None = None
self._context: BrowserContext | None = None
self._page: Page | None = None
self._sem = asyncio.Semaphore(3)
self._sem = asyncio.Semaphore(8)
self._request_count = 0
async def __aenter__(self) -> BrowserSession:

734
backend/app/services/site_finder/best_layouts.py Normal file
View file

@ -0,0 +1,734 @@
"""Анализ лучших планировок конкурентов по velocity (Issue #113 Phase 2.1).
Источники:
cad_parcels_geom / cad_quarters_geom — центроид участка
domrf_kn_objects — ЖК в радиусе (latitude/longitude → geography)
objective_corpus_room_month — ежемесячные сделки по (project_name, room_bucket)
objective_complex_mapping — domrf_obj_id ↔ objective_complex_name
domrf_kn_flats — supply count по (room_bucket, area_bin)
Алгоритм:
Step 1: центроид участка (cad_parcels_geom → cad_quarters_geom fallback).
Step 2: obj_id конкурентов в радиусе (domrf_kn_objects + фильтры).
Step 3: inline SQL из objective_corpus_room_month с честным WHERE report_month фильтром.
Step 4: velocity_per_month = deals_window / months_in_window (честный time_window).
Step 5: supply side из domrf_kn_flats — один батч-запрос.
Step 6: per-row signature + sold_pct.
Step 7: фильтр min_velocity + sort + rank.
Step 8: build recommendation_for_tz (unit-mix, price, rationale).
Step 9: data_quality (coverage + confidence).
Fix SF-01: раньше mv_layout_velocity (24 мес) делился на divisor (4/12) — данные
не менялись при смене time_window. Теперь inline SQL с реальным фильтром report_month.
"""
from __future__ import annotations
import datetime as dt
import logging
from typing import Any
from sqlalchemy import text
from sqlalchemy.orm import Session
from app.schemas.parcel import (
BestLayoutsRequest,
BestLayoutsResponse,
LayoutDataQuality,
LayoutTzMixRow,
LayoutTzRecommendation,
TopLayoutRow,
)
from app.services.site_finder.layout_signature import area_bin, layout_signature
logger = logging.getLogger(__name__)
# Confidence thresholds (per coverage % of objects with MV velocity data)
# Tune via PR if business feedback требует.
LAYOUT_CONFIDENCE_HIGH_PCT = 50.0
LAYOUT_CONFIDENCE_MEDIUM_PCT = 20.0
# Fix SF-09: cap доминирующего bucket чтобы рекомендация не зеркалила перекос рынка.
# Избыток перераспределяется пропорционально остальным bucket'ам.
MAX_BUCKET_SHARE_PCT = 35
# Параметры time_window: (PostgreSQL interval string, months divisor для velocity_per_month).
# Используются в _INLINE_VELOCITY_SQL — реальный фильтр по report_month.
# Fix SF-01: убраны _VELOCITY_DIVISORS, которые делили MV (24 мес) без изменения данных.
_TIME_WINDOW_PARAMS: dict[str, tuple[str, float]] = {
"last_month": ("1 month", 1.0),
"last_quarter": ("3 months", 3.0),
"last_year": ("12 months", 12.0),
}
# ── SQL: центроид участка ─────────────────────────────────────────────────────
_PARCEL_CENTROID_SQL = text("""
SELECT ST_X(pt) AS center_lon,
ST_Y(pt) AS center_lat
FROM (
SELECT ST_Centroid(geom) AS pt
FROM cad_parcels_geom
WHERE cad_num = :cad_num AND geom IS NOT NULL
UNION ALL
SELECT ST_Centroid(geom) AS pt
FROM cad_quarters_geom
WHERE cad_number = :quarter AND geom IS NOT NULL
) sub
LIMIT 1
""")
# ── SQL: obj_id конкурентов в радиусе ─────────────────────────────────────────
# Геометрия domrf_kn_objects вычисляется on-the-fly из (latitude, longitude)
# как ST_SetSRID(ST_MakePoint(longitude, latitude), 4326)::geography
# (consistency с competitors.py).
# obj_class_filter: NULL = все классы.
# filter_competitor_obj_ids: NULL = не фильтровать по списку.
_COMPETITORS_IN_RADIUS_SQL = text("""
SELECT DISTINCT ON (obj_id) obj_id
FROM domrf_kn_objects
WHERE latitude IS NOT NULL AND longitude IS NOT NULL
AND ST_DWithin(
ST_SetSRID(ST_MakePoint(longitude, latitude), 4326)::geography,
ST_SetSRID(
ST_MakePoint(CAST(:center_lon AS float), CAST(:center_lat AS float)),
4326
)::geography,
CAST(:radius_m AS float)
)
AND (
CAST(:obj_class_filter AS text) IS NULL
OR obj_class = CAST(:obj_class_filter AS text)
)
ORDER BY obj_id, snapshot_date DESC NULLS LAST
""")
# ── SQL: inline velocity из objective_corpus_room_month + mapping ─────────────
# Fix SF-01: честный фильтр по report_month вместо деления mv_layout_velocity (24 мес).
# Параметры:
# :window_interval — PostgreSQL interval string ('1 month', '3 months', '12 months')
# :competitor_obj_ids — list[int] obj_id конкурентов в радиусе
# CAST(:window_interval AS interval) — psycopg v3 / SQLAlchemy 2.0 safe (не ::interval).
_INLINE_VELOCITY_SQL = text("""
SELECT
CASE
WHEN crm.room_bucket = 'студия' THEN 'studio'
ELSE crm.room_bucket
END AS room_bucket,
SUM(crm.deals_total_count) AS deals_window,
COALESCE(
SUM(crm.deals_total_avg_area_m2 * crm.deals_total_count)
/ NULLIF(SUM(crm.deals_total_count), 0),
0
)::numeric(10, 2) AS avg_area_m2,
COALESCE(
SUM(crm.deals_total_avg_price_thousand_rub_per_m2 * crm.deals_total_count)
/ NULLIF(SUM(crm.deals_total_count), 0),
0
)::numeric(12, 2) * 1000.0 AS avg_price_per_m2_rub,
array_agg(DISTINCT cm.domrf_obj_id) AS competitor_obj_ids,
COUNT(DISTINCT cm.domrf_obj_id) AS competitor_count,
MIN(crm.report_month) AS window_start,
MAX(crm.report_month) AS window_end
FROM objective_corpus_room_month crm
JOIN objective_complex_mapping cm
ON cm.objective_complex_name = crm.project_name
WHERE crm.report_month >= (NOW() - CAST(:window_interval AS interval))::date
AND cm.domrf_obj_id = ANY(:competitor_obj_ids)
AND crm.room_bucket IS NOT NULL
GROUP BY
CASE
WHEN crm.room_bucket = 'студия' THEN 'studio'
ELSE crm.room_bucket
END
""")
# ── SQL: supply по (room_bucket, area_bin) за последний снимок ───────────────
# Один батч-запрос вместо N — возвращает map (rb, ab) → count.
# room_bucket и area_bin вычисляются в SQL аналогично layout_signature.py.
_SUPPLY_BATCH_SQL = text("""
SELECT
CASE
WHEN f.is_studio = TRUE OR f.flat_type = 'Квартира-студия' THEN 'studio'
WHEN f.rooms = 0 THEN 'studio'
-- Fix SF-08: euro-форматы — DOM.РФ маркирует малогабаритные квартиры как 2-комн.
-- rooms=2 + area<35 → euro-1 (студия с отдельной кухней ~26м²)
-- rooms=2 + area<50 → euro-2 (~35-50м², евро-двушка)
WHEN f.rooms = 2 AND f.total_area < 35 THEN 'euro-1'
WHEN f.rooms = 2 AND f.total_area < 50 THEN 'euro-2'
WHEN f.rooms IN (1, 2, 3) THEN f.rooms::text
WHEN f.rooms >= 4 THEN '4+'
ELSE '1'
END AS rb,
CASE
WHEN f.total_area < 25 THEN '<25'
WHEN f.total_area < 40 THEN '25-40'
WHEN f.total_area < 60 THEN '40-60'
WHEN f.total_area < 80 THEN '60-80'
WHEN f.total_area < 100 THEN '80-100'
ELSE '100+'
END AS ab,
COUNT(*) AS units
FROM domrf_kn_flats f
JOIN domrf_kn_objects o ON f.obj_id = o.obj_id
WHERE o.latitude IS NOT NULL AND o.longitude IS NOT NULL
AND ST_DWithin(
ST_SetSRID(ST_MakePoint(o.longitude, o.latitude), 4326)::geography,
ST_SetSRID(
ST_MakePoint(CAST(:center_lon AS float), CAST(:center_lat AS float)),
4326
)::geography,
CAST(:radius_m AS float)
)
AND f.snapshot_date = CAST(:latest_snap AS date)
GROUP BY rb, ab
""")
# ── Вспомогательные функции ───────────────────────────────────────────────────
def _quarter_from_cad(cad_num: str) -> str:
"""Извлечь кадастровый квартал: '66:41:0303161:123' → '66:41:0303161'."""
parts = cad_num.split(":")
if len(parts) >= 3:
return ":".join(parts[:3])
return cad_num
def _normalize_pct(buckets: dict[str, float]) -> dict[str, int]:
"""Нормировать доли до целых процентов с суммой ровно 100.
Алгоритм largest-remainder (Hamilton method):
1. Floor каждого значения.
2. Остаток 100 − sum_floors распределить в top-bucket по дробной части.
"""
if not buckets:
return {}
total = sum(buckets.values())
if total <= 0:
n = len(buckets)
base = 100 // n
result = {k: base for k in buckets}
# распределить остаток
remainder = 100 - base * n
for k in list(buckets.keys())[:remainder]:
result[k] += 1
return result
raw = {k: v / total * 100.0 for k, v in buckets.items()}
floors = {k: int(v) for k, v in raw.items()}
remainder = 100 - sum(floors.values())
# sort by fractional part desc
fracs = sorted(buckets.keys(), key=lambda k: -(raw[k] - floors[k]))
for k in fracs[:remainder]:
floors[k] += 1
return floors
def _cap_and_redistribute(pct_map: dict[str, int]) -> tuple[dict[str, int], bool]:
"""Fix SF-09 round 2: capacity-aware redistribute, bounded iterations.
Round 1 bug: surplus распределялся пропорционально текущему `v` free bucket'а,
что переливало его выше cap — на 2-bucket вход цикл осциллировал бесконечно.
Round 2 fix: surplus распределяется пропорционально **available capacity**
`(cap - v)` каждого free bucket'а. Тогда free никогда не вылетит выше cap →
цикл сходится за ≤ len(pct_map) итераций. Hard guard `for _ in range(N+1)`.
Если surplus > total_capacity (геометрически невозможно поместить излишек ниже
cap) — забиваем все free к cap, возвращаем `cap_skipped=True` + warning log.
Returns:
(result_map, cap_skipped) — cap_skipped=True если cap не удержан
(pathological: всё хочет > cap, или surplus > available capacity).
"""
if not pct_map:
return pct_map, False
cap = MAX_BUCKET_SHARE_PCT
# Быстрый path: нет доминирующих
if all(v <= cap for v in pct_map.values()):
return pct_map, False
work: dict[str, float] = {k: float(v) for k, v in pct_map.items()}
# Bounded iteration: после k-й итерации число clamped не убывает только если
# surplus > capacity (тогда — pathological). При корректном capacity-aware
# redistribute достаточно ≤ len(pct_map) итераций.
for _ in range(len(pct_map) + 1):
clamped = [k for k, v in work.items() if v > cap]
if not clamped:
break
free = [k for k, v in work.items() if v < cap]
if not free:
# Все bucket'ы либо >cap либо ровно =cap — некуда переливать.
logger.warning(
"MAX_BUCKET_SHARE cap: нет free bucket'ов (%d total) — cap_skipped",
len(pct_map),
)
return pct_map, True
surplus = sum(work[k] - cap for k in clamped)
capacities = {k: cap - work[k] for k in free}
total_capacity = sum(capacities.values())
for k in clamped:
work[k] = float(cap)
if surplus > total_capacity + 1e-9:
# Излишек не помещается ниже cap — pathological.
# Возвращаем оригинал (sum=100 invariant) + флаг для frontend banner.
logger.warning(
"MAX_BUCKET_SHARE cap: surplus %.2f > total_capacity %.2f — cap_skipped",
surplus,
total_capacity,
)
return pct_map, True
for k in free:
work[k] += capacities[k] / total_capacity * surplus
else:
# Hard guard: не сошлись за N+1 итераций — bug. Лог + cap_skipped.
logger.error(
"MAX_BUCKET_SHARE cap: не сошлись за %d итераций — algorithm bug",
len(pct_map) + 1,
)
return pct_map, True
return _hamilton_round(work), False
def _hamilton_round(work: dict[str, float]) -> dict[str, int]:
"""Hamilton apportionment: float → integer pct с суммой ровно 100."""
floors = {k: int(v) for k, v in work.items()}
remainder = 100 - sum(floors.values())
fracs = sorted(work.keys(), key=lambda k: -(work[k] - floors[k]))
for k in fracs[: max(0, remainder)]:
floors[k] += 1
return floors
# ── Главная функция ───────────────────────────────────────────────────────────
def get_best_layouts(
db: Session,
cad_num: str,
request: BestLayoutsRequest,
) -> BestLayoutsResponse:
"""Top layouts (rooms × area_bin) конкурентов с рейтингом по velocity.
Raises:
ValueError: если центроид участка не найден (caller → HTTP 404).
"""
quarter = _quarter_from_cad(cad_num)
radius_m = request.radius_km * 1000.0
# time_window → (interval_str, months divisor)
window_interval, months_in_window = _TIME_WINDOW_PARAMS.get(
request.time_window, ("3 months", 3.0)
)
# ── Step 1: центроид участка ─────────────────────────────────────────────
try:
coord_row = (
db.execute(
_PARCEL_CENTROID_SQL,
{"cad_num": cad_num, "quarter": quarter},
)
.mappings()
.first()
)
except Exception:
logger.exception("best_layouts: centroid query failed for cad_num=%s", cad_num)
raise
if not coord_row:
raise ValueError(f"Геометрия для {cad_num} не найдена")
center_lon = float(coord_row["center_lon"])
center_lat = float(coord_row["center_lat"])
# ── Step 2: obj_id конкурентов в радиусе ────────────────────────────────
try:
id_rows = (
db.execute(
_COMPETITORS_IN_RADIUS_SQL,
{
"center_lon": center_lon,
"center_lat": center_lat,
"radius_m": radius_m,
"obj_class_filter": request.obj_class_filter,
},
)
.mappings()
.all()
)
except Exception:
logger.exception("best_layouts: competitors-in-radius query failed for cad_num=%s", cad_num)
raise
all_obj_ids: list[int] = [int(r["obj_id"]) for r in id_rows]
objects_total_in_radius = len(all_obj_ids)
# Применить exclude / filter из request
exclude_set = set(request.exclude_competitor_obj_ids)
if exclude_set:
all_obj_ids = [oid for oid in all_obj_ids if oid not in exclude_set]
if request.filter_competitor_obj_ids is not None:
filter_set = set(request.filter_competitor_obj_ids)
all_obj_ids = [oid for oid in all_obj_ids if oid in filter_set]
if not all_obj_ids:
return _empty_response(
radius_km=request.radius_km,
time_window=request.time_window,
objects_total_in_radius=objects_total_in_radius,
)
# ── Step 3: inline velocity из objective_corpus_room_month ──────────────
# Fix SF-01: честный фильтр report_month >= NOW() - window_interval.
# Разные time_window → разные deals_window, разный mix.
try:
vel_rows = (
db.execute(
_INLINE_VELOCITY_SQL,
{
"window_interval": window_interval,
"competitor_obj_ids": all_obj_ids,
},
)
.mappings()
.all()
)
except Exception:
logger.exception(
"best_layouts: inline velocity query failed for cad_num=%s obj_count=%d",
cad_num,
len(all_obj_ids),
)
raise
if not vel_rows:
return _empty_response(
radius_km=request.radius_km,
time_window=request.time_window,
objects_total_in_radius=objects_total_in_radius,
)
# ── Step 5: supply side (батч-запрос) ────────────────────────────────────
# Pre-compute последний snapshot_date один раз — избегаем subquery на каждый scan.
latest_snap: dt.date | None = db.scalar(text("SELECT MAX(snapshot_date) FROM domrf_kn_flats"))
if latest_snap is None:
logger.warning("best_layouts: domrf_kn_flats пустой (нет snapshot_date), supply=0 fallback")
supply_rows = []
else:
try:
supply_rows = (
db.execute(
_SUPPLY_BATCH_SQL,
{
"center_lon": center_lon,
"center_lat": center_lat,
"radius_m": radius_m,
"latest_snap": latest_snap,
},
)
.mappings()
.all()
)
except Exception:
logger.warning("best_layouts: supply query failed, supply=0 fallback")
supply_rows = []
supply_map: dict[tuple[str, str], int] = {
(str(r["rb"]), str(r["ab"])): int(r["units"]) for r in supply_rows
}
# ── Step 4 + 6: velocity из реального окна и enrichment per row ─────────
# Fix SF-01: velocity_per_month = deals_window / months_in_window.
# deals_window уже отфильтрован по report_month — разные time_window дают разные данные.
enriched: list[dict[str, Any]] = []
window_start: dt.date | None = None
window_end: dt.date | None = None
# Собираем obj_ids с данными в objective_corpus_room_month (для data_quality)
obj_ids_with_data: set[int] = set()
for r in vel_rows:
room_bucket = str(r["room_bucket"])
deals_window = float(r["deals_window"]) if r["deals_window"] is not None else 0.0
avg_area = float(r["avg_area_m2"]) if r["avg_area_m2"] is not None else 0.0
price_rub = (
float(r["avg_price_per_m2_rub"]) if r["avg_price_per_m2_rub"] is not None else None
)
competitor_obj_ids: list[int] = (
[int(oid) for oid in r["competitor_obj_ids"]] if r["competitor_obj_ids"] else []
)
competitor_count = int(r["competitor_count"])
obj_ids_with_data.update(competitor_obj_ids)
# Step 4: честный velocity = сделки за окно / длина окна в месяцах
velocity_per_month = round(deals_window / months_in_window, 2)
# Step 6: area_bin по avg_area (layout_signature.area_bin)
ab = area_bin(avg_area) if avg_area > 0 else "<25"
sig = layout_signature(room_bucket, ab) # type: ignore[arg-type]
supply_count = supply_map.get((room_bucket, ab), 0)
sold_pct: float | None = None
is_oversold = False
if supply_count > 0:
sold_pct_raw = deals_window / supply_count * 100.0
is_oversold = sold_pct_raw > 100.0
# Clamp at 100%: сделки за 24 мес / текущий snapshot supply несопоставимы.
# Значения >100% артефакт окна, не реальная «распроданность».
sold_pct = round(min(sold_pct_raw, 100.0), 1)
# data window
if r["window_start"] is not None:
ws = r["window_start"]
if isinstance(ws, str):
ws = dt.date.fromisoformat(ws)
elif isinstance(ws, dt.datetime):
ws = ws.date()
window_start = ws if window_start is None else min(window_start, ws)
if r["window_end"] is not None:
we = r["window_end"]
if isinstance(we, str):
we = dt.date.fromisoformat(we)
elif isinstance(we, dt.datetime):
we = we.date()
window_end = we if window_end is None else max(window_end, we)
enriched.append(
{
"room_bucket": room_bucket,
"area_bin": ab,
"signature": sig,
"competitor_obj_ids": competitor_obj_ids,
"competitor_count": competitor_count,
"sum_deals": deals_window,
"velocity_per_month": velocity_per_month,
"avg_price_per_m2_rub": price_rub,
"avg_area_m2": avg_area,
"supply_units_in_radius": supply_count,
"sold_pct_of_supply": sold_pct,
"is_oversold": is_oversold,
}
)
# ── Step 7: фильтр min_velocity + sort + rank ────────────────────────────
filtered = [
row for row in enriched if row["velocity_per_month"] >= request.min_velocity_per_month
]
filtered.sort(key=lambda r: r["velocity_per_month"], reverse=True)
top_layouts: list[TopLayoutRow] = []
for rank_idx, row in enumerate(filtered, start=1):
top_layouts.append(
TopLayoutRow(
rank=rank_idx,
room_bucket=row["room_bucket"],
area_bin=row["area_bin"],
signature=row["signature"],
competitor_obj_ids=row["competitor_obj_ids"],
competitor_count=row["competitor_count"],
total_sold_in_window=int(row["sum_deals"]),
velocity_per_month=row["velocity_per_month"],
avg_price_per_m2_rub=row["avg_price_per_m2_rub"],
avg_area_m2=round(row["avg_area_m2"], 1),
supply_units_in_radius=row["supply_units_in_radius"],
sold_pct_of_supply=row["sold_pct_of_supply"],
is_oversold=row["is_oversold"],
)
)
# ── Step 8: build recommendation_for_tz ─────────────────────────────────
# Используем filtered (только > min_velocity) для recommendation.
# Если после фильтрации всё пустое — используем enriched (все данные без фильтра).
rec_source = filtered if filtered else enriched
today = dt.date.today()
ws_date = window_start if window_start is not None else today
we_date = window_end if window_end is not None else today
recommendation = _build_recommendation(
rows=rec_source,
radius_km=request.radius_km,
time_window=request.time_window,
target_total_flats=request.target_total_flats,
window_start=ws_date,
window_end=we_date,
all_enriched=enriched,
)
# ── Step 9: data_quality ─────────────────────────────────────────────────
# Denominator = post-filter set (effective consideration set после exclude/filter).
objects_total_after_filter = len(all_obj_ids)
objects_with_data = len(obj_ids_with_data & set(all_obj_ids))
coverage_pct = (
round(objects_with_data / objects_total_after_filter * 100.0, 1)
if objects_total_after_filter > 0
else 0.0
)
if coverage_pct >= LAYOUT_CONFIDENCE_HIGH_PCT:
confidence: str = "high"
elif coverage_pct >= LAYOUT_CONFIDENCE_MEDIUM_PCT:
confidence = "medium"
else:
confidence = "low"
data_quality = LayoutDataQuality(
objects_with_velocity_data=objects_with_data,
objects_total_in_radius=objects_total_after_filter,
velocity_coverage_pct=coverage_pct,
confidence=confidence, # type: ignore[arg-type]
)
return BestLayoutsResponse(
top_layouts=top_layouts,
recommendation_for_tz=recommendation,
data_quality=data_quality,
)
def _build_recommendation(
rows: list[dict[str, Any]],
radius_km: float,
time_window: str,
target_total_flats: int | None,
window_start: dt.date,
window_end: dt.date,
all_enriched: list[dict[str, Any]],
) -> LayoutTzRecommendation:
"""Собрать LayoutTzRecommendation из enriched rows."""
if not rows:
return LayoutTzRecommendation(
rationale_text=(
f"В радиусе {radius_km}км: нет layout-паттернов с достаточной velocity."
),
mix=[],
weighted_avg_price_per_m2_rub=None,
based_on_obj_count=0,
based_on_total_deals=0,
data_window_start=window_start,
data_window_end=window_end,
)
# Группировка по room_bucket (строки уже могут быть per-bucket из MV GROUP BY)
rb_deals: dict[str, float] = {}
rb_area_weighted: dict[str, float] = {}
rb_price_weighted: dict[str, float] = {}
rb_price_total_deals: dict[str, float] = {}
all_competitor_ids: set[int] = set()
for row in rows:
rb = row["room_bucket"]
sd = float(row["sum_deals"])
rb_deals[rb] = rb_deals.get(rb, 0.0) + sd
rb_area_weighted[rb] = rb_area_weighted.get(rb, 0.0) + row["avg_area_m2"] * sd
all_competitor_ids.update(row["competitor_obj_ids"])
if row["avg_price_per_m2_rub"] is not None:
rb_price_weighted[rb] = rb_price_weighted.get(rb, 0.0) + (
row["avg_price_per_m2_rub"] * sd
)
rb_price_total_deals[rb] = rb_price_total_deals.get(rb, 0.0) + sd
total_deals = sum(rb_deals.values())
pct_map = _normalize_pct(rb_deals)
pct_map, cap_skipped = _cap_and_redistribute(pct_map)
mix: list[LayoutTzMixRow] = []
for rb, pct in sorted(pct_map.items(), key=lambda x: -x[1]):
avg_area = (
round(rb_area_weighted[rb] / rb_deals[rb], 1) if rb_deals.get(rb, 0) > 0 else None
)
abs_units: int | None = None
if target_total_flats is not None:
abs_units = round(pct / 100.0 * target_total_flats)
mix.append(
LayoutTzMixRow(
room_bucket=rb,
pct=pct,
abs_units=abs_units,
avg_target_area_m2=avg_area,
)
)
# Weighted avg price across all room_buckets
total_price_deals = sum(rb_price_total_deals.values())
weighted_price: float | None = None
if total_price_deals > 0:
weighted_price = round(sum(rb_price_weighted.values()) / total_price_deals, 0)
# Rationale
competitor_count = len(all_competitor_ids)
tw_label = {"last_month": "1 мес", "last_quarter": "квартал", "last_year": "год"}.get(
time_window, time_window
)
rationale_text = (
f"В радиусе {radius_km}км за {tw_label}: "
f"{len(rows)} активных layout-паттернов, "
f"total {int(total_deals)} продаж в {competitor_count} ЖК"
)
# based_on_obj_count из all_enriched (уникальные obj_id с данными MV)
all_mv_obj_ids: set[int] = set()
for row in all_enriched:
all_mv_obj_ids.update(row["competitor_obj_ids"])
return LayoutTzRecommendation(
rationale_text=rationale_text,
mix=mix,
weighted_avg_price_per_m2_rub=weighted_price,
based_on_obj_count=len(all_mv_obj_ids),
based_on_total_deals=int(total_deals),
data_window_start=window_start,
data_window_end=window_end,
cap_skipped=cap_skipped,
)
def _empty_response(
radius_km: float,
time_window: str,
objects_total_in_radius: int,
) -> BestLayoutsResponse:
"""Ответ когда нет конкурентов или нет MV данных."""
today = dt.date.today()
tw_label = {"last_month": "1 мес", "last_quarter": "квартал", "last_year": "год"}.get(
time_window, time_window
)
return BestLayoutsResponse(
top_layouts=[],
recommendation_for_tz=LayoutTzRecommendation(
rationale_text=(
f"В радиусе {radius_km}км за {tw_label}: "
f"конкуренты не найдены или нет данных velocity."
),
mix=[],
weighted_avg_price_per_m2_rub=None,
based_on_obj_count=0,
based_on_total_deals=0,
data_window_start=today,
data_window_end=today,
),
data_quality=LayoutDataQuality(
objects_with_velocity_data=0,
objects_total_in_radius=objects_total_in_radius,
velocity_coverage_pct=0.0,
confidence="low",
),
)

305
backend/app/services/site_finder/competitors.py Normal file
View file

@ -0,0 +1,305 @@
"""Анализ активных конкурентов ЖК в радиусе от участка.
Issue #112 — Demand: активные конкуренты, продажи ЖК в радиусе 1км за квартал.
Источники:
domrf_kn_objects — ЖК с lat/lon, flat_count, obj_class, site_status
objective_complex_mapping — domrf_obj_id → objective_complex_name
objective_corpus_room_month — monthly deals_total_count per project_name
cad_parcels_geom — centroid участка (fallback: cad_quarters_geom)
domrf_kn_flats — avg price_per_m2 по проданным квартирам
Внимание: velocity coverage ~2.5% — большинство ЖК не имеют маппинга в
objective_complex_mapping. LEFT JOIN гарантирует velocity=0 (не ошибку) для
немаппированных объектов.
"""
from __future__ import annotations
import logging
from sqlalchemy import text
from sqlalchemy.orm import Session
from app.schemas.parcel import (
Competitor,
CompetitorsRequest,
CompetitorsResponse,
CompetitorsSummary,
)
logger = logging.getLogger(__name__)
# Маппинг time_window → число месяцев (float для деления velocity)
_TIME_WINDOW_MONTHS: dict[str, float] = {
"last_month": 1.0,
"last_quarter": 3.0,
"last_year": 12.0,
}
# site_status значения, считающиеся «активными»
_ACTIVE_STATUSES = frozenset({"sales", "construction"})
# SQL для получения центроида участка
_PARCEL_CENTROID_SQL = text("""
SELECT ST_X(pt) AS lon, ST_Y(pt) AS lat
FROM (
SELECT ST_Centroid(geom) AS pt
FROM cad_parcels_geom
WHERE cad_num = :cad_num AND geom IS NOT NULL
UNION ALL
SELECT ST_Centroid(geom) AS pt
FROM cad_quarters_geom
WHERE cad_number = :quarter AND geom IS NOT NULL
) sub
LIMIT 1
""")
# Основной запрос конкурентов в радиусе.
# Velocity через objective_corpus_room_month (актуальные данные, обновляется еженедельно).
# domrf_kn_sale_graph устарел (данные до 2026-01) — не используется.
# Coverage velocity ~2.5%: большинство obj_id нет в objective_complex_mapping →
# LEFT JOIN → velocity=0 (не ошибка).
_COMPETITORS_SQL = text("""
WITH latest_obj AS (
SELECT DISTINCT ON (obj_id)
obj_id,
comm_name,
dev_name,
obj_class,
latitude,
longitude,
flat_count,
site_status,
snapshot_date
FROM domrf_kn_objects
WHERE latitude IS NOT NULL
AND longitude IS NOT NULL
ORDER BY obj_id, snapshot_date DESC NULLS LAST
),
mapped AS (
SELECT cm.domrf_obj_id AS obj_id,
cm.objective_complex_name
FROM objective_complex_mapping cm
),
velocity AS (
SELECT
m.obj_id,
SUM(COALESCE(crm.deals_total_count, 0))
/ CAST(:time_window_months AS float) AS velocity_per_month
FROM objective_corpus_room_month crm
JOIN mapped m ON m.objective_complex_name = crm.project_name
WHERE crm.report_month >= (NOW() - CAST(:window_interval AS interval))
GROUP BY m.obj_id
),
distances AS (
SELECT
o.obj_id,
o.comm_name,
o.dev_name,
o.obj_class,
o.latitude,
o.longitude,
o.flat_count,
o.site_status,
ST_Distance(
ST_SetSRID(ST_MakePoint(o.longitude, o.latitude), 4326)::geography,
ST_SetSRID(
ST_MakePoint(CAST(:center_lon AS float), CAST(:center_lat AS float)),
4326
)::geography
) AS distance_m
FROM latest_obj o
)
SELECT
d.obj_id,
d.comm_name,
d.dev_name,
d.obj_class,
d.latitude,
d.longitude,
d.flat_count,
d.site_status,
d.distance_m,
COALESCE(v.velocity_per_month, 0.0) AS velocity_per_month
FROM distances d
LEFT JOIN velocity v ON v.obj_id = d.obj_id
WHERE d.distance_m <= CAST(:radius_m AS float)
AND (
CAST(:obj_class_filter AS text) IS NULL
OR d.obj_class = CAST(:obj_class_filter AS text)
)
ORDER BY d.distance_m ASC
""")
# Средняя цена м² по квартирам с известной ценой для набора obj_id.
# Фильтр status='sold' убран: поле status в domrf_kn_flats заполнено в ~0.2% строк
# (99.8% NULL) — фильтр давал 0 строк и avg_price_per_m2 всегда None (Issue #112/227).
# AVG по всем квартирам с price_per_m2 IS NOT NULL даёт корректную среднюю цену ЖК.
_AVG_PRICE_SQL = text("""
SELECT
f.obj_id,
AVG(f.price_per_m2) AS avg_price_per_m2
FROM domrf_kn_flats f
WHERE f.obj_id = ANY(:obj_ids)
AND f.price_per_m2 IS NOT NULL
GROUP BY f.obj_id
""")
def _quarter_from_cad(cad_num: str) -> str:
"""Извлечь кадастровый квартал из номера участка/здания.
66:41:0303161:123 → 66:41:0303161
Если формат нестандартный — возвращаем cad_num как есть (fallback).
"""
parts = cad_num.split(":")
if len(parts) >= 3:
return ":".join(parts[:3])
return cad_num
def get_competitors(
db: Session,
cad_num: str,
request: CompetitorsRequest,
) -> CompetitorsResponse:
"""Получить список конкурентов ЖК в радиусе от участка.
Шаги:
1. Найти центроид участка (cad_parcels_geom → cad_quarters_geom fallback).
2. Выбрать ЖК из domrf_kn_objects в радиусе с velocity из objective_corpus_room_month.
3. Применить exclude_obj_ids фильтр в Python (избегаем array cast).
4. Подтянуть avg_price_per_m2 из domrf_kn_flats.
5. Собрать CompetitorsResponse.
Raises:
ValueError: если центроид участка не найден (caller должен вернуть 404).
"""
quarter = _quarter_from_cad(cad_num)
# ── 1. Центроид участка ──────────────────────────────────────────────────
try:
coord_row = (
db.execute(
_PARCEL_CENTROID_SQL,
{"cad_num": cad_num, "quarter": quarter},
)
.mappings()
.first()
)
except Exception:
logger.exception("competitors: centroid query failed for cad_num=%s", cad_num)
raise
if not coord_row:
raise ValueError(f"Геометрия для {cad_num} не найдена")
center_lat = float(coord_row["lat"])
center_lon = float(coord_row["lon"])
# ── 2. Конкуренты в радиусе ──────────────────────────────────────────────
time_window_months = _TIME_WINDOW_MONTHS[request.time_window]
window_interval = f"{int(time_window_months)} months"
try:
rows = (
db.execute(
_COMPETITORS_SQL,
{
"center_lat": center_lat,
"center_lon": center_lon,
"radius_m": request.radius_km * 1000.0,
"time_window_months": time_window_months,
"window_interval": window_interval,
"obj_class_filter": request.obj_class_filter,
},
)
.mappings()
.all()
)
except Exception:
logger.exception(
"competitors: main query failed for cad_num=%s radius_km=%.2f",
cad_num,
request.radius_km,
)
raise
# ── 3. Применить exclude_obj_ids ─────────────────────────────────────────
exclude_set = set(request.exclude_obj_ids)
if exclude_set:
rows = [r for r in rows if int(r["obj_id"]) not in exclude_set]
if not rows:
return CompetitorsResponse(
competitors=[],
summary=CompetitorsSummary(
total_competitors=0,
active_count=0,
weighted_avg_velocity=0.0,
radius_km=request.radius_km,
time_window=request.time_window,
),
)
obj_ids: list[int] = [int(r["obj_id"]) for r in rows]
# ── 4. Средняя цена м² (graceful — таблица может быть не заполнена) ──────
avg_price_map: dict[int, float] = {}
try:
price_rows = db.execute(_AVG_PRICE_SQL, {"obj_ids": obj_ids}).mappings().all()
avg_price_map = {
int(r["obj_id"]): float(r["avg_price_per_m2"])
for r in price_rows
if r["avg_price_per_m2"] is not None
}
except Exception:
logger.warning("competitors: avg_price query failed, continuing without prices")
# ── 5. Сборка результата ─────────────────────────────────────────────────
# flats_sold / sold_pct: не доступны из domrf_kn_objects (только flat_count).
# Можно получить через COUNT(domrf_kn_flats WHERE status='sold') —
# отложено за MVP, поля остаются None.
competitors: list[Competitor] = []
for r in rows:
obj_id = int(r["obj_id"])
flats_total = int(r["flat_count"]) if r["flat_count"] is not None else None
site_status = r["site_status"]
is_active = site_status in _ACTIVE_STATUSES if site_status else False
competitors.append(
Competitor(
obj_id=obj_id,
comm_name=r["comm_name"],
dev_name=r["dev_name"],
obj_class=r["obj_class"],
distance_m=round(float(r["distance_m"]), 1),
lat=float(r["latitude"]),
lng=float(r["longitude"]),
stage=site_status,
flats_total=flats_total,
flats_sold=None,
sold_pct=None,
velocity_per_month=round(float(r["velocity_per_month"]), 2),
avg_price_per_m2=avg_price_map.get(obj_id),
is_active=is_active,
)
)
# ── 6. Summary ───────────────────────────────────────────────────────────
active_count = sum(1 for c in competitors if c.is_active)
total_velocity = sum(c.velocity_per_month for c in competitors)
n = len(competitors)
weighted_avg_velocity = round(total_velocity / n, 2) if n > 0 else 0.0
summary = CompetitorsSummary(
total_competitors=n,
active_count=active_count,
weighted_avg_velocity=weighted_avg_velocity,
radius_km=request.radius_km,
time_window=request.time_window,
)
return CompetitorsResponse(competitors=competitors, summary=summary)

276
backend/app/services/site_finder/custom_pois.py Normal file
View file

@ -0,0 +1,276 @@
"""CRUD-сервис для user_custom_pois (#254).
API:
- create_custom_poi(db, user_id, payload) → CustomPoiOut
- list_custom_pois(db, user_id, parcel_cad?) → list[CustomPoiOut]
- get_custom_poi(db, poi_id, user_id) → CustomPoiOut | None
- update_custom_poi(db, poi_id, user_id, payload) → CustomPoiOut | None
- delete_custom_poi(db, poi_id, user_id) → bool
- get_overlaps_for_scoring(db, parcel_geom_wkt, user_id, parcel_cad?) → list[dict]
Паттерн: raw SQL через SQLAlchemy text() + CAST(:x AS type), psycopg v3.
"""
from __future__ import annotations
import logging
from typing import Any
from sqlalchemy import text
from app.schemas.custom_poi import CustomPoiCreate, CustomPoiOut, CustomPoiUpdate
logger = logging.getLogger(__name__)
# ── SQL ────────────────────────────────────────────────────────────────────────
_SELECT_COLS = """
id, user_id, parcel_cad, name, category, weight, lon, lat, notes,
created_at, updated_at
"""
_INSERT_SQL = f"""
INSERT INTO user_custom_pois
(user_id, parcel_cad, name, category, weight, lon, lat, notes)
VALUES
(:user_id, :parcel_cad, :name, :category,
CAST(:weight AS real), CAST(:lon AS double precision),
CAST(:lat AS double precision), :notes)
RETURNING {_SELECT_COLS}
"""
_SELECT_BY_USER_ALL = f"""
SELECT {_SELECT_COLS}
FROM user_custom_pois
WHERE user_id = :user_id
ORDER BY created_at DESC
"""
_SELECT_BY_USER_PARCEL = f"""
SELECT {_SELECT_COLS}
FROM user_custom_pois
WHERE user_id = :user_id
AND (parcel_cad = :parcel_cad OR parcel_cad IS NULL)
ORDER BY created_at DESC
"""
_SELECT_BY_ID = f"""
SELECT {_SELECT_COLS}
FROM user_custom_pois
WHERE id = :poi_id
AND user_id = :user_id
"""
_DELETE_SQL = """
DELETE FROM user_custom_pois
WHERE id = :poi_id
AND user_id = :user_id
RETURNING id
"""
# Запрос для scoring: custom POI в радиусе 1 км от центроида участка.
# Возвращает POI (global + parcel-specific) пользователя с расстоянием.
_OVERLAPS_SQL = """
SELECT p.id, p.name, p.category, p.weight, p.lon, p.lat,
ST_Distance(
p.geom,
ST_Centroid(ST_GeomFromText(CAST(:wkt AS text), 4326))::geography
) AS distance_m
FROM user_custom_pois p
WHERE p.user_id = :user_id
AND (p.parcel_cad IS NULL OR p.parcel_cad = :parcel_cad)
AND ST_DWithin(
p.geom,
ST_Centroid(ST_GeomFromText(CAST(:wkt AS text), 4326))::geography,
1000
)
ORDER BY distance_m ASC
"""
# ── Row mapper ─────────────────────────────────────────────────────────────────
def _row_to_out(r: Any) -> CustomPoiOut:
return CustomPoiOut(
id=int(r["id"]),
user_id=r["user_id"],
parcel_cad=r["parcel_cad"],
name=r["name"],
category=r["category"],
weight=float(r["weight"]),
lon=float(r["lon"]),
lat=float(r["lat"]),
notes=r["notes"],
created_at=r["created_at"],
updated_at=r["updated_at"],
)
# ── CRUD ───────────────────────────────────────────────────────────────────────
def create_custom_poi(db: Any, user_id: str, payload: CustomPoiCreate) -> CustomPoiOut:
"""Создать кастомную POI точку для пользователя."""
row = (
db.execute(
text(_INSERT_SQL),
{
"user_id": user_id,
"parcel_cad": payload.parcel_cad,
"name": payload.name,
"category": payload.category,
"weight": payload.weight,
"lon": payload.lon,
"lat": payload.lat,
"notes": payload.notes,
},
)
.mappings()
.first()
)
db.commit()
assert row is not None, "INSERT RETURNING вернул пустой результат"
logger.info(
"custom_poi created: id=%s user=%s parcel=%s weight=%s",
row["id"],
user_id,
payload.parcel_cad,
payload.weight,
)
return _row_to_out(row)
def list_custom_pois(db: Any, user_id: str, parcel_cad: str | None = None) -> list[CustomPoiOut]:
"""Вернуть custom POI пользователя.
Если parcel_cad задан — возвращает global (parcel_cad IS NULL) + parcel-specific.
Если parcel_cad=None — возвращает все POI пользователя.
"""
if parcel_cad is not None:
rows = (
db.execute(
text(_SELECT_BY_USER_PARCEL),
{"user_id": user_id, "parcel_cad": parcel_cad},
)
.mappings()
.all()
)
else:
rows = db.execute(text(_SELECT_BY_USER_ALL), {"user_id": user_id}).mappings().all()
return [_row_to_out(r) for r in rows]
def get_custom_poi(db: Any, poi_id: int, user_id: str) -> CustomPoiOut | None:
"""Вернуть одну POI по id (scoped к user_id)."""
row = db.execute(text(_SELECT_BY_ID), {"poi_id": poi_id, "user_id": user_id}).mappings().first()
if row is None:
return None
return _row_to_out(row)
def update_custom_poi(
db: Any, poi_id: int, user_id: str, payload: CustomPoiUpdate
) -> CustomPoiOut | None:
"""PATCH-style обновление кастомной POI. Возвращает None если не найдена."""
existing = get_custom_poi(db, poi_id, user_id)
if existing is None:
return None
sets: list[str] = ["updated_at = NOW()"]
params: dict[str, Any] = {"poi_id": poi_id, "user_id": user_id}
if payload.name is not None:
sets.append("name = :name")
params["name"] = payload.name
if payload.category is not None:
sets.append("category = :category")
params["category"] = payload.category
if payload.weight is not None:
sets.append("weight = CAST(:weight AS real)")
params["weight"] = payload.weight
# lon/lat изменяем только вместе — geom GENERATED ALWAYS пересчитается автоматически
if payload.lon is not None:
sets.append("lon = CAST(:lon AS double precision)")
params["lon"] = payload.lon
if payload.lat is not None:
sets.append("lat = CAST(:lat AS double precision)")
params["lat"] = payload.lat
if payload.parcel_cad is not None:
sets.append("parcel_cad = :parcel_cad")
params["parcel_cad"] = payload.parcel_cad
if payload.notes is not None:
sets.append("notes = :notes")
params["notes"] = payload.notes
if len(sets) > 1:
db.execute(
text(
f"UPDATE user_custom_pois SET {', '.join(sets)}"
" WHERE id = :poi_id AND user_id = :user_id"
),
params,
)
db.commit()
return get_custom_poi(db, poi_id, user_id)
def delete_custom_poi(db: Any, poi_id: int, user_id: str) -> bool:
"""Удалить кастомную POI. Возвращает True если удалена, False если не найдена."""
result = db.execute(
text(_DELETE_SQL),
{"poi_id": poi_id, "user_id": user_id},
).first()
if result is None:
return False
db.commit()
logger.info("custom_poi deleted: id=%s user=%s", poi_id, user_id)
return True
def get_overlaps_for_scoring(
db: Any,
parcel_geom_wkt: str,
user_id: str,
parcel_cad: str | None = None,
) -> list[dict[str, Any]]:
"""Вернуть custom POI в радиусе 1 км с расстоянием для scoring.
Включает global POI (parcel_cad IS NULL) + parcel-specific если parcel_cad задан.
Distance decay применяется в parcels.py аналогично OSM POI.
Returns:
list[dict] с ключами: id, name, category, weight, lon, lat, distance_m
"""
_parcel_cad = parcel_cad or ""
try:
rows = (
db.execute(
text(_OVERLAPS_SQL),
{"wkt": parcel_geom_wkt, "user_id": user_id, "parcel_cad": _parcel_cad},
)
.mappings()
.all()
)
return [
{
"id": int(r["id"]),
"name": r["name"],
"category": r["category"],
"weight": float(r["weight"]),
"lon": float(r["lon"]),
"lat": float(r["lat"]),
"distance_m": float(r["distance_m"]),
}
for r in rows
]
except Exception as e:
logger.warning("get_overlaps_for_scoring failed user=%s: %s", user_id, e)
return []

76
backend/app/services/site_finder/gate_verdict.py
View file

@ -18,11 +18,25 @@ RESIDENTIAL_KEYWORDS = ("жил",)
# ── ЗОУИТ taxonomy ────────────────────────────────────────────────────────────
# Subcategory codes которые блокируют МКД (охранные зоны ЛЭП/газа/трубопровода)
# Subcategory codes которые блокируют МКД (охранные зоны ЛЭП/газа/трубопровода).
# Используется для overlaps из NSPD dump (subcategory INT там заполнен).
BLOCKER_SUBCATEGORIES: dict[int, str] = {
17: "Инженерные коммуникации (охранная зона ЛЭП/газа/трубопровода)",
}
# Keyword-based blockers для cad_zouit fallback (#232).
# cad_zouit.subcategory = 100% NULL, поэтому классификация по type_zone substring.
# Порядок: lowercase, substring match (any wins → blocker).
BLOCKER_TYPE_ZONE_KEYWORDS: tuple[str, ...] = (
"охранная зона",
"трубопровод",
"электр",
"газ",
)
# type_zone substrings, которые дают warning (не blocker) из cad_zouit.
WARNING_TYPE_ZONE_KEYWORDS: tuple[str, ...] = ("сзз", "санитарно-защитная")
# Engineering nearby search radius (метры) — совпадает с quarter_dump_lookup.py
ENGINEERING_NEARBY_THRESHOLD_M = 200
@ -136,21 +150,57 @@ def compute_gate_verdict(
# Check 2 — ЗОУИТ overlaps
checks.append("ЗОУИТ пересечения")
for overlap in nspd_zouit_overlaps or []:
sub = overlap.get("subcategory")
if isinstance(sub, int) and sub in BLOCKER_SUBCATEGORIES:
blockers.append(
Blocker(
code=f"ZOUIT_OVERLAP_SUB{sub}",
detail=f"{BLOCKER_SUBCATEGORIES[sub]}: {overlap.get('name', '')}",
src = overlap.get("source", "nspd-quarter-dump")
if src == "cad_zouit":
# cad_zouit fallback path: classify by type_zone keywords (#232).
# subcategory = NULL в cad_zouit, поэтому subcategory-based logic не применяется.
type_zone_lower = (overlap.get("type_zone") or overlap.get("layer") or "").lower()
if any(kw in type_zone_lower for kw in BLOCKER_TYPE_ZONE_KEYWORDS):
blockers.append(
Blocker(
code="ZOUIT_CAD_BLOCKER",
detail=(
f"Охранная зона ({overlap.get('type_zone', '')}): "
f"{overlap.get('name', '')}"
),
)
)
elif any(kw in type_zone_lower for kw in WARNING_TYPE_ZONE_KEYWORDS):
warnings.append(
Warning(
code="ZOUIT_CAD_SZZ",
detail=(
f"СЗЗ ({overlap.get('type_zone', '')}): " f"{overlap.get('name', '')}"
),
)
)
else:
warnings.append(
Warning(
code="ZOUIT_CAD_OTHER",
detail=(
f"ЗОУИТ cad ({overlap.get('type_zone', '')}): "
f"{overlap.get('name', '')}"
),
)
)
)
else:
warnings.append(
Warning(
code=f"ZOUIT_SUB{sub if sub is not None else 'unknown'}",
detail=f"ЗОУИТ {overlap.get('layer', '')}: {overlap.get('name', '')}",
# NSPD dump path: subcategory-based logic (backward-compat).
sub = overlap.get("subcategory")
if isinstance(sub, int) and sub in BLOCKER_SUBCATEGORIES:
blockers.append(
Blocker(
code=f"ZOUIT_OVERLAP_SUB{sub}",
detail=f"{BLOCKER_SUBCATEGORIES[sub]}: {overlap.get('name', '')}",
)
)
else:
warnings.append(
Warning(
code=f"ZOUIT_SUB{sub if sub is not None else 'unknown'}",
detail=f"ЗОУИТ {overlap.get('layer', '')}: {overlap.get('name', '')}",
)
)
)
# Check 3 — Engineering nearby (warning only, not a blocker)
checks.append(f"Инженерные сети в радиусе {ENGINEERING_NEARBY_THRESHOLD_M}м")

66
backend/app/services/site_finder/layout_signature.py Normal file
View file

@ -0,0 +1,66 @@
"""Layout signature extraction для Issue #113 (Phase 2.1 minimal).
Без `layout_type` / `balcony_count` — ждут B2B Объектив (#52).
"""
from __future__ import annotations
from typing import Literal
RoomBucket = Literal["studio", "euro-1", "euro-2", "1", "2", "3", "4+"]
AreaBin = Literal["<25", "25-40", "40-60", "60-80", "80-100", "100+"]
def room_bucket_from_flat(
rooms: int | None,
flat_type: str | None,
is_studio: bool | None,
total_area: float | None = None,
) -> RoomBucket:
"""Determine room_bucket из kn_flats полей.
Priority:
1. is_studio=True OR flat_type='Квартира-студия' → "studio"
2. rooms=0 (без is_studio) → "studio"
3. Fix SF-08: rooms=2 + area<35 → "euro-1" (DOM.РФ маркирует малогабаритные как 2-комн)
4. Fix SF-08: rooms=2 + area<50 → "euro-2" (евро-двушки 35-50м²)
5. rooms IN (1, 2, 3) → str(rooms)
6. rooms >= 4 → "4+"
7. fallback на "1" (rooms is None и не studio)
"""
if is_studio is True or flat_type == "Квартира-студия":
return "studio"
if rooms is None:
return "1"
if rooms == 0:
return "studio"
if rooms == 2 and total_area is not None:
if total_area < 35.0:
return "euro-1"
if total_area < 50.0:
return "euro-2"
if rooms >= 4:
return "4+"
if rooms in (1, 2, 3):
return str(rooms) # type: ignore[return-value]
return "1"
def area_bin(area_m2: float) -> AreaBin:
"""Bucket площади per ARN-buckets (НСПД)."""
if area_m2 < 25.0:
return "<25"
if area_m2 < 40.0:
return "25-40"
if area_m2 < 60.0:
return "40-60"
if area_m2 < 80.0:
return "60-80"
if area_m2 < 100.0:
return "80-100"
return "100+"
def layout_signature(room_bucket_val: RoomBucket, area_bin_val: AreaBin) -> str:
"""Deterministic string-signature для группировки."""
return f"{room_bucket_val}__{area_bin_val}"

57
backend/app/services/site_finder/layout_velocity_refresh.py Normal file
View file

@ -0,0 +1,57 @@
"""Refresh helper for mv_layout_velocity (Issue #113 PR B).
Not scheduled automatically in this PR — intended for manual invocation or
a Celery beat task in a follow-up issue.
Usage example (manual, via psql-connected shell or admin endpoint):
from sqlalchemy.orm import Session
from app.services.site_finder.layout_velocity_refresh import refresh_layout_velocity
count = refresh_layout_velocity(db)
# logs: "mv_layout_velocity refreshed: 459 rows"
"""
from __future__ import annotations
import logging
from sqlalchemy import text
from sqlalchemy.exc import OperationalError
from sqlalchemy.orm import Session
logger = logging.getLogger(__name__)
def refresh_layout_velocity(db: Session, *, concurrently: bool = True) -> int:
"""REFRESH MATERIALIZED VIEW mv_layout_velocity.
Args:
db: SQLAlchemy Session (sync).
concurrently: When True, uses REFRESH CONCURRENTLY — non-blocking but
requires the unique index mv_layout_velocity_pk to exist (created
by 94_mv_layout_velocity.sql). Pass False only for the very first
populate or when the MV was just recreated.
Returns:
Row count in the MV after refresh (for observability / alerting).
"""
try:
if concurrently:
db.execute(text("REFRESH MATERIALIZED VIEW CONCURRENTLY mv_layout_velocity"))
else:
db.execute(text("REFRESH MATERIALIZED VIEW mv_layout_velocity"))
db.commit()
except OperationalError as e:
if concurrently and "cannot refresh materialized view" in str(e).lower():
logger.warning(
"CONCURRENTLY failed (MV likely not populated), falling back to non-concurrent"
)
db.rollback()
db.execute(text("REFRESH MATERIALIZED VIEW mv_layout_velocity"))
db.commit()
else:
raise
row = db.execute(text("SELECT COUNT(*) FROM mv_layout_velocity")).first()
count = int(row[0]) if row else 0
logger.info("mv_layout_velocity refreshed: %d rows", count)
return count

159
backend/app/services/site_finder/poi_score.py Normal file
View file

@ -0,0 +1,159 @@
"""POI weighted score для Site Finder (B6).
Формула: weight = (1 / (distance_m + 100)) * category_weight
Возвращает top-7 ближайших POI из osm_poi_ekb, отсортированных по weight DESC.
Категории и их веса согласованы с _POI_WEIGHTS в parcels.py.
"""
from __future__ import annotations
import logging
from typing import Any
from pydantic import BaseModel
from sqlalchemy import text
logger = logging.getLogger(__name__)
# Веса по категории — согласованы с _POI_WEIGHTS в parcels.py + новые из vault B6.
# Задача: "2GIS-style ranking", метро самое приоритетное.
CATEGORY_WEIGHTS: dict[str, float] = {
"metro_stop": 6.0,
"school": 5.0,
"kindergarten": 4.5,
"hospital": 4.0,
"shop_supermarket": 3.5,
"shop_mall": 4.0,
"park": 3.5,
"bus_stop": 4.5,
"tram_stop": 2.0,
"pharmacy": 2.5,
"shop_small": 2.0,
"default": 1.0,
}
class PoiScoreItem(BaseModel):
"""Один POI в ranked-ответе."""
name: str | None
category: str
distance_m: float
weight: float
address: str | None
class PoiScoreResponse(BaseModel):
cad_num: str
radius_m: int
top_poi: list[PoiScoreItem]
def _category_weight(category: str) -> float:
"""Вернуть вес категории. Если не знаем — default."""
return CATEGORY_WEIGHTS.get(category, CATEGORY_WEIGHTS["default"])
def compute_poi_weighted_top7(
db: Any,
cad_num: str,
lat: float,
lon: float,
radius_m: int = 2000,
top_n: int = 7,
) -> PoiScoreResponse:
"""Найти top-N POI вокруг (lat, lon) в radius_m, ранжировать по weighted score.
Запрос к osm_poi_ekb через ST_DWithin + ST_Distance.
Формула: weight = (1 / (distance_m + 100)) * category_weight
Args:
db: SQLAlchemy Session
cad_num: кадастровый номер (для ответа)
lat: широта центроида участка
lon: долгота центроида участка
radius_m: радиус поиска в метрах (default 2000)
top_n: количество POI в ответе (default 7)
Returns:
PoiScoreResponse с отсортированными по weight DESC POI.
"""
# ST_DWithin с geography=true использует метры напрямую.
# ST_Distance тоже в метрах при geography=true.
rows = (
db.execute(
text("""
SELECT
p.name,
p.category,
p.tags,
CAST(
ST_Distance(
p.geom::geography,
ST_SetSRID(ST_MakePoint(:lon, :lat), 4326)::geography
) AS double precision
) AS distance_m
FROM osm_poi_ekb p
WHERE ST_DWithin(
p.geom::geography,
ST_SetSRID(ST_MakePoint(:lon, :lat), 4326)::geography,
:radius_m
)
ORDER BY distance_m ASC
LIMIT :limit
"""),
{
"lat": lat,
"lon": lon,
"radius_m": radius_m,
"limit": top_n * 10, # запрашиваем больше, потом ранжируем
},
)
.mappings()
.all()
)
logger.debug(
"poi_score: cad=%s lat=%.5f lon=%.5f radius=%dm → %d candidates",
cad_num,
lat,
lon,
radius_m,
len(rows),
)
items: list[PoiScoreItem] = []
for row in rows:
distance_m = float(row["distance_m"])
category = row["category"] or "default"
cat_weight = _category_weight(category)
weight = (1.0 / (distance_m + 100.0)) * cat_weight
# Адрес из tags jsonb если есть
tags: dict[str, str] = row["tags"] or {}
addr_parts = [
tags.get("addr:street"),
tags.get("addr:housenumber"),
]
address = ", ".join(p for p in addr_parts if p) or None
items.append(
PoiScoreItem(
name=row["name"],
category=category,
distance_m=round(distance_m, 1),
weight=round(weight, 6),
address=address,
)
)
# Сортировка по weight DESC, берём top_n
items.sort(key=lambda x: x.weight, reverse=True)
top_items = items[:top_n]
return PoiScoreResponse(
cad_num=cad_num,
radius_m=radius_m,
top_poi=top_items,
)

919
backend/app/services/site_finder/quarter_dump_lookup.py
View file

File diff suppressed because it is too large Load diff

63
backend/app/services/site_finder/quarter_price_refresh.py Normal file
View file

@ -0,0 +1,63 @@
"""Refresh helper for mv_quarter_price_per_m2 (Issue #33 D1).
Not scheduled automatically in this PR — intended for manual invocation or
a Celery beat task in a follow-up issue (Issue #33 PR C).
Usage example (manual, via psql-connected shell or admin endpoint):
from sqlalchemy.orm import Session
from app.services.site_finder.quarter_price_refresh import refresh_quarter_price
count = refresh_quarter_price(db)
# logs: "mv_quarter_price_per_m2 refreshed: 52492 rows"
"""
from __future__ import annotations
import logging
from sqlalchemy import text
from sqlalchemy.exc import OperationalError
from sqlalchemy.orm import Session
logger = logging.getLogger(__name__)
def refresh_quarter_price(db: Session, *, concurrently: bool = True) -> int:
"""REFRESH MATERIALIZED VIEW mv_quarter_price_per_m2.
Args:
db: SQLAlchemy Session (sync).
concurrently: When True, uses REFRESH CONCURRENTLY — non-blocking but
requires the unique index mv_quarter_price_pk to exist (created by
95_mv_quarter_price.sql) and the MV to be already populated.
Pass False only for the very first populate or after MV recreation.
Returns:
Row count in the MV after refresh (for observability / alerting).
Raises:
OperationalError: Re-raised if the error is not the known
"cannot refresh materialized view concurrently" case.
"""
try:
if concurrently:
db.execute(text("REFRESH MATERIALIZED VIEW CONCURRENTLY mv_quarter_price_per_m2"))
else:
db.execute(text("REFRESH MATERIALIZED VIEW mv_quarter_price_per_m2"))
db.commit()
except OperationalError as e:
if concurrently and "cannot refresh materialized view" in str(e).lower():
logger.warning(
"CONCURRENTLY failed (MV likely not populated), falling back to"
" non-concurrent refresh"
)
db.rollback()
db.execute(text("REFRESH MATERIALIZED VIEW mv_quarter_price_per_m2"))
db.commit()
else:
raise
row = db.execute(text("SELECT COUNT(*) FROM mv_quarter_price_per_m2")).first()
count = int(row[0]) if row else 0
logger.info("mv_quarter_price_per_m2 refreshed: %d rows", count)
return count

299
backend/app/services/site_finder/velocity.py
View file

@ -6,10 +6,14 @@ Per #34 D2: утилизация objective_corpus_room_month (еженедель
конкурирующих ЖК в радиусе radius_km от участка, нормированный к
ЕКБ-медиане по данным Objective.
Fallback (SF#17): если Objective coverage <50% конкурентов в радиусе,
использует rosreestr_deals JOIN по cad_quarter участка (100% coverage по кварталам).
Foundation: domrf_kn_objects (lat/lon, comm_name, obj_class, region_cd),
objective_complex_mapping (domrf_obj_id ↔ objective_complex_name),
objective_corpus_room_month (project_name, deals_total_vol_m2,
deals_total_count, report_month).
Fallback: rosreestr_deals (quarter_cad_number, deal_count, period_start_date).
Linkage: domrf_kn_objects.obj_id
→ objective_complex_mapping.domrf_obj_id
@ -32,6 +36,10 @@ logger = logging.getLogger(__name__)
# Эмпирика по ЕКБ: ~4 500 м²/мес на один ЖК (apartments, 2024-2025).
_EKB_MEDIAN_FALLBACK_SQM_PER_MONTH: float = 4500.0
# Порог: если доля конкурентов с Objective-маппингом < этого значения,
# пытаемся rosreestr_fallback.
_OBJECTIVE_COVERAGE_MIN_RATIO: float = 0.50
@dataclass(frozen=True)
class VelocityResult:
@ -47,6 +55,12 @@ class VelocityResult:
period_end: str # YYYY-MM
sample_competitors: list[dict[str, Any]] # top-5 для UI
by_room_bucket: dict[str, dict[str, Any]] # агрегат по комнатности
# True если данные есть (objective или rosreestr_fallback).
# False → нет данных ни из одного источника.
velocity_data_available: bool = True
# Источник данных: objective (основной), rosreestr_fallback (по кадастровому кварталу),
# none (нет данных).
velocity_source: Literal["objective", "rosreestr_fallback", "none"] = "objective"
def as_dict(self) -> dict[str, Any]:
return {
@ -59,6 +73,8 @@ class VelocityResult:
"period": {"start": self.period_start, "end": self.period_end},
"sample_competitors": self.sample_competitors,
"by_room_bucket": self.by_room_bucket,
"velocity_data_available": self.velocity_data_available,
"velocity_source": self.velocity_source,
}
@ -68,6 +84,7 @@ def compute_velocity(
radius_km: float = 3.0,
obj_class: str | None = None,
months_window: int = 6,
cad_quarter: str | None = None,
) -> VelocityResult | None:
"""Вычислить velocity-score для участка.
@ -75,9 +92,14 @@ def compute_velocity(
1. Найти ЖК-конкуренты в радиусе radius_km (через lat/lon ST_DWithin).
2. Взять objective_corpus_room_month за последние months_window месяцев
через objective_complex_mapping (domrf_obj_id → project_name).
3. Посчитать суммарный объём deals_total_vol_m2.
3. Если Objective coverage < 50% конкурентов → rosreestr_fallback:
считаем сделки DDU/ДКП в cad_quarter участка за окно.
4. Нормировать на ЕКБ-медиану → score 0..1.
Параметры:
cad_quarter: кадастровый квартал участка (первые 3 сегмента cad_num,
например "66:41:0702048"). Используется только для fallback.
Возвращает None если parcel_geom_wkt невалиден или конкурентов нет.
"""
# ── Step 1: конкуренты по lat/lon в радиусе ──────────────────────────────
@ -165,6 +187,8 @@ def compute_velocity(
# objective_corpus_room_month.
# deals_total_vol_m2 = DDU + DKP м² за месяц (primary signal).
# deals_total_count > 0 — фильтрует месяцы без сделок.
# LEFT JOIN с objective_complex_mapping: конкуренты без маппинга не
# выпадают — они включаются в список но с total_sqm=NULL (→ 0.0).
# GROUP BY domrf_obj_id чтобы сохранить совместимость с caller (obj_id key).
try:
with db.begin_nested():
@ -172,25 +196,32 @@ def compute_velocity(
db.execute(
text(
"""
WITH mapped AS (
WITH all_competitors AS (
SELECT unnest(CAST(:obj_ids AS int[])) AS obj_id
),
mapped AS (
SELECT cm.domrf_obj_id AS obj_id,
cm.objective_complex_name
FROM objective_complex_mapping cm
WHERE cm.domrf_obj_id = ANY(:obj_ids)
)
SELECT
m.obj_id,
ac.obj_id,
SUM(COALESCE(crm.deals_total_vol_m2,
crm.deals_total_count * 45.0)) AS total_sqm,
COUNT(DISTINCT crm.report_month) AS months_with_data,
MIN(crm.report_month) AS period_start,
MAX(crm.report_month) AS period_end
FROM objective_corpus_room_month crm
JOIN mapped m
ON m.objective_complex_name = crm.project_name
WHERE crm.report_month >= (CURRENT_DATE - CAST(:window_interval AS interval))
AND crm.deals_total_count > 0
GROUP BY m.obj_id
COUNT(DISTINCT crm.report_month) AS months_with_data,
MIN(crm.report_month) AS period_start,
MAX(crm.report_month) AS period_end,
CASE WHEN m.obj_id IS NOT NULL THEN TRUE
ELSE FALSE END AS has_mapping
FROM all_competitors ac
LEFT JOIN mapped m ON m.obj_id = ac.obj_id
LEFT JOIN objective_corpus_room_month crm
ON crm.project_name = m.objective_complex_name
AND crm.report_month >= (
CURRENT_DATE - CAST(:window_interval AS interval))
AND crm.deals_total_count > 0
GROUP BY ac.obj_id, m.obj_id
"""
),
{
@ -209,6 +240,69 @@ def compute_velocity(
if not sales_rows:
return None
# ── Step 2a: проверка Objective coverage ─────────────────────────────────
# Считаем: mapped_with_data — конкуренты с маппингом И реальными данными.
# Если mapped_ratio < _OBJECTIVE_COVERAGE_MIN_RATIO → rosreestr_fallback.
n_total_comps = len(obj_ids)
mapped_with_data = [
r for r in sales_rows if bool(r["has_mapping"]) and (r["total_sqm"] or 0.0) > 0
]
mapped_ratio = len(mapped_with_data) / n_total_comps if n_total_comps > 0 else 0.0
ekb_median = (
_get_ekb_median(db, months_window=months_window) or _EKB_MEDIAN_FALLBACK_SQM_PER_MONTH
)
n_comps = len(comp_rows)
sample_no_data = sorted(
[
{
"obj_id": oid,
**competitor_meta[oid],
"total_sqm_period": 0.0,
"by_room_bucket": {},
}
for oid in obj_ids[:5]
if oid in competitor_meta
],
key=lambda x: x["distance_m"], # type: ignore[index]
)
if mapped_ratio < _OBJECTIVE_COVERAGE_MIN_RATIO:
logger.info(
"velocity: objective coverage %.0f%% (<%d%%) for %d competitors;"
" trying rosreestr_fallback cad_quarter=%s",
mapped_ratio * 100,
int(_OBJECTIVE_COVERAGE_MIN_RATIO * 100),
n_total_comps,
cad_quarter,
)
rr_result = _compute_rosreestr_fallback(
db=db,
cad_quarter=cad_quarter,
months_window=months_window,
n_comps=n_comps,
ekb_median=ekb_median,
sample_competitors=sample_no_data,
)
if rr_result is not None:
return rr_result
# Rosreestr тоже пуст — возвращаем none-state.
logger.info("velocity: rosreestr_fallback also empty for cad_quarter=%s", cad_quarter)
return VelocityResult(
competitors_count=n_comps,
monthly_velocity_sqm=0.0,
ekb_median_sqm=ekb_median,
velocity_score=0.0,
confidence="low",
months_observed=0,
period_start="",
period_end="",
sample_competitors=sample_no_data,
by_room_bucket={},
velocity_data_available=False,
velocity_source="none",
)
# ── Step 2b: разбивка по комнатности (room_bucket) ───────────────────────
# Тот же маппинг domrf_obj_id → project_name. Агрегируем по room_bucket
# для отображения структуры спроса в UI.
@ -278,46 +372,89 @@ def compute_velocity(
for bucket, data in by_bucket_agg.items()
}
total_sqm = sum(float(r["total_sqm"] or 0.0) for r in sales_rows)
months_observed = max((int(r["months_with_data"] or 0) for r in sales_rows), default=0)
period_start_dates = [r["period_start"] for r in sales_rows if r["period_start"]]
period_end_dates = [r["period_end"] for r in sales_rows if r["period_end"]]
# Считаем только по строкам с маппингом — unmapped строки дают total_sqm=NULL.
mapped_sales_rows = [r for r in sales_rows if bool(r["has_mapping"])]
total_sqm = sum(float(r["total_sqm"] or 0.0) for r in mapped_sales_rows)
months_observed = max((int(r["months_with_data"] or 0) for r in mapped_sales_rows), default=0)
period_start_dates = [r["period_start"] for r in mapped_sales_rows if r["period_start"]]
period_end_dates = [r["period_end"] for r in mapped_sales_rows if r["period_end"]]
period_start = min(period_start_dates).strftime("%Y-%m") if period_start_dates else ""
period_end = max(period_end_dates).strftime("%Y-%m") if period_end_dates else ""
# Если mapped-конкурентов нет данных — partial coverage → fallback.
if months_observed == 0 or total_sqm <= 0:
return None
logger.info(
"velocity: %d competitors found, %d mapped, but no sales data in window;"
" trying rosreestr_fallback",
len(obj_ids),
len(mapped_sales_rows),
)
rr_result = _compute_rosreestr_fallback(
db=db,
cad_quarter=cad_quarter,
months_window=months_window,
n_comps=n_comps,
ekb_median=ekb_median,
sample_competitors=sample_no_data,
)
if rr_result is not None:
return rr_result
sample_partial = sorted(
[
{
"obj_id": oid,
**competitor_meta[oid],
"total_sqm_period": 0.0,
"by_room_bucket": {},
}
for oid in obj_ids
if oid in competitor_meta
],
key=lambda x: x["total_sqm_period"], # type: ignore[arg-type]
reverse=True,
)[:5]
return VelocityResult(
competitors_count=n_comps,
monthly_velocity_sqm=0.0,
ekb_median_sqm=ekb_median,
velocity_score=0.0,
confidence="low",
months_observed=0,
period_start="",
period_end="",
sample_competitors=sample_partial,
by_room_bucket={},
velocity_data_available=False,
velocity_source="none",
)
# Среднемесячный объём в расчёте: суммарный по всем конкурентам / месяцев.
# Чем больше конкурентов с данными — тем весомее результат.
monthly_velocity = total_sqm / months_observed
# ── Step 3: ЕКБ-медиана ──────────────────────────────────────────────────
ekb_median = (
_get_ekb_median(db, months_window=months_window) or _EKB_MEDIAN_FALLBACK_SQM_PER_MONTH
)
# ── Step 4: нормализация → score 0..1 ────────────────────────────────────
# ── Step 3: нормализация → score 0..1 ────────────────────────────────────
# Логика: сравниваем суммарный velocity радиуса с «нормой» одного ЖК.
# Если в радиусе продаётся N × ekb_median → рынок горячий.
# Нормируем: score = min(1.0, total_velocity / (n_competitors × ekb_median × 2))
# Cap 2×median = «насыщен». Итоговый score 0..1.
n_with_sales = len(sales_rows)
# n_with_sales — только mapped конкуренты (у unmapped данных нет).
n_with_sales = len(mapped_sales_rows)
denominator = n_with_sales * ekb_median * 2.0 if n_with_sales > 0 else ekb_median * 2.0
velocity_score = min(1.0, max(0.0, monthly_velocity / denominator))
# ── Step 5: confidence ───────────────────────────────────────────────────
n_comps = len(comp_rows)
# ── Step 4: confidence ───────────────────────────────────────────────────
mapped_conf: Literal["high", "medium", "low"]
if n_comps >= 10 and months_observed >= 5:
confidence: Literal["high", "medium", "low"] = "high"
mapped_conf = "high"
elif n_comps >= 5 and months_observed >= 3:
confidence = "medium"
mapped_conf = "medium"
else:
confidence = "low"
mapped_conf = "low"
# ── Step 6: top-5 конкурентов по объёму продаж ───────────────────────────
# ── Step 5: top-5 конкурентов по объёму продаж ───────────────────────────
sales_by_id: dict[int, float] = {
int(r["obj_id"]): float(r["total_sqm"] or 0.0) for r in sales_rows
int(r["obj_id"]): float(r["total_sqm"] or 0.0) for r in mapped_sales_rows
}
sample = sorted(
[
@ -339,12 +476,110 @@ def compute_velocity(
monthly_velocity_sqm=monthly_velocity,
ekb_median_sqm=ekb_median,
velocity_score=velocity_score,
confidence=confidence,
confidence=mapped_conf,
months_observed=months_observed,
period_start=period_start,
period_end=period_end,
sample_competitors=sample,
by_room_bucket=by_room_bucket,
velocity_data_available=True,
velocity_source="objective",
)
def _compute_rosreestr_fallback(
db: Session,
cad_quarter: str | None,
months_window: int,
n_comps: int,
ekb_median: float,
sample_competitors: list[dict[str, Any]],
) -> VelocityResult | None:
"""Fallback velocity через rosreestr_deals JOIN по cad_quarter участка.
Считает суммарное число сделок DDU/ДКП в кадастровом квартале за окно months_window.
Velocity = deal_count / months_window (сделок/мес). Нет разбивки по room_bucket
(rosreestr не даёт комнатность).
Возвращает None если cad_quarter не задан или данных нет.
"""
if not cad_quarter:
return None
try:
with db.begin_nested():
row = (
db.execute(
text(
"""
SELECT
SUM(deal_count) AS total_deals,
MIN(period_start_date) AS period_start,
MAX(period_start_date) AS period_end
FROM rosreestr_deals
WHERE quarter_cad_number = :cad_quarter
AND period_start_date >= (CURRENT_DATE - CAST(:window_interval AS interval))
AND doc_type IN ('ДДУ', 'ДКП')
"""
),
{
"cad_quarter": cad_quarter,
"window_interval": f"{months_window} months",
},
)
.mappings()
.first()
)
except Exception:
logger.warning("velocity: rosreestr_fallback query failed for cad_quarter=%s", cad_quarter)
return None
if row is None or not row["total_deals"] or int(row["total_deals"]) == 0:
return None
total_deals = int(row["total_deals"])
# Сделок/мес — грубый аналог velocity (без м², только count).
# Умножаем на 45 м² (эмпирика) для совместимости с м²/мес единицами.
avg_area_per_deal = 45.0 # м² — консервативная оценка для апартаментов ЕКБ
monthly_velocity_sqm = (total_deals * avg_area_per_deal) / months_window
# Нормализация относительно ekb_median (один ЖК × 2).
velocity_score = min(1.0, max(0.0, monthly_velocity_sqm / (ekb_median * 2.0)))
# Confidence — rosreestr данные менее детализированы, чем Objective.
rr_confidence: Literal["high", "medium", "low"]
if total_deals >= 50:
rr_confidence = "medium" # max medium для rosreestr — нет комнатности
else:
rr_confidence = "low"
period_start_date = row["period_start"]
period_end_date = row["period_end"]
period_start = period_start_date.strftime("%Y-%m") if period_start_date else ""
period_end = period_end_date.strftime("%Y-%m") if period_end_date else ""
logger.info(
"velocity: rosreestr_fallback success cad_quarter=%s"
" total_deals=%d window=%dm velocity=%.1f sqm/mon",
cad_quarter,
total_deals,
months_window,
monthly_velocity_sqm,
)
return VelocityResult(
competitors_count=n_comps,
monthly_velocity_sqm=monthly_velocity_sqm,
ekb_median_sqm=ekb_median,
velocity_score=velocity_score,
confidence=rr_confidence,
months_observed=months_window,
period_start=period_start,
period_end=period_end,
sample_competitors=sample_competitors,
by_room_bucket={}, # rosreestr не даёт room_bucket
velocity_data_available=True,
velocity_source="rosreestr_fallback",
)

40
backend/app/services/site_finder/weight_profiles.py
View file

@ -17,6 +17,7 @@ from __future__ import annotations
import json
import logging
import math
from datetime import datetime
from typing import Any
@ -25,7 +26,11 @@ from sqlalchemy import text
logger = logging.getLogger(__name__)
# Allowed POI categories — mirrors _POI_WEIGHTS keys in api/v1/parcels.py
# Sentinel user_id для системных preset-профилей (не привязаны к реальному пользователю).
# Seed: data/sql/100_user_weight_profiles_default_seed.sql
SYSTEM_USER_ID: str = "__system__"
# Allowed POI categories — single source of truth; imported by api/v1/parcels.py
ALLOWED_CATEGORIES: set[str] = {
"school",
"kindergarten",
@ -44,7 +49,7 @@ ALLOWED_CATEGORIES: set[str] = {
MIN_WEIGHT: float = -2.0
MAX_WEIGHT: float = 3.0
# System defaults — keep in sync with _POI_WEIGHTS in parcels.py
# System defaults — single source of truth; imported as _POI_WEIGHTS by api/v1/parcels.py
_SYSTEM_POI_WEIGHTS: dict[str, float] = {
"school": 1.5,
"kindergarten": 1.5,
@ -73,6 +78,17 @@ _SELECT_BY_USER = f"""
ORDER BY is_default DESC, id ASC
"""
_SELECT_BY_USER_WITH_SYSTEM = f"""
SELECT {_SELECT_COLS}
FROM user_weight_profiles
WHERE user_id = :user_id
OR user_id = :system_user_id
ORDER BY
CASE WHEN user_id = :system_user_id THEN 1 ELSE 0 END ASC,
is_default DESC,
id ASC
"""
_SELECT_BY_ID = f"""
SELECT {_SELECT_COLS}
FROM user_weight_profiles
@ -115,7 +131,7 @@ def _validate_weights_dict(v: dict[str, float]) -> dict[str, float]:
for k, w in v.items():
if not isinstance(w, int | float):
raise ValueError(f"Weight for '{k}' must be number, got {type(w).__name__}")
if w < MIN_WEIGHT or w > MAX_WEIGHT:
if not math.isfinite(w) or w < MIN_WEIGHT or w > MAX_WEIGHT:
raise ValueError(f"Weight for '{k}' = {w} out of bounds [{MIN_WEIGHT}, {MAX_WEIGHT}]")
return v
@ -190,6 +206,24 @@ def list_profiles(db: Any, user_id: str) -> list[WeightProfile]:
return [_row_to_profile(r) for r in rows]
def list_profiles_with_system(db: Any, user_id: str) -> list[WeightProfile]:
"""Вернуть профили пользователя + системные preset-профили.
Пользовательские профили идут первыми (default сверху), затем системные
presets (Эконом, Комфорт, Бизнес). Предназначен для endpoint с
include_system=true — UI dropdown видит и пользовательские, и preset.
"""
rows = (
db.execute(
text(_SELECT_BY_USER_WITH_SYSTEM),
{"user_id": user_id, "system_user_id": SYSTEM_USER_ID},
)
.mappings()
.all()
)
return [_row_to_profile(r) for r in rows]
def get_profile(db: Any, user_id: str, profile_id: int) -> WeightProfile | None:
"""Вернуть профиль по id (scoped к пользователю)."""
row = (

240
backend/app/templates/parcel_snapshot.html Normal file
View file

@ -0,0 +1,240 @@
<!DOCTYPE html>
<html lang="ru">
<head>
<meta charset="UTF-8" />
<title>Карточка участка {{ cad_num }}</title>
<style>
@font-face {
font-family: 'DejaVu Sans';
src: url('{{ font_url }}') format('truetype');
}
* { box-sizing: border-box; margin: 0; padding: 0; }
body {
font-family: 'DejaVu Sans', Arial, sans-serif;
font-size: 10pt;
color: #1a1a2e;
background: #ffffff;
padding: 20mm 18mm 18mm 18mm;
}
/* ── HEADER ── */
.header {
display: flex;
justify-content: space-between;
align-items: flex-start;
border-bottom: 2px solid #2563eb;
padding-bottom: 8px;
margin-bottom: 14px;
}
.header-left h1 {
font-size: 14pt;
font-weight: bold;
color: #2563eb;
}
.header-left .subtitle {
font-size: 9pt;
color: #64748b;
margin-top: 2px;
}
.header-right {
text-align: right;
font-size: 8pt;
color: #64748b;
}
/* ── SECTION TITLE ── */
.section-title {
font-size: 10pt;
font-weight: bold;
color: #1e3a5f;
background: #eff6ff;
padding: 4px 8px;
border-left: 3px solid #2563eb;
margin-bottom: 8px;
}
/* ── KPI GRID ── */
.kpi-grid {
display: flex;
flex-wrap: wrap;
gap: 6px;
margin-bottom: 14px;
}
.kpi-card {
flex: 1 1 140px;
border: 1px solid #cbd5e1;
border-radius: 4px;
padding: 7px 10px;
background: #f8fafc;
}
.kpi-card .kpi-label {
font-size: 7.5pt;
color: #64748b;
margin-bottom: 2px;
}
.kpi-card .kpi-value {
font-size: 11pt;
font-weight: bold;
color: #1e3a5f;
}
/* ── TABLE ── */
table {
width: 100%;
border-collapse: collapse;
font-size: 8.5pt;
margin-bottom: 14px;
}
table thead tr th {
background: #1e3a5f;
color: #ffffff;
padding: 5px 8px;
text-align: left;
font-weight: bold;
}
table tbody tr:nth-child(even) td {
background: #f1f5f9;
}
table tbody tr td {
padding: 4px 8px;
border-bottom: 1px solid #e2e8f0;
color: #1a1a2e;
}
.badge {
display: inline-block;
padding: 1px 6px;
border-radius: 10px;
font-size: 7.5pt;
font-weight: bold;
}
.badge-green { background: #dcfce7; color: #166534; }
.badge-yellow { background: #fef9c3; color: #854d0e; }
.badge-blue { background: #dbeafe; color: #1e40af; }
/* ── FOOTER ── */
.footer {
position: fixed;
bottom: 12mm;
left: 18mm;
right: 18mm;
border-top: 1px solid #cbd5e1;
padding-top: 5px;
display: flex;
justify-content: space-between;
font-size: 7.5pt;
color: #94a3b8;
}
.disclaimer {
font-size: 7pt;
color: #94a3b8;
margin-top: 4px;
font-style: italic;
}
</style>
</head>
<body>
<!-- HEADER -->
<div class="header">
<div class="header-left">
<h1>GenDesign &mdash; Карточка участка</h1>
<div class="subtitle">Данные НСПД / ЕГРНsource: cad_parcels. Не является официальной выпиской ЕГРН.</div>
</div>
<div class="header-right">
<strong>{{ cad_num }}</strong><br/>
{{ district or '&mdash;' }}<br/>
{{ address or '&mdash;' }}
</div>
</div>
<!-- BLOCK 1: KPI -->
<div class="section-title">Основные характеристики</div>
<div class="kpi-grid">
<div class="kpi-card">
<div class="kpi-label">Площадь</div>
<div class="kpi-value">{{ area_ha }} га</div>
</div>
<div class="kpi-card">
<div class="kpi-label">Кадастровая стоимость</div>
<div class="kpi-value">{{ cadastral_cost }}</div>
</div>
<div class="kpi-card">
<div class="kpi-label">Категория земель</div>
<div class="kpi-value">{{ land_category or '&mdash;' }}</div>
</div>
<div class="kpi-card">
<div class="kpi-label">ВРИ</div>
<div class="kpi-value">{{ vri or '&mdash;' }}</div>
</div>
<div class="kpi-card">
<div class="kpi-label">Последнее обновление</div>
<div class="kpi-value">{{ last_update or '&mdash;' }}</div>
</div>
</div>
<!-- BLOCK 2: Top-7 POI -->
<div class="section-title">Ближайшая инфраструктура (топ-7 по взвешенному баллу)</div>
{% if poi_items %}
<table>
<thead>
<tr>
<th>Категория</th>
<th>Название</th>
<th>Расстояние</th>
<th>Пешком</th>
<th>Балл</th>
</tr>
</thead>
<tbody>
{% for poi in poi_items %}
<tr>
<td>{{ poi.category_ru }}</td>
<td>{{ poi.name or '&mdash;' }}</td>
<td>{{ poi.distance_m }} м</td>
<td>{{ poi.walk_min }} мин</td>
<td><span class="badge badge-blue">{{ poi.weighted_score }}</span></td>
</tr>
{% endfor %}
</tbody>
</table>
{% else %}
<p style="color:#64748b; font-size:8.5pt; margin-bottom:14px;">POI в радиусе 1 км не найдены.</p>
{% endif %}
<!-- BLOCK 3: Competitors -->
<div class="section-title">Конкуренты в радиусе 3 км (топ {{ competitors|length }})</div>
{% if competitors %}
<table>
<thead>
<tr>
<th>ЖК / Объект</th>
<th>Застройщик</th>
<th>Класс</th>
<th>Квартир</th>
<th>Расстояние</th>
</tr>
</thead>
<tbody>
{% for c in competitors %}
<tr>
<td>{{ c.comm_name or '&mdash;' }}</td>
<td>{{ c.dev_name or '&mdash;' }}</td>
<td>{{ c.obj_class or '&mdash;' }}</td>
<td>{{ c.flat_count or '&mdash;' }}</td>
<td>{{ c.distance_m }} м</td>
</tr>
{% endfor %}
</tbody>
</table>
{% else %}
<p style="color:#64748b; font-size:8.5pt; margin-bottom:14px;">Конкурентов в радиусе 3 км не обнаружено.</p>
{% endif %}
<div class="disclaimer">
Не является выпиской из ЕГРН. Данные носят аналитический характер.
Для официальной выписки: rosreestr.gov.ru
</div>
<!-- FOOTER -->
<div class="footer">
<span>gendsgn.ru &mdash; GenDesign Analytics</span>
<span>Сформировано: {{ generated_at }}</span>
</div>
</body>
</html>

60
backend/app/workers/beat_schedule.py
View file

@ -219,23 +219,49 @@ def build_beat_schedule() -> dict:
"options": {"queue": "celery"},
}
# NSPD quarter dump refresh — DISABLED 2026-05-14 per Bug_NSPD_WMS_NotBulk
# post-mortem (vault: fixes/Bug_NSPD_WMS_NotBulk_2026_May14.md).
# #105 Phase 4: ЕКБ РНС/РВЭ — ежемесячно 1-го числа в 05:00 МСК (02:00 UTC)
schedule["ekburg-permits-monthly"] = {
"task": "tasks.ekburg_permits_sync.refresh_all",
"schedule": _parse_cron("0 2 1 * *"),
"options": {"queue": "celery"},
}
# ПЗЗ территориальных зон ЕКБ из PKK6 ArcGIS — ежемесячно 1-го числа в 03:00 МСК (00:00 UTC).
# PKK6 нестабилен под нагрузкой, но данные ПЗЗ меняются редко — раз в месяц достаточно.
# Task: tasks/pzz_sync.py → sync_pzz_zones_ekb.
# Admin trigger: POST /api/v1/admin/scrape/pzz-sync.
# Ref: issue #233 (pzz_zones_ekb = 0 rows, задача никогда не запускалась автоматически).
schedule["pzz-sync-monthly"] = {
"task": "tasks.pzz_sync.sync_pzz_zones_ekb",
"schedule": _parse_cron("0 0 1 * *"),
"options": {"queue": "celery"},
}
# Catalog-object scrape — наполняет ~25 NULL колонок domrf_kn_objects из SSR-страниц.
# kn-API не отдаёт wall_type, energy_eff, ceiling_height_m, parking_* и т.д.
# Вторник 04:00 UTC. batch 300/run → 1532 объекта за ~5 недель полного обновления.
schedule["scrape-kn-catalog-objects-weekly"] = {
"task": "tasks.scrape_kn_catalog_objects.scrape_kn_catalog_objects",
"schedule": _parse_cron("0 4 * * 2"), # Tuesday 04:00 UTC
"kwargs": {"region_code": 66, "max_objects": 300},
"options": {"queue": "celery"},
}
# NSPD quarter dump refresh — re-enabled 2026-05-17 после Sub-PR B (#260)
# переключения search_by_quarter на grid-walk. Foundation (#247) + integration
# (#260) теперь возвращают полноценные dumps (territorial_zones, ЗОУИТ, risk
# zones, engineering structures) вместо 0-3 features из single-pixel WMS.
#
# Action item #1: disable beat schedule до Sprint 2 fix (grid sampling
# rewrite). harvest_quarter в текущей реализации пишет почти пустые dumps
# из-за single-pixel WMS GetFeatureInfo bug. Запуск Mon 04:00 МСК потратит
# rate-limit budget и заполнит nspd_quarter_dumps мусором.
#
# Task code остаётся в tasks/nspd_sync.py — re-enable после Sprint 2 grid
# sampling rewrite (см. Bug_NSPD_WMS_NotBulk_2026_May14 → Sprint 2 fix-strategy).
# До тех пор harvest_quarter можно вызывать вручную через admin endpoint.
#
# schedule["nspd-harvest-stale-quarters"] = {
# "task": "tasks.nspd_sync.harvest_stale_quarters",
# "schedule": _parse_cron("0 4 * * mon"),
# "kwargs": {"region_code": 66, "max_age_days": 90, "batch_size": 50},
# "options": {"queue": "celery"},
# }
# Schedule: Mon 04:00 МСК (01:00 UTC). batch_size=50 ограничивает fanout per
# tick — при 11k+ кварталов в cad_quarters_geom полный backfill займёт
# несколько недель, но защищает от WAF rate-limit burst.
# max_age_days=90 — refresh свежее квартала; новые / отсутствующие dumps
# тоже попадают через harvest_stale_quarters (берёт NULL fetched_at).
schedule["nspd-harvest-stale-quarters"] = {
"task": "tasks.nspd_sync.harvest_stale_quarters",
"schedule": _parse_cron("0 1 * * mon"), # 01:00 UTC = 04:00 МСК
"kwargs": {"region_code": 66, "max_age_days": 90, "batch_size": 50},
"options": {"queue": "celery"},
}
return schedule

33
backend/app/workers/celery_app.py
View file

@ -5,19 +5,51 @@ Worker lifecycle hooks (process_init, worker_ready) → app/workers/lifecycle.py
"""
import logging
import os
import sentry_sdk
from celery import Celery
from sentry_sdk.integrations.celery import CeleryIntegration
from sentry_sdk.integrations.httpx import HttpxIntegration
from sentry_sdk.integrations.logging import LoggingIntegration
from sentry_sdk.integrations.sqlalchemy import SqlalchemyIntegration
from app.core.config import settings
from app.observability.sentry_scrub import scrub_sensitive_query
logger = logging.getLogger(__name__)
# SDK инициализируется в обоих процессах (FastAPI-сервер и Celery-воркер),
# чтобы события из тасков попадали в GlitchTip. SDK безопасен для двойного
# вызова — повторный sentry_sdk.init() в одном процессе заменяет клиента.
if settings.glitchtip_dsn:
sentry_sdk.init(
dsn=settings.glitchtip_dsn,
environment=settings.environment,
release=os.getenv("GIT_SHA") or os.getenv("SENTRY_RELEASE") or "unknown",
traces_sample_rate=settings.glitchtip_traces_sample_rate,
profiles_sample_rate=0.0,
send_default_pii=False,
before_send_transaction=scrub_sensitive_query,
integrations=[
CeleryIntegration(monitor_beat_tasks=True),
SqlalchemyIntegration(),
HttpxIntegration(),
LoggingIntegration(level=logging.INFO, event_level=logging.ERROR),
],
)
logger.info(
"GlitchTip SDK initialised in Celery worker (env=%s)",
settings.environment,
)
celery_app = Celery(
"gendesign",
broker=settings.redis_url,
backend=settings.redis_url,
include=[
"app.workers.tasks.scrape_kn",
"app.workers.tasks.scrape_kn_catalog_objects",
"app.workers.tasks.refresh_analytics",
"app.workers.tasks.scrape_objective",
"app.workers.tasks.objective_etl",
@ -27,6 +59,7 @@ celery_app = Celery(
"app.workers.tasks.noise_sync",
"app.workers.tasks.pzz_sync",
"app.workers.tasks.scrape_cadastre",
"app.workers.tasks.ekburg_permits_sync",
],
)
celery_app.conf.timezone = "Europe/Moscow"

151
backend/app/workers/tasks/ekburg_permits_sync.py Normal file
View file

@ -0,0 +1,151 @@
"""Celery task: monthly refresh ekburg permits xlsx (Issue #105).
Запускается через beat (1-е число каждого месяца в 05:00 МСК).
Добавить в beat_schedule через job_settings или hardcoded entry (Phase 2 followup).
"""
from __future__ import annotations
import json
import logging
from sqlalchemy import text
from sqlalchemy.orm import Session
from app.core.db import SessionLocal
from app.services.scrapers.ekburg_permits import (
EKBURG_PERMITS_URLS,
EkburgPermitsClient,
PermitRow,
)
from app.workers.celery_app import celery_app
logger = logging.getLogger(__name__)
def _upsert_permit(db: Session, row: PermitRow) -> None:
"""UPSERT одной строки разрешения в ekburg_construction_permits."""
db.execute(
text("""
INSERT INTO ekburg_construction_permits (
permit_type, permit_number,
issue_date, expiry_date,
developer_inn, developer_name,
object_name, object_type,
construction_address, cadastral_number,
total_area_sqm, living_area_sqm, living_area_fact_sqm,
rve_number, rve_date,
raw_coord_x, raw_coord_y,
source_year, source_url, raw_row
)
VALUES (
:permit_type, :permit_number,
:issue_date, :expiry_date,
:developer_inn, :developer_name,
:object_name, :object_type,
:construction_address, :cadastral_number,
:total_area_sqm, :living_area_sqm, :living_area_fact_sqm,
:rve_number, :rve_date,
:raw_coord_x, :raw_coord_y,
:source_year, :source_url, CAST(:raw_row AS jsonb)
)
ON CONFLICT (permit_type, permit_number) DO UPDATE SET
issue_date = EXCLUDED.issue_date,
expiry_date = EXCLUDED.expiry_date,
developer_inn = EXCLUDED.developer_inn,
developer_name = EXCLUDED.developer_name,
object_name = EXCLUDED.object_name,
object_type = EXCLUDED.object_type,
construction_address = EXCLUDED.construction_address,
cadastral_number = EXCLUDED.cadastral_number,
total_area_sqm = EXCLUDED.total_area_sqm,
living_area_sqm = EXCLUDED.living_area_sqm,
living_area_fact_sqm = EXCLUDED.living_area_fact_sqm,
rve_number = EXCLUDED.rve_number,
rve_date = EXCLUDED.rve_date,
raw_coord_x = EXCLUDED.raw_coord_x,
raw_coord_y = EXCLUDED.raw_coord_y,
raw_row = EXCLUDED.raw_row,
fetched_at = NOW()
"""),
{
"permit_type": row.permit_type,
"permit_number": row.permit_number,
"issue_date": row.issue_date,
"expiry_date": row.expiry_date,
"developer_inn": row.developer_inn,
"developer_name": row.developer_name,
"object_name": row.object_name,
"object_type": row.object_type,
"construction_address": row.construction_address,
"cadastral_number": row.cadastral_number,
"total_area_sqm": row.total_area_sqm,
"living_area_sqm": row.living_area_sqm,
"living_area_fact_sqm": row.living_area_fact_sqm,
"rve_number": row.rve_number,
"rve_date": row.rve_date,
"raw_coord_x": row.raw_coord_x,
"raw_coord_y": row.raw_coord_y,
"source_year": row.source_year,
"source_url": row.source_url,
"raw_row": json.dumps(row.raw_row, ensure_ascii=False),
},
)
@celery_app.task(name="tasks.ekburg_permits_sync.refresh_year", queue="celery")
def refresh_year(year: int) -> dict[str, int]:
"""Скачать + распарсить + upsert РНС/РВЭ за один год.
Возвращает {"inserted": N, "errors": N}.
"""
inserted = 0
errors = 0
url = EKBURG_PERMITS_URLS.get(year)
if not url:
logger.error("ekburg_permits_sync: no URL for year=%d", year)
return {"inserted": 0, "errors": 1}
with EkburgPermitsClient() as client:
try:
content = client.download_xlsx(year)
except Exception as exc:
logger.error("ekburg_permits_sync: download failed year=%d: %s", year, exc)
return {"inserted": 0, "errors": 1}
with SessionLocal() as db:
for row in client.parse_xlsx(content, year, url):
try:
with db.begin_nested():
_upsert_permit(db, row)
inserted += 1
except Exception as exc:
logger.warning(
"ekburg_permits_sync: upsert failed %s/%s year=%d: %s",
row.permit_type,
row.permit_number,
year,
exc,
)
errors += 1
db.commit()
logger.info(
"ekburg_permits_sync: year=%d done inserted=%d errors=%d",
year,
inserted,
errors,
)
return {"inserted": inserted, "errors": errors}
@celery_app.task(name="tasks.ekburg_permits_sync.refresh_all", queue="celery")
def refresh_all() -> dict[str, dict[str, int]]:
"""Обновить все 5 лет (2022-2026). Планируется через Celery beat ежемесячно."""
results: dict[str, dict[str, int]] = {}
for year in sorted(EKBURG_PERMITS_URLS.keys()):
logger.info("ekburg_permits_sync: starting year=%d", year)
results[str(year)] = refresh_year(year)
logger.info("ekburg_permits_sync: refresh_all done: %s", results)
return results

96
backend/app/workers/tasks/nspd_denorm_backfill.py Normal file
View file

@ -0,0 +1,96 @@
"""One-shot backfill: denormalize all existing nspd_quarter_dumps → nspd_parcels/buildings.
Запускается вручную через admin endpoint POST /api/v1/admin/etl/nspd-denorm-backfill
или через Celery CLI:
celery -A app.workers.celery_app call nspd_denorm.backfill_all_dumps
Идемпотентен: повторный запуск обновляет строки через ON CONFLICT DO UPDATE.
"""
from __future__ import annotations
import logging
from celery import shared_task
from sqlalchemy import text
from app.core.db import SessionLocal
from app.services.scrapers.nspd_denorm import denorm_dump
logger = logging.getLogger(__name__)
@shared_task(
name="nspd_denorm.backfill_all_dumps",
bind=True,
soft_time_limit=3600, # 1 час — много кварталов × ~10ms per parcel
max_retries=0, # One-shot, не retry
)
def backfill_all_dumps(self: object, *, limit: int | None = None) -> dict[str, int]: # type: ignore[misc]
"""Backfill all existing nspd_quarter_dumps → nspd_parcels / nspd_buildings.
Читает все строки nspd_quarter_dumps (или limit строк), и для каждой вызывает
denorm_dump. Возвращает агрегированные счётчики по всем кварталам.
Args:
limit: если задан — обработать только первые N кварталов (для тестирования).
Returns:
dict {
"parcels": суммарно вставлено/обновлено parcel строк,
"buildings": суммарно вставлено/обновлено building строк,
"errors": суммарно пропусков (нет cad_num, DB ошибки),
"quarters_processed": количество обработанных кварталов,
}
"""
totals: dict[str, int] = {
"parcels": 0,
"buildings": 0,
"errors": 0,
"quarters_processed": 0,
}
db = SessionLocal()
try:
sql = (
"SELECT quarter_cad, features_json "
"FROM nspd_quarter_dumps "
"WHERE total_features > 0 "
"ORDER BY quarter_cad"
)
if limit is not None:
# Используем параметр через text() чтобы избежать SQL-injection.
rows = db.execute(
text(sql + " LIMIT CAST(:lim AS integer)"), {"lim": int(limit)}
).fetchall()
else:
rows = db.execute(text(sql)).fetchall()
logger.info("backfill_all_dumps: starting, quarters=%d limit=%s", len(rows), limit)
for row in rows:
quarter_cad: str = row[0]
features_json = row[1]
# features_json — уже list[dict] через psycopg v3 JSON decoding.
features: list[dict] = features_json if isinstance(features_json, list) else []
try:
counts = denorm_dump(db, quarter_cad=quarter_cad, features=features)
totals["parcels"] += counts["parcels"]
totals["buildings"] += counts["buildings"]
totals["errors"] += counts["errors"]
totals["quarters_processed"] += 1
except Exception as e:
logger.warning("backfill_all_dumps: quarter=%s failed: %s", quarter_cad, e)
totals["errors"] += 1
# Продолжаем с остальными кварталами
try:
db.rollback()
except Exception:
pass
logger.info("backfill_all_dumps: done totals=%s", totals)
finally:
db.close()
return totals

87
backend/app/workers/tasks/nspd_sync.py
View file

@ -33,6 +33,7 @@ from app.services.scrapers.nspd_client import (
NspdLiteWafError,
QuarterDump,
)
from app.services.scrapers.nspd_denorm import denorm_dump
from app.workers.celery_app import celery_app
logger = logging.getLogger(__name__)
@ -82,6 +83,12 @@ def _build_features_json(dump: QuarterDump) -> list[dict[str, Any]]:
for feat in features:
out.append(_feat_to_dict(layer_name, feat))
# TIER 4 opportunity groups — keys: auction_parcels, scheme_parcels, ...
for short_name, features in dump.opportunity.items():
layer_name = f"opportunity_{short_name}"
for feat in features:
out.append(_feat_to_dict(layer_name, feat))
return out
@ -95,6 +102,16 @@ def _build_risks_count(dump: QuarterDump) -> int:
return sum(len(v) for v in dump.risks.values())
def _build_opportunity_count(dump: QuarterDump) -> int:
"""Сумма features по всем TIER 4 opportunity слоям (issue #94 PR2)."""
return sum(len(v) for v in dump.opportunity.values())
def _build_has_auction_parcels(dump: QuarterDump) -> bool:
"""True если квартал содержит >= 1 feature auction_parcels (layer 37299)."""
return len(dump.opportunity.get("auction_parcels", [])) > 0
# ── UPSERT helper ─────────────────────────────────────────────────────────────
_UPSERT_SQL = text(
@ -103,18 +120,26 @@ _UPSERT_SQL = text(
quarter_cad, quarter_geom, bbox_3857,
parcels_count, buildings_count, territorial_zones_count,
red_lines_count, engineering_count, zouit_count, risks_count, total_features,
has_auction_parcels, opportunity_count,
features_json, layers_fetched, fetched_at_utc, harvest_duration_ms,
harvest_error, region_code
) VALUES (
:quarter_cad,
CASE WHEN :geom_json IS NULL THEN NULL
ELSE ST_Multi(ST_Transform(ST_SetSRID(ST_GeomFromGeoJSON(:geom_json), 3857), 4326))
CASE WHEN CAST(:geom_json AS text) IS NULL THEN NULL
ELSE ST_Multi(ST_Transform(
ST_SetSRID(ST_GeomFromGeoJSON(CAST(:geom_json AS text)), 3857), 4326))
END,
CASE WHEN :bbox_xmin IS NULL THEN NULL
ELSE ST_MakeEnvelope(:bbox_xmin, :bbox_ymin, :bbox_xmax, :bbox_ymax, 3857)
CASE WHEN CAST(:bbox_xmin AS double precision) IS NULL THEN NULL
ELSE ST_MakeEnvelope(
CAST(:bbox_xmin AS double precision),
CAST(:bbox_ymin AS double precision),
CAST(:bbox_xmax AS double precision),
CAST(:bbox_ymax AS double precision),
3857)
END,
:parcels_count, :buildings_count, :territorial_zones_count,
:red_lines_count, :engineering_count, :zouit_count, :risks_count, :total_features,
:has_auction_parcels, :opportunity_count,
CAST(:features_json AS jsonb),
CAST(:layers_fetched AS text[]),
CAST(:fetched_at_utc AS timestamptz),
@ -133,6 +158,8 @@ _UPSERT_SQL = text(
zouit_count = EXCLUDED.zouit_count,
risks_count = EXCLUDED.risks_count,
total_features = EXCLUDED.total_features,
has_auction_parcels = EXCLUDED.has_auction_parcels,
opportunity_count = EXCLUDED.opportunity_count,
features_json = EXCLUDED.features_json,
layers_fetched = EXCLUDED.layers_fetched,
fetched_at_utc = EXCLUDED.fetched_at_utc,
@ -178,6 +205,8 @@ def _upsert_dump(
"engineering_count": len(dump.engineering_structures),
"zouit_count": _build_zouit_count(dump),
"risks_count": _build_risks_count(dump),
"has_auction_parcels": _build_has_auction_parcels(dump),
"opportunity_count": _build_opportunity_count(dump),
"total_features": dump.total_features,
"features_json": json.dumps(features_json or [], ensure_ascii=False),
"layers_fetched": list(dump.layers_fetched),
@ -202,6 +231,8 @@ def _upsert_dump(
"engineering_count": 0,
"zouit_count": 0,
"risks_count": 0,
"has_auction_parcels": False,
"opportunity_count": 0,
"total_features": 0,
"features_json": "[]",
"layers_fetched": [],
@ -227,7 +258,9 @@ def _upsert_dump(
bind=True,
name="tasks.nspd_sync.harvest_quarter",
max_retries=3,
soft_time_limit=120, # 2 мин — worst-case 22 HTTP × ~600ms = ~13s + margin
soft_time_limit=600, # 10 мин — grid-walk: 11 layers × 49 cells × ~70ms ≈ 40s + retries
time_limit=900, # 15 мин hard kill — safety net (PR #260 re-review): task может игнорить
# SoftTimeLimitExceeded → процесс не освобождается. Hard limit гарантирует worker recovery.
autoretry_for=(NspdLiteWafError,),
retry_backoff=True,
retry_backoff_max=120,
@ -239,20 +272,26 @@ def harvest_quarter(
region_code: int = 66,
include_zouit: bool = True,
include_risks: bool = False,
include_opportunity: bool = False,
) -> dict[str, Any]:
"""Single-quarter harvest. NSPDClient.search_by_quarter → UPSERT nspd_quarter_dumps.
Идемпотентен: повторный вызов обновляет строку (ON CONFLICT DO UPDATE).
WAF 403/429 → autoretry с exponential backoff (max 3 попытки).
Другие исключения → запись harvest_error в строку, return error dict (не raise).
Args:
include_opportunity: Фетчить TIER 4 opportunity layers (+5 HTTP запросов).
"""
t0 = time.monotonic()
logger.info(
"harvest_quarter start: cad=%s region=%d include_zouit=%s include_risks=%s",
"harvest_quarter start: cad=%s region=%d include_zouit=%s "
"include_risks=%s include_opportunity=%s",
quarter_cad,
region_code,
include_zouit,
include_risks,
include_opportunity,
)
client = NSPDClient()
@ -264,12 +303,39 @@ def harvest_quarter(
quarter_cad,
include_zouit=include_zouit,
include_risks=include_risks,
include_opportunity=include_opportunity,
)
features_json = _build_features_json(dump)
duration_ms = int((time.monotonic() - t0) * 1000)
_upsert_dump(quarter_cad, region_code, dump, features_json, duration_ms, None)
# Inline denorm: разложить parcels/buildings из dump в nspd_parcels/nspd_buildings.
# Ошибка denorm не должна фейлить весь harvest — только warning.
try:
denorm_db = SessionLocal()
try:
denorm_counts = denorm_dump(
denorm_db,
quarter_cad=quarter_cad,
features=features_json or [],
)
logger.info(
"harvest_quarter denorm: cad=%s parcels=%d buildings=%d errors=%d",
quarter_cad,
denorm_counts["parcels"],
denorm_counts["buildings"],
denorm_counts["errors"],
)
finally:
denorm_db.close()
except Exception as denorm_exc:
logger.warning(
"harvest_quarter denorm failed (non-fatal): cad=%s error=%s",
quarter_cad,
denorm_exc,
)
logger.info(
"harvest_quarter done: cad=%s region=%d duration=%dms total=%d",
quarter_cad,
@ -389,7 +455,14 @@ def harvest_stale_quarters(
enqueued = 0
for cad in stale_cads:
try:
harvest_quarter.apply_async(args=[cad, region_code])
harvest_quarter.apply_async(
args=[cad, region_code],
kwargs={
"include_zouit": True,
"include_risks": True,
"include_opportunity": True,
},
)
enqueued += 1
except Exception as e:
logger.warning("harvest_stale_quarters: enqueue failed for cad=%s: %s", cad, e)

170
backend/app/workers/tasks/scrape_kn_catalog_objects.py Normal file
View file

@ -0,0 +1,170 @@
"""Celery task: periodic catalog-object scrape для DOM.РФ.
Дополняет ~25 NULL колонок в domrf_kn_objects из SSR-страниц каталога.
kn-API эти поля не возвращает — они только на публичных страницах объектов.
Selector logic:
- catalog_scraped_at IS NULL — "новый" объект, всегда грузим
- DATE(catalog_scraped_at) < CURRENT_DATE — сегодня ещё не обновлялся, грузим
- DATE(catalog_scraped_at) = CURRENT_DATE — уже сегодня обновлён, пропускаем
- force=True — игнорирует фильтр, загружает все объекты последнего snapshot
Beat schedule: вторник 04:00 UTC (в beat_schedule.py).
"""
from __future__ import annotations
import asyncio
import logging
from datetime import date
from typing import Any
from sqlalchemy import text
from app.core.db import SessionLocal
from app.workers.celery_app import celery_app
logger = logging.getLogger(__name__)
# Запрос для выбора объектов на скрап.
# Возвращает obj_id + snapshot_date одним запросом, чтобы избежать race condition:
# если между двумя запросами kn-scraper запишет новый snapshot — UPDATE по старой
# snapshot_date не затронет ни одной строки. MAX subquery ограничена тем же
# region_cd чтобы не захватить snapshot другого региона.
#
# Фильтр (:force = false): берём только те, что ещё не обновлялись сегодня
# (catalog_scraped_at IS NULL — никогда не скрапились, либо DATE(...) < today).
# При :force = true фильтр снимается — грузим все объекты последнего snapshot.
_SELECT_TARGETS_SQL = text(
"""
SELECT obj_id, snapshot_date
FROM domrf_kn_objects
WHERE region_cd = :region_code
AND (
CAST(:force AS boolean)
OR catalog_scraped_at IS NULL
OR DATE(catalog_scraped_at) < CURRENT_DATE
)
AND snapshot_date = (
SELECT MAX(snapshot_date)
FROM domrf_kn_objects
WHERE region_cd = :region_code
)
ORDER BY catalog_scraped_at NULLS FIRST
LIMIT :max_objects
"""
)
# Лимит по умолчанию если max_objects не задан явно.
_DEFAULT_MAX_OBJECTS = 300
@celery_app.task(
bind=True,
name="tasks.scrape_kn_catalog_objects.scrape_kn_catalog_objects",
time_limit=3600,
)
def scrape_kn_catalog_objects(
self: Any,
region_code: int = 66,
max_objects: int | None = None,
force: bool = False,
) -> dict[str, Any]:
"""Periodic catalog-object scrape.
Args:
region_code: Код региона (ОКАТО prefix). Default 66 = Свердловская обл.
max_objects: Максимум объектов за один run. Default 300.
force: Если True — игнорирует фильтр "уже сегодня обновлён" и грузит
все объекты последнего snapshot (admin "Загрузить все"). По умолчанию
False — пропускает то, что уже скраплено сегодня.
Returns:
dict с ключами: region_code, snapshot_date, obj_ids_count,
processed, succeeded, failed, skipped.
Concurrency:
No Redis lock — consistent with sibling tasks (scrape_kn_region etc.).
Beat is configured for non-overlapping fire (Tuesday 04:00 UTC, ~5min run),
so concurrent execution is extremely rare. If it occurs:
- UPDATE is idempotent (COALESCE, catalog_scraped_at = NOW())
- Max risk: 2x WAF load on DOM.РФ for the same batch
- Both tasks complete; second update is no-op (catalog_scraped_at расхождение)
Add Redis lock if WAF blocks observed or beat schedule changes to overlap.
"""
from app.services.scrapers.domrf_catalog_object import scrape_catalog_objects
limit = max_objects if max_objects is not None else _DEFAULT_MAX_OBJECTS
db = SessionLocal()
try:
rows = (
db.execute(
_SELECT_TARGETS_SQL,
{"region_code": region_code, "max_objects": limit, "force": force},
)
.mappings()
.all()
)
obj_ids: list[int] = [int(r["obj_id"]) for r in rows]
except Exception as exc:
logger.error("scrape_kn_catalog_objects: failed to fetch obj_ids: %s", exc)
db.close()
raise
if not obj_ids:
logger.info(
"scrape_kn_catalog_objects: nothing to do for region=%d (force=%s)",
region_code,
force,
)
db.close()
return {
"region_code": region_code,
"force": force,
"obj_ids_count": 0,
"processed": 0,
"succeeded": 0,
"failed": 0,
"skipped": 0,
}
# snapshot_date берётся из первой строки результата — все строки одинаковые
# (WHERE snapshot_date = MAX(snapshot_date)). Это атомарно: один SELECT вместо двух,
# что устраняет race condition с kn-scraper.
snapshot_date_val: date = rows[0]["snapshot_date"]
logger.info(
"scrape_kn_catalog_objects: region=%d snapshot_date=%s obj_ids=%d limit=%d force=%s",
region_code,
snapshot_date_val,
len(obj_ids),
limit,
force,
)
try:
stats = asyncio.run(
scrape_catalog_objects(
db=db,
obj_ids=obj_ids,
snapshot_date=snapshot_date_val,
region_code=region_code,
)
)
except Exception as exc:
logger.error("scrape_kn_catalog_objects: scrape failed: %s", exc)
raise
finally:
db.close()
result: dict[str, Any] = {
"region_code": region_code,
"force": force,
"snapshot_date": str(snapshot_date_val),
"obj_ids_count": len(obj_ids),
**stats,
}
logger.info("scrape_kn_catalog_objects done: %s", result)
return result

154
backend/app/workers/tasks/scrape_objective.py
View file

@ -99,9 +99,21 @@ def _save_raw(
end_date: date | None,
use_ddu: bool,
use_dkp: bool,
payload: Any,
payload: Any | None = None,
) -> int:
body = json.dumps(payload, ensure_ascii=False)
"""Сохраняет мета + payload в objective_raw_reports.
payload=None допустим для stream-parsed отчётов (lots_pf 600+ МБ).
В этом случае payload_size=0 и payload=NULL (после миграции 79).
"""
if payload is not None:
body = json.dumps(payload, ensure_ascii=False)
payload_param = body
size = len(body.encode("utf-8"))
else:
payload_param = None
size = 0
row = db.execute(
text(
"""
@ -126,8 +138,8 @@ def _save_raw(
"end_date": end_date,
"use_ddu": use_ddu,
"use_dkp": use_dkp,
"payload": body,
"size": len(body.encode("utf-8")),
"payload": payload_param,
"size": size,
},
).scalar_one()
db.commit()
@ -230,45 +242,44 @@ def sync_objective_group(
),
]
snap = date.today()
try:
for kind, fn_name, params, section, rtype, rname in jobs:
try:
method = getattr(client, fn_name)
payload = method(**params)
n_requests += 1
raw_id = _save_raw(
db,
run_id,
report_section=section,
report_type=rtype,
report_name=rname,
group_name=group,
complex_name=None,
start_date=params.get("start_date"),
end_date=params.get("end_date"),
use_ddu=params.get("use_ddu", True),
use_dkp=params.get("use_dkp", False),
payload=payload,
)
reports_ok += 1
# Inline-нормализация в objective_corpus_room_month / lots / history.
snap = date.today()
try:
if kind == "corp_sum":
n = parser_mod.parse_corp_sum(payload, group, raw_id, db, dry_run=False)
rows_corpus_room += n
db.execute(
text(
"UPDATE objective_raw_reports "
" SET rows_extracted = :n WHERE raw_id = :rid"
),
{"n": n, "rid": raw_id},
)
elif kind == "lots_pf":
n_lots, n_hist = parser_mod.parse_lots_pf(
payload, raw_id, snap, db, dry_run=False
)
if kind == "lots_pf":
# lots_pf: 600+ МБ JSON → streaming через ijson, не грузим в RAM.
# payload пишем как NULL в objective_raw_reports (миграция 79).
raw_id = _save_raw(
db,
run_id,
report_section=section,
report_type=rtype,
report_name=rname,
group_name=group,
complex_name=None,
start_date=params.get("start_date"),
end_date=params.get("end_date"),
use_ddu=params.get("use_ddu", True),
use_dkp=params.get("use_dkp", False),
payload=None,
)
n_requests += 1
reports_ok += 1
try:
with client.stream_report(
report_type="Поквартирные",
report_name="Лоты",
group_name=group,
use_ddu=params.get("use_ddu", True),
use_dkp=params.get("use_dkp") or False,
) as resp:
n_lots, n_hist = parser_mod.parse_lots_pf_stream(
resp.iter_bytes(chunk_size=65536),
raw_id,
snap,
db,
dry_run=False,
)
rows_lots += n_lots
rows_history += n_hist
db.execute(
@ -278,18 +289,58 @@ def sync_objective_group(
),
{"n": n_lots, "rid": raw_id},
)
db.commit()
except Exception as parse_err:
# Парсинг упал — raw уже сохранён, можно re-parse позже.
db.rollback()
logger.exception(
"sync_objective_group: parser failed for %s/%s/%s raw_id=%s: %s",
section,
rtype,
rname,
raw_id,
parse_err,
db.commit()
except Exception as parse_err:
db.rollback()
logger.exception(
"sync_objective_group: lots_pf stream failed raw_id=%s: %s",
raw_id,
parse_err,
)
else:
# corp_sum (7 МБ) — полный load как раньше
method = getattr(client, fn_name)
payload = method(**params)
n_requests += 1
raw_id = _save_raw(
db,
run_id,
report_section=section,
report_type=rtype,
report_name=rname,
group_name=group,
complex_name=None,
start_date=params.get("start_date"),
end_date=params.get("end_date"),
use_ddu=params.get("use_ddu", True),
use_dkp=params.get("use_dkp", False),
payload=payload,
)
reports_ok += 1
try:
if kind == "corp_sum":
n = parser_mod.parse_corp_sum(
payload, group, raw_id, db, dry_run=False
)
rows_corpus_room += n
db.execute(
text(
"UPDATE objective_raw_reports "
" SET rows_extracted = :n WHERE raw_id = :rid"
),
{"n": n, "rid": raw_id},
)
db.commit()
except Exception as parse_err:
db.rollback()
logger.exception(
"sync_objective_group: parser failed for %s/%s/%s " "raw_id=%s: %s",
section,
rtype,
rname,
raw_id,
parse_err,
)
_heartbeat(
db,
@ -429,7 +480,6 @@ def sync_all_groups(
logger.info("[%d/%d] sync_objective_group(group=%r) START", idx + 1, len(eff_groups), group)
try:
res = sync_objective_group(
self,
group_name=group,
triggered_by=f"{triggered_by}-multi",
use_ddu=eff_use_ddu,

5
backend/pyproject.toml
View file

@ -22,13 +22,15 @@ dependencies = [
"tenacity>=9.0.0",
"pillow>=10.4.0",
"weasyprint>=62.0",
"jinja2>=3.1.0",
"ezdxf>=1.3.0",
"openpyxl>=3.1.0",
"pandas>=2.2.0",
"numpy>=2.0.0",
"scikit-learn>=1.5.0",
"sentry-sdk[fastapi]>=2.10.0",
"sentry-sdk[fastapi,celery,sqlalchemy,httpx]>=2.18.0",
"rosreestr2coord>=5.0.0",
"ijson>=3.2.0",
]
[dependency-groups]
@ -83,4 +85,5 @@ addopts = ["-m", "not prod_smoke"]
markers = [
"slow: marks tests as slow (need real network, deselect with -m 'not slow')",
"prod_smoke: production smoke tests against live https://gendsgn.ru (run only post-deploy with -m prod_smoke)",
"integration: phantom column gate tests requiring TEST_DATABASE_URL (SSH tunnel to prod Postgres)",
]

82
backend/tests/api/v1/test_admin_ekburg_permits.py Normal file
View file

@ -0,0 +1,82 @@
"""Тесты для POST /admin/scrape/ekburg-permits.
Проверяет:
- валидный запрос без year → scope all_years_2022_2026, task_id в ответе
- валидный запрос с year=2026 → scope year_2026
- year < 2022 или > 2030 → 422
- отсутствие X-Admin-Token → 401/503
"""
from __future__ import annotations
from unittest.mock import MagicMock, patch
import pytest
from fastapi.testclient import TestClient
from app.main import app
ADMIN_TOKEN = "test-admin-token"
ADMIN_HEADERS = {"X-Admin-Token": ADMIN_TOKEN}
ENDPOINT = "/api/v1/admin/scrape/ekburg-permits"
def _mock_task(task_id: str = "fake-task-id-123") -> MagicMock:
result = MagicMock()
result.id = task_id
return result
@patch("app.core.config.settings.scrape_admin_token", ADMIN_TOKEN)
def test_trigger_refresh_all_returns_task_id() -> None:
"""POST без year → refresh_all queued, scope=all_years_2022_2026."""
mock_result = _mock_task("task-all-001")
with (
patch("app.workers.tasks.ekburg_permits_sync.refresh_all") as mock_refresh_all,
patch("app.workers.tasks.ekburg_permits_sync.refresh_year"),
):
mock_refresh_all.apply_async.return_value = mock_result
client = TestClient(app)
response = client.post(ENDPOINT, json={}, headers=ADMIN_HEADERS)
assert response.status_code == 200, response.text
body = response.json()
assert body["task_id"] == "task-all-001"
assert body["scope"] == "all_years_2022_2026"
assert "queued_at" in body
@patch("app.core.config.settings.scrape_admin_token", ADMIN_TOKEN)
def test_trigger_refresh_year_returns_task_id() -> None:
"""POST year=2026 → refresh_year queued, scope=year_2026."""
mock_result = _mock_task("task-year-002")
with (
patch("app.workers.tasks.ekburg_permits_sync.refresh_year") as mock_refresh_year,
patch("app.workers.tasks.ekburg_permits_sync.refresh_all"),
):
mock_refresh_year.apply_async.return_value = mock_result
client = TestClient(app)
response = client.post(ENDPOINT, json={"year": 2026}, headers=ADMIN_HEADERS)
assert response.status_code == 200, response.text
body = response.json()
assert body["task_id"] == "task-year-002"
assert body["scope"] == "year_2026"
@pytest.mark.parametrize("bad_year", [2021, 2031, 1999, 9999])
@patch("app.core.config.settings.scrape_admin_token", ADMIN_TOKEN)
def test_trigger_invalid_year_returns_422(bad_year: int) -> None:
"""year вне диапазона [2022, 2030] → 422 Unprocessable Entity."""
client = TestClient(app)
response = client.post(ENDPOINT, json={"year": bad_year}, headers=ADMIN_HEADERS)
assert response.status_code == 422, f"year={bad_year} должен возвращать 422"
def test_trigger_no_token_returns_401_or_503() -> None:
"""Без X-Admin-Token → 401 или 503."""
client = TestClient(app)
response = client.post(ENDPOINT, json={})
assert response.status_code in (401, 503), response.text

122
backend/tests/api/v1/test_analyze_competitors_status.py Normal file
View file

@ -0,0 +1,122 @@
"""Тесты: /analyze endpoint возвращает site_status + ready_dt в competitors[].
Mock-based — не требуют живой БД.
Проверяет:
- поля site_status и ready_dt присутствуют в каждом элементе competitors
- первые позиции занимают строящиеся ЖК (site_status='Строящиеся')
- сданные ЖК идут после строящихся
"""
from __future__ import annotations
import datetime
from unittest.mock import MagicMock
# ── Вспомогательные фабрики ───────────────────────────────────────────────────
def _competitor_mapping(
obj_id: int,
comm_name: str,
site_status: str,
ready_dt: datetime.date | None,
flat_count: int,
distance_m: float = 500.0,
) -> MagicMock:
"""Имитирует sqlalchemy RowMapping для строки конкурента."""
data: dict = {
"obj_id": obj_id,
"comm_name": comm_name,
"dev_name": "TestDev",
"obj_class": "комфорт",
"flat_count": flat_count,
"district_name": "Ленинский",
"site_status": site_status,
"ready_dt": ready_dt,
"distance_m": distance_m,
}
m = MagicMock()
m.__getitem__ = lambda self, k: data[k]
m.keys = lambda: data.keys()
m.__iter__ = lambda self: iter(data)
m.items = lambda: data.items()
return m
# ── Тестовые данные ───────────────────────────────────────────────────────────
# Два сданных ЖК с большим flat_count и один строящийся с маленьким.
# До фикса ORDER BY flat_count DESC → сданные шли первыми.
_ROWS_MIXED = [
_competitor_mapping(
1, "ПИК Космонавтов 11 корп.1", "Сданные", datetime.date(2022, 6, 1), 800, 300.0
),
_competitor_mapping(
2, "ПИК Космонавтов 11 корп.2", "Сданные", datetime.date(2023, 3, 1), 750, 310.0
),
_competitor_mapping(
3, "Новый ЖК Строящийся", "Строящиеся", datetime.date(2026, 9, 1), 200, 400.0
),
]
# ── Тесты ─────────────────────────────────────────────────────────────────────
class TestCompetitorsHaveStatusFields:
"""site_status и ready_dt должны присутствовать в competitors[]."""
def test_fields_present(self) -> None:
"""Каждый конкурент содержит site_status и ready_dt."""
competitors = [dict(r.items()) for r in _ROWS_MIXED]
for c in competitors:
assert "site_status" in c, f"site_status отсутствует в {c}"
assert "ready_dt" in c, f"ready_dt отсутствует в {c}"
def test_site_status_values(self) -> None:
"""site_status принимает ожидаемые значения."""
competitors = [dict(r.items()) for r in _ROWS_MIXED]
statuses = {c["site_status"] for c in competitors}
assert "Строящиеся" in statuses
assert "Сданные" in statuses
def test_ready_dt_is_date_or_none(self) -> None:
"""ready_dt — datetime.date или None."""
competitors = [dict(r.items()) for r in _ROWS_MIXED]
for c in competitors:
val = c["ready_dt"]
assert val is None or isinstance(
val, datetime.date
), f"ready_dt имеет неожиданный тип {type(val)}: {val}"
class TestCompetitorsSortOrder:
"""Строящиеся ЖК должны идти первыми независимо от flat_count."""
def test_stroyashchiesya_first(self) -> None:
"""Строящийся ЖК с flat_count=200 должен быть раньше сданных с flat_count=800."""
# Симулируем SQL ORDER BY:
# CASE site_status WHEN 'Строящиеся' THEN 0 ELSE 1 END, distance_m ASC
def _sort_key(r: MagicMock) -> tuple:
data = dict(r.items())
status_order = 0 if data["site_status"] == "Строящиеся" else 1
return (status_order, data["distance_m"])
sorted_rows = sorted(_ROWS_MIXED, key=_sort_key)
first = dict(sorted_rows[0].items())
assert first["site_status"] == "Строящиеся", (
f"Первый конкурент должен быть 'Строящиеся', " f"но получили '{first['site_status']}'"
)
def test_flat_count_desc_would_break_order(self) -> None:
"""Демонстрирует, что старый ORDER BY flat_count DESC ставил сданные первыми."""
sorted_by_flat = sorted(
_ROWS_MIXED,
key=lambda r: dict(r.items())["flat_count"],
reverse=True,
)
first_old = dict(sorted_by_flat[0].items())
# Старая логика: первым шёл ЖК с flat_count=800 (Сданные)
assert first_old["flat_count"] == 800
assert first_old["site_status"] == "Сданные"

331
backend/tests/api/v1/test_analyze_inline_weights.py Normal file
View file

@ -0,0 +1,331 @@
"""Тесты для inline POI-weights в POST /api/v1/parcels/{cad_num}/analyze (#201).
Покрывает:
1. POST /analyze без body → system defaults (no regression)
2. POST /analyze с inline weights → applied (source = "inline")
3. POST /analyze с невалидной категорией → 422
4. POST /analyze с весом вне диапазона → 422
5. POST /analyze с body.weights + profile_id → body.weights wins (priority)
Стратегия mock: DB патчим через dependency_overrides, тяжёлые service-функции
(weather, velocity, dump и т.д.) патчим через unittest.mock.patch — чтобы не
дублировать все 18 db.execute call'ов в каждом тесте.
"""
from __future__ import annotations
from typing import Any
from unittest.mock import MagicMock, patch
import pytest
from fastapi.testclient import TestClient
from app.main import app
# ── Константы ─────────────────────────────────────────────────────────────────
_CAD = "66:41:0204016:10"
_WKT = "POLYGON((60.6 56.838, 60.61 56.838, 60.61 56.845, 60.6 56.845, 60.6 56.838))"
_GEOJSON = '{"type":"Polygon","coordinates":[[[60.6,56.838],[60.61,56.838]]]}'
# ── Mock factories ─────────────────────────────────────────────────────────────
def _make_mapping(data: dict[str, Any]) -> MagicMock:
"""Создать mock-строку (mapping) с __getitem__ + .get() для dict-like доступа."""
m = MagicMock()
m.__getitem__ = lambda self, k: data[k]
m.get = lambda k, default=None: data.get(k, default)
return m
def _make_db_for_analyze(
geom_found: bool = True,
district_found: bool = True,
poi_rows: list[Any] | None = None,
) -> MagicMock:
"""Сконструировать mock DB Session для analyze_parcel.
Порядок db.execute calls в analyze_parcel:
0. UNION ALL geom + source → .mappings().first()
1. WKT query → .mappings().first()
2. District → .mappings().first()
3. POI rows → .mappings().all()
4. Competitor rows → .mappings().all()
5. Pipeline rows → .mappings().all()
6. Centroid lat/lon → .mappings().first()
7. Noise rows → .mappings().all()
8. Hydrology → .mappings().all()
9. Utilities → .mappings().all()
10. parcel_meta (cad_parcels) → .mappings().first() ← #29 G2
11. Market trend → .mappings().first()
12. Zoning (begin_nested) → .mappings().first()
13. Success recommendation (begin_nested) → .mappings().all()
14. Market price (begin_nested) → .mappings().first()
15. Recent permits (begin_nested) → .mappings().all() ← #105 Phase 5
16. _geotech_risk (industrial count) → .scalar()
17. _neighbors_summary (neighbor_rows) → .mappings().all()
18. _neighbors_summary (overlap_row) → .mappings().first()
begin_nested() — возвращаем context manager чтобы поддержать `with` statement.
"""
db = MagicMock()
geom_row = (
_make_mapping({"geom_geojson": _GEOJSON, "geom_wkb": None, "source": "cad_quarter"})
if geom_found
else None
)
wkt_row = _make_mapping({"wkt": _WKT}) if geom_found else None
district_row = (
_make_mapping(
{
"district_name": "Октябрьский",
"median_price_per_m2": 120000,
"dist_to_center": 1500.0,
}
)
if district_found
else None
)
centroid_row = _make_mapping({"lat": 56.84, "lon": 60.605})
_poi_rows = poi_rows or []
# Счётчик вызовов execute — разводим first() / all() / scalar() по очерёдности
call_idx = [0]
# Ответы в порядке вызовов:
responses: list[Any] = [
("first", geom_row), # 0: geom UNION ALL
("first", wkt_row), # 1: WKT
("first", district_row), # 2: district
("all", _poi_rows), # 3: POI rows
("all", []), # 4: competitor rows
("all", []), # 5: pipeline rows
("first", centroid_row), # 6: centroid
("all", []), # 7: noise rows
("all", []), # 8: hydrology rows
("all", []), # 9: utilities rows
("first", None), # 10: parcel_meta (cad_parcels) ← #29 G2
("first", None), # 11: market trend
("first", None), # 12: zoning (inside begin_nested)
("all", []), # 13: success recommendation (inside begin_nested)
("scalar", 0), # 14: geotech_risk industrial count
("all", []), # 15: neighbors
("first", None), # 16: overlap
]
def _execute_side_effect(*args: Any, **kwargs: Any) -> MagicMock:
idx = call_idx[0]
call_idx[0] += 1
if idx >= len(responses):
# Безопасный fallback для непредусмотренных вызовов
r = MagicMock()
r.mappings.return_value.first.return_value = None
r.mappings.return_value.all.return_value = []
r.scalar.return_value = 0
return r
kind, data = responses[idx]
r = MagicMock()
r.mappings.return_value.first.return_value = data
r.mappings.return_value.all.return_value = data if isinstance(data, list) else []
r.scalar.return_value = data if kind == "scalar" else 0
return r
db.execute.side_effect = _execute_side_effect
# begin_nested() → context manager, остальные execute внутри него проходят
# через тот же side_effect (because db.execute is the same mock).
ctx = MagicMock()
ctx.__enter__ = MagicMock(return_value=ctx)
ctx.__exit__ = MagicMock(return_value=False)
db.begin_nested.return_value = ctx
return db
def _override_db(db: MagicMock):
def _get_db_override():
yield db
return _get_db_override
# Патчим тяжёлые внешние вызовы (weather / velocity / nspd-dump),
# чтобы тесты не зависели от сети и не требовали полного mock DB.
_PATCHES = [
patch("app.api.v1.parcels._fetch_air_quality_sync", return_value=None),
patch("app.api.v1.parcels._fetch_weather_sync", return_value=None),
patch("app.api.v1.parcels._fetch_seasonal_weather_sync", return_value=None),
patch(
"app.api.v1.parcels.get_quarter_dump_data",
return_value={
"nspd_zoning": None,
"nspd_zouit_overlaps": [],
"nspd_engineering_nearby": [],
"nspd_dump": {"available": False, "stale": False, "harvest_triggered": False},
},
),
patch("app.api.v1.parcels.compute_velocity", return_value=None),
patch("app.api.v1.parcels.compute_gate_verdict", return_value={"verdict": "unknown"}),
]
def _start_patches() -> list[Any]:
started = [p.start() for p in _PATCHES]
return started
def _stop_patches() -> None:
for p in _PATCHES:
p.stop()
# ── Тесты ─────────────────────────────────────────────────────────────────────
def test_analyze_no_body_uses_system_defaults() -> None:
"""POST /analyze без body → source = 'system', нет регрессии."""
from app.core.db import get_db
db = _make_db_for_analyze()
app.dependency_overrides[get_db] = _override_db(db)
_start_patches()
try:
client = TestClient(app)
resp = client.post(f"/api/v1/parcels/{_CAD}/analyze")
assert resp.status_code == 200, resp.text
body = resp.json()
assert "weights_profile" in body
assert body["weights_profile"]["source"] == "system"
assert body["weights_profile"]["inline_weights"] is None
finally:
app.dependency_overrides.clear()
_stop_patches()
def test_analyze_inline_weights_applied() -> None:
"""POST /analyze с body.weights → source = 'inline', веса применены."""
from app.core.db import get_db
db = _make_db_for_analyze()
app.dependency_overrides[get_db] = _override_db(db)
_start_patches()
try:
client = TestClient(app)
resp = client.post(
f"/api/v1/parcels/{_CAD}/analyze",
json={"weights": {"kindergarten": 2.5}},
)
assert resp.status_code == 200, resp.text
body = resp.json()
wp = body["weights_profile"]
assert wp["source"] == "inline"
assert wp["inline_weights"] == {"kindergarten": 2.5}
# applied weights содержат inline override поверх defaults
assert wp["weights_applied"]["kindergarten"] == pytest.approx(2.5)
finally:
app.dependency_overrides.clear()
_stop_patches()
def test_analyze_invalid_category_returns_422() -> None:
"""POST /analyze с невалидной POI-категорией → 422."""
from app.core.db import get_db
db = _make_db_for_analyze()
app.dependency_overrides[get_db] = _override_db(db)
_start_patches()
try:
client = TestClient(app)
resp = client.post(
f"/api/v1/parcels/{_CAD}/analyze",
json={"weights": {"nonexistent_category": 1.0}},
)
assert resp.status_code == 422, resp.text
detail = resp.json()["detail"]
assert "nonexistent_category" in detail
finally:
app.dependency_overrides.clear()
_stop_patches()
def test_analyze_weight_out_of_range_returns_422() -> None:
"""POST /analyze с весом вне [-2, 3] → 422."""
from app.core.db import get_db
db = _make_db_for_analyze()
app.dependency_overrides[get_db] = _override_db(db)
_start_patches()
try:
client = TestClient(app)
# Слишком большой вес
resp = client.post(
f"/api/v1/parcels/{_CAD}/analyze",
json={"weights": {"school": 99.9}},
)
assert resp.status_code == 422, resp.text
detail = resp.json()["detail"]
assert "school" in detail
# Слишком маленький вес
resp2 = client.post(
f"/api/v1/parcels/{_CAD}/analyze",
json={"weights": {"park": -5.0}},
)
assert resp2.status_code == 422, resp2.text
assert "park" in resp2.json()["detail"]
finally:
app.dependency_overrides.clear()
_stop_patches()
def test_inline_weights_rejects_nan() -> None:
"""NaN weight должен вернуть 422, а не propagate в score."""
from app.core.db import get_db
db = _make_db_for_analyze()
app.dependency_overrides[get_db] = _override_db(db)
_start_patches()
try:
client = TestClient(app)
# Отправляем raw JSON с NaN — httpx.Client не умеет encode float('nan'),
# поэтому используем content= с явным bytes-телом.
raw_body = b'{"weights": {"school": NaN}}'
resp = client.post(
f"/api/v1/parcels/{_CAD}/analyze",
content=raw_body,
headers={"Content-Type": "application/json"},
)
assert (
resp.status_code == 422
), f"Ожидали 422 для NaN-weight, получили {resp.status_code}: {resp.text}"
finally:
app.dependency_overrides.clear()
_stop_patches()
def test_analyze_inline_weights_beats_profile_id() -> None:
"""body.weights + profile_id → body.weights имеет приоритет (source = 'inline')."""
from app.core.db import get_db
db = _make_db_for_analyze()
app.dependency_overrides[get_db] = _override_db(db)
_start_patches()
try:
client = TestClient(app)
# Передаём и profile_id=1, и inline weights — inline должен победить
resp = client.post(
f"/api/v1/parcels/{_CAD}/analyze?profile_id=1",
json={"weights": {"metro_stop": 2.0}},
)
assert resp.status_code == 200, resp.text
wp = resp.json()["weights_profile"]
assert wp["source"] == "inline", f"Ожидали source='inline', получили '{wp['source']}'"
assert wp["weights_applied"]["metro_stop"] == pytest.approx(2.0)
# profile_id всё ещё присутствует в ответе для трассировки
assert wp["profile_id"] == 1
finally:
app.dependency_overrides.clear()
_stop_patches()

267
backend/tests/api/v1/test_analyze_market_price.py Normal file
View file

@ -0,0 +1,267 @@
"""Тесты для market_price в POST /api/v1/parcels/{cad_num}/analyze (#33).
Покрывает:
1. analyze с known quarter в mv_quarter_price_per_m2 → возвращает median/p25/p75/source='quarter_mv'
2. analyze с quarter которого нет в MV → deals_count=0, source='no_data'
3. analyze с invalid cad → 404 (no regression)
Стратегия mock: аналогична test_analyze_inline_weights.py — DB mock через
dependency_overrides, тяжёлые сервисы патчим через unittest.mock.patch.
Порядок db.execute calls в analyze_parcel (v3.7 + #33 + #29 G2):
0. UNION ALL geom + source → .mappings().first()
1. WKT query → .mappings().first()
2. District → .mappings().first()
3. POI rows → .mappings().all()
4. Competitor rows → .mappings().all()
5. Pipeline rows → .mappings().all()
6. Centroid lat/lon → .mappings().first()
7. Noise rows → .mappings().all()
8. Hydrology → .mappings().all()
9. Utilities → .mappings().all()
10. parcel_meta (cad_parcels) → .mappings().first() ← #29 G2
11. Market trend → .mappings().first()
12. Zoning (begin_nested) → .mappings().first()
13. Success recommendation (begin_nested) → .mappings().all()
14. Market price (begin_nested) → .mappings().first() ← #33
15. Recent permits (begin_nested) → .mappings().all() ← #105 Phase 5
16. _geotech_risk (industrial count) → .scalar()
17. _neighbors_summary (neighbor_rows) → .mappings().all()
18. _neighbors_summary (overlap_row) → .mappings().first()
"""
from __future__ import annotations
import datetime as dt
from decimal import Decimal
from typing import Any
from unittest.mock import MagicMock, patch
import pytest
from fastapi.testclient import TestClient
from app.main import app
# ── Константы ─────────────────────────────────────────────────────────────────
_CAD = "66:41:0204016:10" # cad_num с 4 частями — quarter = "66:41:0204016"
_CAD_3PARTS = "66:41:0204016" # cad_num уже является quarter
_WKT = "POLYGON((60.6 56.838, 60.61 56.838, 60.61 56.845, 60.6 56.845, 60.6 56.838))"
_GEOJSON = '{"type":"Polygon","coordinates":[[[60.6,56.838],[60.61,56.838]]]}'
# ── Mock factories ─────────────────────────────────────────────────────────────
def _make_mapping(data: dict[str, Any]) -> MagicMock:
"""Создать mock-строку (mapping) с __getitem__ + .get() для dict-like доступа."""
m = MagicMock()
m.__getitem__ = lambda self, k: data[k]
m.get = lambda k, default=None: data.get(k, default)
return m
def _make_db_for_analyze(
geom_found: bool = True,
district_found: bool = True,
market_price_row: dict[str, Any] | None = None,
) -> MagicMock:
"""Сконструировать mock DB Session для analyze_parcel.
market_price_row=None → имитирует "нет данных в MV" (mp_row is None → source='no_data').
market_price_row={...} → имитирует найденную строку в MV.
"""
db = MagicMock()
geom_row = (
_make_mapping({"geom_geojson": _GEOJSON, "geom_wkb": None, "source": "cad_quarter"})
if geom_found
else None
)
wkt_row = _make_mapping({"wkt": _WKT}) if geom_found else None
district_row = (
_make_mapping(
{
"district_name": "Октябрьский",
"median_price_per_m2": 120000,
"dist_to_center": 1500.0,
}
)
if district_found
else None
)
centroid_row = _make_mapping({"lat": 56.84, "lon": 60.605})
mp_mock = _make_mapping(market_price_row) if market_price_row is not None else None
call_idx = [0]
responses: list[Any] = [
("first", geom_row), # 0: geom UNION ALL
("first", wkt_row), # 1: WKT
("first", district_row), # 2: district
("all", []), # 3: POI rows
("all", []), # 4: competitor rows
("all", []), # 5: pipeline rows
("first", centroid_row), # 6: centroid
("all", []), # 7: noise rows
("all", []), # 8: hydrology rows
("all", []), # 9: utilities rows
("first", None), # 10: parcel_meta (cad_parcels) ← #29 G2
("first", None), # 11: market trend
("first", None), # 12: zoning (begin_nested)
("all", []), # 13: success recommendation (begin_nested)
("first", mp_mock), # 14: market price (begin_nested) ← #33
("all", []), # 15: recent permits (begin_nested)
("scalar", 0), # 16: geotech_risk
("all", []), # 17: neighbors
("first", None), # 18: overlap
]
def _execute_side_effect(*args: Any, **kwargs: Any) -> MagicMock:
idx = call_idx[0]
call_idx[0] += 1
if idx >= len(responses):
r = MagicMock()
r.mappings.return_value.first.return_value = None
r.mappings.return_value.all.return_value = []
r.scalar.return_value = 0
return r
kind, data = responses[idx]
r = MagicMock()
r.mappings.return_value.first.return_value = data
r.mappings.return_value.all.return_value = data if isinstance(data, list) else []
r.scalar.return_value = data if kind == "scalar" else 0
return r
db.execute.side_effect = _execute_side_effect
ctx = MagicMock()
ctx.__enter__ = MagicMock(return_value=ctx)
ctx.__exit__ = MagicMock(return_value=False)
db.begin_nested.return_value = ctx
return db
def _override_db(db: MagicMock):
def _get_db_override():
yield db
return _get_db_override
_PATCHES = [
patch("app.api.v1.parcels._fetch_air_quality_sync", return_value=None),
patch("app.api.v1.parcels._fetch_weather_sync", return_value=None),
patch("app.api.v1.parcels._fetch_seasonal_weather_sync", return_value=None),
patch(
"app.api.v1.parcels.get_quarter_dump_data",
return_value={
"nspd_zoning": None,
"nspd_zouit_overlaps": [],
"nspd_engineering_nearby": [],
"nspd_dump": {"available": False, "stale": False, "harvest_triggered": False},
},
),
patch("app.api.v1.parcels.compute_velocity", return_value=None),
patch("app.api.v1.parcels.compute_gate_verdict", return_value={"verdict": "unknown"}),
]
def _start_patches() -> None:
for p in _PATCHES:
p.start()
def _stop_patches() -> None:
for p in _PATCHES:
p.stop()
# ── Тесты ─────────────────────────────────────────────────────────────────────
def test_market_price_found_in_mv() -> None:
"""analyze с known quarter → market_price содержит median/p25/p75, source='quarter_mv'."""
from app.core.db import get_db
mv_data: dict[str, Any] = {
"p25": Decimal("85000.00"),
"median": Decimal("102000.00"),
"p75": Decimal("118000.00"),
"mean": Decimal("103500.00"),
"deals_count": 47,
"median_6m": Decimal("105000.00"),
"median_12m": Decimal("100000.00"),
"median_24m": Decimal("102000.00"),
"last_deal_date": dt.date(2026, 3, 15),
}
db = _make_db_for_analyze(market_price_row=mv_data)
app.dependency_overrides[get_db] = _override_db(db)
_start_patches()
try:
client = TestClient(app)
resp = client.post(f"/api/v1/parcels/{_CAD}/analyze")
assert resp.status_code == 200, resp.text
body = resp.json()
assert "market_price" in body, "market_price отсутствует в ответе"
mp = body["market_price"]
assert mp["source"] == "quarter_mv"
assert mp["deals_count"] == 47
assert mp["median"] == pytest.approx(102000.0)
assert mp["p25"] == pytest.approx(85000.0)
assert mp["p75"] == pytest.approx(118000.0)
assert mp["mean"] == pytest.approx(103500.0)
assert mp["median_6m"] == pytest.approx(105000.0)
assert mp["median_12m"] == pytest.approx(100000.0)
assert mp["median_24m"] == pytest.approx(102000.0)
assert mp["last_deal_date"] == "2026-03-15"
finally:
app.dependency_overrides.clear()
_stop_patches()
def test_market_price_quarter_not_in_mv() -> None:
"""analyze с quarter которого нет в MV → deals_count=0, source='no_data'."""
from app.core.db import get_db
# market_price_row=None → mp_mock=None → ветка else {"deals_count": 0, "source": "no_data"}
db = _make_db_for_analyze(market_price_row=None)
app.dependency_overrides[get_db] = _override_db(db)
_start_patches()
try:
client = TestClient(app)
resp = client.post(f"/api/v1/parcels/{_CAD}/analyze")
assert resp.status_code == 200, resp.text
body = resp.json()
assert "market_price" in body
mp = body["market_price"]
assert mp["source"] == "no_data"
assert mp["deals_count"] == 0
# price fields absent или None
assert mp.get("median") is None
assert mp.get("p25") is None
finally:
app.dependency_overrides.clear()
_stop_patches()
def test_market_price_invalid_cad_returns_404() -> None:
"""analyze с несуществующим cad → 404 (no regression)."""
from app.core.db import get_db
# geom_found=False → geom_row=None → endpoint вернёт 202 (inline fetch) или 404
# В тестовой среде on-demand fetch задизейблен (нет Celery/Redis) → 404 expected
db = _make_db_for_analyze(geom_found=False)
app.dependency_overrides[get_db] = _override_db(db)
_start_patches()
try:
client = TestClient(app)
resp = client.post("/api/v1/parcels/00:00:0000000:0/analyze")
# При отсутствии геометрии возвращается 202 (on-demand fetch enqueue) или 404
assert resp.status_code in (
202,
404,
), f"Ожидали 202 или 404 для неизвестного cad, получили {resp.status_code}: {resp.text}"
finally:
app.dependency_overrides.clear()
_stop_patches()

210
backend/tests/api/v1/test_analyze_parcel_meta.py Normal file
View file

@ -0,0 +1,210 @@
"""Тесты для parcel_meta в POST /api/v1/parcels/{cad_num}/analyze (#29 G2).
Покрывает:
1. analyze когда cad_parcels has row → parcel_meta содержит permitted_use/land_category/cad_cost
2. analyze когда cad_parcels row отсутствует → parcel_meta == None
Стратегия mock: аналогична test_analyze_market_price.py — DB mock через
dependency_overrides, тяжёлые сервисы патчим через unittest.mock.patch.
Порядок db.execute calls в analyze_parcel (с #29 G2):
0. UNION ALL geom + source → .mappings().first()
1. WKT query → .mappings().first()
2. District → .mappings().first()
3. POI rows → .mappings().all()
4. Competitor rows → .mappings().all()
5. Pipeline rows → .mappings().all()
6. Centroid lat/lon → .mappings().first()
7. Noise rows → .mappings().all()
8. Hydrology → .mappings().all()
9. Utilities → .mappings().all()
10. parcel_meta (cad_parcels) → .mappings().first() ← NEW #29 G2
11. Market trend → .mappings().first()
12. Zoning (begin_nested) → .mappings().first()
13. Success recommendation (begin_nested) → .mappings().all()
14. Market price (begin_nested) → .mappings().first()
15. Recent permits (begin_nested) → .mappings().all()
16. _geotech_risk (industrial count) → .scalar()
17. _neighbors_summary (neighbor_rows) → .mappings().all()
18. _neighbors_summary (overlap_row) → .mappings().first()
"""
from __future__ import annotations
from typing import Any
from unittest.mock import MagicMock, patch
from fastapi.testclient import TestClient
from app.main import app
_CAD = "66:41:0204016:10"
_WKT = "POLYGON((60.6 56.838, 60.61 56.838, 60.61 56.845, 60.6 56.845, 60.6 56.838))"
_GEOJSON = '{"type":"Polygon","coordinates":[[[60.6,56.838],[60.61,56.838]]]}'
def _make_mapping(data: dict[str, Any]) -> MagicMock:
m = MagicMock()
m.__getitem__ = lambda self, k: data[k]
m.get = lambda k, default=None: data.get(k, default)
return m
def _make_db_for_analyze(
parcel_meta_row: dict[str, Any] | None = None,
) -> MagicMock:
"""Сконструировать mock DB Session для analyze_parcel с фокусом на parcel_meta.
parcel_meta_row=None → имитирует "строки нет в cad_parcels" → parcel_meta=None.
parcel_meta_row={...} → имитирует найденную строку.
"""
db = MagicMock()
geom_row = _make_mapping({"geom_geojson": _GEOJSON, "geom_wkb": None, "source": "cad_quarter"})
wkt_row = _make_mapping({"wkt": _WKT})
district_row = _make_mapping(
{
"district_name": "Октябрьский",
"median_price_per_m2": 120000,
"dist_to_center": 1500.0,
}
)
centroid_row = _make_mapping({"lat": 56.84, "lon": 60.605})
pm_mock = _make_mapping(parcel_meta_row) if parcel_meta_row is not None else None
call_idx = [0]
responses: list[Any] = [
("first", geom_row), # 0: geom UNION ALL
("first", wkt_row), # 1: WKT
("first", district_row), # 2: district
("all", []), # 3: POI rows
("all", []), # 4: competitor rows
("all", []), # 5: pipeline rows
("first", centroid_row), # 6: centroid
("all", []), # 7: noise rows
("all", []), # 8: hydrology rows
("all", []), # 9: utilities rows
("first", pm_mock), # 10: parcel_meta ← #29 G2
("first", None), # 11: market trend
("first", None), # 12: zoning (begin_nested)
("all", []), # 13: success recommendation (begin_nested)
("first", None), # 14: market price (begin_nested)
("all", []), # 15: recent permits (begin_nested)
("scalar", 0), # 16: geotech_risk
("all", []), # 17: neighbors
("first", None), # 18: overlap
]
def _execute_side_effect(*args: Any, **kwargs: Any) -> MagicMock:
idx = call_idx[0]
call_idx[0] += 1
if idx >= len(responses):
r = MagicMock()
r.mappings.return_value.first.return_value = None
r.mappings.return_value.all.return_value = []
r.scalar.return_value = 0
return r
kind, data = responses[idx]
r = MagicMock()
r.mappings.return_value.first.return_value = data
r.mappings.return_value.all.return_value = data if isinstance(data, list) else []
r.scalar.return_value = data if kind == "scalar" else 0
return r
db.execute.side_effect = _execute_side_effect
ctx = MagicMock()
ctx.__enter__ = MagicMock(return_value=ctx)
ctx.__exit__ = MagicMock(return_value=False)
db.begin_nested.return_value = ctx
return db
def _override_db(db: MagicMock):
def _get_db_override():
yield db
return _get_db_override
_PATCHES = [
patch("app.api.v1.parcels._fetch_air_quality_sync", return_value=None),
patch("app.api.v1.parcels._fetch_weather_sync", return_value=None),
patch("app.api.v1.parcels._fetch_seasonal_weather_sync", return_value=None),
patch(
"app.api.v1.parcels.get_quarter_dump_data",
return_value={
"nspd_zoning": None,
"nspd_zouit_overlaps": [],
"nspd_engineering_nearby": [],
"nspd_risk_zones": [],
"nspd_opportunity_parcels": [],
"nspd_red_lines": [],
"nspd_dump": {"available": False, "stale": False, "harvest_triggered": False},
},
),
patch("app.api.v1.parcels.compute_velocity", return_value=None),
patch("app.api.v1.parcels.compute_gate_verdict", return_value={"verdict": "unknown"}),
]
def _start_patches() -> None:
for p in _PATCHES:
p.start()
def _stop_patches() -> None:
for p in _PATCHES:
p.stop()
def test_parcel_meta_found_in_cad_parcels() -> None:
"""analyze → cad_parcels has row → parcel_meta содержит permitted_use/land_category/cad_cost."""
from app.core.db import get_db
pm_data: dict[str, Any] = {
"permitted_use": "многоквартирный дом",
"land_category": "Земли населённых пунктов",
"land_subtype": "жилая застройка",
"cad_cost": 5_000_000.0,
}
db = _make_db_for_analyze(parcel_meta_row=pm_data)
app.dependency_overrides[get_db] = _override_db(db)
_start_patches()
try:
client = TestClient(app)
resp = client.post(f"/api/v1/parcels/{_CAD}/analyze")
assert resp.status_code == 200, resp.text
body = resp.json()
assert "parcel_meta" in body, "parcel_meta отсутствует в ответе"
pm = body["parcel_meta"]
assert pm is not None, "parcel_meta должен быть не None при наличии строки"
assert pm["permitted_use"] == "многоквартирный дом"
assert pm["land_category"] == "Земли населённых пунктов"
assert pm["land_subtype"] == "жилая застройка"
assert pm["cad_cost"] == 5_000_000.0
assert pm["source"] == "cad_parcels"
finally:
app.dependency_overrides.clear()
_stop_patches()
def test_parcel_meta_none_when_cad_parcels_missing() -> None:
"""analyze → cad_parcels row отсутствует → parcel_meta == None."""
from app.core.db import get_db
db = _make_db_for_analyze(parcel_meta_row=None)
app.dependency_overrides[get_db] = _override_db(db)
_start_patches()
try:
client = TestClient(app)
resp = client.post(f"/api/v1/parcels/{_CAD}/analyze")
assert resp.status_code == 200, resp.text
body = resp.json()
assert "parcel_meta" in body, "parcel_meta ключ должен присутствовать в ответе"
assert body["parcel_meta"] is None, "parcel_meta должен быть None при отсутствии строки"
finally:
app.dependency_overrides.clear()
_stop_patches()

283
backend/tests/api/v1/test_analyze_recent_permits.py Normal file
View file

@ -0,0 +1,283 @@
"""Тесты для recent_permits_in_quarter в POST /api/v1/parcels/{cad_num}/analyze (#105 Phase 5).
Покрывает:
1. analyze с quarter где есть РНС/РВЭ → recent_permits non-empty + summary calculated
2. analyze с quarter без permits → recent_permits=[], summary={rns_count=0, ...}
3. analyze 404 на invalid cad (no regression)
Порядок db.execute calls в analyze_parcel (после #105 Phase 5 + #29 G2):
0. UNION ALL geom + source → .mappings().first()
1. WKT query → .mappings().first()
2. District → .mappings().first()
3. POI rows → .mappings().all()
4. Competitor rows → .mappings().all()
5. Pipeline rows → .mappings().all()
6. Centroid lat/lon → .mappings().first()
7. Noise rows → .mappings().all()
8. Hydrology → .mappings().all()
9. Utilities → .mappings().all()
10. parcel_meta (cad_parcels) → .mappings().first() ← #29 G2
11. Market trend → .mappings().first()
12. Zoning (begin_nested) → .mappings().first()
13. Success recommendation (begin_nested) → .mappings().all()
14. Market price (begin_nested) → .mappings().first()
15. Recent permits (begin_nested) → .mappings().all() ← #105
16. _geotech_risk (industrial count) → .scalar()
17. _neighbors_summary (neighbor_rows) → .mappings().all()
18. _neighbors_summary (overlap_row) → .mappings().first()
"""
from __future__ import annotations
import datetime as dt
from typing import Any
from unittest.mock import MagicMock, patch
import pytest
from fastapi.testclient import TestClient
from app.main import app
# ── Константы ─────────────────────────────────────────────────────────────────
_CAD = "66:41:0204016:10" # cad_num с 4 частями — quarter prefix = "66:41:0204016"
_WKT = "POLYGON((60.6 56.838, 60.61 56.838, 60.61 56.845, 60.6 56.845, 60.6 56.838))"
_GEOJSON = '{"type":"Polygon","coordinates":[[[60.6,56.838],[60.61,56.838]]]}'
# ── Mock factories ─────────────────────────────────────────────────────────────
def _make_mapping(data: dict[str, Any]) -> MagicMock:
"""Создать mock-строку (mapping) с __getitem__ + .get() для dict-like доступа."""
m = MagicMock()
m.__getitem__ = lambda self, k: data[k]
m.get = lambda k, default=None: data.get(k, default)
return m
def _make_db_for_analyze(
geom_found: bool = True,
permits_rows: list[dict[str, Any]] | None = None,
) -> MagicMock:
"""Сконструировать mock DB Session для analyze_parcel.
permits_rows=None → пустой список (нет разрешений в квартале).
permits_rows=[...] → список разрешений для агрегации.
"""
db = MagicMock()
geom_row = (
_make_mapping({"geom_geojson": _GEOJSON, "geom_wkb": None, "source": "cad_quarter"})
if geom_found
else None
)
wkt_row = _make_mapping({"wkt": _WKT}) if geom_found else None
district_row = _make_mapping(
{
"district_name": "Октябрьский",
"median_price_per_m2": 120000,
"dist_to_center": 1500.0,
}
)
centroid_row = _make_mapping({"lat": 56.84, "lon": 60.605})
raw_permits = [_make_mapping(p) for p in (permits_rows or [])]
call_idx = [0]
responses: list[Any] = [
("first", geom_row), # 0: geom UNION ALL
("first", wkt_row), # 1: WKT
("first", district_row), # 2: district
("all", []), # 3: POI rows
("all", []), # 4: competitor rows
("all", []), # 5: pipeline rows
("first", centroid_row), # 6: centroid
("all", []), # 7: noise rows
("all", []), # 8: hydrology rows
("all", []), # 9: utilities rows
("first", None), # 10: parcel_meta (cad_parcels) ← #29 G2
("first", None), # 11: market trend
("first", None), # 12: zoning (begin_nested)
("all", []), # 13: success recommendation (begin_nested)
("first", None), # 14: market price (begin_nested)
("all", raw_permits), # 15: recent permits (begin_nested) ← #105
("scalar", 0), # 16: geotech_risk
("all", []), # 17: neighbors
("first", None), # 18: overlap
]
def _execute_side_effect(*args: Any, **kwargs: Any) -> MagicMock:
idx = call_idx[0]
call_idx[0] += 1
if idx >= len(responses):
r = MagicMock()
r.mappings.return_value.first.return_value = None
r.mappings.return_value.all.return_value = []
r.scalar.return_value = 0
return r
kind, data = responses[idx]
r = MagicMock()
r.mappings.return_value.first.return_value = data
r.mappings.return_value.all.return_value = data if isinstance(data, list) else []
r.scalar.return_value = data if kind == "scalar" else 0
return r
db.execute.side_effect = _execute_side_effect
ctx = MagicMock()
ctx.__enter__ = MagicMock(return_value=ctx)
ctx.__exit__ = MagicMock(return_value=False)
db.begin_nested.return_value = ctx
return db
def _override_db(db: MagicMock):
def _get_db_override():
yield db
return _get_db_override
_PATCHES = [
patch("app.api.v1.parcels._fetch_air_quality_sync", return_value=None),
patch("app.api.v1.parcels._fetch_weather_sync", return_value=None),
patch("app.api.v1.parcels._fetch_seasonal_weather_sync", return_value=None),
patch(
"app.api.v1.parcels.get_quarter_dump_data",
return_value={
"nspd_zoning": None,
"nspd_zouit_overlaps": [],
"nspd_engineering_nearby": [],
"nspd_dump": {"available": False, "stale": False, "harvest_triggered": False},
},
),
patch("app.api.v1.parcels.compute_velocity", return_value=None),
patch("app.api.v1.parcels.compute_gate_verdict", return_value={"verdict": "unknown"}),
]
def _start_patches() -> None:
for p in _PATCHES:
p.start()
def _stop_patches() -> None:
for p in _PATCHES:
p.stop()
# ── Тесты ─────────────────────────────────────────────────────────────────────
def test_recent_permits_found_in_quarter() -> None:
"""analyze с квартала где есть РНС/РВЭ → recent_permits non-empty + summary calculated."""
from app.core.db import get_db
rns_permit: dict[str, Any] = {
"permit_type": "RNS",
"permit_number": "66-RNS-2024-001",
"issue_date": dt.date(2024, 6, 15),
"developer_name": "Атомстрой",
"developer_inn": "6671234567",
"object_name": "МКД ЖК Тест",
"object_type": "МКД",
"construction_address": "г. Екатеринбург, ул. Ленина, 1",
"total_area_sqm": 45000.0,
}
rve_permit: dict[str, Any] = {
"permit_type": "RVE",
"permit_number": "66-RVE-2025-002",
"issue_date": dt.date(2025, 3, 10),
"developer_name": "Атомстрой",
"developer_inn": "6671234567",
"object_name": "МКД ЖК Тест (1 оч.)",
"object_type": "МКД",
"construction_address": "г. Екатеринбург, ул. Ленина, 1",
"total_area_sqm": None,
}
db = _make_db_for_analyze(permits_rows=[rns_permit, rve_permit])
app.dependency_overrides[get_db] = _override_db(db)
_start_patches()
try:
client = TestClient(app)
resp = client.post(f"/api/v1/parcels/{_CAD}/analyze")
assert resp.status_code == 200, resp.text
body = resp.json()
assert "recent_permits_in_quarter" in body
assert "permits_summary" in body
permits = body["recent_permits_in_quarter"]
assert len(permits) == 2
rns = next(p for p in permits if p["permit_type"] == "RNS")
assert rns["permit_number"] == "66-RNS-2024-001"
assert rns["developer_name"] == "Атомстрой"
assert rns["issue_date"] == "2024-06-15"
assert rns["total_area_sqm"] == pytest.approx(45000.0)
rve = next(p for p in permits if p["permit_type"] == "RVE")
assert rve["issue_date"] == "2025-03-10"
assert "units_count" not in rve
summary = body["permits_summary"]
assert summary["rns_count"] == 1
assert summary["rve_count"] == 1
assert summary["rns_total_area_sqm"] == pytest.approx(45000.0)
assert "rve_total_units" not in summary
assert len(summary["by_developer"]) == 1
assert summary["by_developer"][0]["name"] == "Атомстрой"
assert summary["by_developer"][0]["permits_count"] == 2
finally:
app.dependency_overrides.clear()
_stop_patches()
def test_recent_permits_empty_quarter() -> None:
"""analyze с quarter без permits → recent_permits=[], summary с нулями."""
from app.core.db import get_db
db = _make_db_for_analyze(permits_rows=[])
app.dependency_overrides[get_db] = _override_db(db)
_start_patches()
try:
client = TestClient(app)
resp = client.post(f"/api/v1/parcels/{_CAD}/analyze")
assert resp.status_code == 200, resp.text
body = resp.json()
assert "recent_permits_in_quarter" in body
assert body["recent_permits_in_quarter"] == []
assert "permits_summary" in body
summary = body["permits_summary"]
assert summary["rns_count"] == 0
assert summary["rve_count"] == 0
assert summary["rns_total_area_sqm"] == pytest.approx(0.0)
assert "rve_total_units" not in summary
assert summary["by_developer"] == []
finally:
app.dependency_overrides.clear()
_stop_patches()
def test_recent_permits_invalid_cad_no_regression() -> None:
"""analyze с несуществующим cad → 202 или 404 (no regression от Phase 5)."""
from app.core.db import get_db
db = _make_db_for_analyze(geom_found=False)
app.dependency_overrides[get_db] = _override_db(db)
_start_patches()
try:
client = TestClient(app)
resp = client.post("/api/v1/parcels/00:00:0000000:0/analyze")
assert resp.status_code in (
202,
404,
), f"Ожидали 202 или 404 для неизвестного cad, получили {resp.status_code}: {resp.text}"
finally:
app.dependency_overrides.clear()
_stop_patches()

595
backend/tests/api/v1/test_custom_pois.py Normal file
View file

@ -0,0 +1,595 @@
"""Тесты для custom POI CRUD + scoring integration (#254).
Покрывает:
1. POST /api/v1/custom-pois → 201, correct response, X-Session-Id в response header
2. GET /api/v1/custom-pois → list для user (изоляция по user_id)
3. PATCH /api/v1/custom-pois/{id} → updated, 404 для чужой
4. DELETE /api/v1/custom-pois/{id} → 204, 404 для чужой
5. Validation: weight outside [-5,5] → 422; lon/lat outside range → 422
6. GET /api/v1/custom-pois?parcel_cad=... → filter works
7. Scoring integration: custom POI в 500м с weight=+2 увеличивает score
8. No X-Session-Id → auto-generated UUID в response header
9. Scoring absent when no X-Session-Id header
10. Service-level: db.commit() вызывается в create/update/delete (#261 regression guard)
Стратегия mock: сервисные функции патчим через unittest.mock.patch,
DB — через dependency_overrides (аналогично test_analyze_inline_weights.py).
Service commit tests (#261): MagicMock db + assert db.commit.assert_called_once().
"""
from __future__ import annotations
import uuid
from datetime import UTC, datetime
from typing import Any
from unittest.mock import MagicMock, patch
import pytest
from fastapi.testclient import TestClient
from app.main import app
from app.schemas.custom_poi import CustomPoiCreate, CustomPoiOut, CustomPoiUpdate
# ── Константы ─────────────────────────────────────────────────────────────────
_CAD = "66:41:0204016:10"
_WKT = "POLYGON((60.6 56.838, 60.61 56.838, 60.61 56.845, 60.6 56.845, 60.6 56.838))"
_GEOJSON = '{"type":"Polygon","coordinates":[[[60.6,56.838],[60.61,56.838]]]}'
_SESSION = "test-session-abc123"
_TS = datetime(2026, 5, 17, 10, 0, 0, tzinfo=UTC)
# ── Helpers ────────────────────────────────────────────────────────────────────
def _make_poi_out(
poi_id: int = 1,
user_id: str = _SESSION,
name: str = "Парк Маяковского",
weight: float = 2.0,
lon: float = 60.605,
lat: float = 56.838,
parcel_cad: str | None = None,
) -> CustomPoiOut:
return CustomPoiOut(
id=poi_id,
user_id=user_id,
parcel_cad=parcel_cad,
name=name,
category="park",
weight=weight,
lon=lon,
lat=lat,
notes=None,
created_at=_TS,
updated_at=_TS,
)
def _make_mapping(data: dict[str, Any]) -> MagicMock:
m = MagicMock()
m.__getitem__ = lambda self, k: data[k]
m.get = lambda k, default=None: data.get(k, default)
return m
# ── Tests: CRUD ────────────────────────────────────────────────────────────────
def test_create_poi_returns_201() -> None:
"""POST /custom-pois → 201 + корректный тело + X-Session-Id в header."""
expected = _make_poi_out()
with patch("app.api.v1.custom_pois.create_custom_poi", return_value=expected) as mock_create:
client = TestClient(app)
resp = client.post(
"/api/v1/custom-pois",
json={
"name": "Парк Маяковского",
"category": "park",
"weight": 2.0,
"lon": 60.605,
"lat": 56.838,
},
headers={"X-Session-Id": _SESSION},
)
assert resp.status_code == 201, resp.text
body = resp.json()
assert body["name"] == "Парк Маяковского"
assert body["weight"] == pytest.approx(2.0)
assert body["id"] == 1
assert resp.headers.get("x-session-id") == _SESSION
mock_create.assert_called_once()
def test_create_poi_auto_generates_session_id() -> None:
"""POST без X-Session-Id → 201 + авто-UUID в X-Session-Id header."""
expected = _make_poi_out(user_id="some-uuid")
with patch("app.api.v1.custom_pois.create_custom_poi", return_value=expected):
client = TestClient(app)
resp = client.post(
"/api/v1/custom-pois",
json={"name": "Test", "weight": 1.0, "lon": 60.0, "lat": 56.0},
)
assert resp.status_code == 201, resp.text
sid = resp.headers.get("x-session-id")
assert sid is not None, "Ожидали X-Session-Id в response headers"
# Должен быть валидным UUID
try:
uuid.UUID(sid)
except ValueError:
pytest.fail(f"X-Session-Id '{sid}' не является валидным UUID")
def test_create_poi_weight_out_of_range_returns_422() -> None:
"""weight > 5 или < -5 → 422 от Pydantic (FastAPI body validation)."""
client = TestClient(app)
resp = client.post(
"/api/v1/custom-pois",
json={"name": "Test", "weight": 10.0, "lon": 60.0, "lat": 56.0},
headers={"X-Session-Id": _SESSION},
)
assert resp.status_code == 422, resp.text
resp2 = client.post(
"/api/v1/custom-pois",
json={"name": "Test", "weight": -6.0, "lon": 60.0, "lat": 56.0},
headers={"X-Session-Id": _SESSION},
)
assert resp2.status_code == 422, resp2.text
def test_create_poi_lon_out_of_range_returns_422() -> None:
"""lon > 180 → 422."""
client = TestClient(app)
resp = client.post(
"/api/v1/custom-pois",
json={"name": "Test", "weight": 1.0, "lon": 200.0, "lat": 56.0},
headers={"X-Session-Id": _SESSION},
)
assert resp.status_code == 422, resp.text
def test_create_poi_lat_out_of_range_returns_422() -> None:
"""lat > 90 → 422."""
client = TestClient(app)
resp = client.post(
"/api/v1/custom-pois",
json={"name": "Test", "weight": 1.0, "lon": 60.0, "lat": 100.0},
headers={"X-Session-Id": _SESSION},
)
assert resp.status_code == 422, resp.text
def test_list_pois_returns_user_items() -> None:
"""GET /custom-pois → список POI пользователя."""
pois = [_make_poi_out(1), _make_poi_out(2, name="Школа №5", weight=1.5)]
with patch("app.api.v1.custom_pois.list_custom_pois", return_value=pois):
client = TestClient(app)
resp = client.get("/api/v1/custom-pois", headers={"X-Session-Id": _SESSION})
assert resp.status_code == 200, resp.text
body = resp.json()
assert len(body) == 2
assert body[0]["id"] == 1
assert body[1]["name"] == "Школа №5"
def test_list_pois_filter_by_parcel_cad() -> None:
"""GET /custom-pois?parcel_cad=... → вызывает list_custom_pois с parcel_cad."""
with patch("app.api.v1.custom_pois.list_custom_pois", return_value=[]) as mock_list:
client = TestClient(app)
resp = client.get(
f"/api/v1/custom-pois?parcel_cad={_CAD}",
headers={"X-Session-Id": _SESSION},
)
assert resp.status_code == 200, resp.text
# Проверяем что parcel_cad передан в сервис
call_kwargs = mock_list.call_args
assert call_kwargs[1].get("parcel_cad") == _CAD or (
len(call_kwargs[0]) >= 3 and call_kwargs[0][2] == _CAD
)
def test_patch_poi_returns_updated() -> None:
"""PATCH /custom-pois/{id} → 200 с обновлёнными данными."""
updated = _make_poi_out(weight=3.0, name="Обновлённый парк")
with patch("app.api.v1.custom_pois.update_custom_poi", return_value=updated):
client = TestClient(app)
resp = client.patch(
"/api/v1/custom-pois/1",
json={"weight": 3.0, "name": "Обновлённый парк"},
headers={"X-Session-Id": _SESSION},
)
assert resp.status_code == 200, resp.text
body = resp.json()
assert body["weight"] == pytest.approx(3.0)
assert body["name"] == "Обновлённый парк"
def test_patch_poi_not_found_returns_404() -> None:
"""PATCH для несуществующей/чужой POI → 404."""
with patch("app.api.v1.custom_pois.update_custom_poi", return_value=None):
client = TestClient(app)
resp = client.patch(
"/api/v1/custom-pois/999",
json={"weight": 1.0},
headers={"X-Session-Id": _SESSION},
)
assert resp.status_code == 404, resp.text
def test_delete_poi_returns_204() -> None:
"""DELETE /custom-pois/{id} → 204 при успешном удалении."""
with patch("app.api.v1.custom_pois.delete_custom_poi", return_value=True):
client = TestClient(app)
resp = client.delete("/api/v1/custom-pois/1", headers={"X-Session-Id": _SESSION})
assert resp.status_code == 204, resp.text
def test_delete_poi_not_found_returns_404() -> None:
"""DELETE для несуществующей POI → 404."""
with patch("app.api.v1.custom_pois.delete_custom_poi", return_value=False):
client = TestClient(app)
resp = client.delete("/api/v1/custom-pois/999", headers={"X-Session-Id": _SESSION})
assert resp.status_code == 404, resp.text
# ── Tests: Scoring integration ─────────────────────────────────────────────────
# Вспомогательный mock DB для analyze_parcel (аналогично test_analyze_inline_weights.py).
# Порядок db.execute calls включает #254 custom POI via get_overlaps_for_scoring,
# который вызывается уже после POI loop — ВНУТРИ analyze_parcel через импортированную
# функцию. Мы патчим её через patch() — DB mock остаётся прежним.
def _make_mapping_analyze(data: dict[str, Any]) -> MagicMock:
m = MagicMock()
m.__getitem__ = lambda self, k: data[k]
m.get = lambda k, default=None: data.get(k, default)
return m
def _make_db_for_analyze() -> MagicMock:
"""Mock DB Session для analyze_parcel (без custom POI вызовов — они пропатчены)."""
db = MagicMock()
geom_row = _make_mapping_analyze(
{"geom_geojson": _GEOJSON, "geom_wkb": None, "source": "cad_quarter"}
)
wkt_row = _make_mapping_analyze({"wkt": _WKT})
district_row = _make_mapping_analyze(
{
"district_name": "Октябрьский",
"median_price_per_m2": 120000,
"dist_to_center": 1500.0,
}
)
centroid_row = _make_mapping_analyze({"lat": 56.84, "lon": 60.605})
call_idx = [0]
responses: list[Any] = [
("first", geom_row), # 0: geom UNION ALL
("first", wkt_row), # 1: WKT
("first", district_row), # 2: district
("all", []), # 3: POI rows
("all", []), # 4: competitor rows
("all", []), # 5: pipeline rows
("first", centroid_row), # 6: centroid
("all", []), # 7: noise rows
("all", []), # 8: hydrology rows
("all", []), # 9: utilities rows
("first", None), # 10: parcel_meta
("first", None), # 11: market trend
("first", None), # 12: zoning (begin_nested)
("all", []), # 13: success recommendation (begin_nested)
("first", None), # 14: market price (begin_nested)
("all", []), # 15: recent permits (begin_nested)
("scalar", 0), # 16: geotech_risk
("all", []), # 17: neighbors
("first", None), # 18: overlap
]
def _execute_side_effect(*args: Any, **kwargs: Any) -> MagicMock:
idx = call_idx[0]
call_idx[0] += 1
if idx >= len(responses):
r = MagicMock()
r.mappings.return_value.first.return_value = None
r.mappings.return_value.all.return_value = []
r.scalar.return_value = 0
return r
kind, data = responses[idx]
r = MagicMock()
r.mappings.return_value.first.return_value = data
r.mappings.return_value.all.return_value = data if isinstance(data, list) else []
r.scalar.return_value = data if kind == "scalar" else 0
return r
db.execute.side_effect = _execute_side_effect
ctx = MagicMock()
ctx.__enter__ = MagicMock(return_value=ctx)
ctx.__exit__ = MagicMock(return_value=False)
db.begin_nested.return_value = ctx
return db
def _override_db(db: MagicMock):
def _get_db_override():
yield db
return _get_db_override
_ANALYZE_PATCHES = [
patch("app.api.v1.parcels._fetch_air_quality_sync", return_value=None),
patch("app.api.v1.parcels._fetch_weather_sync", return_value=None),
patch("app.api.v1.parcels._fetch_seasonal_weather_sync", return_value=None),
patch(
"app.api.v1.parcels.get_quarter_dump_data",
return_value={
"nspd_zoning": None,
"nspd_zouit_overlaps": [],
"nspd_engineering_nearby": [],
"nspd_dump": {"available": False, "stale": False, "harvest_triggered": False},
},
),
patch("app.api.v1.parcels.compute_velocity", return_value=None),
patch("app.api.v1.parcels.compute_gate_verdict", return_value={"verdict": "unknown"}),
]
def _start_analyze_patches() -> None:
for p in _ANALYZE_PATCHES:
p.start()
def _stop_analyze_patches() -> None:
for p in _ANALYZE_PATCHES:
p.stop()
def test_analyze_custom_poi_increases_score() -> None:
"""Custom POI с weight=+2 в 500м → score повышается, custom_poi_score_items непустой."""
from app.core.db import get_db
# Симулируем custom POI в 500м
_custom_overlap = {
"id": 42,
"name": "Мой парк",
"category": "park",
"weight": 2.0,
"lon": 60.607,
"lat": 56.840,
"distance_m": 500.0,
}
# decay = 1 - 500/1000 = 0.5; contribution = 2.0 * 0.5 = 1.0
db = _make_db_for_analyze()
app.dependency_overrides[get_db] = _override_db(db)
_start_analyze_patches()
try:
with patch("app.api.v1.parcels._get_custom_poi_overlaps", return_value=[_custom_overlap]):
client = TestClient(app)
resp = client.post(
f"/api/v1/parcels/{_CAD}/analyze",
headers={"X-Session-Id": _SESSION},
)
assert resp.status_code == 200, resp.text
body = resp.json()
# custom_poi_score_items должен содержать нашу точку
assert "custom_poi_score_items" in body
items = body["custom_poi_score_items"]
assert len(items) == 1
assert items[0]["label"] == "Мой парк"
assert items[0]["weight"] == pytest.approx(2.0)
assert items[0]["distance_m"] == 500
# contribution = 2.0 * 0.5 = 1.0
assert items[0]["contribution"] == pytest.approx(1.0, abs=0.01)
# score должен учитывать contribution custom POI
# (базовый score без POI = center_bonus из 1500м → 1.5; + custom 1.0 = 2.5)
assert body["score"] == pytest.approx(2.5, abs=0.1)
finally:
app.dependency_overrides.clear()
_stop_analyze_patches()
def test_analyze_without_session_id_no_custom_poi() -> None:
"""POST /analyze без X-Session-Id → custom_poi_score_items пустой."""
from app.core.db import get_db
db = _make_db_for_analyze()
app.dependency_overrides[get_db] = _override_db(db)
_start_analyze_patches()
try:
with patch("app.api.v1.parcels._get_custom_poi_overlaps", return_value=[]) as mock_overlaps:
client = TestClient(app)
resp = client.post(f"/api/v1/parcels/{_CAD}/analyze")
assert resp.status_code == 200, resp.text
body = resp.json()
# Без session_id custom_poi_score_items пустой (не вызываем get_overlaps)
assert body["custom_poi_score_items"] == []
mock_overlaps.assert_not_called()
finally:
app.dependency_overrides.clear()
_stop_analyze_patches()
def test_analyze_custom_poi_negative_weight_decreases_score() -> None:
"""Custom POI с weight=-3 в 500м → score уменьшается."""
from app.core.db import get_db
_custom_overlap = {
"id": 7,
"name": "Промзона",
"category": "industrial",
"weight": -3.0,
"lon": 60.607,
"lat": 56.840,
"distance_m": 500.0,
}
# contribution = -3.0 * 0.5 = -1.5
db = _make_db_for_analyze()
app.dependency_overrides[get_db] = _override_db(db)
_start_analyze_patches()
try:
with patch("app.api.v1.parcels._get_custom_poi_overlaps", return_value=[_custom_overlap]):
client = TestClient(app)
resp = client.post(
f"/api/v1/parcels/{_CAD}/analyze",
headers={"X-Session-Id": _SESSION},
)
assert resp.status_code == 200, resp.text
body = resp.json()
items = body["custom_poi_score_items"]
assert len(items) == 1
assert items[0]["contribution"] == pytest.approx(-1.5, abs=0.01)
# center_bonus = 1.5 (1500м), custom = -1.5 → итого ≈ 0
assert body["score"] == pytest.approx(0.0, abs=0.1)
finally:
app.dependency_overrides.clear()
_stop_analyze_patches()
# ── Service commit regression tests (#261) ────────────────────────────────────
#
# Проверяем что service-функции вызывают db.commit() после каждой мутации.
# Если db.commit() убрать из сервиса — тест упадёт на assert_called_once().
#
# Паттерн "два сеанса": write_db отдельный от последующего чтения.
# Эмулирует независимый SELECT после HTTP request close:
# get_db.finally → db.close() без commit → rollback pending tx,
# поэтому commit должен быть явным внутри каждой service-функции.
def _make_row_data(
poi_id: int = 1,
user_id: str = _SESSION,
name: str = "Тест",
weight: float = 1.5,
lon: float = 60.605,
lat: float = 56.838,
) -> dict[str, Any]:
return {
"id": poi_id,
"user_id": user_id,
"parcel_cad": None,
"name": name,
"category": "park",
"weight": weight,
"lon": lon,
"lat": lat,
"notes": None,
"created_at": _TS,
"updated_at": _TS,
}
def _make_db_with_row(row_data: dict[str, Any] | None) -> MagicMock:
"""Вернуть MagicMock-сессию где execute().mappings().first() → row_data mapping."""
db = MagicMock()
row_mock: MagicMock | None = None
if row_data is not None:
row_mock = MagicMock()
row_mock.__getitem__ = lambda self, k: row_data[k]
row_mock.get = lambda k, default=None: row_data.get(k, default)
mapping_mock = MagicMock()
mapping_mock.first.return_value = row_mock
exec_mock = MagicMock()
exec_mock.mappings.return_value = mapping_mock
exec_mock.first.return_value = row_mock # для DELETE RETURNING id (не mappings)
db.execute.return_value = exec_mock
return db
def test_service_create_custom_poi_commits() -> None:
"""create_custom_poi вызывает db.commit() — regression guard #261.
Если commit убрать: row существует только в рамках незакоммиченной транзакции,
при db.close() (get_db.finally) соединение вернётся в пул без commit → rollback.
"""
from app.services.site_finder.custom_pois import create_custom_poi
db = _make_db_with_row(_make_row_data())
payload = CustomPoiCreate(name="Тест", weight=1.5, lon=60.605, lat=56.838)
result = create_custom_poi(db, _SESSION, payload)
db.commit.assert_called_once()
assert result.id == 1
assert result.user_id == _SESSION
assert result.weight == pytest.approx(1.5)
def test_service_delete_custom_poi_commits_when_found() -> None:
"""delete_custom_poi вызывает db.commit() когда запись найдена — regression guard #261."""
from app.services.site_finder.custom_pois import delete_custom_poi
db = _make_db_with_row(_make_row_data())
db.execute.return_value.first.return_value = MagicMock() # RETURNING id truthy
deleted = delete_custom_poi(db, poi_id=1, user_id=_SESSION)
db.commit.assert_called_once()
assert deleted is True
def test_service_delete_custom_poi_no_commit_when_not_found() -> None:
"""delete_custom_poi НЕ вызывает db.commit() если запись не найдена."""
from app.services.site_finder.custom_pois import delete_custom_poi
db = _make_db_with_row(None)
db.execute.return_value.first.return_value = None # RETURNING id пусто
deleted = delete_custom_poi(db, poi_id=999, user_id=_SESSION)
db.commit.assert_not_called()
assert deleted is False
def test_service_update_custom_poi_commits_when_fields_given() -> None:
"""update_custom_poi вызывает db.commit() при наличии полей для обновления — regression #261."""
from app.services.site_finder.custom_pois import update_custom_poi
db = _make_db_with_row(_make_row_data(weight=2.0))
payload = CustomPoiUpdate(weight=2.0)
result = update_custom_poi(db, poi_id=1, user_id=_SESSION, payload=payload)
db.commit.assert_called_once()
assert result is not None
def test_service_update_custom_poi_no_commit_when_payload_empty() -> None:
"""update_custom_poi НЕ вызывает db.commit() если payload не содержит изменяемых полей."""
from app.services.site_finder.custom_pois import update_custom_poi
db = _make_db_with_row(_make_row_data())
# Все поля None → sets = ["updated_at = NOW()"], len == 1 → UPDATE не выполняется
payload = CustomPoiUpdate()
update_custom_poi(db, poi_id=1, user_id=_SESSION, payload=payload)
db.commit.assert_not_called()
def test_service_update_custom_poi_no_commit_when_not_found() -> None:
"""update_custom_poi НЕ вызывает db.commit() если POI не найдена."""
from app.services.site_finder.custom_pois import update_custom_poi
db = _make_db_with_row(None) # get_custom_poi вернёт None (POI не найдена)
payload = CustomPoiUpdate(weight=3.0)
result = update_custom_poi(db, poi_id=999, user_id=_SESSION, payload=payload)
db.commit.assert_not_called()
assert result is None

443
backend/tests/api/v1/test_parcel_best_layouts.py Normal file
View file

@ -0,0 +1,443 @@
"""Тесты для POST /api/v1/parcels/{cad_num}/best-layouts (Issue #113 Phase 2.1).
Mock-based — не требуют живой БД.
Паттерн mock DB: аналогично test_parcel_competitors.py — dependency_overrides[get_db].
Порядок вызовов в get_best_layouts (Fix SF-01 — inline velocity):
db.scalar() → MAX(snapshot_date) (только когда vel_rows non-empty)
db.execute() calls:
1. _PARCEL_CENTROID_SQL → .mappings().first()
2. _COMPETITORS_IN_RADIUS_SQL → .mappings().all()
3. _INLINE_VELOCITY_SQL → .mappings().all()
4. _SUPPLY_BATCH_SQL → .mappings().all() (пропускается если latest_snap is None)
"""
from __future__ import annotations
import datetime as dt
from unittest.mock import MagicMock
import pytest
from fastapi.testclient import TestClient
from app.main import app
# ── Фабрики mock-строк ────────────────────────────────────────────────────────
CAD_NUM = "66:41:0303161:123"
_TODAY = dt.date.today()
def _coord_row(lon: float = 60.6, lat: float = 56.85) -> MagicMock:
"""Центроид участка (EPSG:4326 lon/lat)."""
r = MagicMock()
r.__getitem__ = lambda self, k: {"center_lon": lon, "center_lat": lat}[k]
return r
def _obj_id_row(obj_id: int) -> MagicMock:
"""Строка obj_id из _COMPETITORS_IN_RADIUS_SQL."""
r = MagicMock()
r.__getitem__ = lambda self, k: {"obj_id": obj_id}[k]
return r
def _vel_row(
room_bucket: str = "2",
deals_window: float = 48.0,
avg_area: float = 55.0,
avg_price_rub: float | None = 120000.0,
obj_ids: list[int] | None = None,
window_start: dt.date | None = None,
window_end: dt.date | None = None,
) -> MagicMock:
"""Строка из _INLINE_VELOCITY_SQL (Fix SF-01: deals_window за честный интервал)."""
oids = obj_ids if obj_ids is not None else [1]
ws = window_start or _TODAY - dt.timedelta(days=90)
we = window_end or _TODAY
r = MagicMock()
r.__getitem__ = lambda self, k: {
"room_bucket": room_bucket,
"deals_window": deals_window,
"avg_area_m2": avg_area,
"avg_price_per_m2_rub": avg_price_rub,
"competitor_obj_ids": oids,
"competitor_count": len(oids),
"window_start": ws,
"window_end": we,
}[k]
return r
def _supply_row(rb: str, ab: str, units: int) -> MagicMock:
"""Строка из _SUPPLY_BATCH_SQL."""
r = MagicMock()
r.__getitem__ = lambda self, k: {"rb": rb, "ab": ab, "units": units}[k]
return r
# ── Построение mock DB ────────────────────────────────────────────────────────
def _make_db(
coord: MagicMock | None = None,
id_rows: list[MagicMock] | None = None,
vel_rows: list[MagicMock] | None = None,
supply_rows: list[MagicMock] | None = None,
latest_snap: dt.date | None = None,
) -> MagicMock:
"""Сконструировать mock Session.
db.scalar() возвращает latest_snap (MAX snapshot_date) — вызывается перед supply.
Порядок db.execute():
1. centroid → .mappings().first()
2. competitors-in-radius → .mappings().all()
3. velocity → .mappings().all()
4. supply → .mappings().all() (только если latest_snap is not None)
"""
db = MagicMock()
# db.scalar — pre-computed MAX(snapshot_date) для supply query
db.scalar.return_value = latest_snap if latest_snap is not None else _TODAY
results: list[MagicMock] = []
# 1: centroid
r0 = MagicMock()
r0.mappings.return_value.first.return_value = coord
results.append(r0)
# 2: competitors-in-radius
r1 = MagicMock()
r1.mappings.return_value.all.return_value = id_rows or []
results.append(r1)
# 3: velocity (only queried if id_rows non-empty)
r2 = MagicMock()
r2.mappings.return_value.all.return_value = vel_rows or []
results.append(r2)
# 4: supply
r3 = MagicMock()
r3.mappings.return_value.all.return_value = supply_rows or []
results.append(r3)
db.execute.side_effect = results
return db
def _override_db(db: MagicMock):
def _get_db_override():
yield db
return _get_db_override
def _post(client: TestClient, cad: str = CAD_NUM, **body_kwargs) -> dict:
payload = {"radius_km": 1.0, "time_window": "last_quarter", **body_kwargs}
resp = client.post(f"/api/v1/parcels/{cad}/best-layouts", json=payload)
return resp
# ── Тесты ─────────────────────────────────────────────────────────────────────
def test_parcel_not_found_404() -> None:
"""Если центроид не найден → 404."""
db = _make_db(coord=None)
from app.core.db import get_db
app.dependency_overrides[get_db] = _override_db(db)
try:
resp = _post(TestClient(app), cad="99:99:9999999:999")
assert resp.status_code == 404, resp.text
finally:
app.dependency_overrides.clear()
def test_empty_competitor_set_returns_low_confidence() -> None:
"""Нет конкурентов в радиусе → пустые top_layouts + confidence=low."""
db = _make_db(coord=_coord_row(), id_rows=[])
from app.core.db import get_db
app.dependency_overrides[get_db] = _override_db(db)
try:
resp = _post(TestClient(app))
assert resp.status_code == 200, resp.text
body = resp.json()
assert body["top_layouts"] == []
assert body["data_quality"]["confidence"] == "low"
assert body["data_quality"]["objects_total_in_radius"] == 0
rec = body["recommendation_for_tz"]
assert rec["based_on_obj_count"] == 0
assert rec["based_on_total_deals"] == 0
assert rec["mix"] == []
finally:
app.dependency_overrides.clear()
def test_three_obj_ids_ranking_and_pct_sum_100() -> None:
"""3 obj_id, 3 room_buckets — ranking по velocity, sum pct = 100.
last_quarter (3 мес): velocity = deals_window / 3.0
studio: 9/3=3.0, 1: 24/3=8.0, 2: 48/3=16.0 → rank1="2"
"""
id_rows = [_obj_id_row(1), _obj_id_row(2), _obj_id_row(3)]
vel_rows = [
_vel_row("studio", deals_window=9.0, avg_area=26.0, obj_ids=[1]),
_vel_row("1", deals_window=24.0, avg_area=40.0, obj_ids=[2]),
_vel_row("2", deals_window=48.0, avg_area=55.0, obj_ids=[3]),
]
supply_rows = [
_supply_row("studio", "<25", 20),
_supply_row("1", "40-60", 60),
_supply_row("2", "40-60", 80),
]
db = _make_db(coord=_coord_row(), id_rows=id_rows, vel_rows=vel_rows, supply_rows=supply_rows)
from app.core.db import get_db
app.dependency_overrides[get_db] = _override_db(db)
try:
resp = _post(TestClient(app), time_window="last_quarter")
assert resp.status_code == 200, resp.text
body = resp.json()
top = body["top_layouts"]
assert len(top) == 3
# rank 1 = самая высокая velocity (2-комн: 48/3=16.0 per month)
assert top[0]["rank"] == 1
assert top[0]["room_bucket"] == "2"
# все ранги уникальны
assert sorted(t["rank"] for t in top) == [1, 2, 3]
# sum pct = 100
mix = body["recommendation_for_tz"]["mix"]
assert sum(m["pct"] for m in mix) == 100
finally:
app.dependency_overrides.clear()
def test_exclude_competitor_obj_ids_filter() -> None:
"""exclude_competitor_obj_ids исключает obj_id: при all excluded → пустой ответ."""
# Если после исключения obj_id_list пуст → _empty_response → top_layouts=[]
id_rows = [_obj_id_row(20)] # единственный конкурент
db = _make_db(coord=_coord_row(), id_rows=id_rows, vel_rows=[])
from app.core.db import get_db
app.dependency_overrides[get_db] = _override_db(db)
try:
resp = _post(TestClient(app), exclude_competitor_obj_ids=[20])
assert resp.status_code == 200, resp.text
body = resp.json()
# После исключения obj_id=20 список пуст → пустой ответ
assert body["top_layouts"] == []
assert body["data_quality"]["confidence"] == "low"
# objects_total_in_radius = 1 (до исключения)
assert body["data_quality"]["objects_total_in_radius"] == 1
finally:
app.dependency_overrides.clear()
def test_min_velocity_per_month_filters_low_rows() -> None:
"""min_velocity_per_month=5 → строки с velocity<5 не попадают в top_layouts.
last_quarter (3 мес): studio=6/3=2.0 < 5.0 (убран), 1=30/3=10.0 > 5.0 (остаётся).
"""
id_rows = [_obj_id_row(1), _obj_id_row(2)]
vel_rows = [
_vel_row("studio", deals_window=6.0, obj_ids=[1]),
_vel_row("1", deals_window=30.0, obj_ids=[2]),
]
db = _make_db(coord=_coord_row(), id_rows=id_rows, vel_rows=vel_rows)
from app.core.db import get_db
app.dependency_overrides[get_db] = _override_db(db)
try:
resp = _post(TestClient(app), min_velocity_per_month=5.0)
assert resp.status_code == 200, resp.text
body = resp.json()
top = body["top_layouts"]
assert len(top) == 1
assert top[0]["room_bucket"] == "1"
assert top[0]["velocity_per_month"] == pytest.approx(10.0)
finally:
app.dependency_overrides.clear()
def test_time_window_velocity_scaling() -> None:
"""last_month vs last_year дают разный velocity_per_month для одних deals."""
# sum_deals=24 → last_month: 24/24=1.0, last_year: 24/2=12.0
id_rows = [_obj_id_row(1)]
vel_rows_fixed = [_vel_row("2", sum_deals=24.0, obj_ids=[1])]
from app.core.db import get_db
# last_month
db_m = _make_db(coord=_coord_row(), id_rows=id_rows, vel_rows=vel_rows_fixed)
app.dependency_overrides[get_db] = _override_db(db_m)
try:
resp_m = _post(TestClient(app), time_window="last_month")
assert resp_m.status_code == 200, resp_m.text
v_month = resp_m.json()["top_layouts"][0]["velocity_per_month"]
finally:
app.dependency_overrides.clear()
# last_year
db_y = _make_db(coord=_coord_row(), id_rows=id_rows, vel_rows=vel_rows_fixed)
app.dependency_overrides[get_db] = _override_db(db_y)
try:
resp_y = _post(TestClient(app), time_window="last_year")
assert resp_y.status_code == 200, resp_y.text
v_year = resp_y.json()["top_layouts"][0]["velocity_per_month"]
finally:
app.dependency_overrides.clear()
# last_year velocity должна быть выше (делитель меньше: 2 vs 24)
assert v_year > v_month
assert v_month == pytest.approx(1.0)
assert v_year == pytest.approx(12.0)
def test_obj_class_filter_passes_through() -> None:
"""obj_class_filter передаётся в SQL — endpoint не ломается, возвращает 200."""
db = _make_db(
coord=_coord_row(),
id_rows=[_obj_id_row(5)],
vel_rows=[_vel_row("2", obj_ids=[5])],
supply_rows=[],
)
from app.core.db import get_db
app.dependency_overrides[get_db] = _override_db(db)
try:
resp = _post(TestClient(app), obj_class_filter="comfort")
assert resp.status_code == 200, resp.text
body = resp.json()
assert len(body["top_layouts"]) > 0
finally:
app.dependency_overrides.clear()
def test_mv_empty_for_competitors_returns_empty_top_layouts() -> None:
"""Конкуренты есть в радиусе, но MV пустой → top_layouts=[], data_quality.confidence=low."""
id_rows = [_obj_id_row(1), _obj_id_row(2)]
db = _make_db(coord=_coord_row(), id_rows=id_rows, vel_rows=[])
from app.core.db import get_db
app.dependency_overrides[get_db] = _override_db(db)
try:
resp = _post(TestClient(app))
assert resp.status_code == 200, resp.text
body = resp.json()
assert body["top_layouts"] == []
dq = body["data_quality"]
assert dq["objects_total_in_radius"] == 2
assert dq["objects_with_velocity_data"] == 0
assert dq["confidence"] == "low"
finally:
app.dependency_overrides.clear()
def test_target_total_flats_fills_abs_units() -> None:
"""target_total_flats=100 → abs_units заполнен в mix, sum примерно = 100."""
id_rows = [_obj_id_row(1), _obj_id_row(2)]
vel_rows = [
_vel_row("1", sum_deals=60.0, obj_ids=[1]),
_vel_row("2", sum_deals=40.0, obj_ids=[2]),
]
db = _make_db(coord=_coord_row(), id_rows=id_rows, vel_rows=vel_rows)
from app.core.db import get_db
app.dependency_overrides[get_db] = _override_db(db)
try:
resp = _post(TestClient(app), target_total_flats=100)
assert resp.status_code == 200, resp.text
mix = resp.json()["recommendation_for_tz"]["mix"]
# все abs_units заполнены
for m in mix:
assert m["abs_units"] is not None
# сумма abs_units близка к 100 (round-off ±1)
total_abs = sum(m["abs_units"] for m in mix)
assert 98 <= total_abs <= 102
finally:
app.dependency_overrides.clear()
def test_sold_pct_clamped_at_100_and_is_oversold_flag() -> None:
"""raw sold_pct > 100 → returned sold_pct_of_supply=100.0, is_oversold=True."""
id_rows = [_obj_id_row(1)]
# sum_deals=199, supply=100 → raw = 199% (несопоставимые окна)
vel_rows = [_vel_row("2", sum_deals=199.0, obj_ids=[1])]
supply_rows = [_supply_row("2", "40-60", 100)]
db = _make_db(
coord=_coord_row(),
id_rows=id_rows,
vel_rows=vel_rows,
supply_rows=supply_rows,
latest_snap=dt.date.today(),
)
from app.core.db import get_db
app.dependency_overrides[get_db] = _override_db(db)
try:
resp = _post(TestClient(app))
assert resp.status_code == 200, resp.text
body = resp.json()
top = body["top_layouts"]
assert len(top) == 1
row = top[0]
assert row["sold_pct_of_supply"] == 100.0, "sold_pct_of_supply должен быть clamped до 100"
assert row["is_oversold"] is True, "is_oversold должен быть True когда raw > 100"
finally:
app.dependency_overrides.clear()
def test_sold_pct_below_100_is_not_oversold() -> None:
"""raw sold_pct <= 100 → sold_pct_of_supply возвращается as-is, is_oversold=False."""
id_rows = [_obj_id_row(1)]
# sum_deals=50, supply=100 → raw = 50%
vel_rows = [_vel_row("1", sum_deals=50.0, obj_ids=[1])]
supply_rows = [_supply_row("1", "40-60", 100)]
db = _make_db(
coord=_coord_row(),
id_rows=id_rows,
vel_rows=vel_rows,
supply_rows=supply_rows,
latest_snap=dt.date.today(),
)
from app.core.db import get_db
app.dependency_overrides[get_db] = _override_db(db)
try:
resp = _post(TestClient(app))
assert resp.status_code == 200, resp.text
body = resp.json()
top = body["top_layouts"]
assert len(top) == 1
row = top[0]
assert row["sold_pct_of_supply"] == pytest.approx(50.0)
assert row["is_oversold"] is False
finally:
app.dependency_overrides.clear()
def test_filter_competitor_obj_ids_applied() -> None:
"""filter_competitor_obj_ids=[1] оставляет только obj_id=1."""
id_rows = [_obj_id_row(1), _obj_id_row(2), _obj_id_row(3)]
# После фильтрации остаётся только obj_id=1, velocity запрос получит [1]
vel_rows = [_vel_row("2", sum_deals=24.0, obj_ids=[1])]
db = _make_db(coord=_coord_row(), id_rows=id_rows, vel_rows=vel_rows)
from app.core.db import get_db
app.dependency_overrides[get_db] = _override_db(db)
try:
resp = _post(TestClient(app), filter_competitor_obj_ids=[1])
assert resp.status_code == 200, resp.text
body = resp.json()
top = body["top_layouts"]
assert len(top) >= 1
# competitor_obj_ids должен содержать только 1
for row in top:
for oid in row["competitor_obj_ids"]:
assert oid == 1
finally:
app.dependency_overrides.clear()

152
backend/tests/api/v1/test_parcel_by_bbox.py Normal file
View file

@ -0,0 +1,152 @@
"""Тесты GET /parcels/by-bbox (SF-B1).
Проверяют:
1. Валидный bbox → 200 + корректная структура ответа.
2. Некорректный bbox (min_lat >= max_lat) → 400.
3. user_id передан → status заполняется из mock DB; без user_id → status=null.
"""
from __future__ import annotations
from typing import Any
from unittest.mock import MagicMock, patch
import pytest
from fastapi.testclient import TestClient
from app.main import app
def _make_db_row(
cad_num: str = "66:41:0204016:10",
centroid_lat: float = 56.83,
centroid_lon: float = 60.64,
area_m2: float = 1200.0,
land_category: str | None = "land_residential",
user_status: str | None = None,
) -> dict[str, Any]:
return {
"cad_num": cad_num,
"centroid_lat": centroid_lat,
"centroid_lon": centroid_lon,
"area_m2": area_m2,
"land_category": land_category,
"user_status": user_status,
}
def _build_mock_db(rows: list[dict[str, Any]]) -> MagicMock:
"""Сконструировать mock Session, возвращающий rows при execute().mappings().all()."""
mock_db = MagicMock()
mappings_mock = MagicMock()
mappings_mock.all.return_value = rows
execute_result = MagicMock()
execute_result.mappings.return_value = mappings_mock
mock_db.execute.return_value = execute_result
return mock_db
@pytest.fixture()
def client() -> TestClient:
return TestClient(app)
# ── Test 1: валидный bbox возвращает корректную структуру ──────────────────
def test_by_bbox_valid_returns_structure(client: TestClient) -> None:
rows = [_make_db_row()]
mock_db = _build_mock_db(rows)
with patch("app.api.v1.parcels.get_db", return_value=iter([mock_db])):
resp = client.get(
"/api/v1/parcels/by-bbox",
params={
"min_lat": 56.80,
"min_lon": 60.60,
"max_lat": 56.90,
"max_lon": 60.70,
"limit": 200,
},
)
assert resp.status_code == 200
body = resp.json()
assert "parcels" in body
assert "count" in body
assert "limit" in body
assert "bbox_area_km2" in body
assert body["count"] == 1
assert body["limit"] == 200
parcel = body["parcels"][0]
assert parcel["cad_num"] == "66:41:0204016:10"
assert parcel["centroid_lat"] == pytest.approx(56.83, abs=0.01)
assert parcel["centroid_lon"] == pytest.approx(60.64, abs=0.01)
assert parcel["area_m2"] == pytest.approx(1200.0)
assert parcel["land_category"] == "land_residential"
assert parcel["status"] is None # user_id не передан
assert parcel["last_analysis_date"] is None
# ── Test 2: некорректный bbox → 400 ───────────────────────────────────────
def test_by_bbox_invalid_bbox_returns_400(client: TestClient) -> None:
resp = client.get(
"/api/v1/parcels/by-bbox",
params={
"min_lat": 56.90, # min >= max → ошибка
"min_lon": 60.60,
"max_lat": 56.80,
"max_lon": 60.70,
},
)
assert resp.status_code == 400
assert "bbox" in resp.json()["detail"].lower()
# ── Test 3: user_id → status из БД; без user_id → status null ─────────────
def test_by_bbox_status_overlay_with_user_id(client: TestClient) -> None:
rows = [_make_db_row(user_status="in_work")]
mock_db = _build_mock_db(rows)
with patch("app.api.v1.parcels.get_db", return_value=iter([mock_db])):
resp = client.get(
"/api/v1/parcels/by-bbox",
params={
"min_lat": 56.80,
"min_lon": 60.60,
"max_lat": 56.90,
"max_lon": 60.70,
"user_id": "user-abc-123",
},
)
assert resp.status_code == 200
parcel = resp.json()["parcels"][0]
assert parcel["status"] == "in_work"
def test_by_bbox_no_user_id_status_is_null(client: TestClient) -> None:
rows = [_make_db_row(user_status="favorite")] # DB вернёт статус
mock_db = _build_mock_db(rows)
with patch("app.api.v1.parcels.get_db", return_value=iter([mock_db])):
resp = client.get(
"/api/v1/parcels/by-bbox",
params={
"min_lat": 56.80,
"min_lon": 60.60,
"max_lat": 56.90,
"max_lon": 60.70,
# user_id не передан
},
)
assert resp.status_code == 200
parcel = resp.json()["parcels"][0]
# Без user_id статус принудительно null (endpoint не раскрывает чужие статусы)
assert parcel["status"] is None

390
backend/tests/api/v1/test_parcel_competitors.py Normal file
View file

@ -0,0 +1,390 @@
"""Тесты для POST /api/v1/parcels/{cad_num}/competitors (Issue #112).
Mock-based — не требуют живой БД.
Паттерн mock DB: аналогично test_admin_cadastre.py — dependency_overrides[get_db].
"""
from __future__ import annotations
from unittest.mock import MagicMock
import pytest
from fastapi.testclient import TestClient
from app.main import app
# ── Фабрики mock-строк ────────────────────────────────────────────────────────
def _coord_row(lat: float = 56.838, lon: float = 60.605) -> MagicMock:
"""Строка центроида участка."""
r = MagicMock()
r.__getitem__ = lambda self, k: {"lat": lat, "lon": lon}[k]
return r
def _obj_row(
obj_id: int = 1,
distance_m: float = 400.0,
site_status: str = "sales",
obj_class: str | None = "comfort",
velocity: float = 5.0,
flat_count: int | None = 200,
) -> MagicMock:
r = MagicMock()
r.__getitem__ = lambda self, k: {
"obj_id": obj_id,
"comm_name": f"ЖК-{obj_id}",
"dev_name": "TestDev",
"obj_class": obj_class,
"latitude": 56.838 + distance_m / 1_000_000,
"longitude": 60.605 + distance_m / 1_000_000,
"flat_count": flat_count,
"site_status": site_status,
"distance_m": distance_m,
"velocity_per_month": velocity,
}[k]
return r
def _price_row(obj_id: int, price: float) -> MagicMock:
r = MagicMock()
r.__getitem__ = lambda self, k: {"obj_id": obj_id, "avg_price_per_m2": price}[k]
return r
# ── Построение mock DB ────────────────────────────────────────────────────────
def _make_db(
coord: MagicMock | None = None,
obj_rows: list[MagicMock] | None = None,
price_rows: list[MagicMock] | None = None,
) -> MagicMock:
"""Сконструировать mock Session.
Порядок вызовов execute:
1. centroid query → coord
2. competitors query → obj_rows
3. avg_price query → price_rows
"""
db = MagicMock()
results: list[MagicMock] = []
for rows, is_first in [
(coord, True),
(obj_rows or [], False),
(price_rows or [], False),
]:
result = MagicMock()
if is_first:
# centroid → .mappings().first()
result.mappings.return_value.first.return_value = rows
else:
result.mappings.return_value.all.return_value = rows
results.append(result)
db.execute.side_effect = results
return db
def _override_db(db: MagicMock):
def _get_db_override():
yield db
return _get_db_override
# ── Тесты ─────────────────────────────────────────────────────────────────────
def test_competitors_basic() -> None:
"""3 конкурента → корректная форма ответа, сортировка по distance."""
rows = [
_obj_row(obj_id=1, distance_m=200.0, velocity=4.0),
_obj_row(obj_id=2, distance_m=500.0, velocity=6.0),
_obj_row(obj_id=3, distance_m=900.0, velocity=2.0),
]
db = _make_db(coord=_coord_row(), obj_rows=rows)
from app.core.db import get_db
app.dependency_overrides[get_db] = _override_db(db)
try:
client = TestClient(app)
resp = client.post(
"/api/v1/parcels/66:41:0303161:123/competitors",
json={"radius_km": 1.0, "time_window": "last_quarter"},
)
assert resp.status_code == 200, resp.text
body = resp.json()
assert "competitors" in body
assert "summary" in body
assert len(body["competitors"]) == 3
# первый должен быть ближайшим
assert body["competitors"][0]["obj_id"] == 1
assert body["competitors"][0]["distance_m"] == pytest.approx(200.0)
# все поля присутствуют
first = body["competitors"][0]
for key in (
"obj_id",
"comm_name",
"dev_name",
"obj_class",
"distance_m",
"lat",
"lng",
"stage",
"flats_total",
"flats_sold",
"sold_pct",
"velocity_per_month",
"avg_price_per_m2",
"is_active",
):
assert key in first, f"missing key: {key}"
finally:
app.dependency_overrides.clear()
def test_competitors_summary_calc() -> None:
"""summary: total_competitors, active_count, weighted_avg_velocity корректны."""
rows = [
_obj_row(obj_id=1, site_status="sales", velocity=10.0),
_obj_row(obj_id=2, site_status="construction", velocity=6.0),
_obj_row(obj_id=3, site_status="completed", velocity=2.0),
]
db = _make_db(coord=_coord_row(), obj_rows=rows)
from app.core.db import get_db
app.dependency_overrides[get_db] = _override_db(db)
try:
client = TestClient(app)
resp = client.post(
"/api/v1/parcels/66:41:0303161:5/competitors",
json={"radius_km": 1.0, "time_window": "last_quarter"},
)
assert resp.status_code == 200, resp.text
summary = resp.json()["summary"]
assert summary["total_competitors"] == 3
assert summary["active_count"] == 2 # sales + construction
# avg velocity = (10+6+2)/3 = 6.0
assert summary["weighted_avg_velocity"] == pytest.approx(6.0)
assert summary["radius_km"] == pytest.approx(1.0)
assert summary["time_window"] == "last_quarter"
finally:
app.dependency_overrides.clear()
def test_competitors_exclude_obj_ids() -> None:
"""exclude_obj_ids исключает указанные ЖК из результата."""
rows = [
_obj_row(obj_id=1, distance_m=100.0),
_obj_row(obj_id=2, distance_m=200.0),
_obj_row(obj_id=3, distance_m=300.0),
]
db = _make_db(coord=_coord_row(), obj_rows=rows)
from app.core.db import get_db
app.dependency_overrides[get_db] = _override_db(db)
try:
client = TestClient(app)
resp = client.post(
"/api/v1/parcels/66:41:0303161:5/competitors",
json={"radius_km": 1.0, "time_window": "last_quarter", "exclude_obj_ids": [2]},
)
assert resp.status_code == 200, resp.text
ids = [c["obj_id"] for c in resp.json()["competitors"]]
assert 2 not in ids
assert 1 in ids
assert 3 in ids
finally:
app.dependency_overrides.clear()
def test_competitors_obj_class_filter() -> None:
"""obj_class_filter=economy — SQL получает параметр; Python-сторона не ломается."""
rows = [_obj_row(obj_id=10, obj_class="economy")]
db = _make_db(coord=_coord_row(), obj_rows=rows)
from app.core.db import get_db
app.dependency_overrides[get_db] = _override_db(db)
try:
client = TestClient(app)
resp = client.post(
"/api/v1/parcels/66:41:0303161:5/competitors",
json={"radius_km": 1.0, "time_window": "last_quarter", "obj_class_filter": "economy"},
)
assert resp.status_code == 200, resp.text
comps = resp.json()["competitors"]
assert len(comps) == 1
assert comps[0]["obj_class"] == "economy"
finally:
app.dependency_overrides.clear()
def test_competitors_time_window_velocity() -> None:
"""time_window влияет на velocity_per_month (last_month vs last_year)."""
# Здесь проверяем, что endpoint принимает оба варианта без ошибок
# и возвращает velocity из mock-строки (DB-расчёт мокирован).
rows_month = [_obj_row(obj_id=1, velocity=12.0)]
rows_year = [_obj_row(obj_id=1, velocity=3.0)]
from app.core.db import get_db
# last_month
db_m = _make_db(coord=_coord_row(), obj_rows=rows_month)
app.dependency_overrides[get_db] = _override_db(db_m)
try:
client = TestClient(app)
resp_m = client.post(
"/api/v1/parcels/66:41:0303161:5/competitors",
json={"radius_km": 1.0, "time_window": "last_month"},
)
assert resp_m.status_code == 200, resp_m.text
finally:
app.dependency_overrides.clear()
# last_year
db_y = _make_db(coord=_coord_row(), obj_rows=rows_year)
app.dependency_overrides[get_db] = _override_db(db_y)
try:
client = TestClient(app)
resp_y = client.post(
"/api/v1/parcels/66:41:0303161:5/competitors",
json={"radius_km": 1.0, "time_window": "last_year"},
)
assert resp_y.status_code == 200, resp_y.text
v_month = resp_m.json()["competitors"][0]["velocity_per_month"]
v_year = resp_y.json()["competitors"][0]["velocity_per_month"]
# month velocity выше чем year в нашем моке
assert v_month > v_year
finally:
app.dependency_overrides.clear()
def test_competitors_parcel_not_found_404() -> None:
"""Если центроид участка не найден → 404."""
db = _make_db(coord=None) # first() вернёт None
from app.core.db import get_db
app.dependency_overrides[get_db] = _override_db(db)
try:
client = TestClient(app)
resp = client.post(
"/api/v1/parcels/99:99:9999999:999/competitors",
json={"radius_km": 1.0, "time_window": "last_quarter"},
)
assert resp.status_code == 404, resp.text
finally:
app.dependency_overrides.clear()
def test_competitors_empty_radius() -> None:
"""Нет конкурентов в радиусе → пустой список + summary с нулями."""
db = _make_db(coord=_coord_row(), obj_rows=[])
from app.core.db import get_db
app.dependency_overrides[get_db] = _override_db(db)
try:
client = TestClient(app)
resp = client.post(
"/api/v1/parcels/66:41:0303161:5/competitors",
json={"radius_km": 0.1, "time_window": "last_quarter"},
)
assert resp.status_code == 200, resp.text
body = resp.json()
assert body["competitors"] == []
assert body["summary"]["total_competitors"] == 0
assert body["summary"]["active_count"] == 0
assert body["summary"]["weighted_avg_velocity"] == pytest.approx(0.0)
finally:
app.dependency_overrides.clear()
def test_competitors_sold_pct_null() -> None:
"""sold_pct и flats_sold — None (MVP: данные недоступны из domrf_kn_objects).
Полный расчёт продаж требует JOIN с domrf_kn_flats COUNT WHERE status='sold'
— отложен за пределы текущего PR.
"""
rows = [_obj_row(obj_id=1, flat_count=200)]
db = _make_db(coord=_coord_row(), obj_rows=rows)
from app.core.db import get_db
app.dependency_overrides[get_db] = _override_db(db)
try:
client = TestClient(app)
resp = client.post(
"/api/v1/parcels/66:41:0303161:5/competitors",
json={"radius_km": 1.0, "time_window": "last_quarter"},
)
assert resp.status_code == 200, resp.text
comp = resp.json()["competitors"][0]
assert comp["flats_sold"] is None
assert comp["sold_pct"] is None
assert comp["flats_total"] == 200
finally:
app.dependency_overrides.clear()
def test_competitors_avg_price_populated() -> None:
"""avg_price_per_m2 не None если domrf_kn_flats возвращает строки с ценой.
Регрессионный тест для Issue #227: фильтр status='sold' давал 0 строк
(поле status в domrf_kn_flats 99.8% NULL). После фикса — убран, AVG
считается по всем квартирам с known price_per_m2.
"""
rows = [_obj_row(obj_id=1)]
price_rows = [_price_row(obj_id=1, price=150_000.0)]
db = _make_db(coord=_coord_row(), obj_rows=rows, price_rows=price_rows)
from app.core.db import get_db
app.dependency_overrides[get_db] = _override_db(db)
try:
client = TestClient(app)
resp = client.post(
"/api/v1/parcels/66:41:0303161:5/competitors",
json={"radius_km": 1.0, "time_window": "last_quarter"},
)
assert resp.status_code == 200, resp.text
comp = resp.json()["competitors"][0]
assert comp["avg_price_per_m2"] == pytest.approx(
150_000.0
), "avg_price_per_m2 должен быть не None — регрессия #227 status='sold' filter"
finally:
app.dependency_overrides.clear()
def test_competitors_is_active_flag() -> None:
"""is_active=True для sales/construction, False для completed/null."""
rows = [
_obj_row(obj_id=1, site_status="sales"),
_obj_row(obj_id=2, site_status="construction"),
_obj_row(obj_id=3, site_status="completed"),
]
db = _make_db(coord=_coord_row(), obj_rows=rows)
from app.core.db import get_db
app.dependency_overrides[get_db] = _override_db(db)
try:
client = TestClient(app)
resp = client.post(
"/api/v1/parcels/66:41:0303161:5/competitors",
json={"radius_km": 1.0, "time_window": "last_quarter"},
)
assert resp.status_code == 200, resp.text
comps = {c["obj_id"]: c for c in resp.json()["competitors"]}
assert comps[1]["is_active"] is True
assert comps[2]["is_active"] is True
assert comps[3]["is_active"] is False
finally:
app.dependency_overrides.clear()

267
backend/tests/api/v1/test_parcel_connection_points.py Normal file
View file

@ -0,0 +1,267 @@
"""Тесты для GET /{cad_num}/connection-points (issue #115).
Использует FastAPI TestClient с mock DB — без реального Postgres.
"""
from __future__ import annotations
from typing import Any
from unittest.mock import MagicMock, patch
import pytest
from fastapi.testclient import TestClient
from app.main import app
# ── Helpers ───────────────────────────────────────────────────────────────────
_VALID_CAD = "66:41:0204016:10"
_QUARTER = "66:41:0204016"
def _mock_get_db(db: Any):
"""FastAPI dependency override factory."""
def _get_db_override():
yield db
return _get_db_override
def _make_structure(distance_m: float = 120.5) -> dict[str, Any]:
return {
"name": "ТП-101",
"type": "Трансформаторная подстанция",
"cad_num": None,
"distance_to_boundary_m": distance_m,
"geometry_geojson": {"type": "Point", "coordinates": [60.6, 56.8]},
"readable_address": None,
"raw_props": {"name": "ТП-101"},
"source": "nspd_36328",
}
def _make_zouit_overlap() -> dict[str, Any]:
return {
"reg_numb_border": "RN-001",
"type_zone": "Охранная зона ЛЭП",
"subcategory": 5,
"intersects_parcel": True,
"geometry_geojson": {"type": "Polygon", "coordinates": [[]]},
"raw_props": {"type_zone": "Охранная зона ЛЭП"},
"source": "nspd_37578",
}
def _make_summary(
nearest: float | None = 120.5,
in_zone: bool = False,
zones_count: int = 0,
total: int = 1,
) -> dict[str, Any]:
return {
"nearest_structure_distance_m": nearest,
"in_protection_zone": in_zone,
"protection_zones_intersecting": zones_count,
"total_structures_in_radius": total,
}
def _make_full_response(
structures: list[dict[str, Any]] | None = None,
overlaps: list[dict[str, Any]] | None = None,
summary: dict[str, Any] | None = None,
dump_available: bool = True,
dump_fetched_at: str | None = "2026-05-01T12:00:00+00:00",
) -> dict[str, Any]:
if structures is None:
structures = [_make_structure()]
if overlaps is None:
overlaps = []
if summary is None:
summary = _make_summary(total=len(structures))
return {
"engineering_structures": structures,
"zouit_engineering_overlaps": overlaps,
"summary": summary,
"dump_available": dump_available,
"dump_fetched_at": dump_fetched_at,
}
# ── Tests ─────────────────────────────────────────────────────────────────────
def test_connection_points_no_dump_returns_empty() -> None:
"""Квартал без dump → dump_available=false, пустые массивы, 200 OK."""
from app.core.db import get_db
db = MagicMock()
app.dependency_overrides[get_db] = _mock_get_db(db)
empty_response = {
"engineering_structures": [],
"zouit_engineering_overlaps": [],
"summary": {
"nearest_structure_distance_m": None,
"in_protection_zone": False,
"protection_zones_intersecting": 0,
"total_structures_in_radius": 0,
},
"dump_available": False,
"dump_fetched_at": None,
}
try:
with patch(
"app.api.v1.parcels.get_connection_points",
return_value=empty_response,
):
client = TestClient(app)
response = client.get(f"/api/v1/parcels/{_VALID_CAD}/connection-points")
assert response.status_code == 200, response.text
body = response.json()
assert body["dump_available"] is False
assert body["engineering_structures"] == []
assert body["zouit_engineering_overlaps"] == []
assert body["summary"]["total_structures_in_radius"] == 0
assert body["summary"]["nearest_structure_distance_m"] is None
finally:
app.dependency_overrides.clear()
def test_connection_points_parcel_not_found_404() -> None:
"""cad_num не найден в БД (ValueError из сервиса) → 404."""
from app.core.db import get_db
db = MagicMock()
app.dependency_overrides[get_db] = _mock_get_db(db)
try:
with patch(
"app.api.v1.parcels.get_connection_points",
side_effect=ValueError("Участок '66:41:9999999:1' не найден в БД"),
):
client = TestClient(app)
response = client.get("/api/v1/parcels/66:41:9999999:1/connection-points")
assert response.status_code == 404
assert "не найден" in response.json()["detail"]
finally:
app.dependency_overrides.clear()
def test_connection_points_filters_by_radius() -> None:
"""Фичи за radius_m не попадают в ответ (сервис фильтрует).
Мокаем сервис: при radius_m=100 возвращаем только близкую структуру,
при radius_m=500 — обе. Проверяем что endpoint передаёт radius_m в сервис.
"""
from app.core.db import get_db
db = MagicMock()
app.dependency_overrides[get_db] = _mock_get_db(db)
close_only = _make_full_response(structures=[_make_structure(distance_m=80.0)])
far_included = _make_full_response(
structures=[_make_structure(distance_m=80.0), _make_structure(distance_m=450.0)],
summary=_make_summary(nearest=80.0, total=2),
)
try:
with patch("app.api.v1.parcels.get_connection_points") as mock_svc:
mock_svc.return_value = close_only
client = TestClient(app)
r100 = client.get(f"/api/v1/parcels/{_VALID_CAD}/connection-points?radius_m=100")
assert r100.status_code == 200
assert r100.json()["summary"]["total_structures_in_radius"] == 1
# Проверяем что radius_m=100 передан в сервис
call_kwargs = mock_svc.call_args
assert call_kwargs[0][2] == 100 or call_kwargs[1].get("radius_m") == 100
mock_svc.return_value = far_included
r500 = client.get(f"/api/v1/parcels/{_VALID_CAD}/connection-points?radius_m=500")
assert r500.status_code == 200
assert r500.json()["summary"]["total_structures_in_radius"] == 2
finally:
app.dependency_overrides.clear()
def test_connection_points_zouit_intersects_flag() -> None:
"""zouit_engineering_overlaps содержат intersects_parcel=true и правильные поля."""
from app.core.db import get_db
db = MagicMock()
app.dependency_overrides[get_db] = _mock_get_db(db)
overlap = _make_zouit_overlap()
full = _make_full_response(
overlaps=[overlap],
summary=_make_summary(in_zone=True, zones_count=1),
)
try:
with patch(
"app.api.v1.parcels.get_connection_points",
return_value=full,
):
client = TestClient(app)
response = client.get(f"/api/v1/parcels/{_VALID_CAD}/connection-points")
assert response.status_code == 200
body = response.json()
assert body["summary"]["in_protection_zone"] is True
assert body["summary"]["protection_zones_intersecting"] == 1
overlaps = body["zouit_engineering_overlaps"]
assert len(overlaps) == 1
assert overlaps[0]["intersects_parcel"] is True
assert overlaps[0]["reg_numb_border"] == "RN-001"
assert overlaps[0]["source"] == "nspd_37578"
finally:
app.dependency_overrides.clear()
def test_summary_nearest_distance() -> None:
"""summary.nearest_structure_distance_m = расстояние до ближайшей структуры."""
from app.core.db import get_db
db = MagicMock()
app.dependency_overrides[get_db] = _mock_get_db(db)
s1 = _make_structure(distance_m=42.7)
s2 = _make_structure(distance_m=180.0)
full = _make_full_response(
structures=[s1, s2],
summary=_make_summary(nearest=42.7, total=2),
)
try:
with patch(
"app.api.v1.parcels.get_connection_points",
return_value=full,
):
client = TestClient(app)
response = client.get(f"/api/v1/parcels/{_VALID_CAD}/connection-points")
assert response.status_code == 200
body = response.json()
assert body["summary"]["nearest_structure_distance_m"] == pytest.approx(42.7)
assert body["summary"]["total_structures_in_radius"] == 2
assert body["engineering_structures"][0]["distance_to_boundary_m"] == pytest.approx(42.7)
finally:
app.dependency_overrides.clear()
def test_connection_points_radius_out_of_range_422() -> None:
"""radius_m=10 (< min 50) → 422 Unprocessable Entity."""
client = TestClient(app)
response = client.get(f"/api/v1/parcels/{_VALID_CAD}/connection-points?radius_m=10")
assert response.status_code == 422
def test_connection_points_radius_too_large_422() -> None:
"""radius_m=5000 (> max 2000) → 422 Unprocessable Entity."""
client = TestClient(app)
response = client.get(f"/api/v1/parcels/{_VALID_CAD}/connection-points?radius_m=5000")
assert response.status_code == 422

0
backend/tests/integration/__init__.py Normal file
View file

153
backend/tests/integration/conftest.py Normal file
View file

@ -0,0 +1,153 @@
"""Integration test fixtures для phantom column gate.
Использует реальный PostgreSQL через SSH-туннель (localhost:15432).
Тесты пропускаются если TEST_DATABASE_URL не задан.
Подход (Option B): временная schema в prod-Postgres (не трогаем данные).
1. CREATE SCHEMA test_phantom_<random>
2. Копируем DDL public → временная schema через pg_dump --schema-only
3. Выполняем EXPLAIN-запросы (не EXECUTE — readonly)
4. DROP SCHEMA CASCADE после теста
Credentials: TEST_DATABASE_URL (env var, только через SSH-туннель).
Пример: postgresql+psycopg://user:pass@localhost:15432/gendesign
"""
from __future__ import annotations
import logging
import os
import random
import string
import subprocess
from collections.abc import Generator
import pytest
from sqlalchemy import create_engine, text
from sqlalchemy.orm import Session, sessionmaker
logger = logging.getLogger(__name__)
_TEST_DATABASE_URL = os.environ.get("TEST_DATABASE_URL", "")
# Используется во всех тестах этого пакета: пропускаем без TEST_DATABASE_URL
_SKIP_REASON = "TEST_DATABASE_URL не задан — интеграционные тесты требуют SSH-туннель"
requires_test_db = pytest.mark.skipif(
not _TEST_DATABASE_URL,
reason=_SKIP_REASON,
)
def _random_suffix(n: int = 8) -> str:
return "".join(random.choices(string.ascii_lowercase + string.digits, k=n))
def _pg_dump_schema_only(db_url: str, schema: str = "public") -> str:
"""Получить DDL через pg_dump --schema-only --schema=<schema>.
Возвращает SQL как строку. Требует pg_dump в PATH.
Credentials извлекаются из db_url.
Fallback: если pg_dump недоступен — возвращает пустую строку,
и тогда тест работает без создания отдельной schema (тестирует прямо в public).
"""
try:
result = subprocess.run(
[
"pg_dump",
"--schema-only",
f"--schema={schema}",
"--no-owner",
"--no-acl",
db_url.replace("+psycopg", ""), # pg_dump не понимает +psycopg dialect
],
capture_output=True,
text=True,
timeout=30,
)
if result.returncode == 0:
return result.stdout
logger.warning("pg_dump failed (rc=%d): %s", result.returncode, result.stderr[:200])
except FileNotFoundError:
logger.warning("pg_dump не найден в PATH — phantom gate будет тестировать в public schema")
except subprocess.TimeoutExpired:
logger.warning("pg_dump timeout — phantom gate будет тестировать в public schema")
return ""
@pytest.fixture(scope="session")
def phantom_schema_name() -> str:
"""Имя временной schema для phantom column тестов."""
return f"test_phantom_{_random_suffix()}"
@pytest.fixture(scope="session")
def phantom_check_session(phantom_schema_name: str) -> Generator[Session, None, None]:
"""SQLAlchemy Session, настроенная на временную schema.
Если pg_dump доступен — создаёт временную schema с копией DDL из public.
Если нет — работает напрямую с public schema (EXPLAIN не требует изменения данных).
После завершения сессии — DROP SCHEMA CASCADE.
Пропускается если TEST_DATABASE_URL не задан.
"""
if not _TEST_DATABASE_URL:
pytest.skip(_SKIP_REASON)
engine = create_engine(
_TEST_DATABASE_URL,
# Без пула — это одноразовая тестовая сессия
pool_pre_ping=True,
pool_size=1,
max_overflow=0,
echo=False,
)
session_factory = sessionmaker(bind=engine, autoflush=False, autocommit=False)
# Попробуем создать временную schema с копией DDL
ddl_sql = _pg_dump_schema_only(_TEST_DATABASE_URL)
use_temp_schema = bool(ddl_sql)
schema = phantom_schema_name if use_temp_schema else "public"
# Track schema creation independently — needed for teardown even if DDL apply fails
schema_created = False
if use_temp_schema:
with engine.connect() as conn:
conn.execute(text(f'CREATE SCHEMA IF NOT EXISTS "{schema}"'))
conn.commit()
schema_created = True
ddl_for_schema = ddl_sql.replace(
"SET search_path = public", f'SET search_path = "{schema}"'
).replace("search_path TO public", f'search_path TO "{schema}"')
prefixed_ddl = f'SET search_path TO "{schema}", public;\n' + ddl_for_schema
try:
conn.execute(text(prefixed_ddl))
conn.commit()
except Exception as e:
logger.warning("DDL apply to temp schema failed: %s — falling back to public", e)
use_temp_schema = False
schema = "public"
conn.rollback()
db = session_factory()
# Устанавливаем search_path для session: temp schema первая, public как fallback
# (pg_dump DDL содержит объекты только под этой schema)
if use_temp_schema:
db.execute(text(f'SET search_path TO "{schema}", public'))
else:
db.execute(text("SET search_path TO public"))
try:
yield db
finally:
db.close()
# Drop temp schema if it was created — even if DDL apply later failed.
# Otherwise schemas leak when CREATE SCHEMA succeeds but DDL apply raises.
if schema_created:
drop_target = phantom_schema_name
with engine.connect() as conn:
conn.execute(text(f'DROP SCHEMA IF EXISTS "{drop_target}" CASCADE'))
conn.commit()
logger.info("phantom gate: dropped temp schema %s", drop_target)
engine.dispose()

379
backend/tests/integration/test_phantom_columns.py Normal file
View file

@ -0,0 +1,379 @@
"""Phantom column gate — EXPLAIN-тесты против реальной Postgres schema.
Каждый тест выполняет EXPLAIN (не EXECUTE!) на SQL из production services.
Если колонка не существует — Postgres бросит ошибку при планировании → тест fail.
Если таблица не существует — аналогично.
Запуск:
# Без TEST_DATABASE_URL — все тесты skip
uv run pytest tests/integration/ -v
# С SSH-туннелем (ssh -N gendesign → localhost:15432):
export TEST_DATABASE_URL="postgresql+psycopg://user:pass@localhost:15432/db"
uv run pytest tests/integration/test_phantom_columns.py -v -m integration
Детектируемые классы багов:
- Ссылка на несуществующую колонку (domrf_kn_objects.geom_3857 — PR #196)
- Ссылка на несуществующую колонку (ekburg_construction_permits.units_count — PR #213)
- Ссылка на несуществующую таблицу / view
- Опечатки в именах колонок и таблиц
"""
from __future__ import annotations
import pytest
from sqlalchemy import text
from sqlalchemy.orm import Session
from tests.integration.conftest import requires_test_db
# ── Маркеры ───────────────────────────────────────────────────────────────────
pytestmark = [
requires_test_db,
pytest.mark.integration,
]
# ── helpers ───────────────────────────────────────────────────────────────────
def _explain(db: Session, query: str, params: dict | None = None) -> None:
"""Выполнить EXPLAIN для query. Не выполняет DML — только план запроса.
Бросает исключение (тест fail) если:
- колонка не существует
- таблица/view не существует
- синтаксическая ошибка SQL
"""
explain_sql = f"EXPLAIN {query}"
db.execute(text(explain_sql), params or {})
# ── 1. domrf_kn_objects — реальные колонки ────────────────────────────────────
class TestDomrfKnObjects:
"""Проверяем что SQL в services используют реальные колонки domrf_kn_objects.
Исторический баг PR #196: geom_3857 не существует.
Реальная schema: latitude, longitude (float), snapshot_date, obj_class, etc.
"""
def test_competitors_radius_query(self, phantom_check_session: Session) -> None:
"""best_layouts._COMPETITORS_IN_RADIUS_SQL — latitude/longitude/snapshot_date."""
_explain(
phantom_check_session,
"""
SELECT DISTINCT ON (obj_id) obj_id
FROM domrf_kn_objects
WHERE latitude IS NOT NULL AND longitude IS NOT NULL
AND ST_DWithin(
ST_SetSRID(ST_MakePoint(longitude, latitude), 4326)::geography,
ST_SetSRID(
ST_MakePoint(CAST(60.6 AS float), CAST(56.8 AS float)), 4326
)::geography,
CAST(1000 AS float)
)
ORDER BY obj_id, snapshot_date DESC NULLS LAST
""",
)
def test_competitors_full_cte_query(self, phantom_check_session: Session) -> None:
"""competitors._COMPETITORS_SQL — полный CTE с latitude/longitude/flat_count."""
_explain(
phantom_check_session,
"""
WITH latest_obj AS (
SELECT DISTINCT ON (obj_id)
obj_id,
comm_name,
dev_name,
obj_class,
latitude,
longitude,
flat_count,
site_status,
snapshot_date
FROM domrf_kn_objects
WHERE latitude IS NOT NULL
AND longitude IS NOT NULL
ORDER BY obj_id, snapshot_date DESC NULLS LAST
),
distances AS (
SELECT
o.obj_id,
o.comm_name,
o.dev_name,
o.obj_class,
o.latitude,
o.longitude,
o.flat_count,
o.site_status,
ST_Distance(
ST_SetSRID(ST_MakePoint(o.longitude, o.latitude), 4326)::geography,
ST_SetSRID(
ST_MakePoint(CAST(60.6 AS float), CAST(56.8 AS float)),
4326
)::geography
) AS distance_m
FROM latest_obj o
)
SELECT
d.obj_id,
d.comm_name,
d.dev_name,
d.obj_class,
d.latitude,
d.longitude,
d.flat_count,
d.site_status,
d.distance_m
FROM distances d
WHERE d.distance_m <= CAST(1000 AS float)
ORDER BY d.distance_m ASC
""",
)
def test_velocity_competitor_query_columns(self, phantom_check_session: Session) -> None:
"""velocity._competitors query — district_name, region_cd, snapshot_date."""
_explain(
phantom_check_session,
"""
SELECT DISTINCT ON (obj_id)
obj_id,
comm_name,
dev_name,
obj_class,
latitude,
longitude,
district_name
FROM domrf_kn_objects
WHERE latitude IS NOT NULL
AND longitude IS NOT NULL
AND region_cd = 66
ORDER BY obj_id, snapshot_date DESC NULLS LAST
""",
)
# ── 2. ekburg_construction_permits — реальные колонки ────────────────────────
class TestEkburgConstructionPermits:
"""Проверяем что SQL используют реальные колонки ekburg_construction_permits.
Исторический баг PR #213: units_count не существует.
Реальная schema: total_area_sqm, living_area_sqm, living_area_fact_sqm, etc.
"""
def test_recent_permits_query(self, phantom_check_session: Session) -> None:
"""parcels.py recent_permits query — total_area_sqm (не units_count!)."""
_explain(
phantom_check_session,
"""
SELECT
permit_type, permit_number, issue_date,
developer_name, developer_inn, object_name, object_type,
construction_address, total_area_sqm
FROM ekburg_construction_permits
WHERE LEFT(cadastral_number, LENGTH(CAST('66:41:0303161' AS text)))
= CAST('66:41:0303161' AS text)
AND issue_date > NOW() - INTERVAL '24 months'
ORDER BY issue_date DESC
LIMIT 50
""",
)
def test_permits_rns_columns(self, phantom_check_session: Session) -> None:
"""Все колонки RNS-специфичные: living_area_sqm, permit_type."""
_explain(
phantom_check_session,
"""
SELECT
permit_type,
permit_number,
issue_date,
expiry_date,
developer_name,
developer_inn,
object_name,
object_type,
construction_address,
cadastral_number,
total_area_sqm,
living_area_sqm
FROM ekburg_construction_permits
WHERE permit_type = 'RNS'
LIMIT 1
""",
)
def test_permits_rve_columns(self, phantom_check_session: Session) -> None:
"""Все колонки RVE-специфичные: living_area_fact_sqm, rve_number, rve_date."""
_explain(
phantom_check_session,
"""
SELECT
permit_type,
permit_number,
living_area_fact_sqm,
rve_number,
rve_date
FROM ekburg_construction_permits
WHERE permit_type = 'RVE'
LIMIT 1
""",
)
# ── 3. mv_layout_velocity — materialized view ────────────────────────────────
class TestMvLayoutVelocity:
"""Проверяем materialized view mv_layout_velocity и её колонки."""
def test_velocity_by_room_bucket(self, phantom_check_session: Session) -> None:
"""best_layouts._VELOCITY_BY_ROOM_SQL — room_bucket, total_deals_24mo, etc."""
_explain(
phantom_check_session,
"""
SELECT
room_bucket,
SUM(total_deals_24mo) AS sum_deals,
AVG(avg_area_m2) AS avg_area_m2,
AVG(avg_price_thousand_rub_per_m2) * 1000.0 AS avg_price_per_m2_rub,
array_agg(DISTINCT obj_id) AS competitor_obj_ids,
COUNT(DISTINCT obj_id) AS competitor_count,
MIN(window_start) AS window_start,
MAX(window_end) AS window_end
FROM mv_layout_velocity
WHERE obj_id = ANY(ARRAY[1, 2, 3])
GROUP BY room_bucket
""",
)
# ── 4. domrf_kn_flats ─────────────────────────────────────────────────────────
class TestDomrfKnFlats:
"""Проверяем таблицу domrf_kn_flats и её колонки."""
def test_supply_batch_query(self, phantom_check_session: Session) -> None:
"""best_layouts._SUPPLY_BATCH_SQL — total_area, rooms, is_studio, flat_type."""
_explain(
phantom_check_session,
"""
SELECT
CASE
WHEN f.is_studio = TRUE OR f.flat_type = 'Квартира-студия' THEN 'studio'
WHEN f.rooms = 0 THEN 'studio'
WHEN f.rooms IN (1, 2, 3) THEN f.rooms::text
WHEN f.rooms >= 4 THEN '4+'
ELSE '1'
END AS rb,
CASE
WHEN f.total_area < 25 THEN '<25'
WHEN f.total_area < 40 THEN '25-40'
WHEN f.total_area < 60 THEN '40-60'
WHEN f.total_area < 80 THEN '60-80'
WHEN f.total_area < 100 THEN '80-100'
ELSE '100+'
END AS ab,
COUNT(*) AS units
FROM domrf_kn_flats f
JOIN domrf_kn_objects o ON f.obj_id = o.obj_id
WHERE o.latitude IS NOT NULL AND o.longitude IS NOT NULL
AND f.snapshot_date = CAST('2026-01-01' AS date)
GROUP BY rb, ab
""",
)
def test_avg_price_sold_query(self, phantom_check_session: Session) -> None:
"""competitors._AVG_PRICE_SQL — price_per_m2, status='sold'."""
_explain(
phantom_check_session,
"""
SELECT
f.obj_id,
AVG(f.price_per_m2) AS avg_price_per_m2
FROM domrf_kn_flats f
WHERE f.obj_id = ANY(ARRAY[1, 2, 3])
AND f.price_per_m2 IS NOT NULL
AND f.status = 'sold'
GROUP BY f.obj_id
""",
)
# ── 5. objective_complex_mapping / objective_corpus_room_month ────────────────
class TestObjectiveTables:
"""Проверяем objective_* таблицы и их колонки."""
def test_objective_mapping_columns(self, phantom_check_session: Session) -> None:
"""competitors / velocity: domrf_obj_id, objective_complex_name в mapping."""
_explain(
phantom_check_session,
"""
SELECT cm.domrf_obj_id AS obj_id,
cm.objective_complex_name
FROM objective_complex_mapping cm
LIMIT 1
""",
)
def test_objective_corpus_room_month_columns(self, phantom_check_session: Session) -> None:
"""velocity._VELOCITY_BY_ROOM_SQL — deals_total_count, deals_total_vol_m2."""
_explain(
phantom_check_session,
"""
SELECT
m.obj_id,
SUM(COALESCE(crm.deals_total_vol_m2,
crm.deals_total_count * 45.0)) AS total_sqm,
COUNT(DISTINCT crm.report_month) AS months_with_data,
MIN(crm.report_month) AS period_start,
MAX(crm.report_month) AS period_end
FROM objective_corpus_room_month crm
JOIN (
SELECT cm.domrf_obj_id AS obj_id,
cm.objective_complex_name
FROM objective_complex_mapping cm
WHERE cm.domrf_obj_id = ANY(ARRAY[1, 2, 3])
) m
ON m.objective_complex_name = crm.project_name
WHERE crm.report_month >= (CURRENT_DATE - CAST('12 months' AS interval))
AND crm.deals_total_count > 0
GROUP BY m.obj_id
""",
)
# ── 6. cad geo таблицы ────────────────────────────────────────────────────────
class TestCadGeoTables:
"""Проверяем cad_parcels_geom, cad_quarters_geom и их колонки."""
def test_parcel_centroid_query(self, phantom_check_session: Session) -> None:
"""best_layouts._PARCEL_CENTROID_SQL — cad_num, cad_number, geom."""
_explain(
phantom_check_session,
"""
SELECT ST_X(pt) AS center_lon,
ST_Y(pt) AS center_lat
FROM (
SELECT ST_Centroid(geom) AS pt
FROM cad_parcels_geom
WHERE cad_num = '66:41:0303161:123' AND geom IS NOT NULL
UNION ALL
SELECT ST_Centroid(geom) AS pt
FROM cad_quarters_geom
WHERE cad_number = '66:41:0303161' AND geom IS NOT NULL
) sub
LIMIT 1
""",
)

11
backend/tests/scrapers/test_nspd_bulk_client.py
View file

@ -316,7 +316,7 @@ async def test_list_objects_in_building_parses(
@pytest.mark.asyncio
async def test_rate_limit_semaphore_max_3_concurrent() -> None:
"""Не более 3 одновременных запросов через _SEMAPHORE.
"""Не более 3 одновременных запросов через per-instance self._sem (PR #260).
Мокируем httpx.AsyncClient.get (нижний уровень) с задержкой, чтобы
реальный семафор работал. Считаем max in-flight внутри семафора.
@ -332,7 +332,7 @@ async def test_rate_limit_semaphore_max_3_concurrent() -> None:
async def slow_get(*args: Any, **kwargs: Any) -> httpx.Response:
nonlocal max_concurrent, current, call_count
# Фиксируем вход — уже внутри семафора (httpx.get вызывается после async with _SEMAPHORE)
# Фиксируем вход — уже внутри семафора (httpx.get вызывается после async with self._sem)
async with lock:
current += 1
call_count += 1
@ -353,11 +353,12 @@ async def test_rate_limit_semaphore_max_3_concurrent() -> None:
tasks = [client.search_by_quarter(f"66:41:{i:07d}") for i in range(6)]
await asyncio.gather(*tasks)
# _SEMAPHORE(3) → не более 3 одновременно внутри slow_get
# self._sem(3) → не более 3 одновременно внутри slow_get
assert max_concurrent <= 3, f"Expected ≤3 concurrent, got {max_concurrent}"
assert call_count == 6 # все 6 вызовов прошли
# Убедимся что _SEMAPHORE в модуле имеет нужное значение capacity
assert bulk_mod._SEMAPHORE._value >= 0 # семафор сброшен после всех задач
# NB: per-instance self._sem cleanup проверяется через max_concurrent <= 3 выше.
# Module-level _SEMAPHORE удалён в PR #260 (cross-loop binding fix); smoke на capacity:
assert bulk_mod._SEMAPHORE_LIMIT == 3
@pytest.mark.asyncio

449
backend/tests/scrapers/test_nspd_grid_walk.py Normal file
View file

@ -0,0 +1,449 @@
"""Unit-тесты для NSPDClient.get_features_in_bbox_grid + classify_engineering_kind.
Sub-PR A foundation (#126): grid-walk — НЕ live HTTP (all mocked).
Запуск:
cd backend && uv run pytest tests/scrapers/test_nspd_grid_walk.py -v
Live integration (помечены @pytest.mark.integration — пропускаются в CI):
cd backend && uv run pytest tests/scrapers/test_nspd_grid_walk.py -m integration -s
"""
from __future__ import annotations
from typing import Any, ClassVar
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from app.services.scrapers.nspd_client import NSPDClient, NSPDFeature
from app.services.scrapers.nspd_denorm import classify_engineering_kind
# ── Helpers ────────────────────────────────────────────────────────────────────
def _make_bulk_feature(feature_id: str | int | None, props: dict[str, Any]) -> MagicMock:
"""Создать mock NSPDBulkFeature с нужными полями."""
feat = MagicMock()
feat.id = feature_id
feat.geometry = None
feat.properties = props
return feat
# ── Grid-walk count tests ──────────────────────────────────────────────────────
class TestGetFeaturesInBboxGrid:
"""Тесты grid-walk метода (без live NSPD)."""
# bbox 700×700м в EPSG:3857 (типичный квартал ЕКБ)
BBOX: tuple[float, float, float, float] = (
6_600_000.0,
7_700_000.0,
6_600_700.0,
7_700_700.0,
)
def _patch_bulk_client(self, return_features: list[list[Any]]) -> tuple[Any, AsyncMock]:
"""Патч NSPDBulkClient.wms_feature_info.
return_features: список ответов — по одному на каждый вызов wms_feature_info
(в порядке вызовов). Если список короче числа calls — последний элемент
повторяется.
"""
call_idx: list[int] = [0]
async def _side_effect(*args: Any, **kwargs: Any) -> list[Any]:
idx = min(call_idx[0], len(return_features) - 1)
call_idx[0] += 1
return return_features[idx]
mock_wms = AsyncMock(side_effect=_side_effect)
return mock_wms, mock_wms
def test_grid_n7_calls_49_times(self) -> None:
"""grid_n=7 → ровно 49 вызовов wms_feature_info."""
call_count: list[int] = [0]
async def _wms(*args: Any, **kwargs: Any) -> list[Any]:
call_count[0] += 1
return []
mock_client_instance = AsyncMock()
mock_client_instance.wms_feature_info = AsyncMock(side_effect=_wms)
mock_client_instance.__aenter__ = AsyncMock(return_value=mock_client_instance)
mock_client_instance.__aexit__ = AsyncMock(return_value=None)
with patch(
"app.scrapers.nspd_bulk_client.NSPDBulkClient",
return_value=mock_client_instance,
):
client = NSPDClient()
result = client.get_features_in_bbox_grid(36328, self.BBOX, grid_n=7, step_m=1.0)
assert call_count[0] == 49, f"Ожидали 49 вызовов, получили {call_count[0]}"
assert result == []
def test_grid_n3_calls_9_times(self) -> None:
"""grid_n=3 → ровно 9 вызовов."""
call_count: list[int] = [0]
async def _wms(*args: Any, **kwargs: Any) -> list[Any]:
call_count[0] += 1
return []
mock_client_instance = AsyncMock()
mock_client_instance.wms_feature_info = AsyncMock(side_effect=_wms)
mock_client_instance.__aenter__ = AsyncMock(return_value=mock_client_instance)
mock_client_instance.__aexit__ = AsyncMock(return_value=None)
with patch(
"app.scrapers.nspd_bulk_client.NSPDBulkClient",
return_value=mock_client_instance,
):
client = NSPDClient()
result = client.get_features_in_bbox_grid(36328, self.BBOX, grid_n=3, step_m=1.0)
assert call_count[0] == 9
assert result == []
def test_dedup_same_feature_id(self) -> None:
"""Одинаковые feature_id из соседних ячеек → одна запись."""
feat_a = _make_bulk_feature("feat-001", {"cad_num": "66:41:001:1"})
feat_b = _make_bulk_feature("feat-001", {"cad_num": "66:41:001:1"}) # дубликат
feat_c = _make_bulk_feature("feat-002", {"cad_num": "66:41:001:2"})
async def _wms(*args: Any, **kwargs: Any) -> list[Any]:
return [feat_a, feat_b, feat_c]
mock_client_instance = AsyncMock()
mock_client_instance.wms_feature_info = AsyncMock(side_effect=_wms)
mock_client_instance.__aenter__ = AsyncMock(return_value=mock_client_instance)
mock_client_instance.__aexit__ = AsyncMock(return_value=None)
with patch(
"app.scrapers.nspd_bulk_client.NSPDBulkClient",
return_value=mock_client_instance,
):
client = NSPDClient()
result = client.get_features_in_bbox_grid(36328, self.BBOX, grid_n=2, step_m=1.0)
# 4 cells × 3 features = 12 raw, но feature_id уникальных 2
feature_ids = [f.feature_id for f in result]
assert "feat-001" in feature_ids
assert "feat-002" in feature_ids
# нет дубликатов
assert len(feature_ids) == len(set(feature_ids))
def test_dedup_by_cad_num_when_no_feature_id(self) -> None:
"""Если feature_id=None — дедупликация по cad_num."""
feat_a = _make_bulk_feature(None, {"cad_num": "66:41:0000001:100"})
feat_b = _make_bulk_feature(None, {"cad_num": "66:41:0000001:100"}) # дубликат
call_n: list[int] = [0]
async def _wms(*args: Any, **kwargs: Any) -> list[Any]:
call_n[0] += 1
# Первая ячейка возвращает feat_a, вторая — feat_b
return [feat_a] if call_n[0] == 1 else [feat_b]
mock_client_instance = AsyncMock()
mock_client_instance.wms_feature_info = AsyncMock(side_effect=_wms)
mock_client_instance.__aenter__ = AsyncMock(return_value=mock_client_instance)
mock_client_instance.__aexit__ = AsyncMock(return_value=None)
with patch(
"app.scrapers.nspd_bulk_client.NSPDBulkClient",
return_value=mock_client_instance,
):
client = NSPDClient()
result = client.get_features_in_bbox_grid(36328, self.BBOX, grid_n=2, step_m=1.0)
cad_nums = [f.properties.get("cad_num") for f in result]
assert cad_nums.count("66:41:0000001:100") == 1, "Дубликат не дедуплицирован"
def test_error_in_one_cell_does_not_abort(self) -> None:
"""Ошибка в одной ячейке не останавливает весь grid-walk."""
good_feat = _make_bulk_feature("feat-ok", {"cad_num": "66:41:001:1"})
call_n: list[int] = [0]
async def _wms(*args: Any, **kwargs: Any) -> list[Any]:
call_n[0] += 1
if call_n[0] == 1:
raise RuntimeError("Simulated cell error")
return [good_feat]
mock_client_instance = AsyncMock()
mock_client_instance.wms_feature_info = AsyncMock(side_effect=_wms)
mock_client_instance.__aenter__ = AsyncMock(return_value=mock_client_instance)
mock_client_instance.__aexit__ = AsyncMock(return_value=None)
with patch(
"app.scrapers.nspd_bulk_client.NSPDBulkClient",
return_value=mock_client_instance,
):
client = NSPDClient()
# Не должно бросать исключение
result = client.get_features_in_bbox_grid(36328, self.BBOX, grid_n=2, step_m=1.0)
# 4 cells: 1 error + 3 good_feat → 1 unique feature
assert any(f.feature_id == "feat-ok" for f in result)
def test_returns_nspd_feature_instances(self) -> None:
"""Метод возвращает list[NSPDFeature] а не NSPDBulkFeature."""
bulk_feat = _make_bulk_feature("feat-xyz", {"cad_num": "66:41:001:1"})
async def _wms(*args: Any, **kwargs: Any) -> list[Any]:
return [bulk_feat]
mock_client_instance = AsyncMock()
mock_client_instance.wms_feature_info = AsyncMock(side_effect=_wms)
mock_client_instance.__aenter__ = AsyncMock(return_value=mock_client_instance)
mock_client_instance.__aexit__ = AsyncMock(return_value=None)
with patch(
"app.scrapers.nspd_bulk_client.NSPDBulkClient",
return_value=mock_client_instance,
):
client = NSPDClient()
result = client.get_features_in_bbox_grid(36328, self.BBOX, grid_n=1, step_m=1.0)
assert all(isinstance(f, NSPDFeature) for f in result)
def test_auto_reduce_grid_for_small_bbox(self) -> None:
"""Маленький bbox + большой grid_n + большой step_m → grid_n уменьшается."""
# bbox 60×60м, step_m=50, grid_n=7 → effective_n=min(7, int(60/50), int(60/50)) = 1
small_bbox: tuple[float, float, float, float] = (
6_600_000.0,
7_700_000.0,
6_600_060.0,
7_700_060.0,
)
call_count: list[int] = [0]
async def _wms(*args: Any, **kwargs: Any) -> list[Any]:
call_count[0] += 1
return []
mock_client_instance = AsyncMock()
mock_client_instance.wms_feature_info = AsyncMock(side_effect=_wms)
mock_client_instance.__aenter__ = AsyncMock(return_value=mock_client_instance)
mock_client_instance.__aexit__ = AsyncMock(return_value=None)
with patch(
"app.scrapers.nspd_bulk_client.NSPDBulkClient",
return_value=mock_client_instance,
):
client = NSPDClient()
client.get_features_in_bbox_grid(36328, small_bbox, grid_n=7, step_m=50.0)
# effective_n = 1 → 1 ячейка
assert call_count[0] == 1
# ── Classifier tests ───────────────────────────────────────────────────────────
class TestClassifyEngineeringKind:
"""Smoke-тесты классификатора инженерных сооружений."""
@pytest.mark.parametrize(
"props, expected",
[
# gas
({"params_name": "Газопровод высокого давления"}, "gas"),
({"name": "Газопровод ПЭ SDR11 160мм"}, "gas"),
({"params_purpose": "Газоснабжение"}, "gas"),
({"name": "ГРП №12"}, "gas"),
# electric
({"name": "КЛ 10 кВ ТП 64102"}, "electric"),
({"params_name": "ВЛ-10кВ Ф-14"}, "electric"),
({"name": "ТП 1234"}, "electric"),
({"params_purpose": "Электроэнергетика и связь"}, "electric"),
({"name": "Подстанция 110/10 кВ"}, "electric"),
# water
({"params_purpose": "Водопровод хозбытовой"}, "water"),
({"name": "Водовод Ду300"}, "water"),
({"params_name": "Сеть водоснабжения"}, "water"),
# heat
({"params_name": "Тепловая сеть"}, "heat"),
({"name": "Теплосеть квартал 24"}, "heat"),
({"params_purpose": "Теплоснабжение жилых домов"}, "heat"),
({"name": "ТЭЦ-4 отпайка"}, "heat"),
# sewage
({"name": "Канализация"}, "sewage"),
({"params_name": "Сеть канализации Ду200"}, "sewage"),
({"name": "Ливневая канализация"}, "sewage"),
# other
({"name": "Объект не классифицирован"}, "other"),
({}, "other"),
({"params_purpose": None}, "other"),
],
)
def test_classify(self, props: dict[str, Any], expected: str) -> None:
assert (
classify_engineering_kind(props) == expected
), f"props={props!r} → expected {expected!r}"
def test_field_priority_params_name_over_purpose(self) -> None:
"""params_name проверяется раньше purpose."""
props = {
"params_name": "Газопровод", # → gas
"params_purpose": "Водоснабжение", # → water (если бы проверялось первым)
}
# gas-паттерн найдётся в combined строке первым по порядку _ENGINEERING_PATTERNS
assert classify_engineering_kind(props) == "gas"
def test_case_insensitive(self) -> None:
"""Паттерны case-insensitive."""
assert classify_engineering_kind({"name": "ГАЗОПРОВОД ВЫСОКОГО ДАВЛЕНИЯ"}) == "gas"
assert classify_engineering_kind({"name": "канализация бытовая"}) == "sewage"
# ── Sub-PR B: _fetch_layer dispatch tests ─────────────────────────────────────
class TestFetchLayerDispatch:
"""Проверяем что _fetch_layer внутри search_by_quarter правильно диспатчит
area-слои на get_features_in_bbox_grid, а EGRN-слои — на legacy get_features_in_bbox.
Моки: search_by_cad и get_features_in_bbox_grid / get_features_in_bbox
через patch, без live HTTP.
"""
# Минимальный bbox квартала (Web Mercator), который возвращает search_by_cad
_MOCK_GEOM: ClassVar[dict[str, Any]] = {
"type": "Polygon",
"coordinates": [
[
[6_600_000.0, 7_700_000.0],
[6_600_700.0, 7_700_000.0],
[6_600_700.0, 7_700_700.0],
[6_600_000.0, 7_700_700.0],
[6_600_000.0, 7_700_000.0],
]
],
}
def _make_search_result(self) -> MagicMock:
"""Mock NSPDSearchResult с первым feature, у которого есть geometry."""
feat = MagicMock(spec=NSPDFeature)
feat.geometry = self._MOCK_GEOM
feat.properties = {}
feat.feature_id = "q-001"
result = MagicMock()
result.first = feat
return result
def test_fetch_layer_uses_grid_walk_for_area_layers(self) -> None:
"""territorial_zones (area layer) → вызывает get_features_in_bbox_grid."""
client = NSPDClient()
mock_search_result = self._make_search_result()
with (
patch.object(client, "search_by_cad", return_value=mock_search_result),
patch.object(client, "get_features_in_bbox_grid", return_value=[]) as mock_grid,
patch.object(client, "get_features_in_bbox", return_value=[]) as mock_legacy,
):
client.search_by_quarter(
"66:41:0303161",
include_zouit=False,
include_risks=False,
include_opportunity=False,
)
# territorial_zones, red_lines, engineering_structures — все три должны
# быть запрошены через grid-walk
called_layer_ids = [call.args[0] for call in mock_grid.call_args_list]
from app.services.scrapers.nspd_client import LAYERS
assert (
LAYERS["territorial_zones"] in called_layer_ids
), "territorial_zones должен использовать grid-walk"
assert LAYERS["red_lines"] in called_layer_ids, "red_lines должен использовать grid-walk"
assert (
LAYERS["engineering_structures"] in called_layer_ids
), "engineering_structures должен использовать grid-walk"
# parcels и buildings — legacy, не grid
called_legacy_ids = [call.args[0] for call in mock_legacy.call_args_list]
assert LAYERS["parcels"] in called_legacy_ids, "parcels должен идти через legacy"
assert LAYERS["buildings"] in called_legacy_ids, "buildings должен идти через legacy"
def test_fetch_layer_uses_legacy_for_egrn_layers(self) -> None:
"""parcels и buildings → вызывают legacy get_features_in_bbox (не grid)."""
client = NSPDClient()
mock_search_result = self._make_search_result()
with (
patch.object(client, "search_by_cad", return_value=mock_search_result),
patch.object(client, "get_features_in_bbox_grid", return_value=[]) as mock_grid,
patch.object(client, "get_features_in_bbox", return_value=[]) as mock_legacy,
):
client.search_by_quarter(
"66:41:0303161",
include_zouit=False,
include_risks=False,
include_opportunity=False,
)
from app.services.scrapers.nspd_client import LAYERS
called_legacy_ids = [call.args[0] for call in mock_legacy.call_args_list]
called_grid_ids = [call.args[0] for call in mock_grid.call_args_list]
assert LAYERS["parcels"] in called_legacy_ids
assert LAYERS["buildings"] in called_legacy_ids
# EGRN layers должны НЕ попасть в grid-walk
assert LAYERS["parcels"] not in called_grid_ids, "parcels не должен использовать grid"
assert LAYERS["buildings"] not in called_grid_ids, "buildings не должен использовать grid"
def test_engineering_structures_classified_kind_enriched(self) -> None:
"""engineering_structures features получают classified_kind в properties."""
from app.services.scrapers.nspd_client import LAYERS
# Создаём feature с mutable properties (как реальный NSPDFeature)
eng_feat = MagicMock(spec=NSPDFeature)
eng_feat.feature_id = "eng-001"
eng_feat.geometry = None
eng_feat.properties = {"params_name": "Газопровод высокого давления"}
client = NSPDClient()
mock_search_result = self._make_search_result()
def _grid_side_effect(layer_id: int, bbox: Any, **kwargs: Any) -> list[Any]:
if layer_id == LAYERS["engineering_structures"]:
return [eng_feat]
return []
with (
patch.object(client, "search_by_cad", return_value=mock_search_result),
patch.object(client, "get_features_in_bbox_grid", side_effect=_grid_side_effect),
patch.object(client, "get_features_in_bbox", return_value=[]),
):
dump = client.search_by_quarter(
"66:41:0303161",
include_zouit=False,
include_risks=False,
include_opportunity=False,
)
assert len(dump.engineering_structures) == 1
enriched = dump.engineering_structures[0]
got = enriched.properties.get("classified_kind")
assert got == "gas", f"Ожидали classified_kind='gas', получили {got!r}"
# ── Integration marker (skip in CI) ──────────────────────────────────────────
@pytest.mark.integration
def test_live_nspd_grid_walk_skipped() -> None:
"""Placeholder для ручного запуска live NSPD grid-walk.
Запуск:
cd backend && uv run pytest tests/scrapers/test_nspd_grid_walk.py -m integration -s
При запуске — делает реальные HTTP-запросы к nspd.gov.ru.
"""
pytest.skip("Live NSPD integration test — запускать вручную с -m integration")

0
backend/tests/services/cadastre/__init__.py Normal file
View file

198
backend/tests/services/cadastre/test_bulk_harvest_territorial.py Normal file
View file

@ -0,0 +1,198 @@
"""Тесты для _save_territorial_zones (bulk_harvest.py) — mock-based.
Проверяет:
- Успешный UPSERT 3 features → 3 строки вставлены
- Повторный вызов → ON CONFLICT обновляет, не дублирует
- Feature без geometry → строка вставлена с geom=NULL, без краша
- Feature без zone_id → синтетический fallback zone_id используется
"""
from __future__ import annotations
from typing import Any
from unittest.mock import MagicMock
from app.services.cadastre.bulk_harvest import _save_territorial_zones
def _make_feature(
feature_id: Any = "zone_1",
zone_code: str = "Ж-1",
zone_name: str = "Жилая смешанная",
permitted_use: str = "ИЖС",
has_geometry: bool = True,
) -> dict:
"""Создать raw feature dict в формате get_features_in_bbox_grid."""
geom = (
{
"type": "Polygon",
"coordinates": [
[
[6090000.0, 7590000.0],
[6090100.0, 7590000.0],
[6090100.0, 7590100.0],
[6090000.0, 7590100.0],
[6090000.0, 7590000.0],
]
],
}
if has_geometry
else None
)
return {
"id": feature_id,
"geometry": geom,
"properties": {
"zone_code": zone_code,
"zone_name": zone_name,
"permitted_use": permitted_use,
},
}
def _make_db_mock() -> MagicMock:
"""Mock SQLAlchemy Session с begin_nested() savepoint support."""
db = MagicMock()
# begin_nested() используется как context manager
savepoint_ctx = MagicMock()
savepoint_ctx.__enter__ = MagicMock(return_value=savepoint_ctx)
savepoint_ctx.__exit__ = MagicMock(return_value=False)
db.begin_nested.return_value = savepoint_ctx
return db
class TestSaveTerritorialZones:
"""Тесты для _save_territorial_zones."""
def test_three_features_inserted(self) -> None:
"""3 features → returned count == 3, execute вызван 3 раза."""
db = _make_db_mock()
features = [
_make_feature("z1", "Ж-1"),
_make_feature("z2", "ОД-1"),
_make_feature("z3", "П-1"),
]
result = _save_territorial_zones(db, "66:41:0204016", features)
assert result == 3
assert db.execute.call_count == 3
db.commit.assert_called_once()
def test_empty_features_list(self) -> None:
"""Пустой список → 0 inserted, commit всё равно вызван."""
db = _make_db_mock()
result = _save_territorial_zones(db, "66:41:0204016", [])
assert result == 0
db.execute.assert_not_called()
db.commit.assert_called_once()
def test_feature_without_geometry_no_crash(self) -> None:
"""Feature без geometry → geom=NULL, строка вставлена без краша."""
db = _make_db_mock()
features = [_make_feature("zone_no_geom", has_geometry=False)]
result = _save_territorial_zones(db, "66:41:0204016", features)
assert result == 1
# Проверяем что geom параметр передан как None
call_kwargs: dict = db.execute.call_args[0][1]
assert call_kwargs["geom"] is None
def test_feature_without_zone_id_uses_fallback(self) -> None:
"""Feature без id → md5-based fallback zone_id (stable между runs)."""
db = _make_db_mock()
features = [
{
"id": None,
"geometry": None,
"properties": {"zone_code": "Ж-2"},
}
]
result = _save_territorial_zones(db, "66:41:0204016", features)
assert result == 1
call_kwargs = db.execute.call_args[0][1]
zone_id: str = call_kwargs["zone_id"]
# fallback zone_id содержит quarter_cad и стабильный hash (12 hex chars)
assert zone_id.startswith("66:41:0204016_")
suffix = zone_id.split("_", 3)[-1]
assert len(suffix) == 12
assert all(c in "0123456789abcdef" for c in suffix)
# Второй вызов с теми же данными → тот же zone_id (идемпотентность)
db2 = _make_db_mock()
_save_territorial_zones(db2, "66:41:0204016", features)
call_kwargs2 = db2.execute.call_args[0][1]
assert call_kwargs2["zone_id"] == zone_id
def test_zone_id_from_props_id(self) -> None:
"""Если feature.id=None, но props['id'] есть — используется props['id']."""
db = _make_db_mock()
features = [
{
"id": None,
"geometry": None,
"properties": {"id": "props_id_42", "zone_code": "Ж-3"},
}
]
result = _save_territorial_zones(db, "66:41:0204016", features)
assert result == 1
call_kwargs = db.execute.call_args[0][1]
assert call_kwargs["zone_id"] == "props_id_42"
def test_execute_error_logged_not_raised(self) -> None:
"""Exception в execute → строка не вставлена, warning залогирован, не re-raise."""
db = _make_db_mock()
db.execute.side_effect = RuntimeError("DB error")
features = [_make_feature("z_err")]
# Не должен бросить исключение
result = _save_territorial_zones(db, "66:41:0204016", features)
assert result == 0
db.commit.assert_called_once()
def test_savepoint_used_per_row(self) -> None:
"""begin_nested() вызывается для каждой строки (SAVEPOINT паттерн)."""
db = _make_db_mock()
features = [_make_feature(f"z{i}") for i in range(3)]
_save_territorial_zones(db, "66:41:0204016", features)
assert db.begin_nested.call_count == 3
def test_quarter_cad_param_passed(self) -> None:
"""quarter_cad правильно передаётся в SQL параметры."""
db = _make_db_mock()
features = [_make_feature("zone_check")]
_save_territorial_zones(db, "66:41:9999999", features)
call_kwargs = db.execute.call_args[0][1]
assert call_kwargs["quarter_cad"] == "66:41:9999999"
def test_raw_props_serialized(self) -> None:
"""raw_props — JSON строка из properties dict."""
import json
db = _make_db_mock()
features = [
{
"id": "z_props",
"geometry": None,
"properties": {"zone_code": "ОД-2", "extra": "value"},
}
]
_save_territorial_zones(db, "66:41:0204016", features)
call_kwargs = db.execute.call_args[0][1]
raw = json.loads(call_kwargs["raw_props"])
assert raw["zone_code"] == "ОД-2"
assert raw["extra"] == "value"

0
backend/tests/services/scrapers/__init__.py Normal file
View file

290
backend/tests/services/scrapers/test_domrf_catalog_object.py Normal file
View file

@ -0,0 +1,290 @@
"""Тесты для domrf_catalog_object.py (issue #297 sub-task 22d).
Покрывает:
- extract_next_data — парсинг HTML с __NEXT_DATA__
- parse_catalog_object — маппинг pageProps → DB columns
- value helpers (_to_numeric_comma, _to_bool_da_net, _to_date_ddmmyyyy)
- partial responses (partial pageProps → all other fields = None, no crash)
"""
from __future__ import annotations
from datetime import date
from typing import Any
import pytest
from app.services.scrapers.domrf_catalog_object import (
_to_bool_da_net,
_to_bool_int,
_to_date_ddmmyyyy,
_to_numeric_comma,
extract_next_data,
parse_catalog_object,
)
# ── extract_next_data ─────────────────────────────────────────────────────────
def test_extract_next_data_from_html() -> None:
"""Базовый case: тег найден, JSON возвращается как dict."""
html = (
"<html><head>"
'<script id="__NEXT_DATA__" type="application/json">'
'{"props":{"pageProps":{"buildingClass":"Комфорт"}}}'
"</script>"
"</head></html>"
)
result = extract_next_data(html)
assert isinstance(result, dict)
assert result["props"]["pageProps"]["buildingClass"] == "Комфорт"
def test_extract_next_data_single_quotes() -> None:
"""Тег с одинарными кавычками тоже должен парситься."""
html = "<script id='__NEXT_DATA__'>" '{"props":{"pageProps":{}}}' "</script>"
result = extract_next_data(html)
assert "props" in result
def test_extract_next_data_not_found_raises() -> None:
"""Если тег не найден — ValueError."""
with pytest.raises(ValueError, match="__NEXT_DATA__"):
extract_next_data("<html><body>no script here</body></html>")
def test_extract_next_data_invalid_json_raises() -> None:
"""Если JSON некорректный — ValueError."""
html = '<script id="__NEXT_DATA__">{broken json</script>'
with pytest.raises(ValueError):
extract_next_data(html)
# ── parse_catalog_object — full sample ───────────────────────────────────────
def _make_full_next_data() -> dict[str, Any]:
"""Реалистичный full next_data для obj_id=65136 (подтверждён live)."""
return {
"props": {
"pageProps": {
"buildingClass": "Комфорт",
"wallMaterial": "Монолит-кирпич",
"objEnergyEfficiency": "B",
"parkingCount": 246,
"finishTypeCount": 1,
"freePlan": "Нет",
"publicationDate": "31.03.2025",
"additionalInfo": {
"objectParkingPlaces": 43,
"nearbyParkingPlaces": 0,
"ceilingHeight": "2,7",
"passengerElevatorsCount": 0,
"cargoElevatorsCount": 0,
"cargoPassengerElevatorCount": 4,
"playgroundsCount": 6,
"sportsgroundCount": 5,
"bicycleLane": 0,
"trashAreaCount": 3,
"ramp": 0,
"curbLowering": 1,
"wheelchairElevatorsCount": 0,
"parkingAvailabilityPerc": 60,
},
"quartography": {
"objLivElemEntrCnt": 1,
"objLivElemSqAvg": 46.2,
"nonLivFirstFloor": 1,
},
"indexes": {
"infrastructure": 10,
"transport": 6,
},
"projectDeclaration": {
"number": "66-001686",
},
}
}
}
def test_parse_catalog_object_full() -> None:
"""Полный sample: все 25+ полей должны быть замаплены корректно."""
data = parse_catalog_object(_make_full_next_data())
assert data["obj_class"] == "Комфорт"
assert data["wall_type"] == "Монолит-кирпич"
assert data["energy_eff"] == "B"
assert data["section_count"] == 1
assert data["parking_total_slots"] == 246
assert data["guest_parking_inside_count"] == 43
assert data["guest_parking_outside_count"] == 0
assert data["ceiling_height_m"] == pytest.approx(2.7)
assert data["finishing_variants_count"] == 1
assert data["has_free_planning"] is False
assert data["avg_flat_area_m2"] == pytest.approx(46.2)
assert data["elevators_passenger_count"] == 0
assert data["elevators_cargo_count"] == 4 # 0 + 4
assert data["playground_kids_count"] == 6
assert data["playground_sport_count"] == 5
assert data["has_bike_paths"] is False # bicycleLane=0
assert data["trash_areas_count"] == 3
assert data["has_ramp"] is False # ramp=0
assert data["has_low_platforms"] is True # curbLowering=1
assert data["has_wheelchair_lift"] is False # wheelchairElevatorsCount=0
assert data["first_floor_type"] == "нежилой" # nonLivFirstFloor=1
assert data["parking_provision_pct"] == 60
assert data["project_published_at"] == date(2025, 3, 31)
assert data["project_declaration_num"] == "66-001686"
assert data["domrf_score_infrastructure"] == 10
assert data["domrf_score_transport"] == 6
def test_parse_catalog_object_has_free_planning_da() -> None:
"""freePlan='Да' → has_free_planning=True."""
nd: dict[str, Any] = {"props": {"pageProps": {"freePlan": "Да"}}}
data = parse_catalog_object(nd)
assert data["has_free_planning"] is True
def test_parse_catalog_object_first_floor_zhiloj() -> None:
"""nonLivFirstFloor=0 → first_floor_type='жилой'."""
nd: dict[str, Any] = {
"props": {
"pageProps": {
"quartography": {"nonLivFirstFloor": 0},
}
}
}
data = parse_catalog_object(nd)
assert data["first_floor_type"] == "жилой"
def test_parse_catalog_object_elevators_cargo_sum() -> None:
"""elevators_cargo_count = cargoElevatorsCount + cargoPassengerElevatorCount."""
nd: dict[str, Any] = {
"props": {
"pageProps": {
"additionalInfo": {
"cargoElevatorsCount": 2,
"cargoPassengerElevatorCount": 3,
}
}
}
}
data = parse_catalog_object(nd)
assert data["elevators_cargo_count"] == 5
def test_parse_catalog_object_partial() -> None:
"""Только buildingClass → остальные поля None, без исключений."""
nd: dict[str, Any] = {"props": {"pageProps": {"buildingClass": "Бизнес"}}}
data = parse_catalog_object(nd)
assert data["obj_class"] == "Бизнес"
assert data["wall_type"] is None
assert data["energy_eff"] is None
assert data["section_count"] is None
assert data["parking_total_slots"] is None
assert data["ceiling_height_m"] is None
assert data["has_free_planning"] is None
assert data["elevators_cargo_count"] is None
assert data["project_published_at"] is None
assert data["domrf_score_infrastructure"] is None
def test_parse_catalog_object_empty() -> None:
"""Полностью пустой next_data → все поля None, без исключений."""
data = parse_catalog_object({})
for v in data.values():
assert v is None
def test_parking_provision_pct_preserves_float() -> None:
"""parking_provision_pct should preserve fractional values (column is numeric(5,1))."""
next_data: dict[str, Any] = {
"props": {"pageProps": {"id": 65136, "additionalInfo": {"parkingAvailabilityPerc": 60.5}}}
}
result = parse_catalog_object(next_data)
assert result["parking_provision_pct"] == 60.5
# ── _to_numeric_comma ─────────────────────────────────────────────────────────
@pytest.mark.parametrize(
"inp,expected",
[
("2,7", 2.7),
("2.7", 2.7),
("3,50", 3.5),
("", None),
(None, None),
(" ", None),
("abc", None),
],
)
def test_to_numeric_comma(inp: Any, expected: float | None) -> None:
result = _to_numeric_comma(inp)
if expected is None:
assert result is None
else:
assert result == pytest.approx(expected)
# ── _to_bool_da_net ───────────────────────────────────────────────────────────
@pytest.mark.parametrize(
"inp,expected",
[
("Да", True),
("да", True),
("ДА", True),
("Нет", False),
("нет", False),
("НЕТ", False),
("", None),
(None, None),
("maybe", None),
("Yes", None),
],
)
def test_to_bool_da_net(inp: Any, expected: bool | None) -> None:
assert _to_bool_da_net(inp) == expected
# ── _to_bool_int ──────────────────────────────────────────────────────────────
@pytest.mark.parametrize(
"inp,expected",
[
(0, False),
(1, True),
(5, True),
("1", True),
("0", False),
(None, None),
],
)
def test_to_bool_int(inp: Any, expected: bool | None) -> None:
assert _to_bool_int(inp) == expected
# ── _to_date_ddmmyyyy ─────────────────────────────────────────────────────────
@pytest.mark.parametrize(
"inp,expected",
[
("31.03.2025", date(2025, 3, 31)),
("01.01.2024", date(2024, 1, 1)),
("", None),
(None, None),
("2025-03-31", None), # неправильный формат → None
("abc", None),
("31.13.2025", None), # невалидный месяц → None
],
)
def test_to_date_ddmmyyyy(inp: Any, expected: date | None) -> None:
assert _to_date_ddmmyyyy(inp) == expected

497
backend/tests/services/site_finder/test_best_layouts.py Normal file
View file

@ -0,0 +1,497 @@
"""Unit-тесты для get_best_layouts (Fix SF-01: honest time_window velocity).
Проверяет, что разные time_window → разные deals_window → разный velocity_per_month.
Mock-стратегия: патчим db.execute с side_effect, повторяя порядок вызовов
в get_best_layouts:
1. _PARCEL_CENTROID_SQL → .mappings().first()
2. _COMPETITORS_IN_RADIUS_SQL → .mappings().all()
3. _INLINE_VELOCITY_SQL → .mappings().all()
4. db.scalar() → MAX(snapshot_date) — через .return_value
5. _SUPPLY_BATCH_SQL → .mappings().all()
Ключевые asserts:
- last_month (1 мес) → velocity = deals_window / 1.0
- last_quarter (3 мес) → velocity = deals_window / 3.0
- last_year (12 мес) → velocity = deals_window / 12.0
- Разный deals_window при разных time_window → разный mix.
"""
from __future__ import annotations
import datetime as dt
from unittest.mock import MagicMock
import pytest
from app.schemas.parcel import BestLayoutsRequest
from app.services.site_finder.best_layouts import (
_TIME_WINDOW_PARAMS,
MAX_BUCKET_SHARE_PCT,
_cap_and_redistribute,
get_best_layouts,
)
_TODAY = dt.date.today()
CAD_NUM = "66:41:0303161:123"
# ── Фабрики mock-строк ────────────────────────────────────────────────────────
def _coord_row(lon: float = 60.6, lat: float = 56.85) -> MagicMock:
r = MagicMock()
r.__getitem__ = lambda self, k: {"center_lon": lon, "center_lat": lat}[k]
return r
def _obj_id_row(obj_id: int) -> MagicMock:
r = MagicMock()
r.__getitem__ = lambda self, k: {"obj_id": obj_id}[k]
return r
def _vel_row(
room_bucket: str = "2",
deals_window: float = 48.0,
avg_area: float = 55.0,
avg_price_rub: float | None = 120000.0,
obj_ids: list[int] | None = None,
window_start: dt.date | None = None,
window_end: dt.date | None = None,
) -> MagicMock:
"""Строка из _INLINE_VELOCITY_SQL.
deals_window — реальные сделки за честное окно (не 24 мес).
"""
oids = obj_ids if obj_ids is not None else [1]
ws = window_start or _TODAY - dt.timedelta(days=90)
we = window_end or _TODAY
r = MagicMock()
r.__getitem__ = lambda self, k: {
"room_bucket": room_bucket,
"deals_window": deals_window,
"avg_area_m2": avg_area,
"avg_price_per_m2_rub": avg_price_rub,
"competitor_obj_ids": oids,
"competitor_count": len(oids),
"window_start": ws,
"window_end": we,
}[k]
return r
def _supply_row(rb: str, ab: str, units: int) -> MagicMock:
r = MagicMock()
r.__getitem__ = lambda self, k: {"rb": rb, "ab": ab, "units": units}[k]
return r
def _make_db(
coord: MagicMock | None = None,
id_rows: list[MagicMock] | None = None,
vel_rows: list[MagicMock] | None = None,
supply_rows: list[MagicMock] | None = None,
latest_snap: dt.date | None = None,
) -> MagicMock:
"""Сконструировать mock Session.
Порядок db.execute():
1. centroid → .mappings().first()
2. competitors → .mappings().all()
3. velocity → .mappings().all()
4. supply → .mappings().all() (только если latest_snap is not None)
db.scalar() → latest_snap (MAX snapshot_date).
"""
db = MagicMock()
db.scalar.return_value = latest_snap if latest_snap is not None else _TODAY
r0 = MagicMock()
r0.mappings.return_value.first.return_value = coord
r1 = MagicMock()
r1.mappings.return_value.all.return_value = id_rows or []
r2 = MagicMock()
r2.mappings.return_value.all.return_value = vel_rows or []
r3 = MagicMock()
r3.mappings.return_value.all.return_value = supply_rows or []
db.execute.side_effect = [r0, r1, r2, r3]
return db
def _request(**kwargs) -> BestLayoutsRequest:
defaults: dict = {
"radius_km": 1.0,
"time_window": "last_quarter",
"min_velocity_per_month": 0.0,
}
defaults.update(kwargs)
return BestLayoutsRequest(**defaults)
# ── Тесты TIME_WINDOW_PARAMS ──────────────────────────────────────────────────
def test_time_window_params_keys() -> None:
"""Все три time_window определены, months_in_window > 0."""
for key in ("last_month", "last_quarter", "last_year"):
assert key in _TIME_WINDOW_PARAMS
interval_str, months = _TIME_WINDOW_PARAMS[key]
assert isinstance(interval_str, str) and len(interval_str) > 0
assert months > 0
# ── Тест SF-01: разный deals_window → разный velocity ────────────────────────
def test_last_month_velocity_divisor_1() -> None:
"""time_window=last_month: velocity = deals_window / 1.0."""
deals = 30.0
db = _make_db(
coord=_coord_row(),
id_rows=[_obj_id_row(1)],
vel_rows=[_vel_row("1", deals_window=deals, obj_ids=[1])],
)
req = _request(time_window="last_month")
resp = get_best_layouts(db, CAD_NUM, req)
assert len(resp.top_layouts) == 1
assert resp.top_layouts[0].velocity_per_month == pytest.approx(30.0, rel=1e-3)
def test_last_quarter_velocity_divisor_3() -> None:
"""time_window=last_quarter: velocity = deals_window / 3.0."""
deals = 30.0
db = _make_db(
coord=_coord_row(),
id_rows=[_obj_id_row(1)],
vel_rows=[_vel_row("1", deals_window=deals, obj_ids=[1])],
)
req = _request(time_window="last_quarter")
resp = get_best_layouts(db, CAD_NUM, req)
assert len(resp.top_layouts) == 1
assert resp.top_layouts[0].velocity_per_month == pytest.approx(10.0, rel=1e-3)
def test_last_year_velocity_divisor_12() -> None:
"""time_window=last_year: velocity = deals_window / 12.0."""
deals = 60.0
db = _make_db(
coord=_coord_row(),
id_rows=[_obj_id_row(1)],
vel_rows=[_vel_row("1", deals_window=deals, obj_ids=[1])],
)
req = _request(time_window="last_year")
resp = get_best_layouts(db, CAD_NUM, req)
assert len(resp.top_layouts) == 1
assert resp.top_layouts[0].velocity_per_month == pytest.approx(5.0, rel=1e-3)
def test_different_time_windows_produce_different_velocity() -> None:
"""Одни и те же deals_window → разная velocity_per_month для разных time_window.
Главный acceptance-тест SF-01: time_window влияет на velocity, не только на масштаб.
При одном и том же deals_window=30:
last_month → 30.0
last_quarter → 10.0
last_year → 2.5
"""
deals = 30.0
velocities: dict[str, float] = {}
for tw in ("last_month", "last_quarter", "last_year"):
db = _make_db(
coord=_coord_row(),
id_rows=[_obj_id_row(1)],
vel_rows=[_vel_row("2", deals_window=deals, obj_ids=[1])],
)
req = _request(time_window=tw)
resp = get_best_layouts(db, CAD_NUM, req)
assert len(resp.top_layouts) == 1, f"No layouts for {tw}"
velocities[tw] = resp.top_layouts[0].velocity_per_month
# Все три значения различаются
vals = list(velocities.values())
assert vals[0] != vals[1] != vals[2], f"Velocities must differ: {velocities}"
# last_month > last_quarter > last_year (одинаковые deals, разный знаменатель)
assert velocities["last_month"] > velocities["last_quarter"] > velocities["last_year"]
# ── Тест: ranking по velocity и sum pct = 100 ────────────────────────────────
def test_ranking_and_pct_sum_100() -> None:
"""3 room_buckets → ranking по velocity, sum pct = 100."""
id_rows = [_obj_id_row(1), _obj_id_row(2), _obj_id_row(3)]
vel_rows = [
_vel_row("studio", deals_window=9.0, avg_area=26.0, obj_ids=[1]), # 9/3=3.0
_vel_row("1", deals_window=24.0, avg_area=40.0, obj_ids=[2]), # 24/3=8.0
_vel_row("2", deals_window=48.0, avg_area=55.0, obj_ids=[3]), # 48/3=16.0
]
supply_rows = [
_supply_row("studio", "<25", 20),
_supply_row("1", "40-60", 60),
_supply_row("2", "40-60", 80),
]
db = _make_db(coord=_coord_row(), id_rows=id_rows, vel_rows=vel_rows, supply_rows=supply_rows)
req = _request(time_window="last_quarter")
resp = get_best_layouts(db, CAD_NUM, req)
top = resp.top_layouts
assert len(top) == 3
# rank 1 = "2" (наибольший velocity 16.0)
assert top[0].room_bucket == "2"
assert top[0].rank == 1
assert top[0].velocity_per_month == pytest.approx(16.0, rel=1e-3)
# rank 2 = "1" (8.0)
assert top[1].room_bucket == "1"
assert top[1].velocity_per_month == pytest.approx(8.0, rel=1e-3)
# ранги уникальны
assert sorted(t.rank for t in top) == [1, 2, 3]
# sum pct = 100
mix = resp.recommendation_for_tz.mix
assert sum(m.pct for m in mix) == 100
# ── Тест: пустые конкуренты ───────────────────────────────────────────────────
def test_no_competitors_returns_empty_response() -> None:
"""Нет конкурентов в радиусе → пустые top_layouts + confidence=low."""
db = _make_db(coord=_coord_row(), id_rows=[], vel_rows=[])
req = _request()
resp = get_best_layouts(db, CAD_NUM, req)
assert resp.top_layouts == []
assert resp.data_quality.confidence == "low"
assert resp.recommendation_for_tz.based_on_obj_count == 0
# ── Тест: centroid не найден ──────────────────────────────────────────────────
def test_centroid_not_found_raises_value_error() -> None:
"""Геометрия участка не найдена → ValueError."""
db = _make_db(coord=None)
req = _request()
with pytest.raises(ValueError, match="не найдена"):
get_best_layouts(db, "99:99:9999999:999", req)
# ── Тест: min_velocity фильтрует строки ──────────────────────────────────────
def test_min_velocity_filters_low_rows() -> None:
"""min_velocity_per_month=5 → строки с velocity<5 не попадают в top_layouts.
last_quarter (3 мес):
studio: 9 / 3 = 3.0 < 5.0 → отфильтрован
1: 24 / 3 = 8.0 > 5.0 → остаётся
"""
id_rows = [_obj_id_row(1), _obj_id_row(2)]
vel_rows = [
_vel_row("studio", deals_window=9.0, obj_ids=[1]),
_vel_row("1", deals_window=24.0, obj_ids=[2]),
]
db = _make_db(coord=_coord_row(), id_rows=id_rows, vel_rows=vel_rows)
req = _request(time_window="last_quarter", min_velocity_per_month=5.0)
resp = get_best_layouts(db, CAD_NUM, req)
top = resp.top_layouts
assert len(top) == 1
assert top[0].room_bucket == "1"
assert top[0].velocity_per_month == pytest.approx(8.0, rel=1e-3)
# ── Тест: exclude_competitor_obj_ids ─────────────────────────────────────────
def test_exclude_competitor_obj_ids() -> None:
"""exclude_competitor_obj_ids=[20] при единственном конкуренте → пустой ответ."""
id_rows = [_obj_id_row(20)]
db = _make_db(coord=_coord_row(), id_rows=id_rows, vel_rows=[])
req = _request(exclude_competitor_obj_ids=[20])
resp = get_best_layouts(db, CAD_NUM, req)
assert resp.top_layouts == []
assert resp.data_quality.objects_total_in_radius == 1
# ── Тест: total_sold_in_window совпадает с deals_window ──────────────────────
def test_total_sold_in_window_matches_deals_window() -> None:
"""total_sold_in_window в TopLayoutRow = deals_window (целое)."""
deals = 37.0
db = _make_db(
coord=_coord_row(),
id_rows=[_obj_id_row(5)],
vel_rows=[_vel_row("3", deals_window=deals, obj_ids=[5])],
)
req = _request(time_window="last_quarter")
resp = get_best_layouts(db, CAD_NUM, req)
assert len(resp.top_layouts) == 1
assert resp.top_layouts[0].total_sold_in_window == int(deals)
# ── Тесты _cap_and_redistribute (Fix SF-09 review) ───────────────────────────
@pytest.mark.parametrize(
"pct_map, expect_pathological",
[
# 1. normal: одиночный bucket > 35, free достаточно capacity
({"1k": 50, "studio": 30, "2k": 20}, False),
# 2. heavy skew (3-bucket): surplus=40, capacity=20+25=45 — помещается
({"1k": 75, "studio": 15, "2k": 10}, False),
# 3. multiple buckets > 35
({"1k": 50, "studio": 40, "2k": 10}, False),
# 4. all > 35 — pathological
({"1k": 50, "studio": 50}, True),
# 5. граничный: один bucket ровно на cap — не clamp
({"1k": 35, "studio": 35, "2k": 30}, False),
# 6. single bucket 100% — pathological (нет free)
({"1k": 100}, True),
# 7. 2-bucket heavy: surplus=55, capacity=25 — pathological (не помещается)
({"1k": 90, "studio": 10}, True),
# 8. все ≤ cap — fast-path без изменений
({"1k": 30, "studio": 35, "2k": 35}, False),
# 9. 2-bucket: 70/30 → surplus=35, capacity=5 → pathological
({"1k": 70, "studio": 30}, True),
# 10. 2-bucket: 99/1 → surplus=64, capacity=34 → pathological
({"1k": 99, "studio": 1}, True),
],
)
def test_cap_and_redistribute_invariants(
pct_map: dict[str, int],
expect_pathological: bool,
) -> None:
"""Invariant: max(pct) ≤ cap И sum(pct) == 100 (или cap_skipped=True в pathological).
Pathological — `cap_skipped=True`, max МОЖЕТ быть > cap (геометрически surplus
не вмещается в free capacity).
"""
result, cap_skipped = _cap_and_redistribute(pct_map)
assert (
cap_skipped == expect_pathological
), f"cap_skipped={cap_skipped} но ожидали {expect_pathological} для {pct_map}"
assert (
sum(result.values()) == 100
), f"sum={sum(result.values())} != 100 для {pct_map} → {result}"
if not expect_pathological:
assert (
max(result.values()) <= MAX_BUCKET_SHARE_PCT
), f"max={max(result.values())} > cap={MAX_BUCKET_SHARE_PCT} для {pct_map} → {result}"
@pytest.mark.parametrize(
"deals, expect_pathological, label",
[
# 3-bucket с достаточной capacity — surplus помещается, cap соблюдён
({"1k": 75, "studio": 15, "2k": 10}, False, "{1k:75, studio:15, 2k:10}"),
({"1k": 80, "studio": 12, "2k": 8}, False, "{1k:80, studio:12, 2k:8}"),
({"1k": 60, "studio": 30, "2k": 10}, False, "{1k:60, studio:30, 2k:10}"),
({"a": 50, "b": 30, "c": 20}, False, "{50, 30, 20}"),
# 2-bucket — surplus геометрически не помещается, cap_skipped=True
({"1k": 90, "studio": 10}, True, "{1k:90, studio:10}"),
({"1k": 70, "studio": 30}, True, "{1k:70, studio:30}"),
({"1k": 99, "studio": 1}, True, "{1k:99, studio:1}"),
],
)
def test_cap_reproduced_failing_cases(
deals: dict[str, int], expect_pathological: bool, label: str
) -> None:
"""Review round-2 reproduced cases: 2-bucket — pathological, 3-bucket — fit cap."""
result, cap_skipped = _cap_and_redistribute(deals)
assert (
cap_skipped == expect_pathological
), f"cap_skipped={cap_skipped} ожидали {expect_pathological} для {label}"
assert sum(result.values()) == 100, f"sum != 100 для {label} → {result}"
if not expect_pathological:
assert (
max(result.values()) <= MAX_BUCKET_SHARE_PCT
), f"max={max(result.values())} > {MAX_BUCKET_SHARE_PCT} для {label} → {result}"
def test_cap_iteration_count_bounded() -> None:
"""Round 2 regression: алгоритм завершается за ≤ len(pct_map)+1 итераций.
Round 1 bag: на 2-bucket {1k:70, studio:30} цикл осциллировал бесконечно.
Round 2 fix: capacity-aware redistribute + hard `for _ in range(N+1)` guard.
Этот тест гарантирует что вызов не зависает (pytest-timeout не нужен).
"""
import time
pathological_cases = [
{"1k": 70, "studio": 30},
{"1k": 99, "studio": 1},
{"1k": 90, "studio": 10},
{"1k": 50, "studio": 50},
]
for case in pathological_cases:
start = time.perf_counter()
result, cap_skipped = _cap_and_redistribute(case)
elapsed_ms = (time.perf_counter() - start) * 1000
assert elapsed_ms < 100, f"Завис ({elapsed_ms:.0f}ms) на {case}"
assert sum(result.values()) == 100, f"sum != 100 для {case}"
# 2-bucket с одним > cap всегда pathological (surplus > free capacity)
if case != {"1k": 50, "studio": 50}:
assert cap_skipped, f"Ожидали cap_skipped=True для {case}"
def test_cap_and_redistribute_no_dominant_unchanged() -> None:
"""Если все bucket'ы ≤ cap — результат идентичен входу (fast-path)."""
pct_map = {"studio": 20, "1": 35, "2": 30, "3": 15}
result, cap_skipped = _cap_and_redistribute(pct_map)
assert not cap_skipped
assert result == pct_map
def test_cap_and_redistribute_empty() -> None:
"""Пустой dict → возвращается как есть."""
result, cap_skipped = _cap_and_redistribute({})
assert result == {}
assert not cap_skipped
def test_cap_skipped_flag_propagates_to_recommendation() -> None:
"""Pathological case → cap_skipped=True в recommendation_for_tz ответа."""
# 2 bucket'а по 50% — pathological
id_rows = [_obj_id_row(1), _obj_id_row(2)]
vel_rows = [
_vel_row("studio", deals_window=50.0, obj_ids=[1]),
_vel_row("1", deals_window=50.0, obj_ids=[2]),
]
db = _make_db(coord=_coord_row(), id_rows=id_rows, vel_rows=vel_rows)
req = _request(time_window="last_quarter")
resp = get_best_layouts(db, CAD_NUM, req)
# С deals 50/50 → normalize_pct даёт {studio:50, 1:50} — оба выше cap
assert resp.recommendation_for_tz.cap_skipped is True
def test_cap_skipped_false_for_normal_case() -> None:
"""Normal case с capping → cap_skipped=False в recommendation_for_tz."""
id_rows = [_obj_id_row(1), _obj_id_row(2), _obj_id_row(3)]
vel_rows = [
_vel_row("1k", deals_window=75.0, obj_ids=[1]),
_vel_row("studio", deals_window=15.0, obj_ids=[2]),
_vel_row("2k", deals_window=10.0, obj_ids=[3]),
]
db = _make_db(coord=_coord_row(), id_rows=id_rows, vel_rows=vel_rows)
req = _request(time_window="last_quarter")
resp = get_best_layouts(db, CAD_NUM, req)
assert resp.recommendation_for_tz.cap_skipped is False
mix = resp.recommendation_for_tz.mix
assert all(row.pct <= MAX_BUCKET_SHARE_PCT for row in mix)
assert sum(row.pct for row in mix) == 100

4
backend/tests/services/test_cadastre_bulk.py
View file

@ -1162,7 +1162,7 @@ async def test_grid_walk_emits_heartbeat_callbacks() -> None:
progress_states: list[dict[str, Any]] = []
upserted, requests = await _grid_walk_category(
_upserted, requests = await _grid_walk_category(
db=db,
client=client,
quarter="66:41:0303161",
@ -1197,7 +1197,7 @@ async def test_grid_walk_no_heartbeat_when_callback_none() -> None:
client.wms_feature_info = AsyncMock(return_value=[])
# Не передаём update_progress — должно не падать
upserted, requests = await _grid_walk_category(
_upserted, requests = await _grid_walk_category(
db=db,
client=client,
quarter="66:41:0303161",

355
backend/tests/services/test_ekburg_permits.py Normal file
View file

@ -0,0 +1,355 @@
"""Тесты для ekburg_permits.py (Issue #105).
Использует mock httpx + синтетические данные xlsx (openpyxl в памяти).
Не требует реального PostgreSQL или сетевого доступа.
"""
from __future__ import annotations
from datetime import date, datetime
from io import BytesIO
from typing import Any
from unittest.mock import MagicMock
import httpx
import pytest
from openpyxl import Workbook
from app.services.scrapers.ekburg_permits import (
EKBURG_PERMITS_URLS,
EkburgPermitsClient,
_clean_inn,
_detect_permit_type,
_to_date,
_to_float,
_to_str,
)
# ── helpers ───────────────────────────────────────────────────────────────────
def _make_xlsx_bytes(sheets: dict[str, list[tuple[Any, ...]]]) -> bytes:
"""Создать xlsx в памяти с заданными листами и строками."""
wb = Workbook()
wb.remove(wb.active) # удалить дефолтный лист
for sheet_name, rows in sheets.items():
ws = wb.create_sheet(sheet_name)
for row in rows:
ws.append(list(row))
buf = BytesIO()
wb.save(buf)
return buf.getvalue()
def _rns_sheet_rows() -> list[tuple[Any, ...]]:
"""Синтетические строки для листа РНС (Form 3).
Воспроизводит структуру 2024-2026: header row 4, данные с row 7.
"""
return [
# row 1: пусто
(None,) * 14,
# row 2: заголовок таблицы
(
"Таблица 3. Реестр выданных разрешений на строительство "
"объектов капитального строительства",
)
+ (None,) * 13,
# row 3: пусто
(None,) * 14,
# row 4: заголовки колонок
(
"Наименование застройщика",
"ИНН",
"Адрес застройщика",
"Тип строительного объекта1",
"Наименование объекта КС",
"Кадастровый номер ЗУ",
"Координаты X",
None,
"Адрес объекта",
"Реквизиты разрешения (номер)",
None,
"Дата окончания",
"Общая площадь, м2",
"Площадь жилых помещений по проекту, м2",
),
# row 5: подзаголовки
(None,) * 7 + ("X", "Y") + (None,) * 3 + ("номер", "дата") + (None,) * 2,
# row 6: нумерация столбцов
tuple(range(1, 15)),
# row 7: первая строка данных (МКД — ООО "Тест")
(
'ООО "Специализированный застройщик "Тест"',
6685000001,
"620000, г. Екатеринбург, ул. Тестовая, д. 1",
"многоквартирные жилые дома;",
"Многоквартирный жилой дом (№ 1 по ПЗУ)",
"66:41:0101001:123",
"1530000.0000",
"380000.0000",
"Свердловская обл., г. Екатеринбург, ул. Тестовая",
"66-41-99-2024",
datetime(2024, 3, 15),
datetime(2027, 3, 15),
12345.67,
8000.0,
),
# row 8: строка без permit_number (должна быть пропущена)
(
"ООО «Продолжение»",
6685000002,
"620001, г. Екатеринбург",
"производственные здания;",
"Производственный корпус",
"66:41:0101002:456",
"1531000.0000",
"381000.0000",
"Свердловская обл., г. Екатеринбург, ул. Другая",
None, # no permit_number → skip
datetime(2024, 5, 20),
datetime(2026, 5, 20),
500.0,
None,
),
# row 9: empty row (all None) — should be skipped
(None,) * 14,
]
def _rve_sheet_rows() -> list[tuple[Any, ...]]:
"""Синтетические строки для листа РВЭ (Form 4)."""
return [
(None,) * 19,
("Таблица 4. Реестр выданных разрешений на ввод в эксплуатацию",) + (None,) * 18,
(None,) * 19,
# row 4: заголовки
(
"Наименование застройщика",
"ИНН",
"Адрес застройщика",
"Тип строительного объекта1",
"Наименование объекта КС",
"Кадастровый номер ЗУ",
"Координаты X",
None,
"Адрес объекта",
"Реквизиты РНС (номер)",
None,
"Дата окончания РНС",
"Общая площадь, м2",
"Площадь жилых по проекту, м2",
"Площадь жилых фактически, м2",
"Реквизиты РВЭ (номер)",
None,
"Введённые мощности",
None,
),
(None,) * 7 + ("X", "Y") + (None,) * 5 + ("номер", "дата") + (None,) * 4,
tuple(range(1, 19)),
# row 7: первая строка данных
(
'ООО "Застройщик Ввода"',
6658000002,
"620100, г. Екатеринбург, ул. Вводная, д. 2",
"многоквартирные жилые дома;",
"Жилой дом (1 очередь) (№ 1 по ПЗУ)",
"66:41:0303001:789",
"1535000.0000",
"385000.0000",
"Свердловская обл., г. Екатеринбург, ул. Вводная",
"66-41-50-2022",
datetime(2022, 10, 1),
datetime(2026, 1, 1),
30000.0,
20000.0,
19800.5, # living_area_fact
"66-41-5-2026",
datetime(2026, 1, 13),
None,
None,
),
]
# ── unit tests: helpers ───────────────────────────────────────────────────────
class TestHelpers:
def test_to_date_datetime(self) -> None:
dt = datetime(2024, 3, 15, 0, 0)
assert _to_date(dt) == date(2024, 3, 15)
def test_to_date_string_dot(self) -> None:
assert _to_date("15.03.2024") == date(2024, 3, 15)
def test_to_date_none(self) -> None:
assert _to_date(None) is None
def test_to_date_invalid(self) -> None:
assert _to_date("not-a-date") is None
def test_to_float_comma(self) -> None:
assert _to_float("12345,67") == pytest.approx(12345.67)
def test_to_float_dash(self) -> None:
assert _to_float("-") is None
def test_to_float_int(self) -> None:
assert _to_float(1490) == pytest.approx(1490.0)
def test_to_str_nbsp(self) -> None:
assert _to_str("г.\xa0Екатеринбург") == "г. Екатеринбург"
def test_to_str_none(self) -> None:
assert _to_str(None) is None
def test_clean_inn_int(self) -> None:
assert _clean_inn(6685180480) == "6685180480"
def test_clean_inn_float(self) -> None:
# openpyxl иногда выдаёт float для длинных чисел
assert _clean_inn(6685180480.0) == "6685180480"
def test_clean_inn_string(self) -> None:
assert _clean_inn("6685180480") == "6685180480"
def test_clean_inn_none(self) -> None:
assert _clean_inn(None) is None
def test_clean_inn_short(self) -> None:
# 9 цифр — не ИНН
assert _clean_inn(123456789) is None
def test_detect_permit_type_rns(self) -> None:
assert _detect_permit_type("реестр разрешений на строительс") == "RNS"
def test_detect_permit_type_rve(self) -> None:
assert _detect_permit_type("реестр разрешений на ввод") == "RVE"
def test_detect_permit_type_unknown(self) -> None:
assert _detect_permit_type("Справочник") is None
# ── unit tests: xlsx parsing ──────────────────────────────────────────────────
class TestParseXlsx:
def _make_client(self) -> EkburgPermitsClient:
client = EkburgPermitsClient.__new__(EkburgPermitsClient)
# Создаём mock httpx.Client чтобы не открывать реальные соединения
client._client = MagicMock()
return client
def test_parse_rns_yields_permit_row(self) -> None:
content = _make_xlsx_bytes(
{
"реестр разрешений на строительс": _rns_sheet_rows(),
"Справочник": [("многоквартирные жилые дома;",)],
}
)
client = self._make_client()
rows = list(client.parse_xlsx(content, 2024, EKBURG_PERMITS_URLS[2024]))
# Должна быть ровно одна строка (вторая без permit_number пропущена)
assert len(rows) == 1
row = rows[0]
assert row.permit_type == "RNS"
assert row.permit_number == "66-41-99-2024"
assert row.issue_date == date(2024, 3, 15)
assert row.expiry_date == date(2027, 3, 15)
assert row.developer_inn == "6685000001"
assert row.object_type == "многоквартирные жилые дома;"
assert row.total_area_sqm == pytest.approx(12345.67)
assert row.living_area_sqm == pytest.approx(8000.0)
assert row.source_year == 2024
def test_parse_rve_yields_permit_row_with_rve_fields(self) -> None:
content = _make_xlsx_bytes({"реестр разрешений на ввод": _rve_sheet_rows()})
client = self._make_client()
rows = list(client.parse_xlsx(content, 2026, EKBURG_PERMITS_URLS[2026]))
assert len(rows) == 1
row = rows[0]
assert row.permit_type == "RVE"
assert row.permit_number == "66-41-50-2022"
assert row.living_area_fact_sqm == pytest.approx(19800.5)
assert row.rve_number == "66-41-5-2026"
assert row.rve_date == date(2026, 1, 13)
def test_parse_skips_справочник_sheet(self) -> None:
content = _make_xlsx_bytes(
{
"Справочник": [("многоквартирные жилые дома;",)],
"Лист1": [(None,)],
}
)
client = self._make_client()
rows = list(client.parse_xlsx(content, 2026, EKBURG_PERMITS_URLS[2026]))
assert rows == []
def test_raw_row_stored(self) -> None:
content = _make_xlsx_bytes({"реестр разрешений на строительс": _rns_sheet_rows()})
client = self._make_client()
rows = list(client.parse_xlsx(content, 2024, EKBURG_PERMITS_URLS[2024]))
assert len(rows) == 1
# raw_row должен содержать dict с string keys
assert isinstance(rows[0].raw_row, dict)
assert "9" in rows[0].raw_row # колонка 9 = permit_number
# ── unit tests: download_xlsx ─────────────────────────────────────────────────
class TestDownloadXlsx:
def _client_with_mock_http(self, mock_response: MagicMock) -> EkburgPermitsClient:
"""Создать EkburgPermitsClient с замоканным внутренним httpx.Client."""
client = EkburgPermitsClient.__new__(EkburgPermitsClient)
mock_http = MagicMock()
mock_http.get.return_value = mock_response
client._client = mock_http
return client
def test_download_returns_bytes(self) -> None:
fake_content = b"PK\x03\x04fake_xlsx_content"
mock_response = MagicMock()
mock_response.content = fake_content
mock_response.raise_for_status = MagicMock()
client = self._client_with_mock_http(mock_response)
result = client.download_xlsx(2026)
assert result == fake_content
def test_download_raises_for_unknown_year(self) -> None:
client = EkburgPermitsClient.__new__(EkburgPermitsClient)
client._client = MagicMock()
with pytest.raises(ValueError, match="year=9999"):
client.download_xlsx(9999)
def test_download_raises_on_http_error(self) -> None:
mock_response = MagicMock()
mock_response.raise_for_status.side_effect = httpx.HTTPStatusError(
"404", request=MagicMock(), response=MagicMock()
)
client = self._client_with_mock_http(mock_response)
with pytest.raises(httpx.HTTPStatusError):
client.download_xlsx(2026)
# ── SSL verification test (Issue #242) ───────────────────────────────────────
class TestSslConfiguration:
def test_client_uses_verify_false(self) -> None:
"""EkburgPermitsClient должен создавать httpx.Client с verify=False.
екатеринбург.рф использует CA Минцифры РФ — не в certifi bundle.
Issue #242: SSL: CERTIFICATE_VERIFY_FAILED для всех 5 годов.
"""
with EkburgPermitsClient() as client:
# httpx.Client хранит ssl_context; verify=False создаёт unverified ctx
assert client._client._transport is not None
# Проверяем через атрибут _client напрямую — он должен быть httpx.Client
assert isinstance(client._client, httpx.Client)

322
backend/tests/services/test_nspd_denorm.py Normal file
View file

@ -0,0 +1,322 @@
"""Тесты для nspd_denorm.py — coerce helpers + denorm_parcel/building/dump.
Не требует реального PostgreSQL — мокает db.execute() и db.begin_nested().
"""
from __future__ import annotations
from typing import Any
from unittest.mock import MagicMock
import pytest
from app.services.scrapers.nspd_denorm import (
_coerce_int,
_coerce_numeric,
denorm_building_feature,
denorm_dump,
denorm_parcel_feature,
)
# ── _coerce_int ────────────────────────────────────────────────────────────────
def test_coerce_int_str_number() -> None:
assert _coerce_int("5") == 5
def test_coerce_int_int_passthrough() -> None:
assert _coerce_int(17) == 17
def test_coerce_int_none_returns_none() -> None:
assert _coerce_int(None) is None
def test_coerce_int_invalid_returns_none() -> None:
assert _coerce_int("не_число") is None
def test_coerce_int_float_truncates() -> None:
assert _coerce_int(3.9) == 3
# ── _coerce_numeric ────────────────────────────────────────────────────────────
def test_coerce_numeric_float_passthrough() -> None:
assert _coerce_numeric(12.5) == 12.5
def test_coerce_numeric_str_dot() -> None:
assert _coerce_numeric("1234567.89") == pytest.approx(1234567.89)
def test_coerce_numeric_comma_decimal() -> None:
"""Европейский формат с запятой → корректный float."""
assert _coerce_numeric("1234567,89") == pytest.approx(1234567.89)
def test_coerce_numeric_none_returns_none() -> None:
assert _coerce_numeric(None) is None
def test_coerce_numeric_invalid_returns_none() -> None:
assert _coerce_numeric("N/A") is None
# ── Helpers для моков ──────────────────────────────────────────────────────────
def _make_mock_session() -> MagicMock:
"""Мок Session: begin_nested() — context manager, execute() → MagicMock."""
db = MagicMock()
# begin_nested() должен быть context manager
cm = MagicMock()
cm.__enter__ = MagicMock(return_value=None)
cm.__exit__ = MagicMock(return_value=False)
db.begin_nested.return_value = cm
return db
def _parcel_feature(
cad_num: str = "66:41:0204016:10",
area: Any = "500.0",
cost_value: Any = "1500000",
**extra_props: Any,
) -> dict[str, Any]:
props: dict[str, Any] = {
"cad_num": cad_num,
"area": area,
"cost_value": cost_value,
"permitted_use": "Для ИЖС",
"land_category": "Земли населённых пунктов",
"address": "г. Екатеринбург",
**extra_props,
}
return {
"layer": "parcels",
"feature_id": "123",
"geometry": {"type": "Polygon", "coordinates": [[[0, 0], [1, 0], [1, 1], [0, 0]]]},
"properties": props,
}
def _building_feature(
cad_num: str = "66:41:0204016:10:1",
purpose: str = "Многоквартирный дом",
**extra_props: Any,
) -> dict[str, Any]:
props: dict[str, Any] = {
"cad_num": cad_num,
"purpose": purpose,
"floors_above_ground": 9,
"floors_underground": 1,
"year_built": 1985,
"cost_value": "50000000",
"build_record_area": "3200.0",
"address": "ул. Ленина 1",
**extra_props,
}
return {
"layer": "buildings",
"feature_id": "456",
"geometry": {"type": "Polygon", "coordinates": [[[0, 0], [1, 0], [1, 1], [0, 0]]]},
"properties": props,
}
# ── denorm_parcel_feature ──────────────────────────────────────────────────────
def test_denorm_parcel_feature_inserts() -> None:
"""Полный feature с cad_num → вызывает db.execute с правильными параметрами."""
db = _make_mock_session()
result = denorm_parcel_feature(
db,
feature=_parcel_feature(),
quarter_cad="66:41:0204016",
snapshot_date="2026-05-16",
)
assert result is True
db.begin_nested.assert_called_once()
db.execute.assert_called_once()
# Проверяем что параметры содержат нужные ключи
call_kwargs = db.execute.call_args[0][1] # positional dict
assert call_kwargs["cad_num"] == "66:41:0204016:10"
assert call_kwargs["quarter_cad"] == "66:41:0204016"
assert call_kwargs["permitted_use"] == "Для ИЖС"
assert call_kwargs["land_category"] == "Земли населённых пунктов"
assert call_kwargs["area_sqm"] == 500.0
assert call_kwargs["cost_value"] == 1500000.0
assert call_kwargs["cost_per_m2"] == pytest.approx(3000.0)
assert call_kwargs["snapshot_date"] == "2026-05-16"
def test_denorm_parcel_no_cad_num_skipped() -> None:
"""Feature без cad_num → возвращает False, db не вызывается."""
db = _make_mock_session()
feature: dict[str, Any] = {
"layer": "parcels",
"feature_id": None,
"geometry": None,
"properties": {"area": "100"},
}
result = denorm_parcel_feature(
db, feature=feature, quarter_cad="66:41:0204016", snapshot_date="2026-05-16"
)
assert result is False
db.execute.assert_not_called()
def test_denorm_parcel_zero_area_cost_per_m2_none() -> None:
"""area=0 → cost_per_m2 = None (нет деления на ноль)."""
db = _make_mock_session()
result = denorm_parcel_feature(
db,
feature=_parcel_feature(area="0", cost_value="1000"),
quarter_cad="66:41:0204016",
snapshot_date="2026-05-16",
)
assert result is True
call_kwargs = db.execute.call_args[0][1]
assert call_kwargs["cost_per_m2"] is None
def test_denorm_parcel_null_geometry() -> None:
"""feature без geometry → geom_json=None, INSERT всё равно вызывается."""
db = _make_mock_session()
feature = _parcel_feature()
feature["geometry"] = None
result = denorm_parcel_feature(
db, feature=feature, quarter_cad="66:41:0204016", snapshot_date="2026-05-16"
)
assert result is True
call_kwargs = db.execute.call_args[0][1]
assert call_kwargs["geom_json"] is None
def test_denorm_parcel_db_exception_returns_false() -> None:
"""DB execute raises → возвращает False (не propagate)."""
db = _make_mock_session()
db.execute.side_effect = Exception("DB constraint violation")
result = denorm_parcel_feature(
db,
feature=_parcel_feature(),
quarter_cad="66:41:0204016",
snapshot_date="2026-05-16",
)
assert result is False
# ── denorm_building_feature ────────────────────────────────────────────────────
def test_denorm_building_feature_inserts() -> None:
"""Полный building feature → INSERT с правильными параметрами."""
db = _make_mock_session()
result = denorm_building_feature(
db,
feature=_building_feature(),
quarter_cad="66:41:0204016",
snapshot_date="2026-05-16",
)
assert result is True
db.execute.assert_called_once()
call_kwargs = db.execute.call_args[0][1]
assert call_kwargs["cad_num"] == "66:41:0204016:10:1"
assert call_kwargs["purpose"] == "Многоквартирный дом"
assert call_kwargs["floors"] == 9
assert call_kwargs["floors_underground"] == 1
assert call_kwargs["year_built"] == 1985
assert call_kwargs["cost_value"] == 50000000.0
assert call_kwargs["build_record_area"] == 3200.0
def test_denorm_building_no_cad_num_skipped() -> None:
"""Building feature без cad_num → False."""
db = _make_mock_session()
feature: dict[str, Any] = {
"layer": "buildings",
"feature_id": None,
"geometry": None,
"properties": {"purpose": "Нежилое здание"},
}
result = denorm_building_feature(
db, feature=feature, quarter_cad="66:41:0204016", snapshot_date="2026-05-16"
)
assert result is False
db.execute.assert_not_called()
def test_denorm_building_str_floors_coerced() -> None:
"""floors_above_ground строкой → корректно парсится в int."""
db = _make_mock_session()
result = denorm_building_feature(
db,
feature=_building_feature(floors_above_ground="12"),
quarter_cad="66:41:0204016",
snapshot_date="2026-05-16",
)
assert result is True
call_kwargs = db.execute.call_args[0][1]
assert call_kwargs["floors"] == 12
# ── denorm_dump ────────────────────────────────────────────────────────────────
def test_denorm_dump_aggregates() -> None:
"""3 parcels + 2 buildings + 1 unknown layer → правильные счётчики."""
db = _make_mock_session()
features: list[dict[str, Any]] = [
_parcel_feature("66:41:0101001:1"),
_parcel_feature("66:41:0101001:2"),
_parcel_feature("66:41:0101001:3"),
_building_feature("66:41:0101001:1:1"),
_building_feature("66:41:0101001:1:2"),
{
"layer": "territorial_zones",
"feature_id": "tz-1",
"geometry": None,
"properties": {"type_zone": "Ж-1"},
},
]
counts = denorm_dump(db, quarter_cad="66:41:0101001", features=features)
assert counts["parcels"] == 3
assert counts["buildings"] == 2
# territorial_zones не считается как ошибка — просто пропускается
assert counts["errors"] == 0
db.commit.assert_called_once()
def test_denorm_dump_empty_features() -> None:
"""Пустой список features → нули, commit всё равно вызывается."""
db = _make_mock_session()
counts = denorm_dump(db, quarter_cad="66:41:0101001", features=[])
assert counts == {"parcels": 0, "buildings": 0, "errors": 0}
db.commit.assert_called_once()
def test_denorm_dump_no_cad_num_counted_as_error() -> None:
"""Parcel без cad_num → denorm_parcel_feature returns False → errors += 1."""
db = _make_mock_session()
feature: dict[str, Any] = {
"layer": "parcels",
"feature_id": None,
"geometry": None,
"properties": {},
}
counts = denorm_dump(db, quarter_cad="66:41:0101001", features=[feature])
assert counts["parcels"] == 0
assert counts["errors"] == 1

176
backend/tests/services/test_objective_backfill.py Normal file
View file

@ -0,0 +1,176 @@
"""Тесты для objective_backfill.py (#203).
Использует mock DB (unittest.mock) — не требует реального PostgreSQL.
"""
from __future__ import annotations
from typing import Any
from unittest.mock import MagicMock, patch
from app.services.etl.objective_backfill import (
AUTO_ACCEPT_THRESHOLD,
REVIEW_THRESHOLD,
MatchCandidate,
auto_apply_matches,
find_match_candidates,
trigger_mv_refresh,
)
# ── Helpers ───────────────────────────────────────────────────────────────────
def _make_candidate(
obj_id: int,
comm_name: str,
project_name: str,
score: float,
dev_name: str | None = "ООО Девелопер",
) -> MatchCandidate:
return MatchCandidate(
domrf_obj_id=obj_id,
domrf_comm_name=comm_name,
domrf_dev_name=dev_name,
objective_project_name=project_name,
similarity_score=score,
)
def _make_db_row(obj_id: int, comm_name: str, dev_name: str, project: str, score: float) -> Any:
"""Имитирует Row, возвращаемый SQLAlchemy db.execute().all()."""
return (obj_id, comm_name, dev_name, project, score)
# ── Test 1: find_match_candidates возвращает правильные MatchCandidate ────────
def test_find_match_candidates_returns_candidates() -> None:
"""find_match_candidates корректно преобразует DB rows в MatchCandidate."""
mock_rows = [
_make_db_row(100, "Северный квартал", "Брусника", "Северный квартал", 1.0),
_make_db_row(200, "АЛЛЕГРО", "СЗ ГОРЖИЛСТРОЙ", "АЛЛЕГРО", 0.95),
_make_db_row(300, "Некий ЖК", None, "Близкий ЖК", 0.72),
]
mock_execute = MagicMock()
mock_execute.all.return_value = mock_rows
mock_db = MagicMock()
mock_db.execute.return_value = mock_execute
candidates = find_match_candidates(mock_db, only_unmapped=True)
assert len(candidates) == 3
assert candidates[0].domrf_obj_id == 100
assert candidates[0].similarity_score == 1.0
assert candidates[0].domrf_dev_name == "Брусника"
assert candidates[2].domrf_dev_name is None # NULL dev_name → None
assert candidates[2].similarity_score == 0.72
# Убедимся, что db.execute вызывался с параметрами
mock_db.execute.assert_called_once()
call_kwargs = mock_db.execute.call_args[0][1]
assert call_kwargs["only_unmapped"] is True
assert call_kwargs["min_threshold"] == REVIEW_THRESHOLD
# ── Test 2: auto_apply_matches dry_run не вставляет ──────────────────────────
def test_auto_apply_matches_dry_run_no_inserts() -> None:
"""dry_run=True возвращает счётчики без обращения к БД (execute не вызывается)."""
mock_db = MagicMock()
candidates = [
_make_candidate(1, "ЖК А", "ЖК А", 0.95), # auto-accept
_make_candidate(2, "ЖК Б", "ЖК Б", 0.70), # review queue
_make_candidate(3, "ЖК В", "ЖК В", 0.65), # review queue
]
result = auto_apply_matches(mock_db, candidates, dry_run=True)
assert result["auto_accepted"] == 0
assert result["review_queue"] == 2
mock_db.execute.assert_not_called()
mock_db.commit.assert_not_called()
# ── Test 3: auto_apply_matches high-score inserts, low-score → review_queue ──
def test_auto_apply_matches_inserts_high_score_only() -> None:
"""Только кандидаты >= AUTO_ACCEPT_THRESHOLD вставляются в БД."""
# Мокируем begin_nested как context manager
savepoint_cm = MagicMock()
savepoint_cm.__enter__ = MagicMock(return_value=None)
savepoint_cm.__exit__ = MagicMock(return_value=False)
mock_execute_result = MagicMock()
mock_execute_result.rowcount = 1
mock_db = MagicMock()
mock_db.begin_nested.return_value = savepoint_cm
mock_db.execute.return_value = mock_execute_result
high_score = AUTO_ACCEPT_THRESHOLD # ровно на пороге — должен вставляться
low_score = REVIEW_THRESHOLD # ровно REVIEW_THRESHOLD — не auto-accept
candidates = [
_make_candidate(10, "ЖК Альфа", "ЖК Альфа", high_score),
_make_candidate(11, "ЖК Бета", "ЖК Бета", high_score + 0.05),
_make_candidate(20, "ЖК Гамма", "ЖК Гамма", low_score), # review только
]
result = auto_apply_matches(mock_db, candidates, threshold=AUTO_ACCEPT_THRESHOLD)
assert result["auto_accepted"] == 2
assert result["review_queue"] == 1
assert mock_db.execute.call_count == 2 # вызывался только для high-score
mock_db.commit.assert_called_once()
# ── Test 4: trigger_mv_refresh делегирует в layout_velocity_refresh ──────────
def test_trigger_mv_refresh_calls_helper() -> None:
"""trigger_mv_refresh вызывает refresh_layout_velocity с concurrently=True.
refresh_layout_velocity импортируется лениво внутри trigger_mv_refresh,
поэтому патчим по полному пути модуля-источника.
"""
mock_db = MagicMock()
with patch(
"app.services.site_finder.layout_velocity_refresh.refresh_layout_velocity",
return_value=512,
) as mock_refresh:
count = trigger_mv_refresh(mock_db)
assert count == 512
mock_refresh.assert_called_once_with(mock_db, concurrently=True)
# ── Test 5: ON CONFLICT — rowcount=0 считается как skipped ───────────────────
def test_auto_apply_matches_conflict_counted_as_skipped() -> None:
"""Если INSERT вернул rowcount=0 (ON CONFLICT DO NOTHING), считается skipped."""
savepoint_cm = MagicMock()
savepoint_cm.__enter__ = MagicMock(return_value=None)
savepoint_cm.__exit__ = MagicMock(return_value=False)
mock_execute_result = MagicMock()
mock_execute_result.rowcount = 0 # ON CONFLICT DO NOTHING
mock_db = MagicMock()
mock_db.begin_nested.return_value = savepoint_cm
mock_db.execute.return_value = mock_execute_result
candidates = [
_make_candidate(99, "Уже существующий ЖК", "Уже существующий ЖК", 0.99),
]
result = auto_apply_matches(mock_db, candidates)
assert result["auto_accepted"] == 0
assert result["skipped"] == 1
assert result["review_queue"] == 0

620
backend/tests/services/test_quarter_dump_lookup.py Normal file
View file

@ -0,0 +1,620 @@
"""Тесты для quarter_dump_lookup.py — risk zones + generic layer extraction + TIER 4.
Покрывает:
- _extract_features_by_layer: filter by prefix
- _get_risk_zones / extract through get_quarter_dump_data: intersect, no-intersect,
no-risks-in-dump
- _get_opportunity_parcels: auction/scheme/free/future/oopt layers, distance sort, empty
- _get_red_lines: intersecting, nearby-only, empty, early-exit
- EMPTY_DUMP_RESULT / make_empty_result: new keys присутствуют
Не требует реального PostgreSQL — мокает db.execute().
"""
from __future__ import annotations
from typing import Any
from unittest.mock import MagicMock
import pytest
from app.services.site_finder.quarter_dump_lookup import (
EMPTY_DUMP_RESULT,
_extract_features_by_layer,
_get_cad_zouit_overlaps,
_get_opportunity_parcels,
_get_red_lines,
_get_risk_zones,
derive_quarter_cad,
get_quarter_dump_data,
make_empty_result,
)
# ── Fixtures & helpers ────────────────────────────────────────────────────────
def _make_feature(layer: str, geom: dict[str, Any] | None = None) -> dict[str, Any]:
"""Минимальный feature dict в формате features_json."""
return {
"layer": layer,
"feature_id": "test-id",
"geometry": geom or {"type": "Polygon", "coordinates": [[[0, 0], [1, 0], [1, 1], [0, 0]]]},
"properties": {"name": f"test-{layer}", "type_zone": f"zone-{layer}"},
}
# ── _extract_features_by_layer ────────────────────────────────────────────────
def test_extract_features_by_layer_prefix_match() -> None:
"""Возвращает только features с нужным prefix."""
features = [
_make_feature("risk_flooding"),
_make_feature("risk_burns"),
_make_feature("zouit_engineering"),
_make_feature("parcels"),
_make_feature("risk_landslide"),
]
result = _extract_features_by_layer(features, "risk_")
assert len(result) == 3
layers = {f["layer"] for f in result}
assert layers == {"risk_flooding", "risk_burns", "risk_landslide"}
def test_extract_features_by_layer_zouit_prefix() -> None:
"""Работает для zouit_ prefix — generic re-use."""
features = [
_make_feature("zouit_engineering"),
_make_feature("zouit_okn"),
_make_feature("risk_flooding"),
]
result = _extract_features_by_layer(features, "zouit_")
assert len(result) == 2
assert all(f["layer"].startswith("zouit_") for f in result)
def test_extract_features_by_layer_empty_list() -> None:
result = _extract_features_by_layer([], "risk_")
assert result == []
def test_extract_features_by_layer_no_match() -> None:
features = [_make_feature("parcels"), _make_feature("buildings")]
result = _extract_features_by_layer(features, "risk_")
assert result == []
def test_extract_features_by_layer_non_string_layer() -> None:
"""Gracefully skips features with non-string layer."""
features: list[dict[str, Any]] = [
{"layer": None, "feature_id": "x", "geometry": None, "properties": {}},
_make_feature("risk_flooding"),
]
result = _extract_features_by_layer(features, "risk_")
assert len(result) == 1
assert result[0]["layer"] == "risk_flooding"
# ── _get_risk_zones ───────────────────────────────────────────────────────────
def _make_mock_db_with_rows(rows: list[tuple[Any, ...]]) -> MagicMock:
"""Мок Session где execute().fetchall() возвращает rows."""
db = MagicMock()
mock_result = MagicMock()
mock_result.fetchall.return_value = rows
db.execute.return_value = mock_result
return db
def test_get_risk_zones_intersects() -> None:
"""Три risk features → три записи в результате."""
rows: list[tuple[Any, ...]] = [
# (layer, props, geom_wkt, intersection_area_sqm)
(
"risk_flooding",
{"type_zone": "Затопление 1%"},
"POLYGON((0 0,1 0,1 1,0 0))",
1234.5,
),
(
"risk_burns",
{"name": "Гарь"},
"POLYGON((0 0,2 0,2 2,0 0))",
500.0,
),
(
"risk_landslide",
{},
"POLYGON((0 0,3 0,3 3,0 0))",
999.9,
),
]
db = _make_mock_db_with_rows(rows)
result = _get_risk_zones(db, "66:41:0204016", "POLYGON((0 0,1 0,1 1,0 0))")
assert len(result) == 3
flooding = next(r for r in result if r["layer"] == "risk_flooding")
assert flooding["subtype"] == "Затопление 1%"
assert flooding["intersection_area_sqm"] == 1234.5
assert flooding["geom_wkt"] == "POLYGON((0 0,1 0,1 1,0 0))"
burns = next(r for r in result if r["layer"] == "risk_burns")
assert burns["subtype"] == "Гарь"
landslide = next(r for r in result if r["layer"] == "risk_landslide")
# No properties name/type_zone → falls back to _RISK_SUBTYPE_LABELS
assert landslide["subtype"] == "Обвально-осыпные процессы"
def test_get_risk_zones_no_intersect() -> None:
"""Пустой fetchall → пустой список."""
db = _make_mock_db_with_rows([])
result = _get_risk_zones(db, "66:41:0204016", "POLYGON((0 0,1 0,1 1,0 0))")
assert result == []
def test_get_risk_zones_no_risks_in_dump() -> None:
"""risks_count == 0 → early exit, db.execute не вызывается."""
db = MagicMock()
layer_counts = {"risks_count": 0}
result = _get_risk_zones(db, "66:41:0204016", "POLYGON((0 0,1 0,1 1,0 0))", layer_counts)
assert result == []
db.execute.assert_not_called()
def test_get_risk_zones_db_exception_returns_empty() -> None:
"""DB exception → пустой список (не propagate)."""
db = MagicMock()
db.execute.side_effect = Exception("DB connection lost")
result = _get_risk_zones(db, "66:41:0204016", "POLYGON((0 0,1 0,1 1,0 0))")
assert result == []
def test_get_risk_zones_null_area() -> None:
"""NULL intersection_area_sqm → intersection_area_sqm=None в результате."""
rows = [("risk_erosion_water", {}, "POINT(0 0)", None)]
db = _make_mock_db_with_rows(rows)
result = _get_risk_zones(db, "66:41:0204016", "POLYGON((0 0,1 0,1 1,0 0))")
assert len(result) == 1
assert result[0]["intersection_area_sqm"] is None
# ── make_empty_result includes nspd_risk_zones ────────────────────────────────
def test_make_empty_result_has_risk_zones_key() -> None:
"""make_empty_result возвращает nspd_risk_zones: []."""
result = make_empty_result()
assert "nspd_risk_zones" in result
assert result["nspd_risk_zones"] == []
# ── _get_opportunity_parcels ─────────────────────────────────────────────────
def test_get_opportunity_parcels_finds_auction() -> None:
"""3 features (1 auction + 2 other types) — все возвращаются, auction в нужном layer."""
rows: list[tuple[Any, ...]] = [
# (layer, props, geom_wkt, distance_m)
(
"opportunity_auction_parcels",
{"cad_num": "66:41:0204016:101"},
"POLYGON((0 0,1 0,1 1,0 0))",
50.0,
),
(
"opportunity_scheme_parcels",
{"cadastral_number": "66:41:0204016:102"},
"POLYGON((1 1,2 1,2 2,1 1))",
150.0,
),
(
"opportunity_oopt",
{},
"POLYGON((2 2,3 2,3 3,2 2))",
300.0,
),
]
db = _make_mock_db_with_rows(rows)
result = _get_opportunity_parcels(db, "66:41:0204016", "POLYGON((0 0,1 0,1 1,0 0))")
assert len(result) == 3
auction = next(r for r in result if r["layer"] == "auction_parcels")
assert auction["cad_num"] == "66:41:0204016:101"
assert auction["distance_m"] == 50.0
scheme = next(r for r in result if r["layer"] == "scheme_parcels")
assert scheme["cad_num"] == "66:41:0204016:102"
oopt = next(r for r in result if r["layer"] == "oopt")
assert oopt["cad_num"] is None
assert oopt["distance_m"] == 300.0
def test_get_opportunity_parcels_distance_sort() -> None:
"""Результаты отсортированы по distance_m ASC (SQL ORDER BY передаётся DB)."""
rows: list[tuple[Any, ...]] = [
# Порядок: ближайший первым (SQL уже сортирует — тест проверяет что порядок сохраняется)
(
"opportunity_free_parcels",
{"cad_num": "66:41:0204016:10"},
"POINT(0 0)",
10.0,
),
(
"opportunity_auction_parcels",
{"cad_num": "66:41:0204016:20"},
"POINT(1 1)",
200.0,
),
(
"opportunity_future_parcels",
{},
"POINT(2 2)",
450.0,
),
]
db = _make_mock_db_with_rows(rows)
result = _get_opportunity_parcels(db, "66:41:0204016", "POLYGON((0 0,1 0,1 1,0 0))")
assert len(result) == 3
# Порядок из DB (mock возвращает в порядке rows) — ближайший первый
assert result[0]["distance_m"] == 10.0
assert result[1]["distance_m"] == 200.0
assert result[2]["distance_m"] == 450.0
def test_get_opportunity_parcels_empty() -> None:
"""Нет opportunity features — пустой список."""
db = _make_mock_db_with_rows([])
result = _get_opportunity_parcels(db, "66:41:0204016", "POLYGON((0 0,1 0,1 1,0 0))")
assert result == []
def test_get_opportunity_parcels_early_exit() -> None:
"""opportunity_count == 0 → early exit, db.execute не вызывается."""
db = MagicMock()
layer_counts = {"opportunity_count": 0}
result = _get_opportunity_parcels(
db, "66:41:0204016", "POLYGON((0 0,1 0,1 1,0 0))", layer_counts
)
assert result == []
db.execute.assert_not_called()
def test_get_opportunity_parcels_db_exception_returns_empty() -> None:
"""DB exception → пустой список (не propagate)."""
db = MagicMock()
db.execute.side_effect = Exception("connection lost")
result = _get_opportunity_parcels(db, "66:41:0204016", "POLYGON((0 0,1 0,1 1,0 0))")
assert result == []
# ── _get_red_lines ────────────────────────────────────────────────────────────
def test_get_red_lines_intersects() -> None:
"""Red line intersecting parcel → intersection_length_m filled, distance_m=None."""
rows: list[tuple[Any, ...]] = [
# (geom_wkt, does_intersect, intersection_length_m, distance_m)
(
"LINESTRING(0 0, 1 1)",
True,
45.3, # длина пересечения в метрах
0.0, # intersecting → distance_m becomes None in result
),
]
db = _make_mock_db_with_rows(rows)
result = _get_red_lines(db, "66:41:0204016", "POLYGON((0 0,1 0,1 1,0 0))")
assert len(result) == 1
r = result[0]
assert r["geom_wkt"] == "LINESTRING(0 0, 1 1)"
assert r["intersection_length_m"] == 45.3
assert r["distance_m"] is None # null when intersecting
def test_get_red_lines_nearby_only() -> None:
"""Red line nearby only (не intersect) → distance_m filled, intersection_length_m=None."""
rows: list[tuple[Any, ...]] = [
# (geom_wkt, does_intersect, intersection_length_m, distance_m)
(
"LINESTRING(10 10, 20 20)",
False,
0.0, # нет пересечения
85.5,
),
]
db = _make_mock_db_with_rows(rows)
result = _get_red_lines(db, "66:41:0204016", "POLYGON((0 0,1 0,1 1,0 0))")
assert len(result) == 1
r = result[0]
assert r["intersection_length_m"] is None
assert r["distance_m"] == 85.5
def test_get_red_lines_db_exception_returns_empty() -> None:
"""DB exception → пустой список (не propagate)."""
db = MagicMock()
db.execute.side_effect = Exception("DB connection lost")
result = _get_red_lines(db, "66:41:0204016", "POLYGON((0 0,1 0,1 1,0 0))")
assert result == []
def test_get_red_lines_empty() -> None:
"""Нет red lines features — пустой список."""
db = _make_mock_db_with_rows([])
result = _get_red_lines(db, "66:41:0204016", "POLYGON((0 0,1 0,1 1,0 0))")
assert result == []
def test_get_red_lines_early_exit() -> None:
"""red_lines_count == 0 → early exit, db.execute не вызывается."""
db = MagicMock()
layer_counts = {"red_lines_count": 0}
result = _get_red_lines(db, "66:41:0204016", "POLYGON((0 0,1 0,1 1,0 0))", layer_counts)
assert result == []
db.execute.assert_not_called()
# ── EMPTY_DUMP_RESULT / make_empty_result includes new TIER 4 keys ────────────
def test_empty_dump_result_has_opportunity_key() -> None:
"""EMPTY_DUMP_RESULT содержит nspd_opportunity_parcels: []."""
assert "nspd_opportunity_parcels" in EMPTY_DUMP_RESULT
assert EMPTY_DUMP_RESULT["nspd_opportunity_parcels"] == []
def test_empty_dump_result_has_red_lines_key() -> None:
"""EMPTY_DUMP_RESULT содержит nspd_red_lines: []."""
assert "nspd_red_lines" in EMPTY_DUMP_RESULT
assert EMPTY_DUMP_RESULT["nspd_red_lines"] == []
def test_make_empty_result_has_tier4_keys() -> None:
"""make_empty_result() возвращает nspd_opportunity_parcels и nspd_red_lines."""
result = make_empty_result()
assert "nspd_opportunity_parcels" in result
assert result["nspd_opportunity_parcels"] == []
assert "nspd_red_lines" in result
assert result["nspd_red_lines"] == []
# ── Issue #234: harvest_eta_seconds + SETNX dedupe ────────────────────────────
def test_empty_dump_result_has_harvest_eta_seconds_key() -> None:
"""EMPTY_DUMP_RESULT.nspd_dump содержит harvest_eta_seconds (issue #234)."""
assert "harvest_eta_seconds" in EMPTY_DUMP_RESULT["nspd_dump"]
assert EMPTY_DUMP_RESULT["nspd_dump"]["harvest_eta_seconds"] is None
def test_make_empty_result_passes_harvest_eta_seconds() -> None:
"""make_empty_result(harvest_eta_seconds=60) → пробрасывает в nspd_dump."""
result = make_empty_result(harvest_triggered=True, harvest_eta_seconds=60)
assert result["nspd_dump"]["harvest_eta_seconds"] == 60
assert result["nspd_dump"]["harvest_triggered"] is True
def test_make_empty_result_default_eta_is_none() -> None:
"""Если harvest_eta_seconds не передан — поле None (для available=False случая)."""
result = make_empty_result()
assert result["nspd_dump"]["harvest_eta_seconds"] is None
def test_acquire_harvest_lock_succeeds_first_call(monkeypatch: pytest.MonkeyPatch) -> None:
"""SETNX lock: первый вызов получает lock (True), второй — нет (False).
Issue #234: burst N concurrent analyze на один свежетриггеренный квартал
дедуплицируется через Redis SETNX. Мокаем redis.Redis.from_url.
"""
from app.services.site_finder import quarter_dump_lookup as qdl
# Mock redis: первый client.set возвращает True, второй — None (key existed).
calls = {"n": 0}
class _MockRedis:
def set(self, key: str, value: str, *, nx: bool, ex: int) -> bool | None:
calls["n"] += 1
return True if calls["n"] == 1 else None
monkeypatch.setattr(
"redis.Redis.from_url",
lambda _url: _MockRedis(),
)
assert qdl._acquire_harvest_lock("66:41:0204016") is True
assert qdl._acquire_harvest_lock("66:41:0204016") is False
assert calls["n"] == 2
def test_acquire_harvest_lock_redis_unavailable_returns_false(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Redis недоступен → _acquire_harvest_lock возвращает False (graceful).
Лучше не запустить дубль, чем нагрузить WAF при transient redis-fail.
"""
from app.services.site_finder import quarter_dump_lookup as qdl
def _boom(_url: str) -> None:
raise ConnectionError("redis down")
monkeypatch.setattr("redis.Redis.from_url", _boom)
assert qdl._acquire_harvest_lock("66:41:0204016") is False
# ── derive_quarter_cad (smoke tests) ──────────────────────────────────────────
@pytest.mark.parametrize(
"cad,expected",
[
("66:41:0204016", "66:41:0204016"),
("66:41:0204016:10", "66:41:0204016"),
("66:41:0204016:10:1", "66:41:0204016"),
("invalid", None),
("66:41", None),
],
)
def test_derive_quarter_cad(cad: str, expected: str | None) -> None:
assert derive_quarter_cad(cad) == expected
# ── #243: cad_zouit fallback when dump absent ─────────────────────────────────
def _make_mock_db_no_dump_with_cad_zouit(
cad_zouit_rows: list[tuple[Any, ...]],
) -> MagicMock:
"""Мок Session: nspd_quarter_dumps.first() → None; cad_zouit.fetchall() → rows."""
db = MagicMock()
first_result = MagicMock()
first_result.first.return_value = None # нет дампа
fetchall_result = MagicMock()
fetchall_result.fetchall.return_value = cad_zouit_rows
# Первый вызов execute (SELECT nspd_quarter_dumps) → first()=None
# Второй вызов execute (SELECT cad_zouit) → fetchall()=rows
db.execute.side_effect = [first_result, fetchall_result]
return db
def test_cad_zouit_fallback_fires_when_no_dump() -> None:
"""#243: cad_zouit fallback срабатывает когда nspd_quarter_dumps пуст глобально.
Ожидаем: nspd_zouit_overlaps непустой, source='cad_zouit'.
"""
cad_zouit_rows: list[tuple[Any, ...]] = [
# (type_zone, category_name, name_by_doc, reg_numb_border, id)
("Охранная зона трубопровода", "Трубопровод", "ГВС ул. Ленина", "66-66-00/123", 42),
]
db = _make_mock_db_no_dump_with_cad_zouit(cad_zouit_rows)
wkt = "POLYGON((60 56,61 56,61 57,60 57,60 56))"
result = get_quarter_dump_data(db, "66:41:0603016:194", wkt)
overlaps: list[dict[str, Any]] = result["nspd_zouit_overlaps"]
assert len(overlaps) == 1
assert overlaps[0]["source"] == "cad_zouit"
assert overlaps[0]["type_zone"] == "Охранная зона трубопровода"
assert overlaps[0]["name"] == "ГВС ул. Ленина"
# dump.available остаётся False (нет дампа)
assert result["nspd_dump"]["available"] is False
def test_cad_zouit_fallback_no_dump_no_parcel_wkt() -> None:
"""#243: dump=None + parcel_wkt=None → overlaps пустой, не вызываем cad_zouit."""
db = MagicMock()
first_result = MagicMock()
first_result.first.return_value = None
db.execute.return_value = first_result
result = get_quarter_dump_data(db, "66:41:0603016:194", None)
assert result["nspd_zouit_overlaps"] == []
# cad_zouit SELECT не должен вызываться (только один execute для dump lookup)
assert db.execute.call_count == 1
def test_cad_zouit_fallback_empty_when_no_overlaps() -> None:
"""#243: dump=None + cad_zouit возвращает 0 строк → nspd_zouit_overlaps пуст."""
db = _make_mock_db_no_dump_with_cad_zouit([])
wkt = "POLYGON((60 56,61 56,61 57,60 57,60 56))"
result = get_quarter_dump_data(db, "66:41:0603016:194", wkt)
assert result["nspd_zouit_overlaps"] == []
def test_get_cad_zouit_overlaps_returns_correct_schema() -> None:
"""_get_cad_zouit_overlaps: поля group_key/layer/source/type_zone корректны."""
rows: list[tuple[Any, ...]] = [
# (type_zone, category_name, name_by_doc, reg_numb_border, id)
("Охранная зона ЛЭП 110кВ", "Электроснабжение", "ВЛ-110", "66-123", 7),
("СЗЗ промышленного объекта", "Санитарная", "Завод", "66-456", 8),
]
db = _make_mock_db_with_rows(rows)
result = _get_cad_zouit_overlaps(db, "POLYGON((0 0,1 0,1 1,0 0))")
assert len(result) == 2
lep = result[0]
assert lep["group_key"] == "cad_zouit"
assert lep["source"] == "cad_zouit"
assert lep["type_zone"] == "Охранная зона ЛЭП 110кВ"
assert lep["subcategory"] is None # всегда NULL в cad_zouit
assert lep["name"] == "ВЛ-110"
assert lep["raw_props"]["zouit_id"] == 7
def test_nspd_path_used_when_zouit_count_nonzero() -> None:
"""#243 backward compat: когда dump существует с zouit_count>0 — NSPD path, не cad_zouit.
Мок: row.zouit_count=3, fetchall возвращает NSPD features.
Проверяем source='nspd-quarter-dump' (не cad_zouit).
"""
# Строка дампа: все поля через MagicMock
from datetime import UTC, datetime
dump_row = MagicMock()
dump_row.__getitem__ = lambda self, i: [
"66:41:0603016", # quarter_cad
datetime(2026, 1, 1, tzinfo=UTC), # fetched_at_utc (свежий)
100, # total_features
None, # harvest_error
1, # territorial_zones_count
3, # zouit_count > 0 → NSPD path
0, # engineering_count
0, # risks_count
0, # opportunity_count
0, # red_lines_count
][i]
nspd_zouit_rows: list[tuple[Any, ...]] = [
# (layer, props)
("zouit_engineering", {"name": "ЛЭП", "subcategory": 17}),
]
db = MagicMock()
first_result = MagicMock()
first_result.first.return_value = dump_row
# Последующие вызовы execute (spatial queries) — возвращаем пустые кроме zouit
zouit_result = MagicMock()
zouit_result.fetchall.return_value = nspd_zouit_rows
empty_result = MagicMock()
empty_result.fetchall.return_value = []
empty_result.first.return_value = None
# [0]=dumps lookup, [1]=zoning(first), [2]=zouit(fetchall), [3+]=others
db.execute.side_effect = [
first_result,
empty_result, # _get_zoning first()
zouit_result, # _get_zouit_overlaps fetchall()
empty_result, # _get_engineering_nearby fetchall()
empty_result, # _get_risk_zones fetchall()
empty_result, # _get_opportunity_parcels fetchall()
empty_result, # _get_red_lines fetchall()
]
result = get_quarter_dump_data(
db, "66:41:0603016:194", "POLYGON((60 56,61 56,61 57,60 57,60 56))"
)
overlaps: list[dict[str, Any]] = result["nspd_zouit_overlaps"]
assert len(overlaps) == 1
assert overlaps[0]["source"] == "nspd-quarter-dump"
# dump.available=True т.к. свежий дамп
assert result["nspd_dump"]["available"] is True

87
backend/tests/test_admin_weight_profiles.py
View file

@ -314,6 +314,93 @@ def test_delete_not_found(client_with_token: TestClient, monkeypatch: pytest.Mon
_clear_overrides()
# ── include_system preset seed (Issue #114 / PR #229) ─────────────────────────
def test_list_include_system_calls_with_system(
client_with_token: TestClient, monkeypatch: pytest.MonkeyPatch
) -> None:
"""GET ?include_system=true вызывает list_profiles_with_system, возвращает presets."""
system_profile = _make_profile(
100, user_id="__system__", profile_name="Комфорт", weights={"park": 1.8, "school": 1.5}
)
user_profile = _make_profile(1, user_id="user-1", profile_name="Мой профиль")
mock = MagicMock()
_override_db(mock)
try:
monkeypatch.setattr(
"app.api.v1.admin_weight_profiles.list_profiles_with_system",
lambda db, user_id: [user_profile, system_profile],
)
r = client_with_token.get(
"/api/v1/admin/site-finder/weight-profiles",
params={"user_id": "user-1", "include_system": "true"},
headers=_HEADERS,
)
assert r.status_code == 200
body = r.json()
assert len(body) == 2
profile_names = {p["profile_name"] for p in body}
assert "Комфорт" in profile_names
assert "Мой профиль" in profile_names
finally:
_clear_overrides()
def test_list_without_include_system_does_not_call_with_system(
client_with_token: TestClient, monkeypatch: pytest.MonkeyPatch
) -> None:
"""GET без include_system → list_profiles (только пользовательские профили)."""
user_profile = _make_profile(1, user_id="user-1")
called_with_system = []
mock = MagicMock()
_override_db(mock)
try:
monkeypatch.setattr(
"app.api.v1.admin_weight_profiles.list_profiles",
lambda db, user_id: [user_profile],
)
monkeypatch.setattr(
"app.api.v1.admin_weight_profiles.list_profiles_with_system",
lambda db, user_id: called_with_system.append(True) or [],
)
r = client_with_token.get(
"/api/v1/admin/site-finder/weight-profiles",
params={"user_id": "user-1"},
headers=_HEADERS,
)
assert r.status_code == 200
assert len(r.json()) == 1
# list_profiles_with_system НЕ должен вызываться без include_system=true
assert called_with_system == [], "вызван list_profiles_with_system без флага"
finally:
_clear_overrides()
def test_list_profiles_with_system_service(monkeypatch: pytest.MonkeyPatch) -> None:
"""list_profiles_with_system передаёт system_user_id='__system__' в запрос."""
from app.services.site_finder.weight_profiles import SYSTEM_USER_ID, list_profiles_with_system
captured_params: list[dict] = []
class FakeResult:
def mappings(self) -> FakeResult:
return self
def all(self) -> list:
return []
class FakeDb:
def execute(self, query: object, params: dict) -> FakeResult:
captured_params.append(params)
return FakeResult()
list_profiles_with_system(FakeDb(), user_id="user-test")
assert len(captured_params) == 1
assert captured_params[0]["system_user_id"] == SYSTEM_USER_ID
assert captured_params[0]["user_id"] == "user-test"
# ── Auth ───────────────────────────────────────────────────────────────────────

92
backend/tests/test_gate_verdict.py
View file

@ -1,6 +1,9 @@
"""Tests for gate_verdict aggregator — pure function, no DB."""
from app.services.site_finder.gate_verdict import compute_gate_verdict, is_residential_zone
from app.services.site_finder.gate_verdict import (
compute_gate_verdict,
is_residential_zone,
)
# ── is_residential_zone unit tests ────────────────────────────────────────────
@ -115,3 +118,90 @@ def test_pzz_unknown_warning_when_no_zoning():
verdict = compute_gate_verdict(None, [], [{"name": "ТП-1"}], {"available": True})
assert any(w["code"] == "PZZ_UNKNOWN" for w in verdict["warnings"])
assert verdict["can_build_mkd"] is True
# ── cad_zouit fallback path (#232) ────────────────────────────────────────────
def test_cad_zouit_ohranaya_zona_truboprovodov_blocks():
"""cad_zouit overlap с 'охранная зона трубопроводов' → ZOUIT_CAD_BLOCKER → Нельзя."""
nspd_zoning = {"zone_code": "Ж-2", "zone_name": "Жилая"}
overlaps = [
{
"source": "cad_zouit",
"type_zone": "Охранная зона трубопроводов",
"layer": "Охранная зона трубопроводов",
"name": "Газопровод высокого давления",
}
]
verdict = compute_gate_verdict(nspd_zoning, overlaps, [], {"available": True})
assert verdict["can_build_mkd"] is False
assert verdict["verdict_label"] == "Нельзя"
assert any(b["code"] == "ZOUIT_CAD_BLOCKER" for b in verdict["blockers"])
def test_cad_zouit_elektr_blocks():
"""cad_zouit overlap с 'электр' substring в type_zone → blocker."""
nspd_zoning = {"zone_code": "Ж-3", "zone_name": "Жилая"}
overlaps = [
{
"source": "cad_zouit",
"type_zone": "Охранная зона электросетевого объекта",
"layer": "Охранная зона электросетевого объекта",
"name": "ЛЭП 110 кВ",
}
]
verdict = compute_gate_verdict(nspd_zoning, overlaps, [], {"available": True})
assert verdict["can_build_mkd"] is False
assert any(b["code"] == "ZOUIT_CAD_BLOCKER" for b in verdict["blockers"])
def test_cad_zouit_szz_is_warning_not_blocker():
"""cad_zouit overlap с 'СЗЗ' → ZOUIT_CAD_SZZ warning, не blocker."""
nspd_zoning = {"zone_code": "Ж-2", "zone_name": "Жилая"}
overlaps = [
{
"source": "cad_zouit",
"type_zone": "Санитарно-защитная зона предприятия",
"layer": "Санитарно-защитная зона предприятия",
"name": "СЗЗ завода",
}
]
verdict = compute_gate_verdict(nspd_zoning, overlaps, [{"name": "ТП-1"}], {"available": True})
assert verdict["can_build_mkd"] is True
assert verdict["verdict_label"] == "С ограничениями"
assert any(w["code"] == "ZOUIT_CAD_SZZ" for w in verdict["warnings"])
assert verdict["blockers"] == []
def test_cad_zouit_other_is_warning():
"""cad_zouit overlap без blocker/СЗЗ keyword → ZOUIT_CAD_OTHER warning."""
nspd_zoning = {"zone_code": "Ж-2", "zone_name": "Жилая"}
overlaps = [
{
"source": "cad_zouit",
"type_zone": "Зона публичного сервитута",
"layer": "Зона публичного сервитута",
"name": None,
}
]
verdict = compute_gate_verdict(nspd_zoning, overlaps, [{"name": "ТП-1"}], {"available": True})
assert verdict["can_build_mkd"] is True
assert any(w["code"] == "ZOUIT_CAD_OTHER" for w in verdict["warnings"])
assert verdict["blockers"] == []
def test_nspd_subcategory_path_backward_compat():
"""NSPD dump path (source != 'cad_zouit') продолжает работать через subcategory."""
nspd_zoning = {"zone_code": "Ж-3", "zone_name": "Жилая"}
overlaps = [
{
"source": "nspd-quarter-dump",
"subcategory": 17,
"name": "ЛЭП 110кВ",
"layer": "37578",
}
]
verdict = compute_gate_verdict(nspd_zoning, overlaps, [], {"available": True})
assert verdict["can_build_mkd"] is False
assert any(b["code"] == "ZOUIT_OVERLAP_SUB17" for b in verdict["blockers"])

97
backend/tests/test_layout_signature.py Normal file
View file

@ -0,0 +1,97 @@
"""Тесты для layout_signature helpers (Issue #113, Phase 2.1).
Pure-Python, без БД и external calls.
"""
from __future__ import annotations
import pytest
from app.services.site_finder.layout_signature import (
area_bin,
layout_signature,
room_bucket_from_flat,
)
@pytest.mark.parametrize(
"rooms, flat_type, is_studio, expected",
[
(None, None, True, "studio"),
(0, "Квартира-студия", False, "studio"),
(0, "Квартира", None, "studio"),
(1, "Квартира", None, "1"),
(2, "Квартира", None, "2"),
(3, "Квартира", False, "3"),
(4, "Квартира", False, "4+"),
(5, "Квартира", None, "4+"),
(None, "Квартира", None, "1"), # fallback
],
)
def test_room_bucket(
rooms: int | None, flat_type: str | None, is_studio: bool | None, expected: str
) -> None:
assert room_bucket_from_flat(rooms, flat_type, is_studio) == expected
@pytest.mark.parametrize(
"area, expected",
[
(0.5, "<25"),
(24.9, "<25"),
(25.0, "25-40"),
(39.99, "25-40"),
(40.0, "40-60"),
(59.99, "40-60"),
(60.0, "60-80"),
(79.99, "60-80"),
(80.0, "80-100"),
(99.99, "80-100"),
(100.0, "100+"),
(250.0, "100+"),
],
)
def test_area_bin(area: float, expected: str) -> None:
assert area_bin(area) == expected
def test_layout_signature_deterministic() -> None:
assert layout_signature("studio", "<25") == "studio__<25"
assert layout_signature("4+", "100+") == "4+__100+"
# Same input → same output
sig1 = layout_signature("2", "40-60")
sig2 = layout_signature("2", "40-60")
assert sig1 == sig2
def test_room_bucket_is_studio_overrides_rooms() -> None:
"""is_studio=True beats any rooms value."""
assert room_bucket_from_flat(3, "Квартира", True) == "studio"
def test_room_bucket_flat_type_studio_overrides_rooms() -> None:
"""flat_type='Квартира-студия' beats rooms=2."""
assert room_bucket_from_flat(2, "Квартира-студия", None) == "studio"
def test_room_bucket_large_rooms() -> None:
"""rooms=10 → '4+'."""
assert room_bucket_from_flat(10, "Квартира", False) == "4+"
def test_area_bin_boundary_exact() -> None:
"""Граничные значения попадают в правильный бакет (включение левой границы)."""
assert area_bin(25.0) == "25-40"
assert area_bin(40.0) == "40-60"
assert area_bin(60.0) == "60-80"
assert area_bin(80.0) == "80-100"
assert area_bin(100.0) == "100+"
def test_layout_signature_format() -> None:
"""Сигнатура всегда содержит двойное подчёркивание-разделитель."""
sig = layout_signature("1", "25-40")
assert "__" in sig
parts = sig.split("__")
assert parts[0] == "1"
assert parts[1] == "25-40"

136
backend/tests/test_layout_tz_pdf.py Normal file
View file

@ -0,0 +1,136 @@
"""Tests для layout_tz_pdf renderer (Issue #113 PR D).
WeasyPrint requires native GTK/Pango/GObject shared libraries. These are present
in the Docker container (Linux) but absent on Windows dev machines. All tests in
this module are skipped automatically when the native libs are unavailable.
"""
import datetime as dt
import pytest
# Attempt to import the module under test; skip entire module if native libs missing.
try:
from app.services.exporters.layout_tz_pdf import render_layout_tz_pdf
except (OSError, ImportError) as _e: # GTK libs missing on Windows, or weasyprint not installed
pytest.skip(f"WeasyPrint deps missing: {_e}", allow_module_level=True)
from app.schemas.parcel import (
BestLayoutsResponse,
LayoutDataQuality,
LayoutTzMixRow,
LayoutTzRecommendation,
TopLayoutRow,
)
def _sample_response() -> BestLayoutsResponse:
return BestLayoutsResponse(
top_layouts=[
TopLayoutRow(
rank=1,
room_bucket="1",
area_bin="25-40",
signature="1__25-40",
competitor_obj_ids=[1234, 5678],
competitor_count=2,
total_sold_in_window=67,
velocity_per_month=8.4,
avg_price_per_m2_rub=148000.0,
avg_area_m2=38.5,
supply_units_in_radius=312,
sold_pct_of_supply=21.5,
),
TopLayoutRow(
rank=2,
room_bucket="studio",
area_bin="<25",
signature="studio__<25",
competitor_obj_ids=[1234],
competitor_count=1,
total_sold_in_window=40,
velocity_per_month=5.0,
avg_price_per_m2_rub=160000.0,
avg_area_m2=22.0,
supply_units_in_radius=100,
sold_pct_of_supply=40.0,
),
],
recommendation_for_tz=LayoutTzRecommendation(
rationale_text="Test rationale текст с кириллицей",
mix=[
LayoutTzMixRow(room_bucket="studio", pct=10, abs_units=30, avg_target_area_m2=22.0),
LayoutTzMixRow(room_bucket="1", pct=60, abs_units=180, avg_target_area_m2=38.5),
LayoutTzMixRow(room_bucket="2", pct=30, abs_units=90, avg_target_area_m2=55.0),
],
weighted_avg_price_per_m2_rub=152000.0,
based_on_obj_count=5,
based_on_total_deals=107,
data_window_start=dt.date(2026, 2, 1),
data_window_end=dt.date(2026, 5, 1),
),
data_quality=LayoutDataQuality(
objects_with_velocity_data=5,
objects_total_in_radius=8,
velocity_coverage_pct=62.5,
confidence="medium",
),
)
def test_pdf_renders_non_empty_bytes() -> None:
pdf = render_layout_tz_pdf(
_sample_response(),
cad_num="66:41:0204016:10",
radius_km=1.0,
time_window="last_quarter",
)
assert len(pdf) > 1000 # PDF минимум ~1KB
def test_pdf_starts_with_pdf_magic() -> None:
pdf = render_layout_tz_pdf(
_sample_response(),
cad_num="66:41:0204016:10",
radius_km=1.0,
time_window="last_quarter",
)
assert pdf[:4] == b"%PDF"
def test_pdf_renders_cyrillic_correctly() -> None:
"""Smoke — WeasyPrint должен handle кириллический rationale_text без UnicodeEncodeError."""
response = _sample_response()
pdf = render_layout_tz_pdf(
response,
cad_num="66:41:0303161:42",
radius_km=1.5,
time_window="last_year",
)
# Embedded text может быть compressed, но без exception = OK
assert len(pdf) > 1000
def test_pdf_handles_empty_top_layouts() -> None:
response = _sample_response()
response.top_layouts = []
pdf = render_layout_tz_pdf(
response,
cad_num="66:41:0204016:10",
radius_km=1.0,
time_window="last_quarter",
)
assert pdf[:4] == b"%PDF"
def test_pdf_handles_null_avg_price() -> None:
"""avg_price_per_m2_rub=None (ЖК не покрыт Objective) → должно рендериться как '—'."""
response = _sample_response()
response.top_layouts[0].avg_price_per_m2_rub = None
pdf = render_layout_tz_pdf(
response,
cad_num="66:41:0204016:10",
radius_km=1.0,
time_window="last_quarter",
)
assert pdf[:4] == b"%PDF"

49
backend/tests/test_nspd_client.py
View file

@ -507,6 +507,40 @@ def _make_fake_http(
return fake_http
def _make_fake_grid_walk(
layer_feature_counts: dict[str, int] | None = None,
) -> Any:
"""Возвращает fake `NSPDClient.get_features_in_bbox_grid`.
После Sub-PR B (#260) area/linear layers (territorial_zones, red_lines,
engineering_structures, zouit_*, risk_*) идут через grid-walk вместо
legacy `_http_get_json` → тесты обязаны мокать оба пути, иначе грид-walk
бьёт по живому NSPD API.
"""
counts = layer_feature_counts or {}
from app.services.scrapers.nspd_client import LAYERS as _LAYERS
from app.services.scrapers.nspd_client import NSPDFeature
id_to_name: dict[int, str] = {v: k for k, v in _LAYERS.items()}
def fake_grid(
self: Any,
layer_id: int,
bbox: tuple[float, float, float, float],
*,
grid_n: int = 7,
step_m: float = 50.0,
) -> list[NSPDFeature]:
layer_name = id_to_name.get(layer_id, "unknown")
n = counts.get(layer_name, 1)
return [
NSPDFeature.from_raw({"id": f"{layer_name}-{i}", "geometry": None, "properties": {}})
for i in range(n)
]
return fake_grid
def test_search_by_quarter_core_only(monkeypatch: pytest.MonkeyPatch) -> None:
"""core_only (include_zouit=False, include_risks=False): 1 search + 5 bulk."""
search_calls: list[str] = []
@ -520,6 +554,11 @@ def test_search_by_quarter_core_only(monkeypatch: pytest.MonkeyPatch) -> None:
"app.services.scrapers.nspd_client._http_get_json",
_make_fake_http(_LAYER_FEATURE_COUNTS),
)
# Sub-PR B (#260): area layers идут через grid-walk — мокаем оба пути.
monkeypatch.setattr(
"app.services.scrapers.nspd_client.NSPDClient.get_features_in_bbox_grid",
_make_fake_grid_walk(_LAYER_FEATURE_COUNTS),
)
result = NSPDClient().search_by_quarter(
"66:41:0204016", include_zouit=False, include_risks=False
@ -566,6 +605,11 @@ def test_search_by_quarter_with_zouit(monkeypatch: pytest.MonkeyPatch) -> None:
"app.services.scrapers.nspd_client._http_get_json",
_make_fake_http(),
)
# Sub-PR B (#260): zouit_* и area layers идут через grid-walk.
monkeypatch.setattr(
"app.services.scrapers.nspd_client.NSPDClient.get_features_in_bbox_grid",
_make_fake_grid_walk(),
)
result = NSPDClient().search_by_quarter(
"66:41:0204016", include_zouit=True, include_risks=False
@ -628,6 +672,11 @@ def test_search_by_quarter_layers_fetched_with_risks(monkeypatch: pytest.MonkeyP
"app.services.scrapers.nspd_client._http_get_json",
_make_fake_http(),
)
# Sub-PR B (#260): risk_* и area layers идут через grid-walk.
monkeypatch.setattr(
"app.services.scrapers.nspd_client.NSPDClient.get_features_in_bbox_grid",
_make_fake_grid_walk(),
)
result = NSPDClient().search_by_quarter(
"66:41:0204016", include_zouit=False, include_risks=True

71
backend/tests/test_nspd_sync.py
View file

@ -25,6 +25,7 @@ from app.workers.tasks.nspd_sync import (
_build_features_json,
_build_risks_count,
_build_zouit_count,
_upsert_dump,
harvest_quarter,
harvest_stale_quarters,
)
@ -71,6 +72,7 @@ def _make_dump(
engineering_structures=[_make_feat("e1")],
zouit=zouit,
risks=risks,
opportunity={},
layers_fetched=(
"search",
"parcels",
@ -204,6 +206,7 @@ def test_harvest_quarter_empty_quarter(
engineering_structures=[],
zouit={"okn": [], "engineering": [], "natural": [], "protected": [], "other": []},
risks={},
opportunity={},
layers_fetched=("search",),
bbox_3857=None,
fetched_at_utc="2026-05-12T03:00:00+00:00",
@ -365,6 +368,7 @@ def test_features_json_geometry_preserved() -> None:
engineering_structures=[],
zouit={},
risks={},
opportunity={},
layers_fetched=("search", "parcels"),
bbox_3857=(100.0, 200.0, 300.0, 400.0),
fetched_at_utc="2026-05-12T00:00:00+00:00",
@ -375,3 +379,70 @@ def test_features_json_geometry_preserved() -> None:
assert result[0]["layer"] == "parcels"
assert result[0]["feature_id"] == "p1"
assert result[0]["properties"] == {"k": "v"}
# ── _upsert_dump CAST regression tests (#244) ────────────────────────────────
@patch("app.workers.tasks.nspd_sync.SessionLocal")
def test_upsert_dump_null_geom_executes_without_error(mock_session_cls: MagicMock) -> None:
"""Regression #244: _upsert_dump с geom_json=None не падает на AmbiguousParameter.
psycopg3 не может вывести тип голого параметра $N внутри CASE WHEN ... IS NULL.
Fix: CAST(:geom_json AS text) IS NULL даёт явный тип — нет AmbiguousParameter.
"""
mock_db = MagicMock()
mock_session_cls.return_value = mock_db
# dump=None → error-only path: все geo-параметры None, должен execute без ошибок
_upsert_dump(
quarter_cad="66:41:0204016",
region_code=66,
dump=None,
features_json=None,
duration_ms=100,
harvest_error="TestError: simulated",
)
mock_db.execute.assert_called_once()
mock_db.commit.assert_called_once()
@patch("app.workers.tasks.nspd_sync.SessionLocal")
def test_upsert_dump_with_geom_executes_without_error(mock_session_cls: MagicMock) -> None:
"""Regression #244: _upsert_dump с реальным geom_json и bbox не падает.
CAST(:geom_json AS text) передаёт строку, CAST(:bbox_xmin AS double precision) —
число. Оба branch CASE WHEN типизированы корректно.
"""
mock_db = MagicMock()
mock_session_cls.return_value = mock_db
geom = {
"type": "Polygon",
"coordinates": [
[[100.0, 200.0], [300.0, 200.0], [300.0, 400.0], [100.0, 400.0], [100.0, 200.0]]
],
}
dump = _make_dump(
quarter_cad="66:41:0204016",
quarter_feat=_make_feat_with_geom("q1", geom),
bbox=(100.0, 200.0, 300.0, 400.0),
)
_upsert_dump(
quarter_cad="66:41:0204016",
region_code=66,
dump=dump,
features_json=_build_features_json(dump),
duration_ms=500,
harvest_error=None,
)
mock_db.execute.assert_called_once()
# Verify params contain properly-typed values (not None for geo fields)
call_params = mock_db.execute.call_args[0][1]
assert call_params["geom_json"] is not None # JSON string
assert call_params["bbox_xmin"] == 100.0
assert call_params["bbox_ymax"] == 400.0
mock_db.commit.assert_called_once()

168
backend/tests/test_poi_score.py Normal file
View file

@ -0,0 +1,168 @@
"""Tests for POI weighted score service (B6).
Юнит-тесты для чистой функции — без DB.
"""
from app.services.site_finder.poi_score import (
CATEGORY_WEIGHTS,
PoiScoreResponse,
_category_weight,
compute_poi_weighted_top7,
)
# ── unit: _category_weight ─────────────────────────────────────────────────────
def test_category_weight_metro():
"""Метро имеет наибольший вес из всех категорий."""
metro_w = _category_weight("metro_stop")
for cat in CATEGORY_WEIGHTS:
if cat != "metro_stop" and cat != "default":
assert metro_w >= _category_weight(
cat
), f"metro_stop weight {metro_w} должен быть >= {cat} weight {_category_weight(cat)}"
def test_category_weight_unknown_returns_default():
w = _category_weight("unknown_category_xyz")
assert w == CATEGORY_WEIGHTS["default"]
def test_category_weight_all_positive():
"""Все веса в CATEGORY_WEIGHTS должны быть положительными (B6 — ranking, не штраф)."""
for cat, w in CATEGORY_WEIGHTS.items():
assert w > 0, f"Вес {cat}={w} должен быть > 0"
# ── unit: weight formula ratio ─────────────────────────────────────────────────
def test_weight_formula_ratio():
"""Ближний объект той же категории должен иметь больший вес."""
cat = "school"
cw = _category_weight(cat)
w_near = (1.0 / (100.0 + 100.0)) * cw # 100м
w_far = (1.0 / (1000.0 + 100.0)) * cw # 1000м
assert w_near > w_far
def test_weight_formula_category_dominates_at_equal_distance():
"""При одинаковом расстоянии метро должно быть впереди автобусной остановки."""
dist = 500.0
w_metro = (1.0 / (dist + 100.0)) * _category_weight("metro_stop")
w_bus = (1.0 / (dist + 100.0)) * _category_weight("bus_stop")
assert w_metro > w_bus
# ── unit: compute_poi_weighted_top7 with mock DB ───────────────────────────────
class _MockMappings:
def __init__(self, data: list[dict]) -> None:
self._data = data
def all(self) -> list[dict]:
return self._data # type: ignore[return-value]
class _MockResult:
def __init__(self, data: list[dict]) -> None:
self._data = data
def mappings(self) -> "_MockMappings":
return _MockMappings(self._data)
class _MockDb:
"""Минимальный мок SQLAlchemy Session для тестирования без БД."""
def __init__(self, rows: list[dict]) -> None:
self._rows = rows
def execute(self, *_args: object, **_kwargs: object) -> _MockResult:
return _MockResult(self._rows)
def _make_row(name: str, category: str, distance_m: float) -> dict:
return {
"name": name,
"category": category,
"tags": {},
"distance_m": distance_m,
}
def test_top7_returns_at_most_7():
rows = [_make_row(f"POI {i}", "school", float(i * 50)) for i in range(1, 20)]
db = _MockDb(rows)
result = compute_poi_weighted_top7(db, "66:41:0204016:10", 56.838, 60.605)
assert isinstance(result, PoiScoreResponse)
assert len(result.top_poi) <= 7
def test_top7_sorted_by_weight_desc():
rows = [
_make_row("Дальняя школа", "school", 1500.0),
_make_row("Метро", "metro_stop", 300.0),
_make_row("Близкая школа", "school", 100.0),
]
db = _MockDb(rows)
result = compute_poi_weighted_top7(db, "66:41:0204016:10", 56.838, 60.605)
weights = [item.weight for item in result.top_poi]
assert weights == sorted(weights, reverse=True), "top_poi должны быть по weight DESC"
def test_metro_beats_school_at_equal_distance():
"""Метро в 300м должно быть на первом месте перед школой в 300м (равное расстояние)."""
rows = [
_make_row("Школа №1", "school", 300.0),
_make_row("Метро Чкаловская", "metro_stop", 300.0),
]
db = _MockDb(rows)
result = compute_poi_weighted_top7(db, "66:41:0204016:10", 56.838, 60.605)
assert (
result.top_poi[0].category == "metro_stop"
), "При равном расстоянии метро (category_weight=6.0) должно быть выше школы (5.0)"
def test_metro_first_when_close():
"""Метро в 50м должно быть на первом месте перед школой в 300м."""
rows = [
_make_row("Школа №1", "school", 300.0),
_make_row("Метро Чкаловская", "metro_stop", 50.0),
]
db = _MockDb(rows)
result = compute_poi_weighted_top7(db, "66:41:0204016:10", 56.838, 60.605)
assert result.top_poi[0].category == "metro_stop", (
"Метро (weight=6.0) в 50м должно быть впереди школы (weight=5.0) в 300м — "
f"metro_weight={(1/(50+100))*6:.5f} vs school_weight={(1/(300+100))*5:.5f}"
)
def test_empty_db_returns_empty_top_poi():
db = _MockDb([])
result = compute_poi_weighted_top7(db, "66:41:0204016:10", 56.838, 60.605)
assert result.top_poi == []
assert result.cad_num == "66:41:0204016:10"
assert result.radius_m == 2000
def test_address_built_from_tags():
rows = [
{
"name": "Магазин",
"category": "shop_small",
"tags": {"addr:street": "ул. Ленина", "addr:housenumber": "10"},
"distance_m": 200.0,
}
]
db = _MockDb(rows)
result = compute_poi_weighted_top7(db, "test", 56.838, 60.605)
assert result.top_poi[0].address == "ул. Ленина, 10"
def test_address_none_when_no_tags():
rows = [_make_row("Парк", "park", 400.0)]
db = _MockDb(rows)
result = compute_poi_weighted_top7(db, "test", 56.838, 60.605)
assert result.top_poi[0].address is None

158
backend/tests/test_sentry_init.py Normal file
View file

@ -0,0 +1,158 @@
"""Unit-тесты логики инициализации GlitchTip SDK.
Проверяем что init-блок в main.py / celery_app.py вызывает sentry_sdk.init()
только при непустом GLITCHTIP_DSN, что release-fallback работает корректно,
и что scrub_sensitive_query redact-ит api keys из URL spans.
"""
import os
from unittest.mock import patch
import sentry_sdk
def test_sdk_imports_without_error() -> None:
"""Все интеграции импортируются без ModuleNotFoundError."""
from sentry_sdk.integrations.celery import CeleryIntegration
from sentry_sdk.integrations.fastapi import FastApiIntegration
from sentry_sdk.integrations.httpx import HttpxIntegration
from sentry_sdk.integrations.logging import LoggingIntegration
from sentry_sdk.integrations.sqlalchemy import SqlalchemyIntegration
from sentry_sdk.integrations.starlette import StarletteIntegration
assert StarletteIntegration()
assert FastApiIntegration()
assert CeleryIntegration()
assert SqlalchemyIntegration()
assert HttpxIntegration()
assert LoggingIntegration()
def test_no_sdk_init_when_dsn_empty() -> None:
"""Если GLITCHTIP_DSN пустой, sentry_sdk.init() не должен вызываться."""
with patch("sentry_sdk.init") as mock_init:
glitchtip_dsn = None
if glitchtip_dsn:
sentry_sdk.init(dsn=glitchtip_dsn)
mock_init.assert_not_called()
def test_sdk_init_called_when_dsn_set() -> None:
"""Если GLITCHTIP_DSN задан, sentry_sdk.init() вызывается с правильными параметрами."""
dsn = "https://key@errors.gendsgn.ru/1"
with patch("sentry_sdk.init") as mock_init:
glitchtip_dsn = dsn
if glitchtip_dsn:
sentry_sdk.init(
dsn=glitchtip_dsn,
environment="test",
release="unknown",
traces_sample_rate=0.05,
profiles_sample_rate=0.0,
send_default_pii=False,
integrations=[],
)
mock_init.assert_called_once()
call_kwargs = mock_init.call_args.kwargs
assert call_kwargs["dsn"] == dsn
assert call_kwargs["send_default_pii"] is False
assert call_kwargs["profiles_sample_rate"] == 0.0
# ── release fallback ──────────────────────────────────────────────────────────
def test_release_uses_git_sha_when_set() -> None:
"""GIT_SHA имеет приоритет над SENTRY_RELEASE."""
env = {"GIT_SHA": "abc1234", "SENTRY_RELEASE": "v1.0.0"}
with patch.dict(os.environ, env, clear=False):
release = os.getenv("GIT_SHA") or os.getenv("SENTRY_RELEASE") or "unknown"
assert release == "abc1234"
def test_release_falls_back_to_sentry_release() -> None:
"""Если GIT_SHA не задан, используется SENTRY_RELEASE."""
env = {"SENTRY_RELEASE": "v1.2.3"}
with patch.dict(os.environ, env, clear=False):
# Убираем GIT_SHA если он есть
os.environ.pop("GIT_SHA", None)
release = os.getenv("GIT_SHA") or os.getenv("SENTRY_RELEASE") or "unknown"
assert release == "v1.2.3"
def test_release_falls_back_to_unknown() -> None:
"""Если ни одна переменная не задана, release='unknown'."""
with patch.dict(os.environ, {}, clear=False):
os.environ.pop("GIT_SHA", None)
os.environ.pop("SENTRY_RELEASE", None)
release = os.getenv("GIT_SHA") or os.getenv("SENTRY_RELEASE") or "unknown"
assert release == "unknown"
# ── sentry_scrub ──────────────────────────────────────────────────────────────
def test_scrub_redacts_apikey_in_span_url() -> None:
"""scrub_sensitive_query заменяет apiKey= в span data['url']."""
from app.observability.sentry_scrub import scrub_sensitive_query
event: dict = {
"spans": [
{
"data": {
"url": "https://api.objctv.ru/v2/Report?apiKey=supersecret&group=EKB",
"http.url": "https://api.objctv.ru/v2/Report?api_key=topsecret",
}
}
]
}
result = scrub_sensitive_query(event, {})
span_data = result["spans"][0]["data"]
assert "[REDACTED]" in span_data["url"]
assert "supersecret" not in span_data["url"]
assert "[REDACTED]" in span_data["http.url"]
assert "topsecret" not in span_data["http.url"]
def test_scrub_redacts_token_in_description() -> None:
"""scrub_sensitive_query заменяет token= в span description."""
from app.observability.sentry_scrub import scrub_sensitive_query
event: dict = {
"spans": [{"description": "GET https://example.com?token=mysecrettoken&foo=bar"}]
}
result = scrub_sensitive_query(event, {})
assert "[REDACTED]" in result["spans"][0]["description"]
assert "mysecrettoken" not in result["spans"][0]["description"]
def test_scrub_redacts_request_url() -> None:
"""scrub_sensitive_query заменяет token в event['request']['url']."""
from app.observability.sentry_scrub import scrub_sensitive_query
event: dict = {"request": {"url": "https://example.com?access_token=abc123&other=val"}}
result = scrub_sensitive_query(event, {})
assert "[REDACTED]" in result["request"]["url"]
assert "abc123" not in result["request"]["url"]
def test_scrub_passes_through_clean_event() -> None:
"""scrub_sensitive_query не трогает URL без чувствительных параметров."""
from app.observability.sentry_scrub import scrub_sensitive_query
event: dict = {
"spans": [{"data": {"url": "https://example.com?foo=bar&page=1"}}],
"request": {"url": "https://example.com/api/health"},
}
result = scrub_sensitive_query(event, {})
assert result["spans"][0]["data"]["url"] == "https://example.com?foo=bar&page=1"
assert result["request"]["url"] == "https://example.com/api/health"
def test_scrub_handles_missing_spans() -> None:
"""scrub_sensitive_query не падает если 'spans' отсутствует."""
from app.observability.sentry_scrub import scrub_sensitive_query
event: dict = {"request": {"url": "https://example.com"}}
result = scrub_sensitive_query(event, {})
assert result["request"]["url"] == "https://example.com"

Some files were not shown because too many files have changed in this diff Show more