Light1YT
02ad9a1f58
Stratified 200-house Yandex reverse-geocode audit comparing our DB's (address, lat/lon) pair against what Yandex.Карты returns for those coordinates. Лежит фундамент данных для Phases 2-5 (canonical = Yandex per user decision). What ships: - Migration 066: address_mismatch_audit table + FDW foreign table gendesign_ekb_districts_geom для stratified sampling. - Sampling SQL: 8 EKB districts × 25 houses = 200. - Driver scripts/audit_address_mismatch.py с двумя режимами: API (если YANDEX_GEOCODER_API_KEY есть) и Playwright (CAPTCHA-aware, персистентный контекст). Resumable по --batch. - Report SQL: p50/p75/p95 distance, top-20 outliers, per-district breakdown. - 19 unit-тестов: normalize, distance bind, API parser, resume idempotency, captcha → status=blocked. Devops note: после деплоя миграции — выдать tradein_fdw_reader SELECT на gendesign.public.ekb_districts_geom перед первым запуском audit. Refs #582 (Phase 1 of 5). Phase 3 backfill заблокирован до Yandex API key (#402).
91 lines
4.2 KiB
SQL
91 lines
4.2 KiB
SQL
-- address_audit_report.sql
|
|
-- Post-run report for the address-mismatch audit (issue #582 Phase 1).
|
|
--
|
|
-- Sections:
|
|
-- 1. Summary — count, p50/p75/p95/mean distance, % street_differs,
|
|
-- % over 50m / 200m thresholds.
|
|
-- 2. Top-20 outliers by distance (manual triage list).
|
|
-- 3. Per-district breakdown — same metrics grouped by district column.
|
|
--
|
|
-- Run via psql:
|
|
-- psql "$DATABASE_URL" -v batch='2026-05-25_run1' -f scripts/address_audit_report.sql
|
|
--
|
|
-- :batch is a psql client variable substituted via -v.
|
|
|
|
\set ON_ERROR_STOP on
|
|
|
|
\echo '=============================================='
|
|
\echo ' Address mismatch audit — batch:' :batch
|
|
\echo '=============================================='
|
|
|
|
-- ---------------------------------------------------------------------------
|
|
-- 1) Top-level summary
|
|
-- ---------------------------------------------------------------------------
|
|
\echo ''
|
|
\echo '--- Summary (status=ok rows only) ---'
|
|
SELECT
|
|
COUNT(*) AS n_total,
|
|
COUNT(*) FILTER (WHERE audit_status = 'ok') AS n_ok,
|
|
COUNT(*) FILTER (WHERE audit_status = 'no_match') AS n_no_match,
|
|
COUNT(*) FILTER (WHERE audit_status = 'error') AS n_error,
|
|
COUNT(*) FILTER (WHERE audit_status = 'blocked') AS n_blocked,
|
|
ROUND(percentile_cont(0.50)
|
|
WITHIN GROUP (ORDER BY distance_m)::numeric, 1) AS p50_distance_m,
|
|
ROUND(percentile_cont(0.75)
|
|
WITHIN GROUP (ORDER BY distance_m)::numeric, 1) AS p75_distance_m,
|
|
ROUND(percentile_cont(0.95)
|
|
WITHIN GROUP (ORDER BY distance_m)::numeric, 1) AS p95_distance_m,
|
|
ROUND(AVG(distance_m)::numeric, 1) AS mean_distance_m,
|
|
ROUND(100.0 * AVG(CASE WHEN street_differs THEN 1.0 ELSE 0.0 END), 1)
|
|
AS pct_street_differs,
|
|
ROUND(100.0 * AVG(CASE WHEN distance_m > 50 THEN 1.0 ELSE 0.0 END), 1)
|
|
AS pct_over_50m,
|
|
ROUND(100.0 * AVG(CASE WHEN distance_m > 200 THEN 1.0 ELSE 0.0 END), 1)
|
|
AS pct_over_200m
|
|
FROM address_mismatch_audit
|
|
WHERE audit_batch = :'batch'
|
|
AND audit_status = 'ok';
|
|
|
|
-- ---------------------------------------------------------------------------
|
|
-- 2) Top-20 outliers
|
|
-- ---------------------------------------------------------------------------
|
|
\echo ''
|
|
\echo '--- Top-20 outliers by distance ---'
|
|
SELECT
|
|
house_id,
|
|
district,
|
|
ROUND(distance_m::numeric, 1) AS distance_m,
|
|
street_differs,
|
|
LEFT(original_address, 60) AS original_address,
|
|
LEFT(snapped_address, 60) AS snapped_address
|
|
FROM address_mismatch_audit
|
|
WHERE audit_batch = :'batch'
|
|
AND audit_status = 'ok'
|
|
AND distance_m IS NOT NULL
|
|
ORDER BY distance_m DESC NULLS LAST
|
|
LIMIT 20;
|
|
|
|
-- ---------------------------------------------------------------------------
|
|
-- 3) Per-district breakdown
|
|
-- ---------------------------------------------------------------------------
|
|
\echo ''
|
|
\echo '--- Per-district breakdown (status=ok only) ---'
|
|
SELECT
|
|
COALESCE(district, '(no district)') AS district,
|
|
COUNT(*) AS n,
|
|
ROUND(percentile_cont(0.50)
|
|
WITHIN GROUP (ORDER BY distance_m)::numeric, 1) AS p50_distance_m,
|
|
ROUND(percentile_cont(0.95)
|
|
WITHIN GROUP (ORDER BY distance_m)::numeric, 1) AS p95_distance_m,
|
|
ROUND(AVG(distance_m)::numeric, 1) AS mean_distance_m,
|
|
ROUND(100.0 * AVG(CASE WHEN street_differs THEN 1.0 ELSE 0.0 END), 1)
|
|
AS pct_street_differs,
|
|
ROUND(100.0 * AVG(CASE WHEN distance_m > 50 THEN 1.0 ELSE 0.0 END), 1)
|
|
AS pct_over_50m,
|
|
ROUND(100.0 * AVG(CASE WHEN distance_m > 200 THEN 1.0 ELSE 0.0 END), 1)
|
|
AS pct_over_200m
|
|
FROM address_mismatch_audit
|
|
WHERE audit_batch = :'batch'
|
|
AND audit_status = 'ok'
|
|
GROUP BY COALESCE(district, '(no district)')
|
|
ORDER BY n DESC, district;
|