Light1YT
02ad9a1f58
Stratified 200-house Yandex reverse-geocode audit comparing our DB's (address, lat/lon) pair against what Yandex.Карты returns for those coordinates. Лежит фундамент данных для Phases 2-5 (canonical = Yandex per user decision). What ships: - Migration 066: address_mismatch_audit table + FDW foreign table gendesign_ekb_districts_geom для stratified sampling. - Sampling SQL: 8 EKB districts × 25 houses = 200. - Driver scripts/audit_address_mismatch.py с двумя режимами: API (если YANDEX_GEOCODER_API_KEY есть) и Playwright (CAPTCHA-aware, персистентный контекст). Resumable по --batch. - Report SQL: p50/p75/p95 distance, top-20 outliers, per-district breakdown. - 19 unit-тестов: normalize, distance bind, API parser, resume idempotency, captcha → status=blocked. Devops note: после деплоя миграции — выдать tradein_fdw_reader SELECT на gendesign.public.ekb_districts_geom перед первым запуском audit. Refs #582 (Phase 1 of 5). Phase 3 backfill заблокирован до Yandex API key (#402).
117 lines
5 KiB
PL/PgSQL
117 lines
5 KiB
PL/PgSQL
-- 066_address_mismatch_audit.sql
|
||
-- Purpose: Phase 1 of Forgejo issue #582 — empirical audit of houses.address vs
|
||
-- authoritative Yandex Geocoder reverse lookup. Captures distance
|
||
-- between stored coords and Yandex-snapped coords, plus a street-token
|
||
-- comparison flag, so we can quantify systemic address mismatches and
|
||
-- decide which houses need re-geocoding before launch.
|
||
--
|
||
-- Schema:
|
||
-- address_mismatch_audit — one row per (house_id, audit_batch). Snapshot of
|
||
-- the comparison run with batch id, district, distance, status, raw Yandex
|
||
-- payload retained for later diagnostics.
|
||
--
|
||
-- FDW:
|
||
-- gendesign_ekb_districts_geom — read-only foreign table over gendesign
|
||
-- master DB so we can do stratified district sampling without duplicating
|
||
-- OSM polygon data into tradein. Pattern copied from 060_postgres_fdw_extension.sql.
|
||
--
|
||
-- Dependencies:
|
||
-- - houses table (009_houses.sql)
|
||
-- - postgres_fdw extension + gendesign_remote server (060_postgres_fdw_extension.sql)
|
||
-- - PostGIS extension (already enabled)
|
||
--
|
||
-- Idempotent: safe to re-apply.
|
||
|
||
BEGIN;
|
||
|
||
-- ---------------------------------------------------------------------------
|
||
-- 1. address_mismatch_audit — the audit log
|
||
-- ---------------------------------------------------------------------------
|
||
CREATE TABLE IF NOT EXISTS address_mismatch_audit (
|
||
id bigserial PRIMARY KEY,
|
||
house_id bigint NOT NULL REFERENCES houses(id) ON DELETE CASCADE,
|
||
audit_batch text NOT NULL, -- e.g. '2026-05-25_run1'
|
||
district text, -- EKB district name (admin polygon)
|
||
|
||
-- Original (stored in houses)
|
||
original_address text,
|
||
original_lat double precision,
|
||
original_lon double precision,
|
||
|
||
-- Snapped (canonical Yandex reverse-geocode result)
|
||
snapped_address text,
|
||
snapped_lat double precision,
|
||
snapped_lon double precision,
|
||
|
||
-- Diff metrics
|
||
distance_m double precision, -- ST_Distance(geography) meters
|
||
street_differs boolean, -- first non-numeric token compare
|
||
|
||
-- Status / debug
|
||
audit_status text NOT NULL DEFAULT 'ok', -- ok / no_match / error / blocked
|
||
error_message text,
|
||
raw_payload jsonb, -- full Yandex response for forensics
|
||
|
||
created_at timestamptz NOT NULL DEFAULT NOW(),
|
||
|
||
-- one row per house × batch — re-running the same batch skips done rows
|
||
UNIQUE (house_id, audit_batch)
|
||
);
|
||
|
||
-- NOTE: the UNIQUE (house_id, audit_batch) constraint above already creates
|
||
-- a backing unique index, so no explicit CREATE INDEX is needed for that pair.
|
||
|
||
-- Frequent query: outliers by distance (report top-N).
|
||
CREATE INDEX IF NOT EXISTS address_mismatch_audit_distance_idx
|
||
ON address_mismatch_audit (distance_m DESC)
|
||
WHERE distance_m IS NOT NULL;
|
||
|
||
-- Per-batch / per-district aggregation in the report SQL.
|
||
CREATE INDEX IF NOT EXISTS address_mismatch_audit_batch_district_idx
|
||
ON address_mismatch_audit (audit_batch, district);
|
||
|
||
COMMENT ON TABLE address_mismatch_audit IS
|
||
'Phase 1 audit of houses.address vs Yandex reverse geocode. Issue #582.';
|
||
COMMENT ON COLUMN address_mismatch_audit.audit_batch IS
|
||
'Free-form batch label, typically "<date>_run<n>". Same batch re-run = resumable.';
|
||
COMMENT ON COLUMN address_mismatch_audit.audit_status IS
|
||
'ok | no_match | error | blocked (captcha). Drives report summary segmentation.';
|
||
COMMENT ON COLUMN address_mismatch_audit.raw_payload IS
|
||
'Raw Yandex response (API JSON or scraped DOM snapshot). Retained for forensics.';
|
||
|
||
-- ---------------------------------------------------------------------------
|
||
-- 2. FDW foreign table for EKB district polygons
|
||
-- ---------------------------------------------------------------------------
|
||
-- The polygons live in gendesign master DB (table ekb_districts_geom, loaded
|
||
-- from OSM). Re-using the same gendesign_remote server defined in
|
||
-- 060_postgres_fdw_extension.sql — extension + server already created there,
|
||
-- so this migration just declares the additional foreign table.
|
||
|
||
CREATE EXTENSION IF NOT EXISTS postgres_fdw;
|
||
|
||
DO $$
|
||
BEGIN
|
||
IF NOT EXISTS (SELECT 1 FROM pg_foreign_server WHERE srvname = 'gendesign_remote') THEN
|
||
CREATE SERVER gendesign_remote
|
||
FOREIGN DATA WRAPPER postgres_fdw
|
||
OPTIONS (host 'gendesign-postgres', dbname 'gendesign', port '5432',
|
||
extensions 'postgis', connect_timeout '3');
|
||
END IF;
|
||
END$$;
|
||
|
||
-- Drop existing foreign table — keeps re-applies idempotent if column shape
|
||
-- ever changes upstream.
|
||
DROP FOREIGN TABLE IF EXISTS gendesign_ekb_districts_geom;
|
||
|
||
CREATE FOREIGN TABLE gendesign_ekb_districts_geom (
|
||
district_name text,
|
||
geom geometry(MultiPolygon, 4326)
|
||
)
|
||
SERVER gendesign_remote
|
||
OPTIONS (schema_name 'public', table_name 'ekb_districts_geom');
|
||
|
||
COMMENT ON FOREIGN TABLE gendesign_ekb_districts_geom IS
|
||
'Live view of gendesign.ekb_districts_geom (8 EKB admin districts as OSM polygons). '
|
||
'USER MAPPING managed by backend startup (app.core.fdw).';
|
||
|
||
COMMIT;
|