b32628e262
Multi-Source Integration Phase 1.3. Manual operator scripts for legacy data
migration. NOT auto-applied -- run explicitly via uv/psql.
scripts/:
- backfill_001_address_fingerprints.py -- Python, populates houses.address_fingerprint
for rows where NULL (uses matching/normalize.address_fingerprint from PR #470)
- backfill_002_house_sources.sql -- INSERT INTO house_sources FROM houses,
ON CONFLICT (ext_source, ext_id) DO NOTHING (idempotent)
- backfill_003_listing_sources.sql -- INSERT INTO listing_sources FROM listings,
same pattern
- README.md -- order + usage
All idempotent -- re-running safe. confidence=1.0, matched_method='backfill'.
42 lines
1.4 KiB
PL/PgSQL
42 lines
1.4 KiB
PL/PgSQL
-- backfill_002_house_sources.sql
|
|
-- Purpose: Populate house_sources from legacy houses.(source, ext_house_id) pairs.
|
|
-- Idempotent -- ON CONFLICT DO NOTHING.
|
|
-- Run AFTER backfill_001 (fingerprint backfill is independent but logically prior).
|
|
--
|
|
-- Column notes (verified against 009_houses.sql + 029_extend_matching_valuation_dynamics.sql):
|
|
-- houses.last_scraped_at -- timestamptz, NOT NULL DEFAULT NOW() (009_houses.sql)
|
|
-- house_sources.last_seen_at -- added in 029 (nullable timestamptz)
|
|
-- house_sources columns: id, house_id, ext_source, ext_id, confidence, matched_method,
|
|
-- matched_at, ext_url, raw_payload, last_seen_at
|
|
--
|
|
-- Usage:
|
|
-- psql "$DATABASE_URL" -f tradein-mvp/scripts/backfill_002_house_sources.sql
|
|
|
|
BEGIN;
|
|
|
|
INSERT INTO house_sources (
|
|
house_id,
|
|
ext_source,
|
|
ext_id,
|
|
confidence,
|
|
matched_method,
|
|
matched_at,
|
|
last_seen_at
|
|
)
|
|
SELECT
|
|
h.id,
|
|
h.source,
|
|
h.ext_house_id,
|
|
1.0 AS confidence,
|
|
'backfill' AS matched_method,
|
|
NOW() AS matched_at,
|
|
h.last_scraped_at AS last_seen_at
|
|
FROM houses h
|
|
WHERE h.ext_house_id IS NOT NULL
|
|
ON CONFLICT (ext_source, ext_id) DO NOTHING;
|
|
|
|
\echo 'house_sources backfill complete'
|
|
SELECT count(*) AS total_house_sources FROM house_sources;
|
|
SELECT ext_source, count(*) AS rows FROM house_sources GROUP BY ext_source ORDER BY ext_source;
|
|
|
|
COMMIT;
|