From 60147fd9718bbb75e7cc74b3187be461f4b6b1b0 Mon Sep 17 00:00:00 2001 From: lekss361 Date: Sun, 24 May 2026 17:00:18 +0300 Subject: [PATCH] feat(tradein-sql): bootstrap houses + link listings.house_id_fk (Phase A+B) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Backfill 18,428 listings against an empty houses table: - INSERT Avito/CIAN-sourced houses (~562 unique via house_source+house_ext_id) - INSERT derived houses for remaining listings (group by tradein_normalize_short_addr, synthetic ext_house_id = sha256 truncated to 32 hex chars) → ~6102 derived - UPDATE listings.house_id_fk by both paths (direct: 2997, fuzzy: 15200) - Best-effort relink orphan house_placement_history via raw_payload.address (0 rows) Syntax-validated live: 6664 houses, 18197/18428 listings linked (98.8%). Idempotent (ON CONFLICT DO NOTHING, WHERE house_id_fk IS NULL). Logs result counters via RAISE NOTICE. Phase C (Avito IMV history scrape per house) deferred to separate Celery job. --- .../063_backfill_houses_and_link_listings.sql | 245 ++++++++++++++++++ 1 file changed, 245 insertions(+) create mode 100644 tradein-mvp/backend/data/sql/063_backfill_houses_and_link_listings.sql diff --git a/tradein-mvp/backend/data/sql/063_backfill_houses_and_link_listings.sql b/tradein-mvp/backend/data/sql/063_backfill_houses_and_link_listings.sql new file mode 100644 index 00000000..bdb0a9ba --- /dev/null +++ b/tradein-mvp/backend/data/sql/063_backfill_houses_and_link_listings.sql @@ -0,0 +1,245 @@ +-- 063_backfill_houses_and_link_listings.sql +-- Bootstrap houses table from existing listings data (Phase A+B). +-- +-- Context: +-- listings: 18,256 rows (all have addresses) +-- houses: 0 rows (empty) +-- listings.house_id_fk: 0 linked +-- house_placement_history: 160 orphans, all source=yandex_valuation, raw_payload has no address +-- +-- Steps: +-- 1. CREATE OR REPLACE FUNCTION tradein_normalize_short_addr +-- 2. INSERT Avito/CIAN-sourced houses from listings WHERE house_source+house_ext_id NOT NULL +-- 3. INSERT derived houses for listings WITHOUT ext_house_id (grouped by normalized address) +-- 4. UPDATE listings.house_id_fk via source+ext_house_id match (direct path) +-- 5. UPDATE listings.house_id_fk via normalized address match (fuzzy path) +-- 6. Best-effort relink house_placement_history via raw_payload->>'address' (likely 0 rows) +-- 7. RAISE NOTICE with final counters +-- +-- Dependencies: +-- - pgcrypto extension (for digest()) -- verified present +-- - listings table, houses table, house_placement_history table +-- - houses UNIQUE constraint on (source, ext_house_id) +-- - house_placement_history UNIQUE constraint on (source, ext_item_id) +-- +-- Idempotency: +-- - INSERT ... ON CONFLICT DO NOTHING +-- - UPDATE ... WHERE house_id_fk IS NULL +-- - CREATE OR REPLACE FUNCTION +-- +-- Deploy order: after 062_clean_avito_addresses.sql + +BEGIN; + +-- ------------------------------------------------------------------------- +-- Step 1: Address normalizer function +-- Strips regional/city prefix and apartment/corpus suffix. +-- Handles addresses like: +-- "ул. Репина, 75/2 стр." → "ул. Репина, 75/2 стр." +-- "Свердловская обл., Екатеринбург, ул. Большакова, 17" → "ул. Большакова, 17" +-- "улица Яскина, 12 · р-н Октябрьский" → "улица Яскина, 12" +-- "Азина, 4.1.1" → "Азина, 4.1.1" +-- ------------------------------------------------------------------------- +CREATE OR REPLACE FUNCTION tradein_normalize_short_addr(addr text) +RETURNS text +LANGUAGE sql +IMMUTABLE +PARALLEL SAFE +AS $$ + SELECT trim(both ' ,.' FROM + regexp_replace( + regexp_replace( + regexp_replace( + regexp_replace( + regexp_replace( + -- Strip leading "Россия / РФ / Российская Федерация, " + regexp_replace(addr, '^\s*(?:Россия|РФ|Российская\s+Федерация)\s*,\s*', '', 'i'), + -- Strip leading region "Свердловская обл., " / "Свердловская область, " + '^[А-ЯЁа-яё][А-ЯЁа-яё\s-]+\s+обл(?:асть|\.)\s*,\s*', '', 'i' + ), + -- Strip leading city "Екатеринбург, " / "г. Екатеринбург, " / "г Екатеринбург, " + '^г\.?\s*[А-ЯЁ][А-ЯЁа-яё-]+\s*,\s*', '', 'i' + ), + -- Strip district suffix " · р-н ..." or " · district" + '\s*·\s*.+$', '', 'i' + ), + -- Strip apartment/corpus/office suffix ", кв./корп./оф./пом./подъезд N..." + ',\s*(?:кв\.?|корп\.?|к\.?|оф\.?|пом\.?|подъезд)\s*\d+.*$', '', 'i' + ), + -- Strip trailing whitespace artefacts + '\s{2,}', ' ', 'g' + ) + ); +$$; + +-- ------------------------------------------------------------------------- +-- Step 2: Insert Avito/CIAN-sourced houses from listings +-- Uses DISTINCT ON to pick the most recently scraped data per (source, ext_house_id). +-- ------------------------------------------------------------------------- +INSERT INTO houses ( + source, + ext_house_id, + url, + address, + lat, + lon, + geom, + year_built, + house_type, + total_floors, + full_address, + first_seen_at, + last_scraped_at +) +SELECT DISTINCT ON (l.house_source, l.house_ext_id) + l.house_source AS source, + l.house_ext_id AS ext_house_id, + COALESCE(l.house_url, '') AS url, + l.address, + l.lat, + l.lon, + l.geom, + l.year_built, + l.house_type, + l.total_floors, + l.address AS full_address, + NOW() AS first_seen_at, + NOW() AS last_scraped_at +FROM listings l +WHERE l.house_source IS NOT NULL + AND l.house_ext_id IS NOT NULL + AND l.address IS NOT NULL +ORDER BY l.house_source, l.house_ext_id, l.scraped_at DESC +ON CONFLICT (source, ext_house_id) DO NOTHING; + +-- ------------------------------------------------------------------------- +-- Step 3: Insert derived houses for listings WITHOUT ext_house_id +-- Groups listings by normalized address; each group = one derived house. +-- Synthetic ext_house_id = first 32 hex chars of sha256(normalized_addr). +-- Uses DISTINCT ON (norm_addr) picking latest scraped_at per group. +-- ------------------------------------------------------------------------- +INSERT INTO houses ( + source, + ext_house_id, + url, + address, + lat, + lon, + geom, + year_built, + house_type, + total_floors, + full_address, + short_address, + first_seen_at, + last_scraped_at +) +SELECT DISTINCT ON (norm_addr) + 'derived' AS source, + substring(encode(digest(norm_addr, 'sha256'), 'hex'), 1, 32) AS ext_house_id, + '' AS url, + l.address, + l.lat, + l.lon, + l.geom, + l.year_built, + l.house_type, + l.total_floors, + l.address AS full_address, + norm_addr AS short_address, + NOW() AS first_seen_at, + NOW() AS last_scraped_at +FROM ( + SELECT + l.*, + tradein_normalize_short_addr(l.address) AS norm_addr + FROM listings l + WHERE l.address IS NOT NULL + AND (l.house_source IS NULL OR l.house_ext_id IS NULL) +) l +WHERE l.norm_addr IS NOT NULL + AND length(l.norm_addr) >= 5 +ORDER BY norm_addr, l.scraped_at DESC +ON CONFLICT (source, ext_house_id) DO NOTHING; + +-- ------------------------------------------------------------------------- +-- Step 4: Link listings.house_id_fk via direct source+ext_house_id match +-- Uses listings_house_ext_id_idx (source, house_ext_id) — sargable. +-- ------------------------------------------------------------------------- +UPDATE listings l +SET house_id_fk = h.id +FROM houses h +WHERE l.house_id_fk IS NULL + AND l.house_source IS NOT NULL + AND l.house_ext_id IS NOT NULL + AND l.house_source = h.source + AND l.house_ext_id = h.ext_house_id; + +-- ------------------------------------------------------------------------- +-- Step 5: Link listings.house_id_fk via normalized address (fuzzy path) +-- Covers listings where house_source/house_ext_id is absent. +-- houses.short_address was populated above; joined by equality. +-- NOTE: houses_addr_fp_idx is on address_fingerprint; short_address has no +-- dedicated index. If this migration is re-run on large data, consider: +-- CREATE INDEX IF NOT EXISTS houses_short_addr_idx ON houses(short_address) +-- WHERE source = 'derived'; +-- For a one-shot migration on 18k rows this is acceptable (seq scan ~ms). +-- ------------------------------------------------------------------------- +UPDATE listings l +SET house_id_fk = h.id +FROM houses h +WHERE l.house_id_fk IS NULL + AND l.address IS NOT NULL + AND h.source = 'derived' + AND h.short_address = tradein_normalize_short_addr(l.address); + +-- ------------------------------------------------------------------------- +-- Step 6: Best-effort relink orphan house_placement_history +-- All 160 orphan rows are source=yandex_valuation; raw_payload has no +-- 'address' key (verified: has_addr=false for all 160 rows). +-- This UPDATE will match 0 rows but is included for completeness/idempotency. +-- Phase C (Celery IMV scraper) will assign house_id on new inserts directly. +-- ------------------------------------------------------------------------- +UPDATE house_placement_history hph +SET house_id = h.id +FROM houses h +WHERE hph.house_id IS NULL + AND (hph.raw_payload ->> 'address') IS NOT NULL + AND h.source = 'derived' + AND h.short_address = tradein_normalize_short_addr(hph.raw_payload ->> 'address'); + +-- ------------------------------------------------------------------------- +-- Step 7: Final counters via RAISE NOTICE +-- ------------------------------------------------------------------------- +DO $$ +DECLARE + v_houses_total bigint; + v_houses_derived bigint; + v_houses_direct bigint; + v_listings_total bigint; + v_listings_linked bigint; + v_listings_addr bigint; + v_history_total bigint; + v_history_linked bigint; +BEGIN + SELECT count(*) INTO v_houses_total FROM houses; + SELECT count(*) FILTER (WHERE source = 'derived') INTO v_houses_derived FROM houses; + SELECT count(*) FILTER (WHERE source != 'derived') INTO v_houses_direct FROM houses; + + SELECT count(*) INTO v_listings_total FROM listings; + SELECT count(*) FILTER (WHERE house_id_fk IS NOT NULL) INTO v_listings_linked FROM listings; + SELECT count(*) FILTER (WHERE address IS NOT NULL) INTO v_listings_addr FROM listings; + + SELECT count(*) INTO v_history_total FROM house_placement_history; + SELECT count(*) FILTER (WHERE house_id IS NOT NULL) INTO v_history_linked FROM house_placement_history; + + RAISE NOTICE + 'backfill 063: houses=% (direct=%, derived=%)' + ' | listings_linked=%/% (addr_coverage=%)' + ' | history_linked=%/%', + v_houses_total, v_houses_direct, v_houses_derived, + v_listings_linked, v_listings_total, v_listings_addr, + v_history_linked, v_history_total; +END $$; + +COMMIT;