diff --git a/tradein-mvp/backend/data/sql/130_backfill_listings_house_id_fk.sql b/tradein-mvp/backend/data/sql/130_backfill_listings_house_id_fk.sql new file mode 100644 index 00000000..e98bad86 --- /dev/null +++ b/tradein-mvp/backend/data/sql/130_backfill_listings_house_id_fk.sql @@ -0,0 +1,117 @@ +-- 130_backfill_listings_house_id_fk.sql +-- One-time backfill of listings.house_id_fk from the canonical house_sources link, +-- for rows written before the realtime FK mirror existed. +-- +-- WHY (root cause — commit e63b21e, 2026-06-17): +-- The realtime FK mirror in app/services/scrapers/base.py:607-616 +-- UPDATE listings SET house_id_fk = CAST(:hid AS bigint) +-- WHERE id = CAST(:lid AS bigint) +-- AND house_id_fk IS DISTINCT FROM CAST(:hid AS bigint) +-- only landed 2026-06-17. Listings scraped/re-scraped BEFORE that commit were +-- matched into house_sources (ext_source, ext_id -> house_id) but never had the +-- FK mirrored back onto listings.house_id_fk. The worst-affected cohort is the +-- 9205 cian novostroyki frozen at 2026-05-31 (no re-scrape since => never picked +-- up the mirror). A NULL house_id_fk breaks the canonical "same-building via +-- house_id_fk" estimator path (estimator.py: JOIN houses ON house_id_fk = id), +-- silently degrading comparables for those buildings. +-- +-- WHAT: +-- Deterministically fill house_id_fk for every listing where it is NULL but the +-- listing carries house_source/house_ext_id that resolves to a house via +-- house_sources (the canonical match table, UNIQUE(ext_source, ext_id)). +-- This is exactly the linkage the realtime mirror produces — just applied offline +-- to the backlog. General across all sources (avito/cian/yandex/n1), consistent +-- with the mirror, which fires for every source. +-- +-- DETERMINISTIC ONLY — no fuzzy fallback: +-- PROD measurement (read-only, 2026-06-19): of active cian novostroyki with NULL +-- house_id_fk, 9235 / 9235 (100%) resolve via the house_sources source-id join; +-- zero unresolvable. This is a clean source-id join, NOT the address-fuzzy / +-- derived-house path of 063_*.sql Step 5 (deliberately avoided — that path +-- fragments houses). We only ever fill a NULL FK from an EXISTING canonical +-- match, so there is no mis-linking and no guessing. +-- +-- RELATION TO 063_backfill_houses_and_link_listings.sql: +-- 063 Step 4 joined listings -> houses directly on (house_source = houses.source, +-- house_ext_id = houses.ext_house_id), a denormalized path. This migration adds +-- the house_sources-canonical path (listings -> house_sources -> house_id), which +-- matches the realtime mirror's semantics and covers matches that landed in +-- house_sources after 063 ran. Different join, complementary — not a duplicate. +-- +-- DEPENDENCIES: +-- - listings (house_source text, house_ext_id text, house_id_fk bigint REFERENCES +-- houses(id)) -- 011_listings_alter.sql:22-25 +-- - house_sources (house_id bigint, ext_source text, ext_id text, +-- UNIQUE(ext_source, ext_id)) -- 028_matching_tables.sql:36-45 +-- - listings_house_ext_id_idx ON (house_source, house_ext_id) +-- WHERE house_ext_id IS NOT NULL -- 011_listings_alter.sql (sargable lookup) +-- +-- IDEMPOTENCY: +-- - UPDATE ... WHERE house_id_fk IS NULL -> re-running is a strict no-op once +-- the backlog is drained. Touches only NULL FKs; never overwrites an existing +-- linkage. No DDL, no destructive ops. +-- +-- ONE-TIME: future rows are linked in realtime by the base.py mirror (e63b21e), +-- so this file only needs to run once against the existing backlog. +-- +-- Deploy order: after 129_avito_full_load_incremental_split.sql (latest revision). + +BEGIN; + +-- ------------------------------------------------------------------------- +-- Backfill: fill NULL house_id_fk from the canonical house_sources match, +-- then report counters. Run inside one DO block so GET DIAGNOSTICS ROW_COUNT +-- reflects the UPDATE's affected rows (a ROW_COUNT in a separate DO block +-- would NOT see the prior statement). Mirrors app/services/scrapers/base.py: +-- 607-616 (realtime FK mirror), applied offline to the pre-e63b21e backlog. +-- Uses listings_house_ext_id_idx (house_source, house_ext_id) for a sargable +-- lookup into house_sources. +-- +-- residual_with_extid = listings still NULL that DO carry house_source/ +-- house_ext_id but have NO house_sources match (expected ~0 per prod measurement). +-- ------------------------------------------------------------------------- +DO $$ +DECLARE + v_filled_this_run bigint; + v_still_null bigint; + v_residual_with_extid bigint; + v_linked_total bigint; + v_listings_total bigint; +BEGIN + UPDATE listings l + SET house_id_fk = hs.house_id + FROM house_sources hs + WHERE l.house_id_fk IS NULL + AND l.house_source IS NOT NULL + AND l.house_ext_id IS NOT NULL + AND hs.ext_source = l.house_source + AND hs.ext_id = l.house_ext_id + AND hs.house_id IS NOT NULL; + + GET DIAGNOSTICS v_filled_this_run = ROW_COUNT; + + SELECT count(*) INTO v_listings_total FROM listings; + SELECT count(*) FILTER (WHERE house_id_fk IS NOT NULL) INTO v_linked_total FROM listings; + SELECT count(*) FILTER (WHERE house_id_fk IS NULL) INTO v_still_null FROM listings; + SELECT count(*) FILTER ( + WHERE house_id_fk IS NULL + AND house_source IS NOT NULL + AND house_ext_id IS NOT NULL + ) INTO v_residual_with_extid FROM listings; + + RAISE NOTICE + 'backfill 130: filled house_id_fk=% this run' + ' | linked=%/% | still_null=% (of which % carry source+ext_id but lack a house_sources match)', + v_filled_this_run, + v_linked_total, v_listings_total, + v_still_null, v_residual_with_extid; + + IF v_residual_with_extid > 0 THEN + RAISE WARNING + 'backfill 130: % listings still NULL despite carrying house_source+house_ext_id' + ' (no house_sources match) — investigate house matching for these sources', + v_residual_with_extid; + END IF; +END $$; + +COMMIT;