Merge pull request 'fix(db): #1781 backfill listings.house_id_fk from house_sources (cian novostroyki)' (#1784) from fix/1781-cian-novostroyki-fk-backfill into main

Reviewed-on: #1784

tradein-mvp/backend/data/sql/130_backfill_listings_house_id_fk.sql

@@ -0,0 +1,117 @@
+-- 130_backfill_listings_house_id_fk.sql
+-- One-time backfill of listings.house_id_fk from the canonical house_sources link,
+-- for rows written before the realtime FK mirror existed.
+--
+-- WHY (root cause — commit e63b21e, 2026-06-17):
+--   The realtime FK mirror in app/services/scrapers/base.py:607-616
+--     UPDATE listings SET house_id_fk = CAST(:hid AS bigint)
+--      WHERE id = CAST(:lid AS bigint)
+--        AND house_id_fk IS DISTINCT FROM CAST(:hid AS bigint)
+--   only landed 2026-06-17. Listings scraped/re-scraped BEFORE that commit were
+--   matched into house_sources (ext_source, ext_id -> house_id) but never had the
+--   FK mirrored back onto listings.house_id_fk. The worst-affected cohort is the
+--   9205 cian novostroyki frozen at 2026-05-31 (no re-scrape since => never picked
+--   up the mirror). A NULL house_id_fk breaks the canonical "same-building via
+--   house_id_fk" estimator path (estimator.py: JOIN houses ON house_id_fk = id),
+--   silently degrading comparables for those buildings.
+--
+-- WHAT:
+--   Deterministically fill house_id_fk for every listing where it is NULL but the
+--   listing carries house_source/house_ext_id that resolves to a house via
+--   house_sources (the canonical match table, UNIQUE(ext_source, ext_id)).
+--   This is exactly the linkage the realtime mirror produces — just applied offline
+--   to the backlog. General across all sources (avito/cian/yandex/n1), consistent
+--   with the mirror, which fires for every source.
+--
+-- DETERMINISTIC ONLY — no fuzzy fallback:
+--   PROD measurement (read-only, 2026-06-19): of active cian novostroyki with NULL
+--   house_id_fk, 9235 / 9235 (100%) resolve via the house_sources source-id join;
+--   zero unresolvable. This is a clean source-id join, NOT the address-fuzzy /
+--   derived-house path of 063_*.sql Step 5 (deliberately avoided — that path
+--   fragments houses). We only ever fill a NULL FK from an EXISTING canonical
+--   match, so there is no mis-linking and no guessing.
+--
+-- RELATION TO 063_backfill_houses_and_link_listings.sql:
+--   063 Step 4 joined listings -> houses directly on (house_source = houses.source,
+--   house_ext_id = houses.ext_house_id), a denormalized path. This migration adds
+--   the house_sources-canonical path (listings -> house_sources -> house_id), which
+--   matches the realtime mirror's semantics and covers matches that landed in
+--   house_sources after 063 ran. Different join, complementary — not a duplicate.
+--
+-- DEPENDENCIES:
+--   - listings (house_source text, house_ext_id text, house_id_fk bigint REFERENCES
+--     houses(id)) -- 011_listings_alter.sql:22-25
+--   - house_sources (house_id bigint, ext_source text, ext_id text,
+--     UNIQUE(ext_source, ext_id)) -- 028_matching_tables.sql:36-45
+--   - listings_house_ext_id_idx ON (house_source, house_ext_id)
+--     WHERE house_ext_id IS NOT NULL -- 011_listings_alter.sql (sargable lookup)
+--
+-- IDEMPOTENCY:
+--   - UPDATE ... WHERE house_id_fk IS NULL  -> re-running is a strict no-op once
+--     the backlog is drained. Touches only NULL FKs; never overwrites an existing
+--     linkage. No DDL, no destructive ops.
+--
+-- ONE-TIME: future rows are linked in realtime by the base.py mirror (e63b21e),
+--   so this file only needs to run once against the existing backlog.
+--
+-- Deploy order: after 129_avito_full_load_incremental_split.sql (latest revision).
+
+BEGIN;
+
+-- -------------------------------------------------------------------------
+-- Backfill: fill NULL house_id_fk from the canonical house_sources match,
+-- then report counters. Run inside one DO block so GET DIAGNOSTICS ROW_COUNT
+-- reflects the UPDATE's affected rows (a ROW_COUNT in a separate DO block
+-- would NOT see the prior statement). Mirrors app/services/scrapers/base.py:
+-- 607-616 (realtime FK mirror), applied offline to the pre-e63b21e backlog.
+-- Uses listings_house_ext_id_idx (house_source, house_ext_id) for a sargable
+-- lookup into house_sources.
+--
+-- residual_with_extid = listings still NULL that DO carry house_source/
+-- house_ext_id but have NO house_sources match (expected ~0 per prod measurement).
+-- -------------------------------------------------------------------------
+DO $$
+DECLARE
+    v_filled_this_run     bigint;
+    v_still_null          bigint;
+    v_residual_with_extid bigint;
+    v_linked_total        bigint;
+    v_listings_total      bigint;
+BEGIN
+    UPDATE listings l
+    SET house_id_fk = hs.house_id
+    FROM house_sources hs
+    WHERE l.house_id_fk IS NULL
+      AND l.house_source IS NOT NULL
+      AND l.house_ext_id IS NOT NULL
+      AND hs.ext_source = l.house_source
+      AND hs.ext_id     = l.house_ext_id
+      AND hs.house_id IS NOT NULL;
+
+    GET DIAGNOSTICS v_filled_this_run = ROW_COUNT;
+
+    SELECT count(*)                                       INTO v_listings_total FROM listings;
+    SELECT count(*) FILTER (WHERE house_id_fk IS NOT NULL) INTO v_linked_total   FROM listings;
+    SELECT count(*) FILTER (WHERE house_id_fk IS NULL)     INTO v_still_null      FROM listings;
+    SELECT count(*) FILTER (
+        WHERE house_id_fk IS NULL
+          AND house_source IS NOT NULL
+          AND house_ext_id IS NOT NULL
+    )                                                      INTO v_residual_with_extid FROM listings;
+
+    RAISE NOTICE
+        'backfill 130: filled house_id_fk=% this run'
+        ' | linked=%/% | still_null=% (of which % carry source+ext_id but lack a house_sources match)',
+        v_filled_this_run,
+        v_linked_total, v_listings_total,
+        v_still_null, v_residual_with_extid;
+
+    IF v_residual_with_extid > 0 THEN
+        RAISE WARNING
+            'backfill 130: % listings still NULL despite carrying house_source+house_ext_id'
+            ' (no house_sources match) — investigate house matching for these sources',
+            v_residual_with_extid;
+    END IF;
+END $$;
+
+COMMIT;