-- 130_backfill_listings_house_id_fk.sql
-- One-time backfill of listings.house_id_fk from the canonical house_sources link,
-- for rows written before the realtime FK mirror existed.
--
-- WHY (root cause — commit e63b21e, 2026-06-17):
--   The realtime FK mirror in app/services/scrapers/base.py:607-616
--     UPDATE listings SET house_id_fk = CAST(:hid AS bigint)
--      WHERE id = CAST(:lid AS bigint)
--        AND house_id_fk IS DISTINCT FROM CAST(:hid AS bigint)
--   only landed 2026-06-17. Listings scraped/re-scraped BEFORE that commit were
--   matched into house_sources (ext_source, ext_id -> house_id) but never had the
--   FK mirrored back onto listings.house_id_fk. The worst-affected cohort is the
--   9205 cian novostroyki frozen at 2026-05-31 (no re-scrape since => never picked
--   up the mirror). A NULL house_id_fk breaks the canonical "same-building via
--   house_id_fk" estimator path (estimator.py: JOIN houses ON house_id_fk = id),
--   silently degrading comparables for those buildings.
--
-- WHAT:
--   Deterministically fill house_id_fk for every listing where it is NULL but the
--   listing carries house_source/house_ext_id that resolves to a house via
--   house_sources (the canonical match table, UNIQUE(ext_source, ext_id)).
--   This is exactly the linkage the realtime mirror produces — just applied offline
--   to the backlog. General across all sources (avito/cian/yandex/n1), consistent
--   with the mirror, which fires for every source.
--
-- DETERMINISTIC ONLY — no fuzzy fallback:
--   PROD measurement (read-only, 2026-06-19): of active cian novostroyki with NULL
--   house_id_fk, 9235 / 9235 (100%) resolve via the house_sources source-id join;
--   zero unresolvable. This is a clean source-id join, NOT the address-fuzzy /
--   derived-house path of 063_*.sql Step 5 (deliberately avoided — that path
--   fragments houses). We only ever fill a NULL FK from an EXISTING canonical
--   match, so there is no mis-linking and no guessing.
--
-- RELATION TO 063_backfill_houses_and_link_listings.sql:
--   063 Step 4 joined listings -> houses directly on (house_source = houses.source,
--   house_ext_id = houses.ext_house_id), a denormalized path. This migration adds
--   the house_sources-canonical path (listings -> house_sources -> house_id), which
--   matches the realtime mirror's semantics and covers matches that landed in
--   house_sources after 063 ran. Different join, complementary — not a duplicate.
--
-- DEPENDENCIES:
--   - listings (house_source text, house_ext_id text, house_id_fk bigint REFERENCES
--     houses(id)) -- 011_listings_alter.sql:22-25
--   - house_sources (house_id bigint, ext_source text, ext_id text,
--     UNIQUE(ext_source, ext_id)) -- 028_matching_tables.sql:36-45
--   - listings_house_ext_id_idx ON (house_source, house_ext_id)
--     WHERE house_ext_id IS NOT NULL -- 011_listings_alter.sql (sargable lookup)
--
-- IDEMPOTENCY:
--   - UPDATE ... WHERE house_id_fk IS NULL  -> re-running is a strict no-op once
--     the backlog is drained. Touches only NULL FKs; never overwrites an existing
--     linkage. No DDL, no destructive ops.
--
-- ONE-TIME: future rows are linked in realtime by the base.py mirror (e63b21e),
--   so this file only needs to run once against the existing backlog.
--
-- Deploy order: after 129_avito_full_load_incremental_split.sql (latest revision).

BEGIN;

-- -------------------------------------------------------------------------
-- Backfill: fill NULL house_id_fk from the canonical house_sources match,
-- then report counters. Run inside one DO block so GET DIAGNOSTICS ROW_COUNT
-- reflects the UPDATE's affected rows (a ROW_COUNT in a separate DO block
-- would NOT see the prior statement). Mirrors app/services/scrapers/base.py:
-- 607-616 (realtime FK mirror), applied offline to the pre-e63b21e backlog.
-- Uses listings_house_ext_id_idx (house_source, house_ext_id) for a sargable
-- lookup into house_sources.
--
-- residual_with_extid = listings still NULL that DO carry house_source/
-- house_ext_id but have NO house_sources match (expected ~0 per prod measurement).
-- -------------------------------------------------------------------------
DO $$
DECLARE
    v_filled_this_run     bigint;
    v_still_null          bigint;
    v_residual_with_extid bigint;
    v_linked_total        bigint;
    v_listings_total      bigint;
BEGIN
    UPDATE listings l
    SET house_id_fk = hs.house_id
    FROM house_sources hs
    WHERE l.house_id_fk IS NULL
      AND l.house_source IS NOT NULL
      AND l.house_ext_id IS NOT NULL
      AND hs.ext_source = l.house_source
      AND hs.ext_id     = l.house_ext_id
      AND hs.house_id IS NOT NULL;

    GET DIAGNOSTICS v_filled_this_run = ROW_COUNT;

    SELECT count(*)                                       INTO v_listings_total FROM listings;
    SELECT count(*) FILTER (WHERE house_id_fk IS NOT NULL) INTO v_linked_total   FROM listings;
    SELECT count(*) FILTER (WHERE house_id_fk IS NULL)     INTO v_still_null      FROM listings;
    SELECT count(*) FILTER (
        WHERE house_id_fk IS NULL
          AND house_source IS NOT NULL
          AND house_ext_id IS NOT NULL
    )                                                      INTO v_residual_with_extid FROM listings;

    RAISE NOTICE
        'backfill 130: filled house_id_fk=% this run'
        ' | linked=%/% | still_null=% (of which % carry source+ext_id but lack a house_sources match)',
        v_filled_this_run,
        v_linked_total, v_listings_total,
        v_still_null, v_residual_with_extid;

    IF v_residual_with_extid > 0 THEN
        RAISE WARNING
            'backfill 130: % listings still NULL despite carrying house_source+house_ext_id'
            ' (no house_sources match) — investigate house matching for these sources',
            v_residual_with_extid;
    END IF;
END $$;

COMMIT;
