-- 136_backfill_listings_house_id_fk_source_identity.sql
-- One-time backfill of listings.house_id_fk via the PER-LISTING-IDENTITY canonical
-- link (house_sources by ext_source=source, ext_id=source_id), for NULL-FK rows that
-- carry NO house_source/house_ext_id and were therefore NOT covered by migration 130.
--
-- WHY (continuation of #1781 / migration 130):
--   The realtime FK mirror (app/services/scrapers/base.py:704-750) resolves a house
--   for each scraped listing and writes it back to listings.house_id_fk. When the
--   scraper extracted an explicit house catalog id it matches on
--   (house_source, house_ext_id); OTHERWISE it falls back to the listing's own
--   identity:
--       h_src = lot.house_source or lot.source        # -> 'cian' / 'avito' / ...
--       h_ext = lot.house_ext_id or (source_id or dedup_hash)
--       match_or_create_house(ext_source=h_src, ext_id=h_ext, ...)
--   Tier 1 of the matcher (matching/houses.py:96-113, confidence 1.0) is exactly
--       SELECT house_id FROM house_sources WHERE ext_source=:s AND ext_id=:e
--   so the canonical record for these rows lives in house_sources keyed by
--   (source, source_id). Rows last scraped before the mirror landed (2026-06-17),
--   or re-scraped and re-NULLed and not re-scraped since (the cian novostroyki cohort
--   frozen at 2026-05-31, #1781), keep house_id_fk = NULL even though that canonical
--   (source, source_id) -> house_id mapping already exists in house_sources.
--
--   Migration 130 only filled the (house_source, house_ext_id) path. PROD measurement
--   (read-only, 2026-06-27): 1884 ACTIVE NULL-FK listings remain, and 0 of them carry
--   house_source/house_ext_id -> 130's re-run would be a strict no-op. Of those 1884,
--   744 resolve via the (source, source_id) house_sources path covered here; the
--   remaining ~1140 active have no canonical source-id match yet and need a
--   re-scrape / re-match (scraper-coverage work, out of scope — #1781 Лёха).
--   Including inactive/historical rows, the (source, source_id) path resolves ~4608
--   NULL-FK listings total (dry-run BEGIN/ROLLBACK, 2026-06-27) — this migration fills
--   all of them (no is_active filter): correct canonical grouping for history too
--   (helps point-in-time backtest #667), harmless to live scoring (estimator filters
--   is_active). Faithful to the mirror, which links rows regardless of active status.
--
-- WHAT:
--   Fill house_id_fk for every listing where it is NULL but (source, source_id)
--   resolves to a house via house_sources (UNIQUE(ext_source, ext_id) -> exactly one
--   house_id). This is precisely the linkage the realtime mirror produces for these
--   rows (Tier-1 source_exact, confidence 1.0) — applied offline to the backlog.
--   General across all sources (avito/cian/yandex/n1), consistent with the mirror.
--
-- DETERMINISTIC ONLY — no fuzzy fallback:
--   We only ever fill a NULL FK from an EXISTING canonical house_sources match
--   (the matcher's own confidence-1.0 record). No address-fuzzy, no geo guessing,
--   no house creation. Same safety guarantee as 130, complementary join key.
--
-- COLLISION SAFETY:
--   The join is keyed on (ext_source = listings.source, ext_id = listings.source_id).
--   house_sources is written ONLY by match_or_create_house; house-catalog ids live
--   under distinct ext_source values (e.g. 'cian_newbuilding', 'yandex_realty_nb'),
--   so a per-listing 'cian'/'avito' source_id cannot collide with a catalog entry.
--   UNIQUE(ext_source, ext_id) guarantees at most one house_id per (source, source_id).
--   This is the same lookup the trusted live mirror performs every scrape — replaying
--   it offline carries no additional risk.
--
-- DEPENDENCIES:
--   - listings (source text, source_id text, house_id_fk bigint REFERENCES houses(id))
--     -- 002_core_tables.sql, 011_listings_alter.sql
--   - house_sources (house_id bigint, ext_source text, ext_id text,
--     UNIQUE(ext_source, ext_id)) -- 028_matching_tables.sql:36-45
--
-- IDEMPOTENCY:
--   UPDATE ... WHERE house_id_fk IS NULL -> re-running is a strict no-op once drained.
--   Touches only NULL FKs; never overwrites an existing linkage. No DDL.
--
-- ONE-TIME: future rows are linked in realtime by the base.py mirror. The unresolved
--   ~1140 are unblocked only by re-scrape/re-match (scraper coverage, #1781).
--
-- Deploy order: after 135_scrape_schedules_seed_house_dedup_merge.sql.

BEGIN;

-- One DO block so GET DIAGNOSTICS ROW_COUNT reflects the UPDATE. Mirrors
-- app/services/scrapers/base.py:704-750 (realtime FK mirror, per-listing-identity
-- fallback path), applied offline to the NULL-FK backlog migration 130 could not reach.
DO $$
DECLARE
    v_filled_this_run bigint;
    v_still_null      bigint;
    v_residual_resolv bigint;
    v_linked_total    bigint;
    v_listings_total  bigint;
BEGIN
    UPDATE listings l
    SET house_id_fk = hs.house_id
    FROM house_sources hs
    WHERE l.house_id_fk IS NULL
      AND l.source_id IS NOT NULL
      AND hs.ext_source = l.source
      AND hs.ext_id     = l.source_id
      AND hs.house_id IS NOT NULL;

    GET DIAGNOSTICS v_filled_this_run = ROW_COUNT;

    SELECT count(*)                                        INTO v_listings_total FROM listings;
    SELECT count(*) FILTER (WHERE house_id_fk IS NOT NULL) INTO v_linked_total   FROM listings;
    SELECT count(*) FILTER (WHERE house_id_fk IS NULL)     INTO v_still_null      FROM listings;
    -- still-NULL rows that WOULD resolve via (source, source_id) — expected 0 after this run
    SELECT count(*) INTO v_residual_resolv
    FROM listings l
    JOIN house_sources hs
      ON hs.ext_source = l.source AND hs.ext_id = l.source_id AND hs.house_id IS NOT NULL
    WHERE l.house_id_fk IS NULL AND l.source_id IS NOT NULL;

    RAISE NOTICE
        'backfill 136: filled house_id_fk=% this run (source-identity path)'
        ' | linked=%/% | still_null=% (of which % still resolvable via (source,source_id))',
        v_filled_this_run,
        v_linked_total, v_listings_total,
        v_still_null, v_residual_resolv;

    IF v_residual_resolv > 0 THEN
        RAISE WARNING
            'backfill 136: % listings still NULL despite a (source,source_id) house_sources'
            ' match — investigate (unexpected after this UPDATE)',
            v_residual_resolv;
    END IF;
END $$;

COMMIT;
