tradein-mvp/backend/data/sql/136_backfill_listings_house_id_fk_source_identity.sql

@@ -0,0 +1,120 @@
+-- 136_backfill_listings_house_id_fk_source_identity.sql
+-- One-time backfill of listings.house_id_fk via the PER-LISTING-IDENTITY canonical
+-- link (house_sources by ext_source=source, ext_id=source_id), for NULL-FK rows that
+-- carry NO house_source/house_ext_id and were therefore NOT covered by migration 130.
+--
+-- WHY (continuation of #1781 / migration 130):
+--   The realtime FK mirror (app/services/scrapers/base.py:704-750) resolves a house
+--   for each scraped listing and writes it back to listings.house_id_fk. When the
+--   scraper extracted an explicit house catalog id it matches on
+--   (house_source, house_ext_id); OTHERWISE it falls back to the listing's own
+--   identity:
+--       h_src = lot.house_source or lot.source        # -> 'cian' / 'avito' / ...
+--       h_ext = lot.house_ext_id or (source_id or dedup_hash)
+--       match_or_create_house(ext_source=h_src, ext_id=h_ext, ...)
+--   Tier 1 of the matcher (matching/houses.py:96-113, confidence 1.0) is exactly
+--       SELECT house_id FROM house_sources WHERE ext_source=:s AND ext_id=:e
+--   so the canonical record for these rows lives in house_sources keyed by
+--   (source, source_id). Rows last scraped before the mirror landed (2026-06-17),
+--   or re-scraped and re-NULLed and not re-scraped since (the cian novostroyki cohort
+--   frozen at 2026-05-31, #1781), keep house_id_fk = NULL even though that canonical
+--   (source, source_id) -> house_id mapping already exists in house_sources.
+--
+--   Migration 130 only filled the (house_source, house_ext_id) path. PROD measurement
+--   (read-only, 2026-06-27): 1884 ACTIVE NULL-FK listings remain, and 0 of them carry
+--   house_source/house_ext_id -> 130's re-run would be a strict no-op. Of those 1884,
+--   744 resolve via the (source, source_id) house_sources path covered here; the
+--   remaining ~1140 active have no canonical source-id match yet and need a
+--   re-scrape / re-match (scraper-coverage work, out of scope — #1781 Лёха).
+--   Including inactive/historical rows, the (source, source_id) path resolves ~4608
+--   NULL-FK listings total (dry-run BEGIN/ROLLBACK, 2026-06-27) — this migration fills
+--   all of them (no is_active filter): correct canonical grouping for history too
+--   (helps point-in-time backtest #667), harmless to live scoring (estimator filters
+--   is_active). Faithful to the mirror, which links rows regardless of active status.
+--
+-- WHAT:
+--   Fill house_id_fk for every listing where it is NULL but (source, source_id)
+--   resolves to a house via house_sources (UNIQUE(ext_source, ext_id) -> exactly one
+--   house_id). This is precisely the linkage the realtime mirror produces for these
+--   rows (Tier-1 source_exact, confidence 1.0) — applied offline to the backlog.
+--   General across all sources (avito/cian/yandex/n1), consistent with the mirror.
+--
+-- DETERMINISTIC ONLY — no fuzzy fallback:
+--   We only ever fill a NULL FK from an EXISTING canonical house_sources match
+--   (the matcher's own confidence-1.0 record). No address-fuzzy, no geo guessing,
+--   no house creation. Same safety guarantee as 130, complementary join key.
+--
+-- COLLISION SAFETY:
+--   The join is keyed on (ext_source = listings.source, ext_id = listings.source_id).
+--   house_sources is written ONLY by match_or_create_house; house-catalog ids live
+--   under distinct ext_source values (e.g. 'cian_newbuilding', 'yandex_realty_nb'),
+--   so a per-listing 'cian'/'avito' source_id cannot collide with a catalog entry.
+--   UNIQUE(ext_source, ext_id) guarantees at most one house_id per (source, source_id).
+--   This is the same lookup the trusted live mirror performs every scrape — replaying
+--   it offline carries no additional risk.
+--
+-- DEPENDENCIES:
+--   - listings (source text, source_id text, house_id_fk bigint REFERENCES houses(id))
+--     -- 002_core_tables.sql, 011_listings_alter.sql
+--   - house_sources (house_id bigint, ext_source text, ext_id text,
+--     UNIQUE(ext_source, ext_id)) -- 028_matching_tables.sql:36-45
+--
+-- IDEMPOTENCY:
+--   UPDATE ... WHERE house_id_fk IS NULL -> re-running is a strict no-op once drained.
+--   Touches only NULL FKs; never overwrites an existing linkage. No DDL.
+--
+-- ONE-TIME: future rows are linked in realtime by the base.py mirror. The unresolved
+--   ~1140 are unblocked only by re-scrape/re-match (scraper coverage, #1781).
+--
+-- Deploy order: after 135_scrape_schedules_seed_house_dedup_merge.sql.
+
+BEGIN;
+
+-- One DO block so GET DIAGNOSTICS ROW_COUNT reflects the UPDATE. Mirrors
+-- app/services/scrapers/base.py:704-750 (realtime FK mirror, per-listing-identity
+-- fallback path), applied offline to the NULL-FK backlog migration 130 could not reach.
+DO $$
+DECLARE
+    v_filled_this_run bigint;
+    v_still_null      bigint;
+    v_residual_resolv bigint;
+    v_linked_total    bigint;
+    v_listings_total  bigint;
+BEGIN
+    UPDATE listings l
+    SET house_id_fk = hs.house_id
+    FROM house_sources hs
+    WHERE l.house_id_fk IS NULL
+      AND l.source_id IS NOT NULL
+      AND hs.ext_source = l.source
+      AND hs.ext_id     = l.source_id
+      AND hs.house_id IS NOT NULL;
+
+    GET DIAGNOSTICS v_filled_this_run = ROW_COUNT;
+
+    SELECT count(*)                                        INTO v_listings_total FROM listings;
+    SELECT count(*) FILTER (WHERE house_id_fk IS NOT NULL) INTO v_linked_total   FROM listings;
+    SELECT count(*) FILTER (WHERE house_id_fk IS NULL)     INTO v_still_null      FROM listings;
+    -- still-NULL rows that WOULD resolve via (source, source_id) — expected 0 after this run
+    SELECT count(*) INTO v_residual_resolv
+    FROM listings l
+    JOIN house_sources hs
+      ON hs.ext_source = l.source AND hs.ext_id = l.source_id AND hs.house_id IS NOT NULL
+    WHERE l.house_id_fk IS NULL AND l.source_id IS NOT NULL;
+
+    RAISE NOTICE
+        'backfill 136: filled house_id_fk=% this run (source-identity path)'
+        ' | linked=%/% | still_null=% (of which % still resolvable via (source,source_id))',
+        v_filled_this_run,
+        v_linked_total, v_listings_total,
+        v_still_null, v_residual_resolv;
+
+    IF v_residual_resolv > 0 THEN
+        RAISE WARNING
+            'backfill 136: % listings still NULL despite a (source,source_id) house_sources'
+            ' match — investigate (unexpected after this UPDATE)',
+            v_residual_resolv;
+    END IF;
+END $$;
+
+COMMIT;