From 6ce60a0d8c93aa9b9d9642b48d3ec5e989f2f0b8 Mon Sep 17 00:00:00 2001 From: Light1YT Date: Sat, 27 Jun 2026 05:01:02 +0500 Subject: [PATCH] fix(tradein): backfill house_id_fk via source-identity path (#1781) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Migration 130 only filled house_id_fk for listings carrying house_source/house_ext_id. Prod (2026-06-27): 1884 active NULL-FK rows remain, 0 with house_source/ext_id (130 re-run = no-op). They were matched via the per-listing-identity path the realtime mirror uses when no house catalog id exists: match_or_create_house(ext_source=source, ext_id=source_id) -> Tier-1 house_sources lookup (confidence 1.0). 136 replays that exact lookup offline: UPDATE listings SET house_id_fk = house_sources.house_id WHERE (ext_source=source, ext_id=source_id) and house_id_fk IS NULL. Faithful to base.py:704-750; idempotent (NULL guard); deterministic (no fuzzy/geo); collision-safe (catalog ids live under distinct ext_source like 'cian_newbuilding'). Dry-run BEGIN/ROLLBACK fills 4608 NULL-FK rows (744 active), restoring estimator Tier-S "same-building" grouping. The remaining ~1140 active need re-scrape/ re-match (scraper coverage — #1781 Лёха). Refs #1781 --- ...l_listings_house_id_fk_source_identity.sql | 120 ++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 tradein-mvp/backend/data/sql/136_backfill_listings_house_id_fk_source_identity.sql diff --git a/tradein-mvp/backend/data/sql/136_backfill_listings_house_id_fk_source_identity.sql b/tradein-mvp/backend/data/sql/136_backfill_listings_house_id_fk_source_identity.sql new file mode 100644 index 00000000..5b9a363c --- /dev/null +++ b/tradein-mvp/backend/data/sql/136_backfill_listings_house_id_fk_source_identity.sql @@ -0,0 +1,120 @@ +-- 136_backfill_listings_house_id_fk_source_identity.sql +-- One-time backfill of listings.house_id_fk via the PER-LISTING-IDENTITY canonical +-- link (house_sources by ext_source=source, ext_id=source_id), for NULL-FK rows that +-- carry NO house_source/house_ext_id and were therefore NOT covered by migration 130. +-- +-- WHY (continuation of #1781 / migration 130): +-- The realtime FK mirror (app/services/scrapers/base.py:704-750) resolves a house +-- for each scraped listing and writes it back to listings.house_id_fk. When the +-- scraper extracted an explicit house catalog id it matches on +-- (house_source, house_ext_id); OTHERWISE it falls back to the listing's own +-- identity: +-- h_src = lot.house_source or lot.source # -> 'cian' / 'avito' / ... +-- h_ext = lot.house_ext_id or (source_id or dedup_hash) +-- match_or_create_house(ext_source=h_src, ext_id=h_ext, ...) +-- Tier 1 of the matcher (matching/houses.py:96-113, confidence 1.0) is exactly +-- SELECT house_id FROM house_sources WHERE ext_source=:s AND ext_id=:e +-- so the canonical record for these rows lives in house_sources keyed by +-- (source, source_id). Rows last scraped before the mirror landed (2026-06-17), +-- or re-scraped and re-NULLed and not re-scraped since (the cian novostroyki cohort +-- frozen at 2026-05-31, #1781), keep house_id_fk = NULL even though that canonical +-- (source, source_id) -> house_id mapping already exists in house_sources. +-- +-- Migration 130 only filled the (house_source, house_ext_id) path. PROD measurement +-- (read-only, 2026-06-27): 1884 ACTIVE NULL-FK listings remain, and 0 of them carry +-- house_source/house_ext_id -> 130's re-run would be a strict no-op. Of those 1884, +-- 744 resolve via the (source, source_id) house_sources path covered here; the +-- remaining ~1140 active have no canonical source-id match yet and need a +-- re-scrape / re-match (scraper-coverage work, out of scope — #1781 Лёха). +-- Including inactive/historical rows, the (source, source_id) path resolves ~4608 +-- NULL-FK listings total (dry-run BEGIN/ROLLBACK, 2026-06-27) — this migration fills +-- all of them (no is_active filter): correct canonical grouping for history too +-- (helps point-in-time backtest #667), harmless to live scoring (estimator filters +-- is_active). Faithful to the mirror, which links rows regardless of active status. +-- +-- WHAT: +-- Fill house_id_fk for every listing where it is NULL but (source, source_id) +-- resolves to a house via house_sources (UNIQUE(ext_source, ext_id) -> exactly one +-- house_id). This is precisely the linkage the realtime mirror produces for these +-- rows (Tier-1 source_exact, confidence 1.0) — applied offline to the backlog. +-- General across all sources (avito/cian/yandex/n1), consistent with the mirror. +-- +-- DETERMINISTIC ONLY — no fuzzy fallback: +-- We only ever fill a NULL FK from an EXISTING canonical house_sources match +-- (the matcher's own confidence-1.0 record). No address-fuzzy, no geo guessing, +-- no house creation. Same safety guarantee as 130, complementary join key. +-- +-- COLLISION SAFETY: +-- The join is keyed on (ext_source = listings.source, ext_id = listings.source_id). +-- house_sources is written ONLY by match_or_create_house; house-catalog ids live +-- under distinct ext_source values (e.g. 'cian_newbuilding', 'yandex_realty_nb'), +-- so a per-listing 'cian'/'avito' source_id cannot collide with a catalog entry. +-- UNIQUE(ext_source, ext_id) guarantees at most one house_id per (source, source_id). +-- This is the same lookup the trusted live mirror performs every scrape — replaying +-- it offline carries no additional risk. +-- +-- DEPENDENCIES: +-- - listings (source text, source_id text, house_id_fk bigint REFERENCES houses(id)) +-- -- 002_core_tables.sql, 011_listings_alter.sql +-- - house_sources (house_id bigint, ext_source text, ext_id text, +-- UNIQUE(ext_source, ext_id)) -- 028_matching_tables.sql:36-45 +-- +-- IDEMPOTENCY: +-- UPDATE ... WHERE house_id_fk IS NULL -> re-running is a strict no-op once drained. +-- Touches only NULL FKs; never overwrites an existing linkage. No DDL. +-- +-- ONE-TIME: future rows are linked in realtime by the base.py mirror. The unresolved +-- ~1140 are unblocked only by re-scrape/re-match (scraper coverage, #1781). +-- +-- Deploy order: after 135_scrape_schedules_seed_house_dedup_merge.sql. + +BEGIN; + +-- One DO block so GET DIAGNOSTICS ROW_COUNT reflects the UPDATE. Mirrors +-- app/services/scrapers/base.py:704-750 (realtime FK mirror, per-listing-identity +-- fallback path), applied offline to the NULL-FK backlog migration 130 could not reach. +DO $$ +DECLARE + v_filled_this_run bigint; + v_still_null bigint; + v_residual_resolv bigint; + v_linked_total bigint; + v_listings_total bigint; +BEGIN + UPDATE listings l + SET house_id_fk = hs.house_id + FROM house_sources hs + WHERE l.house_id_fk IS NULL + AND l.source_id IS NOT NULL + AND hs.ext_source = l.source + AND hs.ext_id = l.source_id + AND hs.house_id IS NOT NULL; + + GET DIAGNOSTICS v_filled_this_run = ROW_COUNT; + + SELECT count(*) INTO v_listings_total FROM listings; + SELECT count(*) FILTER (WHERE house_id_fk IS NOT NULL) INTO v_linked_total FROM listings; + SELECT count(*) FILTER (WHERE house_id_fk IS NULL) INTO v_still_null FROM listings; + -- still-NULL rows that WOULD resolve via (source, source_id) — expected 0 after this run + SELECT count(*) INTO v_residual_resolv + FROM listings l + JOIN house_sources hs + ON hs.ext_source = l.source AND hs.ext_id = l.source_id AND hs.house_id IS NOT NULL + WHERE l.house_id_fk IS NULL AND l.source_id IS NOT NULL; + + RAISE NOTICE + 'backfill 136: filled house_id_fk=% this run (source-identity path)' + ' | linked=%/% | still_null=% (of which % still resolvable via (source,source_id))', + v_filled_this_run, + v_linked_total, v_listings_total, + v_still_null, v_residual_resolv; + + IF v_residual_resolv > 0 THEN + RAISE WARNING + 'backfill 136: % listings still NULL despite a (source,source_id) house_sources' + ' match — investigate (unexpected after this UPDATE)', + v_residual_resolv; + END IF; +END $$; + +COMMIT;