Merge pull request 'fix(scrapers): deactivate pre-#753 ghost duplicate listings (dedup correctness)' (#1557) from fix/deactivate-ghost-duplicate-listings into main

Reviewed-on: #1557

tradein-mvp/backend/data/sql/113_deactivate_ghost_duplicate_listings.sql

@@ -0,0 +1,56 @@
+-- 113_deactivate_ghost_duplicate_listings.sql
+-- One-shot data-correctness migration: deactivate pre-#753 "ghost" duplicate listings.
+--
+-- Context / root cause (#753):
+--   Before #753, dedup_hash was computed including the listing price, so every price
+--   change on a listing produced a brand-new row with a different hash. The current
+--   scraper computes the canonical hash as:
+--       encode(sha256((source || '|' || source_id)::bytea), 'hex')
+--   and upserts ON CONFLICT (dedup_hash). The old ghost rows carry a price-inclusive
+--   hash that is never matched by an upsert again; their last_seen stays frozen and
+--   they stay is_active=true indefinitely, inflating active-listing counts.
+--
+-- What this migration does:
+--   Within each (source, source_id) group that has >1 rows AND at least one row with
+--   the canonical hash, set is_active=false on every ACTIVE row whose dedup_hash is
+--   NOT the canonical value. Groups containing NO canonical row (35 tiny groups,
+--   all-ghost) are excluded by the HAVING filter — deactivating them would kill the
+--   entire group, which is wrong.
+--
+-- Why it is safe:
+--   - Wrapped in BEGIN/COMMIT — atomic.
+--   - Idempotent: the WHERE clause includes AND is_active=true, so re-running
+--     deactivates nothing more after the first successful run.
+--   - merged_into column is deliberately NOT set (column is dead, no code writer).
+--   - Dry-run on prod (rolled-back transaction, 2026-06-16) returned UPDATE 167:
+--       yandex ~164, cian ~1, n1 ~2, avito 0 — matches expected estimate.
+--
+-- Expected row count: ~167 rows deactivated.
+--
+-- Dependencies:
+--   - 002_core_tables.sql  — listings table, is_active, dedup_hash, source, source_id
+--   - #753 dedup_hash fix  — canonical hash formula must already be in production code
+--
+-- Deploy order: apply before (or alongside) any code that relies on clean active counts.
+-- No application code changes required — this is a pure data cleanup.
+
+BEGIN;
+
+UPDATE listings l
+SET is_active = false
+FROM (
+    SELECT source, source_id
+    FROM listings
+    WHERE source_id IS NOT NULL
+    GROUP BY source, source_id
+    HAVING count(*) > 1
+       AND count(*) FILTER (
+           WHERE dedup_hash = encode(sha256((source || '|' || source_id)::bytea), 'hex')
+       ) >= 1
+) g
+WHERE l.source = g.source
+  AND l.source_id = g.source_id
+  AND l.is_active = true
+  AND l.dedup_hash <> encode(sha256((l.source || '|' || l.source_id)::bytea), 'hex');
+
+COMMIT;