From 59d58cf931cef8d63d787b28ed38035857cf0496 Mon Sep 17 00:00:00 2001 From: bot-backend Date: Tue, 16 Jun 2026 19:43:30 +0300 Subject: [PATCH] feat(tradein/db): price-history trigger (116) + last_seen_at index (117) 116: AFTER UPDATE OF price_rub trigger on listings writes derived offer_price_history rows for avito/yandex; cian excluded to preserve source-provided backfill (cian_price_history.py LEFT JOIN oph IS NULL guard). 117: partial composite index (source, last_seen_at) WHERE is_active to accelerate deactivate_stale_listings seq-scan on ~39k rows. --- ...116_offer_price_history_change_trigger.sql | 100 ++++++++++++++++++ ...17_listings_last_seen_deactivate_index.sql | 35 ++++++ 2 files changed, 135 insertions(+) create mode 100644 tradein-mvp/backend/data/sql/116_offer_price_history_change_trigger.sql create mode 100644 tradein-mvp/backend/data/sql/117_listings_last_seen_deactivate_index.sql diff --git a/tradein-mvp/backend/data/sql/116_offer_price_history_change_trigger.sql b/tradein-mvp/backend/data/sql/116_offer_price_history_change_trigger.sql new file mode 100644 index 00000000..04ab3a25 --- /dev/null +++ b/tradein-mvp/backend/data/sql/116_offer_price_history_change_trigger.sql @@ -0,0 +1,100 @@ +-- Migration 116: derived price-change trigger on listings → offer_price_history +-- +-- Problem: +-- The listings upsert in base.py::save_listings uses ON CONFLICT (dedup_hash) +-- DO UPDATE SET price_rub = EXCLUDED.price_rub (and other fields). This +-- silently overwrites the price with no history record. avito and yandex +-- price changes are therefore lost on every re-scrape. +-- +-- Only cian has offer_price_history data today (source-provided, scraped +-- from cian's embedded priceChanges JSON by cian_detail.py and backfilled +-- by cian_price_history.py). +-- +-- Fix: +-- An AFTER UPDATE OF price_rub trigger on listings that writes a derived +-- history row when price_rub actually changes value. +-- +-- Cian-exclusion rationale (CRITICAL): +-- cian_price_history.py::backfill_cian_price_history selects cian listings +-- to process via: +-- LEFT JOIN offer_price_history oph ON oph.listing_id = l.id +-- WHERE l.source = 'cian' AND oph.listing_id IS NULL +-- i.e. it ONLY picks cian listings with zero existing history rows. +-- If this trigger were to insert a NOW() row for a cian listing, that +-- listing would be permanently excluded from the backfill, losing the full +-- source-provided timeline with accurate historical timestamps. Therefore +-- the trigger MUST fire only for non-cian sources. +-- +-- Other no-ops: +-- deactivate_stale_listings runs UPDATE listings SET is_active=false — +-- it does not touch price_rub, so AFTER UPDATE OF price_rub means the +-- trigger function is not even called for those updates. +-- Geocode backfill (SET geom=...) similarly never touches price_rub. +-- +-- Dependencies: listings table, offer_price_history table, +-- offer_price_history_listing_change_uq unique constraint (listing_id, change_time) +-- +-- Deploy order: standalone; no other migration must precede or follow. +-- Idempotent: safe to run multiple times (CREATE OR REPLACE + DROP IF EXISTS). + +BEGIN; + +CREATE OR REPLACE FUNCTION record_listing_price_change() +RETURNS trigger +LANGUAGE plpgsql +AS $$ +BEGIN + -- Skip if price did not actually change (same-value upsert write). + IF NEW.price_rub IS NOT DISTINCT FROM OLD.price_rub THEN + RETURN NEW; + END IF; + -- Skip if the new price is NULL (defensive; price_rub is NOT NULL in practice). + IF NEW.price_rub IS NULL THEN + RETURN NEW; + END IF; + -- Skip cian: its history is source-provided via cian_price_history.py backfill. + -- That service selects listings WHERE oph.listing_id IS NULL (zero rows in + -- offer_price_history). Inserting a trigger row here would permanently + -- exclude those listings from the backfill, losing accurate historical timestamps. + IF NEW.source = 'cian' THEN + RETURN NEW; + END IF; + + INSERT INTO offer_price_history ( + listing_id, + change_time, + price_rub, + source, + diff_percent, + recorded_at + ) VALUES ( + NEW.id, + now(), + NEW.price_rub, + NEW.source, + CASE + WHEN OLD.price_rub IS NOT NULL AND OLD.price_rub > 0 + THEN round( + (NEW.price_rub::numeric - OLD.price_rub::numeric) + / OLD.price_rub::numeric * 100, + 2 + ) + ELSE NULL + END, + now() + ) + ON CONFLICT ON CONSTRAINT offer_price_history_listing_change_uq + DO NOTHING; + + RETURN NEW; +END; +$$; + +DROP TRIGGER IF EXISTS listings_price_change_trg ON listings; + +CREATE TRIGGER listings_price_change_trg + AFTER UPDATE OF price_rub ON listings + FOR EACH ROW + EXECUTE FUNCTION record_listing_price_change(); + +COMMIT; diff --git a/tradein-mvp/backend/data/sql/117_listings_last_seen_deactivate_index.sql b/tradein-mvp/backend/data/sql/117_listings_last_seen_deactivate_index.sql new file mode 100644 index 00000000..900cbf72 --- /dev/null +++ b/tradein-mvp/backend/data/sql/117_listings_last_seen_deactivate_index.sql @@ -0,0 +1,35 @@ +-- Migration 117: partial composite index to accelerate deactivate_stale_listings +-- +-- Problem: +-- deactivate_stale_listings executes: +-- UPDATE listings SET is_active = false +-- WHERE source = :x +-- AND is_active = true +-- AND last_seen_at < NOW() - interval '...' +-- [AND listing_segment = ANY(...)] +-- With ~39k rows and no index on last_seen_at or is_active, every call +-- performs a sequential scan. +-- +-- Fix: +-- A partial B-tree index on (source, last_seen_at) WHERE is_active. +-- The WHERE is_active partial predicate eliminates already-deactivated rows +-- (typically the majority after steady-state), keeping the index small. +-- The (source, last_seen_at) composite matches the equality+range predicates +-- in the deactivate query so the planner can use an index scan. +-- +-- Note on CONCURRENTLY: +-- NOT used here. The deploy migration runner wraps each file in an explicit +-- transaction (BEGIN/COMMIT). CREATE INDEX CONCURRENTLY cannot run inside +-- a transaction block. Plain CREATE INDEX holds an ACCESS SHARE lock; +-- at ~39k rows the lock duration is sub-second and acceptable. +-- +-- Idempotent: IF NOT EXISTS makes repeated runs safe. +-- Deploy order: standalone. + +BEGIN; + +CREATE INDEX IF NOT EXISTS listings_active_source_lastseen_idx + ON listings (source, last_seen_at) + WHERE is_active; + +COMMIT; -- 2.45.3