From d04ee3834711c233c6531c299a8219945be114e2 Mon Sep 17 00:00:00 2001 From: lekss361 Date: Sun, 24 May 2026 16:34:32 +0300 Subject: [PATCH] fix(tradein/cian): sanitize valuation cache writes (047 UNIQUE, fill 026+044 cols) - 047_cian_history_sanitize.sql: dedup existing rows then add UNIQUE(listing_id, change_time) so ON CONFLICT DO NOTHING in cian_detail.save_detail_enrichment suppresses real duplicates, not just PK collisions. Idempotent via DO $$ pg_constraint guard. - cian_valuation._save_to_cache: fill price_rub/accuracy (mig 026), low_price/high_price (mig 044), house_id/listing_id (mig 044). COALESCE on house_id/listing_id preserves existing FK on cache refresh. - cian_valuation._load_from_cache: restore low_price/high_price from DB into sale_price_from/sale_price_to so callers get the full band. - estimate_via_cian_valuation: add house_id/listing_id kwargs (default None), pass through to _save_to_cache. No callsite changes needed (defaults preserve existing behavior; PR 4 can supply IDs when context is available). --- .../app/services/scrapers/cian_valuation.py | 28 +++++++++++++++++- .../data/sql/047_cian_history_sanitize.sql | 29 +++++++++++++++++++ 2 files changed, 56 insertions(+), 1 deletion(-) create mode 100644 tradein-mvp/backend/data/sql/047_cian_history_sanitize.sql diff --git a/tradein-mvp/backend/app/services/scrapers/cian_valuation.py b/tradein-mvp/backend/app/services/scrapers/cian_valuation.py index af403eeb..f2fe97f9 100644 --- a/tradein-mvp/backend/app/services/scrapers/cian_valuation.py +++ b/tradein-mvp/backend/app/services/scrapers/cian_valuation.py @@ -135,6 +135,8 @@ async def estimate_via_cian_valuation( repair_type: str = "cosmetic", deal_type: str = "sale", use_cache: bool = True, + house_id: int | None = None, + listing_id: int | None = None, ) -> CianValuationResult | None: """Estimate value via Cian's authenticated Valuation Calculator. @@ -219,6 +221,8 @@ async def estimate_via_cian_valuation( repair_type=repair_type, deal_type=deal_type, result=result, + house_id=house_id, + listing_id=listing_id, ) logger.info( @@ -355,7 +359,9 @@ def _load_from_cache(db: Session, cache_key: str) -> CianValuationResult | None: chart_change_pct, chart_change_direction, external_house_id, - filters_hash + filters_hash, + low_price, + high_price FROM external_valuations WHERE source = 'cian_valuation' AND cache_key = :ck @@ -379,6 +385,8 @@ def _load_from_cache(db: Session, cache_key: str) -> CianValuationResult | None: chart_change_direction=row["chart_change_direction"], external_house_id=row["external_house_id"], filters_hash=row["filters_hash"], + sale_price_from=_parse_num(row["low_price"]), + sale_price_to=_parse_num(row["high_price"]), raw_state=row["raw_payload"] if isinstance(row["raw_payload"], dict) else None, is_authenticated=True, # cache только authenticated results ) @@ -396,6 +404,8 @@ def _save_to_cache( repair_type: str, deal_type: str, result: CianValuationResult, + house_id: int | None = None, + listing_id: int | None = None, ) -> None: """INSERT result into external_valuations (source='cian_valuation') с 24h expiry. @@ -409,6 +419,9 @@ def _save_to_cache( source, cache_key, address, total_area, rooms_count, floor, total_floors, repair_type, deal_type, + price_rub, accuracy, + low_price, high_price, + house_id, listing_id, sale_price_rub, sale_accuracy, rent_price_rub, rent_accuracy, chart, chart_change_pct, chart_change_direction, @@ -419,12 +432,21 @@ def _save_to_cache( CAST(:ta AS numeric), :rc, :fl, :tf, :rt, :dt, CAST(:sp AS numeric), CAST(:sa AS numeric), + CAST(:lp AS numeric), CAST(:hp AS numeric), + CAST(:hid AS bigint), CAST(:lid AS bigint), + CAST(:sp AS numeric), CAST(:sa AS numeric), CAST(:rp AS numeric), CAST(:ra AS numeric), CAST(:ch AS jsonb), CAST(:ccp AS numeric), :ccd, :ehi, :fh, CAST(:raw AS jsonb), NOW(), NOW() + INTERVAL '24 hours' ) ON CONFLICT (source, cache_key) DO UPDATE SET + price_rub = EXCLUDED.price_rub, + accuracy = EXCLUDED.accuracy, + low_price = EXCLUDED.low_price, + high_price = EXCLUDED.high_price, + house_id = COALESCE(EXCLUDED.house_id, external_valuations.house_id), + listing_id = COALESCE(EXCLUDED.listing_id, external_valuations.listing_id), sale_price_rub = EXCLUDED.sale_price_rub, sale_accuracy = EXCLUDED.sale_accuracy, rent_price_rub = EXCLUDED.rent_price_rub, @@ -449,6 +471,10 @@ def _save_to_cache( "dt": deal_type, "sp": result.sale_price_rub, "sa": result.sale_accuracy, + "lp": result.sale_price_from, + "hp": result.sale_price_to, + "hid": house_id, + "lid": listing_id, "rp": result.rent_price_rub, "ra": result.rent_accuracy, "ch": json.dumps(result.chart), diff --git a/tradein-mvp/backend/data/sql/047_cian_history_sanitize.sql b/tradein-mvp/backend/data/sql/047_cian_history_sanitize.sql new file mode 100644 index 00000000..db9202ae --- /dev/null +++ b/tradein-mvp/backend/data/sql/047_cian_history_sanitize.sql @@ -0,0 +1,29 @@ +-- 047_cian_history_sanitize.sql +-- Purpose: Add UNIQUE(listing_id, change_time) to offer_price_history so that +-- the `ON CONFLICT DO NOTHING` in cian_detail.save_detail_enrichment +-- is semantically correct (currently only suppresses PK conflicts). +-- Dependencies: 023_offer_price_history.sql +-- Deploy order: Apply after 046. + +BEGIN; + +-- Dedupe existing rows before adding constraint (safe if table empty in prod). +DELETE FROM offer_price_history a +USING offer_price_history b +WHERE a.id > b.id + AND a.listing_id = b.listing_id + AND a.change_time = b.change_time; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conname = 'offer_price_history_listing_change_uq' + ) THEN + ALTER TABLE offer_price_history + ADD CONSTRAINT offer_price_history_listing_change_uq + UNIQUE (listing_id, change_time); + END IF; +END $$; + +COMMIT;