e868b01278
SQL migration 064: adds houses.imv_status + last_imv_attempt_at + imv_error_reason
columns, creates house_imv_evaluations (1:1 with houses, latest IMV result)
and house_suggestions (8 Avito similar lots per house, UNIQUE on (house_id, ext_item_id)).
Script scripts/backfill-house-imv.py: resumable iterator that picks houses with
imv_status='pending' AND lat/lon NOT NULL, runs evaluate_via_imv() per house with
median lot-params from linked listings, saves house_imv_evaluations +
house_placement_history (from result.placement_history items) + house_suggestions
(from result.suggestions items). Configurable --batch, --delay, --limit-total.
Survives restart via imv_status state machine ('pending' -> 'ok'/'not_found'/
'transient_error'/'error'/'no_params'/'no_address'). Long backoff on
IMVAuthError (anti-captcha).
79 lines
2.7 KiB
PL/PgSQL
79 lines
2.7 KiB
PL/PgSQL
-- 064: Phase C — Avito IMV per-house backfill schema.
|
|
-- Idempotent: re-run safe (IF NOT EXISTS).
|
|
BEGIN;
|
|
|
|
-- Track IMV scrape state per house (resumable backfill)
|
|
ALTER TABLE houses
|
|
ADD COLUMN IF NOT EXISTS imv_status text NOT NULL DEFAULT 'pending',
|
|
ADD COLUMN IF NOT EXISTS last_imv_attempt_at timestamptz,
|
|
ADD COLUMN IF NOT EXISTS imv_error_reason text;
|
|
|
|
CREATE INDEX IF NOT EXISTS houses_imv_status_idx
|
|
ON houses(imv_status, last_imv_attempt_at NULLS FIRST)
|
|
WHERE imv_status IN ('pending', 'transient_error');
|
|
|
|
-- House-level IMV evaluation result (1:1 with houses, latest fetch)
|
|
CREATE TABLE IF NOT EXISTS house_imv_evaluations (
|
|
id bigserial PRIMARY KEY,
|
|
house_id bigint NOT NULL REFERENCES houses(id) ON DELETE CASCADE,
|
|
cache_key text,
|
|
recommended_price bigint,
|
|
lower_price bigint,
|
|
higher_price bigint,
|
|
market_count integer,
|
|
raw_response jsonb,
|
|
fetched_at timestamptz NOT NULL DEFAULT NOW(),
|
|
-- Lot-params used (median from listings or default)
|
|
rooms integer,
|
|
area_m2 numeric(8,2),
|
|
floor integer,
|
|
floor_at_home integer,
|
|
house_type text,
|
|
renovation_type text,
|
|
has_balcony boolean,
|
|
has_loggia boolean
|
|
);
|
|
|
|
CREATE UNIQUE INDEX IF NOT EXISTS house_imv_eval_house_uniq_idx
|
|
ON house_imv_evaluations(house_id);
|
|
CREATE INDEX IF NOT EXISTS house_imv_eval_fetched_idx
|
|
ON house_imv_evaluations(fetched_at DESC);
|
|
|
|
-- Avito suggestions[] per house — 8 similar lots nearby
|
|
CREATE TABLE IF NOT EXISTS house_suggestions (
|
|
id bigserial PRIMARY KEY,
|
|
house_id bigint NOT NULL REFERENCES houses(id) ON DELETE CASCADE,
|
|
ext_item_id text NOT NULL, -- Avito item id
|
|
title text,
|
|
address text,
|
|
price_rub bigint,
|
|
area_m2 numeric(8,2),
|
|
rooms integer,
|
|
floor integer,
|
|
total_floors integer,
|
|
exposure_days integer,
|
|
publish_date date,
|
|
item_link text,
|
|
image_link text,
|
|
metro_name text,
|
|
metro_distance text,
|
|
has_good_price_badge boolean,
|
|
raw_payload jsonb,
|
|
fetched_at timestamptz NOT NULL DEFAULT NOW()
|
|
);
|
|
|
|
CREATE UNIQUE INDEX IF NOT EXISTS house_suggestions_uniq_idx
|
|
ON house_suggestions(house_id, ext_item_id);
|
|
CREATE INDEX IF NOT EXISTS house_suggestions_house_idx
|
|
ON house_suggestions(house_id, fetched_at DESC);
|
|
|
|
-- Verify
|
|
DO $$
|
|
DECLARE
|
|
pending_n bigint;
|
|
BEGIN
|
|
SELECT count(*) INTO pending_n FROM houses WHERE imv_status = 'pending';
|
|
RAISE NOTICE 'migration 064: % houses pending IMV backfill', pending_n;
|
|
END $$;
|
|
|
|
COMMIT;
|