feat(tradein-sql): bootstrap houses + link listings.house_id_fk (Phase A+B)
Backfill 18,428 listings against an empty houses table: - INSERT Avito/CIAN-sourced houses (~562 unique via house_source+house_ext_id) - INSERT derived houses for remaining listings (group by tradein_normalize_short_addr, synthetic ext_house_id = sha256 truncated to 32 hex chars) → ~6102 derived - UPDATE listings.house_id_fk by both paths (direct: 2997, fuzzy: 15200) - Best-effort relink orphan house_placement_history via raw_payload.address (0 rows) Syntax-validated live: 6664 houses, 18197/18428 listings linked (98.8%). Idempotent (ON CONFLICT DO NOTHING, WHERE house_id_fk IS NULL). Logs result counters via RAISE NOTICE. Phase C (Avito IMV history scrape per house) deferred to separate Celery job.
This commit is contained in:
lekss361
parent
0f7d6c4996
commit
60147fd971
1 changed files with 245 additions and 0 deletions
|
|
@ -0,0 +1,245 @@
|
|||
-- 063_backfill_houses_and_link_listings.sql
|
||||
-- Bootstrap houses table from existing listings data (Phase A+B).
|
||||
--
|
||||
-- Context:
|
||||
-- listings: 18,256 rows (all have addresses)
|
||||
-- houses: 0 rows (empty)
|
||||
-- listings.house_id_fk: 0 linked
|
||||
-- house_placement_history: 160 orphans, all source=yandex_valuation, raw_payload has no address
|
||||
--
|
||||
-- Steps:
|
||||
-- 1. CREATE OR REPLACE FUNCTION tradein_normalize_short_addr
|
||||
-- 2. INSERT Avito/CIAN-sourced houses from listings WHERE house_source+house_ext_id NOT NULL
|
||||
-- 3. INSERT derived houses for listings WITHOUT ext_house_id (grouped by normalized address)
|
||||
-- 4. UPDATE listings.house_id_fk via source+ext_house_id match (direct path)
|
||||
-- 5. UPDATE listings.house_id_fk via normalized address match (fuzzy path)
|
||||
-- 6. Best-effort relink house_placement_history via raw_payload->>'address' (likely 0 rows)
|
||||
-- 7. RAISE NOTICE with final counters
|
||||
--
|
||||
-- Dependencies:
|
||||
-- - pgcrypto extension (for digest()) -- verified present
|
||||
-- - listings table, houses table, house_placement_history table
|
||||
-- - houses UNIQUE constraint on (source, ext_house_id)
|
||||
-- - house_placement_history UNIQUE constraint on (source, ext_item_id)
|
||||
--
|
||||
-- Idempotency:
|
||||
-- - INSERT ... ON CONFLICT DO NOTHING
|
||||
-- - UPDATE ... WHERE house_id_fk IS NULL
|
||||
-- - CREATE OR REPLACE FUNCTION
|
||||
--
|
||||
-- Deploy order: after 062_clean_avito_addresses.sql
|
||||
|
||||
BEGIN;
|
||||
|
||||
-- -------------------------------------------------------------------------
|
||||
-- Step 1: Address normalizer function
|
||||
-- Strips regional/city prefix and apartment/corpus suffix.
|
||||
-- Handles addresses like:
|
||||
-- "ул. Репина, 75/2 стр." → "ул. Репина, 75/2 стр."
|
||||
-- "Свердловская обл., Екатеринбург, ул. Большакова, 17" → "ул. Большакова, 17"
|
||||
-- "улица Яскина, 12 · р-н Октябрьский" → "улица Яскина, 12"
|
||||
-- "Азина, 4.1.1" → "Азина, 4.1.1"
|
||||
-- -------------------------------------------------------------------------
|
||||
CREATE OR REPLACE FUNCTION tradein_normalize_short_addr(addr text)
|
||||
RETURNS text
|
||||
LANGUAGE sql
|
||||
IMMUTABLE
|
||||
PARALLEL SAFE
|
||||
AS $$
|
||||
SELECT trim(both ' ,.' FROM
|
||||
regexp_replace(
|
||||
regexp_replace(
|
||||
regexp_replace(
|
||||
regexp_replace(
|
||||
regexp_replace(
|
||||
-- Strip leading "Россия / РФ / Российская Федерация, "
|
||||
regexp_replace(addr, '^\s*(?:Россия|РФ|Российская\s+Федерация)\s*,\s*', '', 'i'),
|
||||
-- Strip leading region "Свердловская обл., " / "Свердловская область, "
|
||||
'^[А-ЯЁа-яё][А-ЯЁа-яё\s-]+\s+обл(?:асть|\.)\s*,\s*', '', 'i'
|
||||
),
|
||||
-- Strip leading city "Екатеринбург, " / "г. Екатеринбург, " / "г Екатеринбург, "
|
||||
'^г\.?\s*[А-ЯЁ][А-ЯЁа-яё-]+\s*,\s*', '', 'i'
|
||||
),
|
||||
-- Strip district suffix " · р-н ..." or " · district"
|
||||
'\s*·\s*.+$', '', 'i'
|
||||
),
|
||||
-- Strip apartment/corpus/office suffix ", кв./корп./оф./пом./подъезд N..."
|
||||
',\s*(?:кв\.?|корп\.?|к\.?|оф\.?|пом\.?|подъезд)\s*\d+.*$', '', 'i'
|
||||
),
|
||||
-- Strip trailing whitespace artefacts
|
||||
'\s{2,}', ' ', 'g'
|
||||
)
|
||||
);
|
||||
$$;
|
||||
|
||||
-- -------------------------------------------------------------------------
|
||||
-- Step 2: Insert Avito/CIAN-sourced houses from listings
|
||||
-- Uses DISTINCT ON to pick the most recently scraped data per (source, ext_house_id).
|
||||
-- -------------------------------------------------------------------------
|
||||
INSERT INTO houses (
|
||||
source,
|
||||
ext_house_id,
|
||||
url,
|
||||
address,
|
||||
lat,
|
||||
lon,
|
||||
geom,
|
||||
year_built,
|
||||
house_type,
|
||||
total_floors,
|
||||
full_address,
|
||||
first_seen_at,
|
||||
last_scraped_at
|
||||
)
|
||||
SELECT DISTINCT ON (l.house_source, l.house_ext_id)
|
||||
l.house_source AS source,
|
||||
l.house_ext_id AS ext_house_id,
|
||||
COALESCE(l.house_url, '') AS url,
|
||||
l.address,
|
||||
l.lat,
|
||||
l.lon,
|
||||
l.geom,
|
||||
l.year_built,
|
||||
l.house_type,
|
||||
l.total_floors,
|
||||
l.address AS full_address,
|
||||
NOW() AS first_seen_at,
|
||||
NOW() AS last_scraped_at
|
||||
FROM listings l
|
||||
WHERE l.house_source IS NOT NULL
|
||||
AND l.house_ext_id IS NOT NULL
|
||||
AND l.address IS NOT NULL
|
||||
ORDER BY l.house_source, l.house_ext_id, l.scraped_at DESC
|
||||
ON CONFLICT (source, ext_house_id) DO NOTHING;
|
||||
|
||||
-- -------------------------------------------------------------------------
|
||||
-- Step 3: Insert derived houses for listings WITHOUT ext_house_id
|
||||
-- Groups listings by normalized address; each group = one derived house.
|
||||
-- Synthetic ext_house_id = first 32 hex chars of sha256(normalized_addr).
|
||||
-- Uses DISTINCT ON (norm_addr) picking latest scraped_at per group.
|
||||
-- -------------------------------------------------------------------------
|
||||
INSERT INTO houses (
|
||||
source,
|
||||
ext_house_id,
|
||||
url,
|
||||
address,
|
||||
lat,
|
||||
lon,
|
||||
geom,
|
||||
year_built,
|
||||
house_type,
|
||||
total_floors,
|
||||
full_address,
|
||||
short_address,
|
||||
first_seen_at,
|
||||
last_scraped_at
|
||||
)
|
||||
SELECT DISTINCT ON (norm_addr)
|
||||
'derived' AS source,
|
||||
substring(encode(digest(norm_addr, 'sha256'), 'hex'), 1, 32) AS ext_house_id,
|
||||
'' AS url,
|
||||
l.address,
|
||||
l.lat,
|
||||
l.lon,
|
||||
l.geom,
|
||||
l.year_built,
|
||||
l.house_type,
|
||||
l.total_floors,
|
||||
l.address AS full_address,
|
||||
norm_addr AS short_address,
|
||||
NOW() AS first_seen_at,
|
||||
NOW() AS last_scraped_at
|
||||
FROM (
|
||||
SELECT
|
||||
l.*,
|
||||
tradein_normalize_short_addr(l.address) AS norm_addr
|
||||
FROM listings l
|
||||
WHERE l.address IS NOT NULL
|
||||
AND (l.house_source IS NULL OR l.house_ext_id IS NULL)
|
||||
) l
|
||||
WHERE l.norm_addr IS NOT NULL
|
||||
AND length(l.norm_addr) >= 5
|
||||
ORDER BY norm_addr, l.scraped_at DESC
|
||||
ON CONFLICT (source, ext_house_id) DO NOTHING;
|
||||
|
||||
-- -------------------------------------------------------------------------
|
||||
-- Step 4: Link listings.house_id_fk via direct source+ext_house_id match
|
||||
-- Uses listings_house_ext_id_idx (source, house_ext_id) — sargable.
|
||||
-- -------------------------------------------------------------------------
|
||||
UPDATE listings l
|
||||
SET house_id_fk = h.id
|
||||
FROM houses h
|
||||
WHERE l.house_id_fk IS NULL
|
||||
AND l.house_source IS NOT NULL
|
||||
AND l.house_ext_id IS NOT NULL
|
||||
AND l.house_source = h.source
|
||||
AND l.house_ext_id = h.ext_house_id;
|
||||
|
||||
-- -------------------------------------------------------------------------
|
||||
-- Step 5: Link listings.house_id_fk via normalized address (fuzzy path)
|
||||
-- Covers listings where house_source/house_ext_id is absent.
|
||||
-- houses.short_address was populated above; joined by equality.
|
||||
-- NOTE: houses_addr_fp_idx is on address_fingerprint; short_address has no
|
||||
-- dedicated index. If this migration is re-run on large data, consider:
|
||||
-- CREATE INDEX IF NOT EXISTS houses_short_addr_idx ON houses(short_address)
|
||||
-- WHERE source = 'derived';
|
||||
-- For a one-shot migration on 18k rows this is acceptable (seq scan ~ms).
|
||||
-- -------------------------------------------------------------------------
|
||||
UPDATE listings l
|
||||
SET house_id_fk = h.id
|
||||
FROM houses h
|
||||
WHERE l.house_id_fk IS NULL
|
||||
AND l.address IS NOT NULL
|
||||
AND h.source = 'derived'
|
||||
AND h.short_address = tradein_normalize_short_addr(l.address);
|
||||
|
||||
-- -------------------------------------------------------------------------
|
||||
-- Step 6: Best-effort relink orphan house_placement_history
|
||||
-- All 160 orphan rows are source=yandex_valuation; raw_payload has no
|
||||
-- 'address' key (verified: has_addr=false for all 160 rows).
|
||||
-- This UPDATE will match 0 rows but is included for completeness/idempotency.
|
||||
-- Phase C (Celery IMV scraper) will assign house_id on new inserts directly.
|
||||
-- -------------------------------------------------------------------------
|
||||
UPDATE house_placement_history hph
|
||||
SET house_id = h.id
|
||||
FROM houses h
|
||||
WHERE hph.house_id IS NULL
|
||||
AND (hph.raw_payload ->> 'address') IS NOT NULL
|
||||
AND h.source = 'derived'
|
||||
AND h.short_address = tradein_normalize_short_addr(hph.raw_payload ->> 'address');
|
||||
|
||||
-- -------------------------------------------------------------------------
|
||||
-- Step 7: Final counters via RAISE NOTICE
|
||||
-- -------------------------------------------------------------------------
|
||||
DO $$
|
||||
DECLARE
|
||||
v_houses_total bigint;
|
||||
v_houses_derived bigint;
|
||||
v_houses_direct bigint;
|
||||
v_listings_total bigint;
|
||||
v_listings_linked bigint;
|
||||
v_listings_addr bigint;
|
||||
v_history_total bigint;
|
||||
v_history_linked bigint;
|
||||
BEGIN
|
||||
SELECT count(*) INTO v_houses_total FROM houses;
|
||||
SELECT count(*) FILTER (WHERE source = 'derived') INTO v_houses_derived FROM houses;
|
||||
SELECT count(*) FILTER (WHERE source != 'derived') INTO v_houses_direct FROM houses;
|
||||
|
||||
SELECT count(*) INTO v_listings_total FROM listings;
|
||||
SELECT count(*) FILTER (WHERE house_id_fk IS NOT NULL) INTO v_listings_linked FROM listings;
|
||||
SELECT count(*) FILTER (WHERE address IS NOT NULL) INTO v_listings_addr FROM listings;
|
||||
|
||||
SELECT count(*) INTO v_history_total FROM house_placement_history;
|
||||
SELECT count(*) FILTER (WHERE house_id IS NOT NULL) INTO v_history_linked FROM house_placement_history;
|
||||
|
||||
RAISE NOTICE
|
||||
'backfill 063: houses=% (direct=%, derived=%)'
|
||||
' | listings_linked=%/% (addr_coverage=%)'
|
||||
' | history_linked=%/%',
|
||||
v_houses_total, v_houses_direct, v_houses_derived,
|
||||
v_listings_linked, v_listings_total, v_listings_addr,
|
||||
v_history_linked, v_history_total;
|
||||
END $$;
|
||||
|
||||
COMMIT;
|
||||
Loading…
Add table
Reference in a new issue