9e1c7d5147
Avito DOM leaks inline <style>.css-XXX{...}</style> inside the location <p>,
which gets pulled by .text(strip=True). Add _clean_address() helper that strips
CSS rules + post-address noise (Площадь N года, от N мин.), apply at extraction
site. Also applied to address_full in avito_detail.py via import from avito.py.
Backfill migration cleans the ~hundreds of historical rows.
Source: estimate a0a0b820-e8a8-4eee-aa73-0ab3b98ac233 analog #4.
15 lines
640 B
PL/PgSQL
15 lines
640 B
PL/PgSQL
-- 062_clean_avito_addresses.sql
|
|
-- Strip CSS noise from listings.address for Avito source (one-shot backfill).
|
|
-- Idempotent: re-run is safe (regexp_replace on already-clean string is a no-op).
|
|
BEGIN;
|
|
|
|
UPDATE listings
|
|
SET address = trim(both ' ,.' FROM
|
|
regexp_replace(
|
|
regexp_replace(address, '\.?css-[a-z0-9_-]+\s*\{[^}]*\}', '', 'gi'),
|
|
'\s*(Площадь \d|от \d+\s?мин\.|css-[a-z0-9_-]+).*$', '', 'i'
|
|
))
|
|
WHERE source = 'avito'
|
|
AND (address ~* 'css-[a-z0-9_-]+' OR address ~* 'Площадь \d|от \d+\s?мин\.');
|
|
|
|
COMMIT;
|