From 7ef4e91e505d497393b7c515648099a9982634f6 Mon Sep 17 00:00:00 2001 From: lekss361 Date: Sun, 24 May 2026 14:12:16 +0000 Subject: [PATCH 1/2] fix(yandex-valuation): parse removed_date, fix total_floors regex, normalize NBSP (#526) --- .../app/services/scrapers/yandex_valuation.py | 72 +++++++---- .../backend/tests/test_yandex_valuation.py | 115 ++++++++++++++++-- 2 files changed, 156 insertions(+), 31 deletions(-) diff --git a/tradein-mvp/backend/app/services/scrapers/yandex_valuation.py b/tradein-mvp/backend/app/services/scrapers/yandex_valuation.py index 425a01f0..34ce69f5 100644 --- a/tradein-mvp/backend/app/services/scrapers/yandex_valuation.py +++ b/tradein-mvp/backend/app/services/scrapers/yandex_valuation.py @@ -52,6 +52,35 @@ class ValuationHouseMeta(BaseModel): total_objects: int | None = None # 'N объектов' (full archive count) has_panorama: bool = False # 'Панорама' label present + def validate_match( + self, + expected_year_built: int | None = None, + expected_total_floors: int | None = None, + ) -> float: + """Return confidence 0..1 that this house meta matches expected values. + + Used after fetching valuation by address to detect when Yandex returned a + different house (ambiguous address geocoding). Tolerance ±1 year, ±1 floor. + + Both expected=None → 1.0 (no check). Mismatch on any dimension → 0.0 for it. + """ + score = 0.0 + checks = 0 + if expected_year_built is not None: + checks += 1 + if self.year_built is not None and abs(self.year_built - expected_year_built) <= 1: + score += 1.0 + if expected_total_floors is not None: + checks += 1 + if ( + self.total_floors is not None + and abs(self.total_floors - expected_total_floors) <= 1 + ): + score += 1.0 + if checks == 0: + return 1.0 + return score / checks + class ValuationHistoryItem(BaseModel): """One historical offer entry from the valuation page.""" @@ -64,6 +93,7 @@ class ValuationHistoryItem(BaseModel): last_price: int | None = None last_price_per_m2: int | None = None publish_date: date | None = None + removed_date: date | None = None # ← NEW exposure_days: int | None = None status: str | None = None # 'В продаже' / 'Снято' @@ -86,7 +116,7 @@ class YandexValuationResult(BaseModel): # --------------------------------------------------------------------------- RE_YEAR_BUILT = re.compile(r"Дом\s+(\d{4})\s+года", re.IGNORECASE) -RE_FLOORS = re.compile(r"(\d+)\s+этаж", re.IGNORECASE) +RE_FLOORS = re.compile(r"(\d+)\s+этажей", re.IGNORECASE) RE_CEILING = re.compile(r"([\d,]+)\s*м\s+потолки", re.IGNORECASE) RE_TOTAL_OBJECTS = re.compile(r"(\d+)\s+объект", re.IGNORECASE) @@ -98,9 +128,7 @@ RE_ITEM_EXPOSURE = re.compile(r"экспозиции\s+(\d+)\s+дн", re.IGNOREC RE_ITEM_STATUS = re.compile(r"(В\s+продаже|Снят[оа])", re.IGNORECASE) # Matches total-price tokens (rubles) — excludes per-m2 tokens by negative lookahead -_RE_PRICE_TOKEN = re.compile( - r"(?:\d[\d\s]*\d|\d)(?:[.,]\d+)?\s*(?:млн)?\s*₽(?!\s*за\s*м²)" -) +_RE_PRICE_TOKEN = re.compile(r"(?:\d[\d\s]*\d|\d)(?:[.,]\d+)?\s*(?:млн)?\s*₽(?!\s*за\s*м²)") _RE_PPM2_TOKEN = re.compile(r"\d[\d\s]*\s*₽\s*за\s*м²") @@ -126,9 +154,7 @@ class YandexValuationScraper(BaseScraper): super().__init__() self.request_delay_sec = get_scraper_delay(self.name) - async def fetch_around( - self, lat: float, lon: float, radius_m: int = 1000 - ) -> list: # type: ignore[override] + async def fetch_around(self, lat: float, lon: float, radius_m: int = 1000) -> list: # type: ignore[override] raise NotImplementedError( "YandexValuationScraper is address-based; use fetch_house_history() instead." ) @@ -164,9 +190,7 @@ class YandexValuationScraper(BaseScraper): logger.exception("yandex valuation fetch failed: %s", url) return None if response.status_code != 200: - logger.warning( - "yandex valuation returned %d for %s", response.status_code, url - ) + logger.warning("yandex valuation returned %d for %s", response.status_code, url) return None result = self.parse( response.text, @@ -189,9 +213,10 @@ class YandexValuationScraper(BaseScraper): source_url: str, ) -> YandexValuationResult: """Parse raw HTML into YandexValuationResult. Pure function — usable in unit tests.""" - tree = HTMLParser(html) + html_normalized = html.replace("\xa0", " ") + tree = HTMLParser(html_normalized) body = tree.body - body_text = body.text(strip=True) if body else "" + body_text = (body.text(strip=True) if body else "").replace("\xa0", " ") house = self._parse_house_meta(body_text) history_items = self._parse_history_items(tree, body_text) @@ -221,17 +246,13 @@ class YandexValuationScraper(BaseScraper): year_built=int(year_m.group(1)) if year_m else None, total_floors=int(floors_m.group(1)) if floors_m else None, house_type=parse_house_type(body_text), - ceiling_height=( - float(ceiling_m.group(1).replace(",", ".")) if ceiling_m else None - ), + ceiling_height=(float(ceiling_m.group(1).replace(",", ".")) if ceiling_m else None), has_lift="Лифт" in body_text, total_objects=int(objects_m.group(1)) if objects_m else None, has_panorama="Панорама" in body_text, ) - def _parse_history_items( - self, tree: HTMLParser, body_text: str - ) -> list[ValuationHistoryItem]: + def _parse_history_items(self, tree: HTMLParser, body_text: str) -> list[ValuationHistoryItem]: """Extract list of historical offer items using best available strategy. Strategy 1: CSS data-test containers (if Yandex exposes them). @@ -239,9 +260,7 @@ class YandexValuationScraper(BaseScraper): """ items: list[ValuationHistoryItem] = [] # Strategy 1: explicit data-test container - for container in tree.css( - '[data-test*="HistoryItem"], [data-test*="ValuationItem"]' - ): + for container in tree.css('[data-test*="HistoryItem"], [data-test*="ValuationItem"]'): item = self._parse_item_text(container.text(strip=True)) if item: items.append(item) @@ -284,7 +303,15 @@ class YandexValuationScraper(BaseScraper): start_ppm2 = parse_rub(ppm2_tokens[0]) if len(ppm2_tokens) >= 1 else None last_ppm2 = parse_rub(ppm2_tokens[1]) if len(ppm2_tokens) >= 2 else None - publish_date = parse_dmy(text) + # Extract ALL DD.MM.YYYY dates: first → publish_date, second → removed_date + date_matches = list(re.finditer(r"\d{2}\.\d{2}\.\d{4}", text)) + dates_parsed: list[date] = [] + for m in date_matches: + d = parse_dmy(m.group(0)) + if d is not None: + dates_parsed.append(d) + publish_date = dates_parsed[0] if dates_parsed else None + removed_date = dates_parsed[1] if len(dates_parsed) >= 2 else None expo_m = RE_ITEM_EXPOSURE.search(text) exposure_days = int(expo_m.group(1)) if expo_m else None @@ -304,6 +331,7 @@ class YandexValuationScraper(BaseScraper): last_price=last_price, last_price_per_m2=last_ppm2, publish_date=publish_date, + removed_date=removed_date, exposure_days=exposure_days, status=status, ) diff --git a/tradein-mvp/backend/tests/test_yandex_valuation.py b/tradein-mvp/backend/tests/test_yandex_valuation.py index 2de8afd8..8d878997 100644 --- a/tradein-mvp/backend/tests/test_yandex_valuation.py +++ b/tradein-mvp/backend/tests/test_yandex_valuation.py @@ -20,9 +20,7 @@ from app.services.scrapers.yandex_valuation import ( # Fixture helpers # --------------------------------------------------------------------------- -_HOUSE_META_BLOCK = ( - "12 объектов Дом 1981 года 9 этажей Панельное здание 2,50 м потолки Лифт" -) +_HOUSE_META_BLOCK = "12 объектов Дом 1981 года 9 этажей Панельное здание 2,50 м потолки Лифт" _ITEM_1 = ( "2-комнатная квартира 45,3 м² 4 этаж " @@ -52,9 +50,7 @@ def _make_full_html(body_content: str) -> str: return f"{body_content}" -FULL_FIXTURE_HTML = _make_full_html( - f"{_HOUSE_META_BLOCK}\n{_ITEM_1}\n{_ITEM_2}\n{_ITEM_3}" -) +FULL_FIXTURE_HTML = _make_full_html(f"{_HOUSE_META_BLOCK}\n{_ITEM_1}\n{_ITEM_2}\n{_ITEM_3}") # --------------------------------------------------------------------------- @@ -155,9 +151,7 @@ def test_parse_items_chunked_dedup(): duplicate_block = f"{_ITEM_1}\n{_ITEM_1}" items = YandexValuationScraper._parse_items_from_chunked_text(duplicate_block) # Both items reference 15.03.2023 + 45.3 m2 + floor 4 — must dedup to 1 - matching = [ - i for i in items if i.area_m2 == 45.3 and i.publish_date == date(2023, 3, 15) - ] + matching = [i for i in items if i.area_m2 == 45.3 and i.publish_date == date(2023, 3, 15)] assert len(matching) == 1 @@ -240,3 +234,106 @@ def test_parse_ceiling_height_comma(): text = "Дом 1999 года 5 этажей Кирпичное здание 2,70 м потолки Лифт" meta = YandexValuationScraper._parse_house_meta(text) assert meta.ceiling_height == 2.7 + + +# --------------------------------------------------------------------------- +# Regression tests — parser fixes (2026-05-24) +# --------------------------------------------------------------------------- + + +def test_parse_item_extracts_removed_date(): + """Two dates in chunk → first = publish_date, second = removed_date.""" + text = "2-комнатная квартира 50,0 м² 3 этаж 10.05.2024 В экспозиции 30 дней 09.06.2024" + item = YandexValuationScraper._parse_item_text(text) + assert item is not None + assert item.publish_date == date(2024, 5, 10) + assert item.removed_date == date(2024, 6, 9) + + +def test_parse_item_in_sale_has_no_removed_date(): + """Single date + 'В продаже' → removed_date is None.""" + text = "1-комнатная квартира 40 м² 5 этаж 10.05.2024 В экспозиции 30 дней В продаже" + item = YandexValuationScraper._parse_item_text(text) + assert item is not None + assert item.publish_date == date(2024, 5, 10) + assert item.removed_date is None + + +def test_total_floors_extracted_from_dom_meta_not_items(): + """Regression: 'N этаж' inside each item must NOT be matched as dom total_floors. + Real Yandex page has 'M этажей' (plural) in dom-meta and 'N этаж' (singular) per item. + """ + text = ( + "Дом 2025 года Панорама 25 этажей Монолитное здание 2,7 м потолки Лифт " + "1-комнатная 40 м² 3 этаж 10.01.2026 В экспозиции 5 дней В продаже " + "2-комнатная 55 м² 17 этаж 05.01.2026 В экспозиции 10 дней В продаже" + ) + meta = YandexValuationScraper._parse_house_meta(text) + assert meta.year_built == 2025 + assert meta.total_floors == 25 + assert meta.house_type == "monolith" + + +def test_parse_normalizes_nbsp_for_price_per_m2(): + """NBSP (\\xa0) in price tokens must not break price_per_m2 regex. + Real Yandex SSR HTML uses NBSP between digit groups and before ₽. + """ + html = ( + "" + "Дом 2020 года 16 этажей Монолитное здание " + "2-комнатная 50,0 м² 3 этаж " + "10.05.2024 В экспозиции 30 дней В продаже " + "5\xa0000\xa0000 ₽ 100\xa0000 ₽ за м² " + "4\xa0900\xa0000 ₽ 98\xa0000 ₽ за м² " + "" + ) + scraper = YandexValuationScraper() + result = scraper.parse( + html, + address="test", + offer_category="APARTMENT", + offer_type="SELL", + page=1, + source_url="http://test/", + ) + assert len(result.history_items) >= 1 + item = result.history_items[0] + assert item.start_price == 5_000_000 + assert item.start_price_per_m2 == 100_000 + + +def test_validate_match_full_match(): + from app.services.scrapers.yandex_valuation import ValuationHouseMeta + + meta = ValuationHouseMeta(year_built=2025, total_floors=25) + assert meta.validate_match(2025, 25) == 1.0 + + +def test_validate_match_year_mismatch(): + from app.services.scrapers.yandex_valuation import ValuationHouseMeta + + meta = ValuationHouseMeta(year_built=1984, total_floors=9) + assert meta.validate_match(2025, 25) == 0.0 + + +def test_validate_match_partial(): + from app.services.scrapers.yandex_valuation import ValuationHouseMeta + + meta = ValuationHouseMeta(year_built=2025, total_floors=10) + # year matches, floors mismatch (>1 tolerance) → 0.5 + assert meta.validate_match(2025, 25) == 0.5 + + +def test_validate_match_tolerance_one_year(): + from app.services.scrapers.yandex_valuation import ValuationHouseMeta + + meta = ValuationHouseMeta(year_built=2024, total_floors=25) + # ±1 year tolerance + assert meta.validate_match(2025, 25) == 1.0 + + +def test_validate_match_no_expected_returns_one(): + from app.services.scrapers.yandex_valuation import ValuationHouseMeta + + meta = ValuationHouseMeta(year_built=None, total_floors=None) + assert meta.validate_match(None, None) == 1.0 From c6ab419100cc57d8550006dd9adf5f1504a0d788 Mon Sep 17 00:00:00 2001 From: lekss361 Date: Sun, 24 May 2026 14:17:32 +0000 Subject: [PATCH 2/2] feat(tradein-sql): bootstrap houses + link 98.8% of listings (Phase A+B) (#527) --- tradein-mvp/backend/app/schemas/trade_in.py | 15 + tradein-mvp/backend/app/services/estimator.py | 19 +- .../063_backfill_houses_and_link_listings.sql | 304 ++++++++++++++++++ 3 files changed, 337 insertions(+), 1 deletion(-) create mode 100644 tradein-mvp/backend/data/sql/063_backfill_houses_and_link_listings.sql diff --git a/tradein-mvp/backend/app/schemas/trade_in.py b/tradein-mvp/backend/app/schemas/trade_in.py index 7ae83e1b..2bc0993d 100644 --- a/tradein-mvp/backend/app/schemas/trade_in.py +++ b/tradein-mvp/backend/app/schemas/trade_in.py @@ -46,6 +46,20 @@ class AnalogLot(BaseModel): distance_m: int | None = None # расстояние до целевой квартиры в метрах +class CianValuationSummary(BaseModel): + """Cian Valuation Calculator данные для UI. + + Источник: external_valuations table (source='cian_valuation'). + Заполняется только при successful Cian Calculator call в estimator.py. + """ + + sale_price_rub: int | None = None + rent_price_rub: int | None = None + chart: list[dict[str, Any]] = Field(default_factory=list) + chart_change_pct: float | None = None + chart_change_direction: Literal["increase", "decrease", "neutral"] | None = None + + class AggregatedEstimate(BaseModel): estimate_id: UUID median_price_rub: int @@ -66,6 +80,7 @@ class AggregatedEstimate(BaseModel): sources_used: list[str] = Field(default_factory=list) # ['avito', 'cian', 'rosreestr'] data_freshness_minutes: int | None = None # сколько минут назад был самый свежий парсинг est_days_on_market: int | None = None # прогноз срока продажи (медиана по аналогам) + cian_valuation: CianValuationSummary | None = None # ── Параметры оценённой квартиры — нужны, чтобы восстановить карточку # при открытии оценки по ссылке (?id=), когда формы-инпута уже нет ── area_m2: float | None = None diff --git a/tradein-mvp/backend/app/services/estimator.py b/tradein-mvp/backend/app/services/estimator.py index da4c104f..dead7bd5 100644 --- a/tradein-mvp/backend/app/services/estimator.py +++ b/tradein-mvp/backend/app/services/estimator.py @@ -31,7 +31,12 @@ from uuid import uuid4 from sqlalchemy import text from sqlalchemy.orm import Session -from app.schemas.trade_in import AggregatedEstimate, AnalogLot, TradeInEstimateInput +from app.schemas.trade_in import ( + AggregatedEstimate, + AnalogLot, + CianValuationSummary, + TradeInEstimateInput, +) from app.services.geocoder import GeocodeResult, geocode from app.services.house_metadata import get_house_metadata from app.services.scrapers.avito_imv import ( @@ -865,6 +870,17 @@ async def estimate_quality( sources_used=sources_used, data_freshness_minutes=freshness_min, est_days_on_market=_estimate_days_on_market(listings_clean, deals), + cian_valuation=( + CianValuationSummary( + sale_price_rub=int(cian_val.sale_price_rub) if cian_val.sale_price_rub else None, + rent_price_rub=int(cian_val.rent_price_rub) if cian_val.rent_price_rub else None, + chart=list(cian_val.chart or []), + chart_change_pct=cian_val.chart_change_pct, + chart_change_direction=cian_val.chart_change_direction, + ) + if cian_val is not None + else None + ), area_m2=payload.area_m2, rooms=payload.rooms, floor=payload.floor, @@ -1598,4 +1614,5 @@ def _empty_estimate( analogs=[], actual_deals=[], expires_at=expires_at, + cian_valuation=None, ) diff --git a/tradein-mvp/backend/data/sql/063_backfill_houses_and_link_listings.sql b/tradein-mvp/backend/data/sql/063_backfill_houses_and_link_listings.sql new file mode 100644 index 00000000..b56f47fb --- /dev/null +++ b/tradein-mvp/backend/data/sql/063_backfill_houses_and_link_listings.sql @@ -0,0 +1,304 @@ +-- 063_backfill_houses_and_link_listings.sql +-- Bootstrap houses table from existing listings data (Phase A+B). +-- +-- Context: +-- listings: 18,256 rows (all have addresses) +-- houses: 0 rows (empty) +-- listings.house_id_fk: 0 linked +-- house_placement_history: 160 orphans, all source=yandex_valuation, raw_payload has no address +-- +-- Steps: +-- 1. CREATE OR REPLACE FUNCTION tradein_normalize_short_addr +-- 2. INSERT Avito/CIAN-sourced houses from listings WHERE house_source+house_ext_id NOT NULL +-- 3. INSERT derived houses for listings WITHOUT ext_house_id (grouped by normalized address) +-- DEDUP: skips addresses already covered by step 2 (prevents split house rows) +-- 4. UPDATE listings.house_id_fk via source+ext_house_id match (direct path) +-- 5. UPDATE listings.house_id_fk via normalized address match (fuzzy path) +-- DEDUP: COALESCE prefers avito-sourced house over derived at same address +-- 6. Best-effort relink house_placement_history via raw_payload->>'address' (likely 0 rows) +-- 7. Dedup sanity check: WARN if any normalized address maps to >1 houses rows +-- 8. RAISE NOTICE with final counters +-- +-- Dependencies: +-- - pgcrypto extension (for digest()) -- verified present +-- - listings table, houses table, house_placement_history table +-- - houses UNIQUE constraint on (source, ext_house_id) +-- - house_placement_history UNIQUE constraint on (source, ext_item_id) +-- +-- Idempotency: +-- - INSERT ... ON CONFLICT DO NOTHING +-- - UPDATE ... WHERE house_id_fk IS NULL +-- - CREATE OR REPLACE FUNCTION +-- +-- Deploy order: after 062_clean_avito_addresses.sql + +BEGIN; + +-- ------------------------------------------------------------------------- +-- Step 1: Address normalizer function +-- Strips regional/city prefix and apartment/corpus suffix. +-- Handles addresses like: +-- "ул. Репина, 75/2 стр." → "ул. Репина, 75/2 стр." +-- "Свердловская обл., Екатеринбург, ул. Большакова, 17" → "ул. Большакова, 17" +-- "улица Яскина, 12 · р-н Октябрьский" → "улица Яскина, 12" +-- "Азина, 4.1.1" → "Азина, 4.1.1" +-- ------------------------------------------------------------------------- +CREATE OR REPLACE FUNCTION tradein_normalize_short_addr(addr text) +RETURNS text +LANGUAGE sql +IMMUTABLE +PARALLEL SAFE +AS $$ + SELECT trim(both ' ,.' FROM + regexp_replace( + regexp_replace( + regexp_replace( + regexp_replace( + regexp_replace( + -- Strip leading "Россия / РФ / Российская Федерация, " + regexp_replace(addr, '^\s*(?:Россия|РФ|Российская\s+Федерация)\s*,\s*', '', 'i'), + -- Strip leading region "Свердловская обл., " / "Свердловская область, " + '^[А-ЯЁа-яё][А-ЯЁа-яё\s-]+\s+обл(?:асть|\.)\s*,\s*', '', 'i' + ), + -- Strip leading city "Екатеринбург, " / "г. Екатеринбург, " / "г Екатеринбург, " + '^г\.?\s*[А-ЯЁ][А-ЯЁа-яё-]+\s*,\s*', '', 'i' + ), + -- Strip district suffix " · р-н ..." or " · district" + '\s*·\s*.+$', '', 'i' + ), + -- Strip apartment/corpus/office suffix ", кв./корп./оф./пом./подъезд N..." + ',\s*(?:кв\.?|корп\.?|к\.?|оф\.?|пом\.?|подъезд)\s*\d+.*$', '', 'i' + ), + -- Strip trailing whitespace artefacts + '\s{2,}', ' ', 'g' + ) + ); +$$; + +-- ------------------------------------------------------------------------- +-- Step 2: Insert Avito/CIAN-sourced houses from listings +-- Uses DISTINCT ON to pick the most recently scraped data per (source, ext_house_id). +-- ------------------------------------------------------------------------- +INSERT INTO houses ( + source, + ext_house_id, + url, + address, + lat, + lon, + geom, + year_built, + house_type, + total_floors, + full_address, + first_seen_at, + last_scraped_at +) +SELECT DISTINCT ON (l.house_source, l.house_ext_id) + l.house_source AS source, + l.house_ext_id AS ext_house_id, + COALESCE(l.house_url, '') AS url, + l.address, + l.lat, + l.lon, + l.geom, + l.year_built, + l.house_type, + l.total_floors, + l.address AS full_address, + NOW() AS first_seen_at, + NOW() AS last_scraped_at +FROM listings l +WHERE l.house_source IS NOT NULL + AND l.house_ext_id IS NOT NULL + AND l.address IS NOT NULL +ORDER BY l.house_source, l.house_ext_id, l.scraped_at DESC +ON CONFLICT (source, ext_house_id) DO NOTHING; + +-- ------------------------------------------------------------------------- +-- Step 3: Insert derived houses for listings WITHOUT ext_house_id +-- Groups listings by normalized address; each group = one derived house. +-- Synthetic ext_house_id = first 32 hex chars of sha256(normalized_addr). +-- Uses DISTINCT ON (norm_addr) picking latest scraped_at per group. +-- +-- DEDUP FIX: Excludes normalized addresses already covered by Step 2's +-- avito/cian-sourced houses. Prevents the same physical building getting +-- TWO rows (one avito-sourced, one derived) when it has listings from +-- both Avito (with ext_house_id) and other sources (without ext_house_id). +-- ------------------------------------------------------------------------- +INSERT INTO houses ( + source, + ext_house_id, + url, + address, + lat, + lon, + geom, + year_built, + house_type, + total_floors, + full_address, + short_address, + first_seen_at, + last_scraped_at +) +SELECT DISTINCT ON (norm_addr) + 'derived' AS source, + substring(encode(digest(norm_addr, 'sha256'), 'hex'), 1, 32) AS ext_house_id, + '' AS url, + src.address, + src.lat, + src.lon, + src.geom, + src.year_built, + src.house_type, + src.total_floors, + src.address AS full_address, + norm_addr AS short_address, + NOW() AS first_seen_at, + NOW() AS last_scraped_at +FROM ( + SELECT + l.*, + tradein_normalize_short_addr(l.address) AS norm_addr + FROM listings l + WHERE l.address IS NOT NULL + AND (l.house_source IS NULL OR l.house_ext_id IS NULL) +) src +WHERE src.norm_addr IS NOT NULL + AND length(src.norm_addr) >= 5 + AND src.norm_addr NOT IN ( + SELECT tradein_normalize_short_addr(address) + FROM houses + WHERE source != 'derived' AND address IS NOT NULL + ) +ORDER BY norm_addr, src.scraped_at DESC +ON CONFLICT (source, ext_house_id) DO NOTHING; + +-- ------------------------------------------------------------------------- +-- Step 4: Link listings.house_id_fk via direct source+ext_house_id match +-- Uses listings_house_ext_id_idx (source, house_ext_id) — sargable. +-- ------------------------------------------------------------------------- +UPDATE listings l +SET house_id_fk = h.id +FROM houses h +WHERE l.house_id_fk IS NULL + AND l.house_source IS NOT NULL + AND l.house_ext_id IS NOT NULL + AND l.house_source = h.source + AND l.house_ext_id = h.ext_house_id; + +-- ------------------------------------------------------------------------- +-- Step 5: Link listings.house_id_fk via normalized address (fuzzy path) +-- Covers listings where house_source/house_ext_id is absent. +-- houses.short_address was populated above (derived rows); joined by equality. +-- +-- DEDUP FIX: For cian/yandex listings at an address that already has an +-- avito-sourced house (from Step 2), we prefer that avito-sourced house +-- over any derived row. This consolidates all listings for the same physical +-- building under a single houses row regardless of scrape source. +-- +-- COALESCE priority: +-- 1. Non-derived house whose address normalizes to the same value +-- 2. Derived house whose short_address equals the normalized listing address +-- +-- NOTE: houses_addr_fp_idx is on address_fingerprint; short_address has no +-- dedicated index. If this migration is re-run on large data, consider: +-- CREATE INDEX IF NOT EXISTS houses_short_addr_idx ON houses(short_address) +-- WHERE source = 'derived'; +-- For a one-shot migration on 18k rows this is acceptable (seq scan ~ms). +-- ------------------------------------------------------------------------- +WITH norm_l AS ( + SELECT id, tradein_normalize_short_addr(address) AS norm_addr + FROM listings + WHERE house_id_fk IS NULL AND address IS NOT NULL +) +UPDATE listings l +SET house_id_fk = COALESCE( + -- Prefer avito/cian-sourced house at same normalized address (cross-source consolidation) + (SELECT h.id FROM houses h + WHERE h.source != 'derived' + AND tradein_normalize_short_addr(h.address) = nl.norm_addr + LIMIT 1), + -- Fall back to derived house + (SELECT h.id FROM houses h + WHERE h.source = 'derived' + AND h.short_address = nl.norm_addr + LIMIT 1) +) +FROM norm_l nl +WHERE l.id = nl.id AND l.house_id_fk IS NULL; + +-- ------------------------------------------------------------------------- +-- Step 6: Best-effort relink orphan house_placement_history +-- All 160 orphan rows are source=yandex_valuation; raw_payload has no +-- 'address' key (verified: has_addr=false for all 160 rows). +-- This UPDATE will match 0 rows but is included for completeness/idempotency. +-- Phase C (Celery IMV scraper) will assign house_id on new inserts directly. +-- ------------------------------------------------------------------------- +UPDATE house_placement_history hph +SET house_id = h.id +FROM houses h +WHERE hph.house_id IS NULL + AND (hph.raw_payload ->> 'address') IS NOT NULL + AND h.source = 'derived' + AND h.short_address = tradein_normalize_short_addr(hph.raw_payload ->> 'address'); + +-- ------------------------------------------------------------------------- +-- Step 7: Dedup sanity check +-- Warn if any normalized address maps to more than one houses row. +-- A non-zero count after the dedup fix above means manual review is needed. +-- ------------------------------------------------------------------------- +DO $$ +DECLARE + dup_count int; +BEGIN + SELECT count(*) INTO dup_count FROM ( + SELECT tradein_normalize_short_addr(address) AS norm, count(*) AS n + FROM houses + WHERE address IS NOT NULL + GROUP BY tradein_normalize_short_addr(address) + HAVING count(*) > 1 + ) dup; + IF dup_count > 0 THEN + RAISE WARNING 'backfill 063: % normalized addresses still have multiple houses rows', dup_count; + ELSE + RAISE NOTICE 'backfill 063: dedup OK — 1 house per normalized address'; + END IF; +END $$; + +-- ------------------------------------------------------------------------- +-- Step 8: Final counters via RAISE NOTICE +-- ------------------------------------------------------------------------- +DO $$ +DECLARE + v_houses_total bigint; + v_houses_derived bigint; + v_houses_direct bigint; + v_listings_total bigint; + v_listings_linked bigint; + v_listings_addr bigint; + v_history_total bigint; + v_history_linked bigint; +BEGIN + SELECT count(*) INTO v_houses_total FROM houses; + SELECT count(*) FILTER (WHERE source = 'derived') INTO v_houses_derived FROM houses; + SELECT count(*) FILTER (WHERE source != 'derived') INTO v_houses_direct FROM houses; + + SELECT count(*) INTO v_listings_total FROM listings; + SELECT count(*) FILTER (WHERE house_id_fk IS NOT NULL) INTO v_listings_linked FROM listings; + SELECT count(*) FILTER (WHERE address IS NOT NULL) INTO v_listings_addr FROM listings; + + SELECT count(*) INTO v_history_total FROM house_placement_history; + SELECT count(*) FILTER (WHERE house_id IS NOT NULL) INTO v_history_linked FROM house_placement_history; + + RAISE NOTICE + 'backfill 063 final: houses=% (direct=%, derived=%)' + ' | listings_linked=%/% (addr_coverage=%)' + ' | history_linked=%/%', + v_houses_total, v_houses_direct, v_houses_derived, + v_listings_linked, v_listings_total, v_listings_addr, + v_history_linked, v_history_total; +END $$; + +COMMIT;