From fb5004d5dc64ca445e74f33cb7648330b77c2250 Mon Sep 17 00:00:00 2001 From: bot-backend Date: Sat, 4 Jul 2026 17:47:44 +0300 Subject: [PATCH] =?UTF-8?q?feat(tradein/scraper-kit):=20property=5Ftype/la?= =?UTF-8?q?nd=5Farea=5Fm2=20=D0=B2=20listings=20+=20skip=20house-match=20?= =?UTF-8?q?=D0=B4=D0=BB=D1=8F=20non-flat=20(DomClick=20house=20SKU,=20PR1/?= =?UTF-8?q?2)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../data/sql/174_listings_property_type.sql | 25 ++++ .../backend/data/sql/_manifest_applied.txt | 1 + .../tests/test_scraped_at_bump_2206.py | 117 ++++++++++++++++++ .../scraper-kit/src/scraper_kit/base.py | 31 ++++- 4 files changed, 173 insertions(+), 1 deletion(-) create mode 100644 tradein-mvp/backend/data/sql/174_listings_property_type.sql diff --git a/tradein-mvp/backend/data/sql/174_listings_property_type.sql b/tradein-mvp/backend/data/sql/174_listings_property_type.sql new file mode 100644 index 00000000..7ef24af6 --- /dev/null +++ b/tradein-mvp/backend/data/sql/174_listings_property_type.sql @@ -0,0 +1,25 @@ +-- Migration: listings.property_type / listings.land_area_m2 — DomClick house SKU +-- foundation (PR1/2). `listings` was implicitly flat-only (002_core_tables.sql has +-- no property-type discriminator); DomClick's "частный дом" (private house for sale) +-- category (bff-search-web.domclick.ru?offer_type=house) needs a way to tag a row as +-- a house and carry land plot area. No CHECK constraint — matches project convention +-- (house_type/repair_state/listing_segment are soft-typed text, validated at Pydantic +-- ScrapedLot level, not DB level). Additive-only, PG16 fast-default: no table rewrite +-- even on a large table. +-- +-- Scope note: this migration only adds the columns. The `houses`/`house_sources` +-- tables (009_houses.sql, 010_houses_alter.sql) remain a DIFFERENT entity — an +-- apartment-BUILDING profile cache (total_units, passenger_elevators, material_walls, +-- has_concierge). A standalone house is NOT a building and must never be matched/ +-- inserted there — that short-circuit lives in scraper_kit/base.py, not SQL. +BEGIN; + +ALTER TABLE listings ADD COLUMN IF NOT EXISTS property_type text NOT NULL DEFAULT 'flat'; +ALTER TABLE listings ADD COLUMN IF NOT EXISTS land_area_m2 numeric(10, 2); + +COMMENT ON COLUMN listings.property_type IS + 'flat (default, all existing sources) / house (DomClick "частный дом" SKU). Soft-typed text, no CHECK — validated at ScrapedLot level.'; +COMMENT ON COLUMN listings.land_area_m2 IS + 'Площадь земельного участка, соток→м2. Только для property_type=''house'' (DomClick house SKU); NULL для квартир.'; + +COMMIT; diff --git a/tradein-mvp/backend/data/sql/_manifest_applied.txt b/tradein-mvp/backend/data/sql/_manifest_applied.txt index 0d8889dc..07b403f3 100644 --- a/tradein-mvp/backend/data/sql/_manifest_applied.txt +++ b/tradein-mvp/backend/data/sql/_manifest_applied.txt @@ -176,3 +176,4 @@ 169_osm_poi_ekb_local.sql 170_scrape_schedules_seed_osm_poi_ekb_refresh.sql 172_trade_in_leads.sql +174_listings_property_type.sql diff --git a/tradein-mvp/backend/tests/test_scraped_at_bump_2206.py b/tradein-mvp/backend/tests/test_scraped_at_bump_2206.py index f5e3b247..cf815f68 100644 --- a/tradein-mvp/backend/tests/test_scraped_at_bump_2206.py +++ b/tradein-mvp/backend/tests/test_scraped_at_bump_2206.py @@ -23,6 +23,7 @@ from typing import Any from unittest.mock import MagicMock, patch import psycopg.errors +import pytest from sqlalchemy.exc import IntegrityError os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db") @@ -156,6 +157,122 @@ def test_kit_reconcile_bumps_scraped_at() -> None: assert "last_seen_at = NOW()" in sql +# ── DomClick house SKU (PR1/2): property_type/land_area_m2 + house-match short-circuit +# on a real Postgres. Real DB needed (not a MagicMock) because the assertion is about +# actual row state (house_id_fk IS NULL) and a genuine listing_sources row — a mock +# can't prove save_listings' SAVEPOINT/hook wiring actually persists both. ──────────── + + +def _live_session() -> Any | None: + """Return a SQLAlchemy Session if a non-placeholder Postgres is reachable, else None. + + Same self-skipping pattern as test_house_dedup_merge.py's `_live_session()`. + """ + try: + from sqlalchemy import create_engine + from sqlalchemy.orm import sessionmaker + + dsn = os.environ.get("TEST_DATABASE_URL") or os.environ.get("DATABASE_URL", "") + if not dsn or "localhost:5432/test" in dsn: + return None + engine = create_engine(dsn, future=True) + conn = engine.connect() + from sqlalchemy import text as _t + + conn.execute(_t("SELECT 1")) + conn.close() + return sessionmaker(bind=engine, future=True)() + except Exception: + return None + + +class _HouseShortCircuitMatcher: + """Matcher stub for the property_type='house' short-circuit test. + + match_or_create_house() MUST NOT be called for a house lot — asserting that is + the whole point of the test, so this raises rather than silently no-op'ing. + upsert_listing_source() delegates to the real `app.services.matching` function + against the live DB, so the test can assert a genuine listing_sources row exists + (independent of house linkage, per the analyst plan). + """ + + def match_or_create_house(self, *args: Any, **kwargs: Any) -> Any: + raise AssertionError( + "match_or_create_house MUST NOT be called for property_type='house' " + "(houses table is an apartment-BUILDING cache, not a standalone house)" + ) + + def upsert_listing_source(self, db: Any, **kwargs: Any) -> None: + from app.services.matching import upsert_listing_source as _real_upsert + + _real_upsert(db, **kwargs) + + +@pytest.mark.skipif(_live_session() is None, reason="no reachable Postgres test DB") +def test_kit_house_property_type_skips_house_match_real_db() -> None: + """property_type='house' persists land_area_m2, leaves house_id_fk NULL, and + still registers the listing in listing_sources — WITHOUT ever calling + match_or_create_house() (DomClick house SKU, PR1/2).""" + from sqlalchemy import text as _t + + db = _live_session() + assert db is not None + try: + lot = KitLot( + source="domclick", + source_url="https://ekaterinburg.domclick.ru/card/sale__house__TEST-HOUSE-2206", + source_id="TEST-HOUSE-2206", + address="тестовый дом, ул. Домиковая, 1", + lat=56.84, + lon=60.60, + area_m2=180.0, + price_rub=12_000_000, + property_type="house", + land_area_m2=650.0, + ) + + with patch("scraper_kit.base.upsert_listing_snapshot", return_value=None): + inserted, updated = kit_save_listings( + db, [lot], matcher=_HouseShortCircuitMatcher(), region_code=66 + ) + assert (inserted, updated) == (1, 0) + + row = db.execute( + _t( + "SELECT property_type, land_area_m2, house_id_fk FROM listings " + "WHERE source = 'domclick' AND source_id = 'TEST-HOUSE-2206'" + ) + ).fetchone() + assert row is not None + assert row.property_type == "house" + assert float(row.land_area_m2) == 650.0 + assert row.house_id_fk is None + + ls_row = db.execute( + _t( + "SELECT listing_id FROM listing_sources " + "WHERE ext_source = 'domclick' AND ext_id = 'TEST-HOUSE-2206'" + ) + ).fetchone() + assert ls_row is not None + finally: + db.rollback() + db.execute( + _t( + "DELETE FROM listing_sources WHERE ext_source = 'domclick' " + "AND ext_id = 'TEST-HOUSE-2206'" + ) + ) + db.execute( + _t( + "DELETE FROM listings WHERE source = 'domclick' " + "AND source_id = 'TEST-HOUSE-2206'" + ) + ) + db.commit() + db.close() + + # ── Migration 161: retro-backfill scraped_at ────────────────────────────────── _SQL_DIR = Path(__file__).resolve().parents[1] / "data" / "sql" diff --git a/tradein-mvp/packages/scraper-kit/src/scraper_kit/base.py b/tradein-mvp/packages/scraper-kit/src/scraper_kit/base.py index ec155c5d..70d22834 100644 --- a/tradein-mvp/packages/scraper-kit/src/scraper_kit/base.py +++ b/tradein-mvp/packages/scraper-kit/src/scraper_kit/base.py @@ -97,6 +97,17 @@ class ScrapedLot(BaseModel): has_balcony: bool | None = None kadastr_num: str | None = None + # Property-type discriminator (DomClick house SKU, PR1/2). Default 'flat' — + # every existing source (avito/cian/yandex) is implicitly flat-only and never + # sets this, so their behavior is unchanged. 'house' — DomClick's "частный дом" + # (standalone private house for sale, offer_type=house). NOT a CHECK-constrained + # enum (matches house_type/repair_state convention) — validated at this Pydantic + # level only. See scraper_kit.base._link_listing_to_house: property_type='house' + # short-circuits match_or_create_house (houses table is an apartment-BUILDING + # cache, a standalone house is not a building). + property_type: str = "flat" + land_area_m2: float | None = None # площадь участка — только для property_type='house' + # Detail-поля кухни/потолка (#2007). Обычно из detail-страницы (avito_detail, # cian_detail, yandex_detail), но yandex SERP отдаёт их прямо в карточке — # промоутим из raw_payload в колонки для дашборда покрытия и matching. @@ -370,6 +381,8 @@ def save_listings( "house_type": lot.house_type, "repair_state": lot.repair_state, "has_balcony": lot.has_balcony, + "property_type": lot.property_type, + "land_area_m2": lot.land_area_m2, "kitchen_area_m2": lot.kitchen_area_m2, "ceiling_height_m": lot.ceiling_height_m, "mortgage_available": lot.mortgage_available, @@ -421,6 +434,7 @@ def save_listings( address, lat, lon, region_code, rooms, area_m2, floor, total_floors, year_built, house_type, repair_state, has_balcony, + property_type, land_area_m2, kitchen_area_m2, ceiling_height, ceiling_height_m, mortgage_available, is_apartments, is_rosreestr_checked, house_source, house_ext_id, house_url, listing_segment, @@ -442,6 +456,7 @@ def save_listings( :address, :lat, :lon, :region_code, :rooms, :area_m2, :floor, :total_floors, :year_built, :house_type, :repair_state, :has_balcony, + :property_type, :land_area_m2, -- ceiling: один param :ceiling_height_m пишем в ОБЕ колонки — -- ceiling_height (019, читает coverage-дашборд + yandex_detail/cian_detail) -- и ceiling_height_m (111, живая avito-колонка). См. #2007. @@ -783,6 +798,8 @@ def _link_listing_to_house( Skips silently if: - lot has no source_id AND no address/lat/lon (cannot match house anyway) + - lot.property_type == 'house' (standalone house, NOT an apartment-building — + see house resolution short-circuit below; listing_sources upsert still runs) Raises on DB errors — caller wraps in try/except + SAVEPOINT. """ @@ -790,8 +807,20 @@ def _link_listing_to_house( # House resolution: needs at least address or (lat, lon). Skip otherwise — # listing_sources requires listing_id but not house linkage, so still upsert. + # + # property_type == 'house' short-circuit (DomClick house SKU, PR1/2): `houses`/ + # `house_sources` (009_houses.sql, 010_houses_alter.sql) are an apartment-BUILDING + # profile cache (total_units, passenger_elevators, material_walls, has_concierge) + # — attributes of a multi-unit building, not a standalone private house. Matching + # or creating a `houses` row for a house listing would corrupt that cache with a + # non-building entity. house_id stays None (listings.house_id_fk NULL); the + # ext_id→listing_id registration below still runs (independent of house linkage). + # Cross-source house dedup for standalone houses is out of scope for V1 (only + # DomClick scrapes this type). house_id: int | None = None - if lot.address or (lot.lat is not None and lot.lon is not None): + if lot.property_type == "house": + pass + elif lot.address or (lot.lat is not None and lot.lon is not None): # Use house_source/house_ext_id when scraper extracted them (Avito Houses # catalog link, Cian newbuilding). Falls back to per-listing identity # so each unique listing creates its own row if no canonical house exists.