feat(tradein/scraper-kit): property_type/land_area_m2 в listings + skip house-match для non-flat (DomClick house SKU, PR1/2)
This commit is contained in:
bot-backend
parent
b102c334d8
commit
fb5004d5dc
4 changed files with 173 additions and 1 deletions
25
tradein-mvp/backend/data/sql/174_listings_property_type.sql
Normal file
25
tradein-mvp/backend/data/sql/174_listings_property_type.sql
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
-- Migration: listings.property_type / listings.land_area_m2 — DomClick house SKU
|
||||
-- foundation (PR1/2). `listings` was implicitly flat-only (002_core_tables.sql has
|
||||
-- no property-type discriminator); DomClick's "частный дом" (private house for sale)
|
||||
-- category (bff-search-web.domclick.ru?offer_type=house) needs a way to tag a row as
|
||||
-- a house and carry land plot area. No CHECK constraint — matches project convention
|
||||
-- (house_type/repair_state/listing_segment are soft-typed text, validated at Pydantic
|
||||
-- ScrapedLot level, not DB level). Additive-only, PG16 fast-default: no table rewrite
|
||||
-- even on a large table.
|
||||
--
|
||||
-- Scope note: this migration only adds the columns. The `houses`/`house_sources`
|
||||
-- tables (009_houses.sql, 010_houses_alter.sql) remain a DIFFERENT entity — an
|
||||
-- apartment-BUILDING profile cache (total_units, passenger_elevators, material_walls,
|
||||
-- has_concierge). A standalone house is NOT a building and must never be matched/
|
||||
-- inserted there — that short-circuit lives in scraper_kit/base.py, not SQL.
|
||||
BEGIN;
|
||||
|
||||
ALTER TABLE listings ADD COLUMN IF NOT EXISTS property_type text NOT NULL DEFAULT 'flat';
|
||||
ALTER TABLE listings ADD COLUMN IF NOT EXISTS land_area_m2 numeric(10, 2);
|
||||
|
||||
COMMENT ON COLUMN listings.property_type IS
|
||||
'flat (default, all existing sources) / house (DomClick "частный дом" SKU). Soft-typed text, no CHECK — validated at ScrapedLot level.';
|
||||
COMMENT ON COLUMN listings.land_area_m2 IS
|
||||
'Площадь земельного участка, соток→м2. Только для property_type=''house'' (DomClick house SKU); NULL для квартир.';
|
||||
|
||||
COMMIT;
|
||||
|
|
@ -176,3 +176,4 @@
|
|||
169_osm_poi_ekb_local.sql
|
||||
170_scrape_schedules_seed_osm_poi_ekb_refresh.sql
|
||||
172_trade_in_leads.sql
|
||||
174_listings_property_type.sql
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ from typing import Any
|
|||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import psycopg.errors
|
||||
import pytest
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
|
||||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
|
||||
|
|
@ -156,6 +157,122 @@ def test_kit_reconcile_bumps_scraped_at() -> None:
|
|||
assert "last_seen_at = NOW()" in sql
|
||||
|
||||
|
||||
# ── DomClick house SKU (PR1/2): property_type/land_area_m2 + house-match short-circuit
|
||||
# on a real Postgres. Real DB needed (not a MagicMock) because the assertion is about
|
||||
# actual row state (house_id_fk IS NULL) and a genuine listing_sources row — a mock
|
||||
# can't prove save_listings' SAVEPOINT/hook wiring actually persists both. ────────────
|
||||
|
||||
|
||||
def _live_session() -> Any | None:
|
||||
"""Return a SQLAlchemy Session if a non-placeholder Postgres is reachable, else None.
|
||||
|
||||
Same self-skipping pattern as test_house_dedup_merge.py's `_live_session()`.
|
||||
"""
|
||||
try:
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
dsn = os.environ.get("TEST_DATABASE_URL") or os.environ.get("DATABASE_URL", "")
|
||||
if not dsn or "localhost:5432/test" in dsn:
|
||||
return None
|
||||
engine = create_engine(dsn, future=True)
|
||||
conn = engine.connect()
|
||||
from sqlalchemy import text as _t
|
||||
|
||||
conn.execute(_t("SELECT 1"))
|
||||
conn.close()
|
||||
return sessionmaker(bind=engine, future=True)()
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
class _HouseShortCircuitMatcher:
|
||||
"""Matcher stub for the property_type='house' short-circuit test.
|
||||
|
||||
match_or_create_house() MUST NOT be called for a house lot — asserting that is
|
||||
the whole point of the test, so this raises rather than silently no-op'ing.
|
||||
upsert_listing_source() delegates to the real `app.services.matching` function
|
||||
against the live DB, so the test can assert a genuine listing_sources row exists
|
||||
(independent of house linkage, per the analyst plan).
|
||||
"""
|
||||
|
||||
def match_or_create_house(self, *args: Any, **kwargs: Any) -> Any:
|
||||
raise AssertionError(
|
||||
"match_or_create_house MUST NOT be called for property_type='house' "
|
||||
"(houses table is an apartment-BUILDING cache, not a standalone house)"
|
||||
)
|
||||
|
||||
def upsert_listing_source(self, db: Any, **kwargs: Any) -> None:
|
||||
from app.services.matching import upsert_listing_source as _real_upsert
|
||||
|
||||
_real_upsert(db, **kwargs)
|
||||
|
||||
|
||||
@pytest.mark.skipif(_live_session() is None, reason="no reachable Postgres test DB")
|
||||
def test_kit_house_property_type_skips_house_match_real_db() -> None:
|
||||
"""property_type='house' persists land_area_m2, leaves house_id_fk NULL, and
|
||||
still registers the listing in listing_sources — WITHOUT ever calling
|
||||
match_or_create_house() (DomClick house SKU, PR1/2)."""
|
||||
from sqlalchemy import text as _t
|
||||
|
||||
db = _live_session()
|
||||
assert db is not None
|
||||
try:
|
||||
lot = KitLot(
|
||||
source="domclick",
|
||||
source_url="https://ekaterinburg.domclick.ru/card/sale__house__TEST-HOUSE-2206",
|
||||
source_id="TEST-HOUSE-2206",
|
||||
address="тестовый дом, ул. Домиковая, 1",
|
||||
lat=56.84,
|
||||
lon=60.60,
|
||||
area_m2=180.0,
|
||||
price_rub=12_000_000,
|
||||
property_type="house",
|
||||
land_area_m2=650.0,
|
||||
)
|
||||
|
||||
with patch("scraper_kit.base.upsert_listing_snapshot", return_value=None):
|
||||
inserted, updated = kit_save_listings(
|
||||
db, [lot], matcher=_HouseShortCircuitMatcher(), region_code=66
|
||||
)
|
||||
assert (inserted, updated) == (1, 0)
|
||||
|
||||
row = db.execute(
|
||||
_t(
|
||||
"SELECT property_type, land_area_m2, house_id_fk FROM listings "
|
||||
"WHERE source = 'domclick' AND source_id = 'TEST-HOUSE-2206'"
|
||||
)
|
||||
).fetchone()
|
||||
assert row is not None
|
||||
assert row.property_type == "house"
|
||||
assert float(row.land_area_m2) == 650.0
|
||||
assert row.house_id_fk is None
|
||||
|
||||
ls_row = db.execute(
|
||||
_t(
|
||||
"SELECT listing_id FROM listing_sources "
|
||||
"WHERE ext_source = 'domclick' AND ext_id = 'TEST-HOUSE-2206'"
|
||||
)
|
||||
).fetchone()
|
||||
assert ls_row is not None
|
||||
finally:
|
||||
db.rollback()
|
||||
db.execute(
|
||||
_t(
|
||||
"DELETE FROM listing_sources WHERE ext_source = 'domclick' "
|
||||
"AND ext_id = 'TEST-HOUSE-2206'"
|
||||
)
|
||||
)
|
||||
db.execute(
|
||||
_t(
|
||||
"DELETE FROM listings WHERE source = 'domclick' "
|
||||
"AND source_id = 'TEST-HOUSE-2206'"
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
db.close()
|
||||
|
||||
|
||||
# ── Migration 161: retro-backfill scraped_at ──────────────────────────────────
|
||||
|
||||
_SQL_DIR = Path(__file__).resolve().parents[1] / "data" / "sql"
|
||||
|
|
|
|||
|
|
@ -97,6 +97,17 @@ class ScrapedLot(BaseModel):
|
|||
has_balcony: bool | None = None
|
||||
kadastr_num: str | None = None
|
||||
|
||||
# Property-type discriminator (DomClick house SKU, PR1/2). Default 'flat' —
|
||||
# every existing source (avito/cian/yandex) is implicitly flat-only and never
|
||||
# sets this, so their behavior is unchanged. 'house' — DomClick's "частный дом"
|
||||
# (standalone private house for sale, offer_type=house). NOT a CHECK-constrained
|
||||
# enum (matches house_type/repair_state convention) — validated at this Pydantic
|
||||
# level only. See scraper_kit.base._link_listing_to_house: property_type='house'
|
||||
# short-circuits match_or_create_house (houses table is an apartment-BUILDING
|
||||
# cache, a standalone house is not a building).
|
||||
property_type: str = "flat"
|
||||
land_area_m2: float | None = None # площадь участка — только для property_type='house'
|
||||
|
||||
# Detail-поля кухни/потолка (#2007). Обычно из detail-страницы (avito_detail,
|
||||
# cian_detail, yandex_detail), но yandex SERP отдаёт их прямо в карточке —
|
||||
# промоутим из raw_payload в колонки для дашборда покрытия и matching.
|
||||
|
|
@ -370,6 +381,8 @@ def save_listings(
|
|||
"house_type": lot.house_type,
|
||||
"repair_state": lot.repair_state,
|
||||
"has_balcony": lot.has_balcony,
|
||||
"property_type": lot.property_type,
|
||||
"land_area_m2": lot.land_area_m2,
|
||||
"kitchen_area_m2": lot.kitchen_area_m2,
|
||||
"ceiling_height_m": lot.ceiling_height_m,
|
||||
"mortgage_available": lot.mortgage_available,
|
||||
|
|
@ -421,6 +434,7 @@ def save_listings(
|
|||
address, lat, lon, region_code,
|
||||
rooms, area_m2, floor, total_floors, year_built,
|
||||
house_type, repair_state, has_balcony,
|
||||
property_type, land_area_m2,
|
||||
kitchen_area_m2, ceiling_height, ceiling_height_m,
|
||||
mortgage_available, is_apartments, is_rosreestr_checked,
|
||||
house_source, house_ext_id, house_url, listing_segment,
|
||||
|
|
@ -442,6 +456,7 @@ def save_listings(
|
|||
:address, :lat, :lon, :region_code,
|
||||
:rooms, :area_m2, :floor, :total_floors, :year_built,
|
||||
:house_type, :repair_state, :has_balcony,
|
||||
:property_type, :land_area_m2,
|
||||
-- ceiling: один param :ceiling_height_m пишем в ОБЕ колонки —
|
||||
-- ceiling_height (019, читает coverage-дашборд + yandex_detail/cian_detail)
|
||||
-- и ceiling_height_m (111, живая avito-колонка). См. #2007.
|
||||
|
|
@ -783,6 +798,8 @@ def _link_listing_to_house(
|
|||
|
||||
Skips silently if:
|
||||
- lot has no source_id AND no address/lat/lon (cannot match house anyway)
|
||||
- lot.property_type == 'house' (standalone house, NOT an apartment-building —
|
||||
see house resolution short-circuit below; listing_sources upsert still runs)
|
||||
|
||||
Raises on DB errors — caller wraps in try/except + SAVEPOINT.
|
||||
"""
|
||||
|
|
@ -790,8 +807,20 @@ def _link_listing_to_house(
|
|||
|
||||
# House resolution: needs at least address or (lat, lon). Skip otherwise —
|
||||
# listing_sources requires listing_id but not house linkage, so still upsert.
|
||||
#
|
||||
# property_type == 'house' short-circuit (DomClick house SKU, PR1/2): `houses`/
|
||||
# `house_sources` (009_houses.sql, 010_houses_alter.sql) are an apartment-BUILDING
|
||||
# profile cache (total_units, passenger_elevators, material_walls, has_concierge)
|
||||
# — attributes of a multi-unit building, not a standalone private house. Matching
|
||||
# or creating a `houses` row for a house listing would corrupt that cache with a
|
||||
# non-building entity. house_id stays None (listings.house_id_fk NULL); the
|
||||
# ext_id→listing_id registration below still runs (independent of house linkage).
|
||||
# Cross-source house dedup for standalone houses is out of scope for V1 (only
|
||||
# DomClick scrapes this type).
|
||||
house_id: int | None = None
|
||||
if lot.address or (lot.lat is not None and lot.lon is not None):
|
||||
if lot.property_type == "house":
|
||||
pass
|
||||
elif lot.address or (lot.lat is not None and lot.lon is not None):
|
||||
# Use house_source/house_ext_id when scraper extracted them (Avito Houses
|
||||
# catalog link, Cian newbuilding). Falls back to per-listing identity
|
||||
# so each unique listing creates its own row if no canonical house exists.
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue