fix(tradein/yandex): cap ceiling to numeric(3,2) range + backfill 'other'->NULL
All checks were successful
CI / changes (pull_request) Successful in 8s
CI / backend-tests (pull_request) Has been skipped
CI / frontend-tests (pull_request) Has been skipped
CI / openapi-codegen-check (pull_request) Has been skipped

Pre-push review (#2007):
- ceiling_height_m: clamp to plausible 2.0-6.0 m. yandex SERP emits garbage (seen 18 m)
  -> out-of-range write to ceiling_height numeric(3,2) raised DataError failing the whole
  batch (per-lot SAVEPOINT only catches IntegrityError). Raw value still in raw_payload.
- migration 140: also NULL out legacy 'other' (299 rows) for parity with
  normalize_house_type (unknown->None; NULL neutral in estimator vs 'other' false penalty).
This commit is contained in:
bot-backend 2026-06-27 20:43:31 +03:00
parent a1ee06ef17
commit 024fadac73
3 changed files with 35 additions and 4 deletions

View file

@ -325,7 +325,14 @@ def _entity_to_lot(
total_floors = int(floors_total_raw) if floors_total_raw is not None else None
ceiling_height = entity.get("ceilingHeight")
ceiling_height_m = float(ceiling_height) if ceiling_height is not None else None
# Колонка ceiling_height — numeric(3,2), max 9.99. Yandex SERP отдаёт мусор
# (видели 18 м) → запись out-of-range уронила бы весь батч DataError'ом
# (per-lot SAVEPOINT ловит только IntegrityError). Берём только правдоподобный
# диапазон 2.06.0 м, иначе None; сырое значение остаётся в raw_payload. (#2007)
_ceiling_raw = float(ceiling_height) if ceiling_height is not None else None
ceiling_height_m = (
_ceiling_raw if _ceiling_raw is not None and 2.0 <= _ceiling_raw <= 6.0 else None
)
building = entity.get("building") or {}
year_built_raw = building.get("builtYear")

View file

@ -13,9 +13,14 @@
-- MONOLIT->monolith, BRICK->brick, PANEL->panel, MONOLIT_BRICK->monolith_brick,
-- BLOCK->block, WOOD->wood.
--
-- Idempotency: после первого прогона SCREAMING-значений не остаётся, WHERE-фильтр
-- пуст → повторный apply = no-op (0 rows). Scope строго source='yandex' —
-- cian/avito не трогаем (их канон чинится в коде / отдельных PR, напр. #2008).
-- Плюс 'other' (не-канон легаси-значение, 299 строк) → NULL: паритет с
-- normalize_house_type (unknown→None). В estimator soft-penalty NULL нейтрально,
-- а 'other' != target всегда даёт ложный штраф. Лоуэркейс 'brick'/'panel' уже
-- каноничны — не трогаем.
--
-- Idempotency: после первого прогона SCREAMING/'other'-значений не остаётся,
-- WHERE-фильтры пусты → повторный apply = no-op (0 rows). Scope строго
-- source='yandex' — cian/avito не трогаем (канон в коде / отдельных PR, напр. #2008).
BEGIN;
@ -32,4 +37,10 @@ UPDATE listings
WHERE source = 'yandex'
AND house_type IN ('MONOLIT', 'BRICK', 'PANEL', 'MONOLIT_BRICK', 'BLOCK', 'WOOD');
-- 'other' → NULL (паритет с normalize_house_type: unknown→None, нейтрально в estimator).
UPDATE listings
SET house_type = NULL
WHERE source = 'yandex'
AND house_type = 'other';
COMMIT;

View file

@ -239,6 +239,19 @@ def test_entity_to_lot_full_entity():
assert lot.raw_payload["raw_building_type"] == "MONOLIT" # сырой SCREAMING сохранён
def test_entity_to_lot_ceiling_out_of_range_dropped():
"""ceiling_height — numeric(3,2) (max 9.99); неправдоподобный yandex SERP мусор
(видели 18 м) отсекаем в None, иначе DataError уронил бы весь батч. Сырое
значение остаётся в raw_payload. (#2007)"""
lot_hi = _entity_to_lot({**_ENTITY_FULL, "ceilingHeight": 18})
assert lot_hi is not None
assert lot_hi.ceiling_height_m is None
assert lot_hi.raw_payload["ceiling_height"] == 18 # сырое сохранено
lot_lo = _entity_to_lot({**_ENTITY_FULL, "ceilingHeight": 1.6})
assert lot_lo is not None
assert lot_lo.ceiling_height_m is None
def test_entity_to_lot_segment_vtorichka_default():
"""Default new_flat='NO' -> listing_segment='vtorichka'."""
lot = _entity_to_lot(_ENTITY_FULL)