fix(sf-16): derive price_per_m2 from price_rub in _norm_flat + backfill SQL
Audit confirmed: DOM.RF /portal/table API returns price=null for 99.7% of flats (both price_rub and price_per_m2 are NULL). Root cause is API limitation — per-flat pricing lives in a separate catalog endpoint not yet scraped (BUG #22). Adds defensive fallback in _norm_flat(): when price_rub is present but pricePerSquareMeter is absent, derive price_per_m2 = round(price_rub / total_area, 2). Covers partial API responses from any future endpoint that omits the computed rate. Adds 107_backfill_price_per_m2.sql for historical rows (~849 affected).
This commit is contained in:
parent
e054395464
commit
d3ce5a2f96
2 changed files with 48 additions and 3 deletions
|
|
@ -262,6 +262,11 @@ def _norm_flat(row: dict[str, Any], region_cd: int | None) -> dict[str, Any]:
|
||||||
Real fields: flatId|id (numeric, may be null), odsId, elemId (uuid hash),
|
Real fields: flatId|id (numeric, may be null), odsId, elemId (uuid hash),
|
||||||
type, number, isStudio, totalArea, livingArea, rooms, status (free|booked|sold),
|
type, number, isStudio, totalArea, livingArea, rooms, status (free|booked|sold),
|
||||||
price, pricePerSquareMeter, numberFloors. Plus injected _objId and _floor.
|
price, pricePerSquareMeter, numberFloors. Plus injected _objId and _floor.
|
||||||
|
|
||||||
|
Note: /portal/table returns price=null for most objects (sold/booked flats and
|
||||||
|
objects that don't expose per-flat pricing via this endpoint). price_per_m2 is
|
||||||
|
derived from price_rub / total_area when the API returns price_rub but omits
|
||||||
|
pricePerSquareMeter — defensive fallback for partial API responses.
|
||||||
"""
|
"""
|
||||||
flat_id = _g(row, "flatId", "id")
|
flat_id = _g(row, "flatId", "id")
|
||||||
if flat_id is None:
|
if flat_id is None:
|
||||||
|
|
@ -269,18 +274,34 @@ def _norm_flat(row: dict[str, Any], region_cd: int | None) -> dict[str, Any]:
|
||||||
elem = _g(row, "elemId")
|
elem = _g(row, "elemId")
|
||||||
if elem:
|
if elem:
|
||||||
flat_id = abs(hash(elem)) % (2**63 - 1)
|
flat_id = abs(hash(elem)) % (2**63 - 1)
|
||||||
|
|
||||||
|
price_rub = _g(row, "price")
|
||||||
|
price_per_m2 = _g(row, "pricePerSquareMeter")
|
||||||
|
total_area = _g(row, "totalArea")
|
||||||
|
|
||||||
|
# Derive price_per_m2 when API returns price_rub but omits pricePerSquareMeter.
|
||||||
|
# Covers cases where the table endpoint has the flat price but no pre-computed m² rate.
|
||||||
|
if price_per_m2 is None and price_rub is not None and total_area and total_area > 0:
|
||||||
|
price_per_m2 = round(price_rub / total_area, 2)
|
||||||
|
logger.info(
|
||||||
|
"derive price_per_m2=%.2f for flat ods_id=%s obj_id=%s",
|
||||||
|
price_per_m2,
|
||||||
|
_g(row, "odsId"),
|
||||||
|
_g(row, "_objId", "objId"),
|
||||||
|
)
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"id": flat_id,
|
"id": flat_id,
|
||||||
"ods_id": _g(row, "odsId"),
|
"ods_id": _g(row, "odsId"),
|
||||||
"flat_type": _g(row, "type", "flatType"),
|
"flat_type": _g(row, "type", "flatType"),
|
||||||
"flat_number": _g(row, "number", "flatNumber"),
|
"flat_number": _g(row, "number", "flatNumber"),
|
||||||
"is_studio": _to_bool(_g(row, "isStudio")),
|
"is_studio": _to_bool(_g(row, "isStudio")),
|
||||||
"total_area": _g(row, "totalArea"),
|
"total_area": total_area,
|
||||||
"living_area": _g(row, "livingArea"),
|
"living_area": _g(row, "livingArea"),
|
||||||
"rooms": _g(row, "rooms"),
|
"rooms": _g(row, "rooms"),
|
||||||
"status": _g(row, "status"),
|
"status": _g(row, "status"),
|
||||||
"price_rub": _g(row, "price"),
|
"price_rub": price_rub,
|
||||||
"price_per_m2": _g(row, "pricePerSquareMeter"),
|
"price_per_m2": price_per_m2,
|
||||||
"floor": _g(row, "_floor", "floor"),
|
"floor": _g(row, "_floor", "floor"),
|
||||||
"num_floors": _g(row, "numberFloors"),
|
"num_floors": _g(row, "numberFloors"),
|
||||||
"obj_id": _g(row, "_objId", "objId"),
|
"obj_id": _g(row, "_objId", "objId"),
|
||||||
|
|
|
||||||
24
data/sql/107_backfill_price_per_m2.sql
Normal file
24
data/sql/107_backfill_price_per_m2.sql
Normal file
|
|
@ -0,0 +1,24 @@
|
||||||
|
-- 107_backfill_price_per_m2.sql
|
||||||
|
-- Backfill price_per_m2 = ROUND(price_rub / total_area, 2) for rows where
|
||||||
|
-- price_rub is known but price_per_m2 was not stored by the scraper.
|
||||||
|
--
|
||||||
|
-- Context (SF FixList #16):
|
||||||
|
-- domrf_kn.py _norm_flat() now derives price_per_m2 on ingest.
|
||||||
|
-- This one-time UPDATE covers historical rows scraped before the fix.
|
||||||
|
-- For Малевич (obj_id=64701) and the wider dataset, both price_rub AND
|
||||||
|
-- price_per_m2 are NULL (DOM.RF /portal/table API does not return per-flat
|
||||||
|
-- pricing for most objects). Those rows are unaffected by this UPDATE.
|
||||||
|
-- Only rows where price_rub IS NOT NULL benefit — currently ~849 rows.
|
||||||
|
--
|
||||||
|
-- Safe to re-run: idempotent (WHERE price_per_m2 IS NULL).
|
||||||
|
|
||||||
|
BEGIN;
|
||||||
|
|
||||||
|
UPDATE domrf_kn_flats
|
||||||
|
SET price_per_m2 = ROUND((price_rub / total_area)::numeric, 2)
|
||||||
|
WHERE price_per_m2 IS NULL
|
||||||
|
AND price_rub IS NOT NULL
|
||||||
|
AND total_area IS NOT NULL
|
||||||
|
AND total_area > 0;
|
||||||
|
|
||||||
|
COMMIT;
|
||||||
Loading…
Add table
Reference in a new issue