diff --git a/backend/app/services/scrapers/domrf_kn.py b/backend/app/services/scrapers/domrf_kn.py index f91e5206..1bbe7cd4 100644 --- a/backend/app/services/scrapers/domrf_kn.py +++ b/backend/app/services/scrapers/domrf_kn.py @@ -262,6 +262,11 @@ def _norm_flat(row: dict[str, Any], region_cd: int | None) -> dict[str, Any]: Real fields: flatId|id (numeric, may be null), odsId, elemId (uuid hash), type, number, isStudio, totalArea, livingArea, rooms, status (free|booked|sold), price, pricePerSquareMeter, numberFloors. Plus injected _objId and _floor. + + Note: /portal/table returns price=null for most objects (sold/booked flats and + objects that don't expose per-flat pricing via this endpoint). price_per_m2 is + derived from price_rub / total_area when the API returns price_rub but omits + pricePerSquareMeter — defensive fallback for partial API responses. """ flat_id = _g(row, "flatId", "id") if flat_id is None: @@ -269,18 +274,34 @@ def _norm_flat(row: dict[str, Any], region_cd: int | None) -> dict[str, Any]: elem = _g(row, "elemId") if elem: flat_id = abs(hash(elem)) % (2**63 - 1) + + price_rub = _g(row, "price") + price_per_m2 = _g(row, "pricePerSquareMeter") + total_area = _g(row, "totalArea") + + # Derive price_per_m2 when API returns price_rub but omits pricePerSquareMeter. + # Covers cases where the table endpoint has the flat price but no pre-computed m² rate. + if price_per_m2 is None and price_rub is not None and total_area and total_area > 0: + price_per_m2 = round(price_rub / total_area, 2) + logger.info( + "derive price_per_m2=%.2f for flat ods_id=%s obj_id=%s", + price_per_m2, + _g(row, "odsId"), + _g(row, "_objId", "objId"), + ) + return { "id": flat_id, "ods_id": _g(row, "odsId"), "flat_type": _g(row, "type", "flatType"), "flat_number": _g(row, "number", "flatNumber"), "is_studio": _to_bool(_g(row, "isStudio")), - "total_area": _g(row, "totalArea"), + "total_area": total_area, "living_area": _g(row, "livingArea"), "rooms": _g(row, "rooms"), "status": _g(row, "status"), - "price_rub": _g(row, "price"), - "price_per_m2": _g(row, "pricePerSquareMeter"), + "price_rub": price_rub, + "price_per_m2": price_per_m2, "floor": _g(row, "_floor", "floor"), "num_floors": _g(row, "numberFloors"), "obj_id": _g(row, "_objId", "objId"), diff --git a/data/sql/107_backfill_price_per_m2.sql b/data/sql/107_backfill_price_per_m2.sql new file mode 100644 index 00000000..7fd9946c --- /dev/null +++ b/data/sql/107_backfill_price_per_m2.sql @@ -0,0 +1,24 @@ +-- 107_backfill_price_per_m2.sql +-- Backfill price_per_m2 = ROUND(price_rub / total_area, 2) for rows where +-- price_rub is known but price_per_m2 was not stored by the scraper. +-- +-- Context (SF FixList #16): +-- domrf_kn.py _norm_flat() now derives price_per_m2 on ingest. +-- This one-time UPDATE covers historical rows scraped before the fix. +-- For Малевич (obj_id=64701) and the wider dataset, both price_rub AND +-- price_per_m2 are NULL (DOM.RF /portal/table API does not return per-flat +-- pricing for most objects). Those rows are unaffected by this UPDATE. +-- Only rows where price_rub IS NOT NULL benefit — currently ~849 rows. +-- +-- Safe to re-run: idempotent (WHERE price_per_m2 IS NULL). + +BEGIN; + +UPDATE domrf_kn_flats +SET price_per_m2 = ROUND((price_rub / total_area)::numeric, 2) +WHERE price_per_m2 IS NULL + AND price_rub IS NOT NULL + AND total_area IS NOT NULL + AND total_area > 0; + +COMMIT;