From e961ea9435303724852d5d29920aaa3ce9ce63ad Mon Sep 17 00:00:00 2001 From: Light1YT Date: Wed, 3 Jun 2026 01:05:05 +0500 Subject: [PATCH] feat(domrf): persist domrf_kn_objects.free_flats (#950 PR A) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Parse freeFlatsInfo.numberFlats (currently-listed flat count) from the kn-API payload into a new nullable free_flats INT, mirroring price_min_rub (extraction via _to_int, UPSERT column/bind, ON CONFLICT overwrite). _to_int rejects bool, preserves 0, truncates float, and is overflow-safe. Migration 124 adds the column + COMMENT and backfills historical rows deterministically from the stored raw payload (domrf_raw_endpoints, section kn_api) without re-scraping — same jsonb_array_elements pattern already proven on prod in migration 121, plus a {1,9}/{1,18} digit-cap so a stray numeric string can never overflow int4/bigint at the deploy gate. Unblocks object-level Layer2 (Site Finder v2 EPIC 6): hidden = flat_count (total PD) − free_flats (listed now). --- backend/app/services/scrapers/domrf_kn.py | 31 ++++++++++- data/sql/124_domrf_kn_free_flats.sql | 66 +++++++++++++++++++++++ 2 files changed, 95 insertions(+), 2 deletions(-) create mode 100644 data/sql/124_domrf_kn_free_flats.sql diff --git a/backend/app/services/scrapers/domrf_kn.py b/backend/app/services/scrapers/domrf_kn.py index 6e26498e..9f6f2353 100644 --- a/backend/app/services/scrapers/domrf_kn.py +++ b/backend/app/services/scrapers/domrf_kn.py @@ -91,6 +91,29 @@ def _to_bool(v: Any) -> bool | None: return None +def _to_int(v: Any) -> int | None: + """Coerce a DOM.РФ numeric value into int. None / non-numeric / empty → None. + Accepts int, float (truncated), and numeric strings ('12', '12.0').""" + if v is None or isinstance(v, bool): + return None + if isinstance(v, int): + return v + if isinstance(v, float): + try: + return int(v) # int(inf) → OverflowError, int(nan) → ValueError + except (ValueError, OverflowError): + return None + if isinstance(v, str): + s = v.strip() + if not s: + return None + try: + return int(float(s)) + except (ValueError, OverflowError): + return None + return None + + def _to_date(v: Any) -> date | None: """Coerce date string to date. Accept 'YYYY-MM-DD', 'YYYY-MM-DD HH:MM:SS', 'DD-MM-YYYY HH:MM:SS', or quarter-string like 'IV кв. 2028'. @@ -269,8 +292,10 @@ def _norm_object(row: dict[str, Any], region_cd: int | None = None) -> dict[str, ] # freeFlatsInfo.priceMin — price of cheapest available flat in the object + # freeFlatsInfo.numberFlats — кол-во квартир в текущей продаже (#950 Layer2). free_info = row.get("freeFlatsInfo") if isinstance(row.get("freeFlatsInfo"), dict) else {} price_min_rub = _g(free_info, "priceMin") if free_info else None + free_flats = _to_int(_g(free_info, "numberFlats")) if free_info else None return { "obj_id": _g(row, "objId", "obj_id", "id"), @@ -302,6 +327,7 @@ def _norm_object(row: dict[str, Any], region_cd: int | None = None) -> dict[str, "project_declaration_num": _g(row, "rpdNum"), "project_published_at": _to_date(_g(row, "objPublDt")), "price_min_rub": price_min_rub, + "free_flats": free_flats, "dev_group_name": _g(dev, "groupName"), # 22e + 22h: metro (nearest station from kn-API, single object) "metro_nearest_name": metro_nearest_name, @@ -429,7 +455,7 @@ UPSERT_OBJECT_SQL = text( obj_class, wall_type, energy_eff, latitude, longitude, obj_status, section_count, project_declaration_num, project_published_at, - price_min_rub, dev_group_name, + price_min_rub, free_flats, dev_group_name, metro_nearest_name, metro_nearest_walk_minutes, metro_top3, first_floor_type, elevators_passenger_count, elevators_cargo_count, parking_total_slots, guest_parking_inside_count, guest_parking_outside_count, @@ -448,7 +474,7 @@ UPSERT_OBJECT_SQL = text( :obj_class, :wall_type, :energy_eff, :latitude, :longitude, :obj_status, :section_count, :project_declaration_num, :project_published_at, - :price_min_rub, :dev_group_name, + :price_min_rub, :free_flats, :dev_group_name, :metro_nearest_name, :metro_nearest_walk_minutes, CAST(:metro_top3 AS jsonb), :first_floor_type, :elevators_passenger_count, :elevators_cargo_count, :parking_total_slots, :guest_parking_inside_count, :guest_parking_outside_count, @@ -484,6 +510,7 @@ UPSERT_OBJECT_SQL = text( EXCLUDED.project_published_at, domrf_kn_objects.project_published_at ), price_min_rub = EXCLUDED.price_min_rub, + free_flats = EXCLUDED.free_flats, dev_group_name = COALESCE(EXCLUDED.dev_group_name, domrf_kn_objects.dev_group_name), metro_nearest_name = COALESCE( EXCLUDED.metro_nearest_name, domrf_kn_objects.metro_nearest_name diff --git a/data/sql/124_domrf_kn_free_flats.sql b/data/sql/124_domrf_kn_free_flats.sql new file mode 100644 index 00000000..77b2f5ba --- /dev/null +++ b/data/sql/124_domrf_kn_free_flats.sql @@ -0,0 +1,66 @@ +-- 124_domrf_kn_free_flats.sql +-- #950 (Site Finder v2 / 3-layer склад, EPIC 6) — PR A: persist free_flats. +-- +-- Добавляет domrf_kn_objects.free_flats = freeFlatsInfo.numberFlats из kn-API dom.рф +-- (кол-во квартир ЖК В ТЕКУЩЕЙ ПРОДАЖЕ). Это единственный детерминированный сигнал, +-- разблокирующий object-level Layer2: hidden = flat_count − free_flats +-- (flat_count = всего квартир по проектной декларации; free_flats = выставлено сейчас). +-- +-- До #950 payload-блок freeFlatsInfo парсился ТОЛЬКО ради priceMin → price_min_rub +-- (миграция 113); numberFlats отбрасывался. Поле free_flats — sibling price_min_rub: +-- bigint→int, та же null-safety, тот же EXCLUDED-overwrite в UPSERT_OBJECT_SQL. +-- +-- Backfill: сырой list объектов kn-API сохраняется как JSONB в domrf_raw_endpoints +-- (section='kn_api', endpoint='kn_object_place_'; см. domrf_kn._insert_raw), +-- каждый элемент массива несёт objId + freeFlatsInfo.numberFlats. Поэтому historical +-- строки можно заполнить БЕЗ повторного скрейпа — join raw-элементов к +-- domrf_kn_objects по (obj_id, snapshot_date). Детерминированно, без LLM. +-- +-- Deploy: auto-applied by deploy.yml через _schema_migrations tracking (ровно один раз). +-- Idempotent: ADD COLUMN IF NOT EXISTS + backfill только WHERE free_flats IS NULL. +-- Migration order: эта SQL-миграция применяется ПЕРВОЙ, backend domrf_kn.py — следом. +-- free_flats НЕnullable НЕ делаем: historical строки до бекфилла/без raw остаются NULL, +-- L2-сервис в этом случае падает на fallback (flat_count − sold). + +BEGIN; + +-- ── 1. Колонка ───────────────────────────────────────────────────────────────── +ALTER TABLE domrf_kn_objects + ADD COLUMN IF NOT EXISTS free_flats integer; + +COMMENT ON COLUMN domrf_kn_objects.free_flats IS + 'Кол-во квартир в текущей продаже (freeFlatsInfo.numberFlats из kn-API dom.рф). ' + 'NULL = не заполнено (старый снапшот до #950). Для Layer2 hidden = flat_count − free_flats.'; + +-- ── 2. Backfill из сохранённого raw-payload ──────────────────────────────────── +-- domrf_raw_endpoints.payload — JSONB-массив сырых object-строк kn-API. Разворачиваем +-- его, тянем objId + freeFlatsInfo.numberFlats и матчим к уже записанным строкам по +-- (obj_id, snapshot_date). Только WHERE free_flats IS NULL — повторный прогон no-op. +-- Guard: payload должен быть массивом; objId и numberFlats — числовые строки (регексп +-- отсекает null/мусор ДО CAST, иначе invalid input syntax for type integer). Digit-cap +-- {1,9}/{1,18} держит numberFlats в int4, objId в bigint — отсекает overflow ДО CAST +-- (числа-кандидаты семантически ограничены квартирами одного ЖК, но guard явный). +-- DISTINCT ON (obj_id, snapshot_date): один objId не дублируется в одном снапшоте, но +-- на случай нескольких place-endpoint'ов на дату берём первый детерминированно. +WITH raw_flats AS ( + SELECT DISTINCT ON (obj_id, r.snapshot_date) + CAST(elem->>'objId' AS bigint) AS obj_id, + r.snapshot_date AS snapshot_date, + CAST(elem->'freeFlatsInfo'->>'numberFlats' AS integer) AS free_flats + FROM domrf_raw_endpoints AS r + CROSS JOIN LATERAL jsonb_array_elements(r.payload) AS elem + WHERE r.section = 'kn_api' + AND r.endpoint LIKE 'kn_object_place_%' + AND jsonb_typeof(r.payload) = 'array' + AND elem->>'objId' ~ '^[0-9]{1,18}$' + AND elem->'freeFlatsInfo'->>'numberFlats' ~ '^[0-9]{1,9}$' + ORDER BY obj_id, r.snapshot_date +) +UPDATE domrf_kn_objects AS o + SET free_flats = rf.free_flats + FROM raw_flats AS rf + WHERE o.obj_id = rf.obj_id + AND o.snapshot_date = rf.snapshot_date + AND o.free_flats IS NULL; + +COMMIT; -- 2.45.3