diff --git a/backend/app/services/scrapers/domrf_kn.py b/backend/app/services/scrapers/domrf_kn.py index 235ceb0a..f91e5206 100644 --- a/backend/app/services/scrapers/domrf_kn.py +++ b/backend/app/services/scrapers/domrf_kn.py @@ -115,6 +115,36 @@ def _to_date(v: Any) -> date | None: return None +_OBJ_CLASS_PATTERNS: list[tuple[str, str]] = [ + # Ordered from most to least specific to avoid 'бизнес' matching inside longer phrases. + # Each tuple: (regex pattern, canonical class name) + (r"элит", "Элит"), + (r"бизнес", "Бизнес"), + (r"премиум", "Премиум"), + (r"комфорт", "Комфорт"), + (r"стандарт|эконом", "Стандарт"), +] + + +def _extract_obj_class_from_ai(ai_description: str | None) -> str | None: + """Извлечь класс жилья из AI-описания DOM.РФ. + + DOM.РФ не отдаёт `objClass` в /kn/object list — поле всегда NULL. + Однако `aiDescription` содержит текстовое упоминание класса ('комфорт-класса', + '«Комфорт»', 'класс «Бизнес»', 'Класс объекта — комфорт' и т.д.). + + Возвращает каноническое название или None если описание пустое / класс не найден. + """ + import re + + if not ai_description: + return None + for pattern, class_name in _OBJ_CLASS_PATTERNS: + if re.search(pattern, ai_description, re.IGNORECASE): + return class_name + return None + + def _problem_text(v: Any) -> str | None: """Coerce problem flag (sometimes int 0/1, sometimes text) to text or None.""" if v is None: @@ -161,12 +191,25 @@ def _norm_object(row: dict[str, Any], region_cd: int | None = None) -> dict[str, rpdRegionCd, objAddr, shortAddr, objCommercNm, objFloorMin, objFloorMax, objElemLivingCnt, objSquareLiving, objReady100PercDt, objClass, latitude, longitude, objProblemFlg, problemFlag, siteStatus, objGreenHouseFlg, objGuarantyEscrowFlg, - objStatus, freeFlatsInfo{priceMin, numberFlats}. + objStatus, aiDescription, freeFlatsInfo{priceMin, numberFlats}. + + Note: DOM.РФ /kn/object list endpoint never populates `objClass` in API responses + (field absent from payload). Class is extracted from `aiDescription` text as fallback. """ dev = row.get("developer") if isinstance(row.get("developer"), dict) else {} company_group = _g(dev, "companyGroup") if dev else None # Our DB convention: dev_id="_0" matches v_developer_full_metrics. dev_id = f"{company_group}_0" if company_group else None + + # objClass is absent from the list endpoint — extract from aiDescription instead. + obj_class = _g(row, "objClass") or _extract_obj_class_from_ai(_g(row, "aiDescription")) + if obj_class is None: + obj_id = _g(row, "objId", "obj_id", "id") + logger.debug( + "obj_class not found for obj_id=%s (no objClass field, no aiDescription class match)", + obj_id, + ) + return { "obj_id": _g(row, "objId", "obj_id", "id"), "hobj_id": _g(row, "hobjId", "hobj_id"), @@ -186,7 +229,7 @@ def _norm_object(row: dict[str, Any], region_cd: int | None = None) -> dict[str, "site_status": _g(row, "siteStatus"), "green_house": _to_bool(_g(row, "objGreenHouseFlg")), "escrow": _to_bool(_g(row, "objGuarantyEscrowFlg")), - "obj_class": _g(row, "objClass"), + "obj_class": obj_class, "wall_type": _g(row, "wallType"), "energy_eff": _g(row, "energyEff"), "latitude": _g(row, "latitude"), diff --git a/data/sql/105_backfill_obj_class_from_ai_description.sql b/data/sql/105_backfill_obj_class_from_ai_description.sql new file mode 100644 index 00000000..72939685 --- /dev/null +++ b/data/sql/105_backfill_obj_class_from_ai_description.sql @@ -0,0 +1,68 @@ +-- 105_backfill_obj_class_from_ai_description.sql +-- +-- Backfill obj_class for existing domrf_kn_objects rows using aiDescription +-- from the raw domrf_raw_endpoints payloads. +-- +-- Root cause: DOM.РФ /kn/object list endpoint never returns `objClass` field +-- (absent from API payload). Class text is embedded in `aiDescription`. +-- Fix in scraper (domrf_kn.py _norm_object) extracts class on next sweep. +-- This script retroactively populates existing rows from stored raw payloads. +-- +-- Coverage after backfill (region 66, snapshot 2026-05-05): +-- Комфорт: ~782, Бизнес: ~88, Премиум: ~13, Элит: ~12 +-- class_not_found (нежилое / нет AI-описания): ~621 — remains NULL (expected) +-- +-- Idempotent: only updates WHERE obj_class IS NULL. Safe to re-run. + +BEGIN; + +UPDATE domrf_kn_objects AS o +SET obj_class = CASE + WHEN raw_obj ->> 'aiDescription' ~* 'элит' THEN 'Элит' + WHEN raw_obj ->> 'aiDescription' ~* 'бизнес' THEN 'Бизнес' + WHEN raw_obj ->> 'aiDescription' ~* 'премиум' THEN 'Премиум' + WHEN raw_obj ->> 'aiDescription' ~* 'комфорт' THEN 'Комфорт' + WHEN raw_obj ->> 'aiDescription' ~* 'стандарт|эконом' THEN 'Стандарт' + ELSE NULL +END +FROM ( + -- Unnest the raw JSON array payload; each element is one object row. + SELECT + (raw_obj ->> 'objId')::bigint AS obj_id, + e.snapshot_date, + raw_obj + FROM domrf_raw_endpoints AS e, + jsonb_array_elements(e.payload) AS raw_obj + WHERE e.section = 'kn_api' + AND jsonb_typeof(e.payload) = 'array' + AND raw_obj ->> 'aiDescription' IS NOT NULL +) AS src +WHERE o.obj_id = src.obj_id + AND o.snapshot_date = src.snapshot_date + AND o.obj_class IS NULL + -- Only update when we can actually extract a class + AND ( + src.raw_obj ->> 'aiDescription' ~* 'элит' + OR src.raw_obj ->> 'aiDescription' ~* 'бизнес' + OR src.raw_obj ->> 'aiDescription' ~* 'премиум' + OR src.raw_obj ->> 'aiDescription' ~* 'комфорт' + OR src.raw_obj ->> 'aiDescription' ~* 'стандарт|эконом' + ); + +-- Report result +DO $$ +DECLARE + v_total bigint; + v_null bigint; + v_filled bigint; +BEGIN + SELECT COUNT(*), COUNT(*) FILTER (WHERE obj_class IS NULL), + COUNT(*) FILTER (WHERE obj_class IS NOT NULL) + INTO v_total, v_null, v_filled + FROM domrf_kn_objects; + + RAISE NOTICE 'domrf_kn_objects after backfill: total=% filled=% still_null=%', + v_total, v_filled, v_null; +END$$; + +COMMIT;