From 6f170504baa4a7bebf549bd43e92f95988fba439 Mon Sep 17 00:00:00 2001 From: bot-backend Date: Fri, 3 Jul 2026 23:00:42 +0300 Subject: [PATCH] =?UTF-8?q?fix(tradein/pii):=20=D1=83=D0=B4=D0=B0=D0=BB?= =?UTF-8?q?=D0=B8=D1=82=D1=8C=20=D1=82=D0=B5=D0=BB=D0=B5=D1=84=D0=BE=D0=BD?= =?UTF-8?q?=D1=8B=20=D0=BF=D1=80=D0=BE=D0=B4=D0=B0=D0=B2=D1=86=D0=BE=D0=B2?= =?UTF-8?q?=20(NULL)=20+=20PII=20=D0=BA=D0=BB=D0=B8=D0=B5=D0=BD=D1=82?= =?UTF-8?q?=D0=B0=20(DROP)=20(#2217,=20#1969)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - listings.phones: миграция 166 обнуляет данные; cian.py + scraper-kit serp.py (golden-parity зеркало) перестают читать offer.get("phones") на входе — write-path выключен у источника, колонка/поле в модели остаются. Мёртвая запись "phones" в LISTING_FIELD_PRIORITY убрана. - trade_in_estimates.client_name/client_phone: миграция 167 дропает колонки; синхронно убраны из TradeInEstimateInput, обоих INSERT в estimator.py (основной путь + _empty_estimate), frontend TradeInEstimateInput и CRM-блока EstimateForm (поля "Имя клиента"/"Телефон"). - sentry_scrub.py и management_companies.phones не тронуты — вне scope. --- tradein-mvp/backend/app/schemas/trade_in.py | 3 +- tradein-mvp/backend/app/services/estimator.py | 12 +++---- .../services/matching/conflict_resolution.py | 1 - .../backend/app/services/scrapers/cian.py | 6 +++- .../data/sql/166_purge_listings_phones.sql | 26 ++++++++++++++ .../backend/data/sql/167_drop_client_pii.sql | 24 +++++++++++++ .../backend/data/sql/_manifest_applied.txt | 2 ++ .../backend/tests/test_base_save_listings.py | 9 ++--- .../backend/tests/test_cian_serp_scraper.py | 7 ++-- .../src/components/trade-in/EstimateForm.tsx | 34 ------------------- tradein-mvp/frontend/src/types/trade-in.ts | 3 +- .../src/scraper_kit/providers/cian/serp.py | 6 +++- 12 files changed, 76 insertions(+), 57 deletions(-) create mode 100644 tradein-mvp/backend/data/sql/166_purge_listings_phones.sql create mode 100644 tradein-mvp/backend/data/sql/167_drop_client_pii.sql diff --git a/tradein-mvp/backend/app/schemas/trade_in.py b/tradein-mvp/backend/app/schemas/trade_in.py index 7e97a5f6..e9172a65 100644 --- a/tradein-mvp/backend/app/schemas/trade_in.py +++ b/tradein-mvp/backend/app/schemas/trade_in.py @@ -40,8 +40,7 @@ class TradeInEstimateInput(BaseModel): # CRM-поля (#395) — операционные, на расчёт оценки не влияют ownership_type: str | None = Field(default=None, max_length=100) has_mortgage: bool | None = None - client_name: str | None = Field(default=None, max_length=200) - client_phone: str | None = Field(default=None, max_length=50) + # client_name / client_phone удалены (PII purge #1969, DROP COLUMN 167). class AnalogLot(BaseModel): diff --git a/tradein-mvp/backend/app/services/estimator.py b/tradein-mvp/backend/app/services/estimator.py index a6391be9..547e5f99 100644 --- a/tradein-mvp/backend/app/services/estimator.py +++ b/tradein-mvp/backend/app/services/estimator.py @@ -3302,7 +3302,7 @@ async def estimate_quality( id, address, lat, lon, area_m2, rooms, floor, total_floors, year_built, house_type, repair_state, has_balcony, - ownership_type, has_mortgage, client_name, client_phone, + ownership_type, has_mortgage, median_price, range_low, range_high, median_price_per_m2, confidence, confidence_explanation, n_analogs, analogs, actual_deals, @@ -3319,7 +3319,7 @@ async def estimate_quality( :address, :lat, :lon, :area, :rooms, :floor, :total_floors, :year_built, :house_type, :repair_state, :has_balcony, - :ownership_type, :has_mortgage, :client_name, :client_phone, + :ownership_type, :has_mortgage, :median_price, :range_low, :range_high, :median_ppm2, :confidence, :explanation, :n_analogs, CAST(:analogs_json AS jsonb), @@ -3352,8 +3352,6 @@ async def estimate_quality( "has_balcony": payload.has_balcony, "ownership_type": payload.ownership_type, "has_mortgage": payload.has_mortgage, - "client_name": payload.client_name, - "client_phone": payload.client_phone, "median_price": median_price, "range_low": range_low, "range_high": range_high, @@ -5344,7 +5342,7 @@ def _empty_estimate( id, address, area_m2, rooms, floor, total_floors, year_built, house_type, repair_state, has_balcony, - ownership_type, has_mortgage, client_name, client_phone, + ownership_type, has_mortgage, median_price, range_low, range_high, median_price_per_m2, confidence, confidence_explanation, n_analogs, analogs, actual_deals, @@ -5355,7 +5353,7 @@ def _empty_estimate( CAST(:id AS uuid), :address, :area, :rooms, :floor, :total_floors, :year_built, :house_type, :repair_state, :has_balcony, - :ownership_type, :has_mortgage, :client_name, :client_phone, + :ownership_type, :has_mortgage, 0, 0, 0, 0, 'low', :explanation, 0, '[]'::jsonb, '[]'::jsonb, @@ -5378,8 +5376,6 @@ def _empty_estimate( "has_balcony": payload.has_balcony, "ownership_type": payload.ownership_type, "has_mortgage": payload.has_mortgage, - "client_name": payload.client_name, - "client_phone": payload.client_phone, "explanation": reason, "created_by": created_by, "expires_at": expires_at, diff --git a/tradein-mvp/backend/app/services/matching/conflict_resolution.py b/tradein-mvp/backend/app/services/matching/conflict_resolution.py index cc939352..cd0a83c1 100644 --- a/tradein-mvp/backend/app/services/matching/conflict_resolution.py +++ b/tradein-mvp/backend/app/services/matching/conflict_resolution.py @@ -83,7 +83,6 @@ LISTING_FIELD_PRIORITY: dict[str, list[str] | str] = { "combined_wcs_count": ["cian_detail"], "room_type": ["cian_detail", "avito_detail"], "has_furniture": ["cian_serp", "avito_detail"], - "phones": ["cian_serp"], "description": ["cian_serp", "avito_detail", "yandex_detail"], "photo_urls": "union", # Avito Domoteka unique diff --git a/tradein-mvp/backend/app/services/scrapers/cian.py b/tradein-mvp/backend/app/services/scrapers/cian.py index 2841379b..4b514c6a 100644 --- a/tradein-mvp/backend/app/services/scrapers/cian.py +++ b/tradein-mvp/backend/app/services/scrapers/cian.py @@ -934,7 +934,11 @@ class CianScraper(BaseScraper): repair_state: str | None = normalize_repair_state(offer.get("decoration")) # ── Продавец ───────────────────────────────────────────────────── - phones: list[dict[str, Any]] = offer.get("phones") or [] + # PII purge (#2217): телефоны продавцов больше не сохраняем. + # ScrapedLot.phones остаётся в модели (см. base.py), но на вход + # всегда передаём пустой список — write-path на listings.phones + # выключен здесь, а не на уровне upsert. + phones: list[dict[str, Any]] = [] is_homeowner: bool | None = offer.get("isByHomeowner") is_pro_seller: bool | None = offer.get("isPro") diff --git a/tradein-mvp/backend/data/sql/166_purge_listings_phones.sql b/tradein-mvp/backend/data/sql/166_purge_listings_phones.sql new file mode 100644 index 00000000..d754de21 --- /dev/null +++ b/tradein-mvp/backend/data/sql/166_purge_listings_phones.sql @@ -0,0 +1,26 @@ +-- 166_purge_listings_phones.sql +-- +-- PII PURGE (#2217): телефоны продавцов из listings.phones. +-- ПРОДУКТОВОЕ РЕШЕНИЕ (user): колонку НЕ дропаем — обнуляем данные и +-- останавливаем write-path у скрейперов (cian.py / scraper-kit serp.py +-- больше не читают offer.get("phones") — см. #2217 backend PR). Колонка +-- остаётся в схеме (и в модели ScrapedLot) как задел на будущее, просто +-- больше никогда не заполняется непустым значением. +-- +-- ЧТО ДЕЛАЕТ: +-- UPDATE listings SET phones = NULL — вычищает все ранее сохранённые +-- телефоны продавцов (единственный источник, который их писал — Cian SERP). +-- +-- ИДЕМПОТЕНТНОСТЬ: +-- WHERE phones IS NOT NULL -> повторный прогон трогает 0 строк. +-- +-- НЕ ТРОГАЕТ: management_companies.phones (text[], миграция 021) — контакты +-- УК/ТСЖ, другая сущность, вне scope этого issue. + +BEGIN; + +UPDATE listings + SET phones = NULL + WHERE phones IS NOT NULL; + +COMMIT; diff --git a/tradein-mvp/backend/data/sql/167_drop_client_pii.sql b/tradein-mvp/backend/data/sql/167_drop_client_pii.sql new file mode 100644 index 00000000..2ad419ef --- /dev/null +++ b/tradein-mvp/backend/data/sql/167_drop_client_pii.sql @@ -0,0 +1,24 @@ +-- 167_drop_client_pii.sql +-- +-- PII PURGE (#1969): client_name / client_phone в trade_in_estimates. +-- ПРОДУКТОВОЕ РЕШЕНИЕ (user): DROP COLUMN (не NULL-обнуление, как для +-- listings.phones в 166 — здесь колонки не нужны совсем, CRM-поля +-- ownership_type/has_mortgage остаются). client_email физически не +-- существует (только имя в Sentry-scrub allowlist app/observability/ +-- sentry_scrub.py) — трогать нечего кроме этих двух реальных колонок. +-- +-- ЗАВИСИМОСТИ: нет VIEW / индексов на эти колонки (проверено разведкой +-- перед миграцией) — DROP безопасен. +-- +-- Backend-код (app/schemas/trade_in.py, app/services/estimator.py — оба +-- INSERT INTO trade_in_estimates) деплоится синхронно с этой миграцией. +-- +-- ИДЕМПОТЕНТНОСТЬ: DROP COLUMN IF EXISTS -> повторный прогон no-op. + +BEGIN; + +ALTER TABLE trade_in_estimates + DROP COLUMN IF EXISTS client_name, + DROP COLUMN IF EXISTS client_phone; + +COMMIT; diff --git a/tradein-mvp/backend/data/sql/_manifest_applied.txt b/tradein-mvp/backend/data/sql/_manifest_applied.txt index 3f5f10f2..c6ce94e6 100644 --- a/tradein-mvp/backend/data/sql/_manifest_applied.txt +++ b/tradein-mvp/backend/data/sql/_manifest_applied.txt @@ -170,3 +170,5 @@ 163_disable_deactivate_stale_domklik.sql 164_yandex_url_canonicalize_active_dups.sql 165_remove_n1_source.sql +166_purge_listings_phones.sql +167_drop_client_pii.sql diff --git a/tradein-mvp/backend/tests/test_base_save_listings.py b/tradein-mvp/backend/tests/test_base_save_listings.py index 9fbbf148..9904107d 100644 --- a/tradein-mvp/backend/tests/test_base_save_listings.py +++ b/tradein-mvp/backend/tests/test_base_save_listings.py @@ -92,7 +92,8 @@ def _cian_lot(**overrides) -> ScrapedLot: "description_minhash": "abc123def456abc1", "cadastral_number": "66:41:0204016:1234", "building_cadastral_number": "66:41:0204016:100", - "phones": [{"countryCode": "7", "number": "9012345678"}], + # PII purge (#2217): phones больше не заполняется — по умолчанию пусто + # (ScrapedLot.phones default_factory=list, сериализуется в None). "is_homeowner": True, "is_pro_seller": False, "bargain_allowed": True, @@ -163,10 +164,10 @@ def test_save_listings_persists_cian_specific_fields(): assert params["bargain_allowed"] is True assert params["sale_type"] == "free" - # phones и metro_stations — JSON-строки, проверяем десериализацией - phones = json.loads(params["phones"]) - assert phones == [{"countryCode": "7", "number": "9012345678"}] + # PII purge (#2217): phones больше не заполняется cian-фикстурой → None. + assert params["phones"] is None + # metro_stations — JSON-строка, проверяем десериализацией metro = json.loads(params["metro_stations"]) assert metro[0]["name"] == "Геологическая" assert metro[0]["time"] == 10 diff --git a/tradein-mvp/backend/tests/test_cian_serp_scraper.py b/tradein-mvp/backend/tests/test_cian_serp_scraper.py index a7d3a49f..0bb42868 100644 --- a/tradein-mvp/backend/tests/test_cian_serp_scraper.py +++ b/tradein-mvp/backend/tests/test_cian_serp_scraper.py @@ -265,7 +265,8 @@ def test_cian_serp_description_minhash_list_int(): def test_cian_serp_phones(): - """phones — список dicts из state.""" + """PII purge (#2217): даже если raw offer содержит phones, lot.phones пуст — + скрейпер больше не читает offer.get("phones").""" offer = _make_full_offer(has_phones=True) html = _make_cian_serp_html([offer]) @@ -274,9 +275,7 @@ def test_cian_serp_phones(): lot = lots[0] assert isinstance(lot.phones, list) - assert len(lot.phones) == 1 - assert lot.phones[0]["countryCode"] == "7" - assert lot.phones[0]["number"] == "9012345678" + assert lot.phones == [] def test_cian_serp_house_ext_id_newbuilding(): diff --git a/tradein-mvp/frontend/src/components/trade-in/EstimateForm.tsx b/tradein-mvp/frontend/src/components/trade-in/EstimateForm.tsx index 70fe120c..1bb80431 100644 --- a/tradein-mvp/frontend/src/components/trade-in/EstimateForm.tsx +++ b/tradein-mvp/frontend/src/components/trade-in/EstimateForm.tsx @@ -28,8 +28,6 @@ interface FormState { has_balcony: boolean; ownership_type: string; has_mortgage: boolean; - client_name: string; - client_phone: string; } const INITIAL: FormState = { @@ -44,8 +42,6 @@ const INITIAL: FormState = { has_balcony: false, ownership_type: "", has_mortgage: false, - client_name: "", - client_phone: "", }; function validate(f: FormState): string | null { @@ -100,8 +96,6 @@ function toPayload( if (rs) out.repair_state = rs; if (f.ownership_type) out.ownership_type = f.ownership_type; if (f.has_mortgage) out.has_mortgage = true; - if (f.client_name.trim()) out.client_name = f.client_name.trim(); - if (f.client_phone.trim()) out.client_phone = f.client_phone.trim(); return out; } @@ -389,34 +383,6 @@ export function EstimateForm({ Ипотека / обременение -
-
- - -
-
- - -
-
)} diff --git a/tradein-mvp/frontend/src/types/trade-in.ts b/tradein-mvp/frontend/src/types/trade-in.ts index fe7f7cf1..09327203 100644 --- a/tradein-mvp/frontend/src/types/trade-in.ts +++ b/tradein-mvp/frontend/src/types/trade-in.ts @@ -62,8 +62,7 @@ export interface TradeInEstimateInput { // CRM-поля (#395) ownership_type?: string; has_mortgage?: boolean; - client_name?: string; - client_phone?: string; + // client_name / client_phone удалены (PII purge #1969, DROP COLUMN 167). // Координаты из автокомплита — позволяют бэкенду пропустить геокодинг // для адресов, которые DaData/Yandex не резолвят по строке. lat?: number | null; diff --git a/tradein-mvp/packages/scraper-kit/src/scraper_kit/providers/cian/serp.py b/tradein-mvp/packages/scraper-kit/src/scraper_kit/providers/cian/serp.py index 54aa8fec..c451792e 100644 --- a/tradein-mvp/packages/scraper-kit/src/scraper_kit/providers/cian/serp.py +++ b/tradein-mvp/packages/scraper-kit/src/scraper_kit/providers/cian/serp.py @@ -878,7 +878,11 @@ class CianScraper(BaseScraper): repair_state: str | None = normalize_repair_state(offer.get("decoration")) # ── Продавец ───────────────────────────────────────────────────── - phones: list[dict[str, Any]] = offer.get("phones") or [] + # PII purge (#2217): телефоны продавцов больше не сохраняем. + # ScrapedLot.phones остаётся в модели (см. base.py), но на вход + # всегда передаём пустой список — write-path на listings.phones + # выключен здесь, а не на уровне upsert. + phones: list[dict[str, Any]] = [] is_homeowner: bool | None = offer.get("isByHomeowner") is_pro_seller: bool | None = offer.get("isPro")