lekss361/gendesign
1
0
Fork 0
Code Issues 57 Pull requests Projects Releases Packages Wiki Activity Actions

fix(tradein): filter ДКП-only in rosreestr importer + re-enable deals

Browse source 
rosreestr_deals open dataset contains both ДКП (resale, ~110-120K ₽/м²) and
ДДУ (developer pre-sale, ~177-200K ₽/м²). Mixed import previously skewed
secondary-market median, forcing PR #504 to disable deals output entirely.

This PR fixes the root cause: filter doc_type='ДКП' in import-rosreestr.sh
(WHERE clause) and re-enable _fetch_deals() call in estimator.py (was stubbed
to deals=[] since PR #504). dedup_hash prefix changed to 'ros:dkp:' to allow
future ДДУ import as separate source without collision.

Also fix pre-existing broken mock in test_estimator_cian_integration.py:
_fetch_analogs returns 3-tuple (list, bool, str) but mocks passed ([], False)
— updated to ([], False, 'W') matching actual signature.

Post-merge ops (NOT in this PR):
  DELETE FROM deals WHERE source='rosreestr' AND dedup_hash NOT LIKE 'ros:dkp:%';
  ./deploy/import-rosreestr.sh   # на prod

Tests: _fetch_deals already mocked in test_estimator_cian_integration.py.

Refs: Decision_TradeIn_DataQuality_8PR_Roadmap (PR #504 reversed properly).
This commit is contained in:
lekss361 2026-05-24 22:31:23 +03:00
parent 967e7185ca
commit d3ad896efe
3 changed files with 20 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
tradein-mvp/backend/app/services/estimator.py
View file

@ -834,11 +834,17 @@ async def estimate_quality(payload: TradeInEstimateInput, db: Session) -> Aggreg
except Exception as exc:
logger.warning("cian_valuation: lookup failed (graceful): %s", exc)
# 5. Deals — отключены 2026-05-24 (см. Decision_TradeIn_DataQuality_8PR_Roadmap).
# rosreestr_deals содержит ДДУ первичного рынка (177-200 К/м²), что искажает
# оценку вторички (~114 К/м²). Возвращаемся к этому когда отделим вторичные
# сделки от ДДУ в источнике.
deals: list[dict[str, Any]] = []
# 5. Deals — ДКП-only sales (вторичка) из rosreestr_deals.
# Importer фильтрует doc_type='ДКП' (PR-A 2026-05-24), ДДУ застройщиков
# исключены — больше не скёюят median вторички ~110-120 К/м².
deals = _fetch_deals(
db,
lat=geo.lat,
lon=geo.lon,
rooms=payload.rooms,
area=payload.area_m2,
radius_m=DEFAULT_RADIUS_M,
)
# 6. Сохраняем в trade_in_estimates
estimate_id = uuid4()

10
tradein-mvp/backend/tests/test_estimator_cian_integration.py
View file

@ -102,7 +102,7 @@ def _common_patches(cian_mock):
return [
patch("app.services.estimator.geocode", new=AsyncMock(return_value=_make_fake_geo())),
patch("app.services.estimator.get_house_metadata", new=AsyncMock(return_value=None)),
patch("app.services.estimator._fetch_analogs", return_value=([], False)),
patch("app.services.estimator._fetch_analogs", return_value=([], False, "W")),
patch("app.services.estimator._fetch_deals", return_value=[]),
patch(
"app.services.estimator._get_or_fetch_imv_cached",
@ -134,7 +134,7 @@ def test_estimator_includes_cian_valuation_when_available() -> None:
new=AsyncMock(return_value=_make_fake_geo())),
patch("app.services.estimator.get_house_metadata",
new=AsyncMock(return_value=None)),
patch("app.services.estimator._fetch_analogs", return_value=([], False)),
patch("app.services.estimator._fetch_analogs", return_value=([], False, "W")),
patch("app.services.estimator._fetch_deals", return_value=[]),
patch("app.services.estimator._get_or_fetch_imv_cached",
new=AsyncMock(return_value=None)),
@ -164,7 +164,7 @@ def test_estimator_graceful_when_cian_returns_none() -> None:
new=AsyncMock(return_value=_make_fake_geo())),
patch("app.services.estimator.get_house_metadata",
new=AsyncMock(return_value=None)),
patch("app.services.estimator._fetch_analogs", return_value=([], False)),
patch("app.services.estimator._fetch_analogs", return_value=([], False, "W")),
patch("app.services.estimator._fetch_deals", return_value=[]),
patch("app.services.estimator._get_or_fetch_imv_cached",
new=AsyncMock(return_value=None)),
@ -195,7 +195,7 @@ def test_estimator_graceful_when_cian_raises() -> None:
new=AsyncMock(return_value=_make_fake_geo())),
patch("app.services.estimator.get_house_metadata",
new=AsyncMock(return_value=None)),
patch("app.services.estimator._fetch_analogs", return_value=([], False)),
patch("app.services.estimator._fetch_analogs", return_value=([], False, "W")),
patch("app.services.estimator._fetch_deals", return_value=[]),
patch("app.services.estimator._get_or_fetch_imv_cached",
new=AsyncMock(return_value=None)),
@ -228,7 +228,7 @@ def test_estimator_cian_result_no_sale_price_not_added() -> None:
new=AsyncMock(return_value=_make_fake_geo())),
patch("app.services.estimator.get_house_metadata",
new=AsyncMock(return_value=None)),
patch("app.services.estimator._fetch_analogs", return_value=([], False)),
patch("app.services.estimator._fetch_analogs", return_value=([], False, "W")),
patch("app.services.estimator._fetch_deals", return_value=[]),
patch("app.services.estimator._get_or_fetch_imv_cached",
new=AsyncMock(return_value=None)),

5
tradein-mvp/deploy/import-rosreestr.sh
View file

@ -6,6 +6,8 @@
# rooms выводим из площади (Росреестр не отдаёт кол-во комнат).
# Координаты — NULL, проставляются отдельно (geocode по street).
#
# Только ДКП (вторичка) — ДДУ застройщиков скёюят median, отделены сознательно. См. PR-A.
#
# Запуск на прод-хосте: ./import-rosreestr.sh
# Повторяемо: дедуп по dedup_hash, новый запуск подтянет свежие кварталы.
@ -32,7 +34,7 @@ docker exec "$DST_PG" psql -U tradein -d tradein -v ON_ERROR_STOP=on -c "
docker exec "$SRC_PG" psql -U "$SRC_USER" -d "$SRC_DB" -v ON_ERROR_STOP=on -c "
COPY (
SELECT
md5('ros:' || id::text) AS dedup_hash,
md5('ros:dkp:' || id::text) AS dedup_hash,
'Екатеринбург, ' || trim(street) AS address,
CASE
WHEN area < 30 THEN 0 WHEN area < 44 THEN 1
@ -51,6 +53,7 @@ docker exec "$SRC_PG" psql -U "$SRC_USER" -d "$SRC_DB" -v ON_ERROR_STOP=on -c "
AND area BETWEEN 18 AND 200
AND deal_price BETWEEN 1000000 AND 100000000
AND street IS NOT NULL AND trim(street) <> ''
AND doc_type = 'ДКП'
AND period_start_date >= '$SINCE'
) TO STDOUT WITH CSV
" | docker exec -i "$DST_PG" psql -U tradein -d tradein -v ON_ERROR_STOP=on -c "