d3ad896efe
rosreestr_deals open dataset contains both ДКП (resale, ~110-120K ₽/м²) and ДДУ (developer pre-sale, ~177-200K ₽/м²). Mixed import previously skewed secondary-market median, forcing PR #504 to disable deals output entirely. This PR fixes the root cause: filter doc_type='ДКП' in import-rosreestr.sh (WHERE clause) and re-enable _fetch_deals() call in estimator.py (was stubbed to deals=[] since PR #504). dedup_hash prefix changed to 'ros:dkp:' to allow future ДДУ import as separate source without collision. Also fix pre-existing broken mock in test_estimator_cian_integration.py: _fetch_analogs returns 3-tuple (list, bool, str) but mocks passed ([], False) — updated to ([], False, 'W') matching actual signature. Post-merge ops (NOT in this PR): DELETE FROM deals WHERE source='rosreestr' AND dedup_hash NOT LIKE 'ros:dkp:%'; ./deploy/import-rosreestr.sh # на prod Tests: _fetch_deals already mocked in test_estimator_cian_integration.py. Refs: Decision_TradeIn_DataQuality_8PR_Roadmap (PR #504 reversed properly).
84 lines
4 KiB
Bash
Executable file
84 lines
4 KiB
Bash
Executable file
#!/usr/bin/env bash
|
||
# Импорт реальных сделок Росреестра из gendesign-БД в tradein deals.
|
||
#
|
||
# Источник: gendesign-postgres-1 / rosreestr_deals (6.8М строк, партиц.).
|
||
# Берём ЕКБ квартиры (realestate_type_code=002001003000) за последние ~18 мес.
|
||
# rooms выводим из площади (Росреестр не отдаёт кол-во комнат).
|
||
# Координаты — NULL, проставляются отдельно (geocode по street).
|
||
#
|
||
# Только ДКП (вторичка) — ДДУ застройщиков скёюят median, отделены сознательно. См. PR-A.
|
||
#
|
||
# Запуск на прод-хосте: ./import-rosreestr.sh
|
||
# Повторяемо: дедуп по dedup_hash, новый запуск подтянет свежие кварталы.
|
||
|
||
set -euo pipefail
|
||
|
||
SRC_PG="${SRC_PG:-gendesign-postgres-1}"
|
||
DST_PG="${DST_PG:-tradein-postgres}"
|
||
SRC_DB="${SRC_DB:-gendesign}"
|
||
SRC_USER="${SRC_USER:-gendesign}"
|
||
SINCE="${SINCE:-2024-01-01}"
|
||
|
||
echo "[$(date -u +%H:%M:%S)] import-rosreestr: ЕКБ квартиры с $SINCE"
|
||
|
||
# 1. Staging-таблица в tradein.
|
||
docker exec "$DST_PG" psql -U tradein -d tradein -v ON_ERROR_STOP=on -c "
|
||
CREATE TABLE IF NOT EXISTS deals_ros_staging (
|
||
dedup_hash text PRIMARY KEY, address text, rooms int, area_m2 numeric,
|
||
floor int, year_built int, price_rub bigint, price_per_m2 int, deal_date date
|
||
);
|
||
TRUNCATE deals_ros_staging;
|
||
"
|
||
|
||
# 2. Трансформируем на gendesign-стороне, пайпим CSV в staging.
|
||
docker exec "$SRC_PG" psql -U "$SRC_USER" -d "$SRC_DB" -v ON_ERROR_STOP=on -c "
|
||
COPY (
|
||
SELECT
|
||
md5('ros:dkp:' || id::text) AS dedup_hash,
|
||
'Екатеринбург, ' || trim(street) AS address,
|
||
CASE
|
||
WHEN area < 30 THEN 0 WHEN area < 44 THEN 1
|
||
WHEN area < 62 THEN 2 WHEN area < 85 THEN 3
|
||
ELSE 4 END AS rooms,
|
||
round(area, 2) AS area_m2,
|
||
LEAST(NULLIF(substring(floor from '[0-9]+'), '')::int, 100) AS floor,
|
||
year_build AS year_built,
|
||
round(deal_price)::bigint AS price_rub,
|
||
round(price_per_sqm)::int AS price_per_m2,
|
||
period_start_date AS deal_date
|
||
FROM rosreestr_deals
|
||
WHERE region_code = 66
|
||
AND city ILIKE '%катеринбург%'
|
||
AND realestate_type_code = '002001003000'
|
||
AND area BETWEEN 18 AND 200
|
||
AND deal_price BETWEEN 1000000 AND 100000000
|
||
AND street IS NOT NULL AND trim(street) <> ''
|
||
AND doc_type = 'ДКП'
|
||
AND period_start_date >= '$SINCE'
|
||
) TO STDOUT WITH CSV
|
||
" | docker exec -i "$DST_PG" psql -U tradein -d tradein -v ON_ERROR_STOP=on -c "
|
||
COPY deals_ros_staging FROM STDIN WITH CSV
|
||
"
|
||
|
||
STAGED=$(docker exec "$DST_PG" psql -U tradein -d tradein -tAc "SELECT count(*) FROM deals_ros_staging")
|
||
echo "[$(date -u +%H:%M:%S)] staged: $STAGED строк"
|
||
|
||
# 3. Удаляем старые синтетические сделки, вставляем реальные (дедуп).
|
||
docker exec "$DST_PG" psql -U tradein -d tradein -v ON_ERROR_STOP=on -c "
|
||
DELETE FROM deals WHERE address = 'Екатеринбург, реальная сделка';
|
||
|
||
INSERT INTO deals (
|
||
source, dedup_hash, address, rooms, area_m2, floor,
|
||
year_built, price_rub, price_per_m2, deal_date
|
||
)
|
||
SELECT 'rosreestr', dedup_hash, address, rooms, area_m2, floor,
|
||
year_built, price_rub, price_per_m2, deal_date
|
||
FROM deals_ros_staging
|
||
ON CONFLICT (dedup_hash) DO NOTHING;
|
||
|
||
DROP TABLE deals_ros_staging;
|
||
"
|
||
|
||
TOTAL=$(docker exec "$DST_PG" psql -U tradein -d tradein -tAc "SELECT count(*) FROM deals")
|
||
echo "[$(date -u +%H:%M:%S)] deals всего: $TOTAL"
|
||
echo "Координаты проставит geocode-deals (отдельный шаг)."
|